diff --git a/gulpfile.js b/gulpfile.js new file mode 100644 index 0000000000000000000000000000000000000000..32f291387eeef1a8fa35ff49cfb0a58f8fc93077 --- /dev/null +++ b/gulpfile.js @@ -0,0 +1,191 @@ +'use strict'; + +var gulp = require('gulp'); +var gutil = require('gulp-util'); +var raml = require('gulp-raml'); +var rename = require('gulp-rename'); +var jshint = require('gulp-jshint'); +var size = require('gulp-size'); +var jscs = require('gulp-jscs'); +var stylish = require('gulp-jscs-stylish'); +var mocha = require('gulp-mocha'); +var istanbul = require('gulp-istanbul'); +var nodemon = require('gulp-nodemon'); + +var path = require('path'); +var raml2html = require('raml2html'); +var through = require('through2'); +var yaml = require('js-yaml'); +var map = require('map-stream'); + +var srcFiles = [ + 'src/**/*.js', + 'index.js', + 'gulpfile.js', + 'test/**/*.js' +]; + +function handleError(err) { + console.error(err.toString()); + process.exit(1); +} + +function exitOnError(type) { + return map(function(file, callback) { + if (!file[type].success) { + process.exit(1); + } + + callback(null, file); + }); +} + +function generateDoc(options) { + var simplifyMark = function(mark) { + if (mark) { + mark.buffer = mark.buffer + .split('\n', mark.line + 1)[mark.line] + .trim(); + } + }; + + if (!options) { + options = {}; + } + + switch (options.type) { + case 'json': + options.config = { + template: function(obj) { + return JSON.stringify(obj, null, 2); + } + }; + break; + case 'yaml': + options.config = { + template: function(obj) { + return yaml.safeDump(obj, { + skipInvalid: true + }); + } + }; + break; + default: + options.type = 'html'; + if (!options.config) { + options.config = raml2html.getDefaultConfig( + options.https, + options.template, + options.resourceTemplate, + options.itemTemplate + ); + } + } + + if (!options.extension) { + options.extension = '.' + options.type; + } + + var stream = through.obj(function(file, enc, done) { + var fail = function(message) { + done(new gutil.PluginError('raml2html', message)); + }; + + if (file.isBuffer()) { + var cwd = process.cwd(); + process.chdir(path.resolve(path.dirname(file.path))); + raml2html + .render(file.contents, options.config) + .then(function(output) { + process.chdir(cwd); + stream.push(new gutil.File({ + base: file.base, + cwd: file.cwd, + path: gutil.replaceExtension( + file.path, options.extension), + contents: new Buffer(output) + })); + done(); + }, + function(error) { + process.chdir(cwd); + simplifyMark(error.context_mark); + simplifyMark(error.problem_mark); + process.nextTick(function() { + fail(JSON.stringify(error, null, 2)); + }); + } + ); + } + else if (file.isStream()) { + fail('Streams are not supported: ' + file.inspect()); + } + else if (file.isNull()) { + fail('Input file is null: ' + file.inspect()); + } + }); + + return stream; +} + +gulp.task('raml', function() { + gulp.src('specs/*.raml') + .pipe(raml()) + .pipe(raml.reporter('default')) + .pipe(exitOnError('raml')); +}); + +gulp.task('doc', function() { + return gulp.src('specs/*.raml') + .pipe(generateDoc()) + .on('error', handleError) + .pipe(rename({ extname: '.html' })) + .pipe(gulp.dest('doc/build')); +}); + +gulp.task('pre-test', function() { + return gulp.src(['src/**/*.js']) + .pipe(istanbul()) + .pipe(istanbul.hookRequire()); +}); + +gulp.task('test', ['pre-test'], function() { + return gulp.src('test/**/*.spec.js', { read: false }) + .pipe(mocha({ + require: ['./test/common.js'], + reporter: 'spec', + ui: 'bdd', + recursive: true, + colors: true, + timeout: 60000, + slow: 300, + delay: true + })) + .pipe(istanbul.writeReports()) + .once('error', function() { + process.exit(1); + }) + .once('end', function() { + process.exit(); + }); +}); + +gulp.task('lint', function() { + return gulp.src(srcFiles) + .pipe(jshint()) + .pipe(jscs()) + .pipe(stylish.combineWithHintResults()) + .pipe(jshint.reporter('jshint-stylish')) + .pipe(size()) + .pipe(exitOnError('jshint')); +}); + +gulp.task('check', ['raml', 'lint', 'test']); + +gulp.task('develop', function() { + return nodemon({ + script: 'index.js', + ext: 'js', + tasks: ['lint'] + }); +}); diff --git a/index.js b/index.js new file mode 100755 index 0000000000000000000000000000000000000000..4a39bcfb7e55943ba61ee039b6a2d4443ce5cad7 --- /dev/null +++ b/index.js @@ -0,0 +1,68 @@ +#!/usr/bin/env node +/* + * Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blendb. + * + * blendb is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blendb is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blendb. If not, see <http://www.gnu.org/licenses/>. + */ + +'use strict'; + +// Add the ./src directory to require's search path to facilitate import +// modules later on (avoiding the require('../../../../module') problem). +require('app-module-path').addPath(__dirname + '/src'); + +// external libraries +const osprey = require('osprey'); +const express = require('express'); +const path = require('path'); +const ramlParser = require('raml-parser'); + +// connect to mongodb +const mongo = require('core/mongo'); +mongo.connect('mongodb://pyke/blend'); + +// create a new express app +const app = module.exports = express(); + +// load router +const router = require('api/router-v1.js'); + +// parse the RAML spec and load osprey middleware +ramlParser.loadFile(path.join(__dirname, 'specs/blendb-api-v1.raml')) + .then(raml => { + app.use('/v1', + osprey.security(raml), + osprey.server(raml), + router); + + if (!module.parent) { + let port = process.env.PORT || 3000; + app.listen(port); + + if (app.get('env') === 'development') { + console.log('Server listening on port ' + port + '.'); + } + } + else { + // signalize to the test suite that the server is ready to be tested + app.ready = true; + } + }, + err => { + console.error('RAML Parsing Error: ' + err.message); + process.exit(1); + }); diff --git a/specs/blendb-api-v1.raml b/specs/blendb-api-v1.raml new file mode 100644 index 0000000000000000000000000000000000000000..98cfdc3f8f58764e70ab740ab3b689bffdc303d5 --- /dev/null +++ b/specs/blendb-api-v1.raml @@ -0,0 +1,395 @@ +#%RAML 0.8 + +title: BlenDB API +version: v1 +baseUri: http://blendb.c3sl.ufpr.br/api/{version} +mediaType: application/json + +securitySchemes: + - oauth_2_0: + description: | + OAuth2 is a protocol that lets apps request authorization to + private details in the system while avoiding the use of passwords. + This is preferred over Basic Authentication because tokens can be + limited to specific types of data, and can be revoked by users at + any time. + type: OAuth 2.0 + describedBy: + headers: + Authorization: + description: | + Used to send a valid OAuth 2 access token. Do not use + together with the "access_token" query string parameter. + type: string + queryParameters: + access_token: + description: | + Used to send a valid OAuth 2 access token. Do not use + together with the "Authorization" header. + type: string + responses: + 401: + description: | + Bad or expired token. This can happen if access token + has expired or has been revoked by the user. + body: + application/json: + example: | + { + id: "invalid_oauth_token", + message: "Bad or expired token. This can happen if access token has expired or has been revoked by the user." + } + 403: + description: | + Bad OAuth2 request (wrong consumer key, bad nonce, + expired timestamp, ...). + body: + application/json: + example: | + { + id: "invalid_oauth_request", + message: "Bad OAuth2 request (wrong consumer key, bad nonce, expired timestamp, ...)." + } + settings: + authorizationUri: http://simmc.c3sl.ufpr.br/oauth/authorize + accessTokenUri: http://simmc.c3sl.ufpr.br/oauth/access_token + authorizationGrants: [ code, token ] + scopes: + - "user" + - "user:email" + +resourceTypes: + - base: + get?: &common + responses: + 403: + description: API rate limit exceeded. + headers: + X-RateLimit-Limit: + type: integer + X-RateLimit-Remaining: + type: integer + X-RateLimit-Reset: + type: integer + body: + application/json: + example: | + { + id: "too_many_requests", + message: "API Rate limit exceeded." + } + post?: *common + put?: *common + delete?: *common + - collection: + type: base + get?: + description: | + List all of the <<resourcePathName>> (with optional + filtering). + responses: + 200: + description: | + A list of <<resourcePathName>>. + body: + application/json: + schema: <<collectionSchema>> + example: <<collectionExample>> + post?: + description: | + Create a new <<resourcePathName|!singularize>>. + responses: + 201: + description: | + Sucessfully created a new + <<resourcePathName|!singularize>>. + headers: + Location: + description: | + A link to the newly created + <<resourcePathName|!singularize>>. + type: string + 409: + description: | + Failed to create a new + <<resourcePathName|!singularize>> because a conflict + with an already existing + <<resourcePathName|!singularize>> was detected. + body: + application/json: + example: | + { + "id": "already_exists", + "message": "The <<resourcePathName|!singularize>> could not be created due to a conflict with an already existing <<resourcePathName|!singularize>>." + } + - item: + type: base + get?: + description: | + Return a single <<resourcePathName|!singularize>>. + responses: + 200: + description: | + A single <<resourcePathName|!singularize>>. + body: + application/json: + schema: <<itemSchema>> + example: <<itemExample>> + 404: + description: | + The <<resourcePathName|!singularize>> could not be + found. + body: + application/json: + example: | + { + "id": "not_found", + "message": "The <<resourcePathName|!singularize>> could not be found." + } + put?: + description: | + Update a <<resourcePathName>>. + responses: + 204: + description: | + The <<resourcePathName|!singularize>> was updated. + 404: + description: | + The <<resourcePathName|!singularize>> could not be + found. + body: + application/json: + example: | + { + "id": "not_found", + "message": "The <<resourcePathName|!singularize>> could not be found." + } + 409: + description: | + Failed to update the <<resourcePathName|!singularize>> + because a conflict with another + <<resourcePathName|!singularize>> was detected. + body: + application/json: + example: | + { + "id": "already_exists", + "message": "Failed to update the <<resourcePathName|!singularize>> because a conflict with another <<resourcePathName|!singularize>> was detected." + } + patch?: + description: | + Partially update a <<resourcePathName>>. + responses: + 204: + description: | + The <<resourcePathName|!singularize>> was updated. + 404: + description: | + The <<resourcePathName|!singularize>> could not be + found. + body: + application/json: + example: | + { + "id": "not_found", + "message": "The <<resourcePathName|!singularize>> could not be found." + } + 409: + description: | + Failed to update the <<resourcePathName|!singularize>> + because a conflict with another + <<resourcePathName|!singularize>> was detected. + body: + application/json: + example: | + { + "id": "already_exists", + "message": "Failed to update the <<resourcePathName|!singularize>> because a conflict with another <<resourcePathName|!singularize>> was detected." + } + delete?: + description: | + Removes a <<resourcePathName>>. + responses: + 204: + description: | + The <<resourcePathName|!singularize>> was removed. + 404: + description: | + The <<resourcePathName|!singularize>> could not be + found. + body: + application/json: + example: | + { + "id": "not_found", + "message": "The <<resourcePathName|!singularize>> could not be found." + } + - index: + type: base + get?: + description: | + Return an index on the <<resourcePathName>> collection. + responses: + 200: + description: | + An index on the <<resourcePathName>> collection. + body: + application/json: + +traits: + - paged: + queryParameters: + page: + description: Specify the page that you want to retrieve + type: integer + default: 1 + example: 1 + per_page: + description: The number of items to return per page + type: integer + minimum: 1 + maximum: 50 + default: 10 + example: 20 + - searchable: + queryParameters: + query: + description: | + Query string that filters the data returned for your + request. + type: string + - filtered: + queryParameters: + filters: + description: | + Filters that restrict the data returned for your request. + type: string + + - projectable: + queryParameters: + fields: + description: | + Fields to be returned. + type: string + +/metrics: + description: | + A Metric represents a statistic that can be queried to generate reports. + This collection allows the user to list all the metrics available in the + system and their descriptions. + securedBy: [ null, oauth_2_0 ] + get: + +/dimensions: + description: | + A Dimension allows the data to be aggregated by one or more columns. + This collection allows the user to list all the dimensions available in + the system and their descriptions. + securedBy: [ null, oauth_2_0 ] + get: + +/data: + description: | + This is the main part of the API. You may query it for report + data by specifying metrics (at least one). You may also supply + additional query parameters such as dimensions, filters, and + start/end dates to refine your query. + type: base + get: + is: [ filtered ] + queryParameters: + metrics: + description: | + A list of comma-separated metrics. + type: string + required: true + example: "met:daysSinceLastContact,met:estimatedNetworkBandwidth" + dimensions: + description: | + A list of comma-separated dimensions. + type: string + required: true + example: "dim:project,dim:point" + start-date: + description: | + Start date for fetching data. Requests can specify a + start date formatted as YYYY-MM-DD, or as a relative date + (e.g., today, yesterday, or NdaysAgo where N is a positive + integer). + type: string + required: false + pattern: "[0-9]{4}-[0-9]{2}-[0-9]{2}|today|yesterday|[0-9]+(daysAgo)" + example: 7daysAgo + end-date: + description: | + End date for fetching data. Requests can specify a + end date formatted as YYYY-MM-DD, or as a relative date + (e.g., today, yesterday, or NdaysAgo where N is a positive + integer). + type: string + required: false + pattern: "[0-9]{4}-[0-9]{2}-[0-9]{2}|today|yesterday|[0-9]+(daysAgo)" + example: yesterday + filters: + description: | + Filters that restrict the data returned for your request. + type: string + example: "dim:location(4).id%3D%3D10723" + sort: + description: | + A list of comma-separated dimensions and metrics + indicating the sorting order and sorting direction for + the returned data. + type: string + example: "dim:project" + responses: + 200: + description: | + Query successfully executed. Data is returned in a table format. + body: + application/json: + 400: + description: | + The supplied query is invalid. Specified metric or dimension + doesn't exist, incorrect formatting for a filter, unacceptable + date range, etc. + body: + application/json: + example: | + { + "id": "metric_not_found", + "message": "The specified metric 'met:electricCharge' could not be found." + } + +/collect/{class}: + description: | + This API may be used to send data to the monitoring system. There are a + few available data types (like network bandwidth usage, machine + inventory, etc.) and each of them requires a specific format for the + data being sent. + type: base + uriParameters: + class: + description: The class of data that is being collected. + type: string + minLength: 4 + maxLength: 64 + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + post: + body: + application/json: + responses: + 200: + description: | + Data has been successfully received and stored by the server. + 400: + description: | + An error has been found in your request. You may review your + request and the data that is being sent and try again later. + body: + application/json: + example: | + { + "id": "invalid_attribute", + "message": "Invalid attribute \"memory\" for data type \"network_bandwidth\"." + } diff --git a/src/api/controllers/collect.js b/src/api/controllers/collect.js new file mode 100644 index 0000000000000000000000000000000000000000..518bf502570c334dc6524c70475d8df76bd69e0a --- /dev/null +++ b/src/api/controllers/collect.js @@ -0,0 +1,47 @@ +/* + * Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blendb. + * + * blendb is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blendb is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blendb. If not, see <http://www.gnu.org/licenses/>. + */ + +'use strict'; + +const mongo = require('core/mongo'); + +class Collect { + write(req, res, next) { + let collection = mongo.db.collection('raw.' + req.params.class); + + if ('_id' in req.body) { + res.status(400) + .json({ message: 'Property named \'_id\' is protected.' }); + return; + } + + collection.insertOne(req.body, function (err, r) { + if (err) { + res.status(500) + .json({ message: 'Error while writing to the database.' }); + return; + } + + res.status(200).json({ _id: r.insertedId }); + }); + } +} + +module.exports = new Collect(); diff --git a/src/api/controllers/data.js b/src/api/controllers/data.js new file mode 100644 index 0000000000000000000000000000000000000000..8edbaa99b6d35c4ed4ae06b98788c8af118e4fb2 --- /dev/null +++ b/src/api/controllers/data.js @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blendb. + * + * blendb is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blendb is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blendb. If not, see <http://www.gnu.org/licenses/>. + */ + +'use strict'; + +const aggregator = require('core/aggregator'); + +class Data { + read(req, res, next) { + let metrics = req.query.metrics.split(','); + let dimensions = req.query.dimensions.split(','); + + aggregator.query(metrics, dimensions, (err, data) => { + if (err) { + console.error(err); + res.status(500).json({ message: 'Query execution failed ' + + 'because of an unknown error.' }); + return; + } + + res.json({ data }); + }); + } +} + +module.exports = new Data(); diff --git a/src/api/router-v1.js b/src/api/router-v1.js new file mode 100644 index 0000000000000000000000000000000000000000..7a95b16b7d9ced0ce4f1b2c7f43af19111fc9509 --- /dev/null +++ b/src/api/router-v1.js @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blendb. + * + * blendb is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blendb is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blendb. If not, see <http://www.gnu.org/licenses/>. + */ + +'use strict'; + +const osprey = require('osprey'); + +// import controllers +const data = require('./controllers/data'); +const collect = require('./controllers/collect'); + +const router = module.exports = osprey.Router(); + +router.get('/data', data.read); +router.post('/collect/{class}', collect.write); diff --git a/src/core/aggregator.js b/src/core/aggregator.js new file mode 100644 index 0000000000000000000000000000000000000000..6b04a23338597a1591a9a6809d3987c82b90cb94 --- /dev/null +++ b/src/core/aggregator.js @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blendb. + * + * blendb is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blendb is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blendb. If not, see <http://www.gnu.org/licenses/>. + */ + +'use strict'; + +const mongo = require('core/mongo'); + +class Aggregator { + query(metrics, dimensions, callback) { + this.findClosestAggregate(metrics, dimensions, (err, aggr) => { + if (err) { + callback(err); + return; + } + + callback(null, null); + }); + } + + findClosestAggregate(metrics, dimensions, callback) { + var aggregates = mongo.db.collection('meta.aggregates'); + + aggregates.find({ + metrics: { + $all: metrics + }, + dimensions: { + $all: dimensions + } + }).toArray(function(err, result) { + if (err) { + callback(err); + return; + } + + if ((!result) || (result.length <= 0)) { + callback('Query could not be aswered, no aggregate available.'); + return; + } + + // fetch the closest aggregation available + let closestAggr; + for (const aggr of result) { + if ((!closestAggr) || + (aggr.dimensions.length < closestAggr.dimensions.length)) { + closestAggr = aggr; + } + } + + callback(null, closestAggr); + }); + } +} + +module.exports = new Aggregator(); diff --git a/src/core/mongo.js b/src/core/mongo.js new file mode 100644 index 0000000000000000000000000000000000000000..f6f4dccd44872a32870e6de3790cace342634976 --- /dev/null +++ b/src/core/mongo.js @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blendb. + * + * blendb is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blendb is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blendb. If not, see <http://www.gnu.org/licenses/>. + */ + +'use strict'; + +const MongoClient = require('mongodb').MongoClient; + +class Mongo { + constructor() { + this.db = undefined; + } + + connect(connectionString) { + MongoClient.connect(connectionString, (err, db) => { + if (err) { + console.error(err); + return; + } + + this.db = db; + }); + } +} + +module.exports = new Mongo(); diff --git a/src/util/serializer.js b/src/util/serializer.js new file mode 100644 index 0000000000000000000000000000000000000000..2d601aec9dda548e4d445169684755f50d5ba054 --- /dev/null +++ b/src/util/serializer.js @@ -0,0 +1,56 @@ +/* + * Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blendb. + * + * blendb is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blendb is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blendb. If not, see <http://www.gnu.org/licenses/>. + */ + +'use strict'; + +class Serializer { + dump(obj) { + return JSON.stringify(obj, (key, value) => { + if (typeof value === 'function') { + return value.toString(); + } + + return value; + }); + } + + load(str) { + return JSON.parse(str, (key, value) => { + if (key === '') { + return value; + } + + if (typeof value === 'string') { + let rfunc = /function[^\(]*\(([^\)]*)\)[^\{]*{([^\}]*)\}/; + let match = value.match(rfunc); + + if (match) { + let args = match[1].split(',') + .map((arg) => arg.replace(/\s+/, '')); + return new Function(args, match[2]); + } + } + + return value; + }); + } +} + +module.exports = new Serializer();