diff --git a/package.json b/package.json index 2b6176dcd8d3f40c66ed8bdab47d575c28ef052c..10de3b239ff73326e72ed905ed3767b3a734809d 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "license": "GPL-3.0", "dependencies": { "app-module-path": "^1.1.0", + "async": "^2.0.1", "eslint": "^3.2.2", "express": "^4.14.0", "gulp": "^3.9.1", diff --git a/src/core/aggregator.js b/src/core/aggregator.js index 6b04a23338597a1591a9a6809d3987c82b90cb94..d46a3dd9484381ef771971ac5d5b103e5263df37 100644 --- a/src/core/aggregator.js +++ b/src/core/aggregator.js @@ -20,9 +20,125 @@ 'use strict'; +const async = require('async'); const mongo = require('core/mongo'); +const serializer = require('util/serializer'); +const hash = require('util/hash'); class Aggregator { + removeAggregate(id, callback) { + const aggregates = mongo.db.collection('meta.aggregates'); + aggregates.findOneAndDelete({ _id: id }, (err) => { + if (err) { + callback(err); + return; + } + + let aggr = mongo.db.collection('aggr.' + id); + aggr.remove({}, callback); + }); + } + + createAggregate(dimensions, metrics, callback) { + const aggregates = mongo.db.collection('meta.aggregates'); + + let doc = { + _id: hash.sha1(serializer.dump(dimensions) + + serializer.dump(metrics)), + dimensions: dimensions, + metrics: metrics + }; + + aggregates.insert(doc, (err) => { + if (err) { + callback(err); + return; + } + + callback(null, doc); + }); + } + + rebuildBaseAggregates(callback) { + this.cleanAggregates((err) => { + if (err) { + callback(err); + return; + } + + this.buildBaseAggregates(callback); + }); + } + + buildBaseAggregates(callback) { + let classes = mongo.db.collection('meta.classes'); + + classes.find({}).toArray((err, result) => { + if (err) { + callback(err); + return; + } + + async.map(result, (cls, cb) => { this.buildBaseAggregateFromClass(cls, cb); }, (err) => { + if (err) { + callback(err); + return; + } + + return callback(null); + }); + }); + } + + buildBaseAggregateFromClass(cls, callback) { + this.createAggregate(cls.dimensions, cls.metrics, (err, aggr) => { + const raw = mongo.db.collection('raw.' + cls.name); + const aggrData = mongo.db.collection('aggr.' + aggr._id); + + const functions = serializer.load(cls.functions); + + raw.find({}).forEach((doc) => { + let data = { + dimensions: functions.extractDimensions.apply(doc), + metrics: functions.extractMetrics.apply(doc) + }; + + // TODO: aggrData.insert(data); + }, callback); + }); + } + + cleanAggregates(callback) { + let aggregates = mongo.db.collection('meta.aggregates'); + + aggregates.find({}).toArray((err, result) => { + if (err) { + callback(err); + return; + } + + async.map(result, (aggr, callback) => { + let aggrCol = mongo.db.collection('aggr.' + aggr.name); + aggrCol.remove({}, (err) => { + if (err) { + callback(err); + return; + } + + aggregates.remove({ _id: aggr._id }, callback); + }); + }, + (err) => { + if (err) { + callback(err); + return; + } + + callback(null); + }); + }); + } + query(metrics, dimensions, callback) { this.findClosestAggregate(metrics, dimensions, (err, aggr) => { if (err) { @@ -35,7 +151,7 @@ class Aggregator { } findClosestAggregate(metrics, dimensions, callback) { - var aggregates = mongo.db.collection('meta.aggregates'); + let aggregates = mongo.db.collection('meta.aggregates'); aggregates.find({ metrics: { @@ -44,7 +160,7 @@ class Aggregator { dimensions: { $all: dimensions } - }).toArray(function(err, result) { + }).toArray((err, result) => { if (err) { callback(err); return; diff --git a/src/util/hash.js b/src/util/hash.js new file mode 100644 index 0000000000000000000000000000000000000000..abe9f7a01aa68290151323b95315a0a88d0765ad --- /dev/null +++ b/src/util/hash.js @@ -0,0 +1,33 @@ +/* + * Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blendb. + * + * blendb is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blendb is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blendb. If not, see <http://www.gnu.org/licenses/>. + */ + +'use strict'; + +const crypto = require('crypto'); + +class Hash { + sha1(content) { + let hash = crypto.createHash('sha1'); + hash.update(content); + return hash.digest('hex'); + } +} + +module.exports = new Hash(); diff --git a/src/util/serializer.js b/src/util/serializer.js index 2d601aec9dda548e4d445169684755f50d5ba054..9c10e68cdf45e1389ee6cef8d35ff5465abb224a 100644 --- a/src/util/serializer.js +++ b/src/util/serializer.js @@ -24,7 +24,9 @@ class Serializer { dump(obj) { return JSON.stringify(obj, (key, value) => { if (typeof value === 'function') { - return value.toString(); + return value.toString() + .replace(/[\n\r\t]/g, '') + .replace(/ +/g, ' '); } return value; @@ -38,8 +40,8 @@ class Serializer { } if (typeof value === 'string') { - let rfunc = /function[^\(]*\(([^\)]*)\)[^\{]*{([^\}]*)\}/; - let match = value.match(rfunc); + let rfunc = /function[^\(]*\(([^\)]*)\)[^\{]*\{(.*)\}[^\}]*$/; + let match = value.replace(/\n/g, '').match(rfunc); if (match) { let args = match[1].split(',')