Merge branch 'issue/15' into 'master'

Issue #15: Connect API with Engine

See merge request !11

package.json

@@ -38,10 +38,12 @@
     "typescript": "^2.0.3"
   },
   "devDependencies": {
+    "@types/supertest": "^2.0.0",
     "chai": "^3.4.33",
     "istanbul": "1.1.0-alpha.1",
     "mocha": "^3.1.0",
     "raml2html": "^3.0.1",
+    "supertest": "^3.0.0",
     "tslint": "^3.15.1",
     "tslint-stylish": "^2.1.0-beta"
   }

src/api/controllers/collect.spec.ts

+/*
+ * Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as request from "supertest";
+import { expect } from "chai";
+
+import * as server from "../../main";
+
+describe("API collect controller", () => {
+
+    it("should respond 400 when _id field is in request body", (done) => {
+        request(server)
+            .post("/v1/collect/class")
+            .send({"_id": 1})
+            .expect(400)
+            .expect((res: any) => {
+                const message = "Property named \"_id\" is protected.";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+            })
+            .end(done);
+    });
+
+    it("should respond 500 on error on writing in the database", (done) => {
+        request(server)
+            .post("/v1/collect/class")
+            .send({"field1": 1, "field2": 2})
+            .expect(500)
+            .expect((res: any) => {
+                const message = "Error while writing to the database.";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+            })
+            .end(done);
+    });
+
+});

src/api/controllers/data.spec.ts

+/*
+ * Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as request from "supertest";
+import { expect } from "chai";
+
+import * as server from "../../main";
+import { dataCtrlScenario as tests } from "../../../test/scenario";
+import { Query } from "../../common/query";
+
+interface StrQuery {
+    metrics: string;
+    dimensions: string;
+}
+
+function parseQuery(obj: Query): StrQuery {
+    let r: StrQuery = {
+        metrics: "",
+        dimensions: ""
+    };
+
+    r.metrics = obj.metrics.map ((item) => item.name).join(",");
+    r.dimensions = obj.dimensions.map ((item) => item.name).join(",");
+    return r;
+}
+
+describe("API data controller", () => {
+
+    it("should respond 500 when query has inexistent metric", (done) => {
+        request(server)
+            .get("/v1/data")
+            .query(parseQuery(tests.wrongMet))
+            .expect(500)
+            .expect((res: any) => {
+                const message = "Query execution failed: " +
+                "Could not construct query with the paramters given.";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+            })
+            .end(done);
+    });
+
+    it("should respond 500 when query has inexistent dimension", (done) => {
+        request(server)
+            .get("/v1/data")
+            .query(parseQuery(tests.wrongDim))
+            .expect(500)
+            .expect((res: any) => {
+                const message = "Query execution failed: " +
+                "Could not construct query with the paramters given.";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+            })
+            .end(done);
+    });
+
+    it("should respond 200 and get some data", (done) => {
+        request(server)
+            .get("/v1/data")
+            .query(parseQuery(tests.correct))
+            .expect(200)
+            .expect((res: any) => {
+                let result = res.body;
+                expect(result).to.be.an("array");
+                expect(result).to.have.length(5);
+                expect(result[0]).to.be.an("object");
+                let keys: string[] = [];
+                keys = keys.concat(tests.correct.metrics.map((item) => item.name));
+                keys = keys.concat(tests.correct.dimensions.map((item) => item.name));
+                result.forEach((row: any) => {
+                    expect(row).to.be.an("object");
+                    expect(row).to.have.all.keys(keys);
+                });
+            })
+            .end(done);
+    });
+
+});

src/api/controllers/data.ts

@@ -19,13 +19,41 @@
  */
 
 import * as express from "express";
+import { Request } from "../types";
+import { Query } from "../../common/query";
 
 export class DataCtrl {
-    public static read(req: express.Request, res: express.Response, next: express.NextFunction) {
+    public static read(req: Request, res: express.Response, next: express.NextFunction) {
         let metrics = req.query.metrics.split(",");
         let dimensions = req.query.dimensions.split(",");
+        let view;
 
+        try {
+            let query: Query = { metrics: [], dimensions: [] };
+            for (let i = 0; i < metrics.length; ++i) {
+                query.metrics.push(req.engine.getMetricByName(metrics[i]));
+            }
+
+            for (let i = 0; i < metrics.length; ++i) {
+                query.dimensions.push(req.engine.getDimensionByName(dimensions[i]));
+            }
+            view = req.engine.query(query);
+        }
+        catch (e) {
+            res.status(500).json({ message: "Query execution failed: " +
+                "Could not construct query with the paramters given." });
+            return;
+        }
+
+        req.adapter.getDataFromView(view, (err: Error, result: any[]) => {
+            if (err) {
                 res.status(500).json({ message: "Query execution failed " +
-            "because of an unknown error." });
+                    "failed on execute query on database." });
+                return;
+            }
+
+            res.status(200).json(result);
+            return;
+        });
     }
 }

src/api/middlewares/adapter.ts

+/*
+ * Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Middleware } from "../types";
+import { Adapter } from "../../core/adapter";
+import { PostgresAdapter } from "../../adapter/postgres";
+import { PoolConfig } from "pg";
+
+export function PostgresMw(config: PoolConfig): Middleware {
+    let adapter: Adapter = new PostgresAdapter(config);
+    return function postgresMiddleware(req, res, next) {
+        req.adapter = adapter;
+        next();
+    };
+
+}

src/api/middlewares/engine.ts

+/*
+ * Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Engine } from "../../core/engine";
+import { ParsedConfig } from "../../util/configParser";
+import { Middleware } from "../types";
+
+export function EngineMw (config: ParsedConfig): Middleware {
+    let engine: Engine = new  Engine();
+
+    config.metrics.forEach ((met) => engine.addMetric(met));
+    config.dimensions.forEach ((dim) => engine.addDimension(dim));
+    config.views.forEach ((view) => engine.addView(view));
+
+    return function engineMiddleware(req, res, next) {
+        req.engine = engine;
+        next();
+    };
+}

src/api/types.ts

+/*
+ * Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as express from "express";
+import { Engine } from "../core/engine";
+import { Adapter} from "../core/adpater";
+
+export interface Request extends express.Request {
+    engine: Engine;
+    adapter: Adapter;
+}
+
+export interface Middleware {
+    (req: Request, res: express.Response, next: express.NextFunction): void;
+}

src/main.ts

@@ -30,6 +30,19 @@ import { router } from "./api/router-v1";
 // create a new express app
 const app = module.exports = express();
 
+// Load configuration file
+
+import { ConfigParser } from "./util/configParser";
+const configPath = (app.get("env") === "test") ? "config/test.yaml" : "config/config.yaml";
+const config = ConfigParser.parse(configPath);
+
+// include middlewares
+import { EngineMw } from "./api/middlewares/engine";
+import { PostgresMw } from "./api/middlewares/adapter";
+
+app.use(EngineMw(config));
+app.use(PostgresMw(config.connection));
+
 // parse the RAML spec and load osprey middleware
 ramlParser.loadFile("specs/blendb-api-v1.raml")
     .then((raml: any) => {

src/util/configParser.ts

@@ -21,6 +21,7 @@
 import { Metric, MetricOptions } from "../core/metric";
 import { Dimension, DimensionOptions } from "../core/dimension";
 import { View, ViewOptions, LoadView } from "../core/view";
+import { AggregationType, RelationType } from "../common/types";
 import { PoolConfig } from "pg";
 import * as fs from "fs";
 import * as yaml from "js-yaml";
@@ -32,10 +33,23 @@ export interface ViewParsingOptions {
     metrics: string[];
 }
 
+interface MetricStrOptions {
+    name: string;
+    aggregation: string;
+    dataType: string;
+}
+
+interface DimensionStrOptions {
+    name: string;
+    dataType: string;
+    parent?: string;
+    relation?: string;
+}
+
 interface ConfigSchema {
     views: ViewParsingOptions[];
-    metrics: MetricOptions[];
-    dimensions: DimensionOptions[];
+    metrics: MetricStrOptions[];
+    dimensions: DimensionStrOptions[];
 }
 
 interface ConfigFile {
@@ -80,18 +94,21 @@ export class ConfigParser {
         let dimMap: Map<string, Dimension> = new Map();
 
         for (let i = 0; i < metricsOpts.length; ++i) {
-            let met = new Metric(metricsOpts[i]);
+            let met = new Metric(this.parseMetOpts(metricsOpts[i]));
             parsed.metrics.push(met);
             metMap.set(met.name, met);
         }
 
         for (let i = 0; i < dimensionsOpts.length; ++i) {
-            let dim = new Dimension(dimensionsOpts[i]);
+            let dim = new Dimension(this.parseDimOpts(dimensionsOpts[i]));
             parsed.dimensions.push(dim);
             dimMap.set(dim.name, dim);
         }
 
         for (let i = 0; i < viewsOpts.length; ++i) {
+            if (!viewsOpts[i].metrics) {
+                viewsOpts[i].metrics = [];
+            }
             let viewOpts = ConfigParser.parseViewOpt(viewsOpts[i], metMap, dimMap);
             let view = new View(viewOpts);
             parsed.views.push(view);
@@ -134,4 +151,37 @@ export class ConfigParser {
         }
         return viewOpt;
     }
+
+    private static parseDimOpts (opts: DimensionStrOptions): DimensionOptions {
+        if (opts.parent || opts.relation) {
+            throw new Error("Sub dimensions not implemented in config parser !!!");
+        }
+        return {
+            name: opts.name,
+            dataType: opts.dataType,
+            parent: null,
+            relation: RelationType.NONE
+        };
+    }
+
+    private static parseMetOpts (opts: MetricStrOptions): MetricOptions {
+        return {
+            name: opts.name,
+            aggregation: this.strToAggregationType(opts.aggregation),
+            dataType : opts.dataType
+        };
+    }
+
+    private static strToAggregationType (str: string): AggregationType {
+        switch (str) {
+            case "sum":
+                return AggregationType.SUM;
+            case "avg":
+                return AggregationType.AVG;
+            case "count":
+                return AggregationType.COUNT;
+            default:
+                return AggregationType.NONE;
+        }
+    }
 }

test/scenario.ts

@@ -18,10 +18,12 @@
  * along with blendb.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+import { ConfigParser } from "../src/util/configParser";
 import { Metric } from "../src/core/metric";
 import { Dimension } from "../src/core/dimension";
 import { View } from "../src/core/view";
 import { AggregationType, RelationType } from "../src/common/types";
+import { Query} from "../src/common/query";
 
 interface EngineScenario {
     metrics: Metric[];
@@ -39,31 +41,17 @@ interface AdapterScenario {
     dateView: View;
 }
 
-const mets = [
-    new Metric({ name: "met:1", aggregation: AggregationType.SUM, dataType: "integer" }),
-    new Metric({ name: "met:2", aggregation: AggregationType.AVG, dataType: "integer" }),
-    new Metric({ name: "met:3", aggregation: AggregationType.AVG, dataType: "integer" }),
-    new Metric({ name: "met:4", aggregation: AggregationType.SUM, dataType: "integer" }),
-    new Metric({ name: "met:5", aggregation: AggregationType.SUM, dataType: "integer" }),
-    new Metric({ name: "met:6", aggregation: AggregationType.AVG, dataType: "integer" }),
-    new Metric({ name: "met:7", aggregation: AggregationType.COUNT, dataType: "integer" }),
-    new Metric({ name: "met:8", aggregation: AggregationType.COUNT, dataType: "integer" }),
-    new Metric({ name: "met:9", aggregation: AggregationType.SUM, dataType: "integer" }),
-    new Metric({ name: "met:10", aggregation: AggregationType.COUNT, dataType: "integer" }),
-];
+interface DataCtrlScenario {
+    wrongMet: Query;
+    wrongDim: Query;
+    correct: Query;
+}
 
-const dims = [
-    new Dimension({ name: "dim:1", dataType: "date" }),
-    new Dimension({ name: "dim:2", dataType: "date" }),
-    new Dimension({ name: "dim:3", dataType: "integer" }),
-    new Dimension({ name: "dim:4", dataType: "string" }),
-    new Dimension({ name: "dim:5", dataType: "string" }),
-    new Dimension({ name: "dim:6", dataType: "boolean" }),
-    new Dimension({ name: "dim:7", dataType: "integer" }),
-    new Dimension({ name: "dim:8", dataType: "integer" }),
-    new Dimension({ name: "dim:9", dataType: "date" }),
-    new Dimension({ name: "dim:10", dataType: "string" }),
-];
+const config = ConfigParser.parse("config/test.yaml");
+
+const mets = config.metrics;
+const dims = config.dimensions;
+const views = config.views;
 
 const wrongMet = new Metric({
     name: "met:11",
@@ -134,54 +122,6 @@ const dateSubDim = [
     }),
 ];
 
-const views = [
-    new View({
-        metrics: [mets[0], mets[1], mets[2]],
-        dimensions: [dims[0], dims[1]],
-        materialized: true
-    }),
-    new View({
-        metrics: [mets[0], mets[2], mets[4]],
-        dimensions: [dims[0], dims[1]],
-        materialized: true
-    }),
-    new View({
-        metrics: [mets[2], mets[3], mets[6]],
-        dimensions: [dims[3], dims[4]],
-        materialized: true
-    }),
-    new View({
-        metrics: [mets[5], mets[6]],
-        dimensions: [dims[2], dims[3], dims[4], dims[5]],
-        materialized: true
-    }),
-    new View({
-        metrics: [mets[7], mets[1], mets[2]],
-        dimensions: [dims[0], dims[1], dims[6]],
-        materialized: true
-    }),
-    new View({
-        metrics: [mets[0], mets[3]],
-        dimensions: [dims[0], dims[1]],
-        materialized: true
-    }),
-    new View({
-        metrics: [mets[7]],
-        dimensions: [dims[7], dims[8], dims[9]],
-        materialized: true
-    }),
-    new View({
-        metrics: [mets[8]],
-        dimensions: [dims[7], dims[8], dims[9]],
-        materialized: true
-    }),
-    new View({
-        metrics: [mets[9]],
-        dimensions: [dims[7], dims[8], dims[9]],
-        materialized: true
-    })
-];
-
 const engAuxView = [
     new View({
         metrics: [mets[0], mets[1], mets[2], mets[3], mets[4]],
@@ -356,3 +296,9 @@ export const adapterScenario: AdapterScenario = {
     join4View: join4View,
     dateView: dateView
 };
+
+export const dataCtrlScenario: DataCtrlScenario = {
+    wrongMet: { metrics: [wrongMet], dimensions: [dims[0]] },
+    wrongDim: { metrics: [mets[0]], dimensions: [wrongDim] },
+    correct: { metrics: [mets[0]], dimensions: [dims[0]] }
+};