Merge branch 'issue/62' into 'develop'

Issue #62: Add insert

See merge request !60

config/ci_test.yaml.example

@@ -215,28 +215,29 @@ enumTypes:
     -
         name: "enumtype:0"
         values:
-        - "test:0"
-        - "test:1"
-        - "test:2"
-        - "test:3"
+        - "male"
+        - "female"
+        - "binary"
+        - "undecided"
     -
         name: "enumtype:1"
         values:
-        - "test:4"
-        - "test:5"
-        - "test:6"        
+        - "test_4"
+        - "test_5"
+        - "test_6"  
+        - "string"      
     -
         name: "enumtype:2"
         values:
-        - "test:7"
-        - "test:8"
+        - "test_7"
+        - "test_8"
     -
         name: "enumtype:3"
         values:
-        - "test:9"        
+        - "test_9"     
 sources:
     -
-        name: "source:0"
+        name: "source_0"
         description: "source with 3 entries"
         fields:
         -
@@ -252,30 +253,55 @@ sources:
             description: "third entry"
             dataType: "string"          
     -
-        name: "source:1"
+        name: "source_1"
         description: "source with 2 entries"
         fields:
         -
             name: "fields:0"
             description: "first entry"            
-            dataType: "string"
+            dataType: "enumtype:0"
         -
             name: "fields:1"
             description: "second entry"            
             dataType: "string"   
     -
-        name: "source:2"
-        description: "source with one entry"
+        name: "source_2"
+        description: "source with one entry and undefined dataType"
         fields:
         -
             name: "fields:0"
             description: "first entry"
-            dataType: "string"
+            dataType: "notValid"
     -
-        name: "source:3"
+        name: "source_3"
         description: "source with one entry and without description"
         fields:
         -
-            name: "fields:3"
+            name: "fields:0"
             dataType: "string"
+    -
+        name: "source_4"        
+        description: "source with all core types from blendb"
+        fields:
+        -
+            name: "fields:0"
+            description: "first entry"
+            dataType: "integer"
+        - 
+            name: "fields:1"
+            description: "second entry"
+            dataType: "float"    
+        -
+            name: "fields:2"
+            description: "third entry"
+            dataType: "string"
+        -
+            name: "fields:3"
+            description: "fourth entry"
+            dataType: "boolean"
+        -
+            name: "fields:4"
+            description: "fifth entry"
+            dataType: "date"
+
             
\ No newline at end of file

src/adapter/monet.ts

@@ -20,6 +20,7 @@
 
 import { SQLAdapter } from "./sql";
 import { View } from "../core/view";
+import { Source } from "../core/source";
 import { FilterOperator } from "../core/filter";
 const MDB = require("monetdb")();
 
@@ -45,7 +46,10 @@ export class MonetAdapter extends SQLAdapter {
     }
     public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void {
         const query = this.getQueryFromView(view);
+        this.executeQuery(query, cb);
+    }
 
+    private executeQuery(query: string, cb: (error: Error, result?: any[]) => void): void {
         let pool: any = new MDB(this.config);
         pool.connect();
         pool.query(query).then((result: MonetResult) => {
@@ -78,6 +82,11 @@ export class MonetAdapter extends SQLAdapter {
         return false;
     }
 
+    public insertIntoSource(source: Source, data: any[], cb: (err: Error, result: any[]) =>  void): void {
+        const query = this.getQueryFromSource(source, data);
+        this.executeQuery(query, cb);
+    }
+
     protected typeCast(quotedValue: string, dt: string): string {
         switch (dt) {
             case "date":

src/adapter/postgres.ts

@@ -20,6 +20,7 @@
 
 import { SQLAdapter } from "./sql";
 import { View } from "../core/view";
+import { Source } from "../core/source";
 import { FilterOperator } from "../core/filter";
 import { Pool, PoolConfig } from "pg";
 
@@ -32,7 +33,10 @@ export class PostgresAdapter extends SQLAdapter {
     }
     public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void {
         const query = this.getQueryFromView(view);
+        this.executeQuery(query, cb);
+    }
 
+    private executeQuery(query: string, cb: (err: Error, result?: any[]) =>  void): void{
         this.pool.connect((err, client, done) => {
             if (err) {
                 cb (err);
@@ -45,6 +49,10 @@ export class PostgresAdapter extends SQLAdapter {
             });
         });
     }
+    public insertIntoSource(source: Source, data: any[], cb: (err: Error, result?: any[]) =>  void): void {
+        const query = this.getQueryFromSource(source, data);
+        this.executeQuery(query, cb);
+    }
 
     public materializeView(view: View): boolean {
         return false;

src/adapter/sql.ts

@@ -20,6 +20,7 @@
 
 import { Adapter } from "../core/adapter";
 import { Metric } from "../core/metric";
+import { Source } from "../core/source";
 import { Dimension } from "../core/dimension";
 import { Clause } from "../core/clause";
 import { Filter, FilterOperator } from "../core/filter";
@@ -844,4 +845,22 @@ export abstract class SQLAdapter extends Adapter {
         }
         return dims;
     }
+
+    public getQueryFromSource(source: Source, data: any[]): string {
+        let consult: string;
+        let colums: any[] = [];
+        let values: string[] = [];
+        let tam = Object.keys(data).length;
+        colums = source.fields.map(o => o.name);
+        for (let i = 0; i < tam; i++){
+            values[i] = data[colums[i]];
+        }
+        consult = "INSERT INTO " + source.name + " (\"";
+        consult = consult.concat(colums.join("\",\""));
+        consult = consult.concat("\") VALUES ('");
+        consult = consult.concat(values.join("' , '"));
+        consult = consult.concat("');");
+
+        return consult;
+    }
 }

src/api/controllers/collect.spec.ts

@@ -20,37 +20,259 @@
 
 import * as request from "supertest";
 import { expect } from "chai";
-
 import * as server from "../../main";
+import { Adapter } from "../../core/adapter";
+import { ConfigParser } from "../../util/configParser";
+import { Fixture as FixPostgres } from "../../../test/postgres/fixture";
+import { Fixture as FixMonet } from "../../../test/monet/fixture";
+import { MonetAdapter, MonetConfig } from "../../adapter/monet";
+import { PostgresAdapter } from "../../adapter/postgres";
 
 describe("API collect controller", () => {
 
-    it("should respond 400 when _id field is in request body", (done) => {
+    // Initializing
+    let config: any;
+    let adapter: Adapter;
+    let fixture;
+    before(function (done): void {
+        // Arrow function not used to get acces to this and skip the test
+        config = ConfigParser.parse("config/test.yaml");
+        if (config.adapter === "postgres") {
+            fixture = new FixPostgres(config.connection);
+            fixture.LoadSource(config.sources, config.struct.create, (err) => {
+                if (err) {
+                    throw err;
+                }
+                adapter = new PostgresAdapter(config.connection);
+                done();
+            });
+        }
+        else if (config.adapter === "monet") {
+            fixture = new FixMonet(config.connection);
+            fixture.load(config.sources, config.struct.create, (err) => {
+                if (err) {
+                    throw err;
+                }
+                let parsedConfig: MonetConfig = {
+                    user: config.connection.user,
+                    dbname: config.connection.database,
+                    password: config.connection.password,
+                    host: config.connection.host,
+                    port: config.connection.port
+                };
+                adapter = new MonetAdapter(parsedConfig);
+                done();
+            });
+        }
+        else {
+            this.skip();
+        }
+    });
+    it("should respond 500 when req.params.class does not exist on Sources", (done) => {
+        request(server)
+            .post("/v1/collect/thisisjustatest")
+            .send({"fields:1": 1, "fields:2": 2})
+            .expect(500)
+            .expect((res: any) => {
+
+                const message = "Query execution failed: " +
+                "Could not construct query with the paramters given.";
+                const error = "The source named 'thisisjustatest' was not found";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+                expect(res.body.error).to.be.eql(error);
+
+            })
+            .end(done);
+    });
+    it("should respond 500 when fields:0 does not exist on Source", (done) => {
+        request(server)
+            .post("/v1/collect/source_0")
+            .send({"fields:1": 1, "fields:2": 2})
+            .expect(500)
+            .expect((res: any) => {
+                const message = "Query execution failed: " +
+                "Could not construct query with the paramters given.";
+                const error = "The 'fields:0' wasn't informed on json";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+                expect(res.body.error).to.be.eql(error);
+
+            })
+            .end(done);
+    });
+    it("should respond 200 when data has been stored on source_0", (done) => {
+        request(server)
+            .post("/v1/collect/source_0")
+            .send({"fields:0": "teste", "fields:1": "test1", "fields:2": "teste2"})
+            .expect(200)
+            .expect((res: any) => {
+                const message = "Data has been successfully received and stored by the server";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+
+            })
+            .end(done);
+    });
+    it("should respond 500 when value isn't defined on enumtype:0 ", (done) => {
+        request(server)
+            .post("/v1/collect/source_1")
+            .send({"fields:0": 1, "fields:1": 2})
+            .expect(500)
+            .expect((res: any) => {
+                const message = "Query execution failed: " +
+                "Could not construct query with the paramters given.";
+                const error = "The value '1' from 'fields:0' isn't listed on enumtype:0";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+                expect(res.body.error).to.be.eql(error);
+
+            })
+            .end(done);
+    });
+    it("should respond 200 when data has been stored on source_1", (done) => {
+        request(server)
+            .post("/v1/collect/source_1")
+            .send({"fields:0": "male", "fields:1": "test1"})
+            .expect(200)
+            .expect((res: any) => {
+                const message = "Data has been successfully received and stored by the server";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+
+            })
+            .end(done);
+    });
+    it("should respond 500 when dataType from fields:0 isn't integer", (done) => {
+        request(server)
+            .post("/v1/collect/source_4")
+            .send({"fields:0" : "nope", "fields:1" : 95.5 , "fields:2" : "justabacon"
+            , "fields:3" : "1991-25-03" , "fields:4" : 1})
+            .expect(500)
+            .expect((res: any) => {
+
+                const message = "Query execution failed: " +
+                "Could not construct query with the paramters given.";
+                const error = "The value 'nope' from 'fields:0' isn't a type integer";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+                expect(res.body.error).to.be.eql(error);
+
+            })
+            .end(done);
+    });
+    it("should respond 500 when dataType from fields:1 isn't float", (done) => {
         request(server)
-            .post("/v1/collect/class")
-            .send({"_id": 1})
-            .expect(400)
+            .post("/v1/collect/source_4")
+            .send({"fields:0" : 1 , "fields:1" : "notafloat" , "fields:2" : "justabacon"
+            , "fields:3" : "1991-25-03" , "fields:4" : 1})
+            .expect(500)
             .expect((res: any) => {
-                const message = "Property named \"_id\" is protected.";
+
+                const message = "Query execution failed: " +
+                "Could not construct query with the paramters given.";
+                const error = "The value 'notafloat' from 'fields:1' isn't a type float";
                 expect(res.body).to.be.an("object");
                 expect(res.body).to.have.property("message");
                 expect(res.body.message).to.be.eql(message);
+                expect(res.body.error).to.be.eql(error);
+
             })
             .end(done);
     });
+    it("should respond 500 when dataType from fields:2 isn't string", (done) => {
+        request(server)
+            .post("/v1/collect/source_4")
+            .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : 1
+            , "fields:3" : "1991-25-03" , "fields:4" : 1})
+            .expect(500)
+            .expect((res: any) => {
+
+                const message = "Query execution failed: " +
+                "Could not construct query with the paramters given.";
+                const error = "The value '1' from 'fields:2' isn't a type string";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+                expect(res.body.error).to.be.eql(error);
+
+            })
+            .end(done);
+    });
+    it("should respond 500 when dataType from fields:3 isn't boolean", (done) => {
+        request(server)
+            .post("/v1/collect/source_4")
+            .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : "teste"
+            , "fields:3" : "notaboolean" , "fields:4" : "1999-10-10"})
+            .expect(500)
+            .expect((res: any) => {
 
-    it("should respond 500 on error on writing in the database", (done) => {
+                const message = "Query execution failed: " +
+                "Could not construct query with the paramters given.";
+                const error = "The value 'notaboolean' from 'fields:3' isn't a type boolean";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
+                expect(res.body.error).to.be.eql(error);
+
+            })
+            .end(done);
+    });
+    it("should respond 500 when the first dataType from fields:4 isn't date", (done) => {
         request(server)
-            .post("/v1/collect/class")
-            .send({"field1": 1, "field2": 2})
+            .post("/v1/collect/source_4")
+            .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : "teste"
+            , "fields:3" : "true" , "fields:4" : "1999-25-25"})
             .expect(500)
             .expect((res: any) => {
-                const message = "Error while writing to the database.";
+
+                const message = "Query execution failed: " +
+                "Could not construct query with the paramters given.";
+                const error = "The value '1999-25-25' from 'fields:4' isn't a type date";
                 expect(res.body).to.be.an("object");
                 expect(res.body).to.have.property("message");
                 expect(res.body.message).to.be.eql(message);
+                expect(res.body.error).to.be.eql(error);
+
             })
             .end(done);
     });
+    it("should respond 200 when sucessfull insert data on source_4", (done) => {
+        request(server)
+            .post("/v1/collect/source_4")
+            .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : "teste"
+            , "fields:3" : "true" , "fields:4" : "1999-10-10"})
+            .expect(200)
+            .expect((res: any) => {
+                const message = "Data has been successfully received and stored by the server";
+                expect(res.body).to.be.an("object");
+                expect(res.body).to.have.property("message");
+                expect(res.body.message).to.be.eql(message);
 
+            })
+            .end(done);
+    });
+    it("should respond 500 when dataType does not exist", (done) => {
+        request(server)
+        .post("/v1/collect/source_2")
+        .send({"fields:0" : 1 })
+        .expect(500)
+        .expect((res: any) => {
+            const message = "Query execution failed: " +
+            "Could not construct query with the paramters given.";
+            const error = "The dataType named 'notValid' was not found";
+            expect(res.body).to.be.an("object");
+            expect(res.body).to.have.property("message");
+            expect(res.body.message).to.be.eql(message);
+            expect(res.body.error).to.be.eql(error);
+
+        })
+        .end(done);
+    });
 });

src/api/controllers/collect.ts

@@ -19,16 +19,152 @@
  */
 
 import * as express from "express";
+import { Request } from "../types";
+import { Source, Field } from "../../core/source";
+import { EnumType } from "../../core/enumType";
+
+interface Valid{
+    [key: string]: (value: any) => boolean;
+}
 
 export class CollectCtrl {
-    public static write(req: express.Request, res: express.Response, next: express.NextFunction) {
-        if ("_id" in req.body) {
-            res.status(400)
-                .json({ message: "Property named \"_id\" is protected." });
+    public static write(req: Request, res: express.Response, next: express.NextFunction) {
+
+        const validador: Valid = {
+            "integer": function(value: any) {
+                let test = parseInt(value, 10);
+                if (test === undefined || isNaN(value) ) {
+                    return false;
+                }
+
+                else {
+                    return true;
+                }
+            },
+            "float": function(value: any) {
+                let test = parseFloat(value);
+                if (test === undefined || isNaN(value) ) {
+                    return false;
+                }
+
+                else {
+                    return true;
+                }
+            },
+            "string": function(value: any) {
+                let test = typeof(value);
+                if (test === "string") {
+                    return true;
+                }
+
+                else{
+                    return false;
+                }
+            },
+            "date": function(value: any) {
+
+                let test: string[] = [];
+                let date = new Date(value);
+                try {
+                    test = date.toISOString().split("T");
+                }
+                catch (e) {
+                    return false;
+                }
+                if (test[0] === value){
+                    return true;
+                }
+                else{
+                    return false;
+                }
+
+            },
+            "boolean": function(value: any) {
+                let test: string = value;
+                test = test.toLocaleLowerCase();
+                if (test === "true" || test === "false"){
+                    return true;
+                }
+
+                else{
+                    return false;
+                }
+            }
+        };
+
+        let fields: Field[] = [];
+        let data: string[] = [];
+        let types: string[] = [];
+        let source: Source;
+        let enumType: EnumType;
+
+        const id: string = req.params.class;
+        try {
+            source = req.engine.getSourceByName(id);
+            // If source does not exist them return error
+
+            fields = source.fields;
+
+            for (let i = 0; i < fields.length; i++){
+                data[i] = req.body[fields[i].name];
+                if (!data[i]){
+                    throw new Error(
+                        "The '" + fields[i].name + "' wasn't informed on json");
+                }
+            }
+
+            for (let i = 0; i < fields.length; i++){
+            if (validador[fields[i].dataType] !== undefined){
+                if (!validador[fields[i].dataType](data[i]) === true){
+                    throw new Error(
+                        "The value '" +  data[i] + "' from '" + fields[i].name +
+                        "' isn't a type " + fields[i].dataType);
+                }
+
+            }
+            else {
+                    enumType = req.engine.getEnumTypeByName(fields[i].dataType);
+
+                    types = enumType.values;
+                    let found: boolean = false;
+                    for (let j = 0; j < types.length; j++){
+                        if (data[i] === types[j]){
+                        found = true;
+                        break;
+                        }
+                    }
+                    if (!found) {
+                        throw new Error(
+                            "The value '" +  data[i] + "' from '" + fields[i].name +
+                            "' isn't listed on " + fields[i].dataType);
+                    }
+                }
+            }
+        }
+
+        catch (e) {
+            res.status(500).json({
+                message: "Query execution failed: " +
+                "Could not construct query with the paramters given.",
+                error: e.message
+             });
             return;
         }
 
-        res.status(500)
-            .json({ message: "Error while writing to the database." });
+        req.adapter.insertIntoSource(source, req.body, (err: Error, result: any[]) => {
+            if (err) {
+                res.status(500).json({
+                    message: "Insertion has failed",
+                    error: err
+                });
+                return;
+            }
+            else{
+                res.status(200).json({message: "Data has been successfully received and stored by the server"});
+                return;
+            }
+
+        });
+
     }
 }

src/api/controllers/engine.spec.ts

@@ -43,7 +43,7 @@ describe("API engine controller", () => {
             .expect((res: any) => {
                 let result = res.body;
                 expect(result).to.be.an("array");
-                expect(result).to.have.length(4);
+                expect(result).to.have.length(5);
             })
             .end(done);
     });

src/core/adapter.ts

@@ -19,8 +19,10 @@
  */
 
 import { View } from "./view";
+import { Source } from "./source";
 
 export abstract class Adapter {
     public abstract getDataFromView(view: View, cb: (err: Error, result: any[]) =>  void): void;
     public abstract materializeView(view: View): boolean;
+    public abstract insertIntoSource(source: Source, data: any[], cb: (err: Error, result: any[]) =>  void): void;
 }

src/core/engine.ts

@@ -103,6 +103,26 @@ export class Engine {
         return result;
     }
 
+    public getEnumTypeByName(name: string): EnumType {
+        let result = this.enumTypes.find(EnumType => EnumType.name === name);
+
+        if (!result) {
+            throw new Error("The dataType named '" + name + "' was not found");
+        }
+
+        return result;
+    }
+
+    public getSourceByName(name: string): Source {
+        let result = this.sources.find(source => source.name === name);
+
+        if (!result) {
+            throw new Error("The source named '" + name + "' was not found");
+        }
+
+        return result;
+    }
+
     public addDimension(dimension: Dimension): Dimension {
         if (this.graph.addDimension(dimension)) {
             this.dimensions.push(dimension);

src/util/configParser.ts

@@ -105,7 +105,7 @@ export class ConfigParser {
     public static parse(configPath: string): ParsedConfig {
         let config: ConfigSchema = yaml.safeLoad(fs.readFileSync(configPath, {
             encoding: "utf-8"
-        }));
+        })) as ConfigSchema;
 
         let connection: Connection = {
             user: process.env.BLENDB_DB_USER,
@@ -212,6 +212,7 @@ export class ConfigParser {
         }
 
         return parsed;
+
     }
 
     public static parseViewOpt(opts: ViewParsingOptions,

test/postgres/fixture.ts

@@ -20,6 +20,7 @@
 
 import { Client, PoolConfig } from "pg";
 import { View, LoadView } from "../../src/core/view";
+import { Source } from "../../src/core/source";
 import { each, series } from "async";
 import * as fs from "fs";
 
@@ -28,6 +29,12 @@ interface TransSet {
     data: string[];
 }
 
+interface SoucerParse{
+    name: string;
+    type: string[];
+    fields: string[];
+}
+
 export interface Schema {
     alias?: string;
     query?: string;
@@ -107,7 +114,7 @@ export class Fixture {
             case "boolean":
                 return "BOOLEAN";
             default:
-                return "";
+                return "TEXT";
         }
     }
 
@@ -148,4 +155,59 @@ export class Fixture {
         }
         return transaction;
     }
+    public LoadSource(source: Source[], create: boolean , cb: (err: Error) => void): void {
+
+        let client = new Client(this.config);
+
+        let query: string[] = [];
+        for (let i = 0; i < source.length; i++ ){
+        query[i] = this.ExtractData(source[i], create);
+        }
+
+        client.connect((error) => {
+            if (error) {
+                cb(error);
+                return;
+            }
+
+            series([(callback: (err: Error) => void) => {
+                each(query, (insere, cback) => {
+                    return client.query(insere , [], (err: Error) => cback(err));
+                }, (errQuery: Error) => callback(errQuery));
+             }], (errQuery: Error) => {
+                 if (errQuery) {
+                     client.end();
+                     cb(errQuery);
+                 }
+                 client.end((err) => {
+                     cb(err);
+                 });
+             });
+        });
+
+    }
+    private ExtractData(data: Source , create: boolean): string{
+        let name: string;
+        let type: string[];
+        let fields: string[];
+        let consult: string;
+
+        name = data.name;
+        type = data.fields.map((item) => item.dataType);
+        fields = data.fields.map((item) => item.name);
+
+        if (create){
+        consult = "CREATE TABLE " + name + " (" + '"';
+        for (let i = 0; i < fields.length; i++){
+            fields[i] = fields[i].concat('"' + " " + this.typeConvertion(type[i]));
+        }
+        consult = consult.concat(fields.join(", " + '"'));
+        consult = consult.concat(");");
+        }
+        else{
+        consult = "TRUNCATE TABLE " + name + ";";
+        }
+        return consult;
+
+    }
 }