From 056c9c82b89957d3aeda201b31f594b4fb8db287 Mon Sep 17 00:00:00 2001 From: Rafael <rpd17@inf.ufpr.br> Date: Mon, 9 Apr 2018 10:27:20 -0300 Subject: [PATCH] Issue #62: Add insert Signed-off-by: Rafael <rpd17@inf.ufpr.br> --- config/ci_test.yaml.example | 70 +++++--- src/adapter/monet.ts | 9 ++ src/adapter/postgres.ts | 8 + src/adapter/sql.ts | 19 +++ src/api/controllers/collect.spec.ts | 242 ++++++++++++++++++++++++++-- src/api/controllers/collect.ts | 148 ++++++++++++++++- src/api/controllers/engine.spec.ts | 2 +- src/core/adapter.ts | 2 + src/core/engine.ts | 20 +++ src/util/configParser.ts | 3 +- test/postgres/fixture.ts | 64 +++++++- 11 files changed, 546 insertions(+), 41 deletions(-) diff --git a/config/ci_test.yaml.example b/config/ci_test.yaml.example index 30192280..b319223d 100644 --- a/config/ci_test.yaml.example +++ b/config/ci_test.yaml.example @@ -213,30 +213,31 @@ dimensions: description: "A dimension of Blendb. Has 1 possible value." enumTypes: - - name: "enum type:0" + name: "enumtype:0" values: - - "test:0" - - "test:1" - - "test:2" - - "test:3" + - "male" + - "female" + - "binary" + - "undecided" - - name: "enum type:1" + name: "enumtype:1" values: - - "test:4" - - "test:5" - - "test:6" + - "test_4" + - "test_5" + - "test_6" + - "string" - - name: "enum type:2" + name: "enumtype:2" values: - - "test:7" - - "test:8" + - "test_7" + - "test_8" - - name: "enum type:3" + name: "enumtype:3" values: - - "test:9" + - "test_9" sources: - - name: "source:0" + name: "source_0" description: "source with 3 entries" fields: - @@ -252,30 +253,55 @@ sources: description: "third entry" dataType: "string" - - name: "source:1" + name: "source_1" description: "source with 2 entries" fields: - name: "fields:0" description: "first entry" - dataType: "string" + dataType: "enumtype:0" - name: "fields:1" description: "second entry" dataType: "string" - - name: "source:2" - description: "source with one entry" + name: "source_2" + description: "source with one entry and undefined dataType" fields: - name: "fields:0" description: "first entry" - dataType: "string" + dataType: "notValid" - - name: "source:3" + name: "source_3" description: "source with one entry and without description" fields: - - name: "fields:3" + name: "fields:0" + dataType: "string" + - + name: "source_4" + description: "source with all core types from blendb" + fields: + - + name: "fields:0" + description: "first entry" + dataType: "integer" + - + name: "fields:1" + description: "second entry" + dataType: "float" + - + name: "fields:2" + description: "third entry" dataType: "string" + - + name: "fields:3" + description: "fourth entry" + dataType: "boolean" + - + name: "fields:4" + description: "fifth entry" + dataType: "date" + \ No newline at end of file diff --git a/src/adapter/monet.ts b/src/adapter/monet.ts index aa01de01..321a3872 100644 --- a/src/adapter/monet.ts +++ b/src/adapter/monet.ts @@ -20,6 +20,7 @@ import { SQLAdapter } from "./sql"; import { View } from "../core/view"; +import { Source } from "../core/source"; import { FilterOperator } from "../core/filter"; const MDB = require("monetdb")(); @@ -45,7 +46,10 @@ export class MonetAdapter extends SQLAdapter { } public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void { const query = this.getQueryFromView(view); + this.executeQuery(query, cb); + } + private executeQuery(query: string, cb: (error: Error, result?: any[]) => void): void { let pool: any = new MDB(this.config); pool.connect(); pool.query(query).then((result: MonetResult) => { @@ -78,6 +82,11 @@ export class MonetAdapter extends SQLAdapter { return false; } + public insertIntoSource(source: Source, data: any[], cb: (err: Error, result: any[]) => void): void { + const query = this.getQueryFromSource(source, data); + this.executeQuery(query, cb); + } + protected typeCast(quotedValue: string, dt: string): string { switch (dt) { case "date": diff --git a/src/adapter/postgres.ts b/src/adapter/postgres.ts index 3566542b..e01b2bb3 100644 --- a/src/adapter/postgres.ts +++ b/src/adapter/postgres.ts @@ -20,6 +20,7 @@ import { SQLAdapter } from "./sql"; import { View } from "../core/view"; +import { Source } from "../core/source"; import { FilterOperator } from "../core/filter"; import { Pool, PoolConfig } from "pg"; @@ -32,7 +33,10 @@ export class PostgresAdapter extends SQLAdapter { } public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void { const query = this.getQueryFromView(view); + this.executeQuery(query, cb); + } + private executeQuery(query: string, cb: (err: Error, result?: any[]) => void): void{ this.pool.connect((err, client, done) => { if (err) { cb (err); @@ -45,6 +49,10 @@ export class PostgresAdapter extends SQLAdapter { }); }); } + public insertIntoSource(source: Source, data: any[], cb: (err: Error, result?: any[]) => void): void { + const query = this.getQueryFromSource(source, data); + this.executeQuery(query, cb); + } public materializeView(view: View): boolean { return false; diff --git a/src/adapter/sql.ts b/src/adapter/sql.ts index 0ef7ca22..71dbad4f 100644 --- a/src/adapter/sql.ts +++ b/src/adapter/sql.ts @@ -20,6 +20,7 @@ import { Adapter } from "../core/adapter"; import { Metric } from "../core/metric"; +import { Source } from "../core/source"; import { Dimension } from "../core/dimension"; import { Clause } from "../core/clause"; import { Filter, FilterOperator } from "../core/filter"; @@ -844,4 +845,22 @@ export abstract class SQLAdapter extends Adapter { } return dims; } + + public getQueryFromSource(source: Source, data: any[]): string { + let consult: string; + let colums: any[] = []; + let values: string[] = []; + let tam = Object.keys(data).length; + colums = source.fields.map(o => o.name); + for (let i = 0; i < tam; i++){ + values[i] = data[colums[i]]; + } + consult = "INSERT INTO " + source.name + " (" + '"'; + consult = consult.concat(colums.join('"' + "," + '"')); + consult = consult.concat('"' + ") VALUES ('"); + consult = consult.concat(values.join("' , '")); + consult = consult.concat("');"); + + return consult; + } } diff --git a/src/api/controllers/collect.spec.ts b/src/api/controllers/collect.spec.ts index 958196db..d4b57f0e 100644 --- a/src/api/controllers/collect.spec.ts +++ b/src/api/controllers/collect.spec.ts @@ -20,37 +20,259 @@ import * as request from "supertest"; import { expect } from "chai"; - import * as server from "../../main"; +import { Adapter } from "../../core/adapter"; +import { ConfigParser } from "../../util/configParser"; +import { Fixture as FixPostgres } from "../../../test/postgres/fixture"; +import { Fixture as FixMonet } from "../../../test/monet/fixture"; +import { MonetAdapter, MonetConfig } from "../../adapter/monet"; +import { PostgresAdapter } from "../../adapter/postgres"; describe("API collect controller", () => { - it("should respond 400 when _id field is in request body", (done) => { + // Initializing + let config: any; + let adapter: Adapter; + let fixture; + before(function (done): void { + // Arrow function not used to get acces to this and skip the test + config = ConfigParser.parse("config/test.yaml"); + if (config.adapter === "postgres") { + fixture = new FixPostgres(config.connection); + fixture.LoadSource(config.sources, config.struct.create, (err) => { + if (err) { + throw err; + } + adapter = new PostgresAdapter(config.connection); + done(); + }); + } + else if (config.adapter === "monet") { + fixture = new FixMonet(config.connection); + fixture.load(config.sources, config.struct.create, (err) => { + if (err) { + throw err; + } + let parsedConfig: MonetConfig = { + user: config.connection.user, + dbname: config.connection.database, + password: config.connection.password, + host: config.connection.host, + port: config.connection.port + }; + adapter = new MonetAdapter(parsedConfig); + done(); + }); + } + else { + this.skip(); + } + }); + it("should respond 500 when req.params.class does not exist on Sources", (done) => { + request(server) + .post("/v1/collect/thisisjustatest") + .send({"fields:1": 1, "fields:2": 2}) + .expect(500) + .expect((res: any) => { + + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The source named 'thisisjustatest' was not found"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 500 when fields:0 does not exist on Source", (done) => { + request(server) + .post("/v1/collect/source_0") + .send({"fields:1": 1, "fields:2": 2}) + .expect(500) + .expect((res: any) => { + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The 'fields:0' wasn't informed on json"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 200 when data has been stored on source_0", (done) => { + request(server) + .post("/v1/collect/source_0") + .send({"fields:0": "teste", "fields:1": "test1", "fields:2": "teste2"}) + .expect(200) + .expect((res: any) => { + const message = "Data has been successfully received and stored by the server"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + + }) + .end(done); + }); + it("should respond 500 when value isn't defined on enumtype:0 ", (done) => { request(server) - .post("/v1/collect/class") - .send({"_id": 1}) - .expect(400) + .post("/v1/collect/source_1") + .send({"fields:0": 1, "fields:1": 2}) + .expect(500) .expect((res: any) => { - const message = "Property named \"_id\" is protected."; + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value '1' from 'fields:0' isn't listed on enumtype:0"; expect(res.body).to.be.an("object"); expect(res.body).to.have.property("message"); expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + }) .end(done); }); + it("should respond 200 when data has been stored on source_1", (done) => { + request(server) + .post("/v1/collect/source_1") + .send({"fields:0": "male", "fields:1": "test1"}) + .expect(200) + .expect((res: any) => { + const message = "Data has been successfully received and stored by the server"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); - it("should respond 500 on error on writing in the database", (done) => { + }) + .end(done); + }); + it("should respond 500 when dataType from fields:0 isn't integer", (done) => { request(server) - .post("/v1/collect/class") - .send({"field1": 1, "field2": 2}) + .post("/v1/collect/source_4") + .send({"fields:0" : "nope", "fields:1" : 95.5 , "fields:2" : "justabacon" + , "fields:3" : "1991-25-03" , "fields:4" : 1}) .expect(500) .expect((res: any) => { - const message = "Error while writing to the database."; + + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value 'nope' from 'fields:0' isn't a type integer"; expect(res.body).to.be.an("object"); expect(res.body).to.have.property("message"); expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + }) .end(done); }); + it("should respond 500 when dataType from fields:1 isn't float", (done) => { + request(server) + .post("/v1/collect/source_4") + .send({"fields:0" : 1 , "fields:1" : "notafloat" , "fields:2" : "justabacon" + , "fields:3" : "1991-25-03" , "fields:4" : 1}) + .expect(500) + .expect((res: any) => { + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value 'notafloat' from 'fields:1' isn't a type float"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 500 when dataType from fields:2 isn't string", (done) => { + request(server) + .post("/v1/collect/source_4") + .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : 1 + , "fields:3" : "1991-25-03" , "fields:4" : 1}) + .expect(500) + .expect((res: any) => { + + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value '1' from 'fields:2' isn't a type string"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 500 when dataType from fields:3 isn't boolean", (done) => { + request(server) + .post("/v1/collect/source_4") + .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : "teste" + , "fields:3" : "notaboolean" , "fields:4" : "1999-10-10"}) + .expect(500) + .expect((res: any) => { + + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value 'notaboolean' from 'fields:3' isn't a type boolean"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 500 when the first dataType from fields:4 isn't date", (done) => { + request(server) + .post("/v1/collect/source_4") + .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : "teste" + , "fields:3" : "true" , "fields:4" : "1999-25-25"}) + .expect(500) + .expect((res: any) => { + + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The value '1999-25-25' from 'fields:4' isn't a type date"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); + it("should respond 200 when sucessfull insert data on source_4", (done) => { + request(server) + .post("/v1/collect/source_4") + .send({"fields:0" : 1 , "fields:1" : 95.5 , "fields:2" : "teste" + , "fields:3" : "true" , "fields:4" : "1999-10-10"}) + .expect(200) + .expect((res: any) => { + const message = "Data has been successfully received and stored by the server"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + + }) + .end(done); + }); + it("should respond 500 when dataType does not exist", (done) => { + request(server) + .post("/v1/collect/source_2") + .send({"fields:0" : 1 }) + .expect(500) + .expect((res: any) => { + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The dataType named 'notValid' was not found"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + + }) + .end(done); + }); }); diff --git a/src/api/controllers/collect.ts b/src/api/controllers/collect.ts index a273be52..a27afd4b 100644 --- a/src/api/controllers/collect.ts +++ b/src/api/controllers/collect.ts @@ -19,16 +19,152 @@ */ import * as express from "express"; +import { Request } from "../types"; +import { Source, Field } from "../../core/source"; +import { EnumType } from "../../core/enumType"; + +interface Valid{ + [key: string]: (value: any) => boolean; +} export class CollectCtrl { - public static write(req: express.Request, res: express.Response, next: express.NextFunction) { - if ("_id" in req.body) { - res.status(400) - .json({ message: "Property named \"_id\" is protected." }); + public static write(req: Request, res: express.Response, next: express.NextFunction) { + + const Validador: Valid = { + "integer": function(value: any) { + let test = parseInt(value, 10); + if (test === undefined || isNaN(value) ) { + return false; + } + + else { + return true; + } + }, + "float": function(value: any) { + let test = parseFloat(value); + if (test === undefined || isNaN(value) ) { + return false; + } + + else { + return true; + } + }, + "string": function(value: any) { + let test = typeof(value); + if (test === "string") { + return true; + } + + else{ + return false; + } + }, + "date": function(value: any) { + + let test: string[] = []; + let date = new Date(value); + try { + test = date.toISOString().split("T"); + } + catch (e) { + return false; + } + if (test[0] === value){ + return true; + } + else{ + return false; + } + + }, + "boolean": function(value: any) { + let test: string = value; + test = test.toLocaleLowerCase(); + if (test === "true" || test === "false"){ + return true; + } + + else{ + return false; + } + } + }; + + let fields: Field[] = []; + let data: string[] = []; + let Types: string[] = []; + let source: Source; + let enumType: EnumType; + + const id: string = req.params.class; + try { + source = req.engine.getSourceByName(id); + // If source does not exist them return error + + fields = source.fields; + + for (let i = 0; i < fields.length; i++){ + data[i] = req.body[fields[i].name]; + if (!data[i]){ + throw new Error( + "The '" + fields[i].name + "' wasn't informed on json"); + } + } + + for (let i = 0; i < fields.length; i++){ + if (Validador[fields[i].dataType] !== undefined){ + if (!Validador[fields[i].dataType](data[i]) === true){ + throw new Error( + "The value '" + data[i] + "' from '" + fields[i].name + + "' isn't a type " + fields[i].dataType); + } + + } + else { + enumType = req.engine.getEnumTypeByName(fields[i].dataType); + + Types = enumType.values; + let found: boolean = false; + for (let j = 0; j < Types.length; j++){ + if (data[i] === Types[j]){ + found = true; + break; + } + } + if (!found) { + throw new Error( + "The value '" + data[i] + "' from '" + fields[i].name + + "' isn't listed on " + fields[i].dataType); + } + } + } + } + + catch (e) { + res.status(500).json({ + message: "Query execution failed: " + + "Could not construct query with the paramters given.", + error: e.message + }); return; } - res.status(500) - .json({ message: "Error while writing to the database." }); + req.adapter.insertIntoSource(source, req.body, (err: Error, result: any[]) => { + if (err) { + res.status(500).json({ + message: "Insertion has failed", + error: err + }); + return; + } + else{ + res.status(200).json({message: "Data has been successfully received and stored by the server"}); + return; + } + + }); + } } diff --git a/src/api/controllers/engine.spec.ts b/src/api/controllers/engine.spec.ts index 79e8c180..45cb983c 100644 --- a/src/api/controllers/engine.spec.ts +++ b/src/api/controllers/engine.spec.ts @@ -43,7 +43,7 @@ describe("API engine controller", () => { .expect((res: any) => { let result = res.body; expect(result).to.be.an("array"); - expect(result).to.have.length(4); + expect(result).to.have.length(5); }) .end(done); }); diff --git a/src/core/adapter.ts b/src/core/adapter.ts index 20cb90cf..d70f5615 100644 --- a/src/core/adapter.ts +++ b/src/core/adapter.ts @@ -19,8 +19,10 @@ */ import { View } from "./view"; +import { Source } from "./source"; export abstract class Adapter { public abstract getDataFromView(view: View, cb: (err: Error, result: any[]) => void): void; public abstract materializeView(view: View): boolean; + public abstract insertIntoSource(source: Source, data: any[], cb: (err: Error, result: any[]) => void): void; } diff --git a/src/core/engine.ts b/src/core/engine.ts index e2d4ef14..63331949 100644 --- a/src/core/engine.ts +++ b/src/core/engine.ts @@ -103,6 +103,26 @@ export class Engine { return result; } + public getEnumTypeByName(name: string): EnumType { + let result = this.enumTypes.find(EnumType => EnumType.name === name); + + if (!result) { + throw new Error("The dataType named '" + name + "' was not found"); + } + + return result; + } + + public getSourceByName(name: string): Source { + let result = this.sources.find(source => source.name === name); + + if (!result) { + throw new Error("The source named '" + name + "' was not found"); + } + + return result; + } + public addDimension(dimension: Dimension): Dimension { if (this.graph.addDimension(dimension)) { this.dimensions.push(dimension); diff --git a/src/util/configParser.ts b/src/util/configParser.ts index 21cad7f9..6e15a86f 100644 --- a/src/util/configParser.ts +++ b/src/util/configParser.ts @@ -105,7 +105,7 @@ export class ConfigParser { public static parse(configPath: string): ParsedConfig { let config: ConfigSchema = yaml.safeLoad(fs.readFileSync(configPath, { encoding: "utf-8" - })); + })) as ConfigSchema; let connection: Connection = { user: process.env.BLENDB_DB_USER, @@ -212,6 +212,7 @@ export class ConfigParser { } return parsed; + } public static parseViewOpt(opts: ViewParsingOptions, diff --git a/test/postgres/fixture.ts b/test/postgres/fixture.ts index 4fdc0134..3233453e 100644 --- a/test/postgres/fixture.ts +++ b/test/postgres/fixture.ts @@ -20,6 +20,7 @@ import { Client, PoolConfig } from "pg"; import { View, LoadView } from "../../src/core/view"; +import { Source } from "../../src/core/source"; import { each, series } from "async"; import * as fs from "fs"; @@ -28,6 +29,12 @@ interface TransSet { data: string[]; } +interface SoucerParse{ + name: string; + type: string[]; + fields: string[]; +} + export interface Schema { alias?: string; query?: string; @@ -107,7 +114,7 @@ export class Fixture { case "boolean": return "BOOLEAN"; default: - return ""; + return "TEXT"; } } @@ -148,4 +155,59 @@ export class Fixture { } return transaction; } + public LoadSource(source: Source[], create: boolean , cb: (err: Error) => void): void { + + let client = new Client(this.config); + + let query: string[] = []; + for (let i = 0; i < source.length; i++ ){ + query[i] = this.ExtractData(source[i], create); + } + + client.connect((error) => { + if (error) { + cb(error); + return; + } + + series([(callback: (err: Error) => void) => { + each(query, (insere, cback) => { + return client.query(insere , [], (err: Error) => cback(err)); + }, (errQuery: Error) => callback(errQuery)); + }], (errQuery: Error) => { + if (errQuery) { + client.end(); + cb(errQuery); + } + client.end((err) => { + cb(err); + }); + }); + }); + + } + private ExtractData(data: Source , create: boolean): string{ + let name: string; + let type: string[]; + let fields: string[]; + let consult: string; + + name = data.name; + type = data.fields.map((item) => item.dataType); + fields = data.fields.map((item) => item.name); + + if (create){ + consult = "CREATE TABLE " + name + " (" + '"'; + for (let i = 0; i < fields.length; i++){ + fields[i] = fields[i].concat('"' + " " + this.typeConvertion(type[i])); + } + consult = consult.concat(fields.join(", " + '"')); + consult = consult.concat(");"); + } + else{ + consult = "TRUNCATE TABLE " + name + ";"; + } + return consult; + + } } -- GitLab