diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d517822f1affbe517e26d10599a1ac99370e025c..557eb80834b3822dae1ebe46060c99596d3064e9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -24,22 +24,11 @@ stages: - deploy -run_test_postgres: +run_test: stage: test script: - yarn install --frozen-lockfile --silent --non-interactive - - mv config/ci_postgres_test.env.example config/test.env - - yarn test - - yarn run lint - tags: - - node - - postgres - -run_test_monet: - stage: test - script: - - yarn install --frozen-lockfile --silent --non-interactive - - mv config/ci_monet_test.env.example config/test.env + - mv config/ci_database_test.env.example config/test.env - yarn test - yarn run lint tags: diff --git a/config/ci_database_test.env.example b/config/ci_database_test.env.example new file mode 100644 index 0000000000000000000000000000000000000000..a5be21fa2be2bb1ea0fb0b142772bd223bd745b2 --- /dev/null +++ b/config/ci_database_test.env.example @@ -0,0 +1,15 @@ +PORT=3000 +BLENDB_N_DB=2 +BLENDB_DB0_USER=runner +BLENDB_DB0_NAME=blendb_fixture +BLENDB_DB0_PASSWORD= +BLENDB_DB0_HOST=postgres +BLENDB_DB0_PORT=5432 +BLENDB_DB0_ADAPTER=postgres +BLENDB_DB1_USER=monetdb +BLENDB_DB1_NAME=blendb_fixture +BLENDB_DB1_PASSWORD=monetdb +BLENDB_DB1_HOST=monet +BLENDB_DB1_PORT=50000 +BLENDB_DB1_ADAPTER=monet +BLENDB_SCHEMA_FILE=config/ci_test.yaml.example diff --git a/config/ci_monet_test.env.example b/config/ci_monet_test.env.example deleted file mode 100644 index bbdd0ca9838c9b3c69e69f58ff4d0af8d136c419..0000000000000000000000000000000000000000 --- a/config/ci_monet_test.env.example +++ /dev/null @@ -1,8 +0,0 @@ -BLENDB_DB_USER=monetdb -BLENDB_DB_NAME=blendb_fixture -BLENDB_DB_PASSWORD=monetdb -BLENDB_DB_HOST=monet -BLENDB_DB_PORT=50000 -BLENDB_ADAPTER=monet -BLENDB_SCHEMA_FILE=config/ci_test.yaml.example -PORT=3000 diff --git a/config/ci_postgres_test.env.example b/config/ci_postgres_test.env.example deleted file mode 100644 index 64c1c5976c2e98b50fc42d2f223fa8fbbbfaf73f..0000000000000000000000000000000000000000 --- a/config/ci_postgres_test.env.example +++ /dev/null @@ -1,8 +0,0 @@ -BLENDB_DB_USER=runner -BLENDB_DB_NAME=blendb_fixture -BLENDB_DB_PASSWORD= -BLENDB_DB_HOST=postgres -BLENDB_DB_PORT=5432 -BLENDB_ADAPTER=postgres -BLENDB_SCHEMA_FILE=config/ci_test.yaml.example -PORT=3000 diff --git a/config/ci_test.yaml.example b/config/ci_test.yaml.example index 458dec196eed6805e077a641ff94497a63d71683..595347df26f8d64a400aa0663845b49dfb1eb8c5 100644 --- a/config/ci_test.yaml.example +++ b/config/ci_test.yaml.example @@ -46,6 +46,7 @@ enumTypes: sources: links: - config/market_sources.yaml.example + - config/source_test_only.yaml.example obj: - name: "Seller" diff --git a/config/market_dimensions.yaml.example b/config/market_dimensions.yaml.example index 23155f2c2997750d9788a13f695001b10e159e96..8cd85d5ddb39617b3f3f1ec05c0b63ca6d1d451e 100644 --- a/config/market_dimensions.yaml.example +++ b/config/market_dimensions.yaml.example @@ -19,27 +19,27 @@ - name: "dim:product:name" dataType: "string" - description: "Name of the product from market" + description: "Name of the product from market" - name: "dim:product:validity" dataType: "date" description: "Validity of the product from market" -- +- name: "dim:client:name" dataType: "string" description: "Name of the client from market" -- +- name: "dim:client:cpf" dataType: "string" description: "CPF of the client from market" - name: "dim:sell:registered" dataType: "boolean" - description: "Check if the client is registered" + description: "Check if the client is registered" - name: "dim:seller:id" dataType: "integer" - description: "id of the seller from market" + description: "id of the seller from market" - name: "dim:client:id" dataType: "integer" @@ -47,21 +47,39 @@ - name: "dim:sell:datein" dataType: "date" - description: "Date of the sell was realized" + description: "Date of the sell was realized" - name: "dim:buyout:datein" dataType: "date" - description: "Date of the buyout was realized" + description: "Date of the buyout was realized" - name: "dim:product:id" dataType: "integer" - description: "id of the product from market" + description: "id of the product from market" - name: "dim:provider:name" dataType: "string" - description: "Name of the provider from market" + description: "Name of the provider from market" - name: "dim:provider:id" dataType: "string" - description: "id of the provider from market" + description: "id of the provider from market" +- + name: "dim:buyout:day" + dataType: "integer" + parent: "dim:buyout:datein" + relation: "day" + description: "Date of the buyout was realized" +- + name: "dim:buyout:month" + dataType: "integer" + parent: "dim:buyout:datein" + relation: "month" + description: "Date of the buyout was realized" +- + name: "dim:buyout:year" + dataType: "integer" + parent: "dim:buyout:datein" + relation: year + description: "Date of the buyout was realized" diff --git a/config/market_enum.yaml.example b/config/market_enum.yaml.example index b3d9c0b0eb168a4412198d9d3a3f20c1f059a50b..496f1b9447f1f1ab3ad03033196924133b3674fd 100644 --- a/config/market_enum.yaml.example +++ b/config/market_enum.yaml.example @@ -13,5 +13,4 @@ - "vacation" - "sick leave" - "maternity leave" - - + - "stand by" diff --git a/config/market_views.yaml.example b/config/market_views.yaml.example index f9d9ef2c290d7a30a40f1f1859af914ecd2609e4..c5b681d117a51addace4ddb71a9c0da06224e072 100644 --- a/config/market_views.yaml.example +++ b/config/market_views.yaml.example @@ -79,3 +79,12 @@ - "met:seller:max:age" clauses: - "dim:seller:status==active" +- + alias: "view:SellerStatus" + data: "test/postgres/fixtures/sellerstatus.json" + origin: false + aliasAsName: true + metrics: + - "met:seller:count:age" + dimensions: + - "dim:seller:status" diff --git a/config/source_test_only.yaml.example b/config/source_test_only.yaml.example new file mode 100644 index 0000000000000000000000000000000000000000..02a89e58853841fe3dad433609578f34e5ea33da --- /dev/null +++ b/config/source_test_only.yaml.example @@ -0,0 +1,8 @@ +- + name: "test" + description: + fields: + - + name: "noDescriptionString" + description: + dataType: "string" diff --git a/config/test/postgres/fixtures/sellerstatus.json b/config/test/postgres/fixtures/sellerstatus.json new file mode 100644 index 0000000000000000000000000000000000000000..c9d05e0acd2ccd3cdd3913dd8f4e05a21894d4bc --- /dev/null +++ b/config/test/postgres/fixtures/sellerstatus.json @@ -0,0 +1,4 @@ +[ + {"met:seller:count:age" : 4, "dim:seller:status" : "active"}, + {"met:seller:count:age" : 1, "dim:seller:status" : "sick leave"} +] diff --git a/src/adapter/monet.ts b/src/adapter/monet.ts index 9a8b3229d3dfa9df779829cb92f5d9d4fa3451d6..7e98b3774db5a5a245409184ac5fd4753fa565d3 100644 --- a/src/adapter/monet.ts +++ b/src/adapter/monet.ts @@ -142,14 +142,6 @@ export class MonetAdapter extends SQLAdapter { pool.close(); } - /** - * Materialize a given view. - * @param view - View to be materialized. - */ - public materializeView(view: View): boolean { - return false; - } - /** * Asynchronously insert one register into a given Source. * @param source - Insertion "location". diff --git a/src/adapter/postgres.spec.ts b/src/adapter/postgres.spec.ts index a1d08fe1a96d04cdee0f5926906d29e46a1b01ba..02d722bbd89051b94b3125ed972d618fb2d4c251 100644 --- a/src/adapter/postgres.spec.ts +++ b/src/adapter/postgres.spec.ts @@ -25,246 +25,274 @@ import { Adapter } from "../core/adapter"; import { Fixture as FixPostgres } from "../../test/postgres/fixture"; import { Fixture as FixMonet } from "../../test/monet/fixture"; import { adapterScenario } from "../../test/scenario"; +import { eachOf } from "async"; -describe("Sql adapter", () => { +let adapters: Adapter[] = []; +let fixture; +function loadDb(db: string, index: number, cb: (err: any, result: Adapter) => void): void { let adapter: Adapter; - let fixture; - before(function (done): void { - if (adapterScenario.config.adapter === "postgres") { - fixture = new FixPostgres(adapterScenario.config.connection); - fixture.load(adapterScenario.config.loadViews, (err) => { - if (err) { - throw err; + if (db === "postgres") { + fixture = new FixPostgres(adapterScenario.config.connections[index]); + fixture.load(adapterScenario.config.loadViews, (err) => { + if (err) { + throw err; + } + adapter = new PostgresAdapter(adapterScenario.config.connections[index]); + cb(null, adapter); + }); + } + else if (db === "monet") { + fixture = new FixMonet(adapterScenario.config.connections[index]); + fixture.load(adapterScenario.config.loadViews, (err) => { + if (err) { + throw err; + } + let parsedConfig: MonetConfig = { + user: adapterScenario.config.connections[index].user, + dbname: adapterScenario.config.connections[index].database, + password: adapterScenario.config.connections[index].password, + host: adapterScenario.config.connections[index].host, + port: adapterScenario.config.connections[index].port + }; + adapter = new MonetAdapter(parsedConfig); + cb(null, adapter); + }); + } + else{ + cb("invalid adapter", null); + } +} + +describe("Sql adapter", () => { + // Initializing + before(function (done): void { + // Arrow function not used to get acces to this and skip the test + eachOf(adapterScenario.config.adapters , function(database, key: number, callback) { + loadDb(database, key, function(err, result ) { + if (err){ + return callback(err); } - adapter = new PostgresAdapter(adapterScenario.config.connection); - done(); - }); - } - else if (adapterScenario.config.adapter === "monet") { - fixture = new FixMonet(adapterScenario.config.connection); - fixture.load(adapterScenario.config.loadViews, (err) => { - if (err) { - throw err; + else{ + adapters[key] = result; } - let parsedConfig: MonetConfig = { - user: adapterScenario.config.connection.user, - dbname: adapterScenario.config.connection.database, - password: adapterScenario.config.connection.password, - host: adapterScenario.config.connection.host, - port: adapterScenario.config.connection.port - }; - adapter = new MonetAdapter(parsedConfig); - done(); + callback(); }); - } - else { - this.skip(); - } + }, function (err){ + if (err){ + this.skip(); + } + else{ + done(); + } + }); }); - // Tests - it("should get data from single materialized view", (done) => { - let view = adapterScenario.materializedView; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(5); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + + for (let i = 0; i < adapterScenario.config.ndb; i++ ){ + + it(adapterScenario.config.adapters[i] + ": " + "should get data from single materialized view", (done) => { + let view = adapterScenario.materializedView; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(5); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); }); - }); - it("should sort data from join of two views", (done) => { - let view = adapterScenario.sortView; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(4); - expect(result[0]).to.be.an("object"); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + it(adapterScenario.config.adapters[i] + ": " + "should sort data from join of two views", (done) => { + let view = adapterScenario.sortView; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(4); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); - }); - }); - it("should get data from single view (with sub-dimension)", (done) => { - let view = adapterScenario.subDimensionView; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(5); - expect(result[0]).to.be.an("object"); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + }); + it(adapterScenario.config.adapters[i] + ": " + "should get data from single view (with sub-dimension)", (done) => { + let view = adapterScenario.subDimensionView; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(5); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); }); - }); - it("should get data from join of one view", (done) => { - let view = adapterScenario.joinWithOneView; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(5); - expect(result[0]).to.be.an("object"); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + it(adapterScenario.config.adapters[i] + ": " + "should get data from join of one view", (done) => { + let view = adapterScenario.joinWithOneView; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(5); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); }); - }); - it("should get data from view with a single clause", (done) => { - let view = adapterScenario.filterWithEqual; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(1); - expect(result[0]).to.be.an("object"); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + it(adapterScenario.config.adapters[i] + ": " + "should get data from view with a single clause", (done) => { + let view = adapterScenario.filterWithEqual; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(1); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); }); - }); - it("should get data from reduce with the same attributes of view", (done) => { - let view = adapterScenario.reduceAsView; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(5); - expect(result[0]).to.be.an("object"); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + it(adapterScenario.config.adapters[i] + ": " + "should get data from reduce with the same attributes of view", (done) => { + let view = adapterScenario.reduceAsView; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(5); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); }); - }); - it("should get data from join into reduce with a single clause", (done) => { - let view = adapterScenario.filterAverageBought; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(3); - expect(result[0]).to.be.an("object"); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + it(adapterScenario.config.adapters[i] + ": " + "should get data from join into reduce with a single clause", (done) => { + let view = adapterScenario.filterAverageBought; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(3); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); }); - }); - it("should get data from view with multiple clauses", (done) => { - let view = adapterScenario.multipleClause; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(1); - expect(result[0]).to.be.an("object"); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + it(adapterScenario.config.adapters[i] + ": " + "should get data from view with multiple clauses", (done) => { + let view = adapterScenario.multipleClause; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(1); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); }); - }); - it("should get data from view with single clause, with more than one filter", (done) => { - let view = adapterScenario.singleClause; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(4); - expect(result[0]).to.be.an("object"); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + it(adapterScenario.config.adapters[i] + ": " + "should get data from view with single clause, with more than one filter", (done) => { + let view = adapterScenario.singleClause; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(4); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); }); - }); - it("should get data from view with no metrics", (done) => { - let view = adapterScenario.joinWithNoMetrics; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(5); - expect(result[0]).to.be.an("object"); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + it(adapterScenario.config.adapters[i] + ": " + "should get data from view with no metrics", (done) => { + let view = adapterScenario.joinWithNoMetrics; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(5); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); }); - }); - it("should get data from join with one metric and one view", (done) => { - let view = adapterScenario.growOneView; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(5); - expect(result[0]).to.be.an("object"); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + it(adapterScenario.config.adapters[i] + ": " + "should get data from join with one metric and one view", (done) => { + let view = adapterScenario.growOneView; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(5); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); }); - }); - it("should get data from join with unrelated dimension", (done) => { - let view = adapterScenario.JoinWithAncestors; - adapter.getDataFromView(view, (err, result) => { - expect(err).to.be.a("null"); - expect(result).to.be.an("array"); - expect(result).to.have.length(5); - expect(result[0]).to.be.an("object"); - let keys: string[] = []; - keys = keys.concat(view.metrics.map((item) => item.name)); - keys = keys.concat(view.dimensions.map((item) => item.name)); - result.forEach((row) => { - expect(row).to.be.an("object"); - expect(row).to.have.all.keys(keys); + it(adapterScenario.config.adapters[i] + ": " + "should get data from join with unrelated dimension", (done) => { + let view = adapterScenario.JoinWithAncestors; + adapters[i].getDataFromView(view, (err, result) => { + expect(err).to.be.a("null"); + expect(result).to.be.an("array"); + expect(result).to.have.length(5); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(view.metrics.map((item) => item.name)); + keys = keys.concat(view.dimensions.map((item) => item.name)); + result.forEach((row) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + done(); }); - done(); }); - }); + } }); diff --git a/src/adapter/postgres.ts b/src/adapter/postgres.ts index 3455dcd6f0dd22fa6d8109f266456c660cbe1517..30bf58376359ebe2a62b428831543872f75a24ad 100644 --- a/src/adapter/postgres.ts +++ b/src/adapter/postgres.ts @@ -86,14 +86,6 @@ export class PostgresAdapter extends SQLAdapter { this.executeQuery(query, cb); } - /** - * Materialize a given view. - * @param view - View to be materialized. - */ - public materializeView(view: View): boolean { - return false; - } - /** * Cast BlenDB data types to be used in PostgreSQL queries. * @param quotedValue - SQL query attribute wrapped by quotes. diff --git a/src/api/controllers/collect.spec.ts b/src/api/controllers/collect.spec.ts index c8c3947bf57a37c7efc933d9a7d1ad38463542fe..d189bd571e97c6f3ae2d7040dd7bc16897fff513 100644 --- a/src/api/controllers/collect.spec.ts +++ b/src/api/controllers/collect.spec.ts @@ -22,53 +22,80 @@ import * as request from "supertest"; import { expect } from "chai"; import * as server from "../../main"; import { Adapter } from "../../core/adapter"; -import { ConfigParser } from "../../util/configParser"; +import { ConfigParser, ParsedConfig } from "../../util/configParser"; import { Fixture as FixPostgres } from "../../../test/postgres/fixture"; import { Fixture as FixMonet } from "../../../test/monet/fixture"; import { MonetAdapter, MonetConfig } from "../../adapter/monet"; import { PostgresAdapter } from "../../adapter/postgres"; +import { eachOf } from "async"; +import { Source } from "../../core/source"; -describe("API collect controller", () => { +let adapter: Adapter[] = []; +let fixture; +let config: ParsedConfig; + + +function loadDb(db: string, index: number, cb: (err: any, result: Adapter) => void): void { + let adapter: Adapter; + if (db === "postgres") { + fixture = new FixPostgres(config.connections[index]); + fixture.loadSource(config.sources, (err) => { + if (err) { + throw err; + } + adapter = new PostgresAdapter(config.connections[index]); + cb(null, adapter); + }); + } + else if (db === "monet") { + fixture = new FixMonet(config.connections[index]); + fixture.loadSource(config.sources, (err) => { + if (err) { + throw err; + } + let parsedConfig: MonetConfig = { + user: config.connections[index].user, + dbname: config.connections[index].database, + password: config.connections[index].password, + host: config.connections[index].host, + port: config.connections[index].port + }; + adapter = new MonetAdapter(parsedConfig); + cb(null, adapter); + }); + } + else{ + cb("invalid adapter", null); + } +} +describe("API collect controller", () => { // Initializing - let config: any; - let adapter: Adapter; - let fixture; before(function (done): void { - // Arrow function not used to get acces to this and skip the test + // Arrow function not used to get acces to this and skip the test const configPath = process.env.BLENDB_SCHEMA_FILE; - config = ConfigParser.parse(configPath); - if (config.adapter === "postgres") { - fixture = new FixPostgres(config.connection); - fixture.loadSource(config.sources, (err) => { - if (err) { - throw err; + config = ConfigParser.parse(configPath); + + eachOf(config.adapters , function(database, key: number, callback) { + loadDb(database, key, function(err, result ) { + if (err){ + return callback(err); } - adapter = new PostgresAdapter(config.connection); - done(); - }); - } - else if (config.adapter === "monet") { - fixture = new FixMonet(config.connection); - fixture.loadSource(config.sources, (err) => { - if (err) { - throw err; + else{ + adapter[key] = result; } - let parsedConfig: MonetConfig = { - user: config.connection.user, - dbname: config.connection.database, - password: config.connection.password, - host: config.connection.host, - port: config.connection.port - }; - adapter = new MonetAdapter(parsedConfig); - done(); + callback(); }); - } - else { - this.skip(); - } + }, function (err){ + if (err){ + this.skip(); + } + else{ + done(); + } + }); }); + it("should respond 500 when req.params.class does not exist on Sources", (done) => { request(server) .post("/v1/collect/thisisjustatest") diff --git a/src/api/controllers/collect.ts b/src/api/controllers/collect.ts index edc4d1aa3e1c6e034e0c6cde26be26cb651cae3b..8086a8338eecafcbf31c714ee55294e66d13043e 100644 --- a/src/api/controllers/collect.ts +++ b/src/api/controllers/collect.ts @@ -110,7 +110,7 @@ export class CollectCtrl { }else{ return false; } - } + } } }; @@ -155,16 +155,11 @@ export class CollectCtrl { "The value '" + data[i] + "' from '" + fields[i].name + "' isn't listed on " + fields[i].enumType); } - } - - else if (fields[i].dataType !== DataType.NONE){ - // check if it's a valid datatype from query - if (!validador[EnumHandler.stringfyDataType(fields[i].dataType)](data[i]) === true){ + }else if (!validador[EnumHandler.stringfyDataType(fields[i].dataType)](data[i]) === true){ throw new Error( "The value '" + data[i] + "' from '" + fields[i].name + "' isn't a type " + [EnumHandler.stringfyDataType(fields[i].dataType)]); - } - } + } } } diff --git a/src/api/controllers/data.spec.ts b/src/api/controllers/data.spec.ts index 96a249435dd31a2d2c0c6893a27d062d154cede2..83710e99d13d7bc3041bea4bcf85d19c6b9fea18 100644 --- a/src/api/controllers/data.spec.ts +++ b/src/api/controllers/data.spec.ts @@ -192,7 +192,6 @@ describe("API data controller", () => { }) .end(done); }); - it("should respond 200 and get some data, sorted", (done) => { let query = parseQuery(tests.clausal); query.sort = "dim:product:id,met:product:avg:pricein"; @@ -218,5 +217,94 @@ describe("API data controller", () => { }) .end(done); }); + it("should respond 200 and get some data, using filters with GREATER", (done) => { + let query = parseQuery(tests.clausal); + query.filters = "dim:product:id>55"; + request(server) + .get("/v1/data") + .query(query) + .expect(200) + .expect((res: any) => { + let result = res.body; + expect(result).to.be.an("array"); + expect(result).to.have.length(2); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(tests.clausal.metrics.map((item) => item.name)); + keys = keys.concat(tests.clausal.dimensions.map((item) => item.name)); + result.forEach((row: any) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + }) + .end(done); + }); + it("should respond 200 and get some data, using filters with LOWEREQ", (done) => { + let query = parseQuery(tests.clausal); + query.filters = "dim:product:id<=55"; + request(server) + .get("/v1/data") + .query(query) + .expect(200) + .expect((res: any) => { + let result = res.body; + expect(result).to.be.an("array"); + expect(result).to.have.length(3); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(tests.clausal.metrics.map((item) => item.name)); + keys = keys.concat(tests.clausal.dimensions.map((item) => item.name)); + result.forEach((row: any) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + }) + .end(done); + }); + it("should respond 200 and get some data, using filters with default", (done) => { + let query = parseQuery(tests.seller); + query.filters = "dim:sell:registered==false"; + request(server) + .get("/v1/data") + .query(query) + .expect(200) + .expect((res: any) => { + let result = res.body; + expect(result).to.be.an("array"); + expect(result).to.have.length(1); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(tests.seller.metrics.map((item) => item.name)); + keys = keys.concat(tests.seller.dimensions.map((item) => item.name)); + result.forEach((row: any) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + }) + .end(done); + }); + + it("should respond 200 and get some data, using filters with default", (done) => { + let query = parseQuery(tests.expensive); + query.sort = "met:product:max:pricein" + request(server) + .get("/v1/data") + .query(query) + .expect(200) + .expect((res: any) => { + let result = res.body; + expect(result).to.be.an("array"); + expect(result).to.have.length(5); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(tests.expensive.metrics.map((item) => item.name)); + keys = keys.concat(tests.expensive.dimensions.map((item) => item.name)); + result.forEach((row: any) => { + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + }); + }) + .end(done); + }); }); diff --git a/src/api/controllers/engine.spec.ts b/src/api/controllers/engine.spec.ts index 6a9aa0989047d48d6f7e5de92389695e0417a43c..28f71adfc05b58429e9802e8b74a6dc0ab0beb5b 100644 --- a/src/api/controllers/engine.spec.ts +++ b/src/api/controllers/engine.spec.ts @@ -42,7 +42,7 @@ describe("API engine controller", () => { .expect((res: any) => { let result = res.body; expect(result).to.be.an("array"); - expect(result).to.have.length(7); + expect(result).to.have.length(8); }) .end(done); }); @@ -53,7 +53,7 @@ describe("API engine controller", () => { .expect((res: any) => { let result = res.body; expect(result).to.be.an("array"); - expect(result).to.have.length(16); + expect(result).to.have.length(19); }) .end(done); }); diff --git a/src/api/middlewares/error.spec.ts b/src/api/middlewares/error.spec.ts index be9181ab3f2f90643661c91f29d940b571ed6993..f510594d4d8c6b304e8bd4fab14266e74acf1f9a 100644 --- a/src/api/middlewares/error.spec.ts +++ b/src/api/middlewares/error.spec.ts @@ -53,4 +53,14 @@ describe("API error middleware", () => { }) .end(done); }); + it("should respond 400 when you connect, and do not send data", (done) => { + request(server) + .post("/v1/collect/Seller") + .expect(400) + .expect((res: any) => { + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("error"); + }) + .end(done); + }); }); diff --git a/src/core/adapter.ts b/src/core/adapter.ts index 9929807ccdaf598723773b2e9dfb830dae4c0d89..c293e8f27e7591dbadc28e407131768731e447a9 100644 --- a/src/core/adapter.ts +++ b/src/core/adapter.ts @@ -36,11 +36,6 @@ export abstract class Adapter { * @param cb.result - Data got from view. */ public abstract getDataFromView(view: View, cb: (err: Error, result: any[]) => void): void; - /** - * Materialize a given view. - * @param view - View to be materialized. - */ - public abstract materializeView(view: View): boolean; /** * Asynchronously insert one register into a given Source. * @param source - Insertion "location". diff --git a/src/core/engine.spec.ts b/src/core/engine.spec.ts index 5d459854b9684a75f297f75485204cdb92747e91..70689834a1e51e4f583af7e525cc78c04ee4cb7a 100644 --- a/src/core/engine.spec.ts +++ b/src/core/engine.spec.ts @@ -23,6 +23,8 @@ import { Engine } from "./engine"; import { FilterOperator } from "./filter"; import { View } from "./view"; import { engineScenario } from "../../test/scenario"; +import { EnumType } from "./enumType"; +import { Dimension } from "./dimension"; describe("engine class", () => { @@ -44,7 +46,7 @@ describe("engine class", () => { expect(optimalView.metrics).to.be.an("array"); expect(optimalView.dimensions).to.be.an("array"); expect(optimalView.metrics).to.have.length(16); - expect(optimalView.dimensions).to.have.length(16); + expect(optimalView.dimensions).to.have.length(19); }); it("should throw an exception, query with non-existent metric", () => { let error: boolean = false; @@ -288,4 +290,42 @@ describe("engine class", () => { expect(clause.filters[0].operator).to.be.equal(operators[op]); } }); + it("should return all views", () => { + let views: View[]; + views = engine.getViews(); + expect(views).to.have.length(9); + for (let i = 0 ; i < views.length; i++){ + expect(views[i]).to.have.property("metrics"); + expect(views[i]).to.have.property("dimensions"); + expect(views[i]).to.have.property("clauses"); + expect(views[i]).to.have.property("origin"); + expect(views[i]).to.have.property("operation"); + expect(views[i]).to.have.property("id"); + expect(views[i]).to.have.property("name"); + } + }); + it("should return null from addView", () => { + let view: View; + view = engine.addView(null); + expect(view).to.be.null; + }); + it("should return null from addDimension", () => { + let dimension: Dimension; + dimension = engine.addDimension(null); + expect(dimension).to.be.null; + }); + + it("should return all views", () => { + let enumtype: EnumType; + let error: boolean = false; + try { + enumtype = engine.getEnumTypeByName("test"); + } + catch (e){ + error = true; + expect(e.message).to.be.equal("The dataType named 'test' was not found"); + } + expect(error); + }); + }); diff --git a/src/core/filter.ts b/src/core/filter.ts index 624a3f1c53163ee360c4381dbed021dbadf28a32..336b524ed7c98264b43d2178d2522495732a7e7b 100644 --- a/src/core/filter.ts +++ b/src/core/filter.ts @@ -183,11 +183,12 @@ export class Filter { return false; } - if (op.operator === FilterOperator.GREATER || - op.operator === FilterOperator.LOWER || + if (op.operator === FilterOperator.GREATER || + op.operator === FilterOperator.LOWER || op.operator === FilterOperator.GREATEREQ || - op.operator === FilterOperator.LOWEREQ) { - if (op.target.dataType === DataType.DATE || op.target.dataType === DataType.INTEGER) { + op.operator === FilterOperator.LOWEREQ) { + if (op.target.dataType === DataType.DATE || + op.target.dataType === DataType.INTEGER) { return true; } diff --git a/src/main.ts b/src/main.ts index 7a0133ff93671072e40bfbdc71f0652b8aa965c8..456e82a832904d641c98bc64acbe373950c61012 100755 --- a/src/main.ts +++ b/src/main.ts @@ -47,12 +47,12 @@ import { PostgresMw, MonetMw } from "./api/middlewares/adapter"; import { ErrorMw } from "./api/middlewares/error"; app.use(EngineMw(config)); -if (config.adapter === "postgres") { - app.use(PostgresMw(config.connection)); +if (config.adapters[0] === "postgres") { + app.use(PostgresMw(config.connections[0])); } -else if (config.adapter === "monet") { - app.use(MonetMw(config.connection)); +else if (config.adapters[0] === "monet") { + app.use(MonetMw(config.connections[0])); } else { diff --git a/src/util/configParser.ts b/src/util/configParser.ts index 80b3041e086e2a5350e7be2a195a3f2cac083b57..73e6d5b75799df60e7601885a235f746f16a88bd 100644 --- a/src/util/configParser.ts +++ b/src/util/configParser.ts @@ -91,10 +91,10 @@ interface BuildView { } export interface ParsedConfig { - /** What adapter is in use. */ - adapter: string; - /** Connection parameters to connect in the database. */ - connection: Connection; + /** What adapters are in use. */ + adapters: string[]; + /** Connections parameters to connect in the database. */ + connections: Connection[]; /** Set of all views available. */ views: View[]; /** Set of all sources available. */ @@ -108,6 +108,8 @@ export interface ParsedConfig { loadViews: LoadView[]; /** SQL views build configuration. */ buildViews: BuildView[]; + /** Quantity of database */ + ndb: Number; } /** Required information to connect in a generic database. */ @@ -169,28 +171,37 @@ export class ConfigParser { encoding: "utf-8" })) as ConfigSchema; - let connection: Connection = { - user: process.env.BLENDB_DB_USER, - database: process.env.BLENDB_DB_NAME, - password: process.env.BLENDB_DB_PASSWORD, - host: process.env.BLENDB_DB_HOST, - port: parseInt(process.env.BLENDB_DB_PORT, 10) - }; + let connections: Connection[] = []; + + let ndb: number = parseInt(process.env.BLENDB_N_DB, 10) ? parseInt(process.env.BLENDB_N_DB, 10) : 1; + let adapters: string[] = []; + for ( let i = 0; i < ndb ; i++ ){ + adapters[i] = process.env["BLENDB_DB" + i + "_ADAPTER"]; + connections[i] = { + user: process.env["BLENDB_DB" + i + "_USER"], + database: process.env["BLENDB_DB" + i + "_NAME"], + password: process.env["BLENDB_DB" + i + "_PASSWORD"], + host: process.env["BLENDB_DB" + i + "_HOST"], + port: parseInt(process.env["BLENDB_DB" + i + "_PORT"], 10) + }; + } + let metricsOpts = config.metrics.obj; let viewsOpts = config.views.obj; let dimensionsOpts = config.dimensions.obj; let enumTypesOpts = config.enumTypes.obj; let sourcesOpts = config.sources.obj; let parsed: ParsedConfig = { - adapter: process.env.BLENDB_ADAPTER || "postgres", - connection: connection, + adapters: adapters, + connections: connections, views: [], metrics: [], enumTypes: [], dimensions: [], loadViews: [], buildViews: [], - sources: [] + sources: [], + ndb: ndb }; for (let i = 0; i < config.sources.links.length; i++) { diff --git a/src/util/scenarioHandler.ts b/src/util/scenarioHandler.ts index a36b3459b9dda16557082bbc68dc50ce478e37c9..ae981185caef05afaf3a9488cd0f5dea29b9a7da 100644 --- a/src/util/scenarioHandler.ts +++ b/src/util/scenarioHandler.ts @@ -136,6 +136,10 @@ export interface AdapterScenario { * View that contains the seller id and how many sell's he has done */ JoinWithAncestors: View; + /** + * View with false origin + */ + viewSumCount: View; /** * ParsedConfig that is load in scenario */ @@ -161,4 +165,12 @@ export interface DataCtrlScenario { * and it's id */ clausal: Query; + /** + * A query that returns the seller's that are registered + */ + seller: Query; + /** + * The products that are the most expensive + */ + expensive: Query; } diff --git a/test/postgres/fixtures/sellerstatus.json b/test/postgres/fixtures/sellerstatus.json new file mode 100644 index 0000000000000000000000000000000000000000..c9d05e0acd2ccd3cdd3913dd8f4e05a21894d4bc --- /dev/null +++ b/test/postgres/fixtures/sellerstatus.json @@ -0,0 +1,4 @@ +[ + {"met:seller:count:age" : 4, "dim:seller:status" : "active"}, + {"met:seller:count:age" : 1, "dim:seller:status" : "sick leave"} +] diff --git a/test/scenario.ts b/test/scenario.ts index 9d3831261bf23e218f7574e1e134734f9450624e..0cad7fa5f8fdf35f812b17962f0beb8405585252 100644 --- a/test/scenario.ts +++ b/test/scenario.ts @@ -107,6 +107,11 @@ const filters: { [key: string]: Filter } = { target: dims["dim:seller:status"], operator: FilterOperator.EQUAL, value: "active" + }), + "noneFilter": new Filter({ + target: dims["dim:seller:status"], + operator: FilterOperator.NONE, + value: "active" }) } @@ -158,7 +163,9 @@ const clauses: { [key: string]: Clause } = { // or between filteres => (A) || (B) new Clause ({filters: [filters["lower"],filters["averageBought"]]}), "equalClauseView": - new Clause ({filters: [filters["equalFilterView"]]}) + new Clause ({filters: [filters["equalFilterView"]]}), + "emptyClause": + new Clause ({filters: []}) } @@ -255,12 +262,12 @@ let qOpts : {[key: string]: QueryOpts} = { metrics : Object.keys(mets).map((key) => mets[key]), dimensions : Object.keys(dims).map((key) => dims[key]) }, - "queryNoMets": + "queryNoMets": { metrics: [wrongMet], dimensions: [dims["dim:product:name"]] }, - "queryNoDims": + "queryNoDims": { metrics: [mets["met:buyout:min:quantity"]], dimensions: [wrongDim] @@ -282,17 +289,22 @@ let qOpts : {[key: string]: QueryOpts} = { metrics : [], dimensions : [subdims["subdims_day"],subdims["subdims_month"]] }, - "wrongMet": + "wrongMet": { metrics: [wrongMet], dimensions: [dims["dim:product:id"]] }, - "wrongDim": + "seller": + { + metrics:[mets["met:sell:sum:quantity"]], + dimensions:[dims["dim:sell:registered"],dims["dim:product:id"]] + }, + "wrongDim": { metrics: [mets["met:sell:avg:quantity"]], dimensions: [wrongDim] }, - "correct": + "correct": { metrics: [mets["met:buyout:avg:quantity"]], dimensions: [dims["dim:provider:id"]] @@ -302,7 +314,18 @@ let qOpts : {[key: string]: QueryOpts} = { metrics: [mets["met:product:avg:pricein"]], dimensions: [dims["dim:product:name"], dims["dim:product:id"]] - } + }, + "expensive": + { + metrics: [mets["met:product:max:pricein"]], + dimensions: [dims["dim:product:name"], + dims["dim:product:id"]] + }, + "originfalse": + { + metrics: [mets["met:seller:count:age"]], + dimensions: [] + }, } const queries : {[key: string]: Query} = {}; @@ -362,6 +385,8 @@ const viewProduct = views["view:Product"]; const viewActiveSeller = views["view:ActiveSeller"]; +const viewSumCount = ViewHandler.queryReduce(queries["originfalse"],views["view:SellerStatus"]); + // Exports export const engineScenario: EngineScenario = { @@ -393,12 +418,15 @@ export const adapterScenario: AdapterScenario = { joinWithNoMetrics: joinWithNoMetrics, growOneView: growOneView, JoinWithAncestors:JoinWithAncestors, + viewSumCount: viewSumCount, config: config }; export const dataCtrlScenario: DataCtrlScenario = { + seller: queries["seller"], wrongMet: queries["wrongMet"], wrongDim: queries["wrongDim"], correct: queries["correct"], clausal: queries["clausal"], + expensive: queries["expensive"], };