diff --git a/config/ci_test.yaml.example b/config/ci_test.yaml.example
index eaac4332129dde3d194ebeab01eb4d415d2a6ec7..178999eab5bfbc82269149dc6b3e3b946e7b0489 100644
--- a/config/ci_test.yaml.example
+++ b/config/ci_test.yaml.example
@@ -211,9 +211,3 @@ dimensions:
parent: "dim:0"
relation: "year"
description: "A dimension of Blendb. Has 1 possible value."
- -
- name: "dim:12"
- dataType: "integer"
- parent: "dim:0"
- relation: "dayofweek"
- description: "A dimension of Blendb. Has 7 possible values."
diff --git a/package.json b/package.json
index abc4fca8573142f57681791ca0c40102f6212cb7..373398c84c0a5c02e70ffaa98e5ca758e4a3d096 100644
--- a/package.json
+++ b/package.json
@@ -23,10 +23,11 @@
"@types/chai": "^3.4.33",
"@types/express": "^4.0.33",
"@types/js-yaml": "^3.5.29",
- "@types/pg": "^6.1.38",
+ "@types/pg": "^6.1.45",
"async": "=2.4.1",
"express": "^4.0.33",
"js-yaml": "^3.8.2",
+ "monetdb": "^1.1.4",
"osprey": "^0.3.2",
"pg": "^6.1.5",
"ts-node": "^3.1.0",
diff --git a/src/adapter/monet.ts b/src/adapter/monet.ts
new file mode 100644
index 0000000000000000000000000000000000000000..aa01de01d99b7bc5d833340799886ba6e670e14a
--- /dev/null
+++ b/src/adapter/monet.ts
@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blend.
+ *
+ * blend is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blend is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blend. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { SQLAdapter } from "./sql";
+import { View } from "../core/view";
+import { FilterOperator } from "../core/filter";
+const MDB = require("monetdb")();
+
+export interface MonetConfig {
+ host: string;
+ port: number;
+ dbname: string;
+ user: string;
+ password: string;
+}
+
+interface MonetResult {
+ data: any[];
+ structure: {type: string, column: string, index: number}[];
+}
+
+export class MonetAdapter extends SQLAdapter {
+ private config: MonetConfig;
+
+ constructor (conf: MonetConfig) {
+ super();
+ this.config = conf;
+ }
+ public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void {
+ const query = this.getQueryFromView(view);
+
+ let pool: any = new MDB(this.config);
+ pool.connect();
+ pool.query(query).then((result: MonetResult) => {
+ if (result) {
+ let res = result.data.map((item) => {
+ let obj: any = {};
+ for (let i = 0; i < result.structure.length; ++i) {
+ let struct = result.structure[i];
+ if (struct.type === "timestamp") {
+ obj[struct.column] = new Date(item[struct.index]);
+ }
+ else {
+ obj[struct.column] = item[struct.index];
+ }
+ }
+ return obj;
+ });
+ cb(null, res);
+ }
+ else {
+ cb(null, null);
+ }
+ }).fail((err: Error) => {
+ cb(err, null);
+ });
+ pool.close();
+ }
+
+ public materializeView(view: View): boolean {
+ return false;
+ }
+
+ protected typeCast(quotedValue: string, dt: string): string {
+ switch (dt) {
+ case "date":
+ return "CAST(" + quotedValue + " AS TIMESTAMP)";
+ case "integer":
+ return "CAST(" + quotedValue + " AS INTEGER)";
+ case "boolean":
+ return "CAST(" + quotedValue + " AS BOOLEAN)";
+ default:
+ return quotedValue;
+ }
+ }
+
+ protected applyOperator(lSide: string, rSide: string, op: FilterOperator): string {
+ switch (op) {
+ case FilterOperator.EQUAL:
+ return lSide + " = " + rSide;
+ case FilterOperator.NOTEQUAL:
+ return "NOT(" + lSide + " = " + rSide + ")";
+ case FilterOperator.GREATER:
+ return lSide + " > " + rSide;
+ case FilterOperator.LOWER:
+ return lSide + " < " + rSide;
+ case FilterOperator.GREATEREQ:
+ return lSide + " >= " + rSide;
+ case FilterOperator.LOWEREQ:
+ return lSide + " <= " + rSide;
+ default:
+ return "";
+ }
+ }
+}
diff --git a/src/adapter/postgres.spec.ts b/src/adapter/postgres.spec.ts
index 597692c0ec4cdd718bc148ac4808a30113b5f162..e9b1cafe35ff475761fc4b45f04edbeaf13016de 100644
--- a/src/adapter/postgres.spec.ts
+++ b/src/adapter/postgres.spec.ts
@@ -21,8 +21,10 @@
import { expect } from "chai";
import { PostgresAdapter } from "./postgres";
+import { MonetAdapter, MonetConfig } from "./monet";
import { Adapter } from "../core/adapter";
-import { Fixture } from "../../test/postgres/fixture";
+import { Fixture as FixPostgres } from "../../test/postgres/fixture";
+import { Fixture as FixMonet } from "../../test/monet/fixture";
import { ConfigParser } from "../util/configParser";
import { adapterScenario } from "../../test/scenario";
@@ -33,16 +35,39 @@ describe("postgres adapter", () => {
let config: any;
let adapter: Adapter;
let fixture;
- before((done) => {
+ before(function (done) {
+ // Arrow function not used to get acces to this and skip the test
config = ConfigParser.parse("config/test.yaml");
- fixture = new Fixture(config.connection);
- fixture.load(config.loadViews, config.struct.create, (err) => {
- if (err) {
- throw err;
- }
- adapter = new PostgresAdapter(config.connection);
- done();
- });
+ if (config.adapter === "postgres") {
+ fixture = new FixPostgres(config.connection);
+ fixture.load(config.loadViews, config.struct.create, (err) => {
+ if (err) {
+ throw err;
+ }
+ adapter = new PostgresAdapter(config.connection);
+ done();
+ });
+ }
+ else if (config.adapter === "monet") {
+ fixture = new FixMonet(config.connection);
+ fixture.load(config.loadViews, config.struct.create, (err) => {
+ if (err) {
+ throw err;
+ }
+ let parsedConfig: MonetConfig = {
+ user: config.connection.user,
+ dbname: config.connection.database,
+ password: config.connection.password,
+ host: config.connection.host,
+ port: config.connection.port
+ };
+ adapter = new MonetAdapter(parsedConfig);
+ done();
+ });
+ }
+ else {
+ this.skip();
+ }
});
// Tests
it("should get data from single materialized view", (done) => {
diff --git a/src/adapter/postgres.ts b/src/adapter/postgres.ts
index 436b482eaa930966db202cff5e2c69ddae1e8fd9..3566542b2d1111ecaa8b8d625d1fc914925ff94b 100644
--- a/src/adapter/postgres.ts
+++ b/src/adapter/postgres.ts
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
+ * Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
* Departamento de Informatica - Universidade Federal do Parana
*
* This file is part of blend.
@@ -18,37 +18,12 @@
* along with blend. If not, see <http://www.gnu.org/licenses/>.
*/
-import { Adapter } from "../core/adapter";
-import { Metric } from "../core/metric";
-import { Dimension } from "../core/dimension";
-import { Clause } from "../core/clause";
-import { Filter, FilterOperator } from "../core/filter";
-import { AggregationType, RelationType } from "../common/types";
+import { SQLAdapter } from "./sql";
import { View } from "../core/view";
+import { FilterOperator } from "../core/filter";
import { Pool, PoolConfig } from "pg";
-interface ExpandedView {
- dimensions: Dimension[];
- metrics: Metric[];
- dimMetrics: Metric[];
- keys: Dimension[];
- clauses: Clause[];
- from: string;
- id: string;
- origin: boolean;
-}
-
-interface DimInfo {
- dim: Dimension;
- views: ExpandedView[];
-}
-
-interface DimTranslation {
- aliased: string;
- noalias: string;
-}
-
-export class PostgresAdapter extends Adapter {
+export class PostgresAdapter extends SQLAdapter {
private pool: Pool;
constructor (config: PoolConfig) {
@@ -75,809 +50,35 @@ export class PostgresAdapter extends Adapter {
return false;
}
- public getQueryFromView(view: View): string {
- /*
- Find the base (materialized) views that has this data and exapand
- than (make a parse to the format used in the adapter)
- */
- const materialized = this.searchMaterializedViews(view).sort((a, b) => {
- return (a.id < b.id) ? -1 : 1;
- }).map((item) => {
- return {
- id: item.id,
- from: "view_" + item.id,
- dimMetrics: [],
- metrics: item.metrics.filter((i) => {
- return view.metrics.some((j) => i.name === j.name);
- }),
- dimensions: item.dimensions,
- keys: item.keys,
- clauses: item.clauses,
- origin: item.origin
- };
- });
-
- // Remove repeated views from the result
- let partialJoin = [materialized[0]];
- for (let i = 1; i < materialized.length; ++i) {
- if (materialized[i - 1].id !== materialized[i].id) {
- partialJoin.push(materialized[i]);
- }
- }
-
- const blackList = view.dimensions.map((i) => i.name);
- for (let i = 0; i < view.clauses.length; ++i) {
- if (view.clauses[i].filters.length === 1) {
- let filter = view.clauses[i].filters[0];
- if (filter.operator === FilterOperator.EQUAL) {
- blackList.push(filter.target.name);
- }
- }
- }
-
- /*
- If there is more than one source of data (tables/views)
- a join is needed.
-
- Partial Join represents how many sources still exists,
- every join reduces this number.
- */
- let clausesToCover = view.clauses.map((i) => i);
- while (partialJoin.length > 1) {
- /*
- Variable map finds what dimenensions are still needed to
- complete this query, they are required for 2 reasons.
- 1 - To make joins
- 2 - Because they are in the query
-
- For each view that has this dimension we add one score to
- this dimension, if they are in the query the same.
-
- Automatically if the dimension is in the query there will be
- at least one view with this atribute (or the query could not be
- completed) so dimensions in the query always have score of
- at least 2.
-
- To make a join the dimension must be in 2 different views,
- creating a score of 2 either.
-
- If the score is less than 2 so this dimension is not required
- anymore and can be removed.
- */
- let map: { [key: string]: number } = {};
- let partialsChange = false;
- for (let i = 0; i < partialJoin.length; ++i) {
- const dims = partialJoin[i].dimensions;
- for (let k = 0; k < dims.length; ++k) {
- if (!map[dims[k].name]) {
- map[dims[k].name] = 1;
- }
-
- else {
- ++map[dims[k].name];
- }
- }
- }
-
- for (let i = 0; i < view.dimensions.length; ++i) {
- let dim = view.dimensions[i];
- while (dim !== null) {
- if (map[dim.name]) {
- ++map[dim.name];
- }
- dim = dim.parent;
- }
- }
-
- /*
- Also mark scores for dimensions inside clauses
- */
- for (let i = 0; i < clausesToCover.length; ++i) {
- for (let j = 0; j < clausesToCover[i].targets.length; ++j) {
- if (map[clausesToCover[i].targets[j].name]) {
- ++map[clausesToCover[i].targets[j].name];
- }
- }
- }
-
- for (let i = 0; i < partialJoin.length; ++i) {
- const dims = partialJoin[i].dimensions.filter((item) => {
- return map[item.name] > 1;
- });
- const keys = partialJoin[i].keys.filter((item) => {
- return map[item.name] > 1;
- });
- /*
- At this point the dimensions with less than score 2
- are removed, if this happens the view is agreggated
- again, with less dimensions, removing this dimension
- from the view.
- */
-
- let coveredClauses: Clause[] = [];
- let notCoveredClauses: Clause[] = [];
- /*
- If all dimensions in a clause are a sub set of the
- dimensions of a view, this clause is apllied now,
- propagating the clause to this point.
-
- Then this clause is removed from the set of clauses
- */
- for (let j = 0; j < clausesToCover.length; ++j) {
- if (clausesToCover[j].isCovered(partialJoin[i].dimensions)) {
- coveredClauses.push(clausesToCover[j]);
- }
- else {
- notCoveredClauses.push(clausesToCover[j]);
- }
- }
- clausesToCover = notCoveredClauses.filter((clause) => {
- return !partialJoin[i].clauses.some((c) => c.id === clause.id);
- });
- if (dims.length < partialJoin[i].dimensions.length || coveredClauses.length > 0) {
- const partial = new View({
- metrics: partialJoin[i].metrics,
- dimensions: dims,
- keys: keys,
- origin: false,
- clauses: coveredClauses.concat(partialJoin[i].clauses),
- materialized: false
- });
- const from = "(" +
- this.buildQuery(partial, [partialJoin[i]]) +
- ") AS view_" + partial.id + "\n";
-
- partialJoin[i].id = partial.id;
- partialJoin[i].dimensions = partial.dimensions;
- partialJoin[i].keys = partial.keys;
- partialJoin[i].origin = partial.origin;
- partialJoin[i].from = from;
-
- partialsChange = true;
- }
- }
- /*
- If at least one of the views changed (have the number of
- dimensions reduced) returns to the begining of the loop
- again.
-
- Othewise we need to make a join.
- */
- if (!partialsChange) {
- /*
- Sorting the views by keys.
- If the keys are identical, then they
- will be in sequence, and views with identical
- keys can be joined.
-
- Sort an array of keys is the same as sort a
- array of strings.
- */
- const sorted = partialJoin.sort((a, b) => {
- return this.compareKeys(a.keys, b.keys, blackList);
- });
- /*
- SUPER WARNING: WHEN THE BLACK LIST IS USED THE VIEW IS
- UNMATERIALIZEBLE, BUT THE QUERY CAN AGGREGATE THE VALUES
-
- The blackList is the array of dimensions of the query plus
- the dimensions in filters using the equality operator.
- In further coments is expained that the relation to make
- a join must be one-to-one between the tables.
- However and a dimension is choosed, a sub view is
- created and if the relation is preserved in the sub view
- the query can be agregated, but this view cannot be re-used
- so it is unmaterializeble.
-
- The equality operator is the same as select one subview.
- */
- /*
- First of all, the remaining views are splited in segments.
-
- A segment contains views with the same keys that are great
- to make joins. Joins like this do not create "dimensional
- metrics".
-
- In joins like this one row of each view will be connected
- with at most one row of each other table.
- */
- const segment = [[sorted[0]]];
- let segmentId = 0;
- for (let i = 1; i < sorted.length; ++i) {
- if (this.compareKeys(sorted[i - 1].keys, sorted[i].keys, blackList) === 0) {
- segment[segmentId].push(sorted[i]);
- }
- else {
- ++segmentId;
- segment.push([sorted[i]]);
- }
- }
-
- partialJoin = [];
- let ableToJoin = false;
-
- for (let i = 0; i < segment.length; ++i) {
- /*
- If a segment has more than one view, a join can be made
- */
- if (segment[i].length > 1) {
- let mets: Metric[] = [];
- let clauses: Clause[] = [];
- let dims: Dimension[] = [];
- let dimMetrics: Metric[] = [];
- for (let j = 0; j < segment[i].length; ++j) {
- mets = mets.concat(segment[i][j].metrics);
- clauses = clauses.concat(segment[i][j].clauses);
- dims = dims.concat(segment[i][j].dimensions);
- dimMetrics = dimMetrics.concat(segment[i][j].dimMetrics);
- }
-
- dims = this.removeDuplicatedDimensions(dims);
- /*
- Its atributes are just concatenated and the
- duplicates removed.
- */
-
- const partial = new View({
- metrics: mets,
- dimensions: dims,
- keys: segment[i][0].keys,
- origin: false,
- clauses: clauses,
- materialized: false
- });
- const viewFrom = "(" +
- this.buildQuery(partial, segment[i]) +
- ") AS view_" + partial.id + "\n";
-
- partialJoin.push({
- id: partial.id,
- from: viewFrom,
- dimMetrics: dimMetrics,
- metrics: partial.metrics,
- dimensions: partial.dimensions,
- keys: partial.keys,
- clauses: partial.clauses,
- origin: partial.origin
- });
-
- ableToJoin = true;
- }
-
- else {
- /*
- If the segment has just one view, anything can be
- done at this point, so just reinsert this view in
- set of views.
- */
- partialJoin.push(segment[i][0]);
- }
- }
-
- /*
- If at least one join was made in the last part (a segment
- with more than one view) than return to the begining of the
- loop.
-
- This permits after a join remove the dimensions that were
- only choosen to this join, and are no longer required
-
- Ideally the joins should be restrict the join method used
- above, but in some cases this can not be done.
-
- So if all the segments have only one view inside, move
- to the next method.
- */
- if (!ableToJoin) {
- /*
- At this point 2 views will be joined, first the
- similarity with each pair of views is calculated,
- the pair with the biggedt similarity will be joined.
-
- Similarity is calculated with the number of common
- dimensions in the keys.
- */
- let similarity = 0;
- let idx0 = 0;
- let idx1 = 1;
- for (let i = 0; i < partialJoin.length; ++i) {
- for (let j = i + 1 ; j < partialJoin.length; ++j) {
- const pi = partialJoin[i].keys;
- const pj = partialJoin[j].keys;
- let score = this.similarDimensions (pi, pj);
- if (similarity < score) {
- similarity = score;
- idx0 = i;
- idx1 = j;
- }
- }
- }
-
- const partial0 = partialJoin[idx0];
- const partial1 = partialJoin[idx1];
-
- partialJoin.splice(idx1, 1);
- partialJoin.splice(idx0, 1);
-
- /*
- Once the views are select they are joined with the
- same method, concatenedted its atributes and
- removing duplicates, however the nasty effect of
- this join is the creation of "dimensional metrics".
-
- "Dimensional metrics" are metrics that can no longer
- be aggregated, and at this point to the end
- of a query they will act as dimensions.
-
- This change happens to avoid inconsistency generated
- by a join where one row of one table can be connected
- to more than one of other table.
-
- Take this example.
-
- View0 : metrics [met0], dimensions [dim0]
- values: [{met0: 10, dim0: 1}]
- View1 : metrics [met1], dimensions [dim2]
- values: [{met1: 10, dim2: 1}. {met1: 5, dim2: 2}]
- View2 : metrics [], dimensions [dim0, dim1, dim2]
- values: [
- {dim0: 1, dim1: 1, dim2: 1},
- {dim0: 1, dim1: 1, dim2: 2}
- ]
- The query is metrics [met0, met1] and dimensions [dim1]
- First a join of View0 and View1 is made, the result
- is: [
- {dim0: 1, dim1: 1, dim2: 1, met0: 10},
- {dim0: 1, dim1: 1, dim2: 2, met0: 10}
- ]
- Note that the value of met0 is duplicated.
- Now dim0 is removed, than joined with view2 resulting
- in: [
- {met1: 10, dim1: 1, dim2: 1, met0: 10},
- {met1: 5 , dim1: 1, dim2: 2, met0: 10}
- ]
-
- Lets assume that the agregation is SUM
- If we remove dim2 and re-agregate the result is: [
- {met1: 15, dim1: 1, met0: 20}
- ]
-
- This result is wrong. The replication of the value
- met0 affects the result.
-
- See if met1 was not required, first the dimemnsion would
- be reduced, left dim0 and dim1, than joined that reduced
- again resulting in the value [
- {dim1:1, met0: 10}
- ]
-
- Is this case there is no duplication and the aggregation
- does not include more rows than should.
-
- To solve this problem the met0 must become a dimension,
- in other words, not aggregated again. If the met0 was
- not agregated in the query met0, met1, dim1 the result
- is: [
- {met1: 15, dim1: 1, met0: 10}
- ]
- what is compatible.
-
- After this extreme long explanation what must be
- known is: Joining views with diferent keys
- generate "dimensional metrics".
-
- Views with "dimensional metrics" can not used for future
- queries because can not be re-agregated, so this must be
- avoided and is one-query only views.
- */
-
- let dimMetrics: Metric[];
- let mets: Metric[];
- let dims = partial0.dimensions.concat(partial1.dimensions);
- dims = this.removeDuplicatedDimensions(dims);
- let keys = partial0.keys.concat(partial1.keys);
- keys = this.removeDuplicatedDimensions(keys);
- if (partial0.keys.length === similarity) {
- /*
- Here the metrics become dimensions, but the effect
- can be reduced. If the keys of partial0
- is a sub set of the keys ou partial1
- than the number of rows of partial 1 is not
- affected, in other words the metrics of partial1
- can be aggregated and does not need to become
- dimensions.
- */
- partial0.dimMetrics = partial0.dimMetrics.concat(partial0.metrics);
- partial0.metrics = [];
- mets = partial1.metrics;
- }
-
- else if (partial1.keys.length === similarity) {
- /*
- The same occurs if the keys of partia1 is a subset
- of partial0.
- */
- partial1.dimMetrics = partial1.dimMetrics.concat(partial1.metrics);
- partial1.metrics = [];
- mets = partial0.metrics;
- }
-
- else {
- /*
- But if there is no sub set, than both sides have
- the metrics turned in dimensions.
- */
- partial0.dimMetrics = partial0.dimMetrics.concat(partial0.metrics);
- partial0.metrics = [];
- partial1.dimMetrics = partial1.dimMetrics.concat(partial1.metrics);
- partial1.metrics = [];
- mets = [];
- }
-
- dimMetrics = partial0.dimMetrics.concat(partial1.dimMetrics);
- const partial = new View({
- metrics: mets,
- dimensions: dims,
- keys: keys,
- origin: false,
- clauses: partial0.clauses.concat(partial1.clauses),
- materialized: false
- });
- const id = new View({
- metrics: mets.concat(dimMetrics),
- dimensions: dims,
- keys: keys,
- origin: false,
- clauses: partial0.clauses.concat(partial1.clauses),
- materialized: false
- }).id;
- const viewFrom = "(" +
- this.buildQuery(partial, [partial0, partial1]) +
- ") AS view_" + id + "\n";
- partialJoin.push({
- id: id,
- from: viewFrom,
- dimMetrics: dimMetrics,
- metrics: mets,
- dimensions: dims,
- keys: keys,
- clauses: partial.clauses,
- origin: false
- });
-
- }
- }
- }
-
- /*
- When only one view remain, the query is made and a ;
- is added at the end.
-
- TODO: Probrably this last line adds one more
- layer to the query, that is in fact unnecessary.
- Think a way to remove-it.
- */
- return this.buildQuery(view, partialJoin) + ";";
- }
-
- private searchMaterializedViews(view: View): View[] {
- let r: View[] = [];
- if (view.materialized) {
- return [view];
- }
-
- else {
- let children = view.childViews;
- for (let i = 0; i < children.length; ++i) {
- r = r.concat(this.searchMaterializedViews(children[i]));
- }
- }
-
- return r;
- }
-
- private buildQuery(target: View, views: ExpandedView[]) {
- const metrics = target.metrics;
- const dimensions = target.dimensions;
- const clauses = target.clauses;
-
- let dimMap: {[key: string]: DimInfo} = {};
- let nameMap: {[key: string]: ExpandedView} = {};
-
- for (let i = 0; i < views.length; ++i) {
- const mets = views[i].metrics;
- const dims = views[i].dimensions;
- for (let j = 0; j < mets.length; ++j) {
- if (!nameMap[mets[j].name]) {
- nameMap[mets[j].name] = views[i];
- }
-
- }
-
- for (let j = 0; j < dims.length; ++j) {
- if (!dimMap[dims[j].name]) {
- dimMap[dims[j].name] = {
- dim: dims[j],
- views: [views[i]]
- };
- nameMap[dims[j].name] = views[i];
- }
-
- else {
- dimMap[dims[j].name].views.push(views[i]);
- }
- }
- }
-
- // Projection
- const strMetrics = metrics.map((metric) => {
- const view = nameMap[metric.name];
- if (view) {
- return this.translateMetric(metric, view);
- }
-
- return "";
- }).filter((item) => item !== "");
-
- const parsedDimensions = dimensions.map((dimension) => {
- let dim = dimension;
- while (!nameMap[dim.name]) {
- dim = dim.parent;
- }
- const view = nameMap[dim.name];
- return this.translateDimension(dimension, dim, view);
- });
-
- let parsedDimMetrics: DimTranslation[] = [];
-
- for (let i = 0; i < views.length; ++i) {
- const dimMets = views[i].dimMetrics.map((item) => {
- return this.translateDimMetric(item, views[i]);
- });
-
- parsedDimMetrics = parsedDimMetrics.concat(dimMets);
- }
-
- const totalDimensions = parsedDimensions.concat(parsedDimMetrics);
-
- const strDimensions = totalDimensions.map ((item) => item.aliased);
- const grouped = totalDimensions.map((item) => item.noalias);
- const elements = strMetrics.concat(strDimensions);
-
- // Joins
- let conds: string[] = [];
- for (let i of Object.keys(dimMap)) {
- let remainViews = dimMap[i].views.slice();
- let dim = dimMap[i].dim;
- let leftSide = this.buildColumn(dim, remainViews.shift().id);
- if (remainViews.length > 0) {
- while (remainViews.length > 0) {
- const id = remainViews.shift().id;
- const rightSide = this.buildColumn(dim, id);
- conds.push(leftSide + " = " + rightSide);
- }
- }
-
- }
-
- // Selection
- let covered: Clause[] = [];
- for (let i = 0; i < views.length; ++i) {
- // Get the clauses that children already cover
- covered = covered.concat(views[i].clauses);
- }
-
- const toCover = clauses.filter((item) => !covered.some ((clause) => {
- return clause.id === item.id;
- }));
-
- toCover.forEach((item) => {
- const clause = "(" + this.translateClause(item, nameMap) + ")";
- if (clause !== "()") {
- conds.push(clause);
- }
- });
-
- // Assembly
-
- const projection = "SELECT " + elements.join(",");
- const source = " FROM " + views.map((view) => view.from).join(",");
- const selection = (conds.length > 0) ? " WHERE " + conds.join(" AND ") : "";
- let grouping = "";
- if (grouped.length > 0) {
- grouping = " GROUP BY " + grouped.join(",");
- }
-
- return projection + source + selection + grouping;
-
- }
-
- private getAggregateFunction(aggrType: AggregationType, origin: boolean): string {
- switch (aggrType) {
- case AggregationType.SUM:
- return "SUM";
- case AggregationType.AVG:
- return "AVG";
- case AggregationType.COUNT:
- return (origin) ? "COUNT" : "SUM";
- case AggregationType.MAX:
- return "MAX";
- case AggregationType.MIN:
- return "MIN";
- default:
- return "";
- }
-
- }
-
- private translateRelation(relation: RelationType, arg: string): string {
- switch (relation) {
- case RelationType.DAY:
- return this.applyRelation("EXTRACT", ["DAY FROM "], [arg]);
- case RelationType.MONTH:
- return this.applyRelation("EXTRACT", ["MONTH FROM "], [arg]);
- case RelationType.YEAR:
- return this.applyRelation("EXTRACT", ["YEAR FROM "], [arg]);
- case RelationType.DAYOFWEEK:
- return this.applyRelation("EXTRACT", ["DOW FROM "], [arg]);
+ protected typeCast(quotedValue: string, dt: string): string {
+ switch (dt) {
+ case "date":
+ return quotedValue + "::DATE";
+ case "integer":
+ return quotedValue + "::INTEGER";
+ case "boolean":
+ return quotedValue + "::BOOLEAN";
default:
- return "";
+ return quotedValue;
}
-
- }
-
- private applyRelation(name: string, args: string[], values: string[]): string {
- /*
- This adapter uses the concept of functions in Postgres to
- implement BLENDB sub-dimention relations, this functions
- applys the transformation to build the call of a Postgres
- funtion. Note that this function can be native from postgres,
- like EXTRACT, or even implemented on the database.
- This function is short and only used in the translateRelation
- method however is a bit complex and is possible to be used
- several times, because of that is puted appart to make easyer update
- and avoid problems
- Example
- applyRelation ("EXTRACT", "["DAY FROM"]", ["view_0.date"])
- output: EXTRACT(DAY FROM view_0.date)
- */
- return name + "(" + args.map((item, idx) => item + values[idx]).join(",") + ")";
}
- private buildColumn (item: Metric|Dimension, id: string): string {
- const quotedName = "\"" + item.name + "\"";
- return "view_" + id + "." + quotedName;
- }
-
- private translateClause(clause: Clause, map: {[key: string]: ExpandedView}): string {
- const r = clause.filters.map((item) => {
- return this.translateFilter(item, map);
- }).filter((item) => {
- return item !== "";
- });
- return r.join(" OR ");
- }
-
- private translateFilter(filter: Filter, map: {[key: string]: ExpandedView}): string {
- if (!map[filter.target.name]) {
- return "";
- }
-
- const viewId = map[filter.target.name].id;
- const leftSide = this.buildColumn(filter.target, viewId);
- const op = this.translateOperator(filter.operator);
- const dataType = this.translateDataType(filter.target.dataType);
- const quotedValue = "'" + filter.value + "'";
- return leftSide + op + quotedValue + dataType;
- }
-
- private translateMetric(metric: Metric, view: ExpandedView): string {
- const func = this.getAggregateFunction(metric.aggregation, view.origin);
- const quotedName = "\"" + metric.name + "\"";
- const extMetric = func + "(" + this.buildColumn(metric, view.id) + ")";
- return extMetric + " AS " + quotedName;
- }
-
- private translateDimMetric(metric: Metric, view: ExpandedView): DimTranslation {
- const quotedName = "\"" + metric.name + "\"";
- const extMetric = this.buildColumn(metric, view.id);
- return { aliased: extMetric + " AS " + quotedName, noalias: extMetric };
- }
-
- private translateDimension(dimension: Dimension,
- ancestor: Dimension,
- view: ExpandedView): DimTranslation {
- const quotedName = "\"" + dimension.name + "\"";
- let extDimension = this.buildColumn(ancestor, view.id);
- let aux = dimension;
- while (aux.name !== ancestor.name) {
- extDimension = this.translateRelation(aux.relation, extDimension);
- aux = aux.parent;
- }
- return { aliased: extDimension + " AS " + quotedName, noalias: extDimension };
- }
-
- private translateOperator(op: FilterOperator): string {
+ protected applyOperator(lSide: string, rSide: string, op: FilterOperator): string {
switch (op) {
case FilterOperator.EQUAL:
- return " = ";
+ return lSide + " = " + rSide;
case FilterOperator.NOTEQUAL:
- return " != ";
+ return lSide + " != " + rSide;
case FilterOperator.GREATER:
- return " > ";
+ return lSide + " > " + rSide;
case FilterOperator.LOWER:
- return " < ";
+ return lSide + " < " + rSide;
case FilterOperator.GREATEREQ:
- return " >= ";
+ return lSide + " >= " + rSide;
case FilterOperator.LOWEREQ:
- return " <= ";
+ return lSide + " <= " + rSide;
default:
return "";
}
-
- }
-
- private translateDataType(dt: string ): string {
- switch (dt) {
- case "date":
- return "::DATE";
- case "integer":
- return "::INTEGER";
- case "boolean":
- return "::BOOLEAN";
- default:
- return "";
- }
- }
-
- private compareKeys(a: Dimension[], b: Dimension[], blackList: string[]): number {
- /*
- SUPER WARNING: WHEN THE BLACK LIST IS USED THE VIEW IS
- UNMATERIALIZEBLE, BUT THE QUERY CAN AGGREGATE THE VALUES
- */
- let c = a.filter((i) => !blackList.some((bad) => bad === i.name));
- let d = b.filter((i) => !blackList.some((bad) => bad === i.name));
- let length = 0;
- let res = c.length - d.length;
- if (c.length < d.length) {
- length = c.length;
- }
- else {
- length = d.length;
- }
-
- for (let i = 0; i < length; ++i) {
- if (c[i].name < d[i].name) {
- return -1;
- }
-
- else if (c[i].name > d[i].name) {
- return 1;
- }
- }
-
- return res;
- }
-
- private similarDimensions(a: Dimension[], b: Dimension[]): number {
- let count = 0;
- for (let i = 0; i < a.length; ++i) {
- if (b.some((itemB) => a[i].name === itemB.name)) {
- count++;
- }
- }
- return count;
- }
-
- private removeDuplicatedDimensions(candidateDims: Dimension[]): Dimension[] {
- let filterDims: { [key: string]: boolean } = {};
- const dims = [];
- for (let i = 0; i < candidateDims.length; ++i) {
- if (!filterDims[candidateDims[i].name]) {
- dims.push(candidateDims[i]);
- filterDims[candidateDims[i].name] = true;
- }
- }
- return dims;
}
}
diff --git a/src/adapter/sql.ts b/src/adapter/sql.ts
new file mode 100644
index 0000000000000000000000000000000000000000..244863a3d75ed938e69d3250402bcdb4347fdb41
--- /dev/null
+++ b/src/adapter/sql.ts
@@ -0,0 +1,840 @@
+/*
+ * Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blend.
+ *
+ * blend is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blend is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blend. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Adapter } from "../core/adapter";
+import { Metric } from "../core/metric";
+import { Dimension } from "../core/dimension";
+import { Clause } from "../core/clause";
+import { Filter, FilterOperator } from "../core/filter";
+import { AggregationType, RelationType } from "../common/types";
+import { View } from "../core/view";
+
+interface ExpandedView {
+ dimensions: Dimension[];
+ metrics: Metric[];
+ dimMetrics: Metric[];
+ keys: Dimension[];
+ clauses: Clause[];
+ from: string;
+ id: string;
+ origin: boolean;
+}
+
+interface DimInfo {
+ dim: Dimension;
+ views: ExpandedView[];
+}
+
+interface DimTranslation {
+ aliased: string;
+ noalias: string;
+ alias: string;
+ expanded: boolean;
+}
+
+export abstract class SQLAdapter extends Adapter {
+ public getQueryFromView(view: View): string {
+ /*
+ Find the base (materialized) views that has this data and exapand
+ than (make a parse to the format used in the adapter)
+ */
+ const materialized = this.searchMaterializedViews(view).sort((a, b) => {
+ return (a.id < b.id) ? -1 : 1;
+ }).map((item) => {
+ return {
+ id: item.id,
+ from: "view_" + item.id,
+ dimMetrics: [],
+ metrics: item.metrics.filter((i) => {
+ return view.metrics.some((j) => i.name === j.name);
+ }),
+ dimensions: item.dimensions,
+ keys: item.keys,
+ clauses: item.clauses,
+ origin: item.origin
+ };
+ });
+
+ // Remove repeated views from the result
+ let partialJoin = [materialized[0]];
+ for (let i = 1; i < materialized.length; ++i) {
+ if (materialized[i - 1].id !== materialized[i].id) {
+ partialJoin.push(materialized[i]);
+ }
+ }
+
+ const blackList = view.dimensions.map((i) => i.name);
+ for (let i = 0; i < view.clauses.length; ++i) {
+ if (view.clauses[i].filters.length === 1) {
+ let filter = view.clauses[i].filters[0];
+ if (filter.operator === FilterOperator.EQUAL) {
+ blackList.push(filter.target.name);
+ }
+ }
+ }
+
+ /*
+ If there is more than one source of data (tables/views)
+ a join is needed.
+
+ Partial Join represents how many sources still exists,
+ every join reduces this number.
+ */
+ let clausesToCover = view.clauses.map((i) => i);
+ while (partialJoin.length > 1) {
+ /*
+ Variable map finds what dimenensions are still needed to
+ complete this query, they are required for 2 reasons.
+ 1 - To make joins
+ 2 - Because they are in the query
+
+ For each view that has this dimension we add one score to
+ this dimension, if they are in the query the same.
+
+ Automatically if the dimension is in the query there will be
+ at least one view with this atribute (or the query could not be
+ completed) so dimensions in the query always have score of
+ at least 2.
+
+ To make a join the dimension must be in 2 different views,
+ creating a score of 2 either.
+
+ If the score is less than 2 so this dimension is not required
+ anymore and can be removed.
+ */
+ let map: { [key: string]: number } = {};
+ let partialsChange = false;
+ for (let i = 0; i < partialJoin.length; ++i) {
+ const dims = partialJoin[i].dimensions;
+ for (let k = 0; k < dims.length; ++k) {
+ if (!map[dims[k].name]) {
+ map[dims[k].name] = 1;
+ }
+
+ else {
+ ++map[dims[k].name];
+ }
+ }
+ }
+
+ for (let i = 0; i < view.dimensions.length; ++i) {
+ let dim = view.dimensions[i];
+ while (dim !== null) {
+ if (map[dim.name]) {
+ ++map[dim.name];
+ }
+ dim = dim.parent;
+ }
+ }
+
+ /*
+ Also mark scores for dimensions inside clauses
+ */
+ for (let i = 0; i < clausesToCover.length; ++i) {
+ for (let j = 0; j < clausesToCover[i].targets.length; ++j) {
+ if (map[clausesToCover[i].targets[j].name]) {
+ ++map[clausesToCover[i].targets[j].name];
+ }
+ }
+ }
+
+ for (let i = 0; i < partialJoin.length; ++i) {
+ const dims = partialJoin[i].dimensions.filter((item) => {
+ return map[item.name] > 1;
+ });
+ const keys = partialJoin[i].keys.filter((item) => {
+ return map[item.name] > 1;
+ });
+ /*
+ At this point the dimensions with less than score 2
+ are removed, if this happens the view is agreggated
+ again, with less dimensions, removing this dimension
+ from the view.
+ */
+
+ let coveredClauses: Clause[] = [];
+ let notCoveredClauses: Clause[] = [];
+ /*
+ If all dimensions in a clause are a sub set of the
+ dimensions of a view, this clause is apllied now,
+ propagating the clause to this point.
+
+ Then this clause is removed from the set of clauses
+ */
+ for (let j = 0; j < clausesToCover.length; ++j) {
+ if (clausesToCover[j].isCovered(partialJoin[i].dimensions)) {
+ coveredClauses.push(clausesToCover[j]);
+ }
+ else {
+ notCoveredClauses.push(clausesToCover[j]);
+ }
+ }
+ clausesToCover = notCoveredClauses.filter((clause) => {
+ return !partialJoin[i].clauses.some((c) => c.id === clause.id);
+ });
+ if (dims.length < partialJoin[i].dimensions.length || coveredClauses.length > 0) {
+ const partial = new View({
+ metrics: partialJoin[i].metrics,
+ dimensions: dims,
+ keys: keys,
+ origin: false,
+ clauses: coveredClauses.concat(partialJoin[i].clauses),
+ materialized: false
+ });
+ const from = "(" +
+ this.buildQuery(partial, [partialJoin[i]]) +
+ ") AS view_" + partial.id + "\n";
+
+ partialJoin[i].id = partial.id;
+ partialJoin[i].dimensions = partial.dimensions;
+ partialJoin[i].keys = partial.keys;
+ partialJoin[i].origin = partial.origin;
+ partialJoin[i].from = from;
+
+ partialsChange = true;
+ }
+ }
+ /*
+ If at least one of the views changed (have the number of
+ dimensions reduced) returns to the begining of the loop
+ again.
+
+ Othewise we need to make a join.
+ */
+ if (!partialsChange) {
+ /*
+ Sorting the views by keys.
+ If the keys are identical, then they
+ will be in sequence, and views with identical
+ keys can be joined.
+
+ Sort an array of keys is the same as sort a
+ array of strings.
+ */
+ const sorted = partialJoin.sort((a, b) => {
+ return this.compareKeys(a.keys, b.keys, blackList);
+ });
+ /*
+ SUPER WARNING: WHEN THE BLACK LIST IS USED THE VIEW IS
+ UNMATERIALIZEBLE, BUT THE QUERY CAN AGGREGATE THE VALUES
+
+ The blackList is the array of dimensions of the query plus
+ the dimensions in filters using the equality operator.
+ In further coments is expained that the relation to make
+ a join must be one-to-one between the tables.
+ However and a dimension is choosed, a sub view is
+ created and if the relation is preserved in the sub view
+ the query can be agregated, but this view cannot be re-used
+ so it is unmaterializeble.
+
+ The equality operator is the same as select one subview.
+ */
+ /*
+ First of all, the remaining views are splited in segments.
+
+ A segment contains views with the same keys that are great
+ to make joins. Joins like this do not create "dimensional
+ metrics".
+
+ In joins like this one row of each view will be connected
+ with at most one row of each other table.
+ */
+ const segment = [[sorted[0]]];
+ let segmentId = 0;
+ for (let i = 1; i < sorted.length; ++i) {
+ if (this.compareKeys(sorted[i - 1].keys, sorted[i].keys, blackList) === 0) {
+ segment[segmentId].push(sorted[i]);
+ }
+ else {
+ ++segmentId;
+ segment.push([sorted[i]]);
+ }
+ }
+
+ partialJoin = [];
+ let ableToJoin = false;
+
+ for (let i = 0; i < segment.length; ++i) {
+ /*
+ If a segment has more than one view, a join can be made
+ */
+ if (segment[i].length > 1) {
+ let mets: Metric[] = [];
+ let clauses: Clause[] = [];
+ let dims: Dimension[] = [];
+ let dimMetrics: Metric[] = [];
+ for (let j = 0; j < segment[i].length; ++j) {
+ mets = mets.concat(segment[i][j].metrics);
+ clauses = clauses.concat(segment[i][j].clauses);
+ dims = dims.concat(segment[i][j].dimensions);
+ dimMetrics = dimMetrics.concat(segment[i][j].dimMetrics);
+ }
+
+ dims = this.removeDuplicatedDimensions(dims);
+ /*
+ Its atributes are just concatenated and the
+ duplicates removed.
+ */
+
+ const partial = new View({
+ metrics: mets,
+ dimensions: dims,
+ keys: segment[i][0].keys,
+ origin: false,
+ clauses: clauses,
+ materialized: false
+ });
+ const viewFrom = "(" +
+ this.buildQuery(partial, segment[i]) +
+ ") AS view_" + partial.id + "\n";
+
+ partialJoin.push({
+ id: partial.id,
+ from: viewFrom,
+ dimMetrics: dimMetrics,
+ metrics: partial.metrics,
+ dimensions: partial.dimensions,
+ keys: partial.keys,
+ clauses: partial.clauses,
+ origin: partial.origin
+ });
+
+ ableToJoin = true;
+ }
+
+ else {
+ /*
+ If the segment has just one view, anything can be
+ done at this point, so just reinsert this view in
+ set of views.
+ */
+ partialJoin.push(segment[i][0]);
+ }
+ }
+
+ /*
+ If at least one join was made in the last part (a segment
+ with more than one view) than return to the begining of the
+ loop.
+
+ This permits after a join remove the dimensions that were
+ only choosen to this join, and are no longer required
+
+ Ideally the joins should be restrict the join method used
+ above, but in some cases this can not be done.
+
+ So if all the segments have only one view inside, move
+ to the next method.
+ */
+ if (!ableToJoin) {
+ /*
+ At this point 2 views will be joined, first the
+ similarity with each pair of views is calculated,
+ the pair with the biggedt similarity will be joined.
+
+ Similarity is calculated with the number of common
+ dimensions in the keys.
+ */
+ let similarity = 0;
+ let idx0 = 0;
+ let idx1 = 1;
+ for (let i = 0; i < partialJoin.length; ++i) {
+ for (let j = i + 1 ; j < partialJoin.length; ++j) {
+ const pi = partialJoin[i].keys;
+ const pj = partialJoin[j].keys;
+ let score = this.similarDimensions (pi, pj);
+ if (similarity < score) {
+ similarity = score;
+ idx0 = i;
+ idx1 = j;
+ }
+ }
+ }
+
+ const partial0 = partialJoin[idx0];
+ const partial1 = partialJoin[idx1];
+
+ partialJoin.splice(idx1, 1);
+ partialJoin.splice(idx0, 1);
+
+ /*
+ Once the views are select they are joined with the
+ same method, concatenedted its atributes and
+ removing duplicates, however the nasty effect of
+ this join is the creation of "dimensional metrics".
+
+ "Dimensional metrics" are metrics that can no longer
+ be aggregated, and at this point to the end
+ of a query they will act as dimensions.
+
+ This change happens to avoid inconsistency generated
+ by a join where one row of one table can be connected
+ to more than one of other table.
+
+ Take this example.
+
+ View0 : metrics [met0], dimensions [dim0]
+ values: [{met0: 10, dim0: 1}]
+ View1 : metrics [met1], dimensions [dim2]
+ values: [{met1: 10, dim2: 1}. {met1: 5, dim2: 2}]
+ View2 : metrics [], dimensions [dim0, dim1, dim2]
+ values: [
+ {dim0: 1, dim1: 1, dim2: 1},
+ {dim0: 1, dim1: 1, dim2: 2}
+ ]
+ The query is metrics [met0, met1] and dimensions [dim1]
+ First a join of View0 and View1 is made, the result
+ is: [
+ {dim0: 1, dim1: 1, dim2: 1, met0: 10},
+ {dim0: 1, dim1: 1, dim2: 2, met0: 10}
+ ]
+ Note that the value of met0 is duplicated.
+ Now dim0 is removed, than joined with view2 resulting
+ in: [
+ {met1: 10, dim1: 1, dim2: 1, met0: 10},
+ {met1: 5 , dim1: 1, dim2: 2, met0: 10}
+ ]
+
+ Lets assume that the agregation is SUM
+ If we remove dim2 and re-agregate the result is: [
+ {met1: 15, dim1: 1, met0: 20}
+ ]
+
+ This result is wrong. The replication of the value
+ met0 affects the result.
+
+ See if met1 was not required, first the dimemnsion would
+ be reduced, left dim0 and dim1, than joined that reduced
+ again resulting in the value [
+ {dim1:1, met0: 10}
+ ]
+
+ Is this case there is no duplication and the aggregation
+ does not include more rows than should.
+
+ To solve this problem the met0 must become a dimension,
+ in other words, not aggregated again. If the met0 was
+ not agregated in the query met0, met1, dim1 the result
+ is: [
+ {met1: 15, dim1: 1, met0: 10}
+ ]
+ what is compatible.
+
+ After this extreme long explanation what must be
+ known is: Joining views with diferent keys
+ generate "dimensional metrics".
+
+ Views with "dimensional metrics" can not used for future
+ queries because can not be re-agregated, so this must be
+ avoided and is one-query only views.
+ */
+
+ let dimMetrics: Metric[];
+ let mets: Metric[];
+ let dims = partial0.dimensions.concat(partial1.dimensions);
+ dims = this.removeDuplicatedDimensions(dims);
+ let keys = partial0.keys.concat(partial1.keys);
+ keys = this.removeDuplicatedDimensions(keys);
+ if (partial0.keys.length === similarity) {
+ /*
+ Here the metrics become dimensions, but the effect
+ can be reduced. If the keys of partial0
+ is a sub set of the keys ou partial1
+ than the number of rows of partial 1 is not
+ affected, in other words the metrics of partial1
+ can be aggregated and does not need to become
+ dimensions.
+ */
+ partial0.dimMetrics = partial0.dimMetrics.concat(partial0.metrics);
+ partial0.metrics = [];
+ mets = partial1.metrics;
+ }
+
+ else if (partial1.keys.length === similarity) {
+ /*
+ The same occurs if the keys of partia1 is a subset
+ of partial0.
+ */
+ partial1.dimMetrics = partial1.dimMetrics.concat(partial1.metrics);
+ partial1.metrics = [];
+ mets = partial0.metrics;
+ }
+
+ else {
+ /*
+ But if there is no sub set, than both sides have
+ the metrics turned in dimensions.
+ */
+ partial0.dimMetrics = partial0.dimMetrics.concat(partial0.metrics);
+ partial0.metrics = [];
+ partial1.dimMetrics = partial1.dimMetrics.concat(partial1.metrics);
+ partial1.metrics = [];
+ mets = [];
+ }
+
+ dimMetrics = partial0.dimMetrics.concat(partial1.dimMetrics);
+ const partial = new View({
+ metrics: mets,
+ dimensions: dims,
+ keys: keys,
+ origin: false,
+ clauses: partial0.clauses.concat(partial1.clauses),
+ materialized: false
+ });
+ const id = new View({
+ metrics: mets.concat(dimMetrics),
+ dimensions: dims,
+ keys: keys,
+ origin: false,
+ clauses: partial0.clauses.concat(partial1.clauses),
+ materialized: false
+ }).id;
+ const viewFrom = "(" +
+ this.buildQuery(partial, [partial0, partial1]) +
+ ") AS view_" + id + "\n";
+ partialJoin.push({
+ id: id,
+ from: viewFrom,
+ dimMetrics: dimMetrics,
+ metrics: mets,
+ dimensions: dims,
+ keys: keys,
+ clauses: partial.clauses,
+ origin: false
+ });
+
+ }
+ }
+ }
+
+ /*
+ When only one view remain, the query is made and a ;
+ is added at the end.
+
+ TODO: Probrably this last line adds one more
+ layer to the query, that is in fact unnecessary.
+ Think a way to remove-it.
+ */
+ return this.buildQuery(view, partialJoin) + ";";
+ }
+
+ private searchMaterializedViews(view: View): View[] {
+ let r: View[] = [];
+ if (view.materialized) {
+ return [view];
+ }
+
+ else {
+ let children = view.childViews;
+ for (let i = 0; i < children.length; ++i) {
+ r = r.concat(this.searchMaterializedViews(children[i]));
+ }
+ }
+
+ return r;
+ }
+
+ private buildQuery(target: View, views: ExpandedView[]) {
+ const metrics = target.metrics;
+ const dimensions = target.dimensions;
+ const clauses = target.clauses;
+
+ let dimMap: {[key: string]: DimInfo} = {};
+ let nameMap: {[key: string]: ExpandedView} = {};
+
+ for (let i = 0; i < views.length; ++i) {
+ const mets = views[i].metrics;
+ const dims = views[i].dimensions;
+ for (let j = 0; j < mets.length; ++j) {
+ if (!nameMap[mets[j].name]) {
+ nameMap[mets[j].name] = views[i];
+ }
+
+ }
+
+ for (let j = 0; j < dims.length; ++j) {
+ if (!dimMap[dims[j].name]) {
+ dimMap[dims[j].name] = {
+ dim: dims[j],
+ views: [views[i]]
+ };
+ nameMap[dims[j].name] = views[i];
+ }
+
+ else {
+ dimMap[dims[j].name].views.push(views[i]);
+ }
+ }
+ }
+
+ // Projection
+ const strMetrics = metrics.map((metric) => {
+ const view = nameMap[metric.name];
+ if (view) {
+ return this.translateMetric(metric, view);
+ }
+
+ return "";
+ }).filter((item) => item !== "");
+
+ const parsedDimensions = dimensions.map((dimension) => {
+ let dim = dimension;
+ while (!nameMap[dim.name]) {
+ dim = dim.parent;
+ }
+ const view = nameMap[dim.name];
+ return this.translateDimension(dimension, dim, view);
+ });
+
+ let parsedDimMetrics: DimTranslation[] = [];
+
+ for (let i = 0; i < views.length; ++i) {
+ const dimMets = views[i].dimMetrics.map((item) => {
+ return this.translateDimMetric(item, views[i]);
+ });
+
+ parsedDimMetrics = parsedDimMetrics.concat(dimMets);
+ }
+
+ const totalDimensions = parsedDimensions.concat(parsedDimMetrics);
+
+ const strDimensions = totalDimensions.map ((item) => item.aliased);
+ const grouped = totalDimensions.map((item) => {
+ return (item.expanded) ? item.alias : item.noalias;
+ });
+ const elements = strMetrics.concat(strDimensions);
+
+ // Joins
+ let conds: string[] = [];
+ for (let i of Object.keys(dimMap)) {
+ let remainViews = dimMap[i].views.slice();
+ let dim = dimMap[i].dim;
+ let leftSide = this.buildColumn(dim, remainViews.shift().id);
+ if (remainViews.length > 0) {
+ while (remainViews.length > 0) {
+ const id = remainViews.shift().id;
+ const rightSide = this.buildColumn(dim, id);
+ conds.push(leftSide + " = " + rightSide);
+ }
+ }
+
+ }
+
+ // Selection
+ let covered: Clause[] = [];
+ for (let i = 0; i < views.length; ++i) {
+ // Get the clauses that children already cover
+ covered = covered.concat(views[i].clauses);
+ }
+
+ const toCover = clauses.filter((item) => !covered.some ((clause) => {
+ return clause.id === item.id;
+ }));
+
+ toCover.forEach((item) => {
+ const clause = "(" + this.translateClause(item, nameMap) + ")";
+ if (clause !== "()") {
+ conds.push(clause);
+ }
+ });
+
+ // Assembly
+
+ const projection = "SELECT " + elements.join(",");
+ const source = " FROM " + views.map((view) => view.from).join(",");
+ const selection = (conds.length > 0) ? " WHERE " + conds.join(" AND ") : "";
+ let grouping = "";
+ if (grouped.length > 0) {
+ grouping = " GROUP BY " + grouped.join(",");
+ }
+
+ return projection + source + selection + grouping;
+
+ }
+
+ private getAggregateFunction(aggrType: AggregationType, origin: boolean): string {
+ switch (aggrType) {
+ case AggregationType.SUM:
+ return "SUM";
+ case AggregationType.AVG:
+ return "AVG";
+ case AggregationType.COUNT:
+ return (origin) ? "COUNT" : "SUM";
+ case AggregationType.MAX:
+ return "MAX";
+ case AggregationType.MIN:
+ return "MIN";
+ default:
+ return "";
+ }
+
+ }
+
+ private translateRelation(relation: RelationType, arg: string): string {
+ switch (relation) {
+ case RelationType.DAY:
+ return this.applyRelation("EXTRACT", ["DAY FROM "], [arg]);
+ case RelationType.MONTH:
+ return this.applyRelation("EXTRACT", ["MONTH FROM "], [arg]);
+ case RelationType.YEAR:
+ return this.applyRelation("EXTRACT", ["YEAR FROM "], [arg]);
+ default:
+ return "";
+ }
+
+ }
+
+ private applyRelation(name: string, args: string[], values: string[]): string {
+ /*
+ This adapter uses the concept of functions in Postgres to
+ implement BLENDB sub-dimention relations, this functions
+ applys the transformation to build the call of a Postgres
+ funtion. Note that this function can be native from postgres,
+ like EXTRACT, or even implemented on the database.
+ This function is short and only used in the translateRelation
+ method however is a bit complex and is possible to be used
+ several times, because of that is puted appart to make easyer update
+ and avoid problems
+ Example
+ applyRelation ("EXTRACT", "["DAY FROM"]", ["view_0.date"])
+ output: EXTRACT(DAY FROM view_0.date)
+ */
+ return name + "(" + args.map((item, idx) => item + values[idx]).join(",") + ")";
+ }
+
+ private buildColumn (item: Metric|Dimension, id: string): string {
+ const quotedName = "\"" + item.name + "\"";
+ return "view_" + id + "." + quotedName;
+ }
+
+ private translateClause(clause: Clause, map: {[key: string]: ExpandedView}): string {
+ const r = clause.filters.map((item) => {
+ return this.translateFilter(item, map);
+ }).filter((item) => {
+ return item !== "";
+ });
+ return r.join(" OR ");
+ }
+
+ private translateFilter(filter: Filter, map: {[key: string]: ExpandedView}): string {
+ if (!map[filter.target.name]) {
+ return "";
+ }
+
+ const viewId = map[filter.target.name].id;
+ const leftSide = this.buildColumn(filter.target, viewId);
+ const quotedValue = "'" + filter.value + "'";
+ const castedValue = this.typeCast(quotedValue, filter.target.dataType);
+ return this.applyOperator(leftSide, castedValue, filter.operator);
+ }
+
+ private translateMetric(metric: Metric, view: ExpandedView): string {
+ const func = this.getAggregateFunction(metric.aggregation, view.origin);
+ const quotedName = "\"" + metric.name + "\"";
+ const extMetric = func + "(" + this.buildColumn(metric, view.id) + ")";
+ return extMetric + " AS " + quotedName;
+ }
+
+ private translateDimMetric(metric: Metric, view: ExpandedView): DimTranslation {
+ const quotedName = "\"" + metric.name + "\"";
+ const extMetric = this.buildColumn(metric, view.id);
+ return {
+ aliased: extMetric + " AS " + quotedName,
+ noalias: extMetric,
+ alias: quotedName,
+ expanded: false
+ };
+ }
+
+ private translateDimension(dimension: Dimension,
+ ancestor: Dimension,
+ view: ExpandedView): DimTranslation {
+ const quotedName = "\"" + dimension.name + "\"";
+ let extDimension = this.buildColumn(ancestor, view.id);
+ let aux = dimension;
+ let expanded = false;
+ while (aux.name !== ancestor.name) {
+ extDimension = this.translateRelation(aux.relation, extDimension);
+ aux = aux.parent;
+ expanded = true;
+ }
+ return {
+ aliased: extDimension + " AS " + quotedName,
+ noalias: extDimension,
+ alias: quotedName,
+ expanded: expanded
+ };
+ }
+
+ protected abstract applyOperator(leftSide: string, rightSide: string, op: FilterOperator): string;
+
+ protected abstract typeCast(quotedValue: string, dt: string): string;
+
+ private compareKeys(a: Dimension[], b: Dimension[], blackList: string[]): number {
+ /*
+ SUPER WARNING: WHEN THE BLACK LIST IS USED THE VIEW IS
+ UNMATERIALIZEBLE, BUT THE QUERY CAN AGGREGATE THE VALUES
+ */
+ let c = a.filter((i) => !blackList.some((bad) => bad === i.name));
+ let d = b.filter((i) => !blackList.some((bad) => bad === i.name));
+ let length = 0;
+ let res = c.length - d.length;
+ if (c.length < d.length) {
+ length = c.length;
+ }
+ else {
+ length = d.length;
+ }
+
+ for (let i = 0; i < length; ++i) {
+ if (c[i].name < d[i].name) {
+ return -1;
+ }
+
+ else if (c[i].name > d[i].name) {
+ return 1;
+ }
+ }
+
+ return res;
+ }
+
+ private similarDimensions(a: Dimension[], b: Dimension[]): number {
+ let count = 0;
+ for (let i = 0; i < a.length; ++i) {
+ if (b.some((itemB) => a[i].name === itemB.name)) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ private removeDuplicatedDimensions(candidateDims: Dimension[]): Dimension[] {
+ let filterDims: { [key: string]: boolean } = {};
+ const dims = [];
+ for (let i = 0; i < candidateDims.length; ++i) {
+ if (!filterDims[candidateDims[i].name]) {
+ dims.push(candidateDims[i]);
+ filterDims[candidateDims[i].name] = true;
+ }
+ }
+ return dims;
+ }
+}
diff --git a/src/api/controllers/engine.spec.ts b/src/api/controllers/engine.spec.ts
index dc436712347eafbb1f57da91512ac50c7b5e141d..0d1ea0804b331b7c11197cc826df18bb32687410 100644
--- a/src/api/controllers/engine.spec.ts
+++ b/src/api/controllers/engine.spec.ts
@@ -44,7 +44,7 @@ describe("API engine controller", () => {
.expect((res: any) => {
let result = res.body;
expect(result).to.be.an("array");
- expect(result).to.have.length(13);
+ expect(result).to.have.length(12);
})
.end(done);
});
diff --git a/src/api/middlewares/adapter.ts b/src/api/middlewares/adapter.ts
index 15e883b086d8aa27e27ff65d8c5ca68dc37d2495..1e4bcbf0725a3ffcc74dd0d94886f898a525f944 100644
--- a/src/api/middlewares/adapter.ts
+++ b/src/api/middlewares/adapter.ts
@@ -21,6 +21,7 @@
import { Middleware } from "../types";
import { Adapter } from "../../core/adapter";
import { PostgresAdapter } from "../../adapter/postgres";
+import { MonetAdapter, MonetConfig } from "../../adapter/monet";
import { PoolConfig } from "pg";
import { Connection } from "../../util/configParser";
@@ -41,3 +42,19 @@ export function PostgresMw(config: Connection): Middleware {
};
}
+
+export function MonetMw(config: Connection): Middleware {
+ let parsedConfig: MonetConfig = {
+ user: config.user,
+ dbname: config.database,
+ password: config.password,
+ host: config.host,
+ port: config.port,
+ };
+ let adapter: Adapter = new MonetAdapter(parsedConfig);
+ return function monetMiddleware(req, res, next) {
+ req.adapter = adapter;
+ next();
+ };
+
+}
diff --git a/src/common/types.ts b/src/common/types.ts
index 661fce354b942243046905bcb9b986662e818f8b..504dcd68eee7fee526931867efcbbbf18778e09b 100644
--- a/src/common/types.ts
+++ b/src/common/types.ts
@@ -32,5 +32,4 @@ export enum RelationType {
DAY,
MONTH,
YEAR,
- DAYOFWEEK
};
diff --git a/src/core/dimension.ts b/src/core/dimension.ts
index bda4e07fe43ad15b09e71294ebdd8abe4392746c..946bc43fe4cfa826ece50833155fdee8b2a90efc 100644
--- a/src/core/dimension.ts
+++ b/src/core/dimension.ts
@@ -79,8 +79,6 @@ export class Dimension {
return RelationType.MONTH;
case "year":
return RelationType.YEAR;
- case "dayofweek":
- return RelationType.DAYOFWEEK;
default:
return RelationType.NONE;
}
@@ -94,8 +92,6 @@ export class Dimension {
return "month";
case RelationType.YEAR:
return "year";
- case RelationType.DAYOFWEEK:
- return "dayofweek";
default:
return "";
}
diff --git a/src/core/engine.spec.ts b/src/core/engine.spec.ts
index 66c5c1b52ab7dd7914ad54601bad04d6ade24009..b326b88303bef93c1223563f4a604ff51a9f9542 100644
--- a/src/core/engine.spec.ts
+++ b/src/core/engine.spec.ts
@@ -55,7 +55,7 @@ describe("engine class", () => {
expect(optimalView.dimensions).to.be.an("array");
expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics).to.have.length(12);
- expect(optimalView.dimensions).to.have.length(13);
+ expect(optimalView.dimensions).to.have.length(12);
});
it("should throw an exception, query with non-existent metric", () => {
let error: boolean = false;
diff --git a/src/main.ts b/src/main.ts
index 6214b70217712560a5e582846998998eea9cc8d9..56a5acf4d25df63d151d246a242df35ee023eebe 100755
--- a/src/main.ts
+++ b/src/main.ts
@@ -38,7 +38,7 @@ const config = ConfigParser.parse(configPath);
// include middlewares
import { EngineMw } from "./api/middlewares/engine";
-import { PostgresMw } from "./api/middlewares/adapter";
+import { PostgresMw, MonetMw } from "./api/middlewares/adapter";
import { ErrorMw } from "./api/middlewares/error";
app.use(EngineMw(config));
@@ -46,8 +46,12 @@ if (config.adapter === "postgres") {
app.use(PostgresMw(config.connection));
}
+else if (config.adapter === "monet") {
+ app.use(MonetMw(config.connection));
+}
+
else {
- console.error("Invalid adapter. Options available are: postgres");
+ console.error("Invalid adapter. Options available are: postgres and monet");
process.exit(1);
}
diff --git a/src/util/configParser.spec.ts b/src/util/configParser.spec.ts
index 973bad91619ac6d3f99fe340d36a7c598e1b072a..389c6067fff69a2320d2efb0c072fb6e24ba339a 100644
--- a/src/util/configParser.spec.ts
+++ b/src/util/configParser.spec.ts
@@ -32,8 +32,6 @@ function strToRelationType (str: string): RelationType {
return RelationType.MONTH;
case "year":
return RelationType.YEAR;
- case "dayofweek":
- return RelationType.DAYOFWEEK;
default:
return RelationType.NONE;
}
@@ -181,18 +179,12 @@ describe("configParser utility library", () => {
parent: "dim:0",
relation: "year"
},
- {
- name: "dim:dayofweek",
- dataType: "integer",
- parent: "dim:0",
- relation: "dayofweek"
- },
{
name: "dim:none",
dataType: "integer",
parent: "dim:0",
relation: "none"
- },
+ }
];
let dims: Dimension[] = [
diff --git a/test/monet/fixture.ts b/test/monet/fixture.ts
new file mode 100644
index 0000000000000000000000000000000000000000..3c16edc929e494b53eff2ed4a22e1bf42e58d021
--- /dev/null
+++ b/test/monet/fixture.ts
@@ -0,0 +1,162 @@
+/*
+ * Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blend.
+ *
+ * blend is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blend is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blend. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+const MDB = require("monetdb")();
+import { View, LoadView } from "../../src/core/view";
+import { each, series } from "async";
+import { Connection } from "../../src/util/configParser";
+import * as fs from "fs";
+
+interface TransSet {
+ init: string;
+ data: string[];
+}
+
+export interface Schema {
+ alias?: string;
+ query?: string;
+ data?: string;
+ fields: any[];
+}
+
+export class Fixture {
+ private database: string;
+ private config: Connection;
+
+ constructor(config: Connection) {
+ this.config = config;
+ this.database = config.database;
+ }
+
+ public load(schemas: LoadView[], create: boolean, cb: (err: Error) => void) {
+ let client = new MDB({
+ user: this.config.user,
+ dbname: this.config.database,
+ password: this.config.password,
+ host: this.config.host,
+ port: this.config.port,
+ });
+
+ /*
+ Loading data has 2 steps:
+ 1 - Create a table or truncate a existing one.
+ Create or truncate dependes from a parameter in the configure
+ file, this parameter reaches this function as the create
+ parameter.
+ 2 - Insert data.
+ */
+ let init: string[] = [];
+ let data: string[] = [];
+ for (let i = 0; i < schemas.length; ++i) {
+ let table: TransSet = this.createTransSet(schemas[i].view, schemas[i].data, create);
+ init.push(table.init);
+ data = data.concat(table.data);
+ }
+
+ client.connect();
+ /*
+ Tables must be creates before data could be inserted, so the
+ queries that create and insert are splited in 2 arrays.
+ To garantee that tables exists before start insert data, series
+ is used. Inside series each query is executed, using each;
+ */
+ series([(callback: (err: Error) => void) => {
+ each(init, (query, cback) => {
+ return client.query(query).then((result: any) => cback())
+ .fail((err: Error) => (cback(err)));
+ }, (errQuery: Error) => callback(errQuery));
+ }, (callback: (err: Error) => void) => {
+ each(data, (query, cback) => {
+ return client.query(query).then((result: any) => cback())
+ .fail((err: Error) => (cback(err)));
+ }, (errQuery: Error) => callback(errQuery));
+ }], (errQuery: Error) => {
+ client.close();
+ cb(errQuery);
+ });
+ }
+
+ private typeConvertion(t: string) {
+ switch (t) {
+ case "integer":
+ return "INTEGER";
+ case "date":
+ return "TIMESTAMP";
+ case "string":
+ return "TEXT";
+ case "boolean":
+ return "BOOLEAN";
+ default:
+ return "";
+ }
+ }
+
+ private createTransSet(view: View, filePath: string, create: boolean) {
+ let props = [];
+
+ for (let i = 0; i < view.metrics.length; ++i) {
+ let met = view.metrics[i];
+ props.push("\"" + met.name + "\" " + this.typeConvertion(met.dataType));
+ }
+
+ for (let i = 0; i < view.dimensions.length; ++i) {
+ let dim = view.dimensions[i];
+ props.push("\"" + dim.name + "\" " + this.typeConvertion(dim.dataType));
+ }
+
+ let name = "view_" + view.id;
+ let transaction: TransSet = {init: "", data: []};
+ if (create) {
+ transaction.init = "CREATE TABLE " + name + "(" + props.join(", ") + ")";
+ }
+ else {
+ transaction.init = "DELETE FROM " + name;
+ }
+
+ transaction.data = [];
+ let rows = JSON.parse(fs.readFileSync(filePath, {encoding : "utf8"}));
+ for (let i = 0; i < rows.length; ++i) {
+ let values = [];
+ let keys = [];
+ for (let key of Object.keys(rows[i])) {
+ keys.push("\"" + key + "\"");
+ values.push("'" + this.boolCast(rows[i][key]) + "'");
+ }
+ transaction.data.push("INSERT INTO " + name +
+ "(" + keys.join(", ") + ") " +
+ "VALUES (" + values.join(", ") + ")");
+ }
+ return transaction;
+ }
+
+ private boolCast(text: string): string {
+ if (text === "t") {
+ return "true";
+ }
+
+ else if (text === "f") {
+ return "false";
+ }
+
+ else {
+ return text;
+ }
+ }
+}
diff --git a/test/monet/fixtures b/test/monet/fixtures
new file mode 120000
index 0000000000000000000000000000000000000000..410001b097c384463e158a7a38376382c18a0e41
--- /dev/null
+++ b/test/monet/fixtures
@@ -0,0 +1 @@
+../postgres/fixtures/
\ No newline at end of file
diff --git a/test/scenario.ts b/test/scenario.ts
index 1fb2fac343ffd0ecc5596a7466ccfab1aa3ee0a1..9abe77b2021ce0a3ce233a75e5650452f29baca7 100644
--- a/test/scenario.ts
+++ b/test/scenario.ts
@@ -212,13 +212,7 @@ const dateSubDim = [
dataType: "integer",
parent: dims[0],
relation: RelationType.YEAR
- }),
- new Dimension ({
- name: "dim:0:dow",
- dataType: "integer",
- parent: dims[0],
- relation: RelationType.DAYOFWEEK
- }),
+ })
];
const dateView = new View({