diff --git a/src/adapter/postgres.spec.ts b/src/adapter/postgres.spec.ts
index 2279807dbebaf119f1645df35d6614d52aa6a381..d9c5a2d71e7d649b890a350f79c407f0d1693489 100644
--- a/src/adapter/postgres.spec.ts
+++ b/src/adapter/postgres.spec.ts
@@ -350,7 +350,7 @@ describe("postgres adapter", () => {
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
- expect(result).to.have.length(5);
+ expect(result).to.have.length(2);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
@@ -369,7 +369,7 @@ describe("postgres adapter", () => {
adapter.getDataFromView(view, (err, result) => {
expect(err).to.be.a("null");
expect(result).to.be.an("array");
- expect(result).to.have.length(5);
+ expect(result).to.have.length(1);
expect(result[0]).to.be.an("object");
let keys: string[] = [];
keys = keys.concat(view.metrics.map((item) => item.name));
diff --git a/src/adapter/sql.ts b/src/adapter/sql.ts
index e3b2d89c29878bb52722c89c7fb1885ffe3b63ed..4c10cd3168401f38a9808b1c03de297517f47f78 100644
--- a/src/adapter/sql.ts
+++ b/src/adapter/sql.ts
@@ -25,22 +25,12 @@ import { Dimension } from "../core/dimension";
import { Clause } from "../core/clause";
import { Filter, FilterOperator } from "../core/filter";
import { AggregationType, RelationType, DataType } from "../common/types";
+import { Operation, Opcode } from "../common/expression";
import { View } from "../core/view";
-interface ExpandedView {
- dimensions: Dimension[];
- metrics: Metric[];
- dimMetrics: Metric[];
- keys: Dimension[];
- clauses: Clause[];
- from: string;
- id: string;
- origin: boolean;
-}
-
interface DimInfo {
dim: Dimension;
- views: ExpandedView[];
+ views: View[];
}
interface DimTranslation {
@@ -50,516 +40,152 @@ interface DimTranslation {
expanded: boolean;
}
+interface QueryAndName {
+ query: string;
+ name: string;
+}
+
+interface DimAndNameMap {
+ dimensions: {[key: string]: DimInfo};
+ views: {[key: string]: View};
+}
+
export abstract class SQLAdapter extends Adapter {
public getQueryFromView(view: View): string {
- /*
- Find the base (materialized) views that has this data and exapand
- than (make a parse to the format used in the adapter)
- */
- const materialized = this.searchMaterializedViews(view).sort((a, b) => {
- return (a.id < b.id) ? -1 : 1;
- }).map((item) => {
- return {
- id: item.id,
- from: "view_" + item.id,
- dimMetrics: [],
- metrics: item.metrics.filter((i) => {
- return view.metrics.some((j) => i.name === j.name);
- }),
- dimensions: item.dimensions,
- keys: item.keys,
- clauses: item.clauses,
- origin: item.origin
- };
+ const partials = this.buildPartials(view).filter((i) => {
+ return i.query !== "";
+ }).map((i) => {
+ return i.name + " AS (" + i.query + ")";
});
- // Remove repeated views from the result
- let partialJoin = [materialized[0]];
- for (let i = 1; i < materialized.length; ++i) {
- if (materialized[i - 1].id !== materialized[i].id) {
- partialJoin.push(materialized[i]);
- }
+ let withClause = "";
+ if (partials.length > 0) {
+ withClause = "WITH " + partials.join(", ") + " ";
}
- const blackList = view.dimensions.map((i) => i.name);
- for (let i = 0; i < view.clauses.length; ++i) {
- if (view.clauses[i].filters.length === 1) {
- let filter = view.clauses[i].filters[0];
- if (filter.operator === FilterOperator.EQUAL) {
- blackList.push(filter.target.name);
- }
- }
- }
+ let sort = "";
+ if (view.sort.length > 0) {
+ // Sorting
+ const order = view.sort.map((item) => {
+ return "\"" + item.name + "\"";
+ }).join(",");
- /*
- If there is more than one source of data (tables/views)
- a join is needed.
-
- Partial Join represents how many sources still exists,
- every join reduces this number.
- */
- let clausesToCover = view.clauses.map((i) => i);
- while (partialJoin.length > 1) {
- /*
- Variable map finds what dimenensions are still needed to
- complete this query, they are required for 2 reasons.
- 1 - To make joins
- 2 - Because they are in the query
-
- For each view that has this dimension we add one score to
- this dimension, if they are in the query the same.
-
- Automatically if the dimension is in the query there will be
- at least one view with this atribute (or the query could not be
- completed) so dimensions in the query always have score of
- at least 2.
-
- To make a join the dimension must be in 2 different views,
- creating a score of 2 either.
-
- If the score is less than 2 so this dimension is not required
- anymore and can be removed.
- */
- let map: { [key: string]: number } = {};
- let partialsChange = false;
- for (let i = 0; i < partialJoin.length; ++i) {
- const dims = partialJoin[i].dimensions;
- for (let k = 0; k < dims.length; ++k) {
- if (!map[dims[k].name]) {
- map[dims[k].name] = 1;
- }
+ sort = " ORDER BY " + order;
+ }
- else {
- ++map[dims[k].name];
- }
- }
- }
+ return withClause + "SELECT * FROM " + this.viewName(view) + sort + ";";
+ }
- for (let i = 0; i < view.dimensions.length; ++i) {
- let dim = view.dimensions[i];
- while (dim !== null) {
- if (map[dim.name]) {
- ++map[dim.name];
- }
- dim = dim.parent;
+ private buildPartials(view: View): QueryAndName[] {
+ let op = view.operation;
+ let queue: View[] = op.values.map((i) => i);
+ const output: QueryAndName[] = [{
+ query: this.operation(op, view),
+ name: this.viewName(view)
+ }];
+
+ const map: {[key: string]: boolean } = {};
+
+ while (queue.length > 0) {
+ const partial = queue.shift();
+ if (!map[partial.id]) {
+ const query = this.operation(partial.operation, partial);
+ if (query !== "") {
+ map[partial.id] = true;
+ output.unshift({
+ query: query,
+ name: this.viewName(partial)
+ });
+ queue = queue.concat(partial.operation.values);
}
}
+ }
- /*
- Also mark scores for dimensions inside clauses
- */
- for (let i = 0; i < clausesToCover.length; ++i) {
- for (let j = 0; j < clausesToCover[i].targets.length; ++j) {
- if (map[clausesToCover[i].targets[j].name]) {
- ++map[clausesToCover[i].targets[j].name];
- }
- }
- }
+ return output;
+ }
- for (let i = 0; i < partialJoin.length; ++i) {
- const dims = partialJoin[i].dimensions.filter((item) => {
- return map[item.name] > 1;
- });
- const keys = partialJoin[i].keys.filter((item) => {
- return map[item.name] > 1;
- });
- /*
- At this point the dimensions with less than score 2
- are removed, if this happens the view is agreggated
- again, with less dimensions, removing this dimension
- from the view.
- */
-
- let coveredClauses: Clause[] = [];
- let notCoveredClauses: Clause[] = [];
- /*
- If all dimensions in a clause are a sub set of the
- dimensions of a view, this clause is apllied now,
- propagating the clause to this point.
-
- Then this clause is removed from the set of clauses
- */
- for (let j = 0; j < clausesToCover.length; ++j) {
- if (clausesToCover[j].isCovered(partialJoin[i].dimensions)) {
- coveredClauses.push(clausesToCover[j]);
- }
- else {
- notCoveredClauses.push(clausesToCover[j]);
- }
- }
- clausesToCover = notCoveredClauses.filter((clause) => {
- return !partialJoin[i].clauses.some((c) => c.id === clause.id);
- });
- if (dims.length < partialJoin[i].dimensions.length || coveredClauses.length > 0) {
- const partial = new View({
- metrics: partialJoin[i].metrics,
- dimensions: dims,
- keys: keys,
- origin: false,
- clauses: coveredClauses.concat(partialJoin[i].clauses),
- materialized: false
- });
- const from = "(" +
- this.buildQuery(partial, [partialJoin[i]], false) +
- ") AS view_" + partial.id + "\n";
+ protected viewName(view: View): string {
+ return "view_" + view.id;
+ }
- partialJoin[i].id = partial.id;
- partialJoin[i].dimensions = partial.dimensions;
- partialJoin[i].keys = partial.keys;
- partialJoin[i].origin = partial.origin;
- partialJoin[i].from = from;
+ private operation(op: Operation, view: View): string {
+ switch (op.opcode) {
+ case Opcode.REDUCE:
+ return this.buildOperation(view, op.values, false);
+ case Opcode.JOIN:
+ return this.buildOperation(view, op.values, true);
+ default:
+ // Is unnecessary make a push function. Push = materialized
+ // No need for materialized partials
+ return "";
+ }
+ }
- partialsChange = true;
- }
+ private buildOperation(view: View, partials: View[], isJoin: boolean): string {
+ // Mapping, which views the metrics and dimensions are
+ const mapping = this.buildMaps(partials);
+ // Projection
+ const metrics = view.metrics.map((i) => {
+ const sourceView = mapping.views[i.name];
+ return this.translateMetric(i, sourceView);
+ });
+ const dimensions = view.dimensions.map((dimension) => {
+ let dim = dimension;
+ while (!mapping.views[dim.name]) {
+ dim = dim.parent;
}
- /*
- If at least one of the views changed (have the number of
- dimensions reduced) returns to the begining of the loop
- again.
-
- Othewise we need to make a join.
- */
- if (!partialsChange) {
- /*
- Sorting the views by keys.
- If the keys are identical, then they
- will be in sequence, and views with identical
- keys can be joined.
-
- Sort an array of keys is the same as sort a
- array of strings.
- */
- const sorted = partialJoin.sort((a, b) => {
- return this.compareKeys(a.keys, b.keys, blackList);
- });
- /*
- SUPER WARNING: WHEN THE BLACK LIST IS USED THE VIEW IS
- UNMATERIALIZEBLE, BUT THE QUERY CAN AGGREGATE THE VALUES
-
- The blackList is the array of dimensions of the query plus
- the dimensions in filters using the equality operator.
- In further coments is expained that the relation to make
- a join must be one-to-one between the tables.
- However and a dimension is choosed, a sub view is
- created and if the relation is preserved in the sub view
- the query can be agregated, but this view cannot be re-used
- so it is unmaterializeble.
-
- The equality operator is the same as select one subview.
- */
- /*
- First of all, the remaining views are splited in segments.
-
- A segment contains views with the same keys that are great
- to make joins. Joins like this do not create "dimensional
- metrics".
-
- In joins like this one row of each view will be connected
- with at most one row of each other table.
- */
- const segment = [[sorted[0]]];
- let segmentId = 0;
- for (let i = 1; i < sorted.length; ++i) {
- if (this.compareKeys(sorted[i - 1].keys, sorted[i].keys, blackList) === 0) {
- segment[segmentId].push(sorted[i]);
- }
- else {
- ++segmentId;
- segment.push([sorted[i]]);
- }
- }
-
- partialJoin = [];
- let ableToJoin = false;
-
- for (let i = 0; i < segment.length; ++i) {
- /*
- If a segment has more than one view, a join can be made
- */
- if (segment[i].length > 1) {
- let mets: Metric[] = [];
- let clauses: Clause[] = [];
- let dims: Dimension[] = [];
- let dimMetrics: Metric[] = [];
- for (let j = 0; j < segment[i].length; ++j) {
- mets = mets.concat(segment[i][j].metrics);
- clauses = clauses.concat(segment[i][j].clauses);
- dims = dims.concat(segment[i][j].dimensions);
- dimMetrics = dimMetrics.concat(segment[i][j].dimMetrics);
- }
-
- dims = this.removeDuplicatedDimensions(dims);
- /*
- Its atributes are just concatenated and the
- duplicates removed.
- */
-
- const partial = new View({
- metrics: mets,
- dimensions: dims,
- keys: segment[i][0].keys,
- origin: false,
- clauses: clauses,
- materialized: false
- });
- const viewFrom = "(" +
- this.buildQuery(partial, segment[i], false) +
- ") AS view_" + partial.id + "\n";
-
- partialJoin.push({
- id: partial.id,
- from: viewFrom,
- dimMetrics: dimMetrics,
- metrics: partial.metrics,
- dimensions: partial.dimensions,
- keys: partial.keys,
- clauses: partial.clauses,
- origin: partial.origin
- });
-
- ableToJoin = true;
- }
-
- else {
- /*
- If the segment has just one view, anything can be
- done at this point, so just reinsert this view in
- set of views.
- */
- partialJoin.push(segment[i][0]);
- }
- }
-
- /*
- If at least one join was made in the last part (a segment
- with more than one view) than return to the begining of the
- loop.
-
- This permits after a join remove the dimensions that were
- only choosen to this join, and are no longer required
-
- Ideally the joins should be restrict the join method used
- above, but in some cases this can not be done.
-
- So if all the segments have only one view inside, move
- to the next method.
- */
- if (!ableToJoin) {
- /*
- At this point 2 views will be joined, first the
- similarity with each pair of views is calculated,
- the pair with the biggedt similarity will be joined.
-
- Similarity is calculated with the number of common
- dimensions in the keys.
- */
- let similarity = 0;
- let idx0 = 0;
- let idx1 = 1;
- for (let i = 0; i < partialJoin.length; ++i) {
- for (let j = i + 1 ; j < partialJoin.length; ++j) {
- const pi = partialJoin[i].keys;
- const pj = partialJoin[j].keys;
- let score = this.similarDimensions (pi, pj);
- if (similarity < score) {
- similarity = score;
- idx0 = i;
- idx1 = j;
- }
- }
- }
+ const sourceView = mapping.views[dim.name];
+ return this.translateDimension(dimension, dim, sourceView);
+ });
+ const projElem = dimensions.map((i) => i.aliased).concat(metrics);
- const partial0 = partialJoin[idx0];
- const partial1 = partialJoin[idx1];
-
- partialJoin.splice(idx1, 1);
- partialJoin.splice(idx0, 1);
-
- /*
- Once the views are select they are joined with the
- same method, concatenedted its atributes and
- removing duplicates, however the nasty effect of
- this join is the creation of "dimensional metrics".
-
- "Dimensional metrics" are metrics that can no longer
- be aggregated, and at this point to the end
- of a query they will act as dimensions.
-
- This change happens to avoid inconsistency generated
- by a join where one row of one table can be connected
- to more than one of other table.
-
- Take this example.
-
- View0 : metrics [met0], dimensions [dim0]
- values: [{met0: 10, dim0: 1}]
- View1 : metrics [met1], dimensions [dim2]
- values: [{met1: 10, dim2: 1}. {met1: 5, dim2: 2}]
- View2 : metrics [], dimensions [dim0, dim1, dim2]
- values: [
- {dim0: 1, dim1: 1, dim2: 1},
- {dim0: 1, dim1: 1, dim2: 2}
- ]
- The query is metrics [met0, met1] and dimensions [dim1]
- First a join of View0 and View1 is made, the result
- is: [
- {dim0: 1, dim1: 1, dim2: 1, met0: 10},
- {dim0: 1, dim1: 1, dim2: 2, met0: 10}
- ]
- Note that the value of met0 is duplicated.
- Now dim0 is removed, than joined with view2 resulting
- in: [
- {met1: 10, dim1: 1, dim2: 1, met0: 10},
- {met1: 5 , dim1: 1, dim2: 2, met0: 10}
- ]
-
- Lets assume that the agregation is SUM
- If we remove dim2 and re-agregate the result is: [
- {met1: 15, dim1: 1, met0: 20}
- ]
-
- This result is wrong. The replication of the value
- met0 affects the result.
-
- See if met1 was not required, first the dimemnsion would
- be reduced, left dim0 and dim1, than joined that reduced
- again resulting in the value [
- {dim1:1, met0: 10}
- ]
-
- Is this case there is no duplication and the aggregation
- does not include more rows than should.
-
- To solve this problem the met0 must become a dimension,
- in other words, not aggregated again. If the met0 was
- not agregated in the query met0, met1, dim1 the result
- is: [
- {met1: 15, dim1: 1, met0: 10}
- ]
- what is compatible.
-
- After this extreme long explanation what must be
- known is: Joining views with diferent keys
- generate "dimensional metrics".
-
- Views with "dimensional metrics" can not used for future
- queries because can not be re-agregated, so this must be
- avoided and is one-query only views.
- */
-
- let dimMetrics: Metric[];
- let mets: Metric[];
- let dims = partial0.dimensions.concat(partial1.dimensions);
- dims = this.removeDuplicatedDimensions(dims);
- let keys = partial0.keys.concat(partial1.keys);
- keys = this.removeDuplicatedDimensions(keys);
- if (partial0.keys.length === similarity) {
- /*
- Here the metrics become dimensions, but the effect
- can be reduced. If the keys of partial0
- is a sub set of the keys ou partial1
- than the number of rows of partial 1 is not
- affected, in other words the metrics of partial1
- can be aggregated and does not need to become
- dimensions.
- */
- partial0.dimMetrics = partial0.dimMetrics.concat(partial0.metrics);
- partial0.metrics = [];
- mets = partial1.metrics;
- }
+ // Grouping
+ const grouped = dimensions.map((item) => {
+ return (item.expanded) ? item.alias : item.noalias;
+ });
- else if (partial1.keys.length === similarity) {
- /*
- The same occurs if the keys of partia1 is a subset
- of partial0.
- */
- partial1.dimMetrics = partial1.dimMetrics.concat(partial1.metrics);
- partial1.metrics = [];
- mets = partial0.metrics;
- }
+ // Selection
+ const conds = [];
+ const clauses = this.orphanClauses(view, partials);
+ for (let i = 0; i < clauses.length; ++i) {
+ const trClause = this.translateClause(clauses[i], mapping.views);
+ if (trClause) {
+ conds.push("(" + trClause + ")");
+ }
+ }
- else {
- /*
- But if there is no sub set, than both sides have
- the metrics turned in dimensions.
- */
- partial0.dimMetrics = partial0.dimMetrics.concat(partial0.metrics);
- partial0.metrics = [];
- partial1.dimMetrics = partial1.dimMetrics.concat(partial1.metrics);
- partial1.metrics = [];
- mets = [];
+ // Joinning
+ if (isJoin) {
+ const dimMap = mapping.dimensions;
+ for (let i of Object.keys(dimMap)) {
+ if (dimMap[i].views.length > 1) {
+ const views = dimMap[i].views;
+ const dim = dimMap[i].dim;
+ const leftSide = this.buildColumn(dim, views.shift().id);
+ while (views.length > 0) {
+ const rightSide = this.buildColumn(dim, views.shift().id);
+ conds.push("(" + leftSide + "=" + rightSide + ")");
}
-
- dimMetrics = partial0.dimMetrics.concat(partial1.dimMetrics);
- const partial = new View({
- metrics: mets,
- dimensions: dims,
- keys: keys,
- origin: false,
- clauses: partial0.clauses.concat(partial1.clauses),
- materialized: false
- });
- const id = new View({
- metrics: mets.concat(dimMetrics),
- dimensions: dims,
- keys: keys,
- origin: false,
- clauses: partial0.clauses.concat(partial1.clauses),
- materialized: false
- }).id;
- const viewFrom = "(" +
- this.buildQuery(partial, [partial0, partial1], false) +
- ") AS view_" + id + "\n";
- partialJoin.push({
- id: id,
- from: viewFrom,
- dimMetrics: dimMetrics,
- metrics: mets,
- dimensions: dims,
- keys: keys,
- clauses: partial.clauses,
- origin: false
- });
-
}
}
}
- /*
- When only one view remain, the query is made and a ;
- is added at the end.
-
- TODO: Probrably this last line adds one more
- layer to the query, that is in fact unnecessary.
- Think a way to remove-it.
- */
- return this.buildQuery(view, partialJoin, true) + ";";
- }
-
- private searchMaterializedViews(view: View): View[] {
- let r: View[] = [];
- if (view.materialized) {
- return [view];
- }
+ // Assembly
+ const projection = "SELECT " + projElem.join(", ");
+ const sources = "FROM " + partials.map((i) => this.viewName(i)).join(", ");
+ const selection = (conds.length > 0) ? " WHERE " + conds.join(" AND ") : "";
- else {
- let children = view.childViews;
- for (let i = 0; i < children.length; ++i) {
- r = r.concat(this.searchMaterializedViews(children[i]));
- }
+ let grouping = "";
+ if (grouped.length > 0) {
+ grouping = " GROUP BY " + grouped.join(",");
}
- return r;
+ return projection + sources + selection + grouping;
}
- private buildQuery(target: View, views: ExpandedView[], toSort: boolean): string {
- const metrics = target.metrics;
- const dimensions = target.dimensions;
- const clauses = target.clauses;
- const sort = target.sort;
-
+ private buildMaps(views: View[]): DimAndNameMap {
let dimMap: {[key: string]: DimInfo} = {};
- let nameMap: {[key: string]: ExpandedView} = {};
+ let nameMap: {[key: string]: View} = {};
for (let i = 0; i < views.length; ++i) {
const mets = views[i].metrics;
@@ -586,95 +212,20 @@ export abstract class SQLAdapter extends Adapter {
}
}
- // Projection
- const strMetrics = metrics.map((metric) => {
- const view = nameMap[metric.name];
- if (view) {
- return this.translateMetric(metric, view);
- }
-
- return "";
- }).filter((item) => item !== "");
-
- const parsedDimensions = dimensions.map((dimension) => {
- let dim = dimension;
- while (!nameMap[dim.name]) {
- dim = dim.parent;
- }
- const view = nameMap[dim.name];
- return this.translateDimension(dimension, dim, view);
- });
-
- let parsedDimMetrics: DimTranslation[] = [];
-
- for (let i = 0; i < views.length; ++i) {
- const dimMets = views[i].dimMetrics.map((item) => {
- return this.translateDimMetric(item, views[i]);
- });
-
- parsedDimMetrics = parsedDimMetrics.concat(dimMets);
- }
-
- const totalDimensions = parsedDimensions.concat(parsedDimMetrics);
-
- const strDimensions = totalDimensions.map ((item) => item.aliased);
- const grouped = totalDimensions.map((item) => {
- return (item.expanded) ? item.alias : item.noalias;
- });
- const elements = strMetrics.concat(strDimensions);
-
- // Joins
- let conds: string[] = [];
- for (let i of Object.keys(dimMap)) {
- let remainViews = dimMap[i].views.slice();
- let dim = dimMap[i].dim;
- let leftSide = this.buildColumn(dim, remainViews.shift().id);
- if (remainViews.length > 0) {
- while (remainViews.length > 0) {
- const id = remainViews.shift().id;
- const rightSide = this.buildColumn(dim, id);
- conds.push(leftSide + " = " + rightSide);
- }
- }
-
- }
-
- // Selection
- let covered: Clause[] = [];
- for (let i = 0; i < views.length; ++i) {
- // Get the clauses that children already cover
- covered = covered.concat(views[i].clauses);
- }
-
- const toCover = clauses.filter((item) => !covered.some ((clause) => {
- return clause.id === item.id;
- }));
-
- toCover.forEach((item) => {
- const clause = "(" + this.translateClause(item, nameMap) + ")";
- if (clause !== "()") {
- conds.push(clause);
- }
- });
-
- // Sorting
- const order = sort.map((item) => {
- return "\"" + item.name + "\"";
- }).join(",");
-
- // Assembly
+ return {
+ dimensions: dimMap,
+ views: nameMap
+ };
+ }
- const projection = "SELECT " + elements.join(",");
- const source = " FROM " + views.map((view) => view.from).join(",");
- const selection = (conds.length > 0) ? " WHERE " + conds.join(" AND ") : "";
- let grouping = "";
- if (grouped.length > 0) {
- grouping = " GROUP BY " + grouped.join(",");
+ private orphanClauses(view: View, partials: View[]): Clause[] {
+ let parentClauses: Clause[] = [];
+ for (let i = 0; i < partials.length; ++i) {
+ parentClauses = parentClauses.concat(partials[i].clauses);
}
- const sorting = (toSort && sort.length > 0) ? " ORDER BY " + order : "";
-
- return projection + source + selection + grouping + sorting;
+ // return clauses that does not exist in the partials
+ return view.clauses.filter((i) => !parentClauses.some((j) => j.id === i.id));
}
private getAggregateFunction(aggrType: AggregationType, origin: boolean): string {
@@ -732,7 +283,7 @@ export abstract class SQLAdapter extends Adapter {
return "view_" + id + "." + quotedName;
}
- private translateClause(clause: Clause, map: {[key: string]: ExpandedView}): string {
+ private translateClause(clause: Clause, map: {[key: string]: View}): string {
const r = clause.filters.map((item) => {
return this.translateFilter(item, map);
}).filter((item) => {
@@ -741,7 +292,7 @@ export abstract class SQLAdapter extends Adapter {
return r.join(" OR ");
}
- private translateFilter(filter: Filter, map: {[key: string]: ExpandedView}): string {
+ private translateFilter(filter: Filter, map: {[key: string]: View}): string {
if (!map[filter.target.name]) {
return "";
}
@@ -753,27 +304,16 @@ export abstract class SQLAdapter extends Adapter {
return this.applyOperator(leftSide, castedValue, filter.operator);
}
- private translateMetric(metric: Metric, view: ExpandedView): string {
+ private translateMetric(metric: Metric, view: View): string {
const func = this.getAggregateFunction(metric.aggregation, view.origin);
const quotedName = "\"" + metric.name + "\"";
const extMetric = func + "(" + this.buildColumn(metric, view.id) + ")";
return extMetric + " AS " + quotedName;
}
- private translateDimMetric(metric: Metric, view: ExpandedView): DimTranslation {
- const quotedName = "\"" + metric.name + "\"";
- const extMetric = this.buildColumn(metric, view.id);
- return {
- aliased: extMetric + " AS " + quotedName,
- noalias: extMetric,
- alias: quotedName,
- expanded: false
- };
- }
-
private translateDimension(dimension: Dimension,
ancestor: Dimension,
- view: ExpandedView): DimTranslation {
+ view: View): DimTranslation {
const quotedName = "\"" + dimension.name + "\"";
let extDimension = this.buildColumn(ancestor, view.id);
let aux = dimension;
@@ -791,61 +331,6 @@ export abstract class SQLAdapter extends Adapter {
};
}
- protected abstract applyOperator(leftSide: string, rightSide: string, op: FilterOperator): string;
-
- protected abstract typeCast(quotedValue: string, dt: DataType): string;
-
- private compareKeys(a: Dimension[], b: Dimension[], blackList: string[]): number {
- /*
- SUPER WARNING: WHEN THE BLACK LIST IS USED THE VIEW IS
- UNMATERIALIZEBLE, BUT THE QUERY CAN AGGREGATE THE VALUES
- */
- let c = a.filter((i) => !blackList.some((bad) => bad === i.name));
- let d = b.filter((i) => !blackList.some((bad) => bad === i.name));
- let length = 0;
- let res = c.length - d.length;
- if (c.length < d.length) {
- length = c.length;
- }
- else {
- length = d.length;
- }
-
- for (let i = 0; i < length; ++i) {
- if (c[i].name < d[i].name) {
- return -1;
- }
-
- else if (c[i].name > d[i].name) {
- return 1;
- }
- }
-
- return res;
- }
-
- private similarDimensions(a: Dimension[], b: Dimension[]): number {
- let count = 0;
- for (let i = 0; i < a.length; ++i) {
- if (b.some((itemB) => a[i].name === itemB.name)) {
- count++;
- }
- }
- return count;
- }
-
- private removeDuplicatedDimensions(candidateDims: Dimension[]): Dimension[] {
- let filterDims: { [key: string]: boolean } = {};
- const dims = [];
- for (let i = 0; i < candidateDims.length; ++i) {
- if (!filterDims[candidateDims[i].name]) {
- dims.push(candidateDims[i]);
- filterDims[candidateDims[i].name] = true;
- }
- }
- return dims;
- }
-
public getQueryFromSource(source: Source, data: any[]): string {
let consult: string;
let colums: any[] = [];
@@ -863,4 +348,8 @@ export abstract class SQLAdapter extends Adapter {
return consult;
}
+
+ protected abstract applyOperator(leftSide: string, rightSide: string, op: FilterOperator): string;
+
+ protected abstract typeCast(quotedValue: string, dt: DataType): string;
}
diff --git a/src/common/expression.ts b/src/common/expression.ts
new file mode 100644
index 0000000000000000000000000000000000000000..513203e49a174da879617d657bd190c71fdef28c
--- /dev/null
+++ b/src/common/expression.ts
@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blend.
+ *
+ * blend is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blend is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blend. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { View } from "../core/view";
+
+export enum Opcode {
+ PUSH,
+ JOIN,
+ REDUCE
+}
+
+export interface Operation {
+ opcode: Opcode;
+ values: View[];
+}
diff --git a/src/core/engine.spec.ts b/src/core/engine.spec.ts
index b326b88303bef93c1223563f4a604ff51a9f9542..03098f5eca2970e098880a908e4a0bfcc23f032c 100644
--- a/src/core/engine.spec.ts
+++ b/src/core/engine.spec.ts
@@ -50,10 +50,8 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions");
- expect(optimalView).to.have.property("childViews");
expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array");
- expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics).to.have.length(12);
expect(optimalView.dimensions).to.have.length(12);
});
@@ -92,13 +90,13 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions");
- expect(optimalView).to.have.property("childViews");
+ expect(optimalView).to.have.property("operation");
expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array");
- expect(optimalView.childViews).to.be.an("array");
+ expect(optimalView.operation).to.be.an("object");
expect(optimalView.metrics).to.have.length(4);
expect(optimalView.dimensions).to.have.length(2);
- expect(optimalView.childViews).to.have.length(0);
+ expect(optimalView.operation).to.have.property("opcode");
expect(optimalView.id).to.be.equal(views[0].id);
});
@@ -120,15 +118,13 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions");
- expect(optimalView).to.have.property("childViews");
- expect(optimalView).to.have.property("materialized");
+ expect(optimalView).to.have.property("operation");
expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array");
- expect(optimalView.childViews).to.be.an("array");
+ expect(optimalView.operation).to.be.an("object");
expect(optimalView.metrics).to.have.length(3);
expect(optimalView.dimensions).to.have.length(1);
- expect(optimalView.childViews).to.have.length(0);
- expect(optimalView.materialized).to.be.true;
+ expect(optimalView.operation).to.have.property("opcode");
expect(optimalView.id).to.be.equal(views[9].id);
});
@@ -143,13 +139,10 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions");
- expect(optimalView).to.have.property("childViews");
expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array");
- expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics).to.have.length(0);
expect(optimalView.dimensions).to.have.length(2);
- expect(optimalView.childViews).to.have.length(3);
expect(optimalView).satisfy((optView: View) => {
return optView.dimensions.some((item) => item.name === subdim[0].name);
@@ -169,13 +162,10 @@ describe("engine class", () => {
expect(optimalView).to.be.an("object");
expect(optimalView).to.have.property("metrics");
expect(optimalView).to.have.property("dimensions");
- expect(optimalView).to.have.property("childViews");
expect(optimalView.metrics).to.be.an("array");
expect(optimalView.dimensions).to.be.an("array");
- expect(optimalView.childViews).to.be.an("array");
expect(optimalView.metrics).to.have.length(0);
expect(optimalView.dimensions).to.have.length(2);
- expect(optimalView.childViews).to.have.length(3);
expect(optimalView).satisfy((optView: View) => {
return optView.dimensions.some((item) => item.name === subdim[2].name);
diff --git a/src/core/engine.ts b/src/core/engine.ts
index 8058722d32e8bd36011f6541c1ad714ea5daf9c8..72a554a47eff5f3e9dac364367647855f02e9ece 100644
--- a/src/core/engine.ts
+++ b/src/core/engine.ts
@@ -27,6 +27,16 @@ import { Query } from "../common/query";
import { Graph } from "../util/graph";
import { EnumType, EnumTypeOptions} from "./enumType";
import { Source , SourceStrOptions } from "./source";
+import { ViewHandler } from "../util/viewHandler";
+
+interface Score {
+ [key: string]: number;
+}
+
+interface ViewsAndClauses {
+ views: View[];
+ clauses: Clause[];
+}
export class Engine {
private views: View[] = [];
@@ -197,67 +207,51 @@ export class Engine {
}
private selectOptimalView (q: Query): View {
- let optimalViews = this.graph.cover(q);
- if (optimalViews.length === 0) {
- throw new Error ("Engine views cannot cover the query");
- }
+ let queries: Query[] = [];
+ if (q.metrics.length > 0) {
+ for (let i = 0; i < q.metrics.length; ++i) {
+ queries.push({
+ metrics: [q.metrics[i]],
+ dimensions: q.dimensions,
+ clauses: (q.clauses) ? q.clauses : [],
+ sort: (q.sort) ? q.sort : []
+ });
+ }
+ const views = queries.map((query) => {
+ return ViewHandler.growView(query, this.getCover(query));
+ });
- // If all the metrics and dimensions are the same and only exist one child view
- // return this single child view
- const metrics = q.metrics;
- const dimensions = q.dimensions;
- const clauses = ((q.clauses) ? q.clauses : []);
- const sort = ((q.sort) ? q.sort : []);
- if (optimalViews.length === 1 &&
- optimalViews[0].metrics.length === metrics.length &&
- optimalViews[0].dimensions.length === dimensions.length &&
- optimalViews[0].clauses.length === clauses.length &&
- optimalViews[0].sort.length === sort.length &&
- optimalViews[0].metrics.every((item) => metrics.indexOf(item) !== -1) &&
- optimalViews[0].dimensions.every((item) => dimensions.indexOf(item) !== -1) &&
- perfectMatch(optimalViews[0].clauses, clauses) &&
- perfectOrder(optimalViews[0].sort, sort)) {
- return optimalViews[0];
+ return ViewHandler.queryJoin(q, views);
}
+
else {
- let options = {
- metrics: metrics,
- dimensions: dimensions,
- clauses: clauses,
- sort: sort,
- materialized: false,
- origin: false, // Never a dynamic generated view will be origin
- childViews: optimalViews
+ let query = {
+ metrics: q.metrics,
+ dimensions: q.dimensions,
+ clauses: (q.clauses) ? q.clauses : [],
+ sort: (q.sort) ? q.sort : []
};
-
- let view = new View(options);
- // this.addView(view);
- /*
- This line has been removed for now because not all views can be
- re-used by other views (unmaterializeble views), and so far this
- is only detected in the adapter, when this can be detected in
- engine, than the queries can be added again to the engine
- */
- return view;
+ return ViewHandler.growView(query, this.getCover(query));
}
+
}
-}
-function perfectMatch (array1: Clause[],
- array2: Clause[]): boolean {
- return array1.every((item: Clause) => {
- return array2.some((otherItem: Clause) => item.id === otherItem.id);
- });
-}
+ private getCover (q: Query): View[] {
+ const optimalViews = this.graph.cover(q);
+ if (optimalViews.length === 0) {
+ throw new Error ("Engine views cannot cover the query");
+ }
-function perfectOrder (array1: (Metric|Dimension)[],
- array2: (Metric|Dimension)[]): boolean {
- // Assuming that the arrays have the same length
- for (let i = 0; i < array1.length; ++i) {
- if (array1[i].name !== array2[i].name) {
- return false;
+ let matViews: View[] = optimalViews.sort((a, b) => (a.id < b.id) ? -1 : 1);
+
+ let noRepeat: View[] = [matViews[0]];
+ for (let i = 1; i < matViews.length; ++i) {
+ if (matViews[i - 1].id !== matViews[i].id) {
+ noRepeat.push(matViews[i]);
+ }
}
- }
- return true;
+ return noRepeat;
+
+ }
}
diff --git a/src/core/view.ts b/src/core/view.ts
index 29e6a63551e02ccfa64ffe858b2203815a72c7a9..b1cba66060f122b4bb0af42c240f0c6102070b11 100644
--- a/src/core/view.ts
+++ b/src/core/view.ts
@@ -22,6 +22,7 @@ import { Dimension } from "./dimension";
import { Metric } from "./metric";
import { Hash } from "../util/hash";
import { Clause } from "./clause";
+import { Operation, Opcode } from "../common/expression";
export interface LoadView {
view: View;
@@ -31,38 +32,36 @@ export interface LoadView {
export interface ViewOptions {
metrics: Metric[];
dimensions: Dimension[];
- keys?: Dimension[];
origin: boolean;
clauses?: Clause[];
sort?: (Metric|Dimension)[];
- materialized?: boolean;
- childViews?: View[];
+ operation?: Operation;
}
export class View {
public readonly id: string;
public readonly metrics: Metric[];
public readonly dimensions: Dimension[];
- public readonly keys: Dimension[];
public readonly clauses: Clause[];
public readonly sort: (Metric|Dimension)[];
- public readonly materialized: boolean;
public readonly origin: boolean;
- public childViews: View[];
+ public readonly operation: Operation;
constructor (options: ViewOptions) {
this.metrics = options.metrics.sort();
this.dimensions = options.dimensions.sort();
this.clauses = (options.clauses) ? options.clauses.sort() : [];
this.sort = (options.sort) ? options.sort : [];
- this.materialized = options.materialized || false;
this.origin = options.origin || false;
- this.childViews = (options.childViews) ? options.childViews : [];
- if (options.keys && options.keys.length > 0) {
- this.keys = options.keys.sort();
+ if (options.operation) {
+ this.operation = options.operation;
}
+
else {
- this.keys = this.dimensions;
+ this.operation = {
+ opcode: Opcode.PUSH,
+ values: []
+ };
}
// calculate the id of the view based on it's metrics and dimensions
diff --git a/src/util/configParser.ts b/src/util/configParser.ts
index 8ce6c3520c5befa67c77cc854c6f24f41ec22efb..87f8ef6e25290307891f3b8f73d2977358e2e090 100644
--- a/src/util/configParser.ts
+++ b/src/util/configParser.ts
@@ -23,6 +23,7 @@ import { Dimension, DimensionOptions, DimensionStrOptions } from "../core/dimens
import { View, ViewOptions, LoadView } from "../core/view";
import { EnumType, EnumTypeOptions } from "../core/enumType";
import { RelationType, DataType } from "../common/types";
+import { Opcode } from "../common/expression";
import { Filter } from "../core/filter";
import { Clause } from "../core/clause";
import { Source, SourceOptions, SourceStrOptions} from "../core/source";
@@ -38,7 +39,6 @@ export interface ViewParsingOptions {
dimensions: string[];
metrics: string[];
clauses?: string[];
- keys?: string[];
}
interface ConfigSchema {
@@ -222,15 +222,14 @@ export class ConfigParser {
let viewOpt: ViewOptions = {
metrics: [],
dimensions: [],
- materialized: true,
origin: opts.origin,
- childViews: [],
clauses: [],
- keys: []
+ operation: {
+ opcode: Opcode.PUSH,
+ values: []
+ }
};
- const keys = (opts.keys) ? opts.keys : [];
-
for (let i = 0; i < opts.metrics.length; ++i) {
if (metMap[opts.metrics[i]]) {
viewOpt.metrics.push(metMap[opts.metrics[i]]);
@@ -251,15 +250,6 @@ export class ConfigParser {
throw new Error("[Parsing error] Non exist dimension set to view " + opts.alias);
}
}
- for (let i = 0; i < keys.length; ++i) {
- if (dimMap[keys[i]]) {
- viewOpt.keys.push(dimMap[opts.keys[i]]);
- }
-
- else {
- throw new Error("[Parsing error] Non exist key set to view " + opts.alias);
- }
- }
if (opts.clauses) {
for (let i = 0; i < opts.clauses.length; ++i) {
diff --git a/src/util/graph.spec.ts b/src/util/graph.spec.ts
index 72e372e09888ec204203d6a49ac1c87413cb951e..8dc8ea982468a6e32235da45507d7dae24220dc8 100644
--- a/src/util/graph.spec.ts
+++ b/src/util/graph.spec.ts
@@ -96,20 +96,17 @@ describe("graph class", () => {
new View({
metrics: [],
dimensions: [dims[0], dims[1]],
- origin: true,
- materialized: true
+ origin: true
}),
new View({
metrics: [],
dimensions: [dims[2], dims[3]],
- origin: true,
- materialized: true
+ origin: true
}),
new View({
metrics: [],
dimensions: dims,
- origin: true,
- materialized: true
+ origin: true
})
];
@@ -133,8 +130,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: [dims[0], dims[1]],
- origin: true,
- materialized: true
+ origin: true
});
expect(g.addView(view)).to.be.true;
@@ -155,8 +151,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: dims,
- origin: true,
- materialized: true
+ origin: true
});
expect(g.addView(view)).to.be.false;
@@ -177,8 +172,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: [dim],
- origin: true,
- materialized: true
+ origin: true
});
expect(g.addView(view)).to.be.true;
@@ -195,8 +189,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: [dim],
- origin: true,
- materialized: true
+ origin: true
});
expect(g.addView(view)).to.be.true;
@@ -231,38 +224,32 @@ describe("graph class", () => {
new View({
metrics: [mets[0]],
dimensions: [dims[0]],
- origin: true,
- materialized: true
+ origin: true
}),
new View({
metrics: [mets[1]],
dimensions: [dims[1]],
- origin: true,
- materialized: true
+ origin: true
}),
new View({
metrics: [mets[2]],
dimensions: [dims[2]],
- origin: true,
- materialized: true
+ origin: true
}),
new View({
metrics: [],
dimensions: dims,
- origin: true,
- materialized: true
+ origin: true
}),
new View({
metrics: mets,
dimensions: dims,
- origin: true,
- materialized: true
+ origin: true
}),
new View({
metrics: [mets[0], mets[1]],
dimensions: [dims[0], dims[1]],
- origin: true,
- materialized: true
+ origin: true
}),
];
@@ -304,8 +291,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: [dims[0]],
- origin: true,
- materialized: true
+ origin: true
});
expect(g.addView(view)).to.be.true;
@@ -344,8 +330,7 @@ describe("graph class", () => {
let view = new View({
metrics: [],
dimensions: [dims[0]],
- origin: true,
- materialized: true
+ origin: true
});
expect(g.addView(view)).to.be.true;
@@ -385,23 +370,19 @@ describe("graph class", () => {
let view1 = new View({
metrics: [],
dimensions: [dims[0], dims[1]],
- origin: true,
- materialized: true
+ origin: true
});
let view2 = new View({
metrics: [],
dimensions: [dims[1], dims[2]],
- origin: true,
- materialized: true
+ origin: true
});
let view3 = new View({
metrics: [],
dimensions: dims,
origin: false,
- materialized: false,
- childViews: [view1, view2],
clauses: [clause1]
});
@@ -517,7 +498,6 @@ describe("graph class", () => {
metrics: [],
dimensions: dims,
origin: false,
- materialized: true,
clauses: clauses
});
@@ -630,8 +610,7 @@ describe("graph class", () => {
new View({
metrics: [],
dimensions: dims,
- origin: false,
- materialized: true
+ origin: false
})
];
@@ -640,7 +619,6 @@ describe("graph class", () => {
metrics: [],
dimensions: dims,
origin: false,
- materialized: true,
clauses: [clauses[i]]
}));
}
diff --git a/src/util/viewHandler.ts b/src/util/viewHandler.ts
new file mode 100644
index 0000000000000000000000000000000000000000..9e3aea11348b477f03048b3d805dc240e5f5b5d6
--- /dev/null
+++ b/src/util/viewHandler.ts
@@ -0,0 +1,356 @@
+/*
+ * Copyright (C) 2018 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Opcode } from "../common/expression";
+import { View } from "../core/view";
+import { Clause } from "../core/clause";
+import { Query } from "../common/query";
+import { Dimension } from "../core/dimension";
+import { Metric } from "../core/metric";
+
+interface Score {
+ [key: string]: number;
+}
+
+interface ViewsAndClauses {
+ views: View[];
+ clauses: Clause[];
+}
+
+export class ViewHandler {
+ public static queryJoin(q: Query, views: View[]): View {
+ if (views.length > 1) {
+ return new View({
+ metrics: q.metrics,
+ dimensions: q.dimensions,
+ origin: false,
+ clauses: (q.clauses) ? q.clauses : [],
+ sort: (q.sort) ? q.sort : [],
+ operation: {
+ opcode: Opcode.JOIN,
+ values: views.map((i) => i)
+ }
+ });
+ }
+
+ else {
+ return views[0];
+ }
+ }
+
+ public static queryReduce(q: Query, view: View): View {
+ const v = new View({
+ metrics: q.metrics,
+ dimensions: q.dimensions,
+ origin: false,
+ clauses: (q.clauses) ? q.clauses : [],
+ sort: (q.sort) ? q.sort : [],
+ operation: {
+ opcode: Opcode.REDUCE,
+ values: [view]
+ }
+ });
+
+ return (v.id !== view.id) ? v : view;
+ }
+
+ public static growView(q: Query, views: View[]): View {
+ let clausesToCover = q.clauses.map((i) => i);
+ let metView: View = null;
+ let partialJoin: View[] = null;
+ let reduced: ViewsAndClauses = null;
+ let partialQuery: Query = {
+ metrics: q.metrics,
+ dimensions: q.dimensions,
+ clauses: q.clauses,
+ sort: q.sort
+ };
+
+ partialJoin = views.map((i) => (i));
+ if (q.metrics.length === 0) { // ignore metView if there are 0 metrics
+ while (partialJoin.length > 1) {
+ partialQuery.clauses = clausesToCover;
+ reduced = ViewHandler.applyReduce(partialJoin, partialQuery);
+ clausesToCover = reduced.clauses;
+ partialJoin = ViewHandler.applyJoin(reduced.views);
+ }
+
+ return ViewHandler.applyReduce(partialJoin, partialQuery).views[0];
+ }
+
+ else {
+ const metViewId = partialJoin.findIndex((i) => i.metrics.some((j) => {
+ return j.name === q.metrics[0].name;
+ }));
+
+ metView = partialJoin[metViewId];
+ partialJoin.splice(metViewId, 1);
+ while (partialJoin.length > 1) {
+ partialJoin.push(metView); // MetView is now the last view
+ partialQuery.clauses = clausesToCover;
+ reduced = ViewHandler.applyReduce(partialJoin, partialQuery);
+ clausesToCover = reduced.clauses;
+ metView = reduced.views.pop();
+ partialJoin = ViewHandler.applyJoin(reduced.views);
+ }
+
+ if (partialJoin.length === 0) {
+ return ViewHandler.applyReduce([metView], partialQuery).views[0];
+ }
+
+ else {
+ partialJoin.push(metView);
+ // In this case, there are exactly 2 views
+ reduced = ViewHandler.applyReduce(partialJoin, partialQuery);
+ partialJoin = ViewHandler.applyJoin(reduced.views);
+ partialQuery.clauses = reduced.clauses;
+ return ViewHandler.applyReduce(partialJoin, partialQuery).views[0];
+ }
+ }
+ }
+
+ private static dimensionsScore(views: View[], clauses: Clause[], dimensions: Dimension[]): Score {
+ let map: { [key: string]: number } = {};
+ for (let i = 0; i < views.length; ++i) {
+ const dims = views[i].dimensions;
+ for (let k = 0; k < dims.length; ++k) {
+ if (map[dims[k].name]) {
+ ++map[dims[k].name];
+ }
+
+ else {
+ map[dims[k].name] = 1;
+ }
+ }
+ }
+
+ for (let i = 0; i < dimensions.length; ++i) {
+ let dim = dimensions[i];
+ if (map[dim.name]) {
+ ++map[dim.name];
+ }
+
+ else { // Necessary for sub dimensions
+ map[dim.name] = 1;
+ }
+ }
+
+ /*
+ Also mark scores for dimensions inside clauses
+ */
+ for (let i = 0; i < clauses.length; ++i) {
+ for (let j = 0; j < clauses[i].targets.length; ++j) {
+ if (map[clauses[i].targets[j].name]) {
+ ++map[clauses[i].targets[j].name];
+ }
+ }
+ }
+ return map;
+ }
+
+ private static applyReduce(v: View[], q: Query): ViewsAndClauses {
+ let changed = true;
+ const views = v.map((i) => i);
+ let clausesToCover = q.clauses.map((i) => i);
+ while (changed) {
+ changed = false;
+ let map = ViewHandler.dimensionsScore(views, clausesToCover, q.dimensions);
+ const unfilledSubDims = q.dimensions.filter((item) => {
+ return map[item.name] < 2;
+ });
+ let ancestors: { [key: string]: Dimension[] } = {};
+ for (let i = 0; i < unfilledSubDims.length; ++i) {
+ let dim = unfilledSubDims[i];
+ ancestors[dim.name] = [dim];
+ dim = dim.parent;
+ while (dim !== null) {
+ ancestors[unfilledSubDims[i].name].push(dim);
+ dim = dim.parent;
+ }
+ }
+ for (let i = 0; i < views.length; ++i) {
+ const dims = views[i].dimensions.filter((item) => {
+ return map[item.name] > 1;
+ });
+ /*
+ At this point the dimensions with less than score 2
+ are removed, if this happens the view is agreggated
+ again, with less dimensions, removing this dimension
+ from the view.
+ */
+
+ for (let key of Object.keys(ancestors)) {
+ const intersection = (ancestors[key].some((anc) => {
+ return views[i].dimensions.some((dim) => {
+ return dim.name === anc.name;
+ });
+ }));
+
+ if (intersection) {
+ // The first item is always the sub dimension
+ dims.push(ancestors[key][0]);
+ }
+ }
+
+ let coveredClauses: Clause[] = [];
+ let notCoveredClauses: Clause[] = [];
+ /*
+ If all dimensions in a clause are a sub set of the
+ dimensions of a view, this clause is apllied now,
+ propagating the clause to this point.
+
+ Then this clause is removed from the set of clauses
+ */
+ for (let j = 0; j < clausesToCover.length; ++j) {
+ if (clausesToCover[j].isCovered(views[i].dimensions)) {
+ coveredClauses.push(clausesToCover[j]);
+ }
+ else {
+ notCoveredClauses.push(clausesToCover[j]);
+ }
+ }
+
+ clausesToCover = notCoveredClauses.filter((clause) => {
+ return !views[i].clauses.some((c) => c.id === clause.id);
+ });
+
+ // Intersection between query metrics and view metrics
+ const mets = q.metrics.filter((met) => {
+ return views[i].metrics.some((m) => m.name === met.name);
+ });
+
+ if (dims.length < views[i].dimensions.length || coveredClauses.length > 0) {
+ const partial = ViewHandler.queryReduce({
+ metrics: mets,
+ dimensions: dims,
+ clauses: coveredClauses.concat(views[i].clauses),
+ sort: q.sort
+ }, views[i]);
+
+ views[i] = partial;
+ changed = true;
+ }
+ }
+ }
+
+ return {
+ views: views,
+ clauses: clausesToCover
+ };
+ }
+
+ private static applyJoin(v: View[]): View[] {
+ /*
+ At this point 2 views will be joined, first the
+ similarity with each pair of views is calculated,
+ the pair with the biggedt similarity will be joined.
+
+ Similarity is calculated with the number of common
+ dimensions in the keys.
+ */
+ const views = v.map((i) => i);
+ let similarity = 0;
+ let idx0 = 0;
+ let idx1 = 1;
+ for (let i = 0; i < views.length; ++i) {
+ for (let j = i + 1 ; j < views.length; ++j) {
+ const pi = views[i].dimensions;
+ const pj = views[j].dimensions;
+ let score = this.similarDimensions (pi, pj);
+ if (similarity < score) {
+ similarity = score;
+ idx0 = i;
+ idx1 = j;
+ }
+ }
+ }
+
+ const partial0 = views[idx0];
+ const partial1 = views[idx1];
+
+ views.splice(idx1, 1);
+ views.splice(idx0, 1);
+
+ let dims = partial0.dimensions.concat(partial1.dimensions);
+ dims = ViewHandler.removeDuplicatedDimensions(dims);
+
+ let mets = partial0.metrics.concat(partial1.metrics);
+ mets = ViewHandler.removeDuplicatedMetrics(mets);
+
+ let clauses = partial0.clauses.concat(partial1.clauses);
+ clauses = ViewHandler.removeDuplicatedClauses(clauses);
+
+ const partialQuery: Query = {
+ metrics: mets,
+ dimensions: dims,
+ clauses: clauses
+ };
+
+ const partial = ViewHandler.queryJoin(partialQuery, [partial0, partial1]);
+ views.push(partial);
+ return views;
+ }
+
+ private static similarDimensions(a: Dimension[], b: Dimension[]): number {
+ let count = 0;
+ for (let i = 0; i < a.length; ++i) {
+ if (b.some((itemB) => a[i].name === itemB.name)) {
+ count++;
+ }
+ }
+ return count;
+ }
+
+ private static removeDuplicatedDimensions(candidateDims: Dimension[]): Dimension[] {
+ let filterDims: { [key: string]: boolean } = {};
+ const dims = [];
+ for (let i = 0; i < candidateDims.length; ++i) {
+ if (!filterDims[candidateDims[i].name]) {
+ dims.push(candidateDims[i]);
+ filterDims[candidateDims[i].name] = true;
+ }
+ }
+ return dims;
+ }
+
+ private static removeDuplicatedMetrics(candidateMets: Metric[]): Metric[] {
+ let filterMets: { [key: string]: boolean } = {};
+ const mets = [];
+ for (let i = 0; i < candidateMets.length; ++i) {
+ if (!filterMets[candidateMets[i].name]) {
+ mets.push(candidateMets[i]);
+ filterMets[candidateMets[i].name] = true;
+ }
+ }
+ return mets;
+ }
+
+ private static removeDuplicatedClauses(candidateClauses: Clause[]): Clause[] {
+ let filterClauses: { [key: string]: boolean } = {};
+ const clauses = [];
+ for (let i = 0; i < candidateClauses.length; ++i) {
+ if (!filterClauses[candidateClauses[i].id]) {
+ clauses.push(candidateClauses[i]);
+ filterClauses[candidateClauses[i].id] = true;
+ }
+ }
+ return clauses;
+ }
+ }
diff --git a/test/scenario.ts b/test/scenario.ts
index 2e1bd1b85f8924846f2ede10a6f064cc1a5162a3..59a91b727ebcf6bda866c754c093360a6ba50251 100644
--- a/test/scenario.ts
+++ b/test/scenario.ts
@@ -26,6 +26,7 @@ import { Filter, FilterOperator } from "../src/core/filter";
import { Clause } from "../src/core/clause";
import { AggregationType, RelationType , DataType} from "../src/common/types";
import { Query} from "../src/common/query";
+import { ViewHandler } from "../src/util/viewHandler";
interface EngineScenario {
metrics: Metric[];
@@ -215,158 +216,135 @@ const dateSubDim = [
})
];
-const dateView = new View({
+const dateView = ViewHandler.queryReduce({
metrics: [],
dimensions: dateSubDim,
- materialized: false,
- origin: false,
- childViews: [views[0]]
-});
+}, views[0]);
-const aggrView = new View({
+const aggrView = ViewHandler.queryJoin({
metrics: [mets[0], mets[1], mets[6], mets[10], mets[11]],
dimensions: [],
- materialized: false,
- origin: false,
- childViews: [views[0], views[2], views[3]]
-});
-
-const clauseView = new View({
+}, [
+ ViewHandler.queryReduce({
+ metrics: [mets[0], mets[1], mets[10]],
+ dimensions: []
+ }, views[0]),
+ ViewHandler.queryReduce({
+ metrics: [mets[6], mets[11]],
+ dimensions: []
+ }, views[2])
+]);
+
+const clauseView = ViewHandler.queryReduce({
metrics: [mets[0], mets[1], mets[2]],
dimensions: [dims[0]],
- materialized: false,
- origin: false,
- childViews: [views[0]],
clauses: [clauses.view0dim7]
-});
+}, views[0]);
-const multiFilterView = new View({
+const multiFilterView = ViewHandler.queryReduce({
metrics: [mets[0], mets[1]],
dimensions: [dims[0]],
- materialized: false,
- origin: false,
- childViews: [views[0]],
clauses: [clauses.view0dim0]
-});
+}, views[0]);
-const multiClauseView = new View({
+const multiClauseView = ViewHandler.queryReduce({
metrics: [mets[0], mets[1]],
dimensions: [dims[0]],
- materialized: false,
- origin: false,
- childViews: [views[0]],
clauses: [clauses.view0dim0, clauses.view0dim7]
-});
+}, views[0]);
-const notEqualView = new View({
+const notEqualView = ViewHandler.queryReduce({
metrics: [],
dimensions: [dims[4], dims[5]],
- materialized: false,
- origin: false,
- childViews: [views[7]],
clauses: [clauses.view7dim5]
-});
+}, views[7]);
-const gtltView = new View({
+const gtltView = ViewHandler.queryReduce({
metrics: [],
dimensions: [dims[0]],
- materialized: false,
- origin: false,
- childViews: [views[0]],
clauses: [clauses.view0gt, clauses.view0lt]
-});
+}, views[0]);
-const geleView = new View({
+const geleView = ViewHandler.queryReduce({
metrics: [],
dimensions: [dims[0]],
- materialized: false,
- origin: false,
- childViews: [views[0]],
clauses: [clauses.view0ge, clauses.view0le]
-});
+}, views[0]);
-const notMatchFilterView = new View({
+const notMatchFilterView = ViewHandler.queryReduce({
metrics: [mets[0]],
dimensions: [dims[0]],
- materialized: false,
- origin: false,
- childViews: [views[0]],
clauses: [clauses.view7dim5]
-});
+}, views[0]);
-const subDimView = new View({
+const subDimView = ViewHandler.queryJoin({
metrics: [mets[0]],
- dimensions: [subdims[0], subdims[1], dims[7], dims[8]],
- materialized: false,
- origin: false,
- childViews: [views[0], views[1], views[4]]
-});
-
-const join4View = new View({
+ dimensions: [subdims[0], subdims[1], dims[7], dims[8]]
+}, [
+ ViewHandler.queryReduce({
+ metrics: [mets[0]],
+ dimensions: [subdims[0], dims[7]]
+ }, views[0]),
+ ViewHandler.queryJoin({
+ metrics: [],
+ dimensions: [subdims[1], dims[7], dims[8]]
+ }, [views[1], views[4]])
+]);
+
+const join4View = ViewHandler.queryJoin({
metrics: [mets[0], mets[1], mets[2], mets[3], mets[4], mets[5]],
- dimensions: [dims[2], dims[7], dims[8]],
- materialized: false,
- origin: false,
- childViews: [views[0], views[1], views[2], views[4]]
-});
+ dimensions: [dims[2], dims[7], dims[8]]
+}, [views[0], views[1], views[2], views[4]]);
-const noSelView = new View({
+const noSelView = ViewHandler.queryJoin({
metrics: [mets[0], mets[3]],
- dimensions: [],
- materialized: false,
- origin: false,
- childViews: [views[0], views[1]]
-});
+ dimensions: []
+}, [views[0], views[1]]);
-const withSelView = new View({
+const withSelView = ViewHandler.queryJoin({
metrics: [mets[0], mets[1]],
- dimensions: [dims[7], dims[8]],
- materialized: false,
- origin: false,
- childViews: [views[0], views[4]]
-});
+ dimensions: [dims[7], dims[8]]
+}, [views[0], views[4]]);
const notOriginCount = new View({
metrics: [mets[5], mets[6], mets[7]],
dimensions: [dims[2]],
- materialized: true,
origin: false,
clauses: [clauses.view9dim2]
});
-const unMaterializebleView = new View({
+const unMaterializebleView = ViewHandler.queryJoin({
metrics: [mets[7], mets[9]],
- dimensions: [dims[5]],
- materialized: false,
- origin: false,
- childViews: [views[3], views[5], views[7], views[8]]
-});
-
-const partialJoinView = new View({
+ dimensions: [dims[5]]
+},
+[ViewHandler.queryReduce({
+ metrics: [mets[9]],
+ dimensions: [dims[5]]
+}, views[8]),
+ViewHandler.queryReduce({metrics: [mets[7]], dimensions: [dims[5]]},
+ViewHandler.queryJoin({metrics: [mets[7]], dimensions: [dims[3], dims[5]]}, [
+ViewHandler.queryReduce({metrics: [], dimensions: [dims[3], dims[5]]},
+ViewHandler.queryJoin({metrics: [], dimensions: [dims[3], dims[4], dims[5]]}, [
+ViewHandler.queryReduce({metrics: [], dimensions: [dims[4], dims[5]]}, views[7]),
+ViewHandler.queryReduce({metrics: [], dimensions: [dims[3], dims[4]]}, views[3])
+])), views[5]]))]);
+
+const partialJoinView = ViewHandler.queryJoin({
metrics: [mets[7], mets[8]],
- dimensions: [],
- materialized: false,
- origin: false,
- childViews: [views[3], views[5], views[6]]
-});
+ dimensions: []
+}, [views[3], views[5], views[6]]);
-const propagatedClauseView = new View({
+const propagatedClauseView = ViewHandler.queryJoin({
metrics: [mets[8]],
dimensions: [dims[4]],
- materialized: false,
- origin: false,
- childViews: [views[6], views[7]],
clauses: [clauses.view7dim5, clauses.view6dim4]
-});
+}, [views[6], views[7]]);
-const propagatedClauseAggrView = new View({
+const propagatedClauseAggrView = ViewHandler.queryJoin({
metrics: [mets[8], mets[5]],
dimensions: [dims[2]],
- materialized: false,
- origin: false,
- childViews: [views[9], views[6], views[7]],
clauses: [clauses.view7dim5, clauses.view6dim4, clauses.view9dim2]
-});
+}, [views[9], views[6], views[7]]);
export const engineScenario: EngineScenario = {
metrics: mets,