From 1f5c6c5378421a93a719507680f48294d4cc63ad Mon Sep 17 00:00:00 2001
From: Lucas Fernandes de Oliveira <lfo14@inf.ufpr.br>
Date: Wed, 29 Mar 2017 11:02:42 -0300
Subject: [PATCH] Issue #9: Implement date parent relations in postgres adapter
Signed-off-by: Lucas Fernandes de Oliveira <lfo14@inf.ufpr.br>
---
src/adapter/postgres.ts | 151 +++++++++++++++++++++++++++++-----------
src/common/types.ts | 5 +-
src/core/dimension.ts | 6 +-
src/core/engine.spec.ts | 29 +++++---
src/core/engine.ts | 38 ++++++----
src/core/view.ts | 14 ++--
6 files changed, 172 insertions(+), 71 deletions(-)
diff --git a/src/adapter/postgres.ts b/src/adapter/postgres.ts
index b796d14..4c7e8b9 100644
--- a/src/adapter/postgres.ts
+++ b/src/adapter/postgres.ts
@@ -19,8 +19,8 @@
*/
import { Adapter } from "../core/adapter";
-import { AggregationType } from "../common/types";
-import { View } from "../core/view";
+import { AggregationType, RelationType } from "../common/types";
+import { View, ChildView } from "../core/view";
interface ParsedView {
query: string;
@@ -30,44 +30,47 @@ interface ParsedView {
export class PostgresAdapter extends Adapter{
public getDataFromView(view: View): string {
// buildQueryFromView does not put the final ;, it need to be put apart
- return this.buildQueryFromView(view) + ";\n";
+ return this.buildQueryFromView(view, view.metrics, view.dimensions) + ";\n";
}
public materializeView(view: View): string {
return null;
}
- private buildQueryFromView (view: View): string {
- let sql = "(\n";
- let metrics = view.metrics.map((metric: string) => {
- let aggrType = view.getAggregationtype(metric);
- let func = this.getAggregateFunction(aggrType);
- let extMetric = func + "(" + metric + ")";
+ private buildQueryFromView (view: View, metrics: Metric[], dimensions: Dimession[]): string {
+ /*
+ Reduce metrics and dimensions array to the intersection with the
+ view. So is possible only get useful data in the sub-querys.
+ */
+ let strMetrics = metrics.map((metric) => {
+ let func = this.getAggregateFunction(metric.aggregation);
+ let extMetric = func + "(" + metric.name + ")";
return extMetric;
});
if (view.materialized) {
- sql += "SELECT " + metrics.join(", ") + ", " + dimensions.join(", ") + "\n";
- sql += "FROM " + "view_" + view.id + "\n";
- sql += "GROUP BY " + view.dimensions.join(", ") + "\n";
+ let strDimensions = dimensions.map((dimmension) => dimension.name );
+ let sql = "(SELECT " + strMetrics.join(", ") + ", " + strDimensions.join(", ");
+ sql += " FROM " + "view_" + view.id;
+ sql += " GROUP BY " + strDimensions.join(", ");
sql += ")\n";
return sql;
}
else {
- let children: ParsedView[] = view.childViews.map((item: View, idx: number) => {
+ let children: ParsedView[] = view.childViews.map((item: ChildView) => {
return {
- query: this.queryfyView(item, childAlias, degree + 1),
- view: item
+ query: this.buildQueryFromView(item.view, item.metrics, item.dimensions),
+ view: item.view
};
});
let covered = new Map();
- view.dimensions.forEach((item: string) => covered.set(item, ""));
- view.metrics.forEach((item: string) => covered.set(item, ""));
+ dimensions.forEach((item) => covered.set(item.name, ""));
+ metrics.forEach((item) => covered.set(item.name, ""));
let projection = "SELECT ";
- let viewsFrom = "FROM";
+ let viewsFrom = "FROM ";
let selection = "WHERE ";
let grouping = "GROUP BY ";
@@ -76,31 +79,64 @@ export class PostgresAdapter extends Adapter{
children.forEach((child: ParsedView) => {
let selected = [];
- child.view.dimensions.forEach((dimension: string) => {
- let first = covered.get(dimension);
- let extDimension = "view_" + child.view.id + "." + dimension;
- if (first === "") {
- covered.set(dimension, child.view.id);
- elements.push(extDimension);
- group.push(extDimension);
- }
+ let matchable = child.dimensions.map((item) => {
+ return { match: item, sub: item, viewId: child.view.id };
+ });
- else {
- let extFirst = "view_" + first + "." + dimension;
- selected.push(extDimension + " = " + extFirst);
+ // Running over the dimensions that cover some part of the query
+ child.dimensions.forEach((dimension: Dimension) => {
+ /*
+ If a dimension has a parent, the parent must match with
+ the sub dimension, so must be added in the matchable
+ array, for the selection (WHERE statement).
+ */
+ let dim = dimension;
+ while (dim.relation !== RelationType.NONE) {
+ matchable.push({
+ match: dim.name,
+ sub: dimension,
+ viewId: child.view.id
+ });
+ dim = dim.parent;
}
- });
- child.view.metrics.forEach((metric: string) => {
- let first = covered.get(metric);
- let aggrType = child.view.getAggregateFunction(metric);
- let func = this.geAggregateFunction(aggrType);
- let extMetric = func + "(view_" + child.view.id + "." + metric + ")";
- if (first === "") {
- covered.set(metric, child.view.id);
- elements.push(extMetric);
+ dim = dimension;
+ let extDimension = "view_" + child.view.id + ".\"" + dim.name + "\"";
+
+ // If the view dimension does not match, search for parent
+ while (!child.dimensions.some((item) => item.name === dim.name)) {
+ dim = dim.parent;
+ extDimension = this.translateRelation(dim.relation, extDimension);
}
+ covered.set(dimension.name, extDimension);
+ elements.push(extDimension + " AS " + dimension.name);
+ group.push(dimension.name);
+ });
+ child.view.dimensions.forEach((dimension: Dimension) => {
+ /*
+ Make selection. Search for dimensions, that are in
+ matchable array.
+ */
+ matchable.filter((item) => {
+ return item.viewId !== child.view.id &&
+ item.match === dimension.name;
+ })
+ .forEach((item) => {
+ // Expand the sub-dimension until match with a parent
+ let dim = item.sub;
+ let extDimension = "view_" + child.view.id + ".\"" + dimension.name + "\"";
+ while (dim.name !== item.match) {
+ dim = dim.parent;
+ extDimension = this.translateRelation(dim, extDimension);
+ }
+ selected.push(extDimension + " = " + covered.get(item.sub.name));
+ });
+ });
+ child.metrics.forEach((metric: Metric) => {
+ let func = this.geAggregateFunction(metric.aggregation);
+ let extMetric = func + "(view_" + child.view.id + "." + metric.name + ")";
+ elements.push(extMetric);
});
viewsFrom += "\n" + child.query;
@@ -114,11 +150,8 @@ export class PostgresAdapter extends Adapter{
selection += "\n";
grouping += group.join(", ") + "\n";
- sql += projection + viewsFrom + selection + grouping + ")";
- return sql;
+ return "(" + projection + viewsFrom + selection + grouping + ")";
}
-
- return sql;
}
private getAggregateFunction(aggrType: AggregationType): string {
@@ -134,4 +167,38 @@ export class PostgresAdapter extends Adapter{
}
}
+
+ private translateRelation(relation: RelationType, arg: string): string {
+ switch (relation) {
+ case RelationType.DAY:
+ return this.applyRelation("EXTRACT", ["DAY FROM "], [arg]);
+ case RelationType.MONTH:
+ return this.applyRelation("EXTRACT", ["MONTH FROM "], [arg]);
+ case RelationType.YEAR:
+ return this.applyRelation("EXTRACT", ["YEAR FROM "], [arg]);
+ case RelationType.DAYOFWEEK:
+ return this.applyRelation("EXTRACT", ["DOW FROM "], [arg]);
+ default:
+ return "";
+ }
+
+ }
+
+ private applyRelation(name, args, values): string {
+ /*
+ This adapter uses the concept of functions in Postgres to
+ implement BLENDB sub-dimention relations, this functions
+ applys the transformation to build the call of a Postgres
+ funtion. Note that this function can be native from postgres,
+ like EXTRACT, or even implemented on the database.
+ This function is short and only used in the translateRelation
+ method however is a bit complex and is possible to be used
+ several times, because of that is puted appart to make easyer update
+ and avoid problems
+ Example
+ applyRelation ("EXTRACT", "["DAY FROM"]", ["view_0.date"])
+ output: EXTRACT(DAY FROM view_0.date)
+ */
+ return name + "(" + args.map((item, idx) => item + values[idx]).join(",") + ")";
+ }
}
diff --git a/src/common/types.ts b/src/common/types.ts
index 6f07921..9757e7a 100644
--- a/src/common/types.ts
+++ b/src/common/types.ts
@@ -27,5 +27,8 @@ export enum AggregationType {
export enum RelationType {
NONE,
- DAY
+ DAY,
+ MONTH,
+ YEAR,
+ DAYOFWEEK
};
diff --git a/src/core/dimension.ts b/src/core/dimension.ts
index de6abcc..386d0b5 100644
--- a/src/core/dimension.ts
+++ b/src/core/dimension.ts
@@ -22,18 +22,18 @@ import { RelationType } from "../common/types";
export interface DimensionOptions {
name: string;
- parent?: string;
+ parent?: Dimension;
relation?: RelationType;
}
export class Dimension {
public readonly name: string;
- public readonly parent: string;
+ public readonly parent: Dimension;
public readonly relation: RelationType;
constructor(options: DimensionOptions) {
this.name = options.name;
this.relation = (options.relation) ? options.relation : RelationType.NONE;
- this.parent = (options.parent) ? options.parent : "";
+ this.parent = (options.parent) ? options.parent : null;
}
}
diff --git a/src/core/engine.spec.ts b/src/core/engine.spec.ts
index 5788e7c..8aaaddb 100644
--- a/src/core/engine.spec.ts
+++ b/src/core/engine.spec.ts
@@ -54,11 +54,11 @@ describe("engine class", () => {
const dim10 = new Dimension({ name: "dim:10" });
const dim11 = new Dimension({ name: "dim:11" });
- const subdim1 = new Dimension({ name: "sub:1", parent: "dim:1", relation: RelationType.DAY });
- const subdim2 = new Dimension({ name: "sub:2", parent: "dim:9", relation: RelationType.DAY });
- const subdim3 = new Dimension({ name: "sub:3", parent: "sub:1", relation: RelationType.DAY });
- const subdim4 = new Dimension({ name: "sub:4", parent: "sub:0", relation: RelationType.DAY });
- const subdim5 = new Dimension({ name: "sub:5", parent: "dim:2", relation: RelationType.DAY });
+ const subdim1 = new Dimension({ name: "sub:1", parent: dim1, relation: RelationType.DAY });
+ const subdim2 = new Dimension({ name: "sub:2", parent: dim9, relation: RelationType.DAY });
+ const subdim3 = new Dimension({ name: "sub:3", parent: subdim1, relation: RelationType.DAY });
+ const subdim4 = new Dimension({ name: "sub:4", parent: null, relation: RelationType.DAY });
+ const subdim5 = new Dimension({ name: "sub:5", parent: dim2, relation: RelationType.DAY });
engine.addMetric(met1);
engine.addMetric(met2);
@@ -105,21 +105,32 @@ describe("engine class", () => {
metrics: [met1, met2, met3, met4, met5],
dimensions: [dim1, dim2],
materialized: false,
- childViews: [views[0], views[6]]
+ childViews: [
+ { view: views[0], metrics: [met1, met2, met3], dimensions: [dim1, dim2]},
+ { view: views[6], metrics: [met4], dimensions: []},
+ { view: views[1], metrics: [met5], dimensions: []}
+ ]
}));
views.push(new View({
metrics: [met8, met9, met10],
dimensions: [dim8, dim9, dim10],
materialized: false,
- childViews: [views[7], views[8], views[9]]
+ childViews: [
+ { view: views[7], metrics: [met8], dimensions: [dim8, dim9, dim10]},
+ { view: views[8], metrics: [met9], dimensions: []},
+ { view: views[9], metrics: [met10], dimensions: []}
+ ]
}));
views.push(new View({
metrics: [met1],
dimensions: [subdim1, subdim2],
materialized: false,
- childViews: [views[0], views[9]]
+ childViews: [
+ { view: views[0], metrics: [met1], dimensions: [dim1]},
+ { view: views[9], metrics: [], dimensions: [dim9]}
+ ]
}));
views.forEach((view) => engine.addView(view));
@@ -247,7 +258,7 @@ describe("engine class", () => {
}
catch (e){
error = true;
- expect(e.message).to.be.equal("The dimension named " + subdim4.parent + " was not found");
+ expect(e.message).to.be.equal("Engine sub-dimention " + subdim4.name + " with no parent");
}
expect(error);
diff --git a/src/core/engine.ts b/src/core/engine.ts
index 8fe8091..e60ef53 100644
--- a/src/core/engine.ts
+++ b/src/core/engine.ts
@@ -20,7 +20,7 @@
import { Dimension } from "./dimension";
import { Metric } from "./metric";
-import { View } from "./view";
+import { View, ChildView } from "./view";
import { Query } from "../common/query";
import { RelationType } from "../common/types";
@@ -77,12 +77,12 @@ export class Engine {
private selectOptimalView (q: Query) {
let metUncovered = q.metrics.map((met) => met);
let dimUncovered = q.dimensions.map((dim) => dim);
- let optimalViews: View[] = [];
+ let optimalViews: ChildView[] = [];
let activeViews = this.getViews();
// run this block until all metrics and dimmensions are covered
while (metUncovered.length > 0 || dimUncovered.length > 0) {
- let bestView: View;
+ let bestView: ChildView;
let coverLength = metUncovered.length + dimUncovered.length;
let shortestDistance = coverLength + 1;
@@ -96,8 +96,11 @@ export class Engine {
let dimIntersection = dimUncovered.filter((dim: Dimension) => {
let r: boolean = view.dimensions.some((item) => item.name === dim.name);
while (!r && dim.relation !== RelationType.NONE) {
- r = view.dimensions.some((item) => item.name === dim.parent);
- dim = this.getDimensionByName(dim.parent);
+ if (dim.parent === null) {
+ throw new Error("Engine sub-dimention " + dim.name + " with no parent");
+ }
+ r = view.dimensions.some((item) => item.name === dim.parent.name);
+ dim = dim.parent;
}
return r;
});
@@ -108,14 +111,22 @@ export class Engine {
let distance = coverLength - intersection;
if (distance < shortestDistance) {
- bestView = view;
+ bestView = {
+ view: view,
+ metrics: metIntersection,
+ dimensions: dimIntersection
+ };
shortestDistance = distance;
}
else if (distance === shortestDistance &&
view.dimensions.length < bestView.dimensions.length) {
// priorizes views with less dimensions
- bestView = view;
+ bestView = {
+ view: view,
+ metrics: metIntersection,
+ dimensions: dimIntersection
+ };
}
return true;
@@ -142,8 +153,11 @@ export class Engine {
dimUncovered = dimUncovered.filter((dim: Dimension) => {
let r: boolean = bestView.dimensions.some((item) => item.name === dim.name);
while (!r && dim.relation !== RelationType.NONE) {
- r = bestView.dimensions.some((item) => item.name === dim.parent);
- dim = this.getDimensionByName(dim.parent);
+ if (dim.parent === null) {
+ throw new Error("Engine sub-dimention " + dim.name + " with no parent");
+ }
+ r = bestView.dimensions.some((item) => item.name === dim.parent.name);
+ dim = dim.parent;
}
return !r;
});
@@ -152,9 +166,9 @@ export class Engine {
// If all the metrics and dimensions are the same and only exist one child view
// return this single child view
if (optimalViews.length === 1 &&
- q.metrics.every((item) => optimalViews[0].metrics.indexOf(item) !== -1) &&
- q.dimensions.every((item) => optimalViews[0].dimensions.indexOf(item) !== -1)) {
- return optimalViews[0];
+ q.metrics.every((item) => optimalViews[0].view.metrics.indexOf(item) !== -1) &&
+ q.dimensions.every((item) => optimalViews[0].view.dimensions.indexOf(item) !== -1)) {
+ return optimalViews[0].view;
}
else {
let options = {
diff --git a/src/core/view.ts b/src/core/view.ts
index 823fe24..66fd11d 100644
--- a/src/core/view.ts
+++ b/src/core/view.ts
@@ -22,11 +22,17 @@ import { Dimension } from "./dimension";
import { Metric } from "./metric";
import { Hash } from "../util/hash";
+export interface ChildView {
+ metrics: Metric[];
+ dimensions: Dimension[];
+ view: View;
+}
+
export interface ViewOptions {
metrics: Metric[];
dimensions: Dimension[];
materialized?: boolean;
- childViews?: View[];
+ childViews?: ChildView[];
}
export class View {
@@ -34,7 +40,7 @@ export class View {
public readonly metrics: Metric[];
public readonly dimensions: Dimension[];
public readonly materialized: boolean;
- public childViews: View[];
+ public childViews: ChildView[];
constructor (options: ViewOptions) {
this.metrics = options.metrics.sort();
@@ -43,8 +49,8 @@ export class View {
this.childViews = options.childViews || [];
// calculate the id of the view based on it's metrics and dimensions
- let metricsNames = this.metrics.map(metric => metric.name);
- let dimensionsNames = this.dimensions.map(dimension => dimension.name);
+ let metricsNames = options.metrics.map(metric => metric.name);
+ let dimensionsNames = options.dimensions.map(dimension => dimension.name);
this.id = Hash.sha1(metricsNames.concat(dimensionsNames).sort());
}
}
--
GitLab