From 0aca2492a98fd9f60e693d97f41da5fde5499fe3 Mon Sep 17 00:00:00 2001 From: Lucas Fernandes de Oliveira <lfo14@inf.ufpr.br> Date: Wed, 19 Oct 2016 11:26:03 -0200 Subject: [PATCH] Issue #6: Create selector of optimal View and sql query generator for Postgres adpter Signed-off-by: Lucas Fernandes de Oliveira <lfo14@inf.ufpr.br> --- src/adapter/postgres.ts | 137 ++++++++++++++++++++++++++++++++ src/common/aggregationType.ts | 26 ++++++ src/common/query.ts | 25 ++++++ src/core/adapter.ts | 26 ++++++ src/core/engine.spec.ts | 145 ++++++++++++++++++++++++++++++++++ src/core/engine.ts | 102 ++++++++++++++++++++++++ src/core/view.ts | 57 +++++++++++++ 7 files changed, 518 insertions(+) create mode 100644 src/adapter/postgres.ts create mode 100644 src/common/aggregationType.ts create mode 100644 src/common/query.ts create mode 100644 src/core/adapter.ts create mode 100644 src/core/engine.spec.ts create mode 100644 src/core/engine.ts create mode 100644 src/core/view.ts diff --git a/src/adapter/postgres.ts b/src/adapter/postgres.ts new file mode 100644 index 00000000..3cb982a3 --- /dev/null +++ b/src/adapter/postgres.ts @@ -0,0 +1,137 @@ +/* + * Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blend. + * + * blend is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blend is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blend. If not, see <http://www.gnu.org/licenses/>. + */ + +import { View } from "../core/view"; +import { Adapter } from "../core/adapter"; +import { AggregationType } from "../common/aggregationType"; + +interface ParsedView { + query: string; + view: View; +}; + +export class PostgresAdapter extends Adapter{ + public getDataFromView(view: View): string { + // buildQueryFromView does not put the final ;, it need to be put apart + return this.buildQueryFromView(view) + ";\n"; + } + + public materializeView(view: View): string { + return null; + } + + private buildQueryFromView (view: View): string { + let sql = "(\n"; + let metrics = view.metrics.map((metric: string) => { + let aggrType = view.getAggregationtype(metric); + let func = this.getAggregateFunction(aggrType); + let extMetric = func + "(" + metric + ")"; + return extMetric; + }); + + if (view.materialized) { + sql += "SELECT " + metrics.join(", ") + ", " + dimensions.join(", ") + "\n"; + sql += "FROM " + "view_" + view.id + "\n"; + sql += "GROUP BY " + view.dimensions.join(", ") + "\n"; + sql += ")\n"; + return sql; + } + + else { + let children: ParsedView[] = view.childViews.map((item: View, idx: number) => { + return { + query: this.queryfyView(item, childAlias, degree + 1), + view: item + }; + }); + + let covered = new Map(); + view.dimensions.forEach((item: string) => covered.set(item, "")); + view.metrics.forEach((item: string) => covered.set(item, "")); + + let projection = "SELECT "; + let viewsFrom = "FROM"; + let selection = "WHERE "; + let grouping = "GROUP BY "; + + let elements = []; + let group = []; + + children.forEach((child: ParsedView) => { + let selected = []; + child.view.dimensions.forEach((dimension: string) => { + let first = covered.get(dimension); + let extDimension = "view_" + child.view.id + "." + dimension; + if (first === "") { + covered.set(dimension, child.view.id); + elements.push(extDimension); + group.push(extDimension); + } + + else { + let extFirst = "view_" + first + "." + dimension; + selected.push(extDimension + " = " + extFirst); + } + }); + + child.view.metrics.forEach((metric: string) => { + let first = covered.get(metric); + let aggrType = child.view.getAggregateFunction(metric); + let func = this.geAggregateFunction(aggrType); + let extMetric = func + "(view_" + child.view.id + "." + metric + ")"; + if (first === "") { + covered.set(metric, child.view.id); + elements.push(extMetric); + } + + }); + + viewsFrom += "\n" + child.query; + + if (selected.length > 0) { + selection += selected.join(" AND "); + } + }); + + projection += elements.join(", ") + "\n"; + selection += "\n"; + grouping += group.join(", ") + "\n"; + + sql += projection + viewsFrom + selection + grouping + ")"; + return sql; + } + + return sql; + } + + private getAggregateFunction(aggrType: AggregationType): string { + switch (aggrType) { + case AggregationType.SUM: + return "SUM"; + case AggregationType.AVG: + return "AVG"; + case AggregationType.COUNT: + return "COUNT"; + default: + return ""; + } + + } +} diff --git a/src/common/aggregationType.ts b/src/common/aggregationType.ts new file mode 100644 index 00000000..8197a086 --- /dev/null +++ b/src/common/aggregationType.ts @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blend. + * + * blend is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blend is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blend. If not, see <http://www.gnu.org/licenses/>. + */ + + export enum AggregationType { + SUM, + AVG, + COUNT, + NONE + }; diff --git a/src/common/query.ts b/src/common/query.ts new file mode 100644 index 00000000..70a46d5b --- /dev/null +++ b/src/common/query.ts @@ -0,0 +1,25 @@ +/* + * Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blend. + * + * blend is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blend is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blend. If not, see <http://www.gnu.org/licenses/>. + */ + +export interface Query { + public metrics: string[]; + public dimensions: string[]; + +} diff --git a/src/core/adapter.ts b/src/core/adapter.ts new file mode 100644 index 00000000..f2981953 --- /dev/null +++ b/src/core/adapter.ts @@ -0,0 +1,26 @@ +/* + * Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blend. + * + * blend is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blend is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blend. If not, see <http://www.gnu.org/licenses/>. + */ + +import { View } from "./view"; + +export abstract class Adapter { + public abstract getDataFromView(view: View): string; + public abstract materializeView(view: View): string; +} diff --git a/src/core/engine.spec.ts b/src/core/engine.spec.ts new file mode 100644 index 00000000..94c6456d --- /dev/null +++ b/src/core/engine.spec.ts @@ -0,0 +1,145 @@ +/* + * Copyright (C) 2015 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blendb. + * + * blendb is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blendb is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blendb. If not, see <http://www.gnu.org/licenses/>. + */ + +import { expect } from "chai"; + +import { Engine } from "./engine"; +import { View } from "./view"; +import { ViewOptions } from "./view"; +import { Query } from "../common/query"; +import { AggregationType } from "../common/aggregationType"; + +describe("engine class", () => { + let query: Query = { + metrics: [], + dimensions: [] + }; + + let views: View[] = []; + + let viewBuilder: Query[] = [ + { metrics: ["met:1", "met:2", "met:3"], dimensions: ["dim:1", "dim:2"]}, + { metrics: ["met:1", "met:3", "met:5"], dimensions: ["dim:1", "dim:2"]}, + { metrics: ["met:3", "met:4", "met:7"], dimensions: ["dim:4", "dim:5"]}, + { metrics: ["met:6", "met:7"], dimensions: ["dim:3", "dim:4", "dim:5", "dim:6"]}, + { metrics: ["met:8", "met:2", "met:3"], dimensions: ["dim:1", "dim:2", "dim:7"]}, + { metrics: ["met:1", "met:2", "met:3"], dimensions: ["dim:1", "dim:2"]}, + { metrics: ["met:2", "met:4"], dimensions: ["dim:1", "dim:2"]}, + { metrics: ["met:8"], dimensions: ["dim:8", "dim:9", "dim:10"]}, + { metrics: ["met:9"], dimensions: ["dim:8", "dim:9", "dim:10"]}, + { metrics: ["met:10"], dimensions: ["dim:8", "dim:9", "dim:10"]} + ]; + + let iterable: [string, AggregationType][] = [ + ["met:1", AggregationType.SUM], + ["met:2", AggregationType.AVG], + ["met:3", AggregationType.AVG], + ["met:4", AggregationType.SUM], + ["met:5", AggregationType.SUM], + ["met:6", AggregationType.AVG], + ["met:7", AggregationType.COUNT], + ["met:8", AggregationType.COUNT], + ["met:9", AggregationType.SUM], + ["met:10", AggregationType.COUNT], + ]; + + let map: Map<string, AggregationType> = new Map(iterable); + + for (let i: number = 0; i < 10; ++ i) { + query.metrics.push("met:" + (i + 1)); + query.dimensions.push("dim:" + (i + 1)); + let options: ViewOptions = { + metrics: viewBuilder[i].metrics, + dimensions: viewBuilder[i].dimensions, + materialized: true, + aggregationMap: map, + childViews: [] + }; + views.push(new View (options)); + } + + views.push(new View({ + metrics: ["met:1", "met:2", "met:3", "met:4", "met:5"], + dimensions: ["dim:1", "dim:2"], + materialized: false, + aggregationMap: map, + childViews: [views[0], views[6]] + })); + + views.push(new View({ + metrics: ["met:8", "met:9", "met:10"], + dimensions: ["dim:8", "dim:9", "dim:10"], + materialized: false, + aggregationMap: map, + childViews: [views[7], views[8], views[9]] + })); + + it("should be create a fill that cover the query and has 4 children", () => { + let engine: Engine = new Engine (views); + let optimalView = engine.query(query); + expect(optimalView).to.be.an("object"); + expect(optimalView).to.have.property("metrics"); + expect(optimalView).to.have.property("dimensions"); + expect(optimalView).to.have.property("childViews"); + expect(optimalView.metrics).to.be.an("array"); + expect(optimalView.dimensions).to.be.an("array"); + expect(optimalView.childViews).to.be.an("array"); + expect(optimalView.metrics.length === 10); + expect(optimalView.dimensions.length === 10); + expect(optimalView.childViews.length === 4); + let metAux: number[] = optimalView.metrics.sort().map((item: string) => { + return Number(item.split(":")[1]); + }); + let dimAux: number[] = optimalView.dimensions.sort().map((item: string) => { + return Number(item.split(":")[1]); + }); + for (let i: number = 1; i <= 10; ++i) { + expect(dimAux[i] === i); + expect(metAux[i] === i); + } + }); + it("should throw an exception, query with non-existent metric", () => { + let engine: Engine = new Engine (views); + let error: boolean = false; + try { + engine.query({metrics: ["met:11"], dimensions: ["dim:1"]}); + } + catch (e){ + error = true; + expect(e.message).to.be.equal("Engine views cannot cover the query"); + + } + expect(error); + }); + + it("should throw an exception, query with non-existent dimension", () => { + let engine: Engine = new Engine (views); + let error: boolean = false; + try { + engine.query({metrics: ["met:1"], dimensions: ["dim:11"]}); + } + catch (e){ + error = true; + expect(e.message).to.be.equal("Engine views cannot cover the query"); + + } + expect(error); + }); +}); diff --git a/src/core/engine.ts b/src/core/engine.ts new file mode 100644 index 00000000..a4e4a5fd --- /dev/null +++ b/src/core/engine.ts @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blend. + * + * blend is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blend is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blend. If not, see <http://www.gnu.org/licenses/>. + */ + +import { Query } from "../common/query"; +import { View } from "./view"; +import { AggregationType } from "../common/aggregationType"; + +export class Engine { + private views: View[]; + + constructor (v: View[]) { + this.views = v; + } + + public query (q: Query) { return this.selectOptimalView(q); } + public insert (data: any) { /*to Implement*/ } + private selectOptimalView (q: Query) { + let objective = q.metrics.concat(q.dimensions); + let optimalViews: View[] = []; + let activeViews = this.views; + + // Run this block until all metrics and dimmensions are covered + while (objective.length > 0) { + let bestView: View; + let shortestDistance = objective.length + 1; + activeViews = activeViews.filter((view: View) => { + let cover = view.metrics.concat(view.dimensions); + let intersection = cover.filter((item: string) => { + return objective.indexOf(item) !== -1; + }); + + if (intersection.length > 0) { + let distance = objective.length - intersection.length; + + if (distance < shortestDistance) { + bestView = view; + shortestDistance = distance; + } + return true; + } + + /*If the intersection is empty, + remove this element from future searches*/ + return false; + }); + + if (shortestDistance === objective.length + 1) { + throw new Error ("Engine views cannot cover the query"); + } + + optimalViews.push(bestView); + + //remove metrics and dimensions corevered by the bestView + objective = objective.filter((item: string) => { + let cover = bestView.dimensions.concat(bestView.metrics); + return cover.indexOf(item) === -1; + }); + } + + if (optimalViews.length === 1) { + return optimalViews.pop(); + } + + else { + let map = new Map(); + optimalViews.forEach((view: View) => { + view.aggregationMap.forEach((value: AggregationType, key: string) => { + map.set(key, value); + }); + }); + + let options = { + metrics: q.metrics, + dimensions: q.dimensions, + materialized: false, + aggregationMap: map, + childViews: optimalViews + }; + + let view = new View(options); + this.views.push(view); + return view; + } + } +} diff --git a/src/core/view.ts b/src/core/view.ts new file mode 100644 index 00000000..34716d22 --- /dev/null +++ b/src/core/view.ts @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2016 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blend. + * + * blend is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blend is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blend. If not, see <http://www.gnu.org/licenses/>. + */ + +import { Hash } from "../util/hash"; +import { AggregationType } from "../common/aggregationType"; + +export interface ViewOptions { + metrics: string[]; + dimensions: string[]; + materialized: boolean; + aggregationMap: Map <string, AggregationType>; + childViews: View[]; +} + +export class View { + public readonly id: string; + public readonly metrics: string[]; + public readonly dimensions: string[]; + public readonly materialized: boolean; + public readonly aggregationMap: Map<string, AggregationType>; + + public childViews: View[]; + + constructor (options: ViewOptions) { + this.metrics = options.metrics; + this.dimensions = options.dimensions; + this.materialized = options.materialized; + this.childViews = options.childViews; + this.aggregationMap = options.aggregationMap; + this.id = Hash.sha1(options.metrics.sort(), options.dimensions.sort()); + } + + public getAggregatationType(metric: string): AggregationType { + if (this.aggregationMap.has(metric)) { + return AggregationType.NONE; + } + + return this.aggregationMap.get(metric); + } +} -- GitLab