diff --git a/database/config.ts b/database/config.ts
deleted file mode 100755
index e138ee60c67a8de25fbd39ad05044050525337fc..0000000000000000000000000000000000000000
--- a/database/config.ts
+++ /dev/null
@@ -1,152 +0,0 @@
-#!/usr/bin/env node
-
-/*
- * Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
- * Departamento de Informatica - Universidade Federal do Parana
- *
- * This file is part of blendb.
- *
- * blendb is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * blendb is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with blendb. If not, see <http://www.gnu.org/licenses/>.
- */
-
-import { Hash } from "../src/util/hash";
-import * as fs from "fs";
-
-interface View {
- alias: string;
- query: string;
- fields: any;
-}
-
-/*interface Table {
- create: string;
- load: string;
-}*/
-
-function typeConvertion(t: string) {
- switch (t) {
- case "integer":
- return "INTEGER";
- case "date":
- return "DATE";
- case "string":
- return "TEXT";
- case "boolean":
- return "BOOLEAN";
- default:
- return "";
- }
-}
-
-/*function createTable(view: View) {
- let output = "-- View: " + view.alias + "\n";
- let props = [];
- let keys = [];
- for (let field in view.fields) {
- props.push("\"" + field + "\" " + typeConvertion(view.fields[field]));
- keys.push(field);
- }
- keys.sort();
- let name = "view_" + Hash.sha1(keys);
- output += "CREATE TABLE " + name + "(" + props.join(",") + ");\n";
- output += "GRANT ALL ON " + name + " TO :DBUSER;\n";
- output += "GRANT ALL ON SEQUENCE " + name + "_id_seq TO :DBUSER;\n";
-
- let pathProp = view.alias.toUpperCase() + "_CSV_PATH";
- let path = "\\set " + pathProp +
- " '\\'' :SCRIPT_PATH/postgres/create/data/" +
- view.loadFile + "'\\''";
- let nickProp = view.alias.toUpperCase() + "_VIEW";
- let wrappedKeys = keys.map((item) => "\"" + item + "\"");
- let nick = "\\set " + nickProp + " " + name + "";
- let copy = "COPY :" + nickProp + "(" + wrappedKeys.join(",") + ") FROM :" +
- pathProp + " DELIMITER ','";
- let load = path + "\n" + nick + "\n" + copy + "\n\n";
-
- return { create: output, load: load };
-}*/
-
-function createView(view: View): string {
- let output = "-- View: " + view.alias + "\n";
- let props = [];
- let keys = [];
- for (let field in view.fields) {
- props.push("\"" + field + "\" " + typeConvertion(view.fields[field]));
- keys.push(field);
- }
- keys.sort();
- let name = "view_" + Hash.sha1(keys);
- output += "DROP VIEW IF EXISTS " + name + " CASCADE;\n";
- output += "CREATE OR REPLACE VIEW " + name + " AS\n";
- output += fs.readFileSync(filePath + "/views/" + view.query, {encoding: "utf-8" });
-
- return output;
-}
-
-// let configFile = process.argv[2];
-// let schemaFile = process.argv[3];
-// let loadFile = process.argv[4];
-let filePath = process.argv[2];
-
-let configFile = filePath + "/schema.json";
-let schemaFile = filePath + "/schema.sql";
-
-fs.readFile(configFile, {encoding: "utf-8" }, (err, data) => {
- if (err) {
- console.log("The config file could not be read. Abort");
- process.exitCode = 1;
- return;
- }
-
- let config: View[] = JSON.parse(data);
- let schema = "\\set ON_ERROR_STOP\n\n";
- // let load = "\\set ON_ERROR_STOP\n\n";
-
- /*for (let i = 0; i < config.length; ++i) {
- let view = config[i];
- let table: Table = createTable(view);
- schema += table.create;
- load += table.load;
- }
-
- fs.writeFile(schemaFile, schema, (error) => {
- if (error) {
- console.log("The schema file could not be written. Abort");
- process.exitCode = -1;
- return;
- }
- });
-
- fs.writeFile(loadFile, load, (error) => {
- if (error) {
- console.log("The load file could not be written. Abort");
- process.exitCode = -1;
- return;
- }
- });*/
-
- for (let i = 0; i < config.length; ++i) {
- let view = config[i];
- let stringfyied: string = createView(view);
- schema += stringfyied;
- }
-
- fs.writeFile(schemaFile, schema, (error) => {
- if (error) {
- console.log("The schema file could not be written. Abort");
- process.exitCode = -1;
- return;
- }
- });
-});
diff --git a/database/createSchema.ts b/database/createSchema.ts
new file mode 100755
index 0000000000000000000000000000000000000000..86b1507cc4864e35870f6b9d3a6b7dcbbae8ac04
--- /dev/null
+++ b/database/createSchema.ts
@@ -0,0 +1,91 @@
+#!/usr/bin/env node
+
+/*
+ * Copyright (C) 2017 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Engine } from "../src/core/engine";
+import { PostgresAdapter } from "../src/adapter/postgres";
+import { ConfigParser } from "../src/util/configParser";
+import * as path from "path";
+import * as fs from "fs";
+
+if (process.argv.length !== 4) {
+ console.log("Wrong arguments number.");
+ console.log("Arguments: <configFile> <outputFile>");
+ process.exitCode = 1;
+ process.exit();
+}
+
+const configFile = process.argv[2];
+const schemaFile = process.argv[3];
+const config = ConfigParser.parse(configFile);
+const referencePath = path.dirname(configFile);
+const engine = new Engine();
+const adapter = new PostgresAdapter(config.connection);
+
+let schema = "";
+
+config.metrics.forEach ((met) => engine.addMetric(met));
+config.dimensions.forEach ((dim) => engine.addDimension(dim));
+
+for (let i = 0; i < config.buildViews.length; ++i) {
+ const view = config.buildViews[i].view;
+ const filePath = config.buildViews[i].file;
+ const alias = config.buildViews[i].alias;
+ if (view.origin) {
+ // Cria uma tabela
+ let output = "-- View: " + alias + "\n";
+ output += "DROP VIEW IF EXISTS view_" + view.id + " CASCADE;\n";
+ output += "CREATE OR REPLACE VIEW view_" + view.id + " AS\n";
+ output += fs.readFileSync(referencePath + "/" + filePath, {encoding: "utf-8" });
+ schema += output + "\n";
+ engine.addView(view);
+ }
+
+}
+
+for (let i = 0; i < config.buildViews.length; ++i) {
+ const view = config.buildViews[i].view;
+ const alias = config.buildViews[i].alias;
+ if (!view.origin) {
+ const materializedView = engine.query({
+ metrics: view.metrics,
+ dimensions: view.dimensions,
+ clauses: view.clauses
+ });
+
+ const table = adapter.getQueryFromView(materializedView);
+ const query = "-- View: " + alias + "\n" +
+ "DROP MATERIALIZED VIEW IF EXISTS view_" + view.id + " CASCADE;\n" +
+ "CREATE MATERIALIZED VIEW view_" + materializedView.id +
+ " AS " + table + "\n\n";
+
+ schema += query;
+ engine.addView(materializedView);
+
+ }
+}
+fs.writeFile(schemaFile, schema, (error) => {
+ if (error) {
+ console.log("The schema file could not be written. Abort");
+ process.exitCode = 1;
+ return;
+ }
+});
diff --git a/database/schema.json b/database/schema.json
deleted file mode 100644
index 0fabaafcc400342ca9dc8267d78ffa000054974c..0000000000000000000000000000000000000000
--- a/database/schema.json
+++ /dev/null
@@ -1,78 +0,0 @@
-[
- {
- "alias" : "net_usage"
- , "query" : "fnu.sql"
- , "fields" : {
- "dim:contactDate" : "date",
- "dim:point" : "integer",
- "dim:macAddr" : "string",
- "met:avg:download" : "integer",
- "met:avg:upload" : "integer",
- "met:count:contact" : "integer",
- "met:max:contact" : "date",
- "met:max:download" : "integer",
- "met:max:upload" : "integer",
- "met:sum:download" : "integer",
- "met:sum:upload" : "integer"
- }
- }
- , {
- "alias" : "city"
- , "query" : "city.sql"
- , "fields" : {
- "dim:city" : "integer",
- "dim:cityName" : "string",
- "dim:region" : "string",
- "dim:state" : "string",
- "met:count:city" : "integer"
- }
- }
- , {
- "alias" : "point"
- , "query" : "point.sql"
- , "fields" : {
- "dim:active" : "boolean",
- "dim:city" : "integer",
- "dim:digitalCity" : "boolean",
- "dim:gesac" : "boolean",
- "dim:point" : "integer",
- "dim:pointDate" : "date",
- "dim:pointName" : "string",
- "dim:project" : "string",
- "dim:telecenter" : "boolean",
- "met:count:point" : "integer",
- "met:avg:hiredDownload" : "integer",
- "met:avg:hiredUpload" : "integer"
- }
- }
- , {
- "alias" : "inventory_changes"
- , "query" : "inv_changes.sql"
- , "fields" : {
- "dim:disk" : "integer",
- "dim:inventoryDate" : "date",
- "dim:macAddr" : "string",
- "dim:memory" : "string",
- "dim:oldDisk" : "string",
- "dim:oldInventoryDate" : "date",
- "dim:oldMemory" : "string",
- "dim:oldOs" : "string",
- "dim:oldProcessor" : "string",
- "dim:os" : "string",
- "dim:point" : "integer",
- "dim:processor" : "string",
- "met:count:machines" : "integer",
- "met:count:modified_disk" : "integer",
- "met:count:modified_memory" : "integer"
- }
- }
- , {
- "alias" : "inventory"
- , "query" : "inv.sql"
- , "fields" : {
- "dim:currentInventory" : "boolean",
- "dim:inventoryDate" : "date",
- "dim:point" : "integer"
- }
- }
-]
diff --git a/src/adapter/postgres.ts b/src/adapter/postgres.ts
index 9055d2b998aab7023bb7b4f4c52a77922513d415..0c32689626e17cfed097e5de4169452a30294cb7 100644
--- a/src/adapter/postgres.ts
+++ b/src/adapter/postgres.ts
@@ -40,18 +40,8 @@ export class PostgresAdapter extends Adapter {
this.pool = new Pool(config);
}
public getDataFromView(view: View, cb: (error: Error, result?: any[]) => void): void {
- const materialized = this.searchMaterializedViews(view).sort((a, b) => {
- return (a.id < b.id) ? -1 : 1;
- });
-
- const unique = [materialized[0]];
- for (let i = 1; i < materialized.length; ++i) {
- if (materialized[i - 1].id !== materialized[i].id) {
- unique.push(materialized[i]);
- }
- }
+ const query = this.getQueryFromView(view);
- const query = this.buildQuery(view, unique);
this.pool.connect((err, client, done) => {
if (err) {
cb (err);
@@ -69,6 +59,21 @@ export class PostgresAdapter extends Adapter {
return false;
}
+ public getQueryFromView(view: View): string {
+ const materialized = this.searchMaterializedViews(view).sort((a, b) => {
+ return (a.id < b.id) ? -1 : 1;
+ });
+
+ const unique = [materialized[0]];
+ for (let i = 1; i < materialized.length; ++i) {
+ if (materialized[i - 1].id !== materialized[i].id) {
+ unique.push(materialized[i]);
+ }
+ }
+
+ return this.buildQuery(view, unique);
+ }
+
private searchMaterializedViews(view: View): View[] {
let r: View[] = [];
if (view.materialized) {
diff --git a/src/util/configParser.ts b/src/util/configParser.ts
index 16a70b7e9db6151632d5a83262f612f925e47012..831050361431b421c2e88adc2712e49966ad9333 100644
--- a/src/util/configParser.ts
+++ b/src/util/configParser.ts
@@ -31,6 +31,7 @@ import * as yaml from "js-yaml";
export interface ViewParsingOptions {
alias: string;
data: string;
+ file?: string;
origin?: boolean;
dimensions: string[];
metrics: string[];
@@ -62,6 +63,12 @@ interface ConfigFile {
schema: ConfigSchema;
}
+interface BuildView {
+ view: View;
+ file: string;
+ alias: string;
+}
+
export interface LoadStruct{
create: boolean;
insert: boolean;
@@ -74,6 +81,7 @@ export interface ParsedConfig {
dimensions: Dimension[];
struct: LoadStruct;
loadViews: LoadView[];
+ buildViews: BuildView[];
}
interface DimensionMap {
@@ -99,7 +107,8 @@ export class ConfigParser {
metrics: [],
dimensions: [],
struct: config.struct,
- loadViews: []
+ loadViews: [],
+ buildViews: []
};
let metMap: MetricMap = {};
@@ -126,6 +135,12 @@ export class ConfigParser {
parsed.views.push(view);
let loadView: LoadView = {view: view, data: viewsOpts[i].data};
parsed.loadViews.push(loadView);
+ const buildView: BuildView = {
+ view: view,
+ file: viewsOpts[i].file,
+ alias: viewsOpts[i].alias
+ };
+ parsed.buildViews.push(buildView);
}
return parsed;