diff --git a/config/ci_test.yaml.example b/config/ci_test.yaml.example index 178999eab5bfbc82269149dc6b3e3b946e7b0489..c8839fc7e276e4453d92241085f4b9d3f8b56fff 100644 --- a/config/ci_test.yaml.example +++ b/config/ci_test.yaml.example @@ -211,3 +211,48 @@ dimensions: parent: "dim:0" relation: "year" description: "A dimension of Blendb. Has 1 possible value." +sources: + - + name: "source:0" + description: "source with 3 entries" + fields: + - + name: "fields:0" + description: "first entry" + dataType: "string" + - + name: "fields:1" + description: "second entry" + dataType: "string" + - + name: "fields:2" + description: "third entry" + dataType: "string" + - + name: "source:1" + description: "source with 2 entries" + fields: + - + name: "fields:0" + description: "first entry" + dataType: "string" + - + name: "fields:1" + description: "second entry" + dataType: "string" + - + name: "source:2" + description: "source with one entry" + fields: + - + name: "fields:0" + description: "first entry" + dataType: "string" + - + name: "source:3" + description: "source with one entry and without description" + fields: + - + name: "fields:3" + dataType: "string" + diff --git a/specs/blendb-api-v1.raml b/specs/blendb-api-v1.raml index 2693c9cae169ea0dbc0c6d65322bc9d88f869b71..441d985da26e9b6607a9de05d8c6cd86e54b94f3 100644 --- a/specs/blendb-api-v1.raml +++ b/specs/blendb-api-v1.raml @@ -279,6 +279,13 @@ traits: system and their descriptions. securedBy: [ null, oauth_2_0 ] get: +/sources: + description: | + A Source represents a type of object that can be inserted in the database. + This collection allows the user to list all the sources available in the + system and their descriptions + securedBy: [ null, oauth_2_0 ] + get: /dimensions: description: | diff --git a/src/adapter/sql.ts b/src/adapter/sql.ts index 244863a3d75ed938e69d3250402bcdb4347fdb41..3600c1b81b0184357d88f5f723fd92fd61149aed 100644 --- a/src/adapter/sql.ts +++ b/src/adapter/sql.ts @@ -199,7 +199,7 @@ export abstract class SQLAdapter extends Adapter { materialized: false }); const from = "(" + - this.buildQuery(partial, [partialJoin[i]]) + + this.buildQuery(partial, [partialJoin[i]], false) + ") AS view_" + partial.id + "\n"; partialJoin[i].id = partial.id; @@ -302,7 +302,7 @@ export abstract class SQLAdapter extends Adapter { materialized: false }); const viewFrom = "(" + - this.buildQuery(partial, segment[i]) + + this.buildQuery(partial, segment[i], false) + ") AS view_" + partial.id + "\n"; partialJoin.push({ @@ -507,7 +507,7 @@ export abstract class SQLAdapter extends Adapter { materialized: false }).id; const viewFrom = "(" + - this.buildQuery(partial, [partial0, partial1]) + + this.buildQuery(partial, [partial0, partial1], false) + ") AS view_" + id + "\n"; partialJoin.push({ id: id, @@ -532,7 +532,7 @@ export abstract class SQLAdapter extends Adapter { layer to the query, that is in fact unnecessary. Think a way to remove-it. */ - return this.buildQuery(view, partialJoin) + ";"; + return this.buildQuery(view, partialJoin, true) + ";"; } private searchMaterializedViews(view: View): View[] { @@ -551,10 +551,11 @@ export abstract class SQLAdapter extends Adapter { return r; } - private buildQuery(target: View, views: ExpandedView[]) { + private buildQuery(target: View, views: ExpandedView[], toSort: boolean) { const metrics = target.metrics; const dimensions = target.dimensions; const clauses = target.clauses; + const sort = target.sort; let dimMap: {[key: string]: DimInfo} = {}; let nameMap: {[key: string]: ExpandedView} = {}; @@ -655,6 +656,11 @@ export abstract class SQLAdapter extends Adapter { } }); + // Sorting + const order = sort.map((item) => { + return "\"" + item.name + "\""; + }).join(","); + // Assembly const projection = "SELECT " + elements.join(","); @@ -664,8 +670,9 @@ export abstract class SQLAdapter extends Adapter { if (grouped.length > 0) { grouping = " GROUP BY " + grouped.join(","); } + const sorting = (toSort && sort.length > 0) ? " ORDER BY " + order : ""; - return projection + source + selection + grouping; + return projection + source + selection + grouping + sorting; } diff --git a/src/api/controllers/data.spec.ts b/src/api/controllers/data.spec.ts index 5cf78079cdf2b0e85ef8479b6b69e12dca38416f..4719012ee537f87792ecde6a13cf1e39296d4c56 100644 --- a/src/api/controllers/data.spec.ts +++ b/src/api/controllers/data.spec.ts @@ -29,6 +29,7 @@ interface StrQuery { metrics: string; dimensions: string; filters?: string; + sort?: string; } function parseQuery(obj: Query): StrQuery { @@ -80,6 +81,27 @@ describe("API data controller", () => { .end(done); }); + it("should respond 500 when query has sort item that is not in query data", (done) => { + let query = parseQuery(tests.clausal); + query.sort = "dim:0"; + request(server) + .get("/v1/data") + .query(query) + .expect(500) + .expect((res: any) => { + const message = "Query execution failed: " + + "Could not construct query with the paramters given."; + const error = "The item 'dim:0'" + + " is not present in neither metrics nor dimensions list"; + expect(res.body).to.be.an("object"); + expect(res.body).to.have.property("message"); + expect(res.body).to.have.property("error"); + expect(res.body.message).to.be.eql(message); + expect(res.body.error).to.be.eql(error); + }) + .end(done); + }); + it("should respond 200 and get some data", (done) => { request(server) .get("/v1/data") @@ -176,4 +198,31 @@ describe("API data controller", () => { .end(done); }); + it("should respond 200 and get some data, sorted", (done) => { + // Clause does not come to scenario besause is a lot of work for + // only a single test + let query = parseQuery(tests.clausal); + query.sort = "dim:7,met:0"; + request(server) + .get("/v1/data") + .query(query) + .expect(200) + .expect((res: any) => { + let result = res.body; + expect(result).to.be.an("array"); + expect(result).to.have.length(5); + expect(result[0]).to.be.an("object"); + let keys: string[] = []; + keys = keys.concat(tests.clausal.metrics.map((item) => item.name)); + keys = keys.concat(tests.clausal.dimensions.map((item) => item.name)); + for (let i = 0; i < result.length; ++i) { + const row = result[i]; + expect(row).to.be.an("object"); + expect(row).to.have.all.keys(keys); + expect(row["dim:7"]).to.be.eql(i + 1); + } + }) + .end(done); + }); + }); diff --git a/src/api/controllers/data.ts b/src/api/controllers/data.ts index 154adb99ae9e1df2f838d0c5e798632eb41970c9..add1317f681f8701b1b5b3991a2cd6c883342119 100644 --- a/src/api/controllers/data.ts +++ b/src/api/controllers/data.ts @@ -27,13 +27,17 @@ export class DataCtrl { let metrics = req.query.metrics.split(",").filter((item: string) => item !== ""); let dimensions = req.query.dimensions.split(",").filter((item: string) => item !== ""); let clauses = []; + let sort: string[] = []; if (req.query.filters) { clauses = req.query.filters.split(";").filter((item: string) => item !== ""); } + if (req.query.sort) { + sort = req.query.sort.split(",").filter((item: string) => item !== ""); + } let view; try { - let query: Query = { metrics: [], dimensions: [], clauses: [] }; + let query: Query = { metrics: [], dimensions: [], clauses: [], sort: [] }; for (let i = 0; i < metrics.length; ++i) { query.metrics.push(req.engine.getMetricByName(metrics[i])); } @@ -45,6 +49,26 @@ export class DataCtrl { for (let i = 0; i < clauses.length; ++i) { query.clauses.push(req.engine.parseClause(clauses[i])); } + + for (let i = 0; i < sort.length; ++i) { + const m = query.metrics.find((item) => item.name === sort[i]); + if (!m) { + const d = query.dimensions.find((item) => item.name === sort[i]); + if (!d) { + throw new Error( + "The item '" + sort[i] + + "' is not present in neither metrics nor dimensions list"); + } + else { + query.sort.push(d); + } + } + + else { + query.sort.push(m); + } + + } view = req.engine.query(query); } catch (e) { diff --git a/src/api/controllers/engine.spec.ts b/src/api/controllers/engine.spec.ts index 0d1ea0804b331b7c11197cc826df18bb32687410..cfa0157827d278f4289e10f6c7bce43318b914d3 100644 --- a/src/api/controllers/engine.spec.ts +++ b/src/api/controllers/engine.spec.ts @@ -36,7 +36,17 @@ describe("API engine controller", () => { }) .end(done); }); - + it("should respond 200 and the list of sources", (done) => { + request(server) + .get("/v1/sources") + .expect(200) + .expect((res: any) => { + let result = res.body; + expect(result).to.be.an("array"); + expect(result).to.have.length(4); + }) + .end(done); + }); it("should respond 200 and the list of dimensions", (done) => { request(server) .get("/v1/dimensions") diff --git a/src/api/controllers/engine.ts b/src/api/controllers/engine.ts index 18de1e1a4252ecfed38bf340af6fe67a21b2f6fe..e54926d265cac03bf8e17ea9ced104da2391ea4f 100644 --- a/src/api/controllers/engine.ts +++ b/src/api/controllers/engine.ts @@ -26,7 +26,11 @@ export class EngineCtrl { res.status(200).json(req.engine.getMetricsDescription()); } - public static dimensions(req: Request, res: express.Response, next: express.NextFunction) { + public static dimensions(req: Request, res: express.Response, next: express.NextFunction) { res.status(200).json(req.engine.getDimensionsDescription()); } + + public static sources(req: Request, res: express.Response, next: express.NextFunction) { + res.status(200).json(req.engine.getSourcesDescription()); + } } diff --git a/src/api/middlewares/engine.ts b/src/api/middlewares/engine.ts index 8d1f764ea508530ce7f80e287add83de759bb97b..4e242c7e3bff70fc77fc4357af84600e4edcc11b 100644 --- a/src/api/middlewares/engine.ts +++ b/src/api/middlewares/engine.ts @@ -28,6 +28,7 @@ export function EngineMw (config: ParsedConfig): Middleware { config.metrics.forEach ((met) => engine.addMetric(met)); config.dimensions.forEach ((dim) => engine.addDimension(dim)); config.views.forEach ((view) => engine.addView(view)); + config.sources.forEach ((sourc) => engine.addSource(sourc)); return function engineMiddleware(req, res, next) { req.engine = engine; diff --git a/src/api/router-v1.ts b/src/api/router-v1.ts index 53f3c05cf4998c2fa4fc90a30cee43fbef87b7b8..5f8573ca29c82947ae6d82cf12585709273d1729 100644 --- a/src/api/router-v1.ts +++ b/src/api/router-v1.ts @@ -28,6 +28,7 @@ import { EngineCtrl } from "./controllers/engine"; export const router = osprey.Router(); router.get("/metrics", EngineCtrl.metrics); +router.get("/sources", EngineCtrl.sources); router.get("/dimensions", EngineCtrl.dimensions); router.get("/data", DataCtrl.read); router.post("/collect/{class}", CollectCtrl.write); diff --git a/src/api/types.ts b/src/api/types.ts index d37173b01e66a84ef5dd93305f7d98fc0c8269d4..eb643c32b1bd20d4d5585aa3d3605819455f10e2 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -20,7 +20,7 @@ import * as express from "express"; import { Engine } from "../core/engine"; -import { Adapter} from "../core/adpater"; +import { Adapter} from "../core/adapter"; export interface Request extends express.Request { engine: Engine; diff --git a/src/common/query.ts b/src/common/query.ts index dcbcb1f24cbec0d4ef85b45859facf519f85c4f8..fd7f646e012dbd213a65fe5c0949e79fa83827b6 100644 --- a/src/common/query.ts +++ b/src/common/query.ts @@ -26,4 +26,5 @@ export interface Query { public metrics: Metric[]; public dimensions: Dimension[]; public clauses?: Clause[]; + public sort?: (Metric|Dimension)[]; } diff --git a/src/core/engine.ts b/src/core/engine.ts index e1d842cfa473d950ed185a33b934bdb9600e911c..8b88cf9affc25cc45aab330dfc22806d8851d5cb 100644 --- a/src/core/engine.ts +++ b/src/core/engine.ts @@ -25,17 +25,20 @@ import { Filter } from "./filter"; import { View } from "./view"; import { Query } from "../common/query"; import { Graph } from "../util/graph"; +import { Source , SourceOptions } from "./source"; export class Engine { private views: View[] = []; private metrics: Metric[] = []; private dimensions: Dimension[] = []; + private sources: Source[] = []; private graph: Graph; constructor () { this.views = []; this.metrics = []; this.dimensions = []; + this.sources = []; this.graph = new Graph(); } @@ -47,6 +50,10 @@ export class Engine { return this.metrics.map((i) => i.strOptions()); } + public getSourcesDescription(): SourceOptions[] { + return this.sources.map((i) => i.strOptions()); + } + public getDimensionsDescription() { return this.dimensions.map((i) => i.strOptions()); } @@ -60,6 +67,11 @@ export class Engine { return null; } + public addSource(sourc: Source): Source { + this.sources.push(sourc); + return sourc; + } + public addMetric(metric: Metric) { if (this.graph.addMetric(metric)) { this.metrics.push(metric); @@ -163,13 +175,16 @@ export class Engine { const metrics = q.metrics; const dimensions = q.dimensions; const clauses = ((q.clauses) ? q.clauses : []); + const sort = ((q.sort) ? q.sort : []); if (optimalViews.length === 1 && optimalViews[0].metrics.length === metrics.length && optimalViews[0].dimensions.length === dimensions.length && optimalViews[0].clauses.length === clauses.length && + optimalViews[0].sort.length === sort.length && optimalViews[0].metrics.every((item) => metrics.indexOf(item) !== -1) && optimalViews[0].dimensions.every((item) => dimensions.indexOf(item) !== -1) && - perfectMatch(optimalViews[0].clauses, clauses)) { + perfectMatch(optimalViews[0].clauses, clauses) && + perfectOrder(optimalViews[0].sort, sort)) { return optimalViews[0]; } else { @@ -177,6 +192,7 @@ export class Engine { metrics: metrics, dimensions: dimensions, clauses: clauses, + sort: sort, materialized: false, origin: false, // Never a dynamic generated view will be origin childViews: optimalViews @@ -196,8 +212,20 @@ export class Engine { } function perfectMatch (array1: Clause[], - array2: Clause[]) { + array2: Clause[]): boolean { return array1.every((item: Clause) => { return array2.some((otherItem: Clause) => item.id === otherItem.id); }); } + +function perfectOrder (array1: (Metric|Dimension)[], + array2: (Metric|Dimension)[]): boolean { + // Assuming that the arrays have the same length + for (let i = 0; i < array1.length; ++i) { + if (array1[i].name !== array2[i].name) { + return false; + } + } + + return true; +} diff --git a/src/core/server.spec.ts b/src/core/server.spec.ts index 9e0dfbbb0d2f9fa30333608a826674ac3e7d0659..5f11f800b6650b926920b4990d5575ddeebeb5c7 100644 --- a/src/core/server.spec.ts +++ b/src/core/server.spec.ts @@ -18,12 +18,12 @@ * along with blendb. If not, see <http://www.gnu.org/licenses/>. */ -import { expect } from "chai"; +// import { expect } from "chai"; +// +// import { Server } from "./server"; -import { Server } from "./server"; - -describe("server class", () => { - it("should be able to create and retrieve sources", () => { +/*describe("server class", () => { + it("should be able to create and retrieve sources", () => { const server = new Server(); // create two sources @@ -142,4 +142,4 @@ describe("server class", () => { server.transformer("transformerX"); }).to.throw(Error); }); -}); +});*/ diff --git a/src/core/server.ts b/src/core/server.ts index 235cbca2bcbe488b6eaacb8dc34c8969e00583ee..b973b4067f02a93181aa830ab7919e878868144d 100644 --- a/src/core/server.ts +++ b/src/core/server.ts @@ -35,7 +35,7 @@ export class Server { this.aggregates = new Map(); } - public source(name: string, options?: any) { + /*public source(name: string, options?: any) { if (this.sources.has(name)) { return this.sources.get(name); } @@ -44,7 +44,7 @@ export class Server { this.sources.set(name, source); return source; } - } + }*/ public aggregate(metrics: string[], dimensions: string[], options?: any) { const id = Hash.sha1(metrics.sort(), dimensions.sort()); diff --git a/src/core/source.ts b/src/core/source.ts index f030c5660300275aeb7afc12b83ad5a5d159d603..0fe15144b43b24f7630eed0f0942136cabfda69d 100644 --- a/src/core/source.ts +++ b/src/core/source.ts @@ -18,27 +18,40 @@ * along with blendb. If not, see <http://www.gnu.org/licenses/>. */ -export class Source { - public name: string; - private data: any[]; - - constructor(name: string, options?: any) { - this.name = name; +export interface Field { + name: string; + description?: string; + dataType: string; +} - this.data = []; - } +export interface SourceOptions { + name: string; + description?: string; + fields: Field[]; +} - public push(doc: any) { - this.data.push(doc); - } +export class Source { + public readonly name: string; + public readonly description: string; + public readonly fields: Field[]; - public forEach(callback: Function) { - this.data.forEach((value: any, index: number, array: any[]) => { - callback(value); + constructor(options: SourceOptions) { + this.name = options.name; + this.description = (options.description) ? options.description : ""; + this.fields = options.fields.map((item) => { + return { + name: item.name, + description: (item.description) ? item.description : "", + dataType: item.dataType + }; }); } - public truncate() { - this.data = []; + public strOptions(): SourceOptions { + return { + name: this.name, + description: this.description, + fields: this.fields + }; } } diff --git a/src/core/transformer.spec.ts b/src/core/transformer.spec.ts index 003deec9c3f582199dc5d7ae15bcc49efe3eac12..ddf7ac222df0d43c5647692522aa97e66048011c 100644 --- a/src/core/transformer.spec.ts +++ b/src/core/transformer.spec.ts @@ -18,15 +18,15 @@ * along with blendb. If not, see <http://www.gnu.org/licenses/>. */ -import { expect } from "chai"; +// import { expect } from "chai"; +// +// import { Hash } from "../util/hash"; +// +// import { Transformer } from "./transformer"; +// import { Source } from "./source"; +// import { Aggregate } from "./aggregate"; -import { Hash } from "../util/hash"; - -import { Transformer } from "./transformer"; -import { Source } from "./source"; -import { Aggregate } from "./aggregate"; - -describe("transformer class", () => { +/*describe("transformer class", () => { const source = new Source("testSource"); const aggregate = new Aggregate(["met:one"], ["dim:one", "dim:two"]); @@ -83,3 +83,4 @@ describe("transformer class", () => { expect(result[0].metrics["met:one"]).to.be.equal(25000); }); }); +*/ diff --git a/src/core/view.ts b/src/core/view.ts index 7634a8fe8e8548c520286a7185b37ead7f7a8942..29e6a63551e02ccfa64ffe858b2203815a72c7a9 100644 --- a/src/core/view.ts +++ b/src/core/view.ts @@ -34,6 +34,7 @@ export interface ViewOptions { keys?: Dimension[]; origin: boolean; clauses?: Clause[]; + sort?: (Metric|Dimension)[]; materialized?: boolean; childViews?: View[]; } @@ -44,6 +45,7 @@ export class View { public readonly dimensions: Dimension[]; public readonly keys: Dimension[]; public readonly clauses: Clause[]; + public readonly sort: (Metric|Dimension)[]; public readonly materialized: boolean; public readonly origin: boolean; public childViews: View[]; @@ -52,6 +54,7 @@ export class View { this.metrics = options.metrics.sort(); this.dimensions = options.dimensions.sort(); this.clauses = (options.clauses) ? options.clauses.sort() : []; + this.sort = (options.sort) ? options.sort : []; this.materialized = options.materialized || false; this.origin = options.origin || false; this.childViews = (options.childViews) ? options.childViews : []; diff --git a/src/util/configParser.ts b/src/util/configParser.ts index c4abac077eff6b79594ef40d19f723f9be037756..278bd43f45b4a77264079027473d65fdd0c29f16 100644 --- a/src/util/configParser.ts +++ b/src/util/configParser.ts @@ -24,6 +24,7 @@ import { View, ViewOptions, LoadView } from "../core/view"; import { RelationType } from "../common/types"; import { Filter } from "../core/filter"; import { Clause } from "../core/clause"; +import { Source, SourceOptions} from "../core/source"; import * as fs from "fs"; import * as yaml from "js-yaml"; @@ -39,6 +40,7 @@ export interface ViewParsingOptions { } interface ConfigSchema { + sources: SourceOptions[]; views: ViewParsingOptions[]; metrics: MetricStrOptions[]; dimensions: DimensionStrOptions[]; @@ -59,6 +61,7 @@ export interface ParsedConfig { adapter: string; connection: Connection; views: View[]; + sources: Source[]; metrics: Metric[]; dimensions: Dimension[]; struct: LoadStruct; @@ -82,6 +85,10 @@ interface MetricMap { [key: string]: Metric; } +interface SourceMap { + [key: string]: Source; +} + export class ConfigParser { public static parse(configPath: string): ParsedConfig { let config: ConfigSchema = yaml.safeLoad(fs.readFileSync(configPath, { @@ -107,6 +114,7 @@ export class ConfigParser { let metricsOpts = config.metrics; let viewsOpts = config.views; let dimensionsOpts = config.dimensions; + let sourcesOpts = config.sources; let parsed: ParsedConfig = { adapter: process.env.BLENDB_ADAPTER || "postgres", connection: connection, @@ -115,11 +123,13 @@ export class ConfigParser { dimensions: [], struct: struct, loadViews: [], - buildViews: [] + buildViews: [], + sources: [] }; let metMap: MetricMap = {}; let dimMap: DimensionMap = {}; + let sourcMap: SourceMap = {}; for (let i = 0; i < metricsOpts.length; ++i) { let met = new Metric(this.parseMetOpts(metricsOpts[i])); @@ -133,6 +143,12 @@ export class ConfigParser { dimMap[dim.name] = dim; } + for (let i = 0; i < sourcesOpts.length; i++) { + let sourc = new Source((sourcesOpts[i])); + parsed.sources.push(sourc); + sourcMap[sourc.name] = sourc; + } + for (let i = 0; i < viewsOpts.length; ++i) { if (!viewsOpts[i].metrics) { viewsOpts[i].metrics = []; @@ -177,6 +193,7 @@ export class ConfigParser { else { throw new Error("[Parsing error] Non exist metric set to view " + opts.alias); } + } for (let i = 0; i < opts.dimensions.length; ++i) { @@ -188,7 +205,6 @@ export class ConfigParser { throw new Error("[Parsing error] Non exist dimension set to view " + opts.alias); } } - for (let i = 0; i < keys.length; ++i) { if (dimMap[keys[i]]) { viewOpt.keys.push(dimMap[opts.keys[i]]); @@ -249,6 +265,14 @@ export class ConfigParser { }; } + // private static parseSourcOpts (opts: SourceOptions): SourceOptions { + // return { + // name: opts.name, + // description: opts.description, + // fields: opts.fields + // }; + // } + private static parseClause (opts: string, metMap: MetricMap, dimMap: DimensionMap): Clause { const strFilters = opts.split(","); const filters: Filter[] = strFilters.map((item) => {