From 1c87e68497c9ba8922876b15d7c9a502d1e5ca4e Mon Sep 17 00:00:00 2001 From: Lucas Fernandes de Oliveira <lfoliveira@inf.ufpr.br> Date: Fri, 26 Jul 2019 11:47:04 -0300 Subject: [PATCH] Issue #107: Make metrics and dimensions routes filterable by tags Signed-off-by: Lucas Fernandes de Oliveira <lfoliveira@inf.ufpr.br> --- config/ci_test.yaml.example | 12 +++++ config/config.yaml.example | 3 ++ config/market_dimensions.yaml.example | 4 -- config/market_metrics.yaml.example | 36 +++++++++++-- config/market_tags.yaml.example | 21 ++++++++ specs/blendb-api-v1.raml | 27 ++++++++-- src/api/controllers/engine.spec.ts | 37 ++++++++++++- src/api/controllers/engine.ts | 17 +++++- src/api/router-v1.ts | 1 + src/common/tag.ts | 63 ++++++++++++++++++++++ src/core/dimension.ts | 23 +++++--- src/core/engine.ts | 66 ++++++++++++++++++++--- src/core/metric.ts | 11 +++- src/util/configParser.spec.ts | 53 ++++++++++++++++-- src/util/configParser.ts | 78 ++++++++++++++++++++++----- test/files/metrics.csv | 34 ++++++------ 16 files changed, 421 insertions(+), 65 deletions(-) create mode 100644 config/market_tags.yaml.example create mode 100644 src/common/tag.ts diff --git a/config/ci_test.yaml.example b/config/ci_test.yaml.example index 595347df..4736b4d2 100644 --- a/config/ci_test.yaml.example +++ b/config/ci_test.yaml.example @@ -1,6 +1,13 @@ # gitignore ignores files.yaml in this folder # however a config file for tests in CI is required # so this example file in fact is the CI test file +tags: + links: + - config/market_tags.yaml.example + obj: + - + name: "noDescription" + description: "Related with seller" views: links: - config/market_views.yaml.example @@ -30,6 +37,9 @@ metrics: dataType: "float" aggregation: "avg" description: "The seller average age" + tags: + - "seller" + - "age" dimensions: links: - config/market_dimensions.yaml.example @@ -38,6 +48,8 @@ dimensions: name: "dim:seller:name" dataType: "string" description: "Name of the seller from market" + tags: + - "seller" enumTypes: links: - config/market_enum.yaml.example diff --git a/config/config.yaml.example b/config/config.yaml.example index 11c74511..d4ff1d1a 100644 --- a/config/config.yaml.example +++ b/config/config.yaml.example @@ -1,6 +1,9 @@ # gitignore ignores files.yaml in this folder # however a config file for tests in CI is required # so this example file in fact is the CI test file +tags: + links: [] + obj: [] views: links: - config/market_views.yaml.example diff --git a/config/market_dimensions.yaml.example b/config/market_dimensions.yaml.example index 8cd85d5d..3e85343d 100644 --- a/config/market_dimensions.yaml.example +++ b/config/market_dimensions.yaml.example @@ -1,7 +1,3 @@ -- - name: "dim:seller:name" - dataType: "string" - description: "The name of the seller from market" - name: "dim:seller:sex" dataType: "enumtype" diff --git a/config/market_metrics.yaml.example b/config/market_metrics.yaml.example index 4853839c..320defff 100644 --- a/config/market_metrics.yaml.example +++ b/config/market_metrics.yaml.example @@ -1,53 +1,72 @@ -- - name: "met:seller:avg:age" - dataType: "float" - aggregation: "avg" - description: "The seller average age" - name: "met:seller:max:age" dataType: "integer" aggregation: "max" description: "The seller highest age" + tags: + - "seller" + - "age" + - "max" - name: "met:seller:min:age" dataType: "integer" aggregation: "min" description: "The seller lowest age" + tags: + - "seller" + - "age" - name: "met:seller:count:age" dataType: "integer" aggregation: "count" description: "The number of seller's" + tags: + - "seller" + - "age" - name: "met:product:avg:pricein" dataType: "float" aggregation: "avg" description: "The average product pricein" + tags: + - "product" - name: "met:product:max:pricein" dataType: "float" aggregation: "max" description: "The highest product pricein" + tags: + - "product" + - "max" - name: "met:product:min:pricein" dataType: "float" aggregation: "min" description: "The lowest product pricein" + tags: + - "product" - name: "met:product:avg:priceout" dataType: "float" aggregation: "avg" description: "The average product priceout" + tags: + - "product" - name: "met:product:max:priceout" dataType: "float" aggregation: "max" description: "The highest product priceout" + tags: + - "product" + - "max" - name: "met:product:min:priceout" dataType: "float" aggregation: "min" description: "The lowest product priceout" + tags: + - "product" - name: "met:sell:sum:quantity" dataType: "integer" @@ -68,13 +87,20 @@ dataType: "float" aggregation: "avg" description: "The average of quantity bought" + tags: + - "buyout" - name: "met:buyout:max:quantity" dataType: "integer" aggregation: "max" description: "The highest quantity bought" + tags: + - "buyout" + - "max" - name: "met:buyout:min:quantity" dataType: "integer" aggregation: "min" description: "The lowest quantity bought" + tags: + - "buyout" diff --git a/config/market_tags.yaml.example b/config/market_tags.yaml.example new file mode 100644 index 00000000..48fc078e --- /dev/null +++ b/config/market_tags.yaml.example @@ -0,0 +1,21 @@ +- + name: "seller" + description: "Related with seller" +- + name: "age" + description: "Related with age" +- + name: "product" + description: "Related with product" +- + name: "client" + description: "Related with client" +- + name: "buyout" + description: "Related with buyout" +- + name: "provider" + description: "Related with provider" +- + name: "max" + description: "Aggregation Max" diff --git a/specs/blendb-api-v1.raml b/specs/blendb-api-v1.raml index 4de37301..eac48517 100644 --- a/specs/blendb-api-v1.raml +++ b/specs/blendb-api-v1.raml @@ -283,6 +283,15 @@ traits: required: false pattern: "^json$|^csv$|^ssv$|^tsv$" type: string + - taggable: + queryParameters: + tags: + description: | + Tags that restrict the elements returned for your request. + Similar to a filter, but used in Blendb elements, not in + query results. + required: false + type: string /metrics: description: | @@ -291,13 +300,13 @@ traits: system and their descriptions. securedBy: [ null, oauth_2_0 ] get: - is: [ formatable ] + is: [ formatable, taggable ] /sources: description: | A Source represents a type of object that can be inserted in the database. This collection allows the user to list all the sources available in the system and their descriptions - securedBy: [ null, oauth_2_0 ] + securedBy: [ null, oauth_2_0 ] get: is: [ formatable ] @@ -308,15 +317,23 @@ traits: the system and their descriptions. securedBy: [ null, oauth_2_0 ] get: - is: [ formatable ] + is: [ formatable, taggable ] /enumtypes: description: | A EnumType is short for enumerable type. This is a special data type that only accepts a few possible values. This collection allows the user to list all the enumerable types available in the system, their descriptions and possible values. - get: + get: + is: [ formatable ] + securedBy: [ null, oauth_2_0 ] +/tags: + description: | + A Tag can be placed in a metric or dimension to add some extra meaning + to it. Tags can be used to filter the amount of elements returned by a + route. Tags are like filters, but instead of filtering query results, + filter blendb elements. + get: is: [ formatable ] - securedBy: [ null, oauth_2_0 ] /data: description: | This is the main part of the API. You may query it for report diff --git a/src/api/controllers/engine.spec.ts b/src/api/controllers/engine.spec.ts index 7290c349..ee086ea2 100644 --- a/src/api/controllers/engine.spec.ts +++ b/src/api/controllers/engine.spec.ts @@ -70,7 +70,18 @@ describe("API engine controller", () => { .end(done); }); - it("should respond 200 and the list of metrics (in csv)", (done) => { + it("should respond 200 and the list of tags", (done) => { + request(server) + .get("/v1/tags") + .expect((res: any) => { + let result = res.body; + expect(result).to.be.an("array"); + expect(result).to.have.length(8); + }) + .end(done); + }); + + it("should respond 200 and the list of metrics (in ssv)", (done) => { waterfall([(cb: (err: Error, data: string) => void) => { fs.readFile("test/files/metrics.csv", "utf8", (err, data) => { cb(err, data); @@ -91,4 +102,28 @@ describe("API engine controller", () => { }); }); + it("should respond 200 and the filtered list of metrics", (done) => { + request(server) + .get("/v1/metrics") + .query({tags: "seller,buyout"}) + .expect((res: any) => { + let result = res.body; + expect(result).to.be.an("array"); + expect(result).to.have.length(7); + }) + .end(done); + }); + + it("should respond 200 and the filtered list of dimensions", (done) => { + request(server) + .get("/v1/dimensions") + .query({tags: "seller"}) + .expect((res: any) => { + let result = res.body; + expect(result).to.be.an("array"); + expect(result).to.have.length(1); + }) + .end(done); + }); + }); diff --git a/src/api/controllers/engine.ts b/src/api/controllers/engine.ts index 802acc1a..98594de1 100644 --- a/src/api/controllers/engine.ts +++ b/src/api/controllers/engine.ts @@ -63,6 +63,7 @@ export class EngineCtrl { }); } } + /** * Route that returns the list of available metrics. * @param req - Object with request information @@ -71,7 +72,7 @@ export class EngineCtrl { * by typescript definition of route. */ public static metrics(req: Request, res: express.Response, next: express.NextFunction) { - const metrics = req.engine.getMetricsDescription(); + const metrics = req.engine.getMetricsDescription(req.query.tags); EngineCtrl.respondList(metrics, "metrics", req, res, next); } @@ -83,7 +84,7 @@ export class EngineCtrl { * by typescript definition of route. */ public static dimensions(req: Request, res: express.Response, next: express.NextFunction) { - const dimensions = req.engine.getDimensionsDescription(); + const dimensions = req.engine.getDimensionsDescription(req.query.tags); EngineCtrl.respondList(dimensions, "dimensions", req, res, next); } @@ -110,4 +111,16 @@ export class EngineCtrl { const sources = req.engine.getSourcesDescription(); EngineCtrl.respondList(sources, "sources", req, res, next); } + + /** + * Route that returns the list of available tags. + * @param req - Object with request information + * @param res - Object used to create and send the response + * @param next - Call next middleware or controller. Not used but required + * by typescript definition of route. + */ + public static tags(req: Request, res: express.Response, next: express.NextFunction) { + const tags = req.engine.getTagsDescription(); + EngineCtrl.respondList(tags, "tags", req, res, next); + } } diff --git a/src/api/router-v1.ts b/src/api/router-v1.ts index ca23e6db..e80dea65 100644 --- a/src/api/router-v1.ts +++ b/src/api/router-v1.ts @@ -33,5 +33,6 @@ router.get("/metrics", EngineCtrl.metrics); router.get("/sources", EngineCtrl.sources); router.get("/dimensions", EngineCtrl.dimensions); router.get("/enumtypes", EngineCtrl.enumTypes); +router.get("/tags", EngineCtrl.tags); router.get("/data", DataCtrl.read); router.post("/collect/{class}", CollectCtrl.write); diff --git a/src/common/tag.ts b/src/common/tag.ts new file mode 100644 index 00000000..1b0b3fdf --- /dev/null +++ b/src/common/tag.ts @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2019 Centro de Computacao Cientifica e Software Livre + * Departamento de Informatica - Universidade Federal do Parana + * + * This file is part of blendb. + * + * blend is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * blendb is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with blendb. If not, see <http://www.gnu.org/licenses/>. + */ + +/** + * Parameters used to create a Tag object. + * Parameters used to define tag object in the configuration file. + * Also the string description of a tag. + */ +export interface TagOptions { + /** Tag name. */ + name: string; + /** Breif description of what this tag represents. */ + description?: string; +} + +/** + * A Tag can be attached to other elements (such as metrics and dimensions) + * to create groups of similar elements. A tag can also be used to filter + * these elements, search for itens that only contain the given tags. + */ +export class Tag { + /** Tag name. */ + public readonly name: string; + /** Breif description of what this tag represents. */ + public readonly description: string; + + /** + * Creates a tag. + * @param options - Parameters required to create a tag. + */ + constructor(options: TagOptions) { + this.name = options.name; + this.description = (options.description) ? options.description : ""; + } + + /** + * Creates a object with the same options used to create this + * tag as strings. Used to inform the API users. + */ + public strOptions(): TagOptions { + return { + name: this.name + , description: this.description + }; + } +} diff --git a/src/core/dimension.ts b/src/core/dimension.ts index 24b5ec16..af69a595 100644 --- a/src/core/dimension.ts +++ b/src/core/dimension.ts @@ -20,6 +20,7 @@ import { RelationType, DataType } from "../common/types"; import { EnumHandler } from "../util/enumHandler"; +import { Tag } from "../common/tag"; /** Parameters used to create a Dimension object. */ export interface DimensionOptions { @@ -33,8 +34,10 @@ export interface DimensionOptions { relation?: RelationType; /** Breif description of what this dimension represents. */ description?: string; - /* Enumerable type name, used if data type is enumerable type. */ + /** Enumerable type name, used if data type is enumerable type. */ enumType?: string; + /** List of tags, a complement to attribute description. */ + tags?: Tag[]; } /** @@ -54,6 +57,8 @@ export interface DimensionStrOptions { description?: string; /** Dimension enum type */ enumType?: string; + /** List of tag names, a complement to attribute description. */ + tags?: string[]; } /** @@ -74,8 +79,10 @@ export class Dimension { public readonly relation: RelationType; /** Breif description of what this dimension represents. */ public readonly description: string; - /* Enumerable type name, used if data type is enumerable type. */ + /** Enumerable type name, used if data type is enumerable type. */ public readonly enumType: string; + /** List of tags, a complement to attribute description. */ + public readonly tags: Tag[]; /** * Creates a dimension. @@ -88,6 +95,7 @@ export class Dimension { this.parent = (options.parent) ? options.parent : null; this.description = (options.description) ? options.description : ""; this.enumType = (options.enumType) ? options.enumType : ""; + this.tags = (options.tags) ? options.tags.map((i) => i) : []; } /** @@ -98,16 +106,19 @@ export class Dimension { let o: DimensionStrOptions = { name: this.name, dataType: EnumHandler.stringfyDataType(this.dataType), - description: this.description - + description: this.description, + tags: this.tags.map ((i) => i.name) }; - if (this.relation !== RelationType.NONE){ + + if (this.relation !== RelationType.NONE) { o.relation = EnumHandler.stringifyRelationType(this.relation); o.parent = this.parent.name; } - if (this.dataType === DataType.ENUMTYPE){ + + if (this.dataType === DataType.ENUMTYPE) { o.enumType = this.enumType; } + return o; } } diff --git a/src/core/engine.ts b/src/core/engine.ts index 799b736e..74ca4770 100644 --- a/src/core/engine.ts +++ b/src/core/engine.ts @@ -20,6 +20,7 @@ import { Dimension, DimensionStrOptions } from "./dimension"; import { Metric, MetricStrOptions } from "./metric"; +import { Tag, TagOptions } from "../common/tag"; import { Clause } from "./clause"; import { Filter } from "./filter"; import { View } from "./view"; @@ -49,6 +50,8 @@ export class Engine { private dimensions: Dimension[]; /** Set of sources available in the database. */ private sources: Source[]; + /** Set of tags available in the database. */ + private tags: Tag[]; /** Graph which represents the database schema. */ private graph: Graph; @@ -62,12 +65,14 @@ export class Engine { this.metrics = []; this.dimensions = []; this.sources = []; + this.tags = []; config.metrics.forEach ((met) => this.addMetric(met)); config.dimensions.forEach ((dim) => this.addDimension(dim)); config.views.forEach ((view) => this.addView(view)); config.enumTypes.forEach ((enumt) => this.addEnumType(enumt)); config.sources.forEach ((sourc) => this.addSource(sourc)); + config.tags.forEach ((tag) => this.addTag(tag)); } @@ -76,9 +81,26 @@ export class Engine { return this.views; } - /** Gets a string description for all the available metrics. */ - public getMetricsDescription(): MetricStrOptions[] { - return this.metrics.map((i) => i.strOptions()); + /** + * Gets a string description for all the available metrics. + * @param expression - Expression to be used in the tags + */ + public getMetricsDescription(expression: string): MetricStrOptions[] { + let list = this.metrics.map((i) => i.strOptions()); + if (!expression) { + return list; + } + + const clauses = expression.split(";").filter((item: string) => item !== ""); + + for (let i = 0; i < clauses.length; ++i) { + const tags = clauses[i].split(",").filter((item: string) => item !== ""); + list = list.filter((item) => { + return item.tags.some((o) => tags.some((t) => t === o)); + }); + } + + return list; } /** Gets a string description for all the available enumerable types. */ @@ -91,9 +113,31 @@ export class Engine { return this.sources.map((i) => i.strOptions()); } - /** Gets a string description for all the available dimensions. */ - public getDimensionsDescription(): DimensionStrOptions[] { - return this.dimensions.map((i) => i.strOptions()); + /** + * Gets a string description for all the available dimensions. + * @param expression - Expression to be used in the tags + */ + public getDimensionsDescription(expression: string): DimensionStrOptions[] { + let list = this.dimensions.map((i) => i.strOptions()); + if (!expression) { + return list; + } + + const clauses = expression.split(";").filter((item: string) => item !== ""); + + for (let i = 0; i < clauses.length; ++i) { + const tags = clauses[i].split(",").filter((item: string) => item !== ""); + list = list.filter((item) => { + return item.tags.some((o) => tags.some((t) => t === o)); + }); + } + + return list; + } + + /** Gets a string description for all the available tags. */ + public getTagsDescription(): TagOptions[] { + return this.tags.map((i) => i.strOptions()); } /** @@ -118,6 +162,15 @@ export class Engine { return enumType; } + /** + * Adds a new tag to the database schema (engine). + * @param enumType - Enumerable type to be added. + */ + public addTag(tag: Tag): Tag { + this.tags.push(tag); + return tag; + } + /** * Adds a new source to the database schema (engine). * @param source - Source to be added. @@ -340,4 +393,5 @@ export class Engine { return noRepeat; } + } diff --git a/src/core/metric.ts b/src/core/metric.ts index 9b76b102..3802f1f4 100644 --- a/src/core/metric.ts +++ b/src/core/metric.ts @@ -20,6 +20,7 @@ import { AggregationType, DataType } from "../common/types"; import { EnumHandler } from "../util/enumHandler"; +import { Tag } from "../common/tag"; /** Parameters used to create a metric object. */ export interface MetricOptions { @@ -31,6 +32,8 @@ export interface MetricOptions { dataType: DataType; /** Breif description of what this metric represents. */ description?: string; + /** List of tags, a complement to attribute description. */ + tags?: Tag[]; } /** @@ -46,6 +49,8 @@ export interface MetricStrOptions { dataType: string; /** Breif description of what this metric represents. */ description?: string; + /** List of tag names, a complement to attribute description. */ + tags?: string[]; } /** @@ -64,6 +69,8 @@ export class Metric { public readonly dataType: DataType; /** Breif description of what this metric represents. */ public readonly description: string; + /** List of tags, a complement to attribute description. */ + public readonly tags: Tag[]; /** * Create a metric. @@ -74,6 +81,7 @@ export class Metric { this.aggregation = options.aggregation; this.dataType = options.dataType; this.description = (options.description) ? options.description : ""; + this.tags = (options.tags) ? options.tags.map((i) => i) : []; } /** @@ -85,7 +93,8 @@ export class Metric { name: this.name, aggregation: EnumHandler.stringifyAggrType(this.aggregation), dataType: EnumHandler.stringfyDataType(this.dataType), - description: this.description + description: this.description, + tags: this.tags.map ((i) => i.name) }; } } diff --git a/src/util/configParser.spec.ts b/src/util/configParser.spec.ts index f31d5610..b603e49e 100644 --- a/src/util/configParser.spec.ts +++ b/src/util/configParser.spec.ts @@ -95,7 +95,7 @@ describe("configParser utility library", () => { let error: boolean = false; try { - ConfigParser.parseDimOpts(opts, dims, null); + ConfigParser.parseDimOpts(opts, dims, null, null); } catch (e) { error = true; @@ -196,7 +196,7 @@ describe("configParser utility library", () => { ]; for (let i = 0; i < opts.length; ++i) { - let parsed = ConfigParser.parseDimOpts(opts[i], dims, null); + let parsed = ConfigParser.parseDimOpts(opts[i], dims, null, null); expect(parsed.name).to.be.equal(opts[i].name); expect(EnumHandler.stringfyDataType(parsed.dataType)).to.be.equal(opts[i].dataType); expect(parsed.parent).to.be.equal(dims[1]); @@ -219,7 +219,7 @@ describe("configParser utility library", () => { let enumMap: {[key: string]: EnumType} = { "enumtype:5" : new EnumType({name: "enumtype:5", values: ["nope", "test"]}) }; - let parsed = ConfigParser.parseDimOpts(opts, dims, enumMap); + let parsed = ConfigParser.parseDimOpts(opts, dims, enumMap, null); expect(parsed.enumType).to.be.equal(enumMap["enumtype:5"].name); }); @@ -240,7 +240,7 @@ describe("configParser utility library", () => { }; let error: boolean = false; try { - ConfigParser.parseDimOpts(opts, dims, enumMap); + ConfigParser.parseDimOpts(opts, dims, enumMap, null); } catch (e) { error = true; @@ -261,7 +261,7 @@ describe("configParser utility library", () => { }; let error: boolean = false; try { - ConfigParser.parseMetOpts(met); + ConfigParser.parseMetOpts(met, null); } catch (e) { error = true; @@ -301,4 +301,47 @@ describe("configParser utility library", () => { expect(error).to.be.true; }); + it("should throw expection for inexistent tag in a Dimension", () => { + let opts: DimensionStrOptions = { + name: "dim:0", + dataType: "integer", + tags: ["none"] + }; + + let dims: Dimension[] = []; + + let error: boolean = false; + try { + ConfigParser.parseDimOpts(opts, dims, null, {}); + } + catch (e) { + error = true; + expect(e.message).to.be + .equal("[Parsing error] Tag: 'none' used in dimension: '" + opts.name + "' was not defined. Check tag spelling and configuration files."); + } + + expect(error).to.be.true; + }); + + it("should throw expection for inexistent tag in a Metric", () => { + let opts: MetricStrOptions = { + name: "met:0", + dataType: "integer", + aggregation: "avg", + tags: ["none"] + }; + + let error: boolean = false; + try { + ConfigParser.parseMetOpts(opts, {}); + } + catch (e) { + error = true; + expect(e.message).to.be + .equal("[Parsing error] Tag: 'none' used in metric: '" + opts.name + "' was not defined. Check tag spelling and configuration files."); + } + + expect(error).to.be.true; + }); + }); diff --git a/src/util/configParser.ts b/src/util/configParser.ts index 73e6d5b7..e2c8a5c4 100644 --- a/src/util/configParser.ts +++ b/src/util/configParser.ts @@ -24,6 +24,7 @@ import { View, ViewOptions, LoadView } from "../core/view"; import { EnumType, EnumTypeOptions } from "../core/enumType"; import { RelationType, DataType } from "../common/types"; import { Opcode } from "../common/expression"; +import { Tag, TagOptions } from "../common/tag"; import { Filter } from "../core/filter"; import { Clause } from "../core/clause"; import { Source, SourceOptions, SourceStrOptions} from "../core/source"; @@ -50,7 +51,7 @@ export interface ViewParsingOptions { metrics: string[]; /** Set of (stringified) clauses applied to the view. */ clauses?: string[]; - /** Inform if the view's name will be it's alias or id */ + /** Inform if the view's name will be it's alias or id. */ aliasAsName?: boolean; } @@ -78,6 +79,10 @@ interface ConfigSchema { enumTypes: { obj: EnumTypeOptions[], links: string[], }; + /** Options of all tags types available */ + tags: { obj: TagOptions[], + links: string[], + }; } /** Information required to build a SQL view code. */ @@ -103,6 +108,8 @@ export interface ParsedConfig { metrics: Metric[]; /** Set of all enumerable types available. */ enumTypes: EnumType[]; + /** Set of all tags available. */ + tags: Tag[]; /** Set of all dimensions available. */ dimensions: Dimension[]; loadViews: LoadView[]; @@ -141,6 +148,11 @@ interface MetricMap { [key: string]: Metric; } +/** Dictonary indexed by tag name, that returns the tag object. */ +interface TagMap { + [key: string]: Tag; +} + /** * Dictonary indexed by enumerable type name, * that returns the enumerable type objetct. @@ -186,11 +198,12 @@ export class ConfigParser { }; } - let metricsOpts = config.metrics.obj; - let viewsOpts = config.views.obj; - let dimensionsOpts = config.dimensions.obj; - let enumTypesOpts = config.enumTypes.obj; - let sourcesOpts = config.sources.obj; + let metricsOpts = (config.metrics) ? config.metrics.obj : []; + let viewsOpts = (config.views) ? config.views.obj : []; + let dimensionsOpts = (config.dimensions) ? config.dimensions.obj : []; + let enumTypesOpts = (config.enumTypes) ? config.enumTypes.obj : []; + let sourcesOpts = (config.sources) ? config.sources.obj : []; + let tagOpts = (config.tags) ? config.tags.obj : []; let parsed: ParsedConfig = { adapters: adapters, connections: connections, @@ -201,6 +214,7 @@ export class ConfigParser { loadViews: [], buildViews: [], sources: [], + tags: [], ndb: ndb }; @@ -224,20 +238,30 @@ export class ConfigParser { enumTypesOpts = enumTypesOpts.concat(yaml.safeLoad(fs.readFileSync( config.enumTypes.links[i], { encoding: "utf-8"})) as EnumTypeOptions[]); } + for (let i = 0; i < config.tags.links.length; i++) { + tagOpts = tagOpts.concat(yaml.safeLoad(fs.readFileSync( + config.tags.links[i], { encoding: "utf-8"})) as TagOptions[]); + } let metMap: MetricMap = {}; let dimMap: DimensionMap = {}; let enumMap: EnumTypeMap = {}; let sourcMap: SourceMap = {}; let dimOptsMap: DimensionOptsMap = {}; + let tagMap: TagMap = {}; + for (let i = 0; i < tagOpts.length; i++) { + let tag = new Tag((tagOpts[i])); + parsed.tags.push(tag); + tagMap[tag.name] = tag; + } for (let i = 0; i < enumTypesOpts.length; i++) { let enumT = new EnumType((enumTypesOpts[i])); parsed.enumTypes.push(enumT); enumMap[enumT.name] = enumT; } for (let i = 0; i < metricsOpts.length; ++i) { - let met = new Metric(this.parseMetOpts(metricsOpts[i])); + let met = new Metric(this.parseMetOpts(metricsOpts[i], tagMap)); parsed.metrics.push(met); metMap[met.name] = met; } @@ -267,7 +291,7 @@ export class ConfigParser { }); for (let i = 0; i < dimensionsOpts.length; ++i) { - let dim = new Dimension(this.parseDimOpts(dimensionsOpts[i], parsed.dimensions, enumMap)); + let dim = new Dimension(this.parseDimOpts(dimensionsOpts[i], parsed.dimensions, enumMap, tagMap)); parsed.dimensions.push(dim); dimMap[dim.name] = dim; } @@ -364,8 +388,9 @@ export class ConfigParser { * @param opts - Dimension struct in configuration file. * @param dims - Parsed dimensions. Parent candidates. * @param map - Enumerable types available. + * @param tagMap - Tags available. */ - public static parseDimOpts (opts: DimensionStrOptions, dims: Dimension[], map: EnumTypeMap): DimensionOptions { + public static parseDimOpts (opts: DimensionStrOptions, dims: Dimension[], map: EnumTypeMap, tagMap: TagMap): DimensionOptions { let type = EnumHandler.parseDataType(opts.dataType); if (type === DataType.NONE) { throw new Error("[Parsing error] DataType: '" + opts.dataType + "' does not exist on Dimension"); @@ -375,6 +400,17 @@ export class ConfigParser { throw new Error("[Parsing error] EnumType: '" + opts.enumType + "' does not exist on Dimension"); } } + let tags: Tag[] = []; + + if (opts.tags) { + tags = opts.tags.map((i) => { + if (!(tagMap[i])) { + throw new Error("[Parsing error] Tag: '" + i + "' used in dimension: '" + opts.name + "' was not defined. Check tag spelling and configuration files."); + } + + return tagMap[i]; + }); + } if (opts.parent || opts.relation) { for (let i = 0; i < dims.length; ++i) { if (dims[i].name === opts.parent) { @@ -384,7 +420,8 @@ export class ConfigParser { description: opts.description, parent: dims[i], relation: EnumHandler.parseRelationType(opts.relation), - enumType: opts.enumType + enumType: opts.enumType, + tags: tags }; } } @@ -397,26 +434,41 @@ export class ConfigParser { description: opts.description, parent: null, relation: RelationType.NONE, - enumType: opts.enumType + enumType: opts.enumType, + tags: tags }; } /** * Parse a metric struct in configuration file. * @param opts - Metric struct in configuration file. + * @param tagMap - Tags available. */ - public static parseMetOpts (opts: MetricStrOptions): MetricOptions { + public static parseMetOpts (opts: MetricStrOptions, tagMap: TagMap): MetricOptions { let type = EnumHandler.parseDataType(opts.dataType); if (!(type === DataType.FLOAT || type === DataType.INTEGER)){ throw new Error("[Parsing error] DataType: '" + opts.dataType + "' does not exist on Metric"); } + + let tags: Tag[] = []; + + if (opts.tags) { + tags = opts.tags.map((i) => { + if (!(tagMap[i])) { + throw new Error("[Parsing error] Tag: '" + i + "' used in metric: '" + opts.name + "' was not defined. Check tag spelling and configuration files."); + } + + return tagMap[i]; + }); + } return { name: opts.name, aggregation: EnumHandler.parseAggrType(opts.aggregation), dataType : EnumHandler.parseDataType(opts.dataType), - description: opts.description + description: opts.description, + tags: tags }; } diff --git a/test/files/metrics.csv b/test/files/metrics.csv index 468d0684..321a243d 100644 --- a/test/files/metrics.csv +++ b/test/files/metrics.csv @@ -1,17 +1,17 @@ -name;aggregation;dataType;description -met:seller:avg:age;avg;float;The seller average age -met:seller:max:age;max;integer;The seller highest age -met:seller:min:age;min;integer;The seller lowest age -met:seller:count:age;count;integer;The number of seller's -met:product:avg:pricein;avg;float;The average product pricein -met:product:max:pricein;max;float;The highest product pricein -met:product:min:pricein;min;float;The lowest product pricein -met:product:avg:priceout;avg;float;The average product priceout -met:product:max:priceout;max;float;The highest product priceout -met:product:min:priceout;min;float;The lowest product priceout -met:sell:sum:quantity;sum;integer;The sum of sales quantity -met:sell:avg:quantity;avg;float;The average of sales quantity -met:sell:count:quantity;count;integer;The total number of sales -met:buyout:avg:quantity;avg;float;The average of quantity bought -met:buyout:max:quantity;max;integer;The highest quantity bought -met:buyout:min:quantity;min;integer;The lowest quantity bought \ No newline at end of file +name;aggregation;dataType;description;tags +met:seller:avg:age;avg;float;The seller average age;"[""seller"",""age""]" +met:seller:max:age;max;integer;The seller highest age;"[""seller"",""age"",""max""]" +met:seller:min:age;min;integer;The seller lowest age;"[""seller"",""age""]" +met:seller:count:age;count;integer;The number of seller's;"[""seller"",""age""]" +met:product:avg:pricein;avg;float;The average product pricein;"[""product""]" +met:product:max:pricein;max;float;The highest product pricein;"[""product"",""max""]" +met:product:min:pricein;min;float;The lowest product pricein;"[""product""]" +met:product:avg:priceout;avg;float;The average product priceout;"[""product""]" +met:product:max:priceout;max;float;The highest product priceout;"[""product"",""max""]" +met:product:min:priceout;min;float;The lowest product priceout;"[""product""]" +met:sell:sum:quantity;sum;integer;The sum of sales quantity;[] +met:sell:avg:quantity;avg;float;The average of sales quantity;[] +met:sell:count:quantity;count;integer;The total number of sales;[] +met:buyout:avg:quantity;avg;float;The average of quantity bought;"[""buyout""]" +met:buyout:max:quantity;max;integer;The highest quantity bought;"[""buyout"",""max""]" +met:buyout:min:quantity;min;integer;The lowest quantity bought;"[""buyout""]" \ No newline at end of file -- GitLab