From bc500ad165551f7e5d3569c13ab02d6c3098ebc4 Mon Sep 17 00:00:00 2001
From: Lucas Fernandes de Oliveira <lfoliveira@inf.ufpr.br>
Date: Mon, 1 Jul 2019 10:29:17 -0300
Subject: [PATCH] Issue #106: Add data return as csv in several routes
Signed-off-by: Lucas Fernandes de Oliveira <lfoliveira@inf.ufpr.br>
---
package.json | 1 +
specs/blendb-api-v1.raml | 18 +++++++++++-
src/api/controllers/data.ts | 27 +++++++++++++++++-
src/api/controllers/engine.ts | 48 ++++++++++++++++++++++++++++---
src/api/middlewares/csv.ts | 54 +++++++++++++++++++++++++++++++++++
src/api/types.ts | 2 ++
src/main.ts | 4 ++-
7 files changed, 147 insertions(+), 7 deletions(-)
create mode 100644 src/api/middlewares/csv.ts
diff --git a/package.json b/package.json
index 06d65825..f1b829d7 100644
--- a/package.json
+++ b/package.json
@@ -29,6 +29,7 @@
"async": "=2.4.1",
"express": "^4.0.33",
"js-yaml": "^3.8.2",
+ "json-2-csv": "^3.5.5",
"monetdb": "^1.1.4",
"osprey": "^0.3.2",
"pg": "^6.1.5",
diff --git a/specs/blendb-api-v1.raml b/specs/blendb-api-v1.raml
index e5c69cb5..4de37301 100644
--- a/specs/blendb-api-v1.raml
+++ b/specs/blendb-api-v1.raml
@@ -271,6 +271,18 @@ traits:
description: |
Fields to be returned.
type: string
+ - formatable:
+ queryParameters:
+ format:
+ description: |
+ Response format. Defines if the response objects will be a
+ json or a csv-like file. The default value is json.
+ The csv-like formats are: csv, ssv and tsv which the
+ separator is comma, semi-colon and tab respectively.
+ example: "ssv+"
+ required: false
+ pattern: "^json$|^csv$|^ssv$|^tsv$"
+ type: string
/metrics:
description: |
@@ -279,6 +291,7 @@ traits:
system and their descriptions.
securedBy: [ null, oauth_2_0 ]
get:
+ is: [ formatable ]
/sources:
description: |
A Source represents a type of object that can be inserted in the database.
@@ -286,6 +299,7 @@ traits:
system and their descriptions
securedBy: [ null, oauth_2_0 ]
get:
+ is: [ formatable ]
/dimensions:
description: |
@@ -294,12 +308,14 @@ traits:
the system and their descriptions.
securedBy: [ null, oauth_2_0 ]
get:
+ is: [ formatable ]
/enumtypes:
description: |
A EnumType is short for enumerable type. This is a special data type that only accepts a few possible values. This
collection allows the user to list all the enumerable types available in the system, their descriptions and possible
values.
get:
+ is: [ formatable ]
securedBy: [ null, oauth_2_0 ]
/data:
description: |
@@ -309,7 +325,7 @@ traits:
start/end dates to refine your query.
type: base
get:
- is: [ filtered ]
+ is: [ filtered, formatable ]
queryParameters:
metrics:
description: |
diff --git a/src/api/controllers/data.ts b/src/api/controllers/data.ts
index a6e4b1f3..3cd155c3 100644
--- a/src/api/controllers/data.ts
+++ b/src/api/controllers/data.ts
@@ -45,6 +45,12 @@ export class DataCtrl {
if (req.query.sort) {
sort = req.query.sort.split(",").filter((item: string) => item !== "");
}
+
+ let format = "json";
+ if (req.query.format) {
+ format = req.query.format;
+ }
+
let view;
try {
@@ -102,7 +108,26 @@ export class DataCtrl {
return;
}
- res.status(200).json(result);
+ if (format === "json") {
+ res.status(200).json(result);
+ }
+
+ else {
+ req.csvParser(result, format, (error: Error, csv: string) => {
+ if (error) {
+ res.status(500).json({
+ message: "Error generating csv file. " +
+ "Try json format.",
+ error: error
+ });
+ return;
+ }
+
+ res.setHeader("Content-Type", "text/csv");
+ res.setHeader("Content-disposition", "attachment;filename=data.csv");
+ res.status(200).send(csv);
+ });
+ }
return;
});
}
diff --git a/src/api/controllers/engine.ts b/src/api/controllers/engine.ts
index 4fde6269..802acc1a 100644
--- a/src/api/controllers/engine.ts
+++ b/src/api/controllers/engine.ts
@@ -27,6 +27,42 @@ import { Request } from "../types";
* engine object that API users can use to create queries.
*/
export class EngineCtrl {
+ /**
+ * Auxiliary function that returns engine information.
+ * @param list - List of objects to return
+ * @param req - Object with request information
+ * @param res - Object used to create and send the response
+ * @param next - Call next middleware or controller. Not used but required
+ * by typescript definition of route.
+ */
+ private static respondList(list: any[], fileName: string, req: Request, res: express.Response, next: express.NextFunction) {
+ let format = "json";
+ if (req.query.format) {
+ format = req.query.format;
+ }
+
+ if (format === "json") {
+ res.status(200).json(list);
+ }
+
+ else {
+ req.csvParser(list, format, (error: Error, csv: string) => {
+ if (error) {
+ res.status(500).json({
+ message: "Error generating csv file. " +
+ "Try json format.",
+ error: error
+ });
+ return;
+ }
+
+ const disposition = "attachment;filename=" + fileName + ".csv";
+ res.setHeader("Content-Type", "text/csv");
+ res.setHeader("Content-disposition", disposition);
+ res.status(200).send(csv);
+ });
+ }
+ }
/**
* Route that returns the list of available metrics.
* @param req - Object with request information
@@ -35,7 +71,8 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static metrics(req: Request, res: express.Response, next: express.NextFunction) {
- res.status(200).json(req.engine.getMetricsDescription());
+ const metrics = req.engine.getMetricsDescription();
+ EngineCtrl.respondList(metrics, "metrics", req, res, next);
}
/**
@@ -46,7 +83,8 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static dimensions(req: Request, res: express.Response, next: express.NextFunction) {
- res.status(200).json(req.engine.getDimensionsDescription());
+ const dimensions = req.engine.getDimensionsDescription();
+ EngineCtrl.respondList(dimensions, "dimensions", req, res, next);
}
/**
@@ -57,7 +95,8 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static enumTypes(req: Request, res: express.Response, next: express.NextFunction) {
- res.status(200).json(req.engine.getEnumTypesDescription());
+ const enumTypes = req.engine.getEnumTypesDescription();
+ EngineCtrl.respondList(enumTypes, "enums", req, res, next);
}
/**
@@ -68,6 +107,7 @@ export class EngineCtrl {
* by typescript definition of route.
*/
public static sources(req: Request, res: express.Response, next: express.NextFunction) {
- res.status(200).json(req.engine.getSourcesDescription());
+ const sources = req.engine.getSourcesDescription();
+ EngineCtrl.respondList(sources, "sources", req, res, next);
}
}
diff --git a/src/api/middlewares/csv.ts b/src/api/middlewares/csv.ts
new file mode 100644
index 00000000..42540aae
--- /dev/null
+++ b/src/api/middlewares/csv.ts
@@ -0,0 +1,54 @@
+/*
+ * Copyright (C) 2015-2019 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import * as json2csv from "json-2-csv";
+import { Middleware } from "../types";
+
+/**
+ * Creates a csv parser and middleaew that
+ * inserts the parser into the request objects.
+ */
+export function CsvMw(): Middleware {
+ return function csvMiddleware(req, res, next) {
+ req.csvParser = function parseCsv(json: any, format: string, cb) {
+ const separator = format.substring(0, 1);
+
+ let sep = ",";
+ if (separator === "s") {
+ sep = ";";
+ }
+
+ else if (separator === "t"){
+ sep = "\t";
+ }
+
+ json2csv.json2csv(json, cb, {
+ delimiter: {
+ field: sep
+ , wrap: "\""
+ , eol: "\n"
+ }
+ , prependHeader: true
+ });
+ };
+ next();
+ };
+
+}
diff --git a/src/api/types.ts b/src/api/types.ts
index 0fe099f9..469b6adc 100644
--- a/src/api/types.ts
+++ b/src/api/types.ts
@@ -35,6 +35,8 @@ export interface Request extends express.Request {
engine: Engine;
/** A adapter object. Used to communicate with the database in use. */
adapter: Adapter;
+ /** A csvParser function. Used to parse json object into csv file. */
+ csvParser: (json: any, format: string, cb: (err: Error, csv?: string));
}
/**
diff --git a/src/main.ts b/src/main.ts
index 0710b3cc..4eb837dd 100755
--- a/src/main.ts
+++ b/src/main.ts
@@ -40,7 +40,7 @@ import { ConfigParser } from "./util/configParser";
let configPath;
/** @hidden */
-if(process.env.BLENDB_SCHEMA_FILE){
+if (process.env.BLENDB_SCHEMA_FILE) {
configPath = process.env.BLENDB_SCHEMA_FILE;
}
else{
@@ -54,8 +54,10 @@ const config = ConfigParser.parse(configPath);
import { EngineMw } from "./api/middlewares/engine";
import { PostgresMw, MonetMw } from "./api/middlewares/adapter";
import { ErrorMw } from "./api/middlewares/error";
+import { CsvMw } from "./api/middlewares/csv";
app.use(EngineMw(config));
+app.use(CsvMw());
if (config.adapters[0] === "postgres") {
app.use(PostgresMw(config.connections[0]));
}
--
GitLab