Merge branch 'issue/108' into 'develop'

Issue #108: Add Log

See merge request !91

.gitignore

@@ -21,3 +21,4 @@
 /service
 schema.sql
 /doc/code
+.nyc_output/*

config/config.env.example

 BLENDB_SCHEMA_FILE=config/config.yaml.example
 PORT=3000
+BLENDB_LOG_FILE=/var/log/blendb.log
+BLENDB_LOG_LEVEL=debug
 BLENDB_N_DB=1
 BLENDB_DB0_USER=blendb
 BLENDB_DB0_NAME=blendb-test

config/test.env.example

 PORT=3000
 BLENDB_N_DB=2
+BLENDB_LOG_FILE=/var/log/blendb.log
+BLENDB_LOG_LEVEL=debug
 BLENDB_DB0_USER=runner
 BLENDB_DB0_NAME=blendb_fixture
 BLENDB_DB0_PASSWORD=

package.json

@@ -15,10 +15,20 @@
     "doc-code": "typedoc --mode 'file' --module 'commonjs' --target 'ES6' --ignoreCompilerErrors --exclude '**/*.spec.ts' --out 'doc/code' 'src'"
   },
   "nyc": {
-    "include": ["src/**/*.ts"],
-    "extension": [".ts"],
-    "require": ["ts-node/register"],
-    "reporter": ["text-summary", "html"],
+    "include": [
+      "src/**/*.ts"
+    ],
+    "extension": [
+      ".ts"
+    ],
+    "require": [
+      "ts-node/register"
+    ],
+    "reporter": [
+      "text-summary",
+      "text",
+      "lcov"
+    ],
     "sourceMap": true,
     "instrument": true
   },
@@ -38,6 +48,7 @@
     "express": "^4.17.1",
     "js-yaml": "^3.13.1",
     "json-2-csv": "^3.5.6",
+    "log4js": "^5.1.0",
     "monetdb": "^1.1.4",
     "osprey": "^0.3.2",
     "pg": "^7.12.1",
@@ -60,4 +71,5 @@
   "engines": {
     "node": "^10.16.3"
   }
+  
 }

scripts/service.sh

@@ -61,6 +61,9 @@ echo "To set different user and port use npm run service -- <port> [<user>]"
 
 echo "Run this commands, as root (or sudo) to finish the process and start blendb"
 SYSTEMD_PATH=/etc/systemd/system/blendb.service
+mkdir -p /var/log/
+touch /var/log/blendb.log
+chown root:$REAL_USER /var/log/blendb.log
 echo -n "rm -f $SYSTEMD_PATH && "
 echo -n "ln -s $WORKSPACE/service/blendb.service $SYSTEMD_PATH && "
 echo "systemctl daemon-reload && systemctl restart blendb.service"

src/api/controllers/collect.ts

@@ -133,9 +133,11 @@ export class CollectCtrl {
                 // true/false, it must guarantee that it isn't a boolean
                 // then it'll test if it's empty
                 if (!(typeof(data[i]) === "boolean") && !data[i]){
-                    throw new Error("[Collect error] '" + fields[i].name + 
-                    "' is mandatory, but no data was received. Review the data sent.");
+                    const message = "[Collect error] '" + fields[i].name + 
+                    "' is mandatory, but no data was received. Review the data sent."
+                    throw new Error(message);
                 }
+                req.log.debug("Sucessfuly accepted the data: " + data[i] + " from source: ",source.name);
             }
 
             for (let i = 0; i < fields.length; i++){
@@ -151,22 +153,27 @@ export class CollectCtrl {
                         }
                     }
                     if (!found) {
-                        throw new Error("[Collect error] EnumType: '" +  data[i] + "' from '" + fields[i].name +
-                            "' isn't allowed on " + fields[i].enumType + 
-                            ". Review configuration files.");
+                        const message = "[Collect error] EnumType: '" +  data[i] + "' from '" + fields[i].name +
+                        "' isn't allowed on " + fields[i].enumType + 
+                        ". Review configuration files."                        
+                        throw new Error(message);
                     }
                 }else if (!validador[EnumHandler.stringfyDataType(fields[i].dataType)](data[i]) === true){
-                        throw new Error("[Collect error] Datatype: '" +  data[i] + "' from '" + fields[i].name +
-                            "' could not be converted to type: " + [EnumHandler.stringfyDataType(fields[i].dataType)] +
-                            ". Review configuration files.");
+                        const message = "[Collect error] Datatype: '" +  data[i] + "' from '" + fields[i].name +
+                        "' could not be converted to type: " + [EnumHandler.stringfyDataType(fields[i].dataType)] +
+                        ". Review configuration files."
+                        throw new Error(message);
                 }
+                req.log.debug("Sucessfuly accepted the enumType data: " + data[i] + " from source: ",source.name);
             }
         }
 
         catch (e) {
+            const message = "Query execution failed: " +
+            "Could not construct query with the given parameters."
+            req.log.warn(message);
             res.status(500).json({
-                message: "Query execution failed: " +
-                "Could not construct query with the given parameters.",
+                message: message,
                 error: e.message
              });
             return;
@@ -174,14 +181,18 @@ export class CollectCtrl {
 
         req.adapter.insertIntoSource(source, req.body, (err: Error, result: any[]) => {
             if (err) {
+                const message = "Insertion has failed";
+                req.log.error(message,err);
                 res.status(500).json({
-                    message: "Insertion has failed",
+                    message: message,
                     error: err
                 });
                 return;
             }
             else{
-                res.status(200).json({message: "Data has been successfully received and stored by the server"});
+                const message = "Data has been successfully received and stored by the server";
+                req.log.info(message);
+                res.status(200).json({message: message});
                 return;
             }
 

src/api/controllers/data.ts

@@ -35,6 +35,7 @@ export class DataCtrl {
      * by typescript definition of route.
      */
     public static read(req: Request, res: express.Response, next: express.NextFunction) {
+        req.log.info("Query: ",req.query);
         let metrics = req.query.metrics.split(",").filter((item: string) => item !== "");
         let dimensions = req.query.dimensions.split(",").filter((item: string) => item !== "");
         let clauses = [];
@@ -45,7 +46,6 @@ export class DataCtrl {
         if (req.query.sort) {
             sort = req.query.sort.split(",").filter((item: string) => item !== "");
         }
-
         let format = "json";
         if (req.query.format) {
             format = req.query.format;
@@ -90,9 +90,11 @@ export class DataCtrl {
             view = req.engine.query(query);
         }
         catch (e) {
+            const message = "Query execution failed: " +
+            "Could not construct query with the given parameters."
+            req.log.warn(message,e);
             res.status(500).json({
-                message: "Query execution failed: " +
-                "Could not construct query with the given parameters.",
+                message: message,
                 error: e.message
              });
             return;
@@ -100,29 +102,35 @@ export class DataCtrl {
 
         req.adapter.getDataFromView(view, (err: Error, result: any[]) => {
             if (err) {
+                const message = "Query execution failed " +
+                "failed on execute query on database."
+                req.log.error(message,err);
                 res.status(500).json({
-                    message: "Query execution failed " +
-                    "failed on execute query on database.",
+                    message: message,
                     error: err
                  });
                 return;
             }
 
             if (format === "json") {
+                req.log.info("Response (json) send with success");
                 res.status(200).json(result);
             }
 
             else {
                 req.csvParser(result, format, (error: Error, csv: string) => {
                     if (error) {
+                        const message = "Error generating csv file. " +
+                        "Try json format."
+                        req.log.error(message,error);
                         res.status(500).json({
-                            message: "Error generating csv file. " +
-                            "Try json format.",
+                            message: message,
                             error: error
                         });
                         return;
                     }
 
+                    req.log.info("Response (csv) send with success");
                     res.setHeader("Content-Type", "text/csv");
                     res.setHeader("Content-disposition", "attachment;filename=data.csv");
                     res.status(200).send(csv);

src/api/controllers/engine.ts

@@ -42,12 +42,14 @@ export class EngineCtrl {
         }
 
         if (format === "json") {
+            req.log.info("Response (json) send with success");
             res.status(200).json(list);
         }
 
         else {
             req.csvParser(list, format, (error: Error, csv: string) => {
                 if (error) {
+                    req.log.error("Error generating csv file: ",error);
                     res.status(500).json({
                         message: "Error generating csv file. " +
                         "Try json format.",
@@ -57,6 +59,7 @@ export class EngineCtrl {
                 }
 
                 const disposition = "attachment;filename=" + fileName + ".csv";
+                req.log.info("Response (csv) send with success");
                 res.setHeader("Content-Type", "text/csv");
                 res.setHeader("Content-disposition", disposition);
                 res.status(200).send(csv);

src/api/middlewares/log.ts

+/*
+ * Copyright (C) 2019 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { Log } from "../../util/log";
+import { Middleware } from "../types";
+
+/**
+ * Creates a log and middleware that
+ * inserts the log into the request objects.
+ */
+export function LogMw (): Middleware {
+    let log: Log = new Log();
+
+    return function logMiddleware(req, res, next) {
+        req.log = log;
+        next();
+    };
+}

src/api/types.ts

@@ -21,6 +21,7 @@
 import * as express from "express";
 import { Engine } from "../core/engine";
 import { Adapter} from "../core/adapter";
+import { Log } from "../core/log";
 
 /**
  * Extension of Express requests that suports the addition
@@ -37,6 +38,8 @@ export interface Request extends express.Request {
     adapter: Adapter;
     /** A csvParser function. Used to parse json object into csv file. */
     csvParser: (json: any, format: string, cb: (err: Error, csv?: string));
+    /** A log object. Used store logs into file. */
+    log: Log;
 }
 
 /**

src/main.ts

@@ -55,9 +55,11 @@ import { EngineMw } from "./api/middlewares/engine";
 import { PostgresMw, MonetMw } from "./api/middlewares/adapter";
 import { ErrorMw } from "./api/middlewares/error";
 import { CsvMw } from "./api/middlewares/csv";
+import { LogMw } from "./api/middlewares/log"
 
 app.use(EngineMw(config));
 app.use(CsvMw());
+app.use(LogMw());
 if (config.adapters[0] === "postgres") {
     app.use(PostgresMw(config.connections[0]));
 }

src/util/log.spec.ts

+/*
+ * Copyright (C) 2019 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { expect } from "chai";
+import { Log } from "./log";
+
+describe("log class", () => {
+
+    it("should modifiy loglevel to error if wrong level", () => {
+        process.env.BLENDB_LOG_LEVEL = "test";
+        const logLevelWrong = new Log();
+        expect(logLevelWrong.getLogLevel()).to.be.equals("error");
+    });
+
+    it("should modifiy loglevel to error if empty level", () => {
+        process.env.BLENDB_LOG_LEVEL = "";
+        const logLevelEmpty = new Log();
+        expect(logLevelEmpty.getLogLevel()).to.be.equals("error");
+    });
+
+    it("should modifiy logFile to stderr if empty file", () => {
+        process.env.BLENDB_LOG_FILE = "";
+        const LogFileEmpty = new Log();
+        expect(LogFileEmpty.getLogFile()).to.be.equals("stderr");
+    });
+
+    it("should modifiy logFile to stderr if file does not exist", () => {
+        process.env.BLENDB_LOG_FILE = "norepo/test/shalnotexist";
+        const LogFileEmpty = new Log();
+        expect(LogFileEmpty.getLogFile()).to.be.equals("stderr");
+    });
+
+});

src/util/log.ts

+/*
+ * Copyright (C) 2019 Centro de Computacao Cientifica e Software Livre
+ * Departamento de Informatica - Universidade Federal do Parana
+ *
+ * This file is part of blendb.
+ *
+ * blendb is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * blendb is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with blendb.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import { configure, getLogger } from "log4js";
+import * as fs from "fs";
+/**
+ * Define the levels that BlenDB log must have
+ */
+interface LogInterface {
+    debug(msg: string, ...additionalInfo: any[]): void;
+    info(msg: string, ...additionalInfo: any[]): void;
+    warn(msg: string, ...additionalInfo: any[]): void;
+    error(msg: string, ...additionalInfo: any[]): void;
+}
+/**
+ * Logging for BlenDB using the framework log4js,
+ * using debug, info, warn and error as levels.
+ * Hierarchy of levels for showing messages are:
+ * debug > info > warn > error
+ */
+export class Log implements LogInterface {
+
+    private levels = ["debug" , "info" , "warn" , "error"];
+    private file: string;
+    private level: string;
+    private logger: any;
+    /**
+     * Definitions to use log4js, such as file destination,
+     *  level of log and configure log4js.
+     */
+    constructor(){
+        this.file = (process.env["BLENDB_LOG_FILE"]) ? process.env["BLENDB_LOG_FILE"] : "stderr";
+        this.level = (process.env["BLENDB_LOG_LEVEL"]) ? process.env["BLENDB_LOG_LEVEL"] : "error";
+
+        let find = this.levels.some( element => {
+            return element === this.level;
+        });
+        if (!find) {
+            this.level = "error";
+        }
+
+        if (!(this.file === "stderr")) {
+            try {
+                fs.appendFileSync(this.file, "");
+            }
+            catch (e) {
+                this.file = "stderr";
+            }
+        }
+
+        /**
+         * The reason for the duplication of configure,
+         * is because the filename flag from appenders it start
+         * the empty file if doesn't exist, even if it's not
+         * called from getLogger. Then it won't create an empty
+         * file called stderr.
+         */
+        if (this.file === "stderr") {
+            configure({
+                appenders: {    defaultLog: { type : "stderr" }
+                    },
+                categories: {
+                              default: { appenders: ["defaultLog"], level: this.level },
+                            }
+            });
+            this.logger = getLogger("defaultLog");
+        }else{
+            configure({
+                appenders: {    blendb: { type: "file", layout: {
+                                type: "pattern",
+                                pattern: "%d [%p] host: [%h] user: [%x{user}] %n %[%m%n%]",
+                                tokens: {
+                                    user: function(logEvent) {
+                                      return process.env.USER;
+                                    }
+                                }
+                        },
+                        filename: this.file }
+                    },
+                categories: {
+                              default: { appenders: ["blendb"], level: this.level }
+                            }
+            });
+            this.logger = getLogger("blendb");
+        }
+        this.logger.level = this.level;
+    }
+    /**
+     * Get log level
+     */
+    public getLogLevel(){
+        return this.level;
+    }
+    /**
+     * Get lof filename
+     */
+    public getLogFile(){
+        return this.file;
+    }
+    /**
+     * Log level debug
+     * @param msg description of log
+     * @param additionalInfo useful additional information
+     */
+    public debug(msg: string, ...additionalInfo: any[]){
+        this.emtigLogMessage("debug", msg, additionalInfo);
+    }
+    /**
+     * Log level info
+     * @param msg description of log
+     * @param additionalInfo useful additional information
+     */
+    public info(msg: string, ...additionalInfo: any[]){
+        this.emtigLogMessage("info", msg, additionalInfo);
+    }
+    /**
+     * Log level warning
+     * @param msg description of log
+     * @param additionalInfo useful additional information
+     */
+    public warn(msg: string, ...additionalInfo: any[]){
+        this.emtigLogMessage("warn", msg, additionalInfo);
+    }
+    /**
+     * Log level error
+     * @param msg description of log
+     * @param additionalInfo useful additional information
+     */
+    public error(msg: string, ...additionalInfo: any[]){
+        this.emtigLogMessage("error", msg, additionalInfo);
+    }
+    /**
+     * Handle data between log class and log4js framework
+     * @param msgType level of log being use
+     * @param msg description of log
+     * @param additionalInfo useful additional information
+     */
+    private emtigLogMessage(logLevel: "debug" | "info" | "warn" | "error" , msg: string, additionalInfo: any[]){
+        if (additionalInfo.length){
+            this.logger[logLevel](msg, additionalInfo);
+        }else{
+            this.logger[logLevel](msg);
+        }
+    }
+}

yarn.lock

@@ -991,6 +991,11 @@ date-and-time@0.7.0:
   resolved "https://registry.yarnpkg.com/date-and-time/-/date-and-time-0.7.0.tgz#26273355558877799f9c95888293fccee92fdb94"
   integrity sha512-qPHBPG0AQqbjP7wVf7vLv25/0bZRjYPiJiJtE0t6RqTswJR/6ExCXQLDnL5w4986j7i6470TMtalJxC8/UHrww==
 
+date-format@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/date-format/-/date-format-2.1.0.tgz#31d5b5ea211cf5fd764cd38baf9d033df7e125cf"
+  integrity sha512-bYQuGLeFxhkxNOF3rcMtiZxvCBAquGzZm6oWA1oZ0g2THUzivaRhv8uOhdr19LmoobSOLoIAxeUK2RdbM8IFTA==
+
 debug@2.6.9, debug@2.x.x, debug@^2.2.0, debug@^2.3.3:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -1544,6 +1549,11 @@ flat@^4.1.0:
   dependencies:
     is-buffer "~2.0.3"
 
+flatted@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/flatted/-/flatted-2.0.1.tgz#69e57caa8f0eacbc281d2e2cb458d46fdb449e08"
+  integrity sha512-a1hQMktqW9Nmqr5aktAux3JMNqaucxGcjtjWnZLHX7yyPCmlSV3M54nGYbqT8K+0GhF3NBgmJCc3ma+WOgX8Jg==
+
 for-in@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
@@ -2486,6 +2496,17 @@ log-symbols@^1.0.2:
   dependencies:
     chalk "^1.0.0"
 
+log4js@^5.1.0:
+  version "5.1.0"
+  resolved "https://registry.yarnpkg.com/log4js/-/log4js-5.1.0.tgz#3fa5372055a4c2611ab92d80496bffc100841508"
+  integrity sha512-QtXrBGZiIwfwBrH9zF2uQarvBuJ5+Icqx9fW+nQL4pnmPITJw8n6kh3bck5IkcTDBQatDeKqUMXXX41fp0TIqw==
+  dependencies:
+    date-format "^2.1.0"
+    debug "^4.1.1"
+    flatted "^2.0.1"
+    rfdc "^1.1.4"
+    streamroller "^2.1.0"
+
 loophole@1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/loophole/-/loophole-1.1.0.tgz#37949fea453b6256acc725c320ce0c5a7f70a2bd"
@@ -4000,6 +4021,11 @@ ret@~0.1.10:
   resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc"
   integrity sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==
 
+rfdc@^1.1.4:
+  version "1.1.4"
+  resolved "https://registry.yarnpkg.com/rfdc/-/rfdc-1.1.4.tgz#ba72cc1367a0ccd9cf81a870b3b58bd3ad07f8c2"
+  integrity sha512-5C9HXdzK8EAqN7JDif30jqsBzavB7wLpaubisuQIGHWf2gUXSpzy6ArX/+Da8RjFpagWsCn+pIgxTMAmKw9Zug==
+
 rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3:
   version "2.7.1"
   resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec"
@@ -4007,20 +4033,7 @@ rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3:
   dependencies:
     glob "^7.1.3"
 
-"router@git+https://github.com/blakeembrey/router#router-engine":
-  version "1.1.4"
-  uid "5eb68560e91b302251ff17a70cd1b6af1fc36d30"
-  resolved "git+https://github.com/blakeembrey/router#5eb68560e91b302251ff17a70cd1b6af1fc36d30"
-  dependencies:
-    array-flatten "2.0.0"
-    debug "^3.1.0"
-    methods "~1.1.2"
-    parseurl "~1.3.1"
-    path-to-regexp "0.1.7"
-    setprototypeof "1.0.0"
-    utils-merge "1.0.0"
-
-"router@github:blakeembrey/router#router-engine":
+router@blakeembrey/router#router-engine:
   version "1.1.4"
   resolved "https://codeload.github.com/blakeembrey/router/tar.gz/5eb68560e91b302251ff17a70cd1b6af1fc36d30"
   dependencies:
@@ -4332,6 +4345,15 @@ stream-shift@^1.0.0:
   resolved "https://registry.yarnpkg.com/stream-shift/-/stream-shift-1.0.0.tgz#d5c752825e5367e786f78e18e445ea223a155952"
   integrity sha1-1cdSgl5TZ+eG944Y5EXqIjoVWVI=
 
+streamroller@^2.1.0:
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/streamroller/-/streamroller-2.1.0.tgz#702de4dbba428c82ed3ffc87a75a21a61027e461"
+  integrity sha512-Ps7CuQL0RRG0YAigxNehrGfHrLu+jKSSnhiZBwF8uWi62WmtHDQV1OG5gVgV5SAzitcz1GrM3QVgnRO0mXV2hg==
+  dependencies:
+    date-format "^2.1.0"
+    debug "^4.1.1"
+    fs-extra "^8.1.0"
+
 streamsearch@0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-0.1.2.tgz#808b9d0e56fc273d809ba57338e929919a1a9f1a"