Merge pull request #834 from hpcc-systems/yadhap/logger-config

Added logger configuration to write file in FS for easy access, analy…

Tombolo/docker-compose-with-kafka

@@ -26,6 +26,10 @@ services:
     volumes:
       - /tmp/pm2
       - ./server/cluster-whitelist.js:/app/cluster-whitelist.js
+      - /var/log/tombolo-server:/app/logs
+    logging:
+      driver: "none" # This line disables Docker logs. All logs are written to the file system
+
     networks:
       - tombolo-network
     ports:

Tombolo/docker-compose-without-kafka

@@ -26,6 +26,7 @@ services:
     volumes:
       - /tmp/pm2
       - ./server/cluster-whitelist.js:/app/cluster-whitelist.js
+      - /var/log/tombolo-server:/app/logs
     networks:
       - tombolo-network
     ports:
@@ -34,6 +35,8 @@ services:
       - mysql_db
     env_file:
       - .env
+    logging:
+      driver: "none" # This line disables Docker logs. All logs are written to the file system
 
   mysql_db:
     image: mysql:5.7

Tombolo/server/config/logger.js

@@ -1,58 +1,81 @@
-const { createLogger, format, transports } = require('winston');
-// docs: https://github.com/winstonjs/winston
-const isProduction = process.env.NODE_ENV === 'production';
+const { createLogger, format, transports } = require("winston");
+require("winston-daily-rotate-file"); // Ensure you have this package installed
 
-// Print logs in color depending on log type 
+// Determine if the environment is production
+const isProduction = process.env.NODE_ENV === "production";
+
+// Define custom colors for different log levels
 format.colorize().addColors({
   error: "red",
   warn: "yellow",
   info: "green",
   http: "magenta",
 });
 
-const getFormat = () =>
+// Function to configure log format based on the environment
+const getFormat = (isConsole) =>
   format.combine(
-    isProduction ? format.uncolorize() : format.colorize({ all: true }),  // adding or removing colors depending on logs type;
-    format.timestamp(), // will add UTC timestamp to logs, not enabled by default
+    // Always include a timestamp and log in JSON format for easy parsing
+    format.timestamp(),
+    isConsole ? format.colorize({ all: true }) : format.uncolorize(),
     format.printf((info) => {
-      let { level, message, label, stack, timestamp } = info;
-      if (!isProduction) timestamp = new Date(timestamp).toLocaleString(); // will make timestemp more friendly in development
-      let devlog = `[${timestamp}]-[${level}] ${message}`; // will produce : [5/12/2022, 10:21:41 AM]-[info] ✔️ JOBSCHEDULER IS BOOTSTRAPED
-      let prodlog = `[${level}] ${message}`; // Azure adds its own timestemp to log so we dont need to put it into console twice
-      let log = isProduction ? prodlog : devlog;
-      if (stack) log += `\n ${stack}`; // stack is only available when error is logged, print in on new line
-      return log;
-    })
-  );
+      // Construct the log object
+      const logEntry = {
+        level: info.level,
+        message: info.message,
+        timestamp: info.timestamp,
+      };
 
-const common = { handleExceptions: true, handleRejections: true }; // we want to catch any errors on promise rejections and add them to logs
+      // If in production, return as JSON
+      if (isProduction) {
+        return JSON.stringify(logEntry);
+      }
 
-let DEFAULT_LOG_LEVEL = 'http';
-//! Changing to 'debug' will bring sequelize logs to console;
-// when in productions log into console everything up to PROD_LEVEL level. https://github.com/winstonjs/winston#logging-levels
-// error  0 | warn  1 | info  2 | http  3 | verbose  4 | debug  5 | silly  6 |
+      // Use a friendly format for development
+      return `[${new Date(info.timestamp).toLocaleString()}]-[${info.level}] ${
+        info.message
+      }`;
+    })
+  );
 
-// Initialize logger
+// Create a logger instance
 const logger = createLogger({
-  exitOnError: false,
-  format: format.combine(
-    format.errors({ stack: true }), 
-    format.timestamp()
-    ), // this will be common setting for all transports;
+  exitOnError: false, // Do not exit on handled exceptions
+  format: getFormat(false), // Set the default log format (for file transports)
   transports: [
-     new transports.Console({
-      ...common,
-      format: getFormat(),
-      level: process.env.NODE_LOG_LEVEL || DEFAULT_LOG_LEVEL
-    }) 
+    // Console transport for logging to the console
+    new transports.Console({
+      level: process.env.NODE_LOG_LEVEL || "info", // Log level from environment or default to 'info'
+      format: getFormat(true), // Set the log format for console (colorized)
+    }),
+    // Combined log transport
+    new transports.DailyRotateFile({
+      level: "verbose",
+      filename: "./logs/combined-%DATE%.log",
+      datePattern: "YYYY-MM-DD",
+      zippedArchive: true,
+      maxSize: "20m", // Maximum size
+      maxFiles: "14d", // Maximum of 14 days
+      format: getFormat(false), // Use plain format for files
+    }),
   ],
 });
 
-// If we're not in production then also log to the files
-// if (process.env.NODE_ENV !== 'production') {
-//   const settings = {...common, format: getFormat()}; // will write to files same output as to console but no special char for coloring
-//   logger.add(new transports.File({ ...settings, level: 'http', filename: './logs/combined.log' }));
-//   logger.add(new transports.File({ ...settings, level: 'error', filename: './logs/error.log' })); // only logger.error() will be written here
-// }
+// Additional file transports for development environment
+if (!isProduction) {
+  // Daily rotate file transport for logging errors
+  logger.add(
+    new transports.DailyRotateFile({
+      level: "error",
+      filename: "./logs/error-%DATE%.log",
+      datePattern: "YYYY-MM-DD",
+      zippedArchive: true, // Compress log files
+      maxSize: "20m", // Maximum size of a log file
+      maxFiles: "14d", // Keep logs for a maximum of 14 days
+      format: getFormat(false), // Use plain format for files
+    })
+  );
+}
 
+// Export the logger instance for use in other modules
 module.exports = logger;