feature(core): add new features and code comments (#11)

* feat(mqtt-provider): ✨ add new MQTT provider to providers collection

* perf(general): ⚡ modify some environment variables from some components

* perf(general): ⚡ modify some environment variables from some components

* docs(general): 📝 documentation process update

* docs(general): 📝 add new documentation on some packets

* feat(doorkeeper): ✨ added the options to include dynamic defaults

* feat(utils): ✨ added a timeout option for each try to retry module

* fix(core): 🐛 fix some minor bugs in Ports and Managers

* refactor(logger): ♻️ refactor code to use LoggerInstance instead of Logger

* refactor(middlewares): ♻️ include a prefix in metrics middleware

* fix(logger): 🩹 fix minor bug in logger and dependencies updates

* feat(app-wrapper): ✨ add the option to monitor providers from the app-wrapper

* chore(limiter): 🚧 working on limiter functionality

* feat(limiter): 🚧 work in progress over the limiter/puller functionality

* refactor(general): 🚧 working in new release

* refactor(general): 🚧 refactor the most of the modules

* chore: 🚧 update modules

* chore(general): 🚧 update flaky test

* chore(general): 🚧 update test

* test(general): ✅ correct son flaky test

* ci(general): 💚 fix sonarqube project setup

* test(general): 🧪 fix some flaky test

* fix(general): 🚧 fix flaky tests

* ci(ci): 💚 fix sonarqube config file

* ci(ci): 💚 fix sonar

* ci(ci): 💚 update ci pipeline

* ci(ci): 💚 update pipeline

* ci(ci): 💚 update sonar config

* ci(ci): 💚 fix sonar

* ci(ci): 💚 add snyk to test

* ci(ci): 💚 update for snyk

* ci(ci): 💚 add snyk to pipeline

* ci(ci): 💚 update snyk pipeline

* ci(ci): 💚 update snyk

* ci(ci): 💚 update snyk

* ci(ci): 💚 update ci for snyk

* ci(ci): 💚 update ci for snyk

* ci(ci): 💚 update ci snyk

* chore(general): 🚧 work in progress for 1.4 release

* chore(ci): ⬆️ update dependencies

* fix(app-wrapper): 🐛 update the logger to be public

* fix(app-wrapper): 🐛 fix minor bug in default value for bootstrap settings

* fix(app-wrapper): 🐛 fix minor bug in version/relase tags

* fix(tasks): 🐛 control the the case when start/stop is called several times

* feat(tasks): ✨ add new options to add resources to the scheduler

* feat(service-config): ✨ add new options to include base and default configs

* fix(app-wrapper): 🐛 solve minor bug in port selection in cluster mode

* fix(app-wrapper): 🐛 solve a problem with the port in cluster a no cluster mode

* fix(service-config):

* fix(mqtt-provider): ⬆️ update dependencies

* fix(elastic-provider): 🐛 fix health check for new version of elastic

* fix(app-wrapper): 🐛 fix proxy on cluster mode

* feat(firehose): ✨ add options for metrics in plugs

* refactor(firehose): ♻️ refactor how the sinks and source are initialized

* fix(app-wrapper):

* perf(amqp-provider): ⚡ add own socket management to amqp provider

* chore(amqp-provider):

* chore(amqp-provider): 🚧 over amqp protocol

* chore(amqp-provider): 🚧 amqp protocol

* fix(doorkeeper):

* fix(mqtt-provider): ♻️ refactor code for new change on mqtt module

* fix(tasks): 🐛 fix minor bug related with side effects on tasks

* fix(jobs): 🐛 fix a minor bug on delay time in Limiter

* fix(jobs): 🐛 minor bug on delay time in Limiter

* ci(ci): 💚 update sonarquebe tasks

* feat(): add S3 provider (#10)

* chore(general): ⬆️ update dependencies for all the packages

---------

Co-authored-by: DanielaDurandBartolo <134690786+DanielaDurandBartolo@users.noreply.github.com>

.config/azure-pipelines.yml

@@ -1,4 +1,4 @@
-## Template for a monorepository based on GitFlow, Docker and DockerCompose in Netin Development Ecosystem
+## Template for a monorepository based on GitFlow, Docker and DockerCompose in Mytra Development Ecosystem
 name: mdf-js$(Date:yyyyMMdd)$(Rev:.r)
 ## CI trigger based on GitFlow
 trigger:
@@ -10,7 +10,7 @@ trigger:
 pr:
   - master
   - develop
-## Netin - Variables
+## Mytra - Variables
 variables:
     ## Artifact name
   - name: artifactName
@@ -28,6 +28,8 @@ stages:
     displayName: 'Application Build'
     pool:
       name: NUCHUB
+      demands:
+        - Agent.OS -equals Linux
     jobs:
       - job: 'CODE_QUALITY'
         displayName: 'Code Quality Assurance'

.config/copyMedia.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+DESTINATION_DIR="docs/assets/media" # Ruta de destino
+SOURCE_DIR="packages/*/*/media" # Ruta de origen
+HTML_DIR="docs" # Ruta de destino de los archivos HTML
+
+# Crear la carpeta de destino si no existe
+mkdir -p "$DESTINATION_DIR"
+
+# Copiar todos los archivos de las carpetas media a la carpeta de destino
+find $SOURCE_DIR -type f -exec cp {} "$DESTINATION_DIR" \;
+
+# Modificar los enlaces en los archivos HTML
+find $HTML_DIR -name "*.html" -type f | while read file; do
+    if [ -f "$file" ]; then
+        sed -i'' -e 's/src="media\//src="..\/assets\/media\//g' "$file"
+    fi
+done

.config/envDoc.mjs

@@ -0,0 +1,352 @@
+/**
+ * Copyright 2023 Netin Systems S.L. All rights reserved.
+ * Note: All information contained herein is, and remains the property of Netin Systems S.L. and its
+ * suppliers, if any. The intellectual and technical concepts contained herein are property of
+ * Netin Systems S.L. and its suppliers and may be covered by European and Foreign patents, patents
+ * in process, and are protected by trade secret or copyright.
+ *
+ * Dissemination of this information or the reproduction of this material is strictly forbidden
+ * unless prior written permission is obtained from Netin Systems S.L.
+ */
+
+import fs from 'fs';
+import * as glob from 'glob';
+import path from 'path';
+import { remark } from 'remark';
+import remarkGFM from 'remark-gfm';
+import remarkLint from 'remark-lint';
+import remarkLintUnorderedListMarkerStyle from 'remark-lint-unordered-list-marker-style';
+import remarkParse from 'remark-parse';
+import remarkStringify from 'remark-stringify';
+import remarkToc from 'remark-toc';
+
+/**
+ * This script is used to prepare the artifacts for the distribution.
+ * It will generate an artifact for each each application in apps folder and will include all the
+ * necessary files for official Netin distribution.
+ * All the specific files for each application is stored in the apps folder, the rest of the files
+ * are stored in the .distribution folder.
+ */
+
+const SOURCE_FILES = './src/**/*.ts';
+const README_FILE = 'README.md';
+
+// Expresiones regulares para identificar variables de entorno y comentarios
+const envVarRegex = /process\.env\[['"](?<varName>[^'"]+)['"]\]/;
+const defaultValueRegex = /(default:|@defaultValue) (?<defaultValue>[^,]+)/;
+
+const srcFiles = glob.sync(SOURCE_FILES, {
+  nodir: true, ignore: {
+    ignored: p => /\.test.ts$/.test(p.name),
+  }
+});
+const variables = [];
+
+for (const file of srcFiles) {
+  const fileContent = fs.readFileSync(file, 'utf8');
+  const matches = fileContent.match(envVarRegex);
+  if (matches) {
+    findAllEnvironmentVariablesInFile(fileContent, path.basename(file));
+  }
+}
+// Now we have all the environment variables and their information, we need to update the markdown file
+// with the information. For this we need to generate a new level 2 title **Environment variables**,
+// overwriting the previous one if exists, and then generate a table with the information of each
+// environment variable. We must include in the TOC the new title if it does not exist.
+const readmeFile = fs.readFileSync(README_FILE, 'utf8');
+const readMeMD = remark()
+  .use(remarkParse)
+  .use(remarkLint)
+  .use(remarkLintUnorderedListMarkerStyle, '-')
+  .use(remarkToc)
+  .use(remarkGFM)
+  .parse(readmeFile);
+
+// We need to find all the nodes between the title **Environment variables** and the next title and remove them
+// from the tree, including the title itself. We also need to find the node of the license and next nodes and
+// save it to add it later.
+findAndRemoveEnvironmentVariablesSection(readMeMD);
+if (variables.length > 0) {
+  const licenseNodes = findRemoveAndReturnLinceSection(readMeMD);
+  const envVariablesNodes = generateListOfEnvironmentVariables(variables);
+  readMeMD.children.push(...envVariablesNodes);
+  if (licenseNodes) {
+    readMeMD.children.push(...licenseNodes);
+  }
+}
+const newReadmeFile = remark()
+  .use(remarkParse)
+  .use(remarkLint)
+  .use(remarkLintUnorderedListMarkerStyle, '-')
+  .use(remarkToc)
+  .use(remarkGFM)
+  .use(remarkStringify, { bullet: '-' })
+  .stringify(readMeMD);
+fs.writeFileSync(`${README_FILE}`, newReadmeFile);
+
+/**
+ * Finds all environment variables in a file and extracts their information.
+ *
+ * @param {string} fileContent - The content of the file.
+ * @param {string} path - The path of the file.
+ */
+function findAllEnvironmentVariablesInFile(fileContent, path) {
+  const lines = fileContent.split('\r\n');
+  for (let index = 0; index < lines.length; index++) {
+    const matches = lines[index].match(envVarRegex);
+    if (matches && matches.groups && matches.groups['varName']) {
+      let comment = findUpperCommentInFile(lines, index);
+      const defaultValue = getDefaultValueFromComment(comment);
+      if (defaultValue) {
+        comment = comment.replace(defaultValueRegex, '').trim();
+      }
+      variables.push({ name: matches.groups['varName'], comment, path, defaultValue });
+    }
+  }
+}
+/**
+ * Finds the upper comment in a file based on a given index.
+ * The comment is always located above the declaration of a constant.
+ *
+ * @param {string[]} lines - The lines of the file.
+ * @param {number} index - The index to start searching from.
+ * @returns {string|undefined} - The upper comment or undefined if no comment is found.
+ */
+function findUpperCommentInFile(lines, index) {
+  let endOfComment = 0;
+  let startOfComment = 0;
+  let comment = undefined;
+  if (index < 0) {
+    return;
+  }
+  // Need to find the line where the constant is declared, the comment is always above the declaration
+  // the function isConstantDeclaration is used to find the line where the constant is declared
+  while (index >= 0) {
+    if (isConstantDeclaration(lines[index])) {
+      break;
+    }
+    index--;
+  }
+  if (index < 0) {
+    return;
+  }
+  // Should check if the upper line has the end of comment, in other case, no comment is found
+  if (!lines[index - 1].includes('*/')) {
+    return;
+  }
+  endOfComment = index - 1;
+  // Search for the start of the comment
+  while (index >= 0) {
+    if (lines[index].includes('/**')) {
+      break;
+    }
+    index--;
+  }
+  if (index < 0) {
+    return;
+  }
+  startOfComment = index;
+  // Join all the lines of the comment, cleaning:
+  //  - '/**' in the first line
+  //  - '*/' in the last line
+  //  - '*' in the rest of the lines
+  if (startOfComment === endOfComment) {
+    comment = lines[startOfComment].replace('/**', '').replace('*/', '');
+  } else {
+    const firstLine = lines[startOfComment].replace('/**', '');
+    const lastLine = lines[endOfComment].replace('*/', '');
+    const middleLines = lines.slice(startOfComment + 1, endOfComment).map(line => line.replace('*', ''));
+    comment = [firstLine, ...middleLines, lastLine].join(' ');
+  }
+  // Clean spaces if there are more than one space between words or at the beginning or end of the comment
+  return comment.replace(/\s\s+/g, ' ').trim();
+}
+/**
+ * Obtiene el valor predeterminado de un comentario.
+ * @param {string} comment - El comentario del cual se desea obtener el valor predeterminado.
+ * @returns {string|undefined} El valor predeterminado del comentario, o undefined si no se encuentra.
+ */
+function getDefaultValueFromComment(comment) {
+  if (!comment) {
+    return;
+  }
+  const matches = comment.match(defaultValueRegex);
+  if (matches && matches.groups && matches.groups['defaultValue']) {
+    return matches.groups['defaultValue'];
+  }
+  return;
+}
+/**
+ * Checks if a given line is a constant declaration.
+ *
+ * @param {string} line - The line to check.
+ * @returns {boolean} - True if the line is a constant declaration, false otherwise.
+ */
+function isConstantDeclaration(line) {
+  return line.includes('const');
+}
+/**
+ * Finds and removes the section containing environment variables from a given Markdown document.
+ * 
+ * @param {object} readMeMD - The Markdown document object.
+ */
+function findAndRemoveEnvironmentVariablesSection(readMeMD) {
+  for (let index = 0; index < readMeMD.children.length; index++) {
+    const node = readMeMD.children[index];
+    if (node.type === 'heading') {
+      let nodeTitle = undefined;
+      if (node.children[0].type === 'strong') {
+        nodeTitle = node.children[0].children[0].value;
+      } else if (node.children[0].type === 'text') {
+        nodeTitle = node.children[0].value;
+      }
+      if (nodeTitle === 'Environment variables') {
+        // Remove all the nodes between the title and the next title
+        readMeMD.children.splice(index, 1);
+        while (index < readMeMD.children.length) {
+          if (readMeMD.children[index].type === 'heading' && readMeMD.children[index].depth === 2) {
+            break;
+          }
+          readMeMD.children.splice(index, 1);
+        }
+        break;
+      }
+    }
+  }
+}
+/**
+ * Finds, removes, and returns the license section from the given readMeMD object.
+ * 
+ * @param {Object} readMeMD - The readMeMD object.
+ * @returns {Array} - An array of nodes representing the license section.
+ */
+function findRemoveAndReturnLinceSection(readMeMD) {
+  let licenseNodes = [];
+  for (let index = 0; index < readMeMD.children.length; index++) {
+    const node = readMeMD.children[index];
+    if (node.type === 'heading') {
+      let nodeTitle = undefined;
+      if (node.children[0].type === 'strong') {
+        nodeTitle = node.children[0].children[0].value;
+      } else if (node.children[0].type === 'text') {
+        nodeTitle = node.children[0].value;
+      }
+      if (nodeTitle === 'License') {
+        // Save all the nodes between the title and the next title, including the title
+        licenseNodes.push(node);
+        readMeMD.children.splice(index, 1);
+        while (index < readMeMD.children.length) {
+          if (readMeMD.children[index].type === 'heading' && readMeMD.children[index].depth === 2) {
+            break;
+          }
+          licenseNodes.push(readMeMD.children[index]);
+          readMeMD.children.splice(index, 1);
+        }
+      }
+    }
+  }
+  return deletePositionPropertyRecursively(licenseNodes);
+}
+/**
+ * Recursively deletes the 'position' property from each node in the given array of nodes.
+ * If a node has children, the function is called recursively on each child node.
+ * @param {Array} nodes - The array of nodes to process.
+ * @returns {Array} - The modified array of nodes.
+ */
+function deletePositionPropertyRecursively(nodes) {
+  for (const node of nodes) {
+    delete node.position;
+    if (node.children) {
+      deletePositionPropertyRecursively(node.children);
+    }
+  }
+  return nodes;
+}
+/**
+ * Generates a table of environment variables.
+ * 
+ * @param {Array<Object>} envVariables - An array of environment variables.
+ * @returns {Array<Object>} - An array representing the table of environment variables.
+ */
+function generateTableOfEnvironmentVariables(envVariables) {
+  const envVariablesTitle = {
+    type: 'heading',
+    depth: 2,
+    children: [{ type: 'strong', children: [{ type: 'text', value: 'Environment variables' }] }],
+  };
+  const envVariablesTable = {
+    type: 'table',
+    align: ['left', 'center', 'left'],
+    children: [
+      {
+        type: 'tableRow',
+        children: [
+          { type: 'tableCell', children: [{ type: 'text', value: 'Variable' }] },
+          { type: 'tableCell', children: [{ type: 'text', value: 'Default value' }] },
+          { type: 'tableCell', children: [{ type: 'text', value: 'Description' }] },
+        ],
+      },
+    ],
+  };
+  for (const envVariable of envVariables) {
+    envVariablesTable.children.push({
+      type: 'tableRow',
+      children: [
+        { type: 'tableCell', children: [{ type: 'text', value: envVariable.name }] },
+        { type: 'tableCell', children: [{ type: 'text', value: envVariable.defaultValue }] },
+        { type: 'tableCell', children: [{ type: 'text', value: envVariable.comment }] },
+      ],
+    });
+  }
+  return [envVariablesTitle, envVariablesTable];
+}
+/**
+ * Generates a list of environment variables.
+ * 
+ * @param {Array<Object>} envVariables - The array of environment variables.
+ * @returns {Array<Object>} - The list of environment variables in a specific format.
+ */
+function generateListOfEnvironmentVariables(envVariables) {
+  const envVariablesTitle = {
+    type: 'heading',
+    depth: 2,
+    children: [{ type: 'strong', children: [{ type: 'text', value: 'Environment variables' }] }],
+  };
+  const envVariablesList = {
+    type: 'list',
+    ordered: false,
+    spread: false,
+    children: [],
+  };
+  for (const envVariable of envVariables) {
+    if (!envVariable.comment) {
+      console.log(`WARNING: No comment found for environment variable ${envVariable.name} in file ${envVariable.path}`);
+    };
+    let defaultValue = [];
+    if (envVariable.defaultValue) {
+      defaultValue = [
+        { type: 'text', value: ' (default: ', },
+        { type: 'inlineCode', value: envVariable.defaultValue },
+        { type: 'text', value: `): ${envVariable.comment}` }];
+    } else {
+      defaultValue = [{ type: 'text', value: `: ${envVariable.comment}` }];
+    }
+    envVariablesList.children.push({
+      type: 'listItem',
+      spread: false,
+      checked: null,
+      children: [{
+        type: 'paragraph',
+        children: [
+          {
+            type: 'strong', children: [
+              { type: 'text', value: envVariable.name },
+            ]
+          },
+          ...defaultValue,
+        ],
+      }],
+    });
+  }
+  return [envVariablesTitle, envVariablesList];
+}