diff --git a/backend/package-lock.json b/backend/package-lock.json index 64d1077..4bcc973 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -29,6 +29,7 @@ "mysql2": "^3.6.1", "nodemon": "^3.0.1", "puppeteer": "^21.3.8", + "redis": "^4.6.12", "socket.io": "^4.7.2", "uuid": "^9.0.1", "web-push": "^3.6.6", @@ -3335,6 +3336,59 @@ "node": ">=12" } }, + "node_modules/@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/client": { + "version": "1.5.13", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.13.tgz", + "integrity": "sha512-epkUM9D0Sdmt93/8Ozk43PNjLi36RZzG+d/T1Gdu5AI8jvghonTeLYV69WVWdilvFo+PYxbP0TZ0saMvr6nscQ==", + "dependencies": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + }, + "engines": { + "node": ">=14" + } + }, + "node_modules/@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/json": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz", + "integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/search": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz", + "integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, + "node_modules/@redis/time-series": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz", + "integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==", + "peerDependencies": { + "@redis/client": "^1.0.0" + } + }, "node_modules/@sideway/address": { "version": "4.1.4", "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.4.tgz", @@ -5474,6 +5528,14 @@ "cli-color": "0.3.2" } }, + "node_modules/cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -7126,6 +7188,14 @@ "is-property": "^1.0.2" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -10783,6 +10853,19 @@ "node": ">=8.10.0" } }, + "node_modules/redis": { + "version": "4.6.12", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.12.tgz", + "integrity": "sha512-41Xuuko6P4uH4VPe5nE3BqXHB7a9lkFL0J29AlxKaIfD6eWO8VO/5PDF9ad2oS+mswMsfFxaM5DlE3tnXT+P8Q==", + "dependencies": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.5.13", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.6", + "@redis/search": "1.1.6", + "@redis/time-series": "1.0.5" + } + }, "node_modules/reduce-flatten": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz", @@ -15238,6 +15321,46 @@ } } }, + "@redis/bloom": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@redis/bloom/-/bloom-1.2.0.tgz", + "integrity": "sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==", + "requires": {} + }, + "@redis/client": { + "version": "1.5.13", + "resolved": "https://registry.npmjs.org/@redis/client/-/client-1.5.13.tgz", + "integrity": "sha512-epkUM9D0Sdmt93/8Ozk43PNjLi36RZzG+d/T1Gdu5AI8jvghonTeLYV69WVWdilvFo+PYxbP0TZ0saMvr6nscQ==", + "requires": { + "cluster-key-slot": "1.1.2", + "generic-pool": "3.9.0", + "yallist": "4.0.0" + } + }, + "@redis/graph": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/@redis/graph/-/graph-1.1.1.tgz", + "integrity": "sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==", + "requires": {} + }, + "@redis/json": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@redis/json/-/json-1.0.6.tgz", + "integrity": "sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw==", + "requires": {} + }, + "@redis/search": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@redis/search/-/search-1.1.6.tgz", + "integrity": "sha512-mZXCxbTYKBQ3M2lZnEddwEAks0Kc7nauire8q20oA0oA/LoA+E/b5Y5KZn232ztPb1FkIGqo12vh3Lf+Vw5iTw==", + "requires": {} + }, + "@redis/time-series": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@redis/time-series/-/time-series-1.0.5.tgz", + "integrity": "sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg==", + "requires": {} + }, "@sideway/address": { "version": "4.1.4", "resolved": "https://registry.npmjs.org/@sideway/address/-/address-4.1.4.tgz", @@ -16953,6 +17076,11 @@ "cli-color": "0.3.2" } }, + "cluster-key-slot": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz", + "integrity": "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==" + }, "co": { "version": "4.6.0", "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", @@ -18257,6 +18385,11 @@ "is-property": "^1.0.2" } }, + "generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==" + }, "gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -21002,6 +21135,19 @@ "picomatch": "^2.2.1" } }, + "redis": { + "version": "4.6.12", + "resolved": "https://registry.npmjs.org/redis/-/redis-4.6.12.tgz", + "integrity": "sha512-41Xuuko6P4uH4VPe5nE3BqXHB7a9lkFL0J29AlxKaIfD6eWO8VO/5PDF9ad2oS+mswMsfFxaM5DlE3tnXT+P8Q==", + "requires": { + "@redis/bloom": "1.2.0", + "@redis/client": "1.5.13", + "@redis/graph": "1.1.1", + "@redis/json": "1.0.6", + "@redis/search": "1.1.6", + "@redis/time-series": "1.0.5" + } + }, "reduce-flatten": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/reduce-flatten/-/reduce-flatten-2.0.0.tgz", diff --git a/backend/package.json b/backend/package.json index 4053ccf..35ff501 100644 --- a/backend/package.json +++ b/backend/package.json @@ -39,6 +39,7 @@ "mysql2": "^3.6.1", "nodemon": "^3.0.1", "puppeteer": "^21.3.8", + "redis": "^4.6.12", "socket.io": "^4.7.2", "uuid": "^9.0.1", "web-push": "^3.6.6", diff --git a/backend/src/app.js b/backend/src/app.js index 76acc59..7ebaa68 100644 --- a/backend/src/app.js +++ b/backend/src/app.js @@ -4,9 +4,7 @@ const cors = require("cors"); const logger = require("./config/logger"); const webpush = require("web-push"); const config = require("./config/config"); -// const { captureScreenshot } = require("./puppeteer/puppeteer"); const { calcAQI } = require("./helper/calculateTotalAQI"); -const Agendash = require("agendash"); const v1Route = require("./routes/v1"); const { errors } = require("celebrate"); const { errorHandler } = require("./middlewares/errors"); @@ -15,6 +13,8 @@ const httpStatus = require("http-status"); const { producer } = require("./rabbitmq/producer"); +const { redisClient } = require("./redis/redis"); + const start = async (agenda) => { const app = express(); @@ -45,6 +45,11 @@ const start = async (agenda) => { const apiRoutes = (app, io, mqtt) => { app.use("/v1", v1Route); + app.get("/test-cache", async (req, res, next) => { + const value = await redisClient.get("value"); + res.send(value); + }); + app.post("/amqp", async (req, res, next) => { await producer.publishEmailMessage(req.body.mailList); res.send("Sent"); diff --git a/backend/src/config/config.js b/backend/src/config/config.js index e6228a5..a69d998 100644 --- a/backend/src/config/config.js +++ b/backend/src/config/config.js @@ -29,6 +29,9 @@ const envVarsSchema = Joi.object() RABBITMQ: Joi.string(), RABBITMQ_MAILSERVICE_EXCHANGE_NAME: Joi.string(), RABBITMQ_MAILSERVICE_QUEUE_NAME: Joi.string(), + REDIS_HOST: Joi.string(), + REDIS_PORT: Joi.string(), + REDIS_PASSWORD: Joi.string(), }) .unknown(); @@ -70,4 +73,9 @@ module.exports = { mailServiceExchangeName: envVars.RABBITMQ_MAILSERVICE_EXCHANGE_NAME, mailServiceQueueName: envVars.RABBITMQ_MAILSERVICE_QUEUE_NAME, }, + redis: { + host: envVars.REDIS_HOST, + port: envVars.REDIS_PORT, + password: envVars.REDIS_PASSWORD, + }, }; diff --git a/backend/src/database.js b/backend/src/database.js deleted file mode 100644 index d9bc3ff..0000000 --- a/backend/src/database.js +++ /dev/null @@ -1,42 +0,0 @@ -var mysql = require("mysql2"); -const config = require("./config/config"); -const logger = require("./config/logger"); - -const connectMySql = () => { - var connection = mysql.createConnection({ - host: config.host, - port: config.dbPort, - user: config.dbUserName, - password: config.dbPassword, - database: config.dbName, - }); - - connection.connect(function (err) { - if (err) { - logger.error("Failed to connect to MySQL: " + err.stack); - - return; - } - - logger.info("Connect to MySQL as id: " + connection.threadId); - }); - - connection.query( - "CREATE TABLE AirQuality (\ - id varchar(255),\ - aqi float,\ - humidity float,\ - temperature float,\ - co float,\ - measuredAt datetime);", - function (err, results, fields) { - if (err.code != "ER_TABLE_EXISTS_ERROR") logger.error(err); - } - ); - - return connection; -}; - -module.exports = { - db: connectMySql, -}; diff --git a/backend/src/index.js b/backend/src/index.js index 5a047a9..1f7703b 100644 --- a/backend/src/index.js +++ b/backend/src/index.js @@ -22,7 +22,8 @@ const startAgenda = async () => { // Local // await agenda.every(CronTime.everyDayAt(23), "retrieveDailyAqi"); - await agenda.every(CronTime.everyDayAt(16), "retrieveDailyAqi"); + await agenda.every(CronTime.everyDayAt(16, 5), "retrieveDailyAqi"); + await agenda.every(CronTime.everyHour(), "getHourlyAqi"); }; mongoose diff --git a/backend/src/jobs/definitions/getHourlyAqi.js b/backend/src/jobs/definitions/getHourlyAqi.js new file mode 100644 index 0000000..65fb4d7 --- /dev/null +++ b/backend/src/jobs/definitions/getHourlyAqi.js @@ -0,0 +1,54 @@ +const { aqiApiToken, predictionServiceUrl } = require("../../config/config"); +const axios = require("axios"); +const logger = require("../../config/logger"); +const HourlyAQIModel = require("../../models/HourlyAQI"); + +const AQI_URL = "http://api.waqi.info"; + +const getHourlyAqi = async (agenda) => { + try { + // define(jobName, fn, [options]) + agenda.define( + "getHourlyAqi", + async function (job, done) { + try { + var response = await axios({ + method: "get", + url: `${AQI_URL}/feed/@1583/?token=${aqiApiToken}`, + }); + + response = response.data; + + const aqi = await HourlyAQIModel.findOne({ + dateTime: response.data.time.iso, + }); + + logger.info( + `New AQI retrieved for date ${response.data.time.iso} from ${AQI_URL}: ${response.data.aqi}` + ); + + if (!aqi) { + await HourlyAQIModel.create({ + aqi: response.data.aqi, + dateTime: response.data.time.iso, + }); + } + + done(); + } catch (error) { + console.error(error); + + logger.error( + "Failed to update new daily aqi in job [agenda.retrieveDailyAqi] : " + + error.message + ); + } + }, + { priority: "highest", concurrency: 20 } + ); + } catch (error) { + logger.error("Error in job [agenda.retrieveDailyAqi] : " + error); + } +}; + +module.exports = { getHourlyAqi }; diff --git a/backend/src/jobs/definitions/index.js b/backend/src/jobs/definitions/index.js index 5e8196f..c94b990 100644 --- a/backend/src/jobs/definitions/index.js +++ b/backend/src/jobs/definitions/index.js @@ -1,11 +1,12 @@ const { retrieveDailyAqi } = require("./retrieveDailyAqi"); +const { getHourlyAqi } = require("./getHourlyAqi"); const logger = require("../../config/logger"); -let definitions = [retrieveDailyAqi]; +let definitions = [retrieveDailyAqi, getHourlyAqi]; const loadDefinitions = async (agenda) => { try { - console.log() + console.log(); for (let definition of definitions) { await definition(agenda); } diff --git a/backend/src/jobs/definitions/retrieveDailyAqi.js b/backend/src/jobs/definitions/retrieveDailyAqi.js index eef7996..c5c61a8 100644 --- a/backend/src/jobs/definitions/retrieveDailyAqi.js +++ b/backend/src/jobs/definitions/retrieveDailyAqi.js @@ -1,8 +1,7 @@ const { aqiApiToken, predictionServiceUrl } = require("../../config/config"); const axios = require("axios"); const logger = require("../../config/logger"); - -const AQI_URL = "http://api.waqi.info"; +const HourlyAQIModel = require("../../models/HourlyAQI"); const retrieveDailyAqi = async (agenda) => { try { @@ -11,20 +10,15 @@ const retrieveDailyAqi = async (agenda) => { "retrieveDailyAqi", async function (job, done) { try { - var response = await axios({ - method: "get", - url: `${AQI_URL}/feed/@1583/?token=${aqiApiToken}`, - }); - - response = response.data; + const hourlyAqi = await HourlyAQIModel.find(); - logger.info( - `New AQI retrieved for date ${response.data.time.iso} from ${AQI_URL}: ${response.data.aqi}` + const compositeAqiObject = hourlyAqi.reduce((prev, curr) => + prev > curr ? prev : curr ); var requestBody = { - newAqi: response.data.aqi, - date: response.data.time.iso.substring(0, 10), + newAqi: compositeAqiObject.aqi, + date: compositeAqiObject.dateTime.substring(0, 10), }; console.log(requestBody); diff --git a/backend/src/models/HourlyAQI.js b/backend/src/models/HourlyAQI.js new file mode 100644 index 0000000..de870f4 --- /dev/null +++ b/backend/src/models/HourlyAQI.js @@ -0,0 +1,21 @@ +const mongoose = require("mongoose"); + +const HourlyAQISchema = new mongoose.Schema( + { + aqi: Number, + dateTime: String, + }, + { + timestamps: false, + } +); + +const MONGO_ATLAS_COLLECTION_NAME = "hourly_aqi"; + +var HourlyAQIModel = mongoose.model( + "HourlyAQI", + HourlyAQISchema, + MONGO_ATLAS_COLLECTION_NAME +); + +module.exports = HourlyAQIModel; diff --git a/backend/src/redis/redis.js b/backend/src/redis/redis.js new file mode 100644 index 0000000..bf37154 --- /dev/null +++ b/backend/src/redis/redis.js @@ -0,0 +1,30 @@ +const { createClient } = require("redis"); +const logger = require("../config/logger"); +const config = require("../config/config"); + +var client = createClient({ + socket: { + host: config.redis.host, + port: config.redis.port, + }, + username: "default", + password: config.redis.password, +}); + +client.on("ready", () => { + logger.info("Redis is connected"); +}); + +client.on("error", (err) => { + reject(`Redis Client Error [${err}]`); +}); + +(async () => { + try { + await client.connect(); + } catch (error) { + reject(`error while connecting redis [${error}]`); + } +})(); + +module.exports = { redisClient: client }; diff --git a/client/index.html b/client/index.html index 4f3141c..827e673 100644 --- a/client/index.html +++ b/client/index.html @@ -62,7 +62,7 @@ -
{v.email}
+{v.email}