feat: delay configurations

AtomiCloud · Feb 28, 2024 · a3e8a90 · a3e8a90
1 parent f741f56
commit a3e8a90
Show file tree

Hide file tree

Showing 12 changed files with 123 additions and 3 deletions.
diff --git a/config/app/config.yaml b/config/app/config.yaml
@@ -7,6 +7,10 @@ app:
   service: helium
   module: pollee
   version: "1.0"
+  populator:
+    delay: 0.5
+  watcher:
+    delay: 1
 auth:
   descope:
     id: "P2Ya46UVwFV8nTwNGZWpxRtyL7Jj"

diff --git a/config/app/schema.json b/config/app/schema.json
@@ -12,6 +12,9 @@
         "platform": {
           "type": "string"
         },
+        "populator": {
+          "$ref": "#/definitions/PopulatorConfig"
+        },
         "searcher": {
           "$ref": "#/definitions/SearcherConfig"
         },
@@ -20,6 +23,9 @@
         },
         "version": {
           "type": "string"
+        },
+        "watcher": {
+          "$ref": "#/definitions/WatcherConfig"
         }
       },
       "type": "object"
@@ -182,6 +188,14 @@
       },
       "type": "object"
     },
+    "PopulatorConfig": {
+      "properties": {
+        "delay": {
+          "type": "number"
+        }
+      },
+      "type": "object"
+    },
     "SearcherConfig": {
       "properties": {
         "proxy": {
@@ -198,6 +212,14 @@
       },
       "type": "object"
     },
+    "WatcherConfig": {
+      "properties": {
+        "delay": {
+          "type": "number"
+        }
+      },
+      "type": "object"
+    },
     "ZincConfig": {
       "properties": {
         "domain": {

diff --git a/infra/cron_chart/app/config.yaml b/infra/cron_chart/app/config.yaml
@@ -7,6 +7,10 @@ app:
   service: helium
   module: pollee
   version: "1.0"
+  populator:
+    delay: 0.5
+  watcher:
+    delay: 1
 auth:
   descope:
     id: "P2Ya46UVwFV8nTwNGZWpxRtyL7Jj"

diff --git a/infra/cron_chart/app/schema.json b/infra/cron_chart/app/schema.json
@@ -12,6 +12,9 @@
         "platform": {
           "type": "string"
         },
+        "populator": {
+          "$ref": "#/definitions/PopulatorConfig"
+        },
         "searcher": {
           "$ref": "#/definitions/SearcherConfig"
         },
@@ -20,6 +23,9 @@
         },
         "version": {
           "type": "string"
+        },
+        "watcher": {
+          "$ref": "#/definitions/WatcherConfig"
         }
       },
       "type": "object"
@@ -182,6 +188,14 @@
       },
       "type": "object"
     },
+    "PopulatorConfig": {
+      "properties": {
+        "delay": {
+          "type": "number"
+        }
+      },
+      "type": "object"
+    },
     "SearcherConfig": {
       "properties": {
         "proxy": {
@@ -198,6 +212,14 @@
       },
       "type": "object"
     },
+    "WatcherConfig": {
+      "properties": {
+        "delay": {
+          "type": "number"
+        }
+      },
+      "type": "object"
+    },
     "ZincConfig": {
       "properties": {
         "domain": {

diff --git a/infra/root_chart/app/config.yaml b/infra/root_chart/app/config.yaml
@@ -7,6 +7,10 @@ app:
   service: helium
   module: pollee
   version: "1.0"
+  populator:
+    delay: 0.5
+  watcher:
+    delay: 1
 auth:
   descope:
     id: "P2Ya46UVwFV8nTwNGZWpxRtyL7Jj"

diff --git a/infra/root_chart/app/schema.json b/infra/root_chart/app/schema.json
@@ -12,6 +12,9 @@
         "platform": {
           "type": "string"
         },
+        "populator": {
+          "$ref": "#/definitions/PopulatorConfig"
+        },
         "searcher": {
           "$ref": "#/definitions/SearcherConfig"
         },
@@ -20,6 +23,9 @@
         },
         "version": {
           "type": "string"
+        },
+        "watcher": {
+          "$ref": "#/definitions/WatcherConfig"
         }
       },
       "type": "object"
@@ -182,6 +188,14 @@
       },
       "type": "object"
     },
+    "PopulatorConfig": {
+      "properties": {
+        "delay": {
+          "type": "number"
+        }
+      },
+      "type": "object"
+    },
     "SearcherConfig": {
       "properties": {
         "proxy": {
@@ -198,6 +212,14 @@
       },
       "type": "object"
     },
+    "WatcherConfig": {
+      "properties": {
+        "delay": {
+          "type": "number"
+        }
+      },
+      "type": "object"
+    },
     "ZincConfig": {
       "properties": {
         "domain": {

diff --git a/src/config/app.config.ts b/src/config/app.config.ts
@@ -1,6 +1,8 @@
 import { IsString, MinLength, ValidateNested } from "class-validator";
 import { Type } from "class-transformer";
 import { SearcherConfig } from "./searcher.config";
+import { PopulatorConfig } from "./populator.config";
+import { WatcherConfig } from "./watcher.config";
 
 export class AppConfig {
   @IsString()
@@ -26,4 +28,12 @@ export class AppConfig {
   @ValidateNested()
   @Type(() => SearcherConfig)
   searcher!: SearcherConfig;
+
+  @ValidateNested()
+  @Type(() => PopulatorConfig)
+  populator!: PopulatorConfig;
+
+  @ValidateNested()
+  @Type(() => WatcherConfig)
+  watcher!: WatcherConfig;
 }
diff --git a/src/config/populator.config.ts b/src/config/populator.config.ts
@@ -0,0 +1,7 @@
+import { IsInt, IsPositive } from "class-validator";
+
+export class PopulatorConfig {
+  @IsInt()
+  @IsPositive()
+  delay: number;
+}
diff --git a/src/config/watcher.config.ts b/src/config/watcher.config.ts
@@ -0,0 +1,7 @@
+import { IsInt, IsPositive } from "class-validator";
+
+export class WatcherConfig {
+  @IsInt()
+  @IsPositive()
+  delay: number;
+}
diff --git a/src/domain/search_core.ts b/src/domain/search_core.ts
@@ -5,6 +5,7 @@ import { stringify } from "querystring";
 import moment from "moment";
 import { SearcherConfig } from "../config/searcher.config.ts";
 import { Logger } from "pino";
+import { WatcherConfig } from "../config/watcher.config.ts";
 
 const f = fetchCookie(fetch);
 
@@ -80,13 +81,16 @@ class SearchCore {
 
   async mainKTMBPage(): Promise<MainPageToken> {
     const referer = "https://online.ktmb.com.my/";
+
+    const proxy1 = this.proxy;
+    this.#logger.info({ proxy: proxy1 }, "Using Proxy");
     const resp = await f("https://shuttleonline.ktmb.com.my/Home/Shuttle", {
       headers: {
         ...defaultHeaders,
         ...htmlHeaders,
         Referer: referer,
       },
-      proxy: this.proxy,
+      proxy: proxy1,
       method: "GET",
     });
     const text = await resp.text();
@@ -133,13 +137,16 @@ class SearchCore {
       PassengerCount: 1,
       __RequestVerificationToken: requestVerificationToken,
     };
+
+    const proxy1 = this.proxy;
+    this.#logger.info({ proxy: proxy1 }, "Using Proxy");
     const resp = await f("https://shuttleonline.ktmb.com.my/ShuttleTrip", {
       headers: {
         ...htmlHeaders,
         ...defaultHeaders,
         Referer: referer,
       },
-      proxy: this.proxy,
+      proxy: proxy1,
       body: stringify(queryParams),
       method: "POST",
     });
@@ -167,14 +174,16 @@ class SearchCore {
 
     const referer = "https://shuttleonline.ktmb.com.my/ShuttleTrip";
 
+    const proxy1 = this.proxy;
+    this.#logger.info({ proxy: proxy1 }, "Using Proxy");
     const init = {
       headers: {
         ...jsonHeaders,
         ...defaultHeaders,
         RequestVerificationToken: token,
         Referer: referer,
       },
-      proxy: this.proxy,
+      proxy: proxy1,
       body: JSON.stringify({
         SearchData: searchData,
         FormValidationCode: formValidation,

diff --git a/src/lib/populator.ts b/src/lib/populator.ts
@@ -2,9 +2,12 @@ import { Logger } from "pino";
 import { SearcherBuilder } from "../domain/searcher/builder.ts";
 import { addDays, differenceInDays } from "date-fns";
 import { RetrieveResult } from "./interfaces.ts";
+import { PopulatorConfig } from "../config/populator.config.ts";
+import { __ } from "../utility.ts";
 
 class Populator {
   constructor(
+    private readonly config: PopulatorConfig,
     private readonly logger: Logger,
     private readonly builder: SearcherBuilder,
   ) {}
@@ -34,6 +37,7 @@ class Populator {
 
         const j2wR = j2w.map((s) => `${s.departure_time}:00`);
         const w2jR = w2j.map((s) => `${s.departure_time}:00`);
+
         ret.push({
           date: currentDate,
           jToW: j2wR,
@@ -43,6 +47,7 @@ class Populator {
         this.logger.error({ error: e }, "Failed to retrieve schedule");
         break;
       }
+      await __(this.config.delay);
     }
     return ret;
   }

diff --git a/src/lib/watcher.ts b/src/lib/watcher.ts
@@ -2,9 +2,12 @@ import { Logger } from "pino";
 import Redis from "ioredis";
 import { SearcherBuilder } from "../domain/searcher/builder.ts";
 import { ZincDate } from "../util/zinc_date.ts";
+import { __ } from "../utility.ts";
+import { WatcherConfig } from "../config/watcher.config.ts";
 
 class Watcher {
   constructor(
+    private readonly config: WatcherConfig,
     private readonly logger: Logger,
     private readonly redis: Redis,
     private readonly builder: SearcherBuilder,
@@ -31,6 +34,7 @@ class Watcher {
         for (const s of sch) timing[s.departure_time] = s.available_seats;
         const key = `ktmb:schedule:${f}:${od}`;
         const _ = await this.redis.publish(key, JSON.stringify(timing));
+        await __(this.config.delay);
       } catch (e) {
         this.logger.error({ error: e }, "Failed to poll schedule");
         failureCount++;