diff --git a/config/app/config.yaml b/config/app/config.yaml index 2619c19..c491143 100644 --- a/config/app/config.yaml +++ b/config/app/config.yaml @@ -7,6 +7,10 @@ app: service: helium module: pollee version: "1.0" + populator: + delay: 0.5 + watcher: + delay: 1 auth: descope: id: "P2Ya46UVwFV8nTwNGZWpxRtyL7Jj" diff --git a/config/app/schema.json b/config/app/schema.json index 8fd654f..974db83 100644 --- a/config/app/schema.json +++ b/config/app/schema.json @@ -12,6 +12,9 @@ "platform": { "type": "string" }, + "populator": { + "$ref": "#/definitions/PopulatorConfig" + }, "searcher": { "$ref": "#/definitions/SearcherConfig" }, @@ -20,6 +23,9 @@ }, "version": { "type": "string" + }, + "watcher": { + "$ref": "#/definitions/WatcherConfig" } }, "type": "object" @@ -182,6 +188,14 @@ }, "type": "object" }, + "PopulatorConfig": { + "properties": { + "delay": { + "type": "number" + } + }, + "type": "object" + }, "SearcherConfig": { "properties": { "proxy": { @@ -198,6 +212,14 @@ }, "type": "object" }, + "WatcherConfig": { + "properties": { + "delay": { + "type": "number" + } + }, + "type": "object" + }, "ZincConfig": { "properties": { "domain": { diff --git a/infra/cron_chart/app/config.yaml b/infra/cron_chart/app/config.yaml index 2619c19..c491143 100644 --- a/infra/cron_chart/app/config.yaml +++ b/infra/cron_chart/app/config.yaml @@ -7,6 +7,10 @@ app: service: helium module: pollee version: "1.0" + populator: + delay: 0.5 + watcher: + delay: 1 auth: descope: id: "P2Ya46UVwFV8nTwNGZWpxRtyL7Jj" diff --git a/infra/cron_chart/app/schema.json b/infra/cron_chart/app/schema.json index 8fd654f..974db83 100644 --- a/infra/cron_chart/app/schema.json +++ b/infra/cron_chart/app/schema.json @@ -12,6 +12,9 @@ "platform": { "type": "string" }, + "populator": { + "$ref": "#/definitions/PopulatorConfig" + }, "searcher": { "$ref": "#/definitions/SearcherConfig" }, @@ -20,6 +23,9 @@ }, "version": { "type": "string" + }, + "watcher": { + "$ref": "#/definitions/WatcherConfig" } }, "type": "object" @@ -182,6 +188,14 @@ }, "type": "object" }, + "PopulatorConfig": { + "properties": { + "delay": { + "type": "number" + } + }, + "type": "object" + }, "SearcherConfig": { "properties": { "proxy": { @@ -198,6 +212,14 @@ }, "type": "object" }, + "WatcherConfig": { + "properties": { + "delay": { + "type": "number" + } + }, + "type": "object" + }, "ZincConfig": { "properties": { "domain": { diff --git a/infra/root_chart/app/config.yaml b/infra/root_chart/app/config.yaml index 2619c19..c491143 100644 --- a/infra/root_chart/app/config.yaml +++ b/infra/root_chart/app/config.yaml @@ -7,6 +7,10 @@ app: service: helium module: pollee version: "1.0" + populator: + delay: 0.5 + watcher: + delay: 1 auth: descope: id: "P2Ya46UVwFV8nTwNGZWpxRtyL7Jj" diff --git a/infra/root_chart/app/schema.json b/infra/root_chart/app/schema.json index 8fd654f..974db83 100644 --- a/infra/root_chart/app/schema.json +++ b/infra/root_chart/app/schema.json @@ -12,6 +12,9 @@ "platform": { "type": "string" }, + "populator": { + "$ref": "#/definitions/PopulatorConfig" + }, "searcher": { "$ref": "#/definitions/SearcherConfig" }, @@ -20,6 +23,9 @@ }, "version": { "type": "string" + }, + "watcher": { + "$ref": "#/definitions/WatcherConfig" } }, "type": "object" @@ -182,6 +188,14 @@ }, "type": "object" }, + "PopulatorConfig": { + "properties": { + "delay": { + "type": "number" + } + }, + "type": "object" + }, "SearcherConfig": { "properties": { "proxy": { @@ -198,6 +212,14 @@ }, "type": "object" }, + "WatcherConfig": { + "properties": { + "delay": { + "type": "number" + } + }, + "type": "object" + }, "ZincConfig": { "properties": { "domain": { diff --git a/src/config/app.config.ts b/src/config/app.config.ts index 86ba210..10a7dca 100644 --- a/src/config/app.config.ts +++ b/src/config/app.config.ts @@ -1,6 +1,8 @@ import { IsString, MinLength, ValidateNested } from "class-validator"; import { Type } from "class-transformer"; import { SearcherConfig } from "./searcher.config"; +import { PopulatorConfig } from "./populator.config"; +import { WatcherConfig } from "./watcher.config"; export class AppConfig { @IsString() @@ -26,4 +28,12 @@ export class AppConfig { @ValidateNested() @Type(() => SearcherConfig) searcher!: SearcherConfig; + + @ValidateNested() + @Type(() => PopulatorConfig) + populator!: PopulatorConfig; + + @ValidateNested() + @Type(() => WatcherConfig) + watcher!: WatcherConfig; } diff --git a/src/config/populator.config.ts b/src/config/populator.config.ts new file mode 100644 index 0000000..09a0bae --- /dev/null +++ b/src/config/populator.config.ts @@ -0,0 +1,7 @@ +import { IsInt, IsPositive } from "class-validator"; + +export class PopulatorConfig { + @IsInt() + @IsPositive() + delay: number; +} diff --git a/src/config/watcher.config.ts b/src/config/watcher.config.ts new file mode 100644 index 0000000..2b51fe0 --- /dev/null +++ b/src/config/watcher.config.ts @@ -0,0 +1,7 @@ +import { IsInt, IsPositive } from "class-validator"; + +export class WatcherConfig { + @IsInt() + @IsPositive() + delay: number; +} diff --git a/src/domain/search_core.ts b/src/domain/search_core.ts index 19eae63..6bcb2b7 100644 --- a/src/domain/search_core.ts +++ b/src/domain/search_core.ts @@ -5,6 +5,7 @@ import { stringify } from "querystring"; import moment from "moment"; import { SearcherConfig } from "../config/searcher.config.ts"; import { Logger } from "pino"; +import { WatcherConfig } from "../config/watcher.config.ts"; const f = fetchCookie(fetch); @@ -80,13 +81,16 @@ class SearchCore { async mainKTMBPage(): Promise { const referer = "https://online.ktmb.com.my/"; + + const proxy1 = this.proxy; + this.#logger.info({ proxy: proxy1 }, "Using Proxy"); const resp = await f("https://shuttleonline.ktmb.com.my/Home/Shuttle", { headers: { ...defaultHeaders, ...htmlHeaders, Referer: referer, }, - proxy: this.proxy, + proxy: proxy1, method: "GET", }); const text = await resp.text(); @@ -133,13 +137,16 @@ class SearchCore { PassengerCount: 1, __RequestVerificationToken: requestVerificationToken, }; + + const proxy1 = this.proxy; + this.#logger.info({ proxy: proxy1 }, "Using Proxy"); const resp = await f("https://shuttleonline.ktmb.com.my/ShuttleTrip", { headers: { ...htmlHeaders, ...defaultHeaders, Referer: referer, }, - proxy: this.proxy, + proxy: proxy1, body: stringify(queryParams), method: "POST", }); @@ -167,6 +174,8 @@ class SearchCore { const referer = "https://shuttleonline.ktmb.com.my/ShuttleTrip"; + const proxy1 = this.proxy; + this.#logger.info({ proxy: proxy1 }, "Using Proxy"); const init = { headers: { ...jsonHeaders, @@ -174,7 +183,7 @@ class SearchCore { RequestVerificationToken: token, Referer: referer, }, - proxy: this.proxy, + proxy: proxy1, body: JSON.stringify({ SearchData: searchData, FormValidationCode: formValidation, diff --git a/src/lib/populator.ts b/src/lib/populator.ts index fb91c70..bac8dc8 100644 --- a/src/lib/populator.ts +++ b/src/lib/populator.ts @@ -2,9 +2,12 @@ import { Logger } from "pino"; import { SearcherBuilder } from "../domain/searcher/builder.ts"; import { addDays, differenceInDays } from "date-fns"; import { RetrieveResult } from "./interfaces.ts"; +import { PopulatorConfig } from "../config/populator.config.ts"; +import { __ } from "../utility.ts"; class Populator { constructor( + private readonly config: PopulatorConfig, private readonly logger: Logger, private readonly builder: SearcherBuilder, ) {} @@ -34,6 +37,7 @@ class Populator { const j2wR = j2w.map((s) => `${s.departure_time}:00`); const w2jR = w2j.map((s) => `${s.departure_time}:00`); + ret.push({ date: currentDate, jToW: j2wR, @@ -43,6 +47,7 @@ class Populator { this.logger.error({ error: e }, "Failed to retrieve schedule"); break; } + await __(this.config.delay); } return ret; } diff --git a/src/lib/watcher.ts b/src/lib/watcher.ts index 4d69ad9..e16acea 100644 --- a/src/lib/watcher.ts +++ b/src/lib/watcher.ts @@ -2,9 +2,12 @@ import { Logger } from "pino"; import Redis from "ioredis"; import { SearcherBuilder } from "../domain/searcher/builder.ts"; import { ZincDate } from "../util/zinc_date.ts"; +import { __ } from "../utility.ts"; +import { WatcherConfig } from "../config/watcher.config.ts"; class Watcher { constructor( + private readonly config: WatcherConfig, private readonly logger: Logger, private readonly redis: Redis, private readonly builder: SearcherBuilder, @@ -31,6 +34,7 @@ class Watcher { for (const s of sch) timing[s.departure_time] = s.available_seats; const key = `ktmb:schedule:${f}:${od}`; const _ = await this.redis.publish(key, JSON.stringify(timing)); + await __(this.config.delay); } catch (e) { this.logger.error({ error: e }, "Failed to poll schedule"); failureCount++;