More types for providers

orangecoding · Feb 4, 2024 · 3fa6409 · 3fa6409
1 parent 9a6c6e9
commit 3fa6409
Show file tree

Hide file tree

Showing 18 changed files with 131 additions and 97 deletions.
diff --git a/index.js → index.ts b/index.js → index.ts
@@ -6,20 +6,16 @@ import * as jobStorage from './lib/services/storage/jobStorage.js';
 import FredyRuntime from './lib/FredyRuntime.js';
 import { duringWorkingHoursOrNotSet } from './lib/utils.js';
 import './lib/api/api.js';
+import { ProviderJobInformation, providers } from './lib/provider/provider.js';
 //if db folder does not exist, ensure to create it before loading anything else
 if (!fs.existsSync('./db')) {
   fs.mkdirSync('./db');
 }
-const path = './lib/provider';
-const provider = fs.readdirSync(path).filter((file) => file.endsWith('.js'));
 //assuming interval is always in minutes
 const INTERVAL = config.interval * 60 * 1000;
 /* eslint-disable no-console */
 console.log(`Started Fredy successfully. Ui can be accessed via http://localhost:${config.port}`);
 /* eslint-enable no-console */
-const fetchedProvider = await Promise.all(
-  provider.filter((provider) => provider.endsWith('.js')).map(async (pro) => import(`${path}/${pro}`))
-);
 
 setInterval(
   (function exec() {
@@ -30,21 +26,27 @@ setInterval(
         .getJobs()
         .filter((job) => job.enabled)
         .forEach((job) => {
-          job.provider
-            .filter((p) => fetchedProvider.find((fp) => fp.metaInformation.id === p.id) != null)
-            .forEach(async (prov) => {
-              const pro = fetchedProvider.find((fp) => fp.metaInformation.id === prov.id);
-              pro.init(prov, job.blacklist);
-              await new FredyRuntime(
-                pro.config,
-                job.notificationAdapter,
-                prov.id,
-                job.id,
-                similarityCache,
-                job.listingProcessors
-              ).execute();
-              setLastJobExecution(job.id);
-            });
+          const validJobProviders: ProviderJobInformation[] = job.provider.filter(
+            (provider: ProviderJobInformation) => {
+              const hasExistingProvider =
+                providers.find((loadedProvider) => loadedProvider.metaInformation.id === provider.id) != null;
+              return hasExistingProvider;
+            }
+          );
+          validJobProviders.forEach(async (jobProvider) => {
+            const provider = providers.find((provider) => provider.metaInformation.id === jobProvider.id)!;
+            provider.init(jobProvider, job.blacklist);
+
+            await new FredyRuntime(
+              provider.config,
+              job.notificationAdapter,
+              jobProvider.id,
+              job.id,
+              similarityCache,
+              job.listingProcessors
+            ).execute();
+            setLastJobExecution(job.id);
+          });
         });
     } else {
       /* eslint-disable no-console */

diff --git a/lib/FredyRuntime.ts b/lib/FredyRuntime.ts
@@ -14,7 +14,7 @@ class FredyRuntime {
   private providerId: string;
   private jobKey: string;
   private similarityCache: SimilarityCacheService;
-  private listingProcessors: process.ProcessorConfig[];
+  private listingProcessors?: process.ProcessorConfig[];
 
   /**
    *
@@ -30,14 +30,15 @@ class FredyRuntime {
     providerId: string,
     jobKey: string,
     similarityCache: SimilarityCacheService,
-    listingProcessors: process.ProcessorConfig[]
+    listingProcessors?: process.ProcessorConfig[]
   ) {
     this.providerConfig = providerConfig;
     this.notificationAdapterConfigs = notificationConfig;
     this.providerId = providerId;
     this.jobKey = jobKey;
     this.similarityCache = similarityCache;
     this.listingProcessors = listingProcessors;
+    console.log('Setup freddy runtime');
   }
   execute() {
     return (
@@ -69,40 +70,35 @@ class FredyRuntime {
         const error = 'Immoscout or Immonet can only be used with if you have set an apikey for scrapingAnt.';
         /* eslint-disable no-console */
         console.log(error);
+
         /* eslint-enable no-console */
         reject(error);
         return;
       }
       const u = scrapingAnt.needScrapingAnt(id) ? scrapingAnt.transformUrlForScrapingAnt(url, id) : url;
       try {
+        const xrayPromise = xray(u, this.providerConfig.crawlContainer, [this.providerConfig.crawlFields]);
+
         if (this.providerConfig.paginate != null) {
-          xray(u, this.providerConfig.crawlContainer, [this.providerConfig.crawlFields])
-            //the first 2 pages should be enough here
-            .limit(2)
-            .paginate(this.providerConfig.paginate)
-            .then((listings) => {
-              resolve(listings == null ? [] : listings);
-            })
-            .catch((err) => {
-              reject(err);
-              console.error(err);
-            });
-        } else {
-          xray(u, this.providerConfig.crawlContainer, [this.providerConfig.crawlFields])
-            .then((listings) => {
-              resolve(listings == null ? [] : listings);
-            })
-            .catch((err) => {
-              reject(err);
-              console.error(err);
-            });
+          //the first 2 pages should be enough here
+          xrayPromise.limit(2).paginate(this.providerConfig.paginate);
         }
+
+        xrayPromise
+          .then((listings) => {
+            resolve(listings == null ? [] : listings);
+          })
+          .catch((err) => {
+            reject(err);
+            console.error(err);
+          });
       } catch (error) {
         reject(error);
         console.error(error);
       }
     });
   }
+
   _normalize(listings: Listing[]): Listing[] {
     return listings.map(this.providerConfig.normalize);
   }
@@ -139,6 +135,7 @@ class FredyRuntime {
   _filterBySimilarListings(listings: Listing[]): Listing[] {
     const filteredList = listings.filter((listing) => {
       const similar = this.similarityCache.hasSimilarEntries(this.jobKey, listing.title);
+
       if (similar) {
         /* eslint-disable no-console */
         console.debug(`Filtering similar entry for job with id ${this.jobKey} with title: `, listing.title);

diff --git a/lib/processors/process.ts b/lib/processors/process.ts
@@ -12,7 +12,8 @@ const findProcessor = (processor: ProcessorConfig): Processor => {
   return new ProcessorFile.default();
 };
 
-export function processListings(listings: Listing[], listingProcessorsConfig: ProcessorConfig[]) {
+export function processListings(listings: Listing[], listingProcessorsConfig: ProcessorConfig[]): Promise<Listing[]> {
+  if (!listingProcessorsConfig || listingProcessorsConfig.length === 0) return Promise.resolve(listings);
   const processors = listingProcessorsConfig.map(findProcessor);
   const processedListingsPromises = listings.map(async (listing) => processListingByAllProcessors(listing, processors));
 
@@ -28,7 +29,8 @@ const processListingByAllProcessors = async (listing: Listing, processors: Proce
 
 const processListingByOneProcessor = async (listing: Listing, processor: Processor): Promise<Listing> => {
   const processedListing = await processor.processListing({ listing });
-  const updatedNotificationText = processedListing.notificationText + processor.notificationText({ listing });
+  const previousNotificationText = processedListing.notificationText || '';
+  const updatedNotificationText = previousNotificationText + processor.notificationText({ listing: processedListing });
   return { ...processedListing, notificationText: updatedNotificationText };
 };
 

diff --git a/lib/provider/einsAImmobilien.ts b/lib/provider/einsAImmobilien.ts
@@ -1,15 +1,15 @@
 import utils from '../utils.js';
-import { ProviderConfig } from './provider.js';
+import { Listing, ProviderConfig, ProviderJobInformation } from './provider.js';
 let appliedBlackList = [];
-function normalize(o) {
+function normalize(o: Listing): Listing {
   let size = `${o.size.replace(' Wohnfläche ', '').trim()}`;
   if (o.rooms != null) {
     size += ` / / ${o.rooms.trim()}`;
   }
   const link = `https://www.1a-immobilienmarkt.de/expose/${o.id}.html`;
   return Object.assign(o, { size, link });
 }
-function applyBlacklist(o) {
+function applyBlacklist(o: Listing): boolean {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
@@ -29,8 +29,7 @@ const config: ProviderConfig = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+export const init = (sourceConfig: ProviderJobInformation, blacklist) => {
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };

diff --git a/lib/provider/immobilienDe.ts b/lib/provider/immobilienDe.ts
@@ -1,13 +1,13 @@
 import utils from '../utils.js';
-import { ProviderConfig } from './provider.js';
+import { Listing, ProviderConfig, ProviderJobInformation } from './provider.js';
 let appliedBlackList = [];
 function shortenLink(link) {
   return link.substring(0, link.indexOf('?'));
 }
 function parseId(shortenedLink) {
   return shortenedLink.substring(shortenedLink.lastIndexOf('/') + 1);
 }
-function normalize(o) {
+function normalize(o: Listing): Listing {
   const id = parseId(shortenLink(o.link));
   const size = o.size || 'N/A m²';
   const price = o.price || 'N/A €';
@@ -16,7 +16,7 @@ function normalize(o) {
   const link = shortenLink(o.link);
   return Object.assign(o, { id, price, size, title, address, link });
 }
-function applyBlacklist(o) {
+function applyBlacklist(o: Listing): boolean {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
@@ -37,8 +37,7 @@ const config: ProviderConfig = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+export const init = (sourceConfig: ProviderJobInformation, blacklist) => {
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };

diff --git a/lib/provider/immonet.ts b/lib/provider/immonet.ts
@@ -1,7 +1,7 @@
 import utils from '../utils.js';
-import { ProviderConfig } from './provider.js';
+import { Listing, ProviderConfig, ProviderJobInformation } from './provider.js';
 let appliedBlackList = [];
-function normalize(o) {
+function normalize(o: Listing): Listing {
   const id = o.id.substring(o.id.lastIndexOf('/') + 1, o.id.length);
   const size = o.size != null ? o.size.replace('Wohnfläche ', '') : 'N/A m²';
   const price = o.price.replace('Kaufpreis ', '');
@@ -10,7 +10,7 @@ function normalize(o) {
   const link = o.id;
   return Object.assign(o, { id, address, price, size, title, link });
 }
-function applyBlacklist(o) {
+function applyBlacklist(o: Listing): boolean {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
@@ -30,8 +30,7 @@ const config: ProviderConfig = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+export const init = (sourceConfig: ProviderJobInformation, blacklist) => {
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };

diff --git a/lib/provider/immoscout.ts b/lib/provider/immoscout.ts
@@ -1,16 +1,16 @@
 import utils from '../utils.js';
-import { ProviderConfig } from './provider.js';
+import { Listing, ProviderConfig, ProviderJobInformation } from './provider.js';
 let appliedBlackList = [];
 function nullOrEmpty(val) {
   return val == null || val.length === 0;
 }
-function normalize(o) {
+function normalize(o: Listing): Listing {
   const title = nullOrEmpty(o.title) ? 'NO TITLE FOUND' : o.title.replace('NEU', '');
   const address = nullOrEmpty(o.address) ? 'NO ADDRESS FOUND' : (o.address || '').replace(/\(.*\),.*$/, '').trim();
   const link = `https://www.immobilienscout24.de${o.link.substring(o.link.indexOf('/expose'))}`;
   return Object.assign(o, { title, address, link });
 }
-function applyBlacklist(o) {
+function applyBlacklist(o: Listing): boolean {
   return !utils.isOneOf(o.title, appliedBlackList);
 }
 const config: ProviderConfig = {
@@ -29,8 +29,7 @@ const config: ProviderConfig = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+export const init = (sourceConfig: ProviderJobInformation, blacklist) => {
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };

diff --git a/lib/provider/immoswp.ts b/lib/provider/immoswp.ts
@@ -1,7 +1,7 @@
 import utils from '../utils.js';
-import { ProviderConfig } from './provider.js';
+import { Listing, ProviderConfig, ProviderJobInformation } from './provider.js';
 let appliedBlackList = [];
-function normalize(o) {
+function normalize(o: Listing): Listing {
   const id = o.id.substring(o.id.indexOf('-') + 1, o.id.length);
   const size = o.size || 'N/A m²';
   const price = (o.price || '--- €').replace('Preis auf Anfrage', '--- €');
@@ -11,7 +11,7 @@ function normalize(o) {
   const description = o.description;
   return Object.assign(o, { id, address, price, size, title, link, description });
 }
-function applyBlacklist(o) {
+function applyBlacklist(o: Listing): boolean {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
@@ -32,8 +32,7 @@ const config: ProviderConfig = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+export const init = (sourceConfig: ProviderJobInformation, blacklist) => {
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };

diff --git a/lib/provider/immowelt.ts b/lib/provider/immowelt.ts
@@ -1,10 +1,10 @@
 import utils from '../utils.js';
-import { ProviderConfig } from './provider.js';
+import { Listing, ProviderConfig, ProviderJobInformation } from './provider.js';
 let appliedBlackList = [];
-function normalize(o) {
+function normalize(o: Listing): Listing {
   return o;
 }
-function applyBlacklist(o) {
+function applyBlacklist(o: Listing): boolean {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   return titleNotBlacklisted && descNotBlacklisted;
@@ -25,8 +25,7 @@ const config: ProviderConfig = {
   normalize: normalize,
   filter: applyBlacklist,
 };
-export const init = (sourceConfig, blacklist) => {
-  config.enabled = sourceConfig.enabled;
+export const init = (sourceConfig: ProviderJobInformation, blacklist) => {
   config.url = sourceConfig.url;
   appliedBlackList = blacklist || [];
 };

diff --git a/lib/provider/kleinanzeigen.ts b/lib/provider/kleinanzeigen.ts
@@ -1,12 +1,12 @@
 import utils from '../utils.js';
-import { ProviderConfig } from './provider.js';
+import { Listing, ProviderConfig, ProviderJobInformation } from './provider.js';
 let appliedBlackList = [];
 let appliedBlacklistedDistricts = [];
-function normalize(o) {
+function normalize(o: Listing): Listing {
   const size = o.size || '--- m²';
   return Object.assign(o, { size });
 }
-function applyBlacklist(o) {
+function applyBlacklist(o: Listing): boolean {
   const titleNotBlacklisted = !utils.isOneOf(o.title, appliedBlackList);
   const descNotBlacklisted = !utils.isOneOf(o.description, appliedBlackList);
   const isBlacklistedDistrict =
@@ -36,8 +36,7 @@ export const metaInformation = {
   baseUrl: 'https://www.kleinanzeigen.de/',
   id: 'kleinanzeigen',
 };
-export const init = (sourceConfig, blacklist, blacklistedDistricts) => {
-  config.enabled = sourceConfig.enabled;
+export const init = (sourceConfig: ProviderJobInformation, blacklist, blacklistedDistricts) => {
   config.url = sourceConfig.url;
   appliedBlacklistedDistricts = blacklistedDistricts || [];
   appliedBlackList = blacklist || [];