Merge pull request #7 from ShahnurIslam/develop

Develop

.dockerignore

@@ -0,0 +1,4 @@
+.vscode/
+build
+node_modules
+.env

.gitignore

@@ -1,3 +1,5 @@
 # Dependencies
 node_modules/
-build/
+build/
+.vscode/
+.env

README.md

@@ -0,0 +1,15 @@
+To compile code into build
+tsc
+
+To run code we use `npm start`
+
+To build the docker container we use 
+`docker build -t ps5_scraper .`
+
+To run we use 
+`docker run --env-file .env node_test5`
+
+Gcloud deployment
+`tsc`
+`cd build`
+`gcloud functions deploy runBot --trigger-topic scrapr --runtime nodejs10 --env-vars-file .env.yaml`

config.json

@@ -0,0 +1,7 @@
+{
+    "url":{
+        "Amazon": "https://www.amazon.co.uk/dp/B08H95Y452/",
+        "ShopTo": "https://www.shopto.net/en/ps5hw01-playstation-5-console-p191472/",
+        "Game": "https://www.game.co.uk/en/playstation-5-console-2826338"
+    }
+}

deployment.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: "apps/v1"
+kind: "Deployment"
+metadata:
+  name: "ps5-scraper"
+  namespace: "default"
+  labels:
+    app: "ps5-scraper"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: "ps5-scraper"
+  template:
+    metadata:
+      labels:
+        app: "ps5-scraper"
+    spec:
+      containers:
+      - name: "ps5-scraper-sha256-1"
+        image: "eu.gcr.io/e-resolver-294514/ps5_scraper@sha256:5057c299a63ba09f2395a6db3b401f92bd0ab21bae0da790ffc6a23e2227d4cf"
+

dockerfile

@@ -0,0 +1,16 @@
+FROM node:alpine
+
+# Create our working directory
+WORKDIR /usr/ps5_scraper
+
+# Copy over the package.json to this directory
+COPY package.json .
+
+#Install our packages
+RUN npm install
+
+# Add the current contents of our code to this directory
+ADD . /usr/ps5_scraper/
+
+# Start the code
+CMD [ "npm", "start" ]

index.ts

@@ -1 +1,16 @@
-console.log("hello")
+import {Bot} from "./src/Bot"
+import {Logger,LogLevel} from './src/Logger';
+
+// Start our Bot
+// new Bot(new Logger(LogLevel.Info)).start();
+
+
+
+
+function bot(){
+    new Bot(new Logger(LogLevel.Info)).start()
+};
+
+exports.runBot = bot;
+
+// bot()

package.json

@@ -0,0 +1,21 @@
+{
+  "name": "ps5_scraper",
+  "version": "1.0.0",
+  "description": "To compile code into build tsc",
+  "main": "index.js",
+  "scripts": {
+    "start": "ts-node index.ts"
+  },
+  "author": "Shan Islam",
+  "license": "ISC",
+  "devDependencies": {
+    "ts-node": "^9.0.0",
+    "typescript": "^4.0.5"
+  },
+  "dependencies": {
+    "@types/axios": "^0.14.0",
+    "axios": "^0.20.0",
+    "cheerio": "^1.0.0-rc.3",
+    "nodemailer": "^6.4.13"
+  }
+}

src/Bot.ts

@@ -0,0 +1,71 @@
+import {Logger,LogLevel} from "./Logger"
+import { AmazonUK } from "./Crawlers/AmazonUk"
+import { ShopTo } from "./Crawlers/ShopTo"
+import { Game } from "./Crawlers/Game"
+import { Crawler } from './Crawlers/Crawler'
+import { Email } from './Notifications/Mailer'
+
+export class Bot {
+    stock_dict: any
+    email: Email
+    msg: string
+
+    constructor(private readonly logger: Logger){
+        this.stock_dict = {}
+        this.email = new Email
+        this.msg = ""
+    }
+
+    private async scrape_site(cr:Crawler){
+        this.logger.info(`Starting Crawler on site ${cr.getRetailerName()}`)
+        this.stock_dict[cr.getRetailerName()] = await cr.getStock(this.logger)
+        // this.msg += '\n' + cr.getRetailerName() + ' has stock at url: ' + cr.getUrl()
+        if (this.stock_dict[cr.getRetailerName()] === true){
+            if (this.msg == ""){
+                this.msg = cr.getRetailerName()  + ' has stock <a href="' + cr.getUrl() + '">here</a><br/>'
+            } else {
+                this.msg +=  cr.getRetailerName()  + ' has stock <a href="' + cr.getUrl() + '">here</a><br/>'
+            }
+
+        }
+    }
+
+    private async scrape_site2(cr:Crawler){
+        this.logger.info(`Starting Crawler on site ${cr.getRetailerName()}`)
+        this.stock_dict[cr.getRetailerName()] = cr
+    }
+
+    check_stock(st_dict){
+        var filtered = Object.keys(st_dict).reduce(function (filtered, key) {
+            if (st_dict[key] === true) filtered[key] = st_dict[key];
+            return filtered;
+        }, {});
+        const len = Object.keys(filtered).length
+        return len
+    }
+
+    email_notification(st_dict){
+        if(this.check_stock(st_dict) > 0){
+            var filtered: { [characterName: string]: boolean} = Object.keys(st_dict).reduce(function (filtered, key) {
+                if (st_dict[key] === true) filtered[key] = st_dict[key];
+                return filtered;
+            }, {});
+
+            this.email.main(this.msg)
+
+        } else{
+            this.logger.info("No stock anywhere")
+        }
+    }
+
+    public async start(){
+        this.logger.info("Starting bot scraper")
+        await this.scrape_site(new AmazonUK)
+        await this.scrape_site(new ShopTo)
+        await this.scrape_site(new Game)
+        // this.stock_dict['test'] = true
+        console.log(this.stock_dict)
+        this.email_notification(this.stock_dict)
+    }
+
+}

src/Crawler.ts

@@ -0,0 +1,20 @@
+
+
+
+function resolveAfter2Seconds() {
+    return new Promise(resolve => {
+      setTimeout(() => {
+        resolve('resolved');
+      }, 10000);
+    });
+  }
+
+
+
+async function crawler() {
+    console.log('calling');
+    const result = await resolveAfter2Seconds();
+    console.log(result);
+    // expected output: "resolved"
+  }
+