Feature/improve tests

.github/workflows/test.yaml

@@ -0,0 +1,32 @@
+name: Run Tests
+
+on:
+  push:
+    branches:
+      - 'feature/**'
+      - 'hotfix/**'
+      - 'bugfix/**'
+  pull_request:
+    branches:
+      - 'feature/**'
+      - 'hotfix/**'
+      - 'bugfix/**'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+
+      - name: Install dependencies
+        run: npm install
+
+      - name: Run tests
+        run: npm test

.gitignore

@@ -2,4 +2,7 @@ node_modules/
 .env
 .npmrc
 data/
-next_matches/
+next_matches/
+repomix.config.json
+repomix-output.md
+.repomixignore

jest.config.cjs

@@ -1,7 +1,6 @@
-/** @type {import('jest').Config} */
-const config = {
-    transform: {
-    },
+module.exports = {
+    testEnvironment: 'node',
+    testMatch: ['**/tests/**/*.test.js'],
+    moduleFileExtensions: ['js'],
+    transform: {}
 };
-
-module.exports = config;

lib/puppeteer.js

@@ -1,49 +1,82 @@
 import puppeteerExtra from 'puppeteer-extra';
 import stealthPlugin from 'puppeteer-extra-plugin-stealth';
 import userAgent from 'user-agents';
-import { anonymizeProxy } from 'proxy-chain';
-
-// Original launchPuppeteer function
-puppeteerExtra.use(stealthPlugin());
-
-const configuration = [
-    "--unlimited-storage",
-    "--full-memory-crash-report",
-    "--disable-gpu",
-    "--ignore-certificate-errors",
-    "--no-sandbox",
-    "--disable-setuid-sandbox",
-    "--disable-dev-shm-usage",
-    "--lang=en-US;q=0.9,en;q=0.8",
-];
-
-if (process.env.ODDS_PORTAL_PROXY_URL) {
-    console.log("USING PROXY");
-    const oldProxyUrl = process.env.ODDS_PORTAL_PROXY_URL;
-    const newProxyUrl = await anonymizeProxy(oldProxyUrl);
-    configuration.push(`--proxy-server=${newProxyUrl}`);
-}
-
+import { anonymizeProxy, closeAnonymizedProxy } from 'proxy-chain';
+
+// Default configuration
+const DEFAULT_CONFIG = {
+    headless: 'new',
+    stealth: true,
+    args: [
+        "--unlimited-storage",
+        "--full-memory-crash-report",
+        "--disable-gpu",
+        "--ignore-certificate-errors",
+        "--no-sandbox",
+        "--disable-setuid-sandbox",
+        "--disable-dev-shm-usage",
+        "--lang=en-US;q=0.9,en;q=0.8"
+    ],
+    userAgent: new userAgent(),
+    proxy: process.env.ODDS_PORTAL_PROXY_URL || null,
+    anonymizedProxy: null
+};
 
 class StealthBrowser {
-    constructor() {
-        this.configuration = [...configuration];
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_CONFIG, ...config };
         this.browser = null;
+        this.pages = new Set();
 
+        if (this.config.stealth) {
+            puppeteerExtra.use(stealthPlugin());
+        }
     }
 
     async init() {
-        this.browser = await puppeteerExtra.launch({
-            headless: 'new',
-            args: [...configuration],
-        });
+        try {
+            // Handle proxy configuration
+            if (this.config.proxy) {
+                console.log("USING PROXY");
+                this.config.anonymizedProxy = await anonymizeProxy(this.config.proxy);
+                this.config.args.push(`--proxy-server=${this.config.anonymizedProxy}`);
+            }
+
+            this.browser = await puppeteerExtra.launch({
+                headless: this.config.headless,
+                args: this.config.args,
+                ignoreHTTPSErrors: true,
+                defaultViewport: null
+            });
+
+            // Validate browser instance
+            if (!this.browser || !this.browser.isConnected()) {
+                throw new Error('Browser failed to initialize');
+            }
+
+            return this.browser;
+        } catch (error) {
+            await this.cleanup();
+            throw new Error(`Browser initialization failed: ${error.message}`);
+        }
     }
 
     async newPage() {
-        const page = await this.browser.newPage();
-        await page.setUserAgent(userAgent.toString());
-        await this.pageConfiguration(page);
-        return page;
+        if (!this.browser || !this.browser.isConnected()) {
+            throw new Error('Browser not initialized');
+        }
+
+        try {
+            const page = await this.browser.newPage();
+            this.pages.add(page);
+
+            await page.setUserAgent(this.config.userAgent.toString());
+            await this.pageConfiguration(page);
+
+            return page;
+        } catch (error) {
+            throw new Error(`Failed to create new page: ${error.message}`);
+        }
     }
 
     async pageConfiguration(page) {
@@ -124,13 +157,39 @@ class StealthBrowser {
     }
 
     async close() {
-        await this.browser.close();
+        try {
+            // Close all pages
+            for (const page of this.pages) {
+                if (!page.isClosed()) {
+                    await page.close();
+                }
+            }
+            this.pages.clear();
+
+            // Close browser
+            if (this.browser && this.browser.isConnected()) {
+                await this.browser.close();
+            }
+
+            // Cleanup proxy
+            if (this.config.anonymizedProxy) {
+                await closeAnonymizedProxy(this.config.anonymizedProxy);
+            }
+        } catch (error) {
+            console.error('Error during cleanup:', error);
+        } finally {
+            this.browser = null;
+        }
+    }
+
+    async cleanup() {
+        await this.close();
     }
 }
 
 
-export default async function launchPuppeteer() {
-    const browser = new StealthBrowser()
+export default async function launchPuppeteer(config = {}) {
+    const browser = new StealthBrowser(config);
     await browser.init();
-    return browser
-}
+    return browser;
+}

lib/scrapers.js

@@ -1,6 +1,6 @@
 import logger from "./logger.js";
 import { leaguesUrlsMap, oddsFormatMap } from "./constants.js";
-import fs from 'fs'
+
 
 function getCurrentDateTimeString() {
     const now = new Date();

tests/cli.test.js

@@ -3,7 +3,6 @@ import fs from 'fs';
 import { exec } from 'child_process'
 import { promisify } from 'util';
 import { expect } from '@jest/globals';
-import { exportToDir } from '../lib/exporters';
 
 
 const promisifiedExec = promisify(exec);

tests/exporters.test.js

@@ -0,0 +1,82 @@
+import { exportToS3, exportToDir } from '../lib/exporters.js';
+import fs from 'fs';
+import path from 'path';
+import { jest } from '@jest/globals';
+import AWS from 'aws-sdk';
+
+describe('exportToS3', () => {
+    let mockPutObject;
+
+    beforeEach(() => {
+        mockPutObject = jest.fn((params, callback) => callback(null, {}));
+        AWS.S3 = jest.fn(() => ({
+            putObject: mockPutObject
+        }));
+    });
+
+    it('should export data to S3 successfully', async () => {
+        const bucketName = 'test-bucket';
+        const fileName = 'test.json';
+        const testData = { key: 'value' };
+
+        const exporter = exportToS3(bucketName);
+        await exporter(testData, fileName);
+
+        expect(mockPutObject).toHaveBeenCalledWith({
+            Bucket: bucketName,
+            Key: fileName,
+            Body: JSON.stringify(testData),
+            ContentType: 'application/json'
+        }, expect.any(Function));
+    });
+
+    it('should reject when S3 upload fails', async () => {
+        const error = new Error('S3 upload failed');
+        mockPutObject = jest.fn((params, callback) => callback(error, null));
+        AWS.S3 = jest.fn(() => ({
+            putObject: mockPutObject
+        }));
+
+        const exporter = exportToS3('test-bucket');
+        await expect(exporter({}, 'test.json')).rejects.toThrow(error);
+    });
+});
+
+describe('exportToDir', () => {
+    const testDir = 'test-output';
+
+    beforeEach(() => {
+        try {
+            fs.mkdirSync(testDir, { recursive: true });
+        } catch (err) {
+            // Ignore errors if directory already exists
+        }
+    });
+
+    afterEach(() => {
+        try {
+            fs.rmSync(testDir, { recursive: true, force: true });
+        } catch (err) {
+            // Ignore errors if directory doesn't exist
+        }
+    });
+
+    it('should export data to directory successfully', async () => {
+        const fileName = 'test.json';
+        const testData = { key: 'value' };
+        const expectedPath = path.join(testDir, fileName);
+
+        const exporter = exportToDir(testDir);
+        await exporter(testData, fileName);
+
+        const fileContent = fs.readFileSync(expectedPath, 'utf8');
+        expect(JSON.parse(fileContent)).toEqual(testData);
+    });
+
+    it('should reject when directory write fails', async () => {
+        const invalidDir = '/invalid/directory';
+        const exporter = exportToDir(invalidDir);
+
+        await expect(exporter({}, 'test.json')).rejects.toThrow();
+    });
+});

tests/puppeteer.test.js

@@ -0,0 +1,40 @@
+import launchPuppeteer from '../lib/puppeteer.js';
+import { anonymizeProxy } from 'proxy-chain';
+import { jest } from '@jest/globals';
+
+describe('StealthBrowser', () => {
+    let browser;
+
+    afterEach(async () => {
+        if (browser) {
+            await browser.cleanup();
+        }
+    });
+
+    it('should initialize browser with default configuration', async () => {
+        browser = await launchPuppeteer();
+
+        expect(browser).toBeDefined();
+    });
+
+    it('should use proxy configuration if provided', async () => {
+        browser = await launchPuppeteer({ proxy: 'http://test-proxy' });
+        expect(browser.config.proxy).toBe('http://test-proxy');
+
+    });
+
+    it('should create a new page with user agent and configurations', async () => {
+        browser = await launchPuppeteer();
+        const page = await browser.newPage();
+
+        expect(page).toBeDefined();
+    });
+
+    it('should close all pages and browser during cleanup', async () => {
+        browser = await launchPuppeteer();
+        const page = await browser.newPage();
+        await browser.cleanup();
+
+        expect(page.isClosed()).toBe(true);
+    });
+});

tests/scrapers.test.js

@@ -0,0 +1,21 @@
+import { jest } from '@jest/globals';
+import { nextMatchesScraper } from '../lib/scrapers.js';
+import launchPuppeteer from '../lib/puppeteer.js';
+
+describe('Scrapers Integration Tests', () => {
+    let browser;
+
+    beforeAll(async () => {
+        browser = await launchPuppeteer();
+    });
+
+    afterAll(async () => {
+        await browser.close();
+    });
+
+    it('should scrape next matches successfully', async () => {
+        const mockCallback = jest.fn();
+        await nextMatchesScraper(browser, 'liga-portugal', 'eu', mockCallback);
+        expect(mockCallback).toHaveBeenCalled();
+    }, 60000);
+});