diff --git a/.github/workflows/scraper.yaml b/.github/workflows/scraper.yaml index b95ba32..f1a7de9 100644 --- a/.github/workflows/scraper.yaml +++ b/.github/workflows/scraper.yaml @@ -1,10 +1,10 @@ -name: Run scraper and update SQLite database +name: Scraper on: -# push: -# branches: [ "main" ] schedule: - - cron: "45 02 * * THU" # at 02:45 every Thursday + - cron: "45 02 * * THU" # rerun at UTC 02:45 every Thursday + # allow manual trigger + workflow_dispatch: permissions: actions: write diff --git a/.github/workflows/cicd.yaml b/.github/workflows/tests.yaml similarity index 95% rename from .github/workflows/cicd.yaml rename to .github/workflows/tests.yaml index f473f51..cfe787d 100644 --- a/.github/workflows/cicd.yaml +++ b/.github/workflows/tests.yaml @@ -1,4 +1,4 @@ -name: Run CI/CD pipeline +name: Tests on: push: diff --git a/Makefile b/Makefile index 49e8f20..237f34b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ install: @echo ">>> Installing dependencies" - pip install --upgrade pip && pip install -r requirements.txt + pip install --upgrade pip + pip install -r requirements.txt format: @echo ">>> Formatting files using isort and Black" @@ -13,10 +14,9 @@ lint: lint-container: @echo ">>> Linting Dockerfile(s)" - docker run --rm -i hadolint/hadolint < Dockerfile + docker run --rm -i hadolint/hadolint < webapp/Dockerfile -refactor: - format lint lint-container +refactor: format lint lint-container coverage: @echo ">>> Displaying pytest coverage report" diff --git a/README.md b/README.md index f43af85..f322476 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,7 @@ # ⚽ Futsal Friend: Your Digital Futsal Companion -[![Run CI/CD pipeline](https://github.com/sborms/futsalfriend/actions/workflows/cicd.yaml/badge.svg)](https://github.com/sborms/futsalfriend/actions/workflows/cicd.yaml) +[![Tests](https://github.com/sborms/futsalfriend/actions/workflows/cicd.yaml/badge.svg)](https://github.com/sborms/futsalfriend/actions/workflows/tests.yaml) +[![Scraper](https://github.com/sborms/futsalfriend/actions/workflows/scraper.yaml/badge.svg)](https://github.com/sborms/futsalfriend/actions/workflows/scraper.yaml) For those in love with futsal as a way to enjoy sports and to maximize time with friends. diff --git a/requirements.txt b/requirements.txt index 0ac7ae6..7b0c458 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,4 @@ sqlalchemy geopy langchain openai -hugchat \ No newline at end of file +# hugchat \ No newline at end of file diff --git a/scraper/config/config_for_testing.json b/scraper/config/config_for_testing.json deleted file mode 100644 index 8c0b54b..0000000 --- a/scraper/config/config_for_testing.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "url_base": "https://www.lzvcup.be", - "steps": { - "historical_players": true, - "csv": false - }, - "areas": { - "BRUSSELS GEWEST": "results/2" - }, - "dir_output": "data", - "postprocessing": { - "competitions": [["area", "region", "competition"], []], - "teams": [["area", "region", "competition", "team"], []], - "sportshalls": [["area", "region", "sportshall", "url_sportshall"], []], - "stats_players": [["name", "team", "number", "url"], ["area", "region", "competition"]], - "stats_players_historical": [["name", "team", "seizoen", "reeks", "stand"], []], - "schedules": [["area", "region", "competition", "sportshall"], []], - "standings": [["area", "region", "competition", "team"], []], - "palmares": [["team", "seizoen", "reeks", "positie"], ["area", "region", "competition"]] - }, - "database": "database/futsalfriend.db", - "dir_last_updated": "webapp/last_updated.txt" -} \ No newline at end of file diff --git a/scraper/main.py b/scraper/main.py index 2fddabf..c9a50c9 100644 --- a/scraper/main.py +++ b/scraper/main.py @@ -159,7 +159,6 @@ def store(config, dict_tables, log_main): log.info(f"Running script from {DIR_SCRIPT}") config = DataStorage.load_json(f"{DIR_SCRIPT}/config/config.json") - # config = DataStorage.load_json(f"{DIR_SCRIPT}/config/config_for_testing.json") log.info("Config loaded") dict_tables = scrape(config, log_main=log)