diff --git a/.github/workflows/cd-backend.yaml b/.github/workflows/cd-backend.yaml index 1c947be..99fb918 100644 --- a/.github/workflows/cd-backend.yaml +++ b/.github/workflows/cd-backend.yaml @@ -36,6 +36,10 @@ jobs: run: | pip3 install --quiet -r requirements.txt -r tests/requirements.txt + - name: 'Static analysis (Lint)' + working-directory: ./backend + run: ./run_lint.sh + - name: 'Run tests and check coverage' working-directory: ./backend run: | diff --git a/backend/job_fetch_offers.py b/backend/job_fetch_offers.py index 7cde5c0..7278e4e 100644 --- a/backend/job_fetch_offers.py +++ b/backend/job_fetch_offers.py @@ -1,7 +1,6 @@ from datetime import datetime from scrapy.utils.project import get_project_settings -from twisted.internet import reactor -from scrapy.crawler import CrawlerRunner +from scrapy.crawler import CrawlerProcess import pprint from my_logging import * @@ -13,7 +12,7 @@ if __name__ == '__main__': try: settings = get_project_settings() - runner = CrawlerRunner(settings) + process = CrawlerProcess(settings) spiders = { SoaringDeSpider.SoaringDeSpider: None, @@ -21,13 +20,11 @@ #PlaneCheckComSpider.PlaneCheckComSpider: None } for spider_cls in spiders.keys(): - crawler = runner.create_crawler(spider_cls) + crawler = process.create_crawler(spider_cls) spiders[spider_cls] = crawler - runner.crawl(crawler) + process.crawl(crawler) - d = runner.join() - d.addBoth(lambda _: reactor.stop()) - reactor.run() # the script will block here until all crawling jobs are finished + process.start() # the script will block here until all crawling jobs are finished stats_per_spider = {} diff --git a/backend/mailer.py b/backend/mailer.py index c61b5ba..f3414d8 100644 --- a/backend/mailer.py +++ b/backend/mailer.py @@ -9,8 +9,7 @@ def send_mail(text=""): if not SEND_RESULT_MAIL: return msg = email.mime.text.MIMEText(text) - # TODO put your mail address here - # me = u'ralf.thaenert@googlemail.com' + me = 'dev@aerooffers.pl' msg['Subject'] = 'Aircraft Offers Crawling Result' msg['From'] = SMTP_USER msg['To'] = me diff --git a/backend/requirements.txt b/backend/requirements.txt index 991b895..0b15cf3 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,6 +1,7 @@ coverage==7.6.0 Twisted==24.3.0 psycopg2-binary==2.9.9 +pylint==3.2.6 SQLAlchemy==2.0.31 price-parser==0.3.4 Scrapy==2.11.2 diff --git a/backend/run_lint.sh b/backend/run_lint.sh new file mode 100755 index 0000000..1dbb35a --- /dev/null +++ b/backend/run_lint.sh @@ -0,0 +1,5 @@ +export PYTHONPATH=$PYTHONPATH':./' + +set -e + +pylint --fail-on=E --errors-only ./ \ No newline at end of file diff --git a/backend/run_tests.sh b/backend/run_tests.sh index 2cef050..ef38f17 100755 --- a/backend/run_tests.sh +++ b/backend/run_tests.sh @@ -1,9 +1,6 @@ export PYTHONPATH=$PYTHONPATH':./' -coverage run --source ./ -m xmlrunner -o ./test-results +set -e -if [[ $? -ne 0 ]]; then - exit 1 -else - coverage report --fail-under=75 -fi \ No newline at end of file +coverage run --source ./ --omit="tests/*" -m xmlrunner -o ./test-results +coverage report --fail-under=65 \ No newline at end of file