Dev: added backend code linter (pylint)

.github/workflows/cd-backend.yaml

@@ -36,6 +36,10 @@ jobs:
         run: |
           pip3 install --quiet -r requirements.txt -r tests/requirements.txt
 
+      - name: 'Static analysis (Lint)'
+        working-directory: ./backend
+        run: ./run_lint.sh
+
       - name: 'Run tests and check coverage'
         working-directory: ./backend
         run: |

backend/job_fetch_offers.py

@@ -1,7 +1,6 @@
 from datetime import datetime
 from scrapy.utils.project import get_project_settings
-from twisted.internet import reactor
-from scrapy.crawler import CrawlerRunner
+from scrapy.crawler import CrawlerProcess
 
 import pprint
 from my_logging import *
@@ -13,21 +12,19 @@
 if __name__ == '__main__':
     try:
         settings = get_project_settings()
-        runner = CrawlerRunner(settings)
+        process = CrawlerProcess(settings)
 
         spiders = {
             SoaringDeSpider.SoaringDeSpider: None,
             FlugzeugMarktDeSpider.FlugzeugMarktDeSpider: None,
             #PlaneCheckComSpider.PlaneCheckComSpider: None
         }
         for spider_cls in spiders.keys():
-            crawler = runner.create_crawler(spider_cls)
+            crawler = process.create_crawler(spider_cls)
             spiders[spider_cls] = crawler
-            runner.crawl(crawler)
+            process.crawl(crawler)
 
-        d = runner.join()
-        d.addBoth(lambda _: reactor.stop())
-        reactor.run()  # the script will block here until all crawling jobs are finished
+        process.start() # the script will block here until all crawling jobs are finished
 
         stats_per_spider = {}
 

backend/mailer.py

@@ -9,8 +9,7 @@ def send_mail(text=""):
     if not SEND_RESULT_MAIL:
         return
     msg = email.mime.text.MIMEText(text)
-    # TODO put your mail address here
-    # me = u'ralf.thaenert@googlemail.com'
+    me = 'dev@aerooffers.pl'
     msg['Subject'] = 'Aircraft Offers Crawling Result'
     msg['From'] = SMTP_USER
     msg['To'] = me

backend/requirements.txt

@@ -1,6 +1,7 @@
 coverage==7.6.0
 Twisted==24.3.0
 psycopg2-binary==2.9.9
+pylint==3.2.6
 SQLAlchemy==2.0.31
 price-parser==0.3.4
 Scrapy==2.11.2

backend/run_lint.sh

@@ -0,0 +1,5 @@
+export PYTHONPATH=$PYTHONPATH':./'
+
+set -e
+
+pylint --fail-on=E --errors-only ./

backend/run_tests.sh

@@ -1,9 +1,6 @@
 export PYTHONPATH=$PYTHONPATH':./'
 
-coverage run --source ./ -m xmlrunner -o ./test-results
+set -e
 
-if [[ $? -ne 0 ]]; then
-    exit 1
-else
-  coverage report --fail-under=75
-fi
+coverage run --source ./ --omit="tests/*" -m xmlrunner -o ./test-results
+coverage report --fail-under=65