Fix lint errors, broke coverage requireemnts

backend/job_fetch_offers.py

@@ -1,8 +1,6 @@
 from datetime import datetime
 from scrapy.utils.project import get_project_settings
-from twisted.internet import reactor
 from scrapy.crawler import CrawlerProcess
-from scrapy.crawler import CrawlerRunner
 
 import pprint
 from my_logging import *
@@ -14,7 +12,6 @@
 if __name__ == '__main__':
     try:
         settings = get_project_settings()
-        runner = CrawlerRunner(settings)
         process = CrawlerProcess(settings)
 
         spiders = {
@@ -27,9 +24,6 @@
             spiders[spider_cls] = crawler
             process.crawl(crawler)
 
-        d = runner.join()
-        d.addBoth(lambda _: reactor.stop())
-
         process.start() # the script will block here until all crawling jobs are finished
 
         stats_per_spider = {}

backend/run_tests.sh

@@ -3,4 +3,4 @@ export PYTHONPATH=$PYTHONPATH':./'
 set -e
 
 coverage run --source ./ -m xmlrunner -o ./test-results
-coverage report --fail-under=75
+coverage report --fail-under=80