Temporarily enabled crawling all 6 pages from soaring.de to capture a…

…s many ads as possible before they are gone
lwitkowski · Jul 30, 2024 · e2f7c1c · e2f7c1c
1 parent e231645
commit e2f7c1c
Showing 1 changed file with 9 additions and 1 deletion.
diff --git a/backend/spiders/SoaringDeSpider.py b/backend/spiders/SoaringDeSpider.py
@@ -14,7 +14,15 @@ class SoaringDeSpider(scrapy.Spider):
     name = "segelflug_de_kleinanzeigen" # fixme, possibly needs db rows update
     logger = logging.getLogger(name)
 
-    start_urls = [GLIDER_OFFERS_URL, ENGINE_OFFERS_URL]
+    start_urls = [
+        GLIDER_OFFERS_URL,
+        GLIDER_OFFERS_URL + "&iPage=2",
+        GLIDER_OFFERS_URL + "&iPage=3",
+        GLIDER_OFFERS_URL + "&iPage=4",
+        GLIDER_OFFERS_URL + "&iPage=5",
+        GLIDER_OFFERS_URL + "&iPage=6",
+        ENGINE_OFFERS_URL
+    ]
 
     AD_SELECTOR = ".listing-thumb"