Mior fixes in crawler

bohdanbobrowski · Nov 15, 2024 · 216059b · 216059b
1 parent 0e7737d
commit 216059b
Show file tree

Hide file tree

Showing 3 changed files with 21 additions and 14 deletions.
diff --git a/blog2epub/common/cover.py b/blog2epub/common/cover.py
@@ -108,7 +108,7 @@ def _split_too_long_parts(self, parts: list[str]) -> list[str]:
 
     def _split_long_title(self) -> list[str]:
         title = []
-        for char in ["|", "-", ":", " "]:
+        for char in ["|", "-", ":", "–", " "]:
             if self.title.find(char) > -1:
                 title = self.title.split(char)
             if len(title) > 0:

diff --git a/blog2epub/crawlers/default.py b/blog2epub/crawlers/default.py
@@ -113,6 +113,9 @@ def __init__(self, **kwargs):
                 Pattern(
                     xpath="//article//img",
                 ),
+                Pattern(
+                    xpath="//img[contains(@class, 'wp-post-image')]",
+                ),
             ],
         )
         self.downloader = Downloader(
@@ -187,6 +190,7 @@ def _get_header_images(self, tree) -> list[ImageModel]:
         xpaths = [
             '//*[contains(@class, "wp-block-image")]//img/@src',
             '//div[@id="header"]/div/div/div/p[@class="description"]/span/img/@src',
+            "//img[contains(@class, 'wp-post-image')]/@src",
         ]
         for xpath in xpaths:
             for img in tree.xpath(xpath):

diff --git a/blog2epub/models/configuration.py b/blog2epub/models/configuration.py
@@ -5,27 +5,30 @@
 
 example_blogs = [
     "http://archaia-ellada.blogspot.com",
-    "http://historicaltidbits.blogspot.com/",
-    "http://starybezpiek.blogspot.com/",
-    "https://19thcentury.wordpress.com/",
-    "https://cyclehistory.wordpress.com/",
+    "http://historicaltidbits.blogspot.com",
+    "http://starybezpiek.blogspot.com",
+    "https://19thcentury.wordpress.com",
+    "https://cyclehistory.wordpress.com",
+    "https://klubjagiellonski.pl",
     "https://knippsen.blogspot.com",
     "https://ksgedania.blogspot.com",
     "https://motorbikes.blog",
     "https://oldcam.wordpress.com",
-    "https://oldcamera.blog/",
+    "https://oldcamera.blog",
     "https://python-bloggers.com",
     "https://rocket-garage.blogspot.com",
-    "https://thevictoriancyclist.wordpress.com/",
+    "https://swiatmotocykli.pl",
+    "https://thevictoriancyclist.wordpress.com",
     "https://velosov.blogspot.com",
-    "https://vintagebicycle.wordpress.com/",
-    "https://vowe.net/",
+    "https://vintagebicycle.wordpress.com",
+    "https://vowe.net",
     "https://www.blog.homebrewing.pl",
-    "https://www.historyoftheancientworld.com/",
-    "https://www.mikeanderson.biz/",
-    "https://www.nomadicmatt.com/",
-    "https://www.returnofthecaferacers.com/",
-    "https://www.szarmant.pl/",
+    "https://www.historyoftheancientworld.com",
+    "https://www.infolotnicze.pl",
+    "https://www.mikeanderson.biz",
+    "https://www.nomadicmatt.com",
+    "https://www.returnofthecaferacers.com",
+    "https://www.szarmant.pl",
 ]