Merge pull request #267 from alan-turing-institute/website/191-dailywire

add dailywire
alan-turing-institute · Jun 19, 2019 · 3828a25 · 3828a25
2 parents 6c5a8ae + 9033cb8
commit 3828a25
Show file tree

Hide file tree

Showing 4 changed files with 1,303 additions and 0 deletions.
diff --git a/misinformation/middlewares/jsloadbuttonmiddleware.py b/misinformation/middlewares/jsloadbuttonmiddleware.py
@@ -37,6 +37,7 @@ def __init__(self):
             '//form[@class="gdpr-form"]/input[@class="btn"]',
             '//div[contains(@class, "load-btn")]/a',
             '//ul[contains(@class, "pager-load-more")]/li/a',
+            '//a[text()="Show More"]'
         ]
 
     def first_load_button_xpath(self):

diff --git a/site_configs.yml b/site_configs.yml
@@ -562,6 +562,29 @@ dailykos.com:
       select_expression: '//div[@class="story-intro story-content"]'
       match_rule: 'single'
 
+dailywire.com:
+  site_name: 'dailywire.com'
+  start_url: 'https://www.dailywire.com/'
+  crawl_strategy:
+    method: 'index_page'
+    index_page:
+      url_must_contain: 'dailywire.com'
+      url_must_not_contain: 'dailywire.com/'
+      article_links: '//article/div[contains(@class, "media")]/a'
+  article:
+    url_must_contain: '/news/'
+    byline:
+      select_method: 'xpath'
+      select_expression: '//div[@class="author-information"]/a/text()'
+      match_rule: 'first'
+    content:
+      select_method: 'xpath'
+      select_expression: '//div[@class="field-body"]'
+      match_rule: 'single'
+      remove_expressions:
+        - '//div[@id="powr_1"]'
+        - '//div[contains(@class, "ad-unit")]'
+
 dallasobserver.com:
   site_name: 'dallasobserver.com'
   start_url: 'https://www.dallasobserver.com/topic/politics-7019564'