Add support for 'language' field in scraped recipes

recipe_scrapers/__init__.py

@@ -138,7 +138,7 @@ class WebsiteNotImplementedError(NotImplementedError):
     pass
 
 
-def scrape_me(url_path):
+def scrape_me(url_path, **kwargs):
 
     host_name = url_path_to_dict(url_path.replace('://www.', '://'))['host']
 
@@ -149,7 +149,7 @@ def scrape_me(url_path):
             "Website ({}) is not supported".format(host_name)
         )
 
-    return scraper(url_path)
+    return scraper(url_path, **kwargs)
 
 
 __all__ = ['scrape_me']

recipe_scrapers/_abstract.py

@@ -1,5 +1,6 @@
 import requests
 from bs4 import BeautifulSoup
+from language_tags import tags
 
 from ._schemaorg import (
     SchemaOrg,
@@ -19,30 +20,46 @@ class Decorators:
         Define decorators for AbstractScraper methods here.
         """
         @staticmethod
-        def schema_org_priority(function):
+        def schema_org_priority(decorated):
             """
             Use SchemaOrg parser with priority (if there's data in it)
             On exception raised - continue by default.
             If there's no data (no schema implemented on the site) - continue by default
             """
             def schema_org_priority_wrapper(self, *args, **kwargs):
-                if self.schema.data:
-                    try:
-                        return self.schema.__getattribute__(
-                            function.__name__
-                        )(*args, **kwargs)
-                    except SchemaOrgException:
-                        pass
-                return function(self, *args, **kwargs)
+                function = getattr(self.schema, decorated.__name__)
+                if not function:
+                    raise SchemaOrgException(
+                        "Function '{}' not found in schema"
+                        .format(decorated.__name)
+                    )
+
+                if not self.schema.data:
+                    return decorated(self, *args, **kwargs)
+
+                try:
+                    value = function(*args, **kwargs)
+                except SchemaOrgException:
+                    return decorated(self, *args, **kwargs)
+                return value or decorated(self, *args, **kwargs)
+
             return schema_org_priority_wrapper
 
-    def __init__(self, url, test=False):
+        @staticmethod
+        def bcp47_validate(function):
+            def bcp47_validate_wrapper(self, *args, **kwargs):
+                tag = tags.tag(function(self, *args, **kwargs))
+                return str(tag) if tag.valid else None
+            return bcp47_validate_wrapper
+
+    def __init__(self, url, test=False, **kwargs):
         if test:  # when testing, we load a file
             with url:
                 page_data = url.read()
         else:
             page_data = requests.get(url, headers=HEADERS).content
 
+        self.meta_http_equiv = kwargs.get('meta_http_equiv', False)
         self.soup = BeautifulSoup(page_data, "html.parser")
         self.schema = SchemaOrg(page_data)
         self.url = url
@@ -88,6 +105,43 @@ def image(self):
         except AttributeError:  # if image not found
             raise NotImplementedError("This should be implemented.")
 
+    @Decorators.bcp47_validate
+    @Decorators.schema_org_priority
+    def language(self):
+        """
+        Human language the recipe is written in.
+
+        May be overridden by individual scrapers.
+        """
+        candidate_languages = set()
+        html = self.soup.find(
+            'html',
+            {'lang': True}
+        )
+        candidate_languages.add(html.get('lang'))
+
+        # Deprecated: check for a meta http-equiv header
+        # See: https://www.w3.org/International/questions/qa-http-and-lang
+        meta_language = self.soup.find(
+            'meta',
+            {
+                'http-equiv': lambda x: x and x.lower() == 'content-language',
+                'content': True
+            }
+        ) if self.meta_http_equiv else None
+        if meta_language:
+            for language in meta_language.get('content').split(','):
+                candidate_languages.add(language)
+                break
+
+        # If other langs exist, remove 'en' commonly generated by HTML editors
+        if len(candidate_languages) > 1 and 'en' in candidate_languages:
+            candidate_languages.remove('en')
+
+        # Return the first candidate language
+        for language in candidate_languages:
+            return language
+
     @Decorators.schema_org_priority
     def ingredients(self):
         raise NotImplementedError("This should be implemented.")

recipe_scrapers/_schemaorg.py

@@ -36,6 +36,9 @@ def __init__(self, page_data):
                         self.data = self.data.get('mainEntity')
                     return
 
+    def language(self):
+        return self.data.get("inLanguage") or self.data.get("language")
+
     def title(self):
         return self.data.get("name")
 

recipe_scrapers/tests/test_inspiralized.py

@@ -20,6 +20,12 @@ def test_host(self):
             self.harvester_class.host()
         )
 
+    def test_language(self):
+        self.assertEqual(
+            'en-US',
+            self.harvester_class.language()
+        )
+
     def test_title(self):
         self.assertEqual(
             self.harvester_class.title(),

recipe_scrapers/tests/test_mindmegette.py

@@ -14,14 +14,20 @@ def setUp(self):
                 'test_data',
                 'mindmegette.testhtml'
         )) as file_opened:
-            self.harvester_class = Mindmegette(file_opened, test=True)
+            self.harvester_class = Mindmegette(file_opened, meta_http_equiv=True, test=True)
 
     def test_host(self):
         self.assertEqual(
             'mindmegette.hu',
             self.harvester_class.host()
         )
 
+    def test_language(self):
+        self.assertEqual(
+            'hu-HU',
+            self.harvester_class.language()
+        )
+
     def test_title(self):
         self.assertEqual(
             self.harvester_class.title(),

recipe_scrapers/tests/test_przepisy.py

@@ -23,6 +23,12 @@ def test_host(self):
             self.harvester_class.host()
         )
 
+    def test_language(self):
+        self.assertEqual(
+            'pl',
+            self.harvester_class.language()
+        )
+
     def test_title(self):
         self.assertEqual(
             'Placki ziemniaczane',

requirements.txt

@@ -1,4 +1,5 @@
 beautifulsoup4>=4.6.0
 coverage>=4.5.1
 extruct>=0.8.0
+language-tags>=1.0.0
 requests>=2.19.1