From aee926eda6233adb680e0166abebade1d1c2fca6 Mon Sep 17 00:00:00 2001 From: Hails Date: Fri, 21 Oct 2022 02:48:31 -0400 Subject: [PATCH 1/5] Update cle_city_council.py --- city_scrapers/spiders/cle_city_council.py | 37 ++++++++++++----------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/city_scrapers/spiders/cle_city_council.py b/city_scrapers/spiders/cle_city_council.py index 773163b..3eac421 100644 --- a/city_scrapers/spiders/cle_city_council.py +++ b/city_scrapers/spiders/cle_city_council.py @@ -18,23 +18,26 @@ def parse_legistar(self, events): needs. """ for event in events: - meeting = Meeting( - title=event["Name"]["label"], - description=self._parse_description(event), - classification=self._parse_classification(event), - start=self.legistar_start(event), - end=None, - all_day=False, - time_notes="", - location=self._parse_location(event), - links=self.legistar_links(event), - source=self.legistar_source(event), - ) - - meeting["status"] = self._get_status(meeting) - meeting["id"] = self._get_id(meeting) - - yield meeting + if (self.legistar_start(event) is NoneType): + return + else: + meeting = Meeting( + title=event["Name"]["label"], + description=self._parse_description(event), + classification=self._parse_classification(event), + start=self.legistar_start(event), + end=None, + all_day=False, + time_notes="", + location=self._parse_location(event), + links=self.legistar_links(event), + source=self.legistar_source(event), + ) + + meeting["status"] = self._get_status(meeting) + meeting["id"] = self._get_id(meeting) + + yield meeting def _parse_description(self, item): """Parse or generate meeting description.""" From f7dfdfc32694dd64916bf6d2fd79e3dbd384e181 Mon Sep 17 00:00:00 2001 From: Hails Date: Sat, 22 Oct 2022 19:12:31 -0400 Subject: [PATCH 2/5] Update cle_city_council.py line#21: change from NoneType to None --- city_scrapers/spiders/cle_city_council.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/city_scrapers/spiders/cle_city_council.py b/city_scrapers/spiders/cle_city_council.py index 3eac421..968a85e 100644 --- a/city_scrapers/spiders/cle_city_council.py +++ b/city_scrapers/spiders/cle_city_council.py @@ -18,7 +18,7 @@ def parse_legistar(self, events): needs. """ for event in events: - if (self.legistar_start(event) is NoneType): + if (self.legistar_start(event) is None): return else: meeting = Meeting( From 02d5e73e01649c78036581bdb3f1052020e8cbbd Mon Sep 17 00:00:00 2001 From: Hails Date: Sat, 22 Oct 2022 20:02:05 -0400 Subject: [PATCH 3/5] Add files via upload admittedly slightly hippie-dippy way of fixing this but tbh at this point i just want this bug to be over with. if self.legistar_start(event) is None, then make the start a passed time that will register the meeting on the Documenters.org website as 'passed' (i.e. a meeting that is already passed and shouldn't be assigned). --- city_scrapers/spiders/cle_city_council.py | 38 +++++++++++++---------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/city_scrapers/spiders/cle_city_council.py b/city_scrapers/spiders/cle_city_council.py index 968a85e..59da167 100644 --- a/city_scrapers/spiders/cle_city_council.py +++ b/city_scrapers/spiders/cle_city_council.py @@ -1,3 +1,5 @@ +from datetime import datetime +from tracemalloc import start from city_scrapers_core.constants import CITY_COUNCIL, COMMITTEE from city_scrapers_core.items import Meeting from city_scrapers_core.spiders import LegistarSpider @@ -18,26 +20,30 @@ def parse_legistar(self, events): needs. """ for event in events: + if (self.legistar_start(event) is None): - return + start = datetime.strptime("01-01-01 00:00:00", '%y-%m-%d %H:%M:%S') else: - meeting = Meeting( - title=event["Name"]["label"], - description=self._parse_description(event), - classification=self._parse_classification(event), - start=self.legistar_start(event), - end=None, - all_day=False, - time_notes="", - location=self._parse_location(event), - links=self.legistar_links(event), - source=self.legistar_source(event), - ) + start = self.legistar_start(event) + + print(self.legistar_start(event)) + meeting = Meeting( + title=event["Name"]["label"], + description=self._parse_description(event), + classification=self._parse_classification(event), + start=start, + end=None, + all_day=False, + time_notes="", + location=self._parse_location(event), + links=self.legistar_links(event), + source=self.legistar_source(event), + ) - meeting["status"] = self._get_status(meeting) - meeting["id"] = self._get_id(meeting) + meeting["status"] = self._get_status(meeting) + meeting["id"] = self._get_id(meeting) - yield meeting + yield meeting def _parse_description(self, item): """Parse or generate meeting description.""" From 021ffd94aca1355ed4dfbfa28ad70f235a8a39f0 Mon Sep 17 00:00:00 2001 From: Hails Date: Sat, 22 Oct 2022 20:21:28 -0400 Subject: [PATCH 4/5] isort/black/flake8 --- city_scrapers/spiders/cle_city_council.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/city_scrapers/spiders/cle_city_council.py b/city_scrapers/spiders/cle_city_council.py index 59da167..9d92da0 100644 --- a/city_scrapers/spiders/cle_city_council.py +++ b/city_scrapers/spiders/cle_city_council.py @@ -1,5 +1,5 @@ from datetime import datetime -from tracemalloc import start + from city_scrapers_core.constants import CITY_COUNCIL, COMMITTEE from city_scrapers_core.items import Meeting from city_scrapers_core.spiders import LegistarSpider @@ -20,12 +20,12 @@ def parse_legistar(self, events): needs. """ for event in events: - - if (self.legistar_start(event) is None): - start = datetime.strptime("01-01-01 00:00:00", '%y-%m-%d %H:%M:%S') + + if self.legistar_start(event) is None: + start = datetime.strptime("01-01-01 00:00:00", "%y-%m-%d %H:%M:%S") else: start = self.legistar_start(event) - + print(self.legistar_start(event)) meeting = Meeting( title=event["Name"]["label"], From 14fbe5239e3c113facdf7fb02b80cbe38ec05743 Mon Sep 17 00:00:00 2001 From: Hails Date: Tue, 25 Oct 2022 10:56:47 -0400 Subject: [PATCH 5/5] redo cle_city_council use solution from Lien Dang isort/black/flake8 --- city_scrapers/spiders/cle_city_council.py | 41 ++++++++++------------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/city_scrapers/spiders/cle_city_council.py b/city_scrapers/spiders/cle_city_council.py index 9d92da0..f0922c5 100644 --- a/city_scrapers/spiders/cle_city_council.py +++ b/city_scrapers/spiders/cle_city_council.py @@ -1,5 +1,3 @@ -from datetime import datetime - from city_scrapers_core.constants import CITY_COUNCIL, COMMITTEE from city_scrapers_core.items import Meeting from city_scrapers_core.spiders import LegistarSpider @@ -20,30 +18,25 @@ def parse_legistar(self, events): needs. """ for event in events: + start = self.legistar_start(event) + if start: + meeting = Meeting( + title=event["Name"]["label"], + description=self._parse_description(event), + classification=self._parse_classification(event), + start=start, + end=None, + all_day=False, + time_notes="", + location=self._parse_location(event), + links=self.legistar_links(event), + source=self.legistar_source(event), + ) - if self.legistar_start(event) is None: - start = datetime.strptime("01-01-01 00:00:00", "%y-%m-%d %H:%M:%S") - else: - start = self.legistar_start(event) - - print(self.legistar_start(event)) - meeting = Meeting( - title=event["Name"]["label"], - description=self._parse_description(event), - classification=self._parse_classification(event), - start=start, - end=None, - all_day=False, - time_notes="", - location=self._parse_location(event), - links=self.legistar_links(event), - source=self.legistar_source(event), - ) - - meeting["status"] = self._get_status(meeting) - meeting["id"] = self._get_id(meeting) + meeting["status"] = self._get_status(meeting) + meeting["id"] = self._get_id(meeting) - yield meeting + yield meeting def _parse_description(self, item): """Parse or generate meeting description."""