From 047db74794b752ba8fc940bcd6c8ae0116abdaef Mon Sep 17 00:00:00 2001 From: alfenn <52635707+alfenn@users.noreply.github.com> Date: Thu, 26 May 2022 14:52:18 -0500 Subject: [PATCH 1/4] finish mental health response scraper --- .../spiders/cuya_mental_health_response.py | 108 ++ tests/files/cuya_mental_health_response.html | 1246 +++++++++++++++++ tests/test_cuya_mental_health_response.py | 86 ++ 3 files changed, 1440 insertions(+) create mode 100644 city_scrapers/spiders/cuya_mental_health_response.py create mode 100644 tests/files/cuya_mental_health_response.html create mode 100644 tests/test_cuya_mental_health_response.py diff --git a/city_scrapers/spiders/cuya_mental_health_response.py b/city_scrapers/spiders/cuya_mental_health_response.py new file mode 100644 index 0000000..69ca758 --- /dev/null +++ b/city_scrapers/spiders/cuya_mental_health_response.py @@ -0,0 +1,108 @@ +from datetime import datetime +from typing import Tuple +from city_scrapers_core.constants import COMMITTEE +from city_scrapers_core.items import Meeting +from city_scrapers_core.spiders import CityScrapersSpider + + +class CuyaMentalHealthResponseSpider(CityScrapersSpider): + name = "cuya_mental_health_response" + agency = "Mental Health Response Advisory Committee" + timezone = "America/Chicago" + start_urls = ["https://www.adamhscc.org/about-us/current-initiatives/task-forces-and-coalitions/mental-health-response-advisory-committee-mhrac"] + + def parse(self, response): + """ + `parse` should always `yield` Meeting items. + + Change the `_parse_title`, `_parse_start`, etc methods to fit your scraping + needs. + """ + for i in range(len(response.css("tr"))): + if i == 0: continue # Skip the first element (table header) + item = response.css("tr")[i] + meeting = Meeting( + title=self._parse_title(item), + description=self._parse_description(item), + classification=self._parse_classification(item), + start=self._parse_start(item), + end=self._parse_end(item), + all_day=self._parse_all_day(item), + time_notes=self._parse_time_notes(item), + location=self._parse_location(item), + links=self._parse_links(item), + source=self._parse_source(response), + ) + + meeting["status"] = self._get_status(meeting) + meeting["id"] = self._get_id(meeting) + + yield meeting + + def _parse_title(self, item): + """Parse or generate meeting title.""" + return item.css("span::text").get() + + def _parse_description(self, item): + """Parse or generate meeting description.""" + return "" + + def _parse_classification(self, item): + """Parse or generate classification from allowed options.""" + return COMMITTEE + + def _to_24h_time(self, time_s: str) -> Tuple[int, int]: + """Helper functionto convert 12-hour time to 24-hour time""" + time_s = time_s.strip() + time_tokens = time_s.split(":") + hour = int(time_tokens[0]) + minute = int(time_tokens[1].split(" ")[0]) + if time_s[-2] == "P" and hour != 12: + hour += 12 + return (hour, minute) + + def _parse_start_end(self, item) -> dict: + """Helper function to generate start and end time.""" + parsed_raw_l = item.css('td.event_datetime::text').get().split(' ') + start_time_24_t = self._to_24h_time(' '.join(parsed_raw_l[1:3])) + end_time_24_t = self._to_24h_time(' '.join(parsed_raw_l[4:6])) + parsed_raw_date_l = parsed_raw_l[0].split('/') + year_i = int(parsed_raw_date_l[2]) + month_i = int(parsed_raw_date_l[0]) + day_i = int(parsed_raw_date_l[1]) + start_time_dt = datetime(year_i, month_i, day_i, start_time_24_t[0], start_time_24_t[1]) + end_time_dt = datetime(year_i, month_i, day_i, end_time_24_t[0], end_time_24_t[1]) + return {"start": start_time_dt, + "end": end_time_dt} + + def _parse_start(self, item): + """Parse start datetime as a naive datetime object.""" + return self._parse_start_end(item)["start"] + + def _parse_end(self, item): + """Parse end datetime as a naive datetime object. Added by pipeline if None""" + return self._parse_start_end(item)["end"] + + def _parse_time_notes(self, item): + """Parse any additional notes on the timing of the meeting""" + return "" + + def _parse_all_day(self, item): + """Parse or generate all-day status. Defaults to False.""" + return False + + def _parse_location(self, item): + """Parse or generate location.""" + return { + "address": "", + "name": "Zoom link in meeting links.", + } + + def _parse_links(self, item): + """Parse or generate links.""" + href: str = 'https://www.adamhscc.org' + item.css('a').attrib['href'] + return [{"href": href, "title": "Meeting details"}] + + def _parse_source(self, response): + """Parse or generate source.""" + return response.url diff --git a/tests/files/cuya_mental_health_response.html b/tests/files/cuya_mental_health_response.html new file mode 100644 index 0000000..6c6324b --- /dev/null +++ b/tests/files/cuya_mental_health_response.html @@ -0,0 +1,1246 @@ + + + + + + + Mental Health Response Advisory Committee (MHRAC) | ADAMHS Board of Cuyahoga County + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ +
+ + Skip to page body + + + Home + + + About Us + + + Resources + + + Training + + + Board Meetings + + + Job Opportunities + +
+ + + + + +
+ +
+ + + +
+ +
+ + + + + + + + + + + +
+ + +
+ + + + + + + + + + + + +
+ + +
+ + +
+ + +

+ About Us » Current Initiatives » Task Forces and Coalitions +

+ + +

Mental Health Response Advisory Committee (MHRAC)

+ + +
+ + Print + + + + +
+ + Share & Bookmark, Press Enter to show all options, press Tab go to next option + +
+ +
+ Font Size: + + + - +
+ +
+ + + +
+ + + + + +

About the MHRAC

+

The Settlement Agreement between the City of Cleveland and the Department of Justice required that a Mental Health Response Advisory Committee (MHRAC) be developed by the City and the Cleveland Division of Police (CDP). 

+

Scott S. Osiecki, ADAMHS Board CEO; Nicole A. Carlton, Commissioner of the Division of Emergency Medical Service, Cleveland Department of Public Safety; and the Cleveland Division of Police CIT Coordinator (vacant, but currently being filled by Deputy Chief Joellen O’Neill) serve as Tri-Chairs of this committee that has the following charge:

+
    +
  • Fostering relationships and support between the police, community and mental health providers.
    • +
  • Identifying problems and developing solutions to improve crisis outcomes.
    • +
  • Providing guidance to improving, expanding and sustaining the CPD Crisis Intervention Program.
    • +
  • Conducting a yearly analysis of incidents to determine if the CPD has enough specialized CIT officers, if they are deployed effectively and responding appropriately and recommending changes to policies and procedures regarding training.
    • +
+
+

2021 MHRAC Annual Report

+

In accordance with the Memorandum of Understanding between the City of Cleveland and the ADAMHS Board of Cuyahoga County, the Tri-chairs of the Mental Health Response Advisory Committee (MHRAC) - Captain James McPike, Nicole Carlton and Mr. Scott S. Osiecki submitted the MHRAC 2021 Annual Report on January 31, 2022. You can view all of the MHRAC Annual Reports here.

+
+

Subcommittees

+

There are three subcommittees of the MHRAC and its structure continues to mirror the core elements, process for implementation and the coordination for a successful CIT Program.

+
    +
  • Training Subcommittee: Reviews and makes recommendations for mental health and alcohol or other drug (AOD) training for all Cleveland law enforcement officers and personnel, as well as for the 40-hour specialized Crisis Intervention Team (CIT) training for officers who volunteer and are approved for the training. 
    • +
  • Community Involvement/Engagement Subcommittee: Fosters relationships between the Cleveland Division of Police (CDP) and the community by engaging the behavioral health community, police and the general public in meaningful dialogue that builds knowledge, sensitivity, understanding and trust. 
    • +
  • Quality Improvement Subcommittee: Reviews and discusses the data submitted from the Crisis Intervention Team (CIT) Stat Sheets and other data sources, and makes recommendations on improving the quality and quantity of data collected, as well as potential changes to policy and procedures based in part on the data review. 
    • +
+

+

+

Meeting Schedule

+

See upcoming MHRAC meetings below. You can download the list of 2021 meeting dates here.

+ + + +
EventDate/TimeAgenda
05/17/2022 9:00 AM - 10:00 AM Not Included
06/21/2022 9:00 AM - 10:30 AM Not Included
07/11/2022 9:00 AM - 10:30 AM Not Included
07/11/2022 10:30 AM - 11:30 AM Not Included
07/18/2022 9:00 AM - 10:30 AM Not Included
07/19/2022 9:00 AM - 10:30 AM Not Included
08/15/2022 9:00 AM - 10:30 AM Not Included
08/16/2022 9:00 AM - 10:30 AM Not Included
09/12/2022 9:00 AM - 10:30 AM Not Included
09/12/2022 10:30 AM - 11:30 AM Not Included
09/19/2022 9:00 AM - 10:30 AM Not Included
09/20/2022 9:00 AM - 10:30 AM Not Included
10/17/2022 9:00 AM - 10:30 AM Not Included
10/18/2022 9:00 AM - 10:30 AM Not Included
11/07/2022 9:00 AM - 10:30 AM Not Included
11/07/2022 10:30 AM - 11:30 AM Not Included
11/15/2022 9:00 AM - 10:30 AM Not Included
+ + + + + +

All meetings of the MHRAC and its subcommittees are open to the public. Please email Clare Rosser, Chief of Strategic Initiatives, if you'd like to be added to the email list. 

+

+

+

Meeting Summaries/Highlights

+ + + +
+
+

+ Free viewers are required for some of the attached documents.
They can be downloaded by clicking on the icons below. +

+ + Acrobat Reader + Download Acrobat Reader + + + Windows Media Player + Download Windows Media Player + + + + Word Viewer + Download Word Viewer + + + Excel Viewer + Download Excel Viewer + + + PowerPoint Viewer + Download PowerPoint Viewer + +
+ + + + + +
+ + + +

+

+DOJ/CDP Settlement Agreement

+

The United States Department of Justice and the City of Cleveland have entered into an agreement to create widespread reforms and changes within the Cleveland Division of Police.

+

The changes focus on building community trust, creating a culture of community and problem-oriented policing, officer safety and training, officer accountability and technological upgrades. In addition, a Mental Health Response Advisory Committee was formed to foster relationships and build support between police, the community and mental health providers. Click here to read the MOU between the City of Cleveland Department of Public Safety and the ADAMHS Board. There will be broad data collection regarding many of CDP’s activities, including its use of force and stop and search practices, and public reporting of that data on this website.

+

Under the agreement, the parties have jointly selected an independent monitor to assess and report whether the requirements of the agreement have been implemented for a term of at least five years.

+

Learn more about the City’s Settlement agreement.

+

+

+

Mental Health Task Force Recommendations for Consent Decree

+

The report Mental Health Task Force Recommendations to Insure all Clevelanders with Mental Illness – and All Citizens – are Treated Safely with Dignity and Respect providing recommendations focusing on community involvement, training and collaboration was submitted to Cleveland Mayor Frank Jackson and Steven Dettlebach of the U.S. Attorney General’s Office by Members of the Mental Health Task Force convened by the ADAMHS Board of Cuyahoga County. The Mental Health Task Force was formed to respond to issues pertaining to mental health that were identified in the U.S. Department of Justice Investigation of the Cleveland Division of Police.

+

View the recommendations, including a roster of Mental Health Task Force members.

+

+

+

Cleveland Division of Police CIT

+

Crisis Intervention Team training is a national program designed to assist police officers encountering individuals living with mental illness. CIT is a community-based collaborative between individuals, families, the behavioral health system and mental health treatment providers, law enforcement agencies, advocacy organization and the medical community. Click here to learn more about Cleveland Division of Police CIT.

+

+

+ + + +
+ +
+ + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/test_cuya_mental_health_response.py b/tests/test_cuya_mental_health_response.py new file mode 100644 index 0000000..592c227 --- /dev/null +++ b/tests/test_cuya_mental_health_response.py @@ -0,0 +1,86 @@ +from datetime import datetime +from os.path import dirname, join + +import pytest +from city_scrapers_core.constants import COMMITTEE +from city_scrapers_core.utils import file_response +from freezegun import freeze_time + +from city_scrapers.spiders.cuya_mental_health_response import CuyaMentalHealthResponseSpider + +test_response = file_response( + join(dirname(__file__), "files", "cuya_mental_health_response.html"), + url="https://www.adamhscc.org/about-us/current-initiatives/task-forces-and-coalitions/mental-health-response-advisory-committee-mhrac", +) +spider = CuyaMentalHealthResponseSpider() + +freezer = freeze_time("2022-05-17") +freezer.start() + +parsed_items = [item for item in spider.parse(test_response)] + +freezer.stop() + + +# def test_tests(): +# print("Please write some tests for this spider or at least disable this one.") +# assert False + + +""" +Uncomment below +""" + +def test_title(): + assert parsed_items[0]["title"] == "MHRAC QI Subcommittee Meeting" + + +def test_description(): + assert parsed_items[0]["description"] == "" + + +def test_start(): + assert parsed_items[0]["start"] == datetime(2022, 5, 17, 9, 0) + + +def test_end(): + assert parsed_items[0]["end"] == datetime(2022, 5, 17, 10, 0m) + + +def test_time_notes(): + assert parsed_items[0]["time_notes"] == "" + + +def test_id(): + assert parsed_items[0]["id"] == 'cuya_mental_health_response/202205170900/x/mhrac_qi_subcommittee_meeting' # noqa + + +def test_status(): + assert parsed_items[0]["status"] == "tentative" + + +def test_location(): + assert parsed_items[0]["location"] == { + "name": "Zoom link in meeting links.", + "address": "" + } + + +def test_source(): + assert parsed_items[0]["source"] == "https://www.adamhscc.org/about-us/current-initiatives/task-forces-and-coalitions/mental-health-response-advisory-committee-mhrac" # noqa + + +def test_links(): + assert parsed_items[0]["links"] == [{ + "href": "https://www.adamhscc.org/Home/Components/Calendar/Event/854/110", # noqa + "title": "Meeting details" + }] + + +def test_classification(): + assert parsed_items[0]["classification"] == COMMITTEE + + +@pytest.mark.parametrize("item", parsed_items) +def test_all_day(item): + assert item["all_day"] is False From 93731b7a4b30ef665a450dbd97edc5b8f9c34c9d Mon Sep 17 00:00:00 2001 From: alfenn <52635707+alfenn@users.noreply.github.com> Date: Thu, 26 May 2022 15:11:00 -0500 Subject: [PATCH 2/4] run and fix style checks --- .../spiders/cuya_mental_health_response.py | 32 ++++++++++------- tests/test_cuya_mental_health_response.py | 36 +++++++++++-------- 2 files changed, 40 insertions(+), 28 deletions(-) diff --git a/city_scrapers/spiders/cuya_mental_health_response.py b/city_scrapers/spiders/cuya_mental_health_response.py index 69ca758..c207f2f 100644 --- a/city_scrapers/spiders/cuya_mental_health_response.py +++ b/city_scrapers/spiders/cuya_mental_health_response.py @@ -9,7 +9,9 @@ class CuyaMentalHealthResponseSpider(CityScrapersSpider): name = "cuya_mental_health_response" agency = "Mental Health Response Advisory Committee" timezone = "America/Chicago" - start_urls = ["https://www.adamhscc.org/about-us/current-initiatives/task-forces-and-coalitions/mental-health-response-advisory-committee-mhrac"] + start_urls = [ + "https://www.adamhscc.org/about-us/current-initiatives/task-forces-and-coalitions/mental-health-response-advisory-committee-mhrac" # noqa + ] def parse(self, response): """ @@ -19,7 +21,8 @@ def parse(self, response): needs. """ for i in range(len(response.css("tr"))): - if i == 0: continue # Skip the first element (table header) + if i == 0: + continue # Skip the first element (table header) item = response.css("tr")[i] meeting = Meeting( title=self._parse_title(item), @@ -60,21 +63,24 @@ def _to_24h_time(self, time_s: str) -> Tuple[int, int]: if time_s[-2] == "P" and hour != 12: hour += 12 return (hour, minute) - + def _parse_start_end(self, item) -> dict: """Helper function to generate start and end time.""" - parsed_raw_l = item.css('td.event_datetime::text').get().split(' ') - start_time_24_t = self._to_24h_time(' '.join(parsed_raw_l[1:3])) - end_time_24_t = self._to_24h_time(' '.join(parsed_raw_l[4:6])) - parsed_raw_date_l = parsed_raw_l[0].split('/') + parsed_raw_l = item.css("td.event_datetime::text").get().split(" ") + start_time_24_t = self._to_24h_time(" ".join(parsed_raw_l[1:3])) + end_time_24_t = self._to_24h_time(" ".join(parsed_raw_l[4:6])) + parsed_raw_date_l = parsed_raw_l[0].split("/") year_i = int(parsed_raw_date_l[2]) month_i = int(parsed_raw_date_l[0]) day_i = int(parsed_raw_date_l[1]) - start_time_dt = datetime(year_i, month_i, day_i, start_time_24_t[0], start_time_24_t[1]) - end_time_dt = datetime(year_i, month_i, day_i, end_time_24_t[0], end_time_24_t[1]) - return {"start": start_time_dt, - "end": end_time_dt} - + start_time_dt = datetime( + year_i, month_i, day_i, start_time_24_t[0], start_time_24_t[1] + ) + end_time_dt = datetime( + year_i, month_i, day_i, end_time_24_t[0], end_time_24_t[1] + ) + return {"start": start_time_dt, "end": end_time_dt} + def _parse_start(self, item): """Parse start datetime as a naive datetime object.""" return self._parse_start_end(item)["start"] @@ -100,7 +106,7 @@ def _parse_location(self, item): def _parse_links(self, item): """Parse or generate links.""" - href: str = 'https://www.adamhscc.org' + item.css('a').attrib['href'] + href: str = "https://www.adamhscc.org" + item.css("a").attrib["href"] return [{"href": href, "title": "Meeting details"}] def _parse_source(self, response): diff --git a/tests/test_cuya_mental_health_response.py b/tests/test_cuya_mental_health_response.py index 592c227..aa15450 100644 --- a/tests/test_cuya_mental_health_response.py +++ b/tests/test_cuya_mental_health_response.py @@ -6,11 +6,13 @@ from city_scrapers_core.utils import file_response from freezegun import freeze_time -from city_scrapers.spiders.cuya_mental_health_response import CuyaMentalHealthResponseSpider +from city_scrapers.spiders.cuya_mental_health_response import ( + CuyaMentalHealthResponseSpider, +) test_response = file_response( join(dirname(__file__), "files", "cuya_mental_health_response.html"), - url="https://www.adamhscc.org/about-us/current-initiatives/task-forces-and-coalitions/mental-health-response-advisory-committee-mhrac", + url="https://www.adamhscc.org/about-us/current-initiatives/task-forces-and-coalitions/mental-health-response-advisory-committee-mhrac", # noqa ) spider = CuyaMentalHealthResponseSpider() @@ -22,15 +24,11 @@ freezer.stop() -# def test_tests(): -# print("Please write some tests for this spider or at least disable this one.") -# assert False - - """ Uncomment below """ + def test_title(): assert parsed_items[0]["title"] == "MHRAC QI Subcommittee Meeting" @@ -44,7 +42,7 @@ def test_start(): def test_end(): - assert parsed_items[0]["end"] == datetime(2022, 5, 17, 10, 0m) + assert parsed_items[0]["end"] == datetime(2022, 5, 17, 10, 0) def test_time_notes(): @@ -52,7 +50,10 @@ def test_time_notes(): def test_id(): - assert parsed_items[0]["id"] == 'cuya_mental_health_response/202205170900/x/mhrac_qi_subcommittee_meeting' # noqa + assert ( + parsed_items[0]["id"] + == "cuya_mental_health_response/202205170900/x/mhrac_qi_subcommittee_meeting" + ) def test_status(): @@ -62,19 +63,24 @@ def test_status(): def test_location(): assert parsed_items[0]["location"] == { "name": "Zoom link in meeting links.", - "address": "" + "address": "", } def test_source(): - assert parsed_items[0]["source"] == "https://www.adamhscc.org/about-us/current-initiatives/task-forces-and-coalitions/mental-health-response-advisory-committee-mhrac" # noqa + assert ( + parsed_items[0]["source"] + == "https://www.adamhscc.org/about-us/current-initiatives/task-forces-and-coalitions/mental-health-response-advisory-committee-mhrac" # noqa + ) def test_links(): - assert parsed_items[0]["links"] == [{ - "href": "https://www.adamhscc.org/Home/Components/Calendar/Event/854/110", # noqa - "title": "Meeting details" - }] + assert parsed_items[0]["links"] == [ + { + "href": "https://www.adamhscc.org/Home/Components/Calendar/Event/854/110", # noqa + "title": "Meeting details", + } + ] def test_classification(): From a14821b0e2ef1cca397e33bf18705d2656c6e549 Mon Sep 17 00:00:00 2001 From: alfenn <52635707+alfenn@users.noreply.github.com> Date: Thu, 26 May 2022 15:16:34 -0500 Subject: [PATCH 3/4] run isort --- city_scrapers/spiders/cuya_mental_health_response.py | 1 + 1 file changed, 1 insertion(+) diff --git a/city_scrapers/spiders/cuya_mental_health_response.py b/city_scrapers/spiders/cuya_mental_health_response.py index c207f2f..3dca0f5 100644 --- a/city_scrapers/spiders/cuya_mental_health_response.py +++ b/city_scrapers/spiders/cuya_mental_health_response.py @@ -1,5 +1,6 @@ from datetime import datetime from typing import Tuple + from city_scrapers_core.constants import COMMITTEE from city_scrapers_core.items import Meeting from city_scrapers_core.spiders import CityScrapersSpider From 91cf8e9796c7c11e06c43051dcfa9ab9809a428e Mon Sep 17 00:00:00 2001 From: alfenn <52635707+alfenn@users.noreply.github.com> Date: Wed, 1 Jun 2022 16:21:28 -0500 Subject: [PATCH 4/4] change timezone --- city_scrapers/spiders/cuya_mental_health_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/city_scrapers/spiders/cuya_mental_health_response.py b/city_scrapers/spiders/cuya_mental_health_response.py index 3dca0f5..f88b599 100644 --- a/city_scrapers/spiders/cuya_mental_health_response.py +++ b/city_scrapers/spiders/cuya_mental_health_response.py @@ -9,7 +9,7 @@ class CuyaMentalHealthResponseSpider(CityScrapersSpider): name = "cuya_mental_health_response" agency = "Mental Health Response Advisory Committee" - timezone = "America/Chicago" + timezone = "America/Detroit" start_urls = [ "https://www.adamhscc.org/about-us/current-initiatives/task-forces-and-coalitions/mental-health-response-advisory-committee-mhrac" # noqa ]