Add function and data for validating PF medications

lib/functions/get_dataset_nhsbsa_table_schema.R

@@ -0,0 +1,54 @@
+library("httr")
+library("jsonlite")
+library("crul")
+library("here")
+library("rvest")
+library("dplyr")
+library("lubridate")
+library("tidyverse")
+library("readr")
+
+source(here("lib/functions/get_pf_medication_validation_data.R"))
+
+# This may be useful for writing SQL queries but reading the schema from
+# JSON in the metadata seems more tricky and maybe there's another way
+get_dataset_table_schema <- function(dataset_id) {
+  base_endpoint <- "https://opendata.nhsbsa.net/api/3/action/"
+  package_show_method <- "package_show?id="
+
+  available_datasets <- get_available_datasets()
+
+  if (!dataset_id %in% available_datasets) {
+    stop("The provided 'dataset_id' is not available. Run 'get_available_datasets()' to see all available datasets.", call. = FALSE)
+  }
+
+  metadata_response <- GET(paste0(base_endpoint, package_show_method, dataset_id))
+  resources <- content(metadata_response)$result$resources
+
+  schema_raw <- resources[[1]]$schema
+
+  # There seems to be a lot of odd strings in the JSON
+  # that needs to be fixed before we can read it
+  schema_fixed <- schema_raw
+  schema_fixed <- gsub("u\'", '"', schema_fixed)
+  schema_fixed <- gsub("u\"", '"', schema_fixed)
+  schema_fixed <- gsub("':", '":', schema_fixed)
+  schema_fixed <- gsub("',", '",', schema_fixed)
+  schema_fixed <- gsub("'}", '"}', schema_fixed)
+  schema_fixed <- gsub("']", '"]', schema_fixed)
+  schema_fixed <- gsub("-", "", schema_fixed)
+  schema_fixed <- gsub("-", "", schema_fixed)
+
+  schema_list <- fromJSON(schema_fixed, flatten = TRUE)
+
+  tibble(schema_list$fields) |>
+    select(name, title, type, description)
+}
+
+nhsbsa_table_schemas <- map(
+  set_names(get_available_datasets()),
+  safely(get_dataset_table_schema)
+)
+
+nhsbsa_table_schemas_results <- map(nhsbsa_table_schemas, "result")
+nhsbsa_table_schemas_errors <- map(nhsbsa_table_schemas, "error")

lib/functions/function_get_validation_data.R → lib/functions/get_pf_consultation_validation_data.R

@@ -115,19 +115,21 @@ df_dispensing_data_summary <- df_dispensing_data |>
     # n_pf_urgent_medicine_supply_consultations = sum(n_pf_urgent_medicine_supply_consultations, na.rm = TRUE),
     # n_pf_minor_illness_referral_consultations = sum(n_pf_minor_illness_referral_consultations, na.rm = TRUE)
   ) |>
-  pivot_longer(cols = c(
-    n_pf_consultation_acute_otitis_media,
-    n_pf_consultation_acute_sore_throat,
-    n_pf_consultation_impetigo,
-    n_pf_consultation_infected_insect_bites,
-    n_pf_consultation_shingles,
-    n_pf_consultation_sinusitis,
-    n_pf_consultation_uncomplicated_uti,
-    # n_pf_urgent_medicine_supply_consultations,
-    # n_pf_minor_illness_referral_consultations
-  ),
-  names_to = "consultation_type",
-  values_to = "count") |>
+  pivot_longer(
+    cols = c(
+      n_pf_consultation_acute_otitis_media,
+      n_pf_consultation_acute_sore_throat,
+      n_pf_consultation_impetigo,
+      n_pf_consultation_infected_insect_bites,
+      n_pf_consultation_shingles,
+      n_pf_consultation_sinusitis,
+      n_pf_consultation_uncomplicated_uti,
+      # n_pf_urgent_medicine_supply_consultations,
+      # n_pf_minor_illness_referral_consultations
+    ),
+    names_to = "consultation_type",
+    values_to = "count"
+  ) |>
   mutate(consultation_type = str_replace(consultation_type, "^n_pf_consultation_", ""))
 
 write_csv(df_dispensing_data_summary, here("lib", "validation", "data", "pf_consultation_validation_data.csv"))

lib/functions/get_pf_medication_validation_data.R

@@ -0,0 +1,118 @@
+library("httr")
+library("jsonlite")
+library("crul")
+library("here")
+library("rvest")
+library("dplyr")
+library("lubridate")
+library("tidyverse")
+library("readr")
+
+base_endpoint <- "https://opendata.nhsbsa.net/api/3/action/"
+package_list_method <- "package_list"
+package_show_method <- "package_show?id="
+action_method <- "datastore_search_sql?"
+
+get_available_datasets <- function() {
+  base_endpoint <- "https://opendata.nhsbsa.net/api/3/action/"
+  package_list_method <- "package_list"
+
+  datasets_response <- fromJSON(paste0(
+    base_endpoint,
+    package_list_method
+  ))$result
+
+  # Remove datasets with FOI and starting with a number
+  # There does not seem to be any data that we can query from these tables
+  datasets_response <- datasets_response[!grepl("foi", datasets_response)]
+  datasets_response <- datasets_response[!grepl("^[0-9]", datasets_response)]
+
+  datasets_response
+}
+
+get_available_datasets()
+
+get_dataset_table_names <- function(dataset_id, start_date = NULL, end_date = NULL) {
+  available_datasets <- get_available_datasets()
+
+  if (!dataset_id %in% available_datasets) {
+    stop("The provided 'dataset_id' is not available. Run 'get_available_datasets()' to see all available datasets.", call. = FALSE)
+  }
+
+  base_endpoint <- "https://opendata.nhsbsa.net/api/3/action/"
+  package_show_method <- "package_show?id="
+
+  metadata_response <- GET(paste0(base_endpoint, package_show_method, dataset_id))
+  resources_table <- content(metadata_response)$result$resources
+
+  dataset_tables <- tibble(
+    table_name = map_chr(
+      resources_table,
+      "bq_table_name",
+      .default = NA_character_
+    ),
+    date = ym(str_extract(table_name, "\\d{6}"))
+  ) |>
+    relocate(date)
+
+  if (is.null(start_date)) {
+    start_date <- as.Date(min(dataset_tables$date))
+  }
+
+  if (is.null(end_date)) {
+    end_date <- as.Date(max(dataset_tables$date))
+  }
+
+  dataset_tables <- dataset_tables |>
+    filter(between(date, as.Date(start_date), as.Date(end_date)))
+
+  dataset_tables
+}
+
+get_dataset_table_names("prescription-cost-analysis-pca-monthly-data", "2024-09-01")
+
+construct_sql_query <- function(table_name, sql_query) {
+  gsub("{FROM_TABLE}", sprintf("FROM `%s`", table_name), sql_query, fixed = TRUE)
+}
+
+get_nhsbsa_data <- function(dataset_id, sql, start_date = NULL, end_date = NULL) {
+  base_endpoint <- "https://opendata.nhsbsa.net/api/3/action/"
+  action_method <- "datastore_search_sql?"
+
+  table_names <- get_dataset_table_names(dataset_id, start_date, end_date)$table_name
+
+  async_api_calls <- paste0(
+    base_endpoint,
+    action_method,
+    "resource_id=", table_names,
+    "&sql=", URLencode(map_chr(table_names, construct_sql_query, sql))
+  )
+
+  responses <- crul::Async$new(urls = async_api_calls)$get()
+
+  df_tmp <- bind_rows(map(responses, ~ as_tibble(jsonlite::fromJSON(.x$parse("UTF-8"))$result$result$records)))
+
+  df_tmp |>
+    janitor::clean_names() |>
+    mutate(year_month = ym(year_month)) |>
+    select(date = year_month, everything())
+}
+
+sql <- ("
+  SELECT *
+  {FROM_TABLE}
+  WHERE PHARMACY_ADVANCED_SERVICE = 'Pharmacy First Clinical Pathways'
+  ")
+
+df_validate <- get_nhsbsa_data("prescription-cost-analysis-pca-monthly-data", sql, start_date = "2024-02-01")
+
+names(df_validate)
+unique(df_validate$pharmacy_advanced_service)
+
+pf_medication_validation_data <- df_validate |>
+  select(date, snomed_code, pharmacy_advanced_service, bnf_section, bnf_paragraph, items) |>
+  group_by(date, pharmacy_advanced_service, bnf_paragraph) |>
+  summarise(count = sum(items, na.rm = TRUE)) |>
+  ungroup()
+
+write_csv(pf_medication_validation_data, here("lib", "validation", "data", "pf_medication_validation_data.csv"))

lib/validation/data/pf_medication_validation_data.csv

@@ -0,0 +1,78 @@
+date,pharmacy_advanced_service,bnf_paragraph,count
+2024-02-01,Pharmacy First Clinical Pathways,Antibacterial preparations,3113
+2024-02-01,Pharmacy First Clinical Pathways,Drugs used in nasal allergy,7476
+2024-02-01,Pharmacy First Clinical Pathways,Herpesvirus infections,3336
+2024-02-01,Pharmacy First Clinical Pathways,Individually formulated preparations bought in,13
+2024-02-01,Pharmacy First Clinical Pathways,Macrolides,4528
+2024-02-01,Pharmacy First Clinical Pathways,Otitis externa,3831
+2024-02-01,Pharmacy First Clinical Pathways,Penicillins,42008
+2024-02-01,Pharmacy First Clinical Pathways,Preparations for minor cuts and abrasions,2251
+2024-02-01,Pharmacy First Clinical Pathways,Tetracyclines,1002
+2024-02-01,Pharmacy First Clinical Pathways,Urinary-tract infections,26877
+2024-03-01,Pharmacy First Clinical Pathways,Antibacterial preparations,3386
+2024-03-01,Pharmacy First Clinical Pathways,Drugs used in nasal allergy,6588
+2024-03-01,Pharmacy First Clinical Pathways,Herpesvirus infections,3451
+2024-03-01,Pharmacy First Clinical Pathways,Individually formulated preparations bought in,32
+2024-03-01,Pharmacy First Clinical Pathways,Macrolides,6201
+2024-03-01,Pharmacy First Clinical Pathways,Otitis externa,5325
+2024-03-01,Pharmacy First Clinical Pathways,Penicillins,56231
+2024-03-01,Pharmacy First Clinical Pathways,Preparations for minor cuts and abrasions,1820
+2024-03-01,Pharmacy First Clinical Pathways,Tetracyclines,1028
+2024-03-01,Pharmacy First Clinical Pathways,Urinary-tract infections,29345
+2024-04-01,Pharmacy First Clinical Pathways,Antibacterial preparations,3834
+2024-04-01,Pharmacy First Clinical Pathways,Drugs used in nasal allergy,6598
+2024-04-01,Pharmacy First Clinical Pathways,Herpesvirus infections,3818
+2024-04-01,Pharmacy First Clinical Pathways,Individually formulated preparations bought in,5
+2024-04-01,Pharmacy First Clinical Pathways,Macrolides,6530
+2024-04-01,Pharmacy First Clinical Pathways,Otitis externa,4375
+2024-04-01,Pharmacy First Clinical Pathways,Penicillins,58070
+2024-04-01,Pharmacy First Clinical Pathways,Preparations for minor cuts and abrasions,1673
+2024-04-01,Pharmacy First Clinical Pathways,Tetracyclines,1036
+2024-04-01,Pharmacy First Clinical Pathways,Urinary-tract infections,35592
+2024-05-01,Pharmacy First Clinical Pathways,Antibacterial preparations,3224
+2024-05-01,Pharmacy First Clinical Pathways,Drugs used in nasal allergy,5789
+2024-05-01,Pharmacy First Clinical Pathways,Herpesvirus infections,4065
+2024-05-01,Pharmacy First Clinical Pathways,Macrolides,7864
+2024-05-01,Pharmacy First Clinical Pathways,Otitis externa,5457
+2024-05-01,Pharmacy First Clinical Pathways,Penicillins,68200
+2024-05-01,Pharmacy First Clinical Pathways,Preparations for minor cuts and abrasions,1558
+2024-05-01,Pharmacy First Clinical Pathways,Tetracyclines,935
+2024-05-01,Pharmacy First Clinical Pathways,Urinary-tract infections,38165
+2024-06-01,Pharmacy First Clinical Pathways,Antibacterial preparations,3262
+2024-06-01,Pharmacy First Clinical Pathways,Drugs used in nasal allergy,5671
+2024-06-01,Pharmacy First Clinical Pathways,Herpesvirus infections,3979
+2024-06-01,Pharmacy First Clinical Pathways,Macrolides,7910
+2024-06-01,Pharmacy First Clinical Pathways,Otitis externa,4541
+2024-06-01,Pharmacy First Clinical Pathways,Penicillins,67691
+2024-06-01,Pharmacy First Clinical Pathways,Preparations for minor cuts and abrasions,1625
+2024-06-01,Pharmacy First Clinical Pathways,Tetracyclines,813
+2024-06-01,Pharmacy First Clinical Pathways,Urinary-tract infections,36995
+2024-07-01,Pharmacy First Clinical Pathways,Antibacterial preparations,3327
+2024-07-01,Pharmacy First Clinical Pathways,Drugs used in nasal allergy,5022
+2024-07-01,Pharmacy First Clinical Pathways,Herpesvirus infections,4332
+2024-07-01,Pharmacy First Clinical Pathways,Macrolides,9023
+2024-07-01,Pharmacy First Clinical Pathways,Otitis externa,4950
+2024-07-01,Pharmacy First Clinical Pathways,Penicillins,76597
+2024-07-01,Pharmacy First Clinical Pathways,Preparations for minor cuts and abrasions,1664
+2024-07-01,Pharmacy First Clinical Pathways,Tetracyclines,819
+2024-07-01,Pharmacy First Clinical Pathways,Urinary-tract infections,43848
+2024-08-01,Pharmacy First Clinical Pathways,Antibacterial preparations,3628
+2024-08-01,Pharmacy First Clinical Pathways,Drugs used in nasal allergy,3996
+2024-08-01,Pharmacy First Clinical Pathways,Herpesvirus infections,4372
+2024-08-01,Pharmacy First Clinical Pathways,Individually formulated preparations bought in,9
+2024-08-01,Pharmacy First Clinical Pathways,Macrolides,8538
+2024-08-01,Pharmacy First Clinical Pathways,Otitis externa,5554
+2024-08-01,Pharmacy First Clinical Pathways,Penicillins,70102
+2024-08-01,Pharmacy First Clinical Pathways,Preparations for minor cuts and abrasions,1761
+2024-08-01,Pharmacy First Clinical Pathways,Tetracyclines,572
+2024-08-01,Pharmacy First Clinical Pathways,Urinary-tract infections,45658
+2024-09-01,Pharmacy First Clinical Pathways,Antibacterial preparations,4165
+2024-09-01,Pharmacy First Clinical Pathways,Drugs used in nasal allergy,5464
+2024-09-01,Pharmacy First Clinical Pathways,Herpesvirus infections,4042
+2024-09-01,Pharmacy First Clinical Pathways,Individually formulated preparations bought in,11
+2024-09-01,Pharmacy First Clinical Pathways,Macrolides,7177
+2024-09-01,Pharmacy First Clinical Pathways,Otitis externa,4644
+2024-09-01,Pharmacy First Clinical Pathways,Penicillins,58958
+2024-09-01,Pharmacy First Clinical Pathways,Preparations for minor cuts and abrasions,1939
+2024-09-01,Pharmacy First Clinical Pathways,Tetracyclines,735
+2024-09-01,Pharmacy First Clinical Pathways,Urinary-tract infections,47327