Merge pull request #95 from opensafely/viv3ckj/extend-study-dates

Viv3ckj/extend-validation-and-study-periods

analysis/config.py

@@ -1,19 +1,19 @@
-# measures_definition_pf_breakdown.py
+# measures_definition_pf_breakdown.py (01/11/2023 - 01/11/2024)
 start_date_measure_pf_breakdown = "2023-11-01"
 monthly_intervals_measure_pf_breakdown = 12
 
-# measures_definition_pf_condition_provider.py
+# measures_definition_pf_condition_provider.py (01/01/2023 - 01/11/2024)
 start_date_measure_condition_provider = "2023-01-01"
 monthly_intervals_measure_condition_provider = 22
 
-# measures_definition_pf_descriptive_stats.py
+# measures_definition_pf_descriptive_stats.py (01/02/2024 - 01/11/2024)
 start_date_measure_descriptive_stats = "2024-02-01"
 monthly_intervals_measure_descriptive_stats = 9
 
-# measures_definition_pf_medications.py
+# measures_definition_pf_medications.py (01/11/2023 - 01/11/2024)
 start_date_measure_medications = "2023-11-01"
-monthly_intervals_measure_medications = 9
+monthly_intervals_measure_medications = 12
 
-# measures_definition_pf_consultation_pf_counts.py
+# measures_definition_pf_consultation_pf_counts.py (01/02/2024 - 01/11/2024)
 start_date_measure_med_counts = "2024-02-01"
-monthly_intervals_measure_med_counts = 6
+monthly_intervals_measure_med_counts = 9

lib/functions/get_pf_medication_validation_data.R

@@ -7,6 +7,7 @@ library("dplyr")
 library("lubridate")
 library("tidyverse")
 library("readr")
+library("retry")
 
 base_endpoint <- "https://opendata.nhsbsa.net/api/3/action/"
 package_list_method <- "package_list"
@@ -79,18 +80,61 @@ get_nhsbsa_data <- function(dataset_id, sql, start_date = NULL, end_date = NULL)
   base_endpoint <- "https://opendata.nhsbsa.net/api/3/action/"
   action_method <- "datastore_search_sql?"
 
+  # Retrieve table names for specified dataset and data range
   table_names <- get_dataset_table_names(dataset_id, start_date, end_date)$table_name
 
+  # Construct URLs for querying the API, one URL per table, modifying each table with table_name
   async_api_calls <- paste0(
     base_endpoint,
     action_method,
     "resource_id=", table_names,
     "&sql=", URLencode(map_chr(table_names, construct_sql_query, sql))
   )
 
-  responses <- crul::Async$new(urls = async_api_calls)$get()
+  # Initialise an empty list to store API responses
+  responses <- list()
+
+  # Loop over each API call URL, by:
+  # 1) getting URL for current table, 
+  # 2) initialise retry counter, 
+  # 3) set max number of retry attempts, 
+  # 4) flag to track if request was successful
+  for (i in seq_along(async_api_calls)) {
+    url <- async_api_calls[i]
+    attempt <- 1
+    max_attempts <- 5
+    success <- FALSE
+
+    # Retry logic: keep trying until request is successful or max attempts reached
+    while (attempt <= max_attempts && !success) {
+      response <- tryCatch({
+        crul::HttpClient$new(url)$get()
+      }, error = function(e) {
+        NULL
+      })
+
+      if (!is.null(response) && response$status_code == 200) {
+        # If response is valid and status code = 200, mark as successful
+        success <- TRUE
+        responses[[i]] <- response
+      } else {
+        message(sprintf("Attempt %d failed for URL: %s", attempt, url))
+        Sys.sleep(2 ^ attempt)
+        attempt <- attempt + 1
+      }
+    }
+
+    if (!success) {
+      stop(sprintf("Failed to fetch data from URL: %s after %d attempts", url, max_attempts))
+    }
+  }
+  # Parse the successful responses to extract the data
+  results <- map(responses, ~ {
+    content_raw <- .x$parse("UTF-8")
+    fromJSON(readLines(textConnection(content_raw), warn = FALSE))$result$result$records
+  })
 
-  df_tmp <- bind_rows(map(responses, ~ as_tibble(jsonlite::fromJSON(.x$parse("UTF-8"))$result$result$records)))
+  df_tmp <- bind_rows(map(results, ~ as_tibble(.x, .default = tibble())))
 
   df_tmp |>
     janitor::clean_names() |>

lib/validation/data/pf_consultation_validation_data.csv

@@ -69,3 +69,10 @@ date,consultation_type,count
 2024-08-01,shingles,5151
 2024-08-01,sinusitis,10970
 2024-08-01,uncomplicated_uti,53389
+2024-09-01,acute_otitis_media,15680
+2024-09-01,acute_sore_throat,49991
+2024-09-01,impetigo,8390
+2024-09-01,infected_insect_bites,25348
+2024-09-01,shingles,4867
+2024-09-01,sinusitis,15059
+2024-09-01,uncomplicated_uti,56235

lib/validation/data/pf_medication_validation_data.csv

@@ -76,3 +76,13 @@ date,pharmacy_advanced_service,bnf_paragraph,count
 2024-09-01,Pharmacy First Clinical Pathways,Preparations for minor cuts and abrasions,1939
 2024-09-01,Pharmacy First Clinical Pathways,Tetracyclines,735
 2024-09-01,Pharmacy First Clinical Pathways,Urinary-tract infections,47327
+2024-10-01,Pharmacy First Clinical Pathways,Antibacterial preparations,4491
+2024-10-01,Pharmacy First Clinical Pathways,Drugs used in nasal allergy,7939
+2024-10-01,Pharmacy First Clinical Pathways,Herpesvirus infections,4458
+2024-10-01,Pharmacy First Clinical Pathways,Individually formulated preparations bought in,12
+2024-10-01,Pharmacy First Clinical Pathways,Macrolides,7949
+2024-10-01,Pharmacy First Clinical Pathways,Otitis externa,5888
+2024-10-01,Pharmacy First Clinical Pathways,Penicillins,65765
+2024-10-01,Pharmacy First Clinical Pathways,Preparations for minor cuts and abrasions,2349
+2024-10-01,Pharmacy First Clinical Pathways,Tetracyclines,1182
+2024-10-01,Pharmacy First Clinical Pathways,Urinary-tract infections,52648