Merge pull request #799 from communitiesuk/FPASF-622_pathfinders-r2-e…

…xtra-year FPASF-622 - Support additional financial year for Pathfinders R2
communitiesuk · Oct 25, 2024 · da3706f · da3706f
2 parents 3742dad + 04abac0
commit da3706f
Show file tree

Hide file tree

Showing 9 changed files with 127 additions and 21 deletions.
diff --git a/data_store/table_extraction/config/pf_r2_config.py b/data_store/table_extraction/config/pf_r2_config.py
@@ -212,6 +212,7 @@
                 "Total cumulative outputs to date, Total",
                 "Financial year 2024 to 2025, Total",
                 "Financial year 2025 to 2026, Total",
+                "Financial year 2026 to 2027, Total",
                 "Grand total",
                 "Variance % (Between MR and Baseline)",
             ],
@@ -232,7 +233,11 @@
                 "Financial year 2025 to 2026, (Jul to Sep), Forecast": float_column(),
                 "Financial year 2025 to 2026, (Oct to Dec), Forecast": float_column(),
                 "Financial year 2025 to 2026, (Jan to Mar), Forecast": float_column(),
-                "April 2026 and after, Total": float_column(),
+                "Financial year 2026 to 2027, (Apr to Jun), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Jul to Sep), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Oct to Dec), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Jan to Mar), Forecast": float_column(),
+                "April 2027 and after, Total": float_column(),
             },
             unique=["Intervention theme", "Output", "Unit of measurement"],
             report_duplicates="exclude_first",
@@ -251,6 +256,7 @@
                 "Total cumulative outputs to date, Total",
                 "Financial year 2024 to 2025, Total",
                 "Financial year 2025 to 2026, Total",
+                "Financial year 2026 to 2027, Total",
                 "Grand total",
                 "Variance % (Between MR and Baseline)",
             ],
@@ -271,7 +277,11 @@
                 "Financial year 2025 to 2026, (Jul to Sep), Forecast": float_column(),
                 "Financial year 2025 to 2026, (Oct to Dec), Forecast": float_column(),
                 "Financial year 2025 to 2026, (Jan to Mar), Forecast": float_column(),
-                "April 2026 and after, Total": float_column(),
+                "Financial year 2026 to 2027, (Apr to Jun), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Jul to Sep), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Oct to Dec), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Jan to Mar), Forecast": float_column(),
+                "April 2027 and after, Total": float_column(),
             },
             unique=["Intervention theme", "Output", "Unit of measurement"],
             report_duplicates="exclude_first",
@@ -290,6 +300,7 @@
                 "Total cumulative outcomes to date, Total",
                 "Financial year 2024 to 2025, Total",
                 "Financial year 2025 to 2026, Total",
+                "Financial year 2026 to 2027, Total",
                 "Grand total",
                 "Variance % (Between MR and Baseline)",
             ],
@@ -310,7 +321,11 @@
                 "Financial year 2025 to 2026, (Jul to Sep), Forecast": float_column(),
                 "Financial year 2025 to 2026, (Oct to Dec), Forecast": float_column(),
                 "Financial year 2025 to 2026, (Jan to Mar), Forecast": float_column(),
-                "April 2026 and after, Total": float_column(),
+                "Financial year 2026 to 2027, (Apr to Jun), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Jul to Sep), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Oct to Dec), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Jan to Mar), Forecast": float_column(),
+                "April 2027 and after, Total": float_column(),
             },
             unique=["Intervention theme", "Outcome", "Unit of measurement"],
             report_duplicates="exclude_first",
@@ -329,6 +344,7 @@
                 "Total cumulative outcomes to date, Total",
                 "Financial year 2024 to 2025, Total",
                 "Financial year 2025 to 2026, Total",
+                "Financial year 2026 to 2027, Total",
                 "Grand total",
                 "Variance % (Between MR and Baseline)",
             ],
@@ -349,7 +365,11 @@
                 "Financial year 2025 to 2026, (Jul to Sep), Forecast": float_column(),
                 "Financial year 2025 to 2026, (Oct to Dec), Forecast": float_column(),
                 "Financial year 2025 to 2026, (Jan to Mar), Forecast": float_column(),
-                "April 2026 and after, Total": float_column(),
+                "Financial year 2026 to 2027, (Apr to Jun), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Jul to Sep), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Oct to Dec), Forecast": float_column(),
+                "Financial year 2026 to 2027, (Jan to Mar), Forecast": float_column(),
+                "April 2027 and after, Total": float_column(),
             },
             unique=["Intervention theme", "Outcome", "Unit of measurement"],
             report_duplicates="exclude_first",
@@ -382,6 +402,7 @@
                 "Total cumulative actuals to date, Total",
                 "Financial year 2024 to 2025, Total",
                 "Financial year 2025 to 2026, Total",
+                "Financial year 2026 to 2027, Total",
                 "Grand total",
             ],
         ),
@@ -399,6 +420,10 @@
                 "Financial year 2025 to 2026, (Jul to Sep), Forecast": float_column(checks.greater_than_or_equal_to(0)),
                 "Financial year 2025 to 2026, (Oct to Dec), Forecast": float_column(checks.greater_than_or_equal_to(0)),
                 "Financial year 2025 to 2026, (Jan to Mar), Forecast": float_column(checks.greater_than_or_equal_to(0)),
+                "Financial year 2026 to 2027, (Apr to Jun), Forecast": float_column(checks.greater_than_or_equal_to(0)),
+                "Financial year 2026 to 2027, (Jul to Sep), Forecast": float_column(checks.greater_than_or_equal_to(0)),
+                "Financial year 2026 to 2027, (Oct to Dec), Forecast": float_column(checks.greater_than_or_equal_to(0)),
+                "Financial year 2026 to 2027, (Jan to Mar), Forecast": float_column(checks.greater_than_or_equal_to(0)),
             },
         ),
     ),
@@ -415,6 +440,7 @@
                 "Total cumulative actuals to date, Total",
                 "Financial year 2024 to 2025, Total",
                 "Financial year 2025 to 2026, Total",
+                "Financial year 2026 to 2027, Total",
                 "Grand total",
             ],
         ),
@@ -432,6 +458,10 @@
                 "Financial year 2025 to 2026, (Jul to Sep), Forecast": float_column(checks.greater_than_or_equal_to(0)),
                 "Financial year 2025 to 2026, (Oct to Dec), Forecast": float_column(checks.greater_than_or_equal_to(0)),
                 "Financial year 2025 to 2026, (Jan to Mar), Forecast": float_column(checks.greater_than_or_equal_to(0)),
+                "Financial year 2026 to 2027, (Apr to Jun), Forecast": float_column(checks.greater_than_or_equal_to(0)),
+                "Financial year 2026 to 2027, (Jul to Sep), Forecast": float_column(checks.greater_than_or_equal_to(0)),
+                "Financial year 2026 to 2027, (Oct to Dec), Forecast": float_column(checks.greater_than_or_equal_to(0)),
+                "Financial year 2026 to 2027, (Jan to Mar), Forecast": float_column(checks.greater_than_or_equal_to(0)),
             },
         ),
     ),

diff --git a/data_store/transformation/pathfinders/pf_transform_r2.py b/data_store/transformation/pathfinders/pf_transform_r2.py
@@ -45,6 +45,22 @@
         "start": datetime(2026, 1, 1),
         "end": datetime(2026, 3, 31, 23, 59, 59),
     },
+    "Financial year 2026 to 2027, (Apr to Jun)": {
+        "start": datetime(2026, 4, 1),
+        "end": datetime(2026, 6, 30, 23, 59, 59),
+    },
+    "Financial year 2026 to 2027, (Jul to Sep)": {
+        "start": datetime(2026, 7, 1),
+        "end": datetime(2026, 9, 30, 23, 59, 59),
+    },
+    "Financial year 2026 to 2027, (Oct to Dec)": {
+        "start": datetime(2026, 10, 1),
+        "end": datetime(2026, 12, 31, 23, 59, 59),
+    },
+    "Financial year 2026 to 2027, (Jan to Mar)": {
+        "start": datetime(2027, 1, 1),
+        "end": datetime(2027, 3, 31, 23, 59, 59),
+    },
 }
 
 
@@ -85,7 +101,23 @@
         "start": datetime(2026, 1, 1),
         "end": datetime(2026, 3, 31, 23, 59, 59),
     },
-    "April 2026 and after": {"start": datetime(2026, 4, 1), "end": None},
+    "Financial year 2026 to 2027, (Apr to Jun)": {
+        "start": datetime(2026, 4, 1),
+        "end": datetime(2026, 6, 30, 23, 59, 59),
+    },
+    "Financial year 2026 to 2027, (Jul to Sep)": {
+        "start": datetime(2026, 7, 1),
+        "end": datetime(2026, 9, 30, 23, 59, 59),
+    },
+    "Financial year 2026 to 2027, (Oct to Dec)": {
+        "start": datetime(2026, 10, 1),
+        "end": datetime(2026, 12, 31, 23, 59, 59),
+    },
+    "Financial year 2026 to 2027, (Jan to Mar)": {
+        "start": datetime(2027, 1, 1),
+        "end": datetime(2027, 3, 31, 23, 59, 59),
+    },
+    "April 2027 and after": {"start": datetime(2027, 4, 1), "end": None},
 }
 
 OUTCOME_REPORTING_PERIOD_HEADERS_TO_DATES = {
@@ -125,7 +157,23 @@
         "start": datetime(2026, 1, 1),
         "end": datetime(2026, 3, 31, 23, 59, 59),
     },
-    "April 2026 and after": {"start": datetime(2026, 4, 1), "end": None},
+    "Financial year 2026 to 2027, (Apr to Jun)": {
+        "start": datetime(2026, 4, 1),
+        "end": datetime(2026, 6, 30, 23, 59, 59),
+    },
+    "Financial year 2026 to 2027, (Jul to Sep)": {
+        "start": datetime(2026, 7, 1),
+        "end": datetime(2026, 9, 30, 23, 59, 59),
+    },
+    "Financial year 2026 to 2027, (Oct to Dec)": {
+        "start": datetime(2026, 10, 1),
+        "end": datetime(2026, 12, 31, 23, 59, 59),
+    },
+    "Financial year 2026 to 2027, (Jan to Mar)": {
+        "start": datetime(2027, 1, 1),
+        "end": datetime(2027, 3, 31, 23, 59, 59),
+    },
+    "April 2027 and after": {"start": datetime(2027, 4, 1), "end": None},
 }
 
 

diff --git a/data_store/validation/initial_validation/schemas.py b/data_store/validation/initial_validation/schemas.py
@@ -324,8 +324,8 @@
         sheet="Metadata",
         row=1,
         column=0,
-        expected_values=("2",),
-        error_message="The expected reporting round is 2",
+        expected_values=("2v2",),
+        error_message="The expected reporting round is 2v2",
     ),
     BasicCheck(
         sheet="Metadata",

diff --git a/data_store/validation/pathfinders/r2_consts.py b/data_store/validation/pathfinders/r2_consts.py
@@ -28,6 +28,10 @@ class PFEnums:
         "Q2 2025/26: Jul 2025 - Sep 2025",
         "Q3 2025/26: Oct 2025 - Dec 2025",
         "Q4 2025/26: Jan 2026 - Mar 2026",
+        "Q1 2026/27: Apr 2026 - Jun 2026",
+        "Q2 2026/27: Jul 2026 - Sep 2026",
+        "Q3 2026/27: Oct 2026 - Dec 2026",
+        "Q4 2026/27: Jan 2027 - Mar 2027",
     ]
     RISK_CATEGORIES = [
         "Arm’s length body risks",

diff --git a/tests/data_store_tests/transformation_tests/pathfinders/test_pf_transform_r2.py b/tests/data_store_tests/transformation_tests/pathfinders/test_pf_transform_r2.py
@@ -200,9 +200,13 @@ def test__funding_data(
         "Financial year 2025 to 2026, (Jul to Sep), Forecast",
         "Financial year 2025 to 2026, (Oct to Dec), Forecast",
         "Financial year 2025 to 2026, (Jan to Mar), Forecast",
+        "Financial year 2026 to 2027, (Apr to Jun), Forecast",
+        "Financial year 2026 to 2027, (Jul to Sep), Forecast",
+        "Financial year 2026 to 2027, (Oct to Dec), Forecast",
+        "Financial year 2026 to 2027, (Jan to Mar), Forecast",
     ]
     first_start_date = "2024-01-01"
-    last_start_date = "2026-01-01"
+    last_start_date = "2027-01-01"
     start_dates = list(pd.date_range(start=first_start_date, end=last_start_date, freq="QS"))
 
     # The actual start date for "Total cumulative actuals to date, (Up to and including Mar 2024)" is 2019-01-01,
@@ -213,7 +217,7 @@ def test__funding_data(
         ((start_dates[i] - pd.Timedelta(days=1)).replace(hour=23, minute=59, second=59))
         for i in range(1, len(start_dates))
     ]
-    end_dates.append(pd.Timestamp("2026-03-31 23:59:59"))
+    end_dates.append(pd.Timestamp("2027-03-31 23:59:59"))
     expected_df = pd.DataFrame(
         {
             "Programme ID": ["PF-BOL"] * len(funding_source_types) * len(reporting_periods) * 2,
@@ -243,7 +247,7 @@ def test__outputs(
         programme_name_to_id_mapping=mock_programme_name_to_id_mapping,
     )
     first_start_date = "2024-01-01"
-    last_start_date = "2026-04-01"
+    last_start_date = "2027-04-01"
     start_dates = list(pd.date_range(start=first_start_date, end=last_start_date, freq="QS"))
 
     # The actual start date for "Total cumulative actuals to date, (Up to and including Mar 2024)" is 2019-01-01,
@@ -291,7 +295,7 @@ def test__outcomes(
         programme_name_to_id_mapping=mock_programme_name_to_id_mapping,
     )
     first_start_date = "2024-01-01"
-    last_start_date = "2026-04-01"
+    last_start_date = "2027-04-01"
     start_dates = list(pd.date_range(start=first_start_date, end=last_start_date, freq="QS"))
 
     # The actual start date for "Total cumulative actuals to date, (Up to and including Mar 2024)" is 2019-01-01,

diff --git a/tests/integration_tests/mock_pf_returns/PF_Round_2_General_Validation_Failures.xlsx b/tests/integration_tests/mock_pf_returns/PF_Round_2_General_Validation_Failures.xlsx
diff --git a/tests/integration_tests/mock_pf_returns/PF_Round_2_Success.xlsx b/tests/integration_tests/mock_pf_returns/PF_Round_2_Success.xlsx
diff --git a/tests/integration_tests/test_ingest_component_pathfinders.py b/tests/integration_tests/test_ingest_component_pathfinders.py
@@ -89,7 +89,7 @@ def test_ingest_pf_r2_file_success(test_client, pathfinders_round_2_file_success
         reporting_round=2,
         do_load=False,
         auth={
-            "Programme": ("Test Council",),
+            "Programme": ("Bolton Council",),
             "Fund Types": ("Pathfinders",),
         },
     )
@@ -100,9 +100,9 @@ def test_ingest_pf_r2_file_success(test_client, pathfinders_round_2_file_success
         "loaded": False,
         "metadata": {
             "FundType_ID": "PF",
-            "Organisation": "Test Council",
-            "Programme ID": "PF-ANO",
-            "Programme Name": "Test Council",
+            "Organisation": "Bolton Council",
+            "Programme ID": "PF-BOL",
+            "Programme Name": "Bolton Council",
         },
         "status": 200,
         "title": "success",
@@ -311,7 +311,7 @@ def test_ingest_pf_r2_general_validation_errors(
         reporting_round=2,
         do_load=False,
         auth={
-            "Programme": ("Test Council",),
+            "Programme": ("Bolton Council",),
             "Fund Types": ("Pathfinders",),
         },
     )

diff --git a/tests/resources/pathfinders/pf_r2_extracted_data.py b/tests/resources/pathfinders/pf_r2_extracted_data.py
@@ -54,7 +54,11 @@ def get_pf_r2_extracted_data():
             "Financial year 2025 to 2026, (Jul to Sep), Forecast": [1.0],
             "Financial year 2025 to 2026, (Oct to Dec), Forecast": [1.0],
             "Financial year 2025 to 2026, (Jan to Mar), Forecast": [1.0],
-            "April 2026 and after, Total": [1.0],
+            "Financial year 2026 to 2027, (Apr to Jun), Forecast": [1.0],
+            "Financial year 2026 to 2027, (Jul to Sep), Forecast": [1.0],
+            "Financial year 2026 to 2027, (Oct to Dec), Forecast": [1.0],
+            "Financial year 2026 to 2027, (Jan to Mar), Forecast": [1.0],
+            "April 2027 and after, Total": [1.0],
         }
     )
     bespoke_outputs = pd.DataFrame(
@@ -71,7 +75,11 @@ def get_pf_r2_extracted_data():
             "Financial year 2025 to 2026, (Jul to Sep), Forecast": [5.0],
             "Financial year 2025 to 2026, (Oct to Dec), Forecast": [5.0],
             "Financial year 2025 to 2026, (Jan to Mar), Forecast": [5.0],
-            "April 2026 and after, Total": [5.0],
+            "Financial year 2026 to 2027, (Apr to Jun), Forecast": [5.0],
+            "Financial year 2026 to 2027, (Jul to Sep), Forecast": [5.0],
+            "Financial year 2026 to 2027, (Oct to Dec), Forecast": [5.0],
+            "Financial year 2026 to 2027, (Jan to Mar), Forecast": [5.0],
+            "April 2027 and after, Total": [5.0],
         }
     )
     outcomes = pd.DataFrame(
@@ -88,7 +96,11 @@ def get_pf_r2_extracted_data():
             "Financial year 2025 to 2026, (Jul to Sep), Forecast": [1.0],
             "Financial year 2025 to 2026, (Oct to Dec), Forecast": [1.0],
             "Financial year 2025 to 2026, (Jan to Mar), Forecast": [1.0],
-            "April 2026 and after, Total": [1.0],
+            "Financial year 2026 to 2027, (Apr to Jun), Forecast": [1.0],
+            "Financial year 2026 to 2027, (Jul to Sep), Forecast": [1.0],
+            "Financial year 2026 to 2027, (Oct to Dec), Forecast": [1.0],
+            "Financial year 2026 to 2027, (Jan to Mar), Forecast": [1.0],
+            "April 2027 and after, Total": [1.0],
         }
     )
     bespoke_outcomes = pd.DataFrame(
@@ -105,7 +117,11 @@ def get_pf_r2_extracted_data():
             "Financial year 2025 to 2026, (Jul to Sep), Forecast": [],
             "Financial year 2025 to 2026, (Oct to Dec), Forecast": [],
             "Financial year 2025 to 2026, (Jan to Mar), Forecast": [],
-            "April 2026 and after, Total": [],
+            "Financial year 2026 to 2027, (Apr to Jun), Forecast": [],
+            "Financial year 2026 to 2027, (Jul to Sep), Forecast": [],
+            "Financial year 2026 to 2027, (Oct to Dec), Forecast": [],
+            "Financial year 2026 to 2027, (Jan to Mar), Forecast": [],
+            "April 2027 and after, Total": [],
         }
     )
     risks = pd.DataFrame(
@@ -139,6 +155,10 @@ def get_pf_r2_extracted_data():
             "Financial year 2025 to 2026, (Jul to Sep), Forecast": [1.0, 0.0, 0.0, 0.0, 0.0],
             "Financial year 2025 to 2026, (Oct to Dec), Forecast": [1.0, 0.0, 0.0, 0.0, 0.0],
             "Financial year 2025 to 2026, (Jan to Mar), Forecast": [1.0, 0.0, 0.0, 0.0, 0.0],
+            "Financial year 2026 to 2027, (Apr to Jun), Forecast": [1.0, 0.0, 0.0, 0.0, 0.0],
+            "Financial year 2026 to 2027, (Jul to Sep), Forecast": [1.0, 0.0, 0.0, 0.0, 0.0],
+            "Financial year 2026 to 2027, (Oct to Dec), Forecast": [1.0, 0.0, 0.0, 0.0, 0.0],
+            "Financial year 2026 to 2027, (Jan to Mar), Forecast": [1.0, 0.0, 0.0, 0.0, 0.0],
         }
     )
     forecast_and_actual_spend_revenue = forecast_and_actual_spend_capital.copy()