Skip to content

Commit

Permalink
added files from FluSight-forecast-hub for tests, and for running on …
Browse files Browse the repository at this point in the history 
…actual hub
  • Loading branch information
@matthewcornell
matthewcornell committed Oct 15, 2024
1 parent 2c5dbbc commit 3fabadc
Show file tree
Hide file tree
Showing 49 changed files with 2,115 additions and 0 deletions.
81 changes: 81 additions & 0 deletions tests/expected/FluSight-forecast-hub/predtimechart-options.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
{"target_variables": [
{
"value": "wk inc flu hosp",
"text": "incident influenza hospitalizations",
"plot_text": "incident influenza hospitalizations"
}
],
"initial_target_var": "wk inc flu hosp",
"task_ids": {
"location": [
{"value": "US", "text": "US"},
{"value": "01", "text": "01"},
{"value": "02", "text": "02"},
{"value": "04", "text": "04"},
{"value": "05", "text": "05"},
{"value": "06", "text": "06"},
{"value": "08", "text": "08"},
{"value": "09", "text": "09"},
{"value": "10", "text": "10"},
{"value": "11", "text": "11"},
{"value": "12", "text": "12"},
{"value": "13", "text": "13"},
{"value": "15", "text": "15"},
{"value": "16", "text": "16"},
{"value": "17", "text": "17"},
{"value": "18", "text": "18"},
{"value": "19", "text": "19"},
{"value": "20", "text": "20"},
{"value": "21", "text": "21"},
{"value": "22", "text": "22"},
{"value": "23", "text": "23"},
{"value": "24", "text": "24"},
{"value": "25", "text": "25"},
{"value": "26", "text": "26"},
{"value": "27", "text": "27"},
{"value": "28", "text": "28"},
{"value": "29", "text": "29"},
{"value": "30", "text": "30"},
{"value": "31", "text": "31"},
{"value": "32", "text": "32"},
{"value": "33", "text": "33"},
{"value": "34", "text": "34"},
{"value": "35", "text": "35"},
{"value": "36", "text": "36"},
{"value": "37", "text": "37"},
{"value": "38", "text": "38"},
{"value": "39", "text": "39"},
{"value": "40", "text": "40"},
{"value": "41", "text": "41"},
{"value": "42", "text": "42"},
{"value": "44", "text": "44"},
{"value": "45", "text": "45"},
{"value": "46", "text": "46"},
{"value": "47", "text": "47"},
{"value": "48", "text": "48"},
{"value": "49", "text": "49"},
{"value": "50", "text": "50"},
{"value": "51", "text": "51"},
{"value": "53", "text": "53"},
{"value": "54", "text": "54"},
{"value": "55", "text": "55"},
{"value": "56", "text": "56"},
{"value": "72", "text": "72"}
]
},
"initial_task_ids": {"location": "US"},
"intervals": ["0%", "50%", "95%"],
"initial_interval": "95%",
"available_as_ofs": {
"wk inc flu hosp": [
"2023-10-07", "2023-10-14", "2023-10-21", "2023-10-28", "2023-11-04", "2023-11-11", "2023-11-18", "2023-11-25", "2023-12-02", "2023-12-09", "2023-12-16", "2023-12-23", "2023-12-30", "2024-01-06", "2024-01-13", "2024-01-20", "2024-01-27", "2024-02-03", "2024-02-10", "2024-02-17", "2024-02-24", "2024-03-02", "2024-03-09", "2024-03-16", "2024-03-23", "2024-03-30", "2024-04-06", "2024-04-13", "2024-04-20", "2024-04-27", "2024-05-04", "2024-05-11", "2024-11-16", "2024-11-23", "2024-11-30", "2024-12-07", "2024-12-14", "2024-12-21", "2024-12-28", "2025-01-04", "2025-01-11", "2025-01-18", "2025-01-25", "2025-02-01", "2025-02-08", "2025-02-15", "2025-02-22", "2025-03-01", "2025-03-08", "2025-03-15", "2025-03-22", "2025-03-29", "2025-04-05", "2025-04-12", "2025-04-19", "2025-04-26", "2025-05-03", "2025-05-10", "2025-05-17", "2025-05-24", "2025-05-31"]
},
"initial_as_of": "2025-05-31",
"current_date": "2025-05-31",
"models": [
"CADPH-FluCAT_Ensemble", "CEPH-Rtrend_fluH", "CMU-TimeSeries", "CU-ensemble", "GT-FluFNP", "ISU_NiemiLab-NLH", "JHU_CSSE-CSSE_Ensemble", "LUcompUncertLab-chimera", "LosAlamos_NAU-CModel_Flu", "MIGHTE-Nsemble", "MOBS-GLEAM_FLUH", "NIH-Flu_ARIMA", "NU_UCSD-GLEAM_AI_FLUH", "PSI-PROF", "SGroup-RandomForest", "SigSci-CREG", "SigSci-TSENS", "Stevens-GBR", "UGA_flucast-Copycat", "UGA_flucast-INFLAenza", "UGuelph-CompositeCurve", "UGuelphensemble-GRYPHON", "UM-DeepOutbreak", "UMass-flusion", "UMass-trends_ensemble", "UNC_IDD-InfluPaint", "UVAFluX-Ensemble", "VTSanghani-Ensemble", "cfa-flumech", "cfarenewal-cfaepimlight", "fjordhest-ensemble"
],
"initial_checked_models": ["UMass-flusion", "UMass-trends_ensemble"],
"disclaimer": "Most forecasts have failed to reliably predict rapid changes in the trends of reported cases and hospitalizations. Due to this limitation, they should not be relied upon for decisions about the possibility or timing of rapid changes in trends.",
"initial_xaxis_range": null
}
9 changes: 9 additions & 0 deletions tests/hub_predtimechart/test_generate_options.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,12 @@ def test_generate_options_complex_forecast_hub():
assert act_options[exp_field] == exp_options[exp_field]

assert act_options == exp_options


def test_generate_options_flusight_forecast_hub():
hub_dir = Path('tests/hubs/FluSight-forecast-hub')
hub_config = HubConfig(hub_dir, hub_dir / 'hub-config/predtimechart-config.yml')
with open('tests/expected/FluSight-forecast-hub/predtimechart-options.json') as fp:
exp_options = json.load(fp)
act_options = ptc_options_for_hub(hub_config)
assert act_options == exp_options
129 changes: 129 additions & 0 deletions tests/hubs/FluSight-forecast-hub/hub-config/model-metadata-schema.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "Schema for Modeling Hub model metadata",
"description": "This is the schema for model metadata files, please refer to https://github.com/cdcepi/FluSight-forecast-hub/blob/main/model-metadata/README.md for more information.",
"type": "object",
"properties": {
"team_name": {
"description": "The name of the team submitting the model",
"type": "string"
},
"team_abbr": {
"description": "Abbreviated name of the team submitting the model",
"type": "string",
"pattern": "^[a-zA-Z0-9_+]+$",
"maxLength": 16
},
"model_name": {
"description": "The name of the model",
"type": "string"
},
"model_abbr": {
"description": "Abbreviated name of the model",
"type": "string",
"pattern": "^[a-zA-Z0-9_+]+$",
"maxLength": 16
},
"model_version": {
"description": "Identifier of the version of the model",
"type": "string"
},
"model_contributors": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"affiliation": {
"type": "string"
},
"email": {
"type": "string",
"format": "email"
},
"orcid": {
"type": "string",
"pattern": "^\\d{4}\\-\\d{4}\\-\\d{4}\\-[\\dX]{4}$"
}
},
"additionalProperties": false,
"required": ["name", "affiliation", "email"]
}
},
"website_url": {
"description": "Public facing website for the model",
"type": "string",
"format": "uri"
},
"repo_url": {
"description": "Repository containing code for the model",
"type": "string",
"format": "uri"
},
"license": {
"description": "License for use of model output data",
"type": "string",
"enum": [
"CC0-1.0",
"CC-BY-4.0",
"CC-BY_SA-4.0",
"PPDL",
"ODC-by",
"ODbL",
"OGL-3.0"
]
},
"designated_model": {
"description": "Team-specified indicator for whether the model should be eligible for inclusion in a Hub ensemble and public visualization. A team may designate up to two models.",
"type": "boolean"
},
"citation": {
"description": "One or more citations for this model",
"type": "string",
"examples": ["Gibson GC , Reich NG , Sheldon D. Real-time mechanistic bayesian forecasts of Covid-19 mortality. medRxiv. 2020. https://doi.org/10.1101/2020.12.22.20248736"]
},
"team_funding": {
"description": "Any information about funding source for the team or members of the team.",
"type": "string",
"examples": ["National Institutes of General Medical Sciences (R01GM123456). The content is solely the responsibility of the authors and does not necessarily represent the official views of NIGMS."]
},
"data_inputs": {
"description": "List or description of data inputs used by the model",
"type": "string"
},
"methods": {
"description": "A brief (200 char.) description of the methods used by this model",
"type": "string",
"maxLength": 200
},
"methods_long": {
"description": "A full description of the methods used by this model. Among other details, this should include whether spatial correlation is considered and how the model accounts for uncertainty.",
"type": "string"
},
"ensemble_of_models": {
"description": "Indicator for whether this model is an ensemble of any separate component models",
"type": "boolean"
},
"ensemble_of_hub_models": {
"description": "Indicator for whether this model is an ensemble specifically of other models submitted to this Hub",
"type": "boolean"
}
},
"additionalProperties": false,
"required": [
"team_name",
"team_abbr",
"model_name",
"model_abbr",
"model_contributors",
"license",
"designated_model",
"data_inputs",
"methods",
"methods_long",
"ensemble_of_models",
"ensemble_of_hub_models"
]
}
8 changes: 8 additions & 0 deletions tests/hubs/FluSight-forecast-hub/hub-config/predtimechart-config.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
---
rounds_idx: 0
model_tasks_idx: 1
reference_date_col_name: 'reference_date'
target_date_col_name: 'target_end_date'
horizon_col_name: 'horizon'
initial_checked_models: ['UMass-flusion', 'UMass-trends_ensemble']
disclaimer: Most forecasts have failed to reliably predict rapid changes in the trends of reported cases and hospitalizations. Due to this limitation, they should not be relied upon for decisions about the possibility or timing of rapid changes in trends.
Loading

0 comments on commit 3fabadc

Please sign in to comment.