diff --git a/tests/expected/FluSight-forecast-hub/predtimechart-options.json b/tests/expected/FluSight-forecast-hub/predtimechart-options.json new file mode 100644 index 0000000..ba897ad --- /dev/null +++ b/tests/expected/FluSight-forecast-hub/predtimechart-options.json @@ -0,0 +1,81 @@ +{"target_variables": [ + { + "value": "wk inc flu hosp", + "text": "incident influenza hospitalizations", + "plot_text": "incident influenza hospitalizations" + } +], + "initial_target_var": "wk inc flu hosp", + "task_ids": { + "location": [ + {"value": "US", "text": "US"}, + {"value": "01", "text": "01"}, + {"value": "02", "text": "02"}, + {"value": "04", "text": "04"}, + {"value": "05", "text": "05"}, + {"value": "06", "text": "06"}, + {"value": "08", "text": "08"}, + {"value": "09", "text": "09"}, + {"value": "10", "text": "10"}, + {"value": "11", "text": "11"}, + {"value": "12", "text": "12"}, + {"value": "13", "text": "13"}, + {"value": "15", "text": "15"}, + {"value": "16", "text": "16"}, + {"value": "17", "text": "17"}, + {"value": "18", "text": "18"}, + {"value": "19", "text": "19"}, + {"value": "20", "text": "20"}, + {"value": "21", "text": "21"}, + {"value": "22", "text": "22"}, + {"value": "23", "text": "23"}, + {"value": "24", "text": "24"}, + {"value": "25", "text": "25"}, + {"value": "26", "text": "26"}, + {"value": "27", "text": "27"}, + {"value": "28", "text": "28"}, + {"value": "29", "text": "29"}, + {"value": "30", "text": "30"}, + {"value": "31", "text": "31"}, + {"value": "32", "text": "32"}, + {"value": "33", "text": "33"}, + {"value": "34", "text": "34"}, + {"value": "35", "text": "35"}, + {"value": "36", "text": "36"}, + {"value": "37", "text": "37"}, + {"value": "38", "text": "38"}, + {"value": "39", "text": "39"}, + {"value": "40", "text": "40"}, + {"value": "41", "text": "41"}, + {"value": "42", "text": "42"}, + {"value": "44", "text": "44"}, + {"value": "45", "text": "45"}, + {"value": "46", "text": "46"}, + {"value": "47", "text": "47"}, + {"value": "48", "text": "48"}, + {"value": "49", "text": "49"}, + {"value": "50", "text": "50"}, + {"value": "51", "text": "51"}, + {"value": "53", "text": "53"}, + {"value": "54", "text": "54"}, + {"value": "55", "text": "55"}, + {"value": "56", "text": "56"}, + {"value": "72", "text": "72"} + ] + }, + "initial_task_ids": {"location": "US"}, + "intervals": ["0%", "50%", "95%"], + "initial_interval": "95%", + "available_as_ofs": { + "wk inc flu hosp": [ + "2023-10-07", "2023-10-14", "2023-10-21", "2023-10-28", "2023-11-04", "2023-11-11", "2023-11-18", "2023-11-25", "2023-12-02", "2023-12-09", "2023-12-16", "2023-12-23", "2023-12-30", "2024-01-06", "2024-01-13", "2024-01-20", "2024-01-27", "2024-02-03", "2024-02-10", "2024-02-17", "2024-02-24", "2024-03-02", "2024-03-09", "2024-03-16", "2024-03-23", "2024-03-30", "2024-04-06", "2024-04-13", "2024-04-20", "2024-04-27", "2024-05-04", "2024-05-11", "2024-11-16", "2024-11-23", "2024-11-30", "2024-12-07", "2024-12-14", "2024-12-21", "2024-12-28", "2025-01-04", "2025-01-11", "2025-01-18", "2025-01-25", "2025-02-01", "2025-02-08", "2025-02-15", "2025-02-22", "2025-03-01", "2025-03-08", "2025-03-15", "2025-03-22", "2025-03-29", "2025-04-05", "2025-04-12", "2025-04-19", "2025-04-26", "2025-05-03", "2025-05-10", "2025-05-17", "2025-05-24", "2025-05-31"] + }, + "initial_as_of": "2025-05-31", + "current_date": "2025-05-31", + "models": [ + "CADPH-FluCAT_Ensemble", "CEPH-Rtrend_fluH", "CMU-TimeSeries", "CU-ensemble", "GT-FluFNP", "ISU_NiemiLab-NLH", "JHU_CSSE-CSSE_Ensemble", "LUcompUncertLab-chimera", "LosAlamos_NAU-CModel_Flu", "MIGHTE-Nsemble", "MOBS-GLEAM_FLUH", "NIH-Flu_ARIMA", "NU_UCSD-GLEAM_AI_FLUH", "PSI-PROF", "SGroup-RandomForest", "SigSci-CREG", "SigSci-TSENS", "Stevens-GBR", "UGA_flucast-Copycat", "UGA_flucast-INFLAenza", "UGuelph-CompositeCurve", "UGuelphensemble-GRYPHON", "UM-DeepOutbreak", "UMass-flusion", "UMass-trends_ensemble", "UNC_IDD-InfluPaint", "UVAFluX-Ensemble", "VTSanghani-Ensemble", "cfa-flumech", "cfarenewal-cfaepimlight", "fjordhest-ensemble" + ], + "initial_checked_models": ["UMass-flusion", "UMass-trends_ensemble"], + "disclaimer": "Most forecasts have failed to reliably predict rapid changes in the trends of reported cases and hospitalizations. Due to this limitation, they should not be relied upon for decisions about the possibility or timing of rapid changes in trends.", + "initial_xaxis_range": null +} diff --git a/tests/hub_predtimechart/test_generate_options.py b/tests/hub_predtimechart/test_generate_options.py index 91904ab..3c5ed4d 100644 --- a/tests/hub_predtimechart/test_generate_options.py +++ b/tests/hub_predtimechart/test_generate_options.py @@ -18,3 +18,12 @@ def test_generate_options_complex_forecast_hub(): assert act_options[exp_field] == exp_options[exp_field] assert act_options == exp_options + + +def test_generate_options_flusight_forecast_hub(): + hub_dir = Path('tests/hubs/FluSight-forecast-hub') + hub_config = HubConfig(hub_dir, hub_dir / 'hub-config/predtimechart-config.yml') + with open('tests/expected/FluSight-forecast-hub/predtimechart-options.json') as fp: + exp_options = json.load(fp) + act_options = ptc_options_for_hub(hub_config) + assert act_options == exp_options diff --git a/tests/hubs/FluSight-forecast-hub/hub-config/model-metadata-schema.json b/tests/hubs/FluSight-forecast-hub/hub-config/model-metadata-schema.json new file mode 100644 index 0000000..20088ee --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/hub-config/model-metadata-schema.json @@ -0,0 +1,129 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Schema for Modeling Hub model metadata", + "description": "This is the schema for model metadata files, please refer to https://github.com/cdcepi/FluSight-forecast-hub/blob/main/model-metadata/README.md for more information.", + "type": "object", + "properties": { + "team_name": { + "description": "The name of the team submitting the model", + "type": "string" + }, + "team_abbr": { + "description": "Abbreviated name of the team submitting the model", + "type": "string", + "pattern": "^[a-zA-Z0-9_+]+$", + "maxLength": 16 + }, + "model_name": { + "description": "The name of the model", + "type": "string" + }, + "model_abbr": { + "description": "Abbreviated name of the model", + "type": "string", + "pattern": "^[a-zA-Z0-9_+]+$", + "maxLength": 16 + }, + "model_version": { + "description": "Identifier of the version of the model", + "type": "string" + }, + "model_contributors": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "affiliation": { + "type": "string" + }, + "email": { + "type": "string", + "format": "email" + }, + "orcid": { + "type": "string", + "pattern": "^\\d{4}\\-\\d{4}\\-\\d{4}\\-[\\dX]{4}$" + } + }, + "additionalProperties": false, + "required": ["name", "affiliation", "email"] + } + }, + "website_url": { + "description": "Public facing website for the model", + "type": "string", + "format": "uri" + }, + "repo_url": { + "description": "Repository containing code for the model", + "type": "string", + "format": "uri" + }, + "license": { + "description": "License for use of model output data", + "type": "string", + "enum": [ + "CC0-1.0", + "CC-BY-4.0", + "CC-BY_SA-4.0", + "PPDL", + "ODC-by", + "ODbL", + "OGL-3.0" + ] + }, + "designated_model": { + "description": "Team-specified indicator for whether the model should be eligible for inclusion in a Hub ensemble and public visualization. A team may designate up to two models.", + "type": "boolean" + }, + "citation": { + "description": "One or more citations for this model", + "type": "string", + "examples": ["Gibson GC , Reich NG , Sheldon D. Real-time mechanistic bayesian forecasts of Covid-19 mortality. medRxiv. 2020. https://doi.org/10.1101/2020.12.22.20248736"] + }, + "team_funding": { + "description": "Any information about funding source for the team or members of the team.", + "type": "string", + "examples": ["National Institutes of General Medical Sciences (R01GM123456). The content is solely the responsibility of the authors and does not necessarily represent the official views of NIGMS."] + }, + "data_inputs": { + "description": "List or description of data inputs used by the model", + "type": "string" + }, + "methods": { + "description": "A brief (200 char.) description of the methods used by this model", + "type": "string", + "maxLength": 200 + }, + "methods_long": { + "description": "A full description of the methods used by this model. Among other details, this should include whether spatial correlation is considered and how the model accounts for uncertainty.", + "type": "string" + }, + "ensemble_of_models": { + "description": "Indicator for whether this model is an ensemble of any separate component models", + "type": "boolean" + }, + "ensemble_of_hub_models": { + "description": "Indicator for whether this model is an ensemble specifically of other models submitted to this Hub", + "type": "boolean" + } + }, + "additionalProperties": false, + "required": [ + "team_name", + "team_abbr", + "model_name", + "model_abbr", + "model_contributors", + "license", + "designated_model", + "data_inputs", + "methods", + "methods_long", + "ensemble_of_models", + "ensemble_of_hub_models" + ] +} diff --git a/tests/hubs/FluSight-forecast-hub/hub-config/predtimechart-config.yml b/tests/hubs/FluSight-forecast-hub/hub-config/predtimechart-config.yml new file mode 100644 index 0000000..dd2fe21 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/hub-config/predtimechart-config.yml @@ -0,0 +1,8 @@ +--- +rounds_idx: 0 +model_tasks_idx: 1 +reference_date_col_name: 'reference_date' +target_date_col_name: 'target_end_date' +horizon_col_name: 'horizon' +initial_checked_models: ['UMass-flusion', 'UMass-trends_ensemble'] +disclaimer: Most forecasts have failed to reliably predict rapid changes in the trends of reported cases and hospitalizations. Due to this limitation, they should not be relied upon for decisions about the possibility or timing of rapid changes in trends. diff --git a/tests/hubs/FluSight-forecast-hub/hub-config/tasks.json b/tests/hubs/FluSight-forecast-hub/hub-config/tasks.json new file mode 100644 index 0000000..8621ed3 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/hub-config/tasks.json @@ -0,0 +1,229 @@ +{ + "schema_version": "https://raw.githubusercontent.com/Infectious-Disease-Modeling-Hubs/schemas/main/v3.0.1/tasks-schema.json", + "rounds": [ + { + "round_id_from_variable": true, + "round_id": "reference_date", + "model_tasks": [ + { + "task_ids": { + "reference_date": { + "required": null, + "optional": ["2023-10-07", "2023-10-14", "2023-10-21", "2023-10-28", "2023-11-04", "2023-11-11", "2023-11-18", "2023-11-25", "2023-12-02", "2023-12-09", "2023-12-16", "2023-12-23", "2023-12-30", "2024-01-06", "2024-01-13", "2024-01-20", "2024-01-27", "2024-02-03", "2024-02-10", "2024-02-17", "2024-02-24", "2024-03-02", "2024-03-09", "2024-03-16", "2024-03-23", "2024-03-30", "2024-04-06", "2024-04-13", "2024-04-20", "2024-04-27", "2024-05-04", "2024-05-11", "2024-11-16", "2024-11-23", "2024-11-30", "2024-12-07", "2024-12-14", "2024-12-21", "2024-12-28", "2025-01-04", "2025-01-11", "2025-01-18", "2025-01-25", "2025-02-01", "2025-02-08", "2025-02-15", "2025-02-22", "2025-03-01", "2025-03-08", "2025-03-15", "2025-03-22", "2025-03-29", "2025-04-05", "2025-04-12", "2025-04-19", "2025-04-26", "2025-05-03", "2025-05-10", "2025-05-17", "2025-05-24", "2025-05-31"] + }, + "target": { + "required": null, + "optional": ["wk flu hosp rate change"] + }, + "horizon": { + "required": null, + "optional": [-1, 0, 1, 2, 3] + }, + "location": { + "required": null, + "optional": ["US", "01", "02", "04", "05", "06", "08", "09", "10", "11", "12", "13", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "44", "45", "46", "47", "48", "49", "50", "51", "53", "54", "55", "56", "72"] + }, + "target_end_date": { + "required": null, + "optional": ["2023-09-23", "2023-09-30", "2023-10-07", "2023-10-14", "2023-10-21", "2023-10-28", "2023-11-04", "2023-11-11", "2023-11-18", "2023-11-25", "2023-12-02", "2023-12-09", "2023-12-16", "2023-12-23", "2023-12-30", "2024-01-06", "2024-01-13", "2024-01-20", "2024-01-27", "2024-02-03", "2024-02-10", "2024-02-17", "2024-02-24", "2024-03-02", "2024-03-09", "2024-03-16", "2024-03-23", "2024-03-30", "2024-04-06", "2024-04-13", "2024-04-20", "2024-04-27", "2024-05-04", "2024-05-11", "2024-05-18", "2024-05-25", "2024-06-01", "2024-11-09", "2024-11-16", "2024-11-23", "2024-11-30", "2024-12-07", "2024-12-14", "2024-12-21", "2024-12-28", "2025-01-04", "2025-01-11", "2025-01-18", "2025-01-25", "2025-02-01", "2025-02-08", "2025-02-15", "2025-02-22", "2025-03-01", "2025-03-08", "2025-03-15", "2025-03-22", "2025-03-29", "2025-04-05", "2025-04-12", "2025-04-19", "2025-04-26", "2025-05-03", "2025-05-10", "2025-05-17", "2025-05-24", "2025-05-31", "2025-06-07", "2025-06-14", "2025-06-21"] + } + }, + "output_type": { + "pmf": { + "output_type_id": { + "required": ["large_decrease", "decrease", "stable", "increase", "large_increase"], + "optional": null + }, + "value": { + "type": "double", + "minimum": 0, + "maximum": 1 + } + } + }, + "target_metadata": [ + { + "target_id": "flu hosp rate change", + "target_name": "week ahead weekly influenza hospitalization rate change", + "target_units": "rate per 100,000 population", + "target_keys": { + "target": "wk flu hosp rate change" + }, + "target_type": "ordinal", + "description": "This target represents the change in the rate of new hospitalizations per week comparing the week ending on the reference_date to the week ending [horizon] weeks after the reference_date, on target_end_date.", + "is_step_ahead": true, + "time_unit": "week" + } + ] + }, + { + "task_ids": { + "reference_date": { + "required": null, + "optional": ["2023-10-07", "2023-10-14", "2023-10-21", "2023-10-28", "2023-11-04", "2023-11-11", "2023-11-18", "2023-11-25", "2023-12-02", "2023-12-09", "2023-12-16", "2023-12-23", "2023-12-30", "2024-01-06", "2024-01-13", "2024-01-20", "2024-01-27", "2024-02-03", "2024-02-10", "2024-02-17", "2024-02-24", "2024-03-02", "2024-03-09", "2024-03-16", "2024-03-23", "2024-03-30", "2024-04-06", "2024-04-13", "2024-04-20", "2024-04-27", "2024-05-04", "2024-05-11", "2024-11-16", "2024-11-23", "2024-11-30", "2024-12-07", "2024-12-14", "2024-12-21", "2024-12-28", "2025-01-04", "2025-01-11", "2025-01-18", "2025-01-25", "2025-02-01", "2025-02-08", "2025-02-15", "2025-02-22", "2025-03-01", "2025-03-08", "2025-03-15", "2025-03-22", "2025-03-29", "2025-04-05", "2025-04-12", "2025-04-19", "2025-04-26", "2025-05-03", "2025-05-10", "2025-05-17", "2025-05-24", "2025-05-31"] + }, + "target": { + "required": null, + "optional": ["wk inc flu hosp"] + }, + "horizon": { + "required": null, + "optional": [-1, 0, 1, 2, 3] + }, + "location": { + "required": null, + "optional": ["US", "01", "02", "04", "05", "06", "08", "09", "10", "11", "12", "13", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "44", "45", "46", "47", "48", "49", "50", "51", "53", "54", "55", "56", "72"] + }, + "target_end_date": { + "required": null, + "optional": ["2023-09-23", "2023-09-30", "2023-10-07", "2023-10-14", "2023-10-21", "2023-10-28", "2023-11-04", "2023-11-11", "2023-11-18", "2023-11-25", "2023-12-02", "2023-12-09", "2023-12-16", "2023-12-23", "2023-12-30", "2024-01-06", "2024-01-13", "2024-01-20", "2024-01-27", "2024-02-03", "2024-02-10", "2024-02-17", "2024-02-24", "2024-03-02", "2024-03-09", "2024-03-16", "2024-03-23", "2024-03-30", "2024-04-06", "2024-04-13", "2024-04-20", "2024-04-27", "2024-05-04", "2024-05-11", "2024-05-18", "2024-05-25", "2024-06-01", "2024-11-09", "2024-11-16", "2024-11-23", "2024-11-30", "2024-12-07", "2024-12-14", "2024-12-21", "2024-12-28", "2025-01-04", "2025-01-11", "2025-01-18", "2025-01-25", "2025-02-01", "2025-02-08", "2025-02-15", "2025-02-22", "2025-03-01", "2025-03-08", "2025-03-15", "2025-03-22", "2025-03-29", "2025-04-05", "2025-04-12", "2025-04-19", "2025-04-26", "2025-05-03", "2025-05-10", "2025-05-17", "2025-05-24", "2025-05-31", "2025-06-07", "2025-06-14", "2025-06-21"] + } + }, + "output_type": { + "quantile": { + "output_type_id": { + "required": [0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 0.975, 0.99], + "optional": null + }, + "value": { + "type": "double", + "minimum": 0 + } + }, + "sample": { + "output_type_id_params": { + "is_required": false, + "type": "integer", + "min_samples_per_task": 100, + "max_samples_per_task": 100, + "compound_taskid_set" : ["reference_date", "location", "target"] + }, + "value": { + "type": "integer", + "minimum": 0 + } + } + }, + "target_metadata": [ + { + "target_id": "wk inc flu hosp", + "target_name": "incident influenza hospitalizations", + "target_units": "count", + "target_keys": { + "target": "wk inc flu hosp" + }, + "target_type": "continuous", + "description": "This target represents the count of new hospitalizations in the week ending on the date [horizon] weeks after the reference_date, on the target_end_date.", + "is_step_ahead": true, + "time_unit": "week" + } + ] + }, + { + "task_ids": { + "reference_date": { + "required": null, + "optional": ["2023-10-07", "2023-10-14", "2023-10-21", "2023-10-28", "2023-11-04", "2023-11-11", "2023-11-18", "2023-11-25", "2023-12-02", "2023-12-09", "2023-12-16", "2023-12-23", "2023-12-30", "2024-01-06", "2024-01-13", "2024-01-20", "2024-01-27", "2024-02-03", "2024-02-10", "2024-02-17", "2024-02-24", "2024-03-02", "2024-03-09", "2024-03-16", "2024-03-23", "2024-03-30", "2024-04-06", "2024-04-13", "2024-04-20", "2024-04-27", "2024-05-04", "2024-05-11", "2024-11-16", "2024-11-23", "2024-11-30", "2024-12-07", "2024-12-14", "2024-12-21", "2024-12-28", "2025-01-04", "2025-01-11", "2025-01-18", "2025-01-25", "2025-02-01", "2025-02-08", "2025-02-15", "2025-02-22", "2025-03-01", "2025-03-08", "2025-03-15", "2025-03-22", "2025-03-29", "2025-04-05", "2025-04-12", "2025-04-19", "2025-04-26", "2025-05-03", "2025-05-10", "2025-05-17", "2025-05-24", "2025-05-31"] + }, + "target": { + "required": null, + "optional": ["peak inc flu hosp"] + }, + "horizon": { + "required": null, + "optional": null + }, + "location": { + "required": null, + "optional": ["US", "01", "02", "04", "05", "06", "08", "09", "10", "11", "12", "13", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "44", "45", "46", "47", "48", "49", "50", "51", "53", "54", "55", "56", "72"] + }, + "target_end_date": { + "required": null, + "optional": null + } + }, + "output_type": { + "quantile": { + "output_type_id": { + "required": [0.01, 0.025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95, 0.975, 0.99], + "optional": null + }, + "value": { + "type": "double", + "minimum": 0 + } + } + }, + "target_metadata": [ + { + "target_id": "peak inc flu hosp", + "target_name": "incident influenza hospitalizations in the season week with the highest hospitalizations", + "target_units": "count", + "target_keys": { + "target": "peak inc flu hosp" + }, + "target_type": "continuous", + "description": "This target represents the count of new hospitalizations in the week of the season with the highest reported hospitalizations.", + "is_step_ahead": false + } + ] + }, + { + "task_ids": { + "reference_date": { + "required": null, + "optional": ["2023-10-07", "2023-10-14", "2023-10-21", "2023-10-28", "2023-11-04", "2023-11-11", "2023-11-18", "2023-11-25", "2023-12-02", "2023-12-09", "2023-12-16", "2023-12-23", "2023-12-30", "2024-01-06", "2024-01-13", "2024-01-20", "2024-01-27", "2024-02-03", "2024-02-10", "2024-02-17", "2024-02-24", "2024-03-02", "2024-03-09", "2024-03-16", "2024-03-23", "2024-03-30", "2024-04-06", "2024-04-13", "2024-04-20", "2024-04-27", "2024-05-04", "2024-05-11", "2024-11-16", "2024-11-23", "2024-11-30", "2024-12-07", "2024-12-14", "2024-12-21", "2024-12-28", "2025-01-04", "2025-01-11", "2025-01-18", "2025-01-25", "2025-02-01", "2025-02-08", "2025-02-15", "2025-02-22", "2025-03-01", "2025-03-08", "2025-03-15", "2025-03-22", "2025-03-29", "2025-04-05", "2025-04-12", "2025-04-19", "2025-04-26", "2025-05-03", "2025-05-10", "2025-05-17", "2025-05-24", "2025-05-31"] + }, + "target": { + "required": null, + "optional": ["peak week inc flu hosp"] + }, + "horizon": { + "required": null, + "optional": null + }, + "location": { + "required": null, + "optional": ["US", "01", "02", "04", "05", "06", "08", "09", "10", "11", "12", "13", "15", "16", "17", "18", "19", "20", "21", "22", "23", "24", "25", "26", "27", "28", "29", "30", "31", "32", "33", "34", "35", "36", "37", "38", "39", "40", "41", "42", "44", "45", "46", "47", "48", "49", "50", "51", "53", "54", "55", "56", "72"] + }, + "target_end_date": { + "required": null, + "optional": null + } + }, + "output_type": { + "pmf": { + "output_type_id": { + "required": ["2024-11-09", "2024-11-16", "2024-11-23", "2024-11-30", "2024-12-07", "2024-12-14", "2024-12-21", "2024-12-28", "2025-01-04", "2025-01-11", "2025-01-18", "2025-01-25", "2025-02-01", "2025-02-08", "2025-02-15", "2025-02-22", "2025-03-01", "2025-03-08", "2025-03-15", "2025-03-22", "2025-03-29", "2025-04-05", "2025-04-12", "2025-04-19", "2025-04-26", "2025-05-03", "2025-05-10", "2025-05-17", "2025-05-24", "2025-05-31"], + "optional": null + }, + "value": { + "type": "double", + "minimum": 0, + "maximum": 1 + } + } + }, + "target_metadata": [ + { + "target_id": "peak week inc flu hosp", + "target_name": "season week with the highest hospitalizations", + "target_units": "count", + "target_keys": { + "target": "peak week inc flu hosp" + }, + "target_type": "date", + "description": "This target represents the week of the season with the highest reported hospitalizations. This is the Saturday ending the epidemic week with the highest reported hospitalization count.", + "is_step_ahead": false + } + ] + } + ], + "submissions_due": { + "relative_to": "reference_date", + "start": -6, + "end": -3 + } + } + ] +} diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/CADPH-FluCAT_Ensemble.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/CADPH-FluCAT_Ensemble.yml new file mode 100644 index 0000000..b950389 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/CADPH-FluCAT_Ensemble.yml @@ -0,0 +1,32 @@ +team_name: "California Department of Public Health" +team_abbr: "CADPH" +model_name: "FluCAT Ensemble" +model_abbr: "FluCAT_Ensemble" +model_contributors: [ + { + "name": "Lauren White", + "affiliation": "California Department of Public Health", + "email": "lauren.white@cdph.ca.gov" + }, + { + "name": "Erin Murray", + "affiliation": "California Department of Public Health", + "email": "erin.murray@cdph.ca.gov" + }, + { + "name": "Tomas M. Leon", + "affiliation": "California Department of Public Health", + "email": "tomas.leon@cdph.ca.gov" + } +] +license: "CC-BY-4.0" +designated_model: true +data_inputs: "Daily incident flu hospitalizations (NHSN), clinical lab surveillance data, Change healthcare claims data (accessed via covidcast)" +methods: "Ensemble of time series forecasts approaches including ARIMA, Holt's, damped Holt's and neural networks" +methods_long: "An ensemble of several time series forecasts including; (1) autoregressive integrated moving average (ARIMA), which uses a weighted linear sum of recent past observations or lags, (2) exponential smoothing (ETS), which uses weighted averages of past observations, with the weights decaying exponentially as the observations get older, and (3) neural networks using the forecast package in R." +ensemble_of_models: true +ensemble_of_hub_models: false +model_version: "1.1" +website_url: "https://calcat.covid19.ca.gov/cacovidmodels/" +team_funding: "The views and opinions expressed by the authors are their own and do not necessarily represent the views and opinions of the California Department of Public Health or the California Health and Human Services Agency." + diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/CEPH-Rtrend_fluH.yaml b/tests/hubs/FluSight-forecast-hub/model-metadata/CEPH-Rtrend_fluH.yaml new file mode 100644 index 0000000..f46042c --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/CEPH-Rtrend_fluH.yaml @@ -0,0 +1,39 @@ +team_name: "CEPH Lab at Indiana University" +team_abbr: "CEPH" +model_name: "Rtrend FluH" +model_abbr: "Rtrend_fluH" +model_contributors: [ + { + "name": "Marco Ajelli", + "affiliation": "Indiana University Bloomington", + "email": "majelli@iu.edu", + }, + { + "name": "Paulo C. Ventura", + "affiliation": "Indiana University Bloomington", + "email": "pventura@iu.edu", + }, + { + "name": "Maria Litvinova", + "affiliation": "Indiana University Bloomington", + "email": "malitv@iu.edu", + }, + { + "name": "Allisandra G. Kummer", + "affiliation": "Indiana University Bloomington", + "email": "alkummer@iu.edu", + }, + { + "name": "Alessandro Vespignani", + "affiliation": "Northeastern University", + "email": "a.vespignani@northeastern.edu", + }, +] +license: "CC-BY-4.0" +designated_model: true +data_inputs: "Daily incident flu hospitalizations, queried through HealthData" +methods: "A renewal equation method based on Bayesian estimation of Rt from hospitalization data." +methods_long: "Model forecasts are obtained by using a renewal equation based on the estimated net reproduction number Rt. We apply a lowpass filter to the time series of daily hospitalizations, extracting the main trend. We then use MCMC Metropolis-Hastings sampling to estimate the posterior distribution of Rt based on the filtered data, considering an informed prior on Rt based on influenza literature. The estimated Rt in the last weeks of available data is used to forecast Rt in the upcoming weeks, with a drift term proportional to the current incidence. Finally, we use the renewal equation with the posterior distribution and trend of the estimated Rt in the most recent weeks of influenza data." +ensemble_of_models: false +ensemble_of_hub_models: false +website_url: https://publichealth.indiana.edu/research/faculty-directory/profile.html?user=majelli diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/CMU-TimeSeries.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/CMU-TimeSeries.yml new file mode 100644 index 0000000..b84553d --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/CMU-TimeSeries.yml @@ -0,0 +1,56 @@ +team_name: "Carnegie Mellon Delphi Group" +team_abbr: "CMU" +model_name: "AR ensemble with auxiliary data" +model_abbr: "TimeSeries" +model_version: "1.0" +model_contributors: + [ + { + "name": "Logan Brooks", + "affiliation": "UC Berkeley", + "email": "lcbrooks@berkeley.edu", + }, + { + "name": "Dmitry Shemetov", + "affiliation": "Carnegie Mellon University", + "email": "dshemeto@andrew.cmu.edu", + }, + { + "name": "Nat DeFries", + "affiliation": "Carnegie Mellon University", + "email": "ndefries@andrew.cmu.edu", + }, + { + "name": "Addison Hu", + "affiliation": "Carnegie Mellon University", + "email": "addison@stat.cmu.edu", + }, + { + "name": "David Weber", + "affiliation": "Carnegie Mellon University", + "email": "davidweb@andrew.cmu.edu", + }, + { + "name": "Daniel McDonald", + "affiliation": "University of British Columbia", + "email": "daniel@stat.ubc.ca", + }, + { + "name": "Ryan Tibshirani", + "affiliation": "UC Berkeley", + "email": "ryantibs@berkeley.edu", + }, + ] +website_url: "https://github.com/cmu-delphi/flu-hosp-forecast/" +repo_url: "https://github.com/cmu-delphi/flu-hosp-forecast/" +license: "CC-BY-4.0" +designated_model: true +team_funding: "Centers for Disease Control and Prevention Awards: U011P001121, 75D30123C15907, NU38FT000005" +include_viz: true +include_ensemble: true +include_eval: true +methods: "An ensemble of AR-based time-series models." +data_inputs: "Daily and weekly incident flu hospitalizations, queried through Delphi Epidata API." +methods_long: "A basic quantile autoregression fit using lagged values of influenza-related hospitalization counts (normalized by population). The data are smoothed in time and the latter data source is adjusted to remove systematic day-of-week effects. The model is fit jointly across all 50 US states, the District of Columbia, Puerto Rico, and the Virgin Islands, using the most recently available 21 days of training data. Each of the 23 quantiles is learned using a separate quantile regression with nonnegativity and quantile sorting constraints applied post hoc. All data signals are available as indicators through the Delphi Epidata API (https://cmu-delphi.github.io/delphi-epidata)." +ensemble_of_models: true +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/CU-ensemble.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/CU-ensemble.yml new file mode 100644 index 0000000..3410014 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/CU-ensemble.yml @@ -0,0 +1,35 @@ +team_name: "Columbia University" +team_abbr: "CU" +model_name: "Ensemble" +model_abbr: "ensemble" +model_contributors: [ + { + "name": "Rami Yaari", + "affiliation": "Columbia University", + "email": "ry2460@cumc.columbia.edu" + }, + { + "name": "Teresa Yamana", + "affiliation": "Columbia University", + "email": "tky2104@cumc.columbia.edu" + }, + { + "name": "Sen Pei", + "affiliation": "Columbia University", + "email": "sp3449@cumc.columbia.edu" + }, + { + "name": "Jeffrey Shaman", + "affiliation": "Columbia University", + "email": "jls106@cumc.columbia.edu" + } +] +website_url: "https://blogs.cuit.columbia.edu/jls106/" +license: "CC-BY-4.0" +team_funding: "US NIH grant AI163023 and CDC 75D30122C14289" +designated_model: true +data_inputs: "State and national-level daily confirmed influenza hospital admissions, queried using covidcast R package. State and national-level ILINet surveillance data, queried using cdcfluview R package." +methods: "An inverse-WIS weighted ensemble of several component models - an SEIRS compartmental model with EAKF, an ARIMA model, a random walk with drift, and the N-HiTS and N-BEATS deep-learning models." +methods_long: "The dynamical model simulates influenza transmission in each state and the US using a humidity-driven SEIRS dynamics. Model variables and parameters are sequentially updated each week using the ensemble adjustment Kalman filter and new observations. Forecasts are generated by integrating the optimized model into the future. Autoregressive Integrated Moving Average model and baseline models use implementations available in the fable R package (ARIMA and RW, respectively). We employ multivariate versions of N-HiTS and N-BEATS models as implemented in the darts python package, trained on modified state-level ILI data and hospitalization data. To build ensemble, the quantile distributions of the component models are weighted by the sum of inverse-WIS scores, over last 4 weeks. The 4-week window is target and location-specific and are recomputed at each forecast week." +ensemble_of_models: true +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-baseline.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-baseline.yml new file mode 100644 index 0000000..e022f56 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-baseline.yml @@ -0,0 +1,55 @@ +team_name: "FluSight-baseline" +team_abbr: "FluSight" +model_name: "Baseline model using reported weekly admissions" +model_abbr: "baseline" +model_version: "1.0" +model_contributors: [ + { + "name": "Daniel McDonald", + "affiliation": "University of British Columbia", + "email": "daniel@stat.ubc.ca" + }, + { + "name": "Logan Brooks", + "affiliation": "UC Berkeley", + "email": "lcbrooks@berkeley.edu" + }, + { + "name": "Sarabeth Mathis", + "affiliation": "CDC", + "email": "nqr2@cdc.gov" + }, + { + "name": "Alexander Webber", + "affiliation": "CDC", + "email": "rpe5@cdc.gov" + }, + { + "name": "Rebecca Borchering", + "affiliation": "CDC", + "email": "xhq2@cdc.gov" + } +] +website_url: "https://github.com/cdcepi/FluSight-forecast-hub" +license: "CC-BY-4.0" +designated_model: false +methods: "Simple time-series baseline model predicting forward the most recent week of reported laboratory confirmed influenza hospital admissions." +data_inputs: "Weekly incident flu hospitalizations from HealthData.gov COVID-19 Reported Patient Impact and Hospital Capacity by State Timeseries." +methods_long: "The median prediction at all future horizons + is the most recent observed weekly incidence. To get a distribution around + the median, we look at how much incidence has changed from week to week in + the past, and we allow for the possibility that similar changes will occur + again in the future. + + In more technical detail, forecasts of incidence are generated through the + following procedure: for each location, 1) Calculate first differences of + incidence; 2) Collect all first differences and their negatives; 3) For each + time step, i) Sample first differences in incidence and add to the most recent + observed incidence; ii) Enforce that the median of the predictive distribution + for incidence is equal to the most recent observed incidence; iii) truncate + the predictive distribution for incidence at 0. + + Reimplementation of the 2022-23 baseline model contributed to by Evan L. Ray, + Nutcha Wattanachit, and Ryan Tibshirani." +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-baseline_cat.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-baseline_cat.yml new file mode 100644 index 0000000..8b80fa2 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-baseline_cat.yml @@ -0,0 +1,36 @@ +team_name: "FluSight-baseline_cat" +team_abbr: "FluSight" +model_name: "Reference model translating FluSight-baseline quantile forecasts to pmf" +model_abbr: "baseline_cat" +model_version: "1.0" +model_contributors: [ + { + "name": "Rebecca Borchering", + "affiliation": "CDC", + "email": "xhq2@cdc.gov" + }, + { + "name": "Sarabeth Mathis", + "affiliation": "CDC", + "email": "nqr2@cdc.gov" + }, + { + "name": "Li Shandross", + "affiliation": "UMass Amherst", + "email": "lshandross@umass.edu" + }, + { + "name": "Evan Ray", + "affiliation": "UMass Amherst", + "email": "elray@umass.edu" + } +] +website_url: "https://github.com/cdcepi/FluSight-forecast-hub" +license: "CC-BY-4.0" +designated_model: false +methods: "Simple reference model translating FluSight baseline quantile forecasts to pmf." +data_inputs: "None" +methods_long: "Simple reference model translating FluSight baseline quantile forecasts to pmf using a customized version of a function to translate quantile forecasts to pmf forecasts." + +ensemble_of_models: false +ensemble_of_hub_models: false \ No newline at end of file diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-ens_q_cat.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-ens_q_cat.yml new file mode 100644 index 0000000..0b379dd --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-ens_q_cat.yml @@ -0,0 +1,36 @@ +team_name: "FluSight-ens_q_cat" +team_abbr: "FluSight" +model_name: "Reference model translating FluSight-ensemble quantile forecasts to pmf" +model_abbr: "q_ens_cat" +model_version: "1.0" +model_contributors: [ + { + "name": "Rebecca Borchering", + "affiliation": "CDC", + "email": "xhq2@cdc.gov" + }, + { + "name": "Sarabeth Mathis", + "affiliation": "CDC", + "email": "nqr2@cdc.gov" + }, + { + "name": "Li Shandross", + "affiliation": "UMass Amherst", + "email": "lshandross@umass.edu" + }, + { + "name": "Evan Ray", + "affiliation": "UMass Amherst", + "email": "elray@umass.edu" + } +] +website_url: "https://github.com/cdcepi/FluSight-forecast-hub" +license: "CC-BY-4.0" +designated_model: false +methods: "Simple reference model translating FluSight ensemble quantile forecasts to pmf." +data_inputs: "None" +methods_long: "Simple reference model translating FluSight ensemble quantile forecasts to pmf using a customized version of a function to translate quantile forecasts to pmf forecasts." + +ensemble_of_models: false +ensemble_of_hub_models: false \ No newline at end of file diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-ensemble.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-ensemble.yml new file mode 100644 index 0000000..9ab7fd9 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-ensemble.yml @@ -0,0 +1,40 @@ +team_name: "FluSight-ensemble" +team_abbr: "FluSight" +model_name: "Ensemble models using FluSight forecast submissions" +model_abbr: "ensemble" +model_version: "1.0" +model_contributors: [ + { + "name": "Li Shandross", + "affiliation": "University of Massachusetts Amherst", + "email": "lshandross@umass.edu" + }, + { + "name": "Evan Ray", + "affiliation": "University of Massachusetts Amherst", + "email": "elray@umass.edu" + }, + { + "name": "Alexander Webber", + "affiliation": "CDC", + "email": "rpe5@cdc.gov" + }, + { + "name": "Sarabeth Mathis", + "affiliation": "CDC", + "email": "nqr2@cdc.gov" + }, + { + "name": "Rebecca Borchering", + "affiliation": "CDC", + "email": "xhq2@cdc.gov" + } +] +website_url: "https://github.com/Infectious-Disease-Modeling-Hubs/hubEnsembles" +license: "CC-BY-4.0" +designated_model: false +methods: "Median-based ensemble of quantile forecasts submissions mean ensemble for categorical rate-trend forecasts." +data_inputs: "Weekly incident flu hospitalizations from HealthData.gov COVID-19 Reported Patient Impact and Hospital Capacity by State Timeseries." +methods_long: "The Hubverse package hubEnsembles is used to generate the FluSight-ensemble forecast A median of quantile outputs is used for all eligible quantile-based forecasts. The mean is used to ensemble probabilities from eligible categorical rate-trend forecasts." +ensemble_of_models: true +ensemble_of_hub_models: true diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-equal_cat.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-equal_cat.yml new file mode 100644 index 0000000..b711b7d --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-equal_cat.yml @@ -0,0 +1,21 @@ +team_name: "FluSight-equal_cat" +team_abbr: "FluSight" +model_name: "Baseline model with equal probabilities for each rate-trend category" +model_abbr: "equal_cat" +model_version: "1.0" +model_contributors: [ + { + "name": "Rebecca Borchering", + "affiliation": "CDC", + "email": "xhq2@cdc.gov" + } +] +website_url: "https://github.com/cdcepi/FluSight-forecast-hub" +license: "CC-BY-4.0" +designated_model: false +methods: "Simple baseline model predicting that the probability for each rate-trend category will equal 0.2 for all jurisdictions and horizons." +data_inputs: "None" +methods_long: "Simple baseline model predicting that the probability for each rate-trend category will be equal. Since there are five categories, for each jurisdiction and rate-difference horizon each category recieves a probability of 0.2 of occurring." + +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-lop_norm.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-lop_norm.yml new file mode 100644 index 0000000..4b1a179 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-lop_norm.yml @@ -0,0 +1,40 @@ +team_name: "FluSight-lop_norm" +team_abbr: "FluSight" +model_name: "Linear opinion pool ensemble of FluSight forecast submissions" +model_abbr: "lop_norm" +model_version: "1.0" +model_contributors: [ + { + "name": "Li Shandross", + "affiliation": "University of Massachusetts Amherst", + "email": "lshandross@umass.edu" + }, + { + "name": "Evan Ray", + "affiliation": "University of Massachusetts Amherst", + "email": "elray@umass.edu" + }, + { + "name": "Alexander Webber", + "affiliation": "CDC", + "email": "rpe5@cdc.gov" + }, + { + "name": "Sarabeth Mathis", + "affiliation": "CDC", + "email": "nqr2@cdc.gov" + }, + { + "name": "Rebecca Borchering", + "affiliation": "CDC", + "email": "xhq2@cdc.gov" + } +] +website_url: "https://github.com/Infectious-Disease-Modeling-Hubs/hubEnsembles" +license: "CC-BY-4.0" +designated_model: false +methods: "Linear opinion pool based ensemble of quantile forecasts submissions." +data_inputs: "Weekly incident flu hospitalizations from HealthData.gov COVID-19 Reported Patient Impact and Hospital Capacity by State Timeseries." +methods_long: "The Hubverse package hubEnsembles is used to generate the FluSight-lop_norm forecast. A linear opinion pool of quantile outputs is used for all eligible quantile-based forecasts." +ensemble_of_models: true +ensemble_of_hub_models: true diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-national_cat.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-national_cat.yml new file mode 100644 index 0000000..31721f4 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/FluSight-national_cat.yml @@ -0,0 +1,23 @@ +team_name: "FluSight-national_cat" +team_abbr: "FluSight" +model_name: "National baseline model with the rate-trend forecasts" +model_abbr: "national_cat" +model_version: "1.0" +model_contributors: [ + {"name": "Jessica Davis", + "affiliation": "MOBS Lab, Northeastern University", + "email": "jes.davis@northeastern.edu"}, + + {"name": "Rebecca Borchering", + "affiliation": "CDC", + "email": "xhq2@cdc.gov"} +] +website_url: "https://github.com/cdcepi/FluSight-forecast-hub" +license: "CC-BY-4.0" +designated_model: false +methods: "Simple baseline/reference model for the rate-trend forecasts that utilizes the national rate-trend data from previous weeks" +data_inputs: "None" +methods_long: "The probability that a jurisdiction is in each category depends on the percentage of location falling in each category for the most recent observed one week and two week differences. This probability distribution is applied to each location." + +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/GH-model.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/GH-model.yml new file mode 100644 index 0000000..8981be3 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/GH-model.yml @@ -0,0 +1,22 @@ +team_name: "Guidehouse" +team_abbr: "GH" +model_name: "model" +model_abbr: "model" +model_version: "1.0" +model_contributors: [ + { + "name": "Akilan Meiyappan", + "affiliation": "Guidehouse", + "email": "ameiyappan@guidehouse.com" + } +] +website_url: "https://github.com/gh-ai-solu/flusight-21-22/" +license: "CC-BY-4.0" +citation: "" +team_funding: "n/a" +designated_model: false +methods: "SVM model trained on HHS Protect hosps." +data_inputs: "Daily indcident flu hospitlization" +methods_long: "" +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/GT-FluFNP.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/GT-FluFNP.yml new file mode 100644 index 0000000..b39296a --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/GT-FluFNP.yml @@ -0,0 +1,29 @@ +team_name: "GT-AdityaLab" +team_abbr: "GT" +model_name: "FluFNP" +model_abbr: "FluFNP" +model_contributors: [ + { + "name": "B.Aditya Prakash", + "affiliation": "GeorgiaTech", + "email": "badityap@cc.gatech.edu" + }, + { + "name": "Harshavardhan Kamarthi", + "affiliation": "GeorgiaTech", + "email": "hkamarthi3@gatech.edu" + }, + { + "name": "Zhiyuan Zhao", + "affiliation": "GeorgiaTech", + "email": "leozhao1997@gatech.edu" + }, +] +website_url: "https://adityalab.cc.gatech.edu/" +license: "CC-BY-4.0" +designated_model: True +data_inputs: "Daily and weekly incident flu hospitalizations, queried through covidData" +methods: "Calibrated and Accurate Multi-view Time-Series Forecasting" +methods_long: "Integrate the knowledge and uncertainty from each data view in a dynamic context-specific manner assigning more importance to useful views to model a well-calibrated forecast distribution" +ensemble_of_models: False +ensemble_of_hub_models: False \ No newline at end of file diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/ISU_NiemiLab-ENS.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/ISU_NiemiLab-ENS.yml new file mode 100644 index 0000000..b0afe2d --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/ISU_NiemiLab-ENS.yml @@ -0,0 +1,27 @@ +team_name: "ISU_NiemiLab" +team_abbr: "ISU_NiemiLab" +model_name: "Nonlinear ASG hierarchical model with discrepancy" +model_abbr: "ENS" +model_version: "1.0" +model_contributors: [ + { + "name": "Spencer Wadsworth", + "affiliation": "Iowa State", + "email": "sgw96@iastate.edu" + }, + { + "name": "Jarad Niemi", + "affiliation": "Iowa State", + "email": "niemi@iastate.edu" + } +] +website_url: "https://jarad.me" +license: "CC-BY-4.0" +citation: "citation" +team_funding: "funding" +designated_model: false +methods: "An ensemble of models of ILI data which is then mapped linearly to hospitalizations" +data_inputs: "weekly incident flu hospitalizations from HHS, and weekly ILI data" +methods_long: "nonlinear model" +ensemble_of_models: true +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/ISU_NiemiLab-NLH.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/ISU_NiemiLab-NLH.yml new file mode 100644 index 0000000..1e7b2fc --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/ISU_NiemiLab-NLH.yml @@ -0,0 +1,27 @@ +team_name: "ISU_NiemiLab" +team_abbr: "ISU_NiemiLab" +model_name: "Nonlinear ASG hierarchical model with discrepancy" +model_abbr: "NLH" +model_version: "1.0" +model_contributors: [ + { + "name": "Spencer Wadsworth", + "affiliation": "Iowa State", + "email": "sgw96@iastate.edu" + }, + { + "name": "Jarad Niemi", + "affiliation": "Iowa State", + "email": "niemi@iastate.edu" + } +] +website_url: "https://jarad.me" +license: "CC-BY-4.0" +citation: "citation" +team_funding: "funding" +designated_model: true +methods: "A nonlinear hierarchical model of ILI data which is then mapped linearly to hospitalizations" +data_inputs: "weekly incident flu hospitalizations from HHS, and weekly ILI data" +methods_long: "nonlinear model" +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/ISU_NiemiLab-SIR.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/ISU_NiemiLab-SIR.yml new file mode 100644 index 0000000..5b61afc --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/ISU_NiemiLab-SIR.yml @@ -0,0 +1,27 @@ +team_name: "ISU_NiemiLab" +team_abbr: "ISU_NiemiLab" +model_name: "Nonlinear ASG hierarchical model with discrepancy" +model_abbr: "SIR" +model_version: "1.0" +model_contributors: [ + { + "name": "Spencer Wadsworth", + "affiliation": "Iowa State", + "email": "sgw96@iastate.edu" + }, + { + "name": "Jarad Niemi", + "affiliation": "Iowa State", + "email": "niemi@iastate.edu" + } +] +website_url: "https://jarad.me/" +license: "CC-BY-4.0" +citation: "citation" +team_funding: "funding" +designated_model: false +methods: "An SIR based model of ILI data which is then mapped linearly to hospitalizations" +data_inputs: "weekly incident flu hospitalizations from HHS, and weekly ILI data" +methods_long: "nonlinear model" +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/JHU_CSSE-CSSE_Ensemble.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/JHU_CSSE-CSSE_Ensemble.yml new file mode 100644 index 0000000..30981d4 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/JHU_CSSE-CSSE_Ensemble.yml @@ -0,0 +1,49 @@ +team_name: "The Center for Systems Science and Engineering at Johns Hopkins University" +team_abbr: "JHU_CSSE" +model_name: "CSSE Ensemble" +model_abbr: "CSSE_Ensemble" +model_contributors: [ + { + "name": "Lauren Gardner", + "affiliation": "Johns Hopkins University", + "email": "l.gardner@jhu.edu" + }, + { + "name": "Hongru Du", + "affiliation": "Johns Hopkins University", + "email": "hdu9@jh.edu" + }, + { + "name": "Hao Frank Yang", + "affiliation": "Johns Hopkins University", + "email": "haofrankyang@jhu.edu" + }, + { + "name": "Shaochong Xu", + "affiliation": "Johns Hopkins University", + "email": "sxu75@jh.edu" + }, + { + "name": "Xianglong Wang", + "affiliation": "Johns Hopkins University", + "email": "xwang344@jh.edu" + }, + { + "name": "Yang Zhao", + "affiliation": "Johns Hopkins University", + "email": "yzhao229@jh.edu" + }, + { + "name": "Pu Wang", + "affiliation": "New York University", + "email": "pw2425@nyu.edu" + } +] +license: "CC-BY-4.0" +designated_model: true +data_inputs: "Weekly flu hospitalizations, Google search volume for flu-related symptoms, Change healthcare claims data (accessed via covidcast)" +methods: "Three-tiered time series forecasting approach using Holt-Winters Exponential Smoothing and LSTM" +methods_long: "This model forecasts state-level influenza hospitalizations using a combination of time series forecasting methods, organized across three hierarchical levels. At the individual state level, forecasts are generated using Holt-Winters Exponential Smoothing. For regional predictions, which group states based on recent flu activity trends identified through the Louvain method, Long Short-Term Memory (LSTM) models are employed. Additionally, a LSTM model that covers all states is implemented. These three-tiered model outputs are integrated, selecting weights based on their recent performance in terms of Mean Absolute Error (MAE) to produce the final prediction." +ensemble_of_models: true +ensemble_of_hub_models: false + diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/LUcompUncertLab-chimera.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/LUcompUncertLab-chimera.yml new file mode 100644 index 0000000..79db649 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/LUcompUncertLab-chimera.yml @@ -0,0 +1,34 @@ +team_name: "LUcompUncertLab" +team_abbr: "LUcompUncertLab" +model_name: "chimera" +model_abbr: "chimera" +model_version: "1.0" +model_contributors: [ + { + "name": "tom mcandrew", + "affiliation": "lehigh university", + "email": "mcandrew@lehigh.edu" + } +] +website_url: "https://github.com/computationalUncertaintyLab" +license: "CC-BY-4.0" +citation: "citation" +team_funding: "CSTE/CDC" +methods: "KM27 model trained on pct positive,HHS hosps,Human judgment" +designated_model: true +data_inputs: "Daily flu hospitalizations,Daily percent positive cases,metaculus human judgment forecasts" +methods_long: "2023-10-11: SEIRH model variant was fit bc KM27 model was not yet complete. +2023-10-12: KM27 model is completed and ready for training/submission for the following week. +2023-10-18: KM27 model was incorrect last week. i spernt the entire week correcting the model. Thete is a final bug in the mehtod i use to aggregate from day forecasts to week forecasts. +i will need to solve this bug before the next submission date. +2023-11-01:Corrected KM27 and added in knowledge about flu: R0 is constrained to be between 1.1 and 2.5, vaccine proprtion is between 0.2 and 0.8. Generatioanl interval is Neg Bin and on average 3.6 days. +Changed from Normal observational process to GammaPoisson. The GP is good but very computationally slow. +Still cranky about how sensitive this model is to changes. This round ill try to focus on assigning seasons to distinct clusters. +2023-11-08:Added in priors for peak intensity and timing, and for the first two weeks of the season (MMWR 40 and 41). This adds stability to the model and speed. +2023-11-23:Found two major bugs in the code. One is frustrating and the other very subtle. First bug was training on hospitalization data the same as last year. i missed the message about a change in the hosp data and so have been training my model on the wrong data up until now. The second bug is a subtle fix in the code that took some time to find. +2023-12-09: Added back QO data, truncated to be two weeks before the forecast submission date. Forecasts look ok except for FIP=72. i'll need to investigate. +2023-12-16: Ran versions without and without the QO data. For the majority of locations i chose the model that included QO data. There were 3 locations, 30,37,56 where i thought the model without QO data was better and so chose those forecasts instead. +2023-12-30: Not much to report. Still running a few slightly different versions of the model and then hand picking which model to submit for which location. +2024-03-20:Not in love with Loc 2 but otherwise forecasts look ok" +ensemble_of_models: false +ensemble_of_hub_models: false \ No newline at end of file diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/LosAlamos_NAU-CModel_Flu.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/LosAlamos_NAU-CModel_Flu.yml new file mode 100644 index 0000000..7b66f79 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/LosAlamos_NAU-CModel_Flu.yml @@ -0,0 +1,56 @@ +team_name: "Los Alamos National Lab and Northern Arizona University" +team_abbr: "LosAlamos_NAU" +model_name: "Compartmental Model with Bayesian Inference" +model_abbr: "CModel_Flu" +model_version: "2.0" +model_contributors: [ + { + "name": "Ye Chen", + "affiliation": "Northern Arizona University", + "email": "Ye.Chen@nau.edu", + }, + { + "name": "Richard Posner", + "affiliation": "Northern Arizona University", + "email": "Richard.Posner@nau.edu", + }, + { + "name": "Jaechoul Lee", + "affiliation": "Northern Arizona University", + "email": "Jaechoul.Lee@nau.edu", + }, + { + "name": "Joseph R Mihaljevic", + "affiliation": "Northern Arizona University", + "email": "Joseph.Mihaljevic@nau.edu", + }, + { + "name": "Avery Drennan", + "affiliation": "Northern Arizona University", + "email": "aad473@nau.edu", + }, + { + "name": "William Scott Hlavacek", + "affiliation": "Los Alamos National Lab", + "email": "hlavacek@lanl.gov", + }, + { + "name": "Yen Ting Lin", + "affiliation": "Los Alamos National Lab", + "email": "yentingl@lanl.gov", + }, + { + "name": "Abhishek Mallela", + "affiliation": "Los Alamos National Lab", + "email": "abhishek.mallela@gmail.com", + } + ] +website_url: "https://github.com/ye-chen-netsci/FluSight-forecast-hub-2023" +repo_url: "https://github.com/ye-chen-netsci/FluSight-forecast-hub-2023" +license: "CC-BY-4.0" +designated_model: true +methods: "A compartmental model with Bayesian inference, uncertainty quantification and calibration." +data_inputs: "State level daily hospitalization data from HealthData.gov." +methods_long: "We employed a compartmental model that features a time-dependent transmission rate as the base model. Using Bayesian inference, we parametrize the transmission rate from hospitalization data. Subsequent steps involved uncertainty quantification and model calibration." +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/MIGHTE-Nsemble.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/MIGHTE-Nsemble.yml new file mode 100644 index 0000000..564d0c4 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/MIGHTE-Nsemble.yml @@ -0,0 +1,37 @@ +team_name: "MIGHTE" +team_abbr: "MIGHTE" +model_name: "Time series ensemble" +model_abbr: "Nsemble" +model_version: "1.0" +model_contributors: [ + { + "name": "Austin Meyer", + "affiliation": "Northeastern University", + "email": "austin.g.meyer@gmail.com" + }, + { + "name": "Leonardo Clemente", + "affiliation": "Northeastern University", + "email": "c.clementelopez@northeastern.edu" + }, + { + "name": "Fred Lu", + "affiliation": "Northeastern University", + "email": "fredlu1618@gmail.com" + }, + { + "name": "Mauricio Santillana", + "affiliation": "Northeastern University", + "email": "m.santillana@northeastern.edu" + } +] +website_url: "https://github.com/MIGHTE-lab/" +license: "CC-BY-4.0" +citation: "citation" +team_funding: "funding" +designated_model: true +methods: "Weighted ensemble of SOTA time-series models especially lightGBM" +data_inputs: "Weekly incident flu hospitalizations, Google Trends" +methods_long: "Weighted ensemble of SOTA time-series models, including lightGBM with hyperparameters tuned to the previous season, ARIMA, vector auto-regression, and ARGO." +ensemble_of_models: true +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/MOBS-GLEAM_FLUH.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/MOBS-GLEAM_FLUH.yml new file mode 100644 index 0000000..e684cf0 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/MOBS-GLEAM_FLUH.yml @@ -0,0 +1,53 @@ +team_name: "MOBS Lab at Northeastern University" +team_abbr: "MOBS" +model_name: "GLEAM Flu Forecasting Model" +model_abbr: "GLEAM_FLUH" +model_version: "1.0" +model_contributors: [ + { + "name": "Alessandro Vespignani", + "affiliation": "MOBS Lab, Northeastern University", + "email": "alexves@gmail.com" + }, + { + "name": "Matteo Chinazzi", + "affiliation": "MOBS Lab, Northeastern University", + "email": "m.chinazzi@northeastern.edu" + }, + { + "name": "Jessica T. Davis", + "affiliation": "MOBS Lab, Northeastern University", + "email": "jes.davis@northeastern.edu" + }, + { + "name": "Kunpeng Mu", + "affiliation": "MOBS Lab, Northeastern University", + "email": "mu.k@northeastern.edu" + }, + + { + "name": "Clara Bay", + "affiliation": "MOBS Lab, Northeastern University", + "email": "bay.c@northeastern.edu" + }, + { + "name": "Guillaume St-Onge", + "affiliation": "MOBS Lab, Northeastern University", + "email": "g.st-onge@northeastern.edu" + }, + { + "name": "Remy LeWinter", + "affiliation": "MOBS Lab, Northeastern University", + "email": "lewinter.r@northeastern.edu" + }, +] +website_url: "https://www.mobs-lab.org/" +license: "CC-BY_SA-4.0" +designated_model: true +citation: "https://uploads-ssl.webflow.com/58e6558acc00ee8e4536c1f5/5e8bab44f5baae4c1c2a75d2_GLEAM_web.pdf" +team_funding: "We acknowledge support from grant HHS/CDC 6U01IP001137, HHS/CDC 5U01IP0001137." +methods: "Metapopulation, age structured SLIR model." +data_inputs: "Weekly incident flu hospitalizations, queried through HHS" +methods_long: "The GLEAM framework is based on a metapopulation approach in which the US is divided into geographical subpopulations. Human mobility between subpopulations is represented on a network. This mobility data layer identifies the numbers of individuals traveling from one sub-population to another. The mobility network is made up of different kinds of mobility processes from short-range commuting between nearby subpopulations to flights. To model short-range mobility such as commuting or car travel, we rely on databases collected from the Offices of Statistics of 30 countries on five continents. Superimposed on the US population and mobility layers is an compartmental epidemic model that defines the infection and population dynamics. " +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/NIH-Flu_ARIMA.yaml b/tests/hubs/FluSight-forecast-hub/model-metadata/NIH-Flu_ARIMA.yaml new file mode 100644 index 0000000..f80fbef --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/NIH-Flu_ARIMA.yaml @@ -0,0 +1,27 @@ +team_name: "Fogarty International Center, NIH" +team_abbr: "NIH" +model_name: "ARIMA model with flu-related covariates" +model_abbr: "Flu_ARIMA" +model_version: "2.0" +model_contributors: [ + { + "name": "Amanda Perofsky", + "affiliation": "Fogarty International Center, National Institutes of Health", + "email": "amanda.perofsky@nih.gov" + }, + { + "name": "Cécile Viboud", + "affiliation": "Fogarty International Center, National Institutes of Health", + "email": "viboudc@mail.nih.gov" + } +] +website_url: "https://github.com/midas-network/flu-scenario-modeling-hub/tree/main/data-processed/NIH-Flu_TS" +license: "CC-BY-4.0" +citation: "citation" +team_funding: "not applicable" +designated_model: true +methods: "Seasonal ARIMA model with exogenous covariates for cumulative vaccination coverage, seasonal vaccine effectiveness, weekly A/H3N2 circulation, and influenza transmissibility." +data_inputs: "Forecasts use HHS influenza hospitalization data, CDC FluSurv-NET hospitalization rates, syndromic and virologic surveillance data from CDC FluView, vaccination coverage estimates from the National Center for Immunization and Respiratory Diseases, and seasonal vaccine effectiveness estimates from published observational studies and the CDC." +methods_long: "We use dynamic regression models with ARIMA errors and exogenous covariates for seasonal cumulative vaccination coverage, seasonal vaccine effectiveness, weekly A/H3N2 circulation, and weekly influenza transmission rates. We used 6 years of historical data on influenza hospitalizations (incidence, epidemic size, and timing), influenza-like illness rates, influenza A subtype circulation, weekly Rt, and vaccine coverage and effectiveness to calibrate our model. To estimate weekly Rt, we fit semi-mechanistic Bayesian epidemiological models to observed ILI x % positive rates during each season, assuming a case ascertainment rate of 0.3." +ensemble_of_models: false +ensemble_of_hub_models: false \ No newline at end of file diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/NU_UCSD-GLEAM_AI_FLUH.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/NU_UCSD-GLEAM_AI_FLUH.yml new file mode 100644 index 0000000..78bcdb5 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/NU_UCSD-GLEAM_AI_FLUH.yml @@ -0,0 +1,57 @@ +team_name: Northeastern University & University of California San Diego +team_abbr: NU_UCSD +model_name: GLEAM_AI Flu Forecasting Model +model_abbr: GLEAM_AI_FLUH +model_contributors: +- name: Mohammadmehdi Zahedi + affiliation: Network Science Institute, Northeastern University, Boston, MA, USA; + The Roux Institute, Northeastern University, Portland, ME, USA + email: zahedi.m@northeastern.edu + orcid: 0009-0002-7064-3258 +- name: Dongxia (Allen) Wu + affiliation: University of California, San Diego, La Jolla, CA, USA + email: dowu@ucsd.edu + orcid: 0000-0003-2412-6049 +- name: Jessica T. Davis + affiliation: Network Science Institute, Northeastern University, Boston, MA, USA + email: jes.davis@northeastern.edu + orcid: 0000-0003-0726-1855 +- name: Yi-An Ma + affiliation: University of California, San Diego, La Jolla, CA, USA + email: yianma@ucsd.edu + orcid: 0000-0001-6074-6638 +- name: Rose Yu + affiliation: University of California, San Diego, La Jolla, CA, USA + email: roseyu@ucsd.edu + orcid: 0000-0002-8491-7937 +- name: Alessandro Vespignani + affiliation: Network Science Institute, Northeastern University, Boston, MA, USA + email: a.vespignani@northeastern.edu + orcid: 0000-0003-3419-4205 +- name: Matteo Chinazzi + affiliation: Network Science Institute, Northeastern University, Boston, MA, USA; + The Roux Institute, Northeastern University, Portland, ME, USA + email: m.chinazzi@northeastern.edu + orcid: 0000-0002-5955-1929 +license: "CC-BY_SA-4.0" +designated_model: true +data_inputs: COVID-19 Reported Patient Impact and Hospital Capacity by State Timeseries + (https://healthdata.gov/Hospital/COVID-19-Reported-Patient-Impact-and-Hospital-Capa/g62h-syeh). + Data Provided by U.S. Department of Health & Human Services. +methods: This a deep surrogate model trained to mimic MOBS-GLEAM_FLUH model. +methods_long: "This a deep surrogate model trained to mimic MOBS-GLEAM_FLUH model.\ + \ GLEAM_FLUH is based on a metapopulation approach in which the world is divided\ + \ into geographical subpopulations. \nHuman mobility between subpopulations is represented\ + \ on a network. This mobility data layer identifies the numbers of individuals traveling\ + \ from one sub-population to another. \nThe mobility network is made up of different\ + \ kinds of mobility processes, from short-range commuting between nearby subpopulations\ + \ to intercontinental flights. \nTo model short-range mobility such as commuting\ + \ or car travel, we rely on databases collected from the Offices of Statistics of\ + \ 30 countries on five continents. \nSuperimposed on the worldwide population and\ + \ mobility layers is an agent-based epidemic model that defines the infection and\ + \ population dynamics." +ensemble_of_models: false +ensemble_of_hub_models: false +model_version: '0.01' +website_url: http://www.gleamproject.org/ +team_funding: JTD, AV, and MC acknowledge support from grant HHS/CDC 6U01IP001137, HHS/CDC 5U01IP0001137; DW, YM, RY acknowledge support from U. S. Army Research Office under Army-ECASE award W911NF-07-R-0003-03, the U.S. Department Of Energy, Office of Science, IARPA HAYSTAC Program, NSF Grants \#2205093, \#2146343, and \#2134274. diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/PSI-PROF.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/PSI-PROF.yml new file mode 100644 index 0000000..f15abf3 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/PSI-PROF.yml @@ -0,0 +1,32 @@ +team_name: "Predictive Science Inc." +team_abbr: "PSI" +model_name: "Package for Respiratory Disease Open-source Forecasting" +model_abbr: "PROF" +model_version: "1.0" +model_contributors: [ + { + "name": "Michal Ben-Nun", + "affiliation": "Predictive Science Inc", + "email": "mbennun@predsci.com" + }, + { + "name": "James Turtle", + "affiliation": "Predictive Science Inc", + "email": "jturtle@predsci.com" + }, + { + "name": "Pete Riley", + "affiliation": "Predictive Science Inc", + "email": "pete@predsci.com" + } +] +website_url: "https://www.predsci.com/usa-flu-hosp/" +license: "CC-BY-4.0" +citation: "https://doi.org/10.1371/journal.pcbi.1010375" +team_funding: "CSTE/CDC: Development of Forecasts and/or Scenario Projections for Influenza to Inform Public Health Decision Making (Cooperative Agreement number NU38OT000297)." +designated_model: true +methods: "A stochastic/deterministic, SIR[H]2 model with a time-dependent transmission rate, that includes compartments for hospitalizations. Parameter posteriors are inferred using an MCMC procedure." +data_inputs: "Daily and weekly incident flu hospitalizations from the HHS COVID-19 Reported Patient Impact and Hospital Capacity by State Timeseries. US Census population estimates." +methods_long: "The PROF routines perform a deterministic fit of our compartmental SIR[H]2 model to daily hospitalization incidence profiles. The model includes a hospitalization compartment which is split into two sub-compartments. This split ensures that the model preserves the correct generation time (Tg) and that the ratio between cumulative recovered and hospitalized individuals is determined by the infection-hospitalization-ratio. The transmission-rate coefficient (beta) is a time-dependent function of 2 or more arc-tangents. The model fit is inferred by an MCMC procedure. It is followed by stochastic simulations through the forecast time-window using the inferred parameter distributions and a daily cadence which is then aggregated to weekly incidence to produce the forecasts. Where there is little-to-no epi signal in the data, a baseline statistical model is substituted for the mechanistic model. This model is similar to the previous model 'PSI-DICE', but in an updated computational framework (https://github.com/predsci/PROF)." +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/PSI-PROF_beta.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/PSI-PROF_beta.yml new file mode 100644 index 0000000..f810d4c --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/PSI-PROF_beta.yml @@ -0,0 +1,33 @@ +team_name: "Predictive Science Inc." +team_abbr: "PSI" +model_name: "Package for Respiratory Disease Open-source Forecasting (beta version)" +model_abbr: "PROF_beta" +model_version: "1.0" +model_contributors: [ + { + "name": "Michal Ben-Nun", + "affiliation": "Predictive Science Inc", + "email": "mbennun@predsci.com" + }, + { + "name": "James Turtle", + "affiliation": "Predictive Science Inc", + "email": "jturtle@predsci.com" + }, + { + "name": "Pete Riley", + "affiliation": "Predictive Science Inc", + "email": "pete@predsci.com" + } +] +website_url: "https://www.predsci.com/usa-flu-hosp/" +license: "CC-BY-4.0" +citation: "https://doi.org/10.1371/journal.pcbi.1010375" +team_funding: "CSTE/CDC: Development of Forecasts and/or Scenario Projections for Influenza to Inform Public Health Decision Making (Cooperative Agreement number NU38OT000297)." +designated_model: false +methods: "A beta-version of PROF, for testing new features in a prospective forecasting environment. This model adds compartments for vaccinated subjects to the existing PROF methodology." +data_inputs: "Daily and weekly incident flu hospitalizations from the HHS COVID-19 Reported Patient Impact and Hospital Capacity by State Timeseries. US Census population estimates. Previous season vaccination time-series from CDC/NCIRD." +methods_long: "The PROF routines perform a deterministic fit of our compartmental SIR[H]2 model to daily hospitalization incidence profiles. The model includes a hospitalization compartment which is split into two sub-compartments. This split ensures that the model preserves the correct generation time (Tg) and that the ratio between cumulative recovered and hospitalized individuals is determined by the infection-hospitalization-ratio. The transmission-rate coefficient (Beta) is a time-dependent function of 2 or more arc-tangents. The model fit is inferred by an MCMC procedure. It is followed by stochastic simulations through the forecast time-window using the inferred parameter distributions and a daily cadence which is then aggregated to weekly incidence to produce the forecasts. Where there is little-to-no epi signal in the data, a baseline statistical model is substituted for the mechanistic model. This model is similar to our standard model 'PSI-PROF', but with the addition of vaccinated compartments for Susceptibles and Infectious, and with deterministic simulations through the forecast time window. +Update 2023-10-10: Vaccine efficacy against severe disease set at 0.52 based on Southern Hemisphere results." +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/README.md b/tests/hubs/FluSight-forecast-hub/model-metadata/README.md new file mode 100644 index 0000000..1639f11 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/README.md @@ -0,0 +1,143 @@ +# Model metadata + +This folder contains metadata files for the models submitting to the FluSight forecasting collaboration. The specification for these files has been adapted to be consistent with [model metadata guidelines in the hubverse documentation](https://hubdocs.readthedocs.io/en/latest/format/model-metadata.html). + +Each model is required to have metadata in +[yaml format](https://docs.ansible.com/ansible/latest/reference_appendices/YAMLSyntax.html), +e.g. [see this metadata file](./UMass-trends_ensemble.yml). + +These instructions provide detail about the [data +format](#Data-format) as well as [validation](#Data-validation) that +you can do prior to a pull request with a metadata file. + +# Data format + +## Required variables + +This section describes each of the variables (keys) in the yaml document. +Please order the variables in this order. + +### team_name +The name of your team that is less than 50 characters. + +### team_abbr +The name of your team that is less than 16 characters. + +### model_name +The name of your model that is less than 50 characters. + +### model_abbr +An abbreviated name for your model that is less than 16 alphanumeric characters. + +### model_contributors + +A list of all individuals involved in the forecasting effort. +A names, affiliations, and email address is required for each contributor. Individuals may also include an optional orcid identifiers. +All email addresses provided will be added to an email distribution list for model contributors. + +The syntax of this field should be +``` +model_contributors: [ + { + "name": "Modeler Name 1", + "affiliation": "Institution Name 1", + "email": "modeler1@example.com", + "orcid": "1234-1234-1234-1234" + }, + { + "name": "Modeler Name 2", + "affiliation": "Institution Name 2", + "email": "modeler2@example.com", + "orcid": "1234-1234-1234-1234" + } +] +``` + +### license + +One of the [accepted licenses](https://github.com/cdcepi/FluSight-forecast-hub/blob/673e983fee54f3a21448071ac46a9f78d27dd164/hub-config/model-metadata-schema.json#L69-L75). + +We encourage teams to submit as a "cc-by-4.0" to allow the broadest possible uses +including private vaccine production (which would be excluded by the "cc-by-nc-4.0" license). + +### designated_model + +A team-specified boolean indicator (`true` or `false`) for whether the model should be considered eligible for inclusion in a Hub ensemble and public visualization. A team may specify up to two models as a designated_model for inclusion. Models which have a designated_model value of 'False' will still be included in internal forecasting hub evaluations. + +### data_inputs + +List or description of the data sources used to inform the model. Particularly those used beyond the target data of confirmed influenza hospital admissions. + +### methods + +A brief description of your forecasting methodology that is less than 200 +characters. + +### methods_long + +A full description of the methods used by this model. Among other details, this should include whether spatial correlation is considered and how the model accounts for uncertainty. If the model is modified, this field can also be used to provide the date of the modification and a description of the change. + +### ensemble_of_models + +A boolean value (`true` or `false`) that indicates whether a model is an ensemble of any separate component models. + +### ensemble_of_hub_models + +A boolean value (`true` or `false`) that indicates whether a model is an ensemble specifically of other models submited to the FluSight forecasting hub. + +## Optional + +### model_version +An identifier of the version of the model + +### website_url + +A url to a website that has additional data about your model. +We encourage teams to submit the most user-friendly version of your +model, e.g. a dashboard, or similar, that displays your model forecasts. + +### repo_url + +A github (or similar) repository url containing code for the model. + +### citation + +One or more citations to manuscripts or preprints with additional model details. For example, "Gibson GC , Reich NG , Sheldon D. Real-time mechanistic bayesian forecasts of Covid-19 mortality. medRxiv. 2020. https://doi.org/10.1101/2020.12.22.20248736". + +### team_funding + +Any information about funding source(s) for the team or members of the team that would be natural to include on any resulting FluSight publications. For example, "National Institutes of General Medical Sciences (R01GM123456). The content is solely the responsibility of the authors and does not necessarily represent the official views of NIGMS." + +# Data validation + +Optionally, you may validate a model metadata file locally before submitting it to the hub in a pull request. Note that this is not required, since the validations will also run on the pull request. To run the validations locally, follow these steps: + +1. Create a fork of the `FluSight-forecast-hub` repository and then clone the fork to your computer. +2. Create a draft of the model metadata file for your model and place it in the `model-metadata` folder of this clone. +3. Install the hubValidations package for R by running the following command from within an R session: +``` r +remotes::install_github("Infectious-Disease-Modeling-Hubs/hubValidations") +``` +4. Validate your draft metadata file by running the following command in an R session: +``` r +hubValidations::validate_model_metadata( + hub_path="", + file_path="") +``` + +For example, if your working directory is the root of the hub repository, you can use a command similar to the following: +``` r +hubValidations::validate_model_metadata(hub_path=".", file_path="UMass-trends_ensemble.yml") +``` + +If all is well, you should see output similar to the following: +``` +✔ model-metadata-schema.json: File exists at path hub-config/model-metadata-schema.json. +✔ UMass-trends_ensemble.yml: File exists at path model-metadata/UMass-trends_ensemble.yml. +✔ UMass-trends_ensemble.yml: Metadata file extension is "yml" or "yaml". +✔ UMass-trends_ensemble.yml: Metadata file directory name matches "model-metadata". +✔ UMass-trends_ensemble.yml: Metadata file contents are consistent with schema specifications. +✔ UMass-trends_ensemble.yml: Metadata file name matches the `model_id` specified within the metadata file. +``` + +If there are any errors, you will see a message describing the problem. diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/SGroup-RandomForest.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/SGroup-RandomForest.yml new file mode 100644 index 0000000..fb2e33d --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/SGroup-RandomForest.yml @@ -0,0 +1,27 @@ +team_name: Srivastava Group +team_abbr: SGroup +model_name: SGroup-RandomForest +model_abbr: RandomForest +model_contributors: [ + { + "name": "Ajitesh Srivastava", + "affiliation": "University of Southern California", + "email": "ajiteshs@usc.edu", + "orcid": "1234-1234-1234-1234" + }, + { + "name": " Majd Al Aawar", + "affiliation": "University of Southern California", + "email": "malaawar@usc.edu", + "orcid": "1234-1234-1234-1234" + } +] +license: "CC-BY-4.0" +website_url: "https://github.com/maa989" +designated_model: true +data_inputs: HHS, FluSurv-NET +methods: Random Forest ensemble of the predictors generated from the SGroup-SIkJalpha submission, HHS data, as well as historical FluSurv-NET data. +methods_long: Random Forest is an ensemble of the individual predictors generated from a variation of the SIkJalpha compartmental model, where each predictor corresponds to a certain set of hyperparameters, as well as some HHS data, as well as historical FluSurv-NET data. +ensemble_of_models: true +ensemble_of_hub_models: false + diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/SigSci-CREG.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/SigSci-CREG.yml new file mode 100644 index 0000000..5c85cc8 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/SigSci-CREG.yml @@ -0,0 +1,33 @@ +team_name: "Signature Science" +team_abbr: "SigSci" +model_name: "Count Regression" +model_abbr: "CREG" +model_version: "1.0" +model_contributors: [ + { + "name": "VP Nagraj", + "affiliation": "Signature Science, LLC", + "email": "pnagraj@signaturescience.com" + }, + { + "name": "Amy Benefield", + "affiliation": "Signature Science, LLC", + "email": "abenefield@signaturescience.com" + }, + { + "name": "Desiree Williams", + "affiliation": "Signature Science, LLC", + "email": "dwilliams@signaturescience.com" + } +] +website_url: "https://github.com/signaturescience/fiphde" +license: "CC-BY-4.0" +citation: "" +team_funding: "CSTE subaward" +designated_model: true +methods: "Count regression modeling." +data_inputs: "Weekly incident flu hospitalizations by state retrieved from HHS Protect API. Percent positivity of clinical lab tests by state retrieved from NREVSS. Percentage of outpatient visits characterized as influenza-like illness (ILI) by state from ILInet." +methods_long: "Count regression modeling approach. The procedure independently fits multiple count regression models for each location every week. Possible models are defined by a grid of all combinations of potential covariates (e.g., ILI, percent positivity of clinical lab flu tests) and model families (e.g., negative binomial, quasipoisson). The best fit model for the given forecast week and location is carried forward to generate forecasts. Forecasted values are rounded to the nearest integer and truncated at zero to align with hospitalization count target. All forecasts are reviewed for plausibility by a human prior to submission." +ensemble_of_models: false +ensemble_of_hub_models: false + diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/SigSci-TSENS.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/SigSci-TSENS.yml new file mode 100644 index 0000000..6fa08aa --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/SigSci-TSENS.yml @@ -0,0 +1,32 @@ +team_name: "Signature Science" +team_abbr: "SigSci" +model_name: "Time Series Ensemble" +model_abbr: "TSENS" +model_version: "1.0" +model_contributors: [ + { + "name": "VP Nagraj", + "affiliation": "Signature Science, LLC", + "email": "pnagraj@signaturescience.com" + }, + { + "name": "Amy Benefield", + "affiliation": "Signature Science, LLC", + "email": "abenefield@signaturescience.com" + }, + { + "name": "Desiree Williams", + "affiliation": "Signature Science, LLC", + "email": "dwilliams@signaturescience.com" + } +] +website_url: "https://github.com/signaturescience/fiphde" +license: "CC-BY-4.0" +citation: "" +team_funding: "CSTE subaward" +designated_model: true +methods: "Ensemble of time series models." +data_inputs: "Weekly incident flu hospitalizations by state retrieved from HHS Protect API." +methods_long: "Time series ensemble with equal weights for each component. The ensemble includes models that are fit independently for each location every week. The component models include different classes of time series models, some of which have automated parameter selection (e.g., ARIMA). Models are first combined before generating an ensemble forecast, as opposed to generating individual forecasts for each model and then combining forecasts. Forecasted values are rounded to the nearest integer and truncated at zero to align with hospitalization count target. All forecasts are reviewed for plausibility by a human prior to submission." +ensemble_of_models: true +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/Stevens-GBR.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/Stevens-GBR.yml new file mode 100644 index 0000000..afd75d2 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/Stevens-GBR.yml @@ -0,0 +1,40 @@ +team_name: "Stevens Institute of Technology" +team_abbr: "Stevens" +model_name: "An ensemble of gradient boosting regressors" +model_abbr: "GBR" +model_version: "1.0" +model_contributors: [ + { + "name": "Aditya Brahme", + "affiliation": "Stevens Institute of Technology", + "email": "abrahme@stevens.edu" + }, + { + "name": "Rutvik Raut", + "affiliation": "Stevens Institute of Technology", + "email": "rraut@stevens.edu" + }, + { + "name": "Bhumiti Gohel", + "affiliation": "Stevens Institute of Technology", + "email": "bgohel@stevens.edu" + }, + { + "name": "Sanika Mhadgut", + "affiliation": "Stevens Institute of Technology", + "email": "smhadgut@stevens.edu" + }, + { + "name": "Nikhil Muralidhar", + "affiliation": "Stevens Institute of Technology", + "email": "nmurali1@stevens.edu" + } +] +website_url: "https://sites.google.com/view/nikhil-muralidhar" +license: "CC-BY-4.0" +designated_model: true +methods: "An ensemble of basic tree models with equal weighting is used to create a robust and powerful tree-based model." +data_inputs: "State-aggregated data weekly incident flu hospitalizations, queried through covidData" +methods_long: "This model generates forecasts for state-level weekly influenza hospitalizations. The model comprises an ensemble of tree-based regression models based employing the gradient boosting learning paradigm." +ensemble_of_models: true +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/UGA_flucast-Copycat.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/UGA_flucast-Copycat.yml new file mode 100644 index 0000000..ac263c4 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/UGA_flucast-Copycat.yml @@ -0,0 +1,22 @@ +team_name: UGA_flucast +team_abbr: UGA_flucast +model_name: Copycat +model_abbr: Copycat +model_contributors: [ + { + "name": "Spencer J. Fox", + "affiliation": "University of Georgia", + "email": "sjfox@uga.edu", + "orcid": "0000-0003-1969-3778" + } +] +license: "CC-BY-4.0" +designated_model: true +data_inputs: HHS, ILIp, Fluserv +methods: A pattern matching model that matches growth rate trends to historic growth rate curves. +methods_long: Matches seasonal growth rate trends against historic growth rate curves to identify the closest matches. Makes forecasts based on nearest neighbor trajectories. +website_url: https://thefoxlab.wordpress.com/ +ensemble_of_models: false +ensemble_of_hub_models: false +team_funding: CSTE/CDC award NU38OT000297 + diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/UGA_flucast-INFLAenza.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/UGA_flucast-INFLAenza.yml new file mode 100644 index 0000000..da1be27 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/UGA_flucast-INFLAenza.yml @@ -0,0 +1,34 @@ +team_name: UGA_flucast +team_abbr: UGA_flucast +model_name: INFLAenza +model_abbr: INFLAenza +model_contributors: [ + { + "name": "Spencer J. Fox", + "affiliation": "University of Georgia", + "email": "sjfox@uga.edu", + "orcid": "0000-0003-1969-3778" + }, + { + "name": "Bren Case", + "affiliation": "University of Georgia", + "email": "bcase@uga.edu", + "orcid": "1234-1234-1234-1234" + }, +{ + "name": "Mariah Salcedo", + "affiliation": "University of Georgia", + "email": "mariah.salcedo@uga.edu", + "orcid": "1234-1234-1234-1234" + } +] +license: "CC-BY-4.0" +designated_model: true +data_inputs: HHS +methods: A spatial time-series model that uses the R-INLA package for estimating forecast posterior distributions. +methods_long: A spatial time-series model that uses the R-INLA package for estimating a jointly fitted time-series model that includes seasonality terms, state-specific connectivity, and a random walk component. +website_url: https://thefoxlab.wordpress.com/ +ensemble_of_models: false +ensemble_of_hub_models: false +team_funding: CSTE/CDC award NU38OT000297 + diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/UGA_flucast-OKeeffe.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/UGA_flucast-OKeeffe.yml new file mode 100644 index 0000000..f86e01a --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/UGA_flucast-OKeeffe.yml @@ -0,0 +1,34 @@ +team_name: UGA_flucast +team_abbr: UGA_flucast +model_name: OKeeffe +model_abbr: OKeeffe +model_contributors: [ + { + "name": "Spencer Fox", + "affiliation": "University of Georgia", + "email": "sjfox@uga.edu", + "orcid": "0000-0003-1969-3778" + }, + { + "name": "Graham C. Gibson", + "affiliation": "Los Alamos National Laboratories", + "email": "gcgibson@lanl.gov", + "orcid": "1234-1234-1234-1234" + }, + { + "name": "Lauren Meyers", + "affiliation": "The University of Texas at Austin", + "email": "laurenmeyers@austin.utexas.edu", + "orcid": "1234-1234-1234-1234" + } +] +license: "CC-BY-4.0" +designated_model: false +data_inputs: HHS +methods: An epidemic model with an underlying machine learning component, coupled with a tempering term that acts as susceptible depletion and is estimated from historical data. +methods_long: A class of epidemic models with a flexible underlying machine learning component, coupled with a tempering term that functionally acts as susceptible depletion, but can be estimated from historical data. We use historic HHS influenza wave data to make future forecasts +website_url: https://github.com/gcgibson/semimech_package +ensemble_of_models: false +ensemble_of_hub_models: false +team_funding: CSTE/CDC award NU38OT000297 + diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/UGuelph-CompositeCurve.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/UGuelph-CompositeCurve.yml new file mode 100644 index 0000000..d651bf6 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/UGuelph-CompositeCurve.yml @@ -0,0 +1,45 @@ +team_name: "University of Guelph Dynamics Training Lab" +team_abbr: "UGuelph" +model_name: "Composite Curve" +model_abbr: "CompositeCurve" +model_version: "1.0" +model_contributors: [ + { + "name": "Edward Thommes", + "affiliation": "Sanofi/University of Guelph", + "email": "ethommes@uoguelph.ca" + }, + { + "name": "Christopher van Bommel", + "affiliation": "University of Guelph", + "email": "cvanbomm@uoguelph.ca" + }, + { + "name": "Rhiannon Loster", + "affiliation": "University of Guelph", + "email": "rloster@uoguelph.ca" + }, + { + "name": "Benjamin Benteke", + "affiliation": "University of Guelph", + "email": "bbenteke@uoguelph.ca" + }, + { + "name": "Pengfei Yue", + "affiliation": "University of Guelph", + "email": "yuep@uoguelph.ca" + }, + { + "name": "Monica Cojocaru", + "affiliation": "University of Guelph", + "email": "mcojocar@uoguelph.ca" + } +] +website_url: "https://sites.google.com/site/mgcojocarumath/networks-and-dynamics-lab" +license: "CC-BY-4.0" +designated_model: true +methods: "Composite of past epidemic curves" +data_inputs: "target-hospital-admissions.csv" +methods_long: "Uses a midpoint-aligned composite of epidemic curves across past seasons and geographies (here states), following the approach of Schanzer et al., Influenza and Other Respiratory Viruses 2010 " +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/UGuelphensemble-GRYPHON.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/UGuelphensemble-GRYPHON.yml new file mode 100644 index 0000000..1e6f965 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/UGuelphensemble-GRYPHON.yml @@ -0,0 +1,45 @@ +team_name: "University of Guelph Dynamics Training Lab" +team_abbr: "UGuelphensemble" +model_name: "GRYPHON" +model_abbr: "GRYPHON" +model_version: "1.0" +model_contributors: [ + { + "name": "Edward Thommes", + "affiliation": "Sanofi/University of Guelph", + "email": "ethommes@uoguelph.ca" + }, + { + "name": "Christopher van Bommel", + "affiliation": "University of Guelph", + "email": "cvanbomm@uoguelph.ca" + }, + { + "name": "Rhiannon Loster", + "affiliation": "University of Guelph", + "email": "rloster@uoguelph.ca" + }, + { + "name": "Benjamin Benteke", + "affiliation": "University of Guelph", + "email": "bbenteke@uoguelph.ca" + }, + { + "name": "Pengfei Yue", + "affiliation": "University of Guelph", + "email": "yuep@uoguelph.ca" + }, + { + "name": "Monica Cojocaru", + "affiliation": "University of Guelph", + "email": "mcojocar@uoguelph.ca" + } +] +website_url: "https://sites.google.com/site/mgcojocarumath/networks-and-dynamics-lab" +license: "CC-BY-4.0" +designated_model: true +methods: "Ensemble" +data_inputs: "hospitalization data for US " +methods_long: "" +ensemble_of_models: True +ensemble_of_hub_models: True diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/UM-DeepOutbreak.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/UM-DeepOutbreak.yml new file mode 100644 index 0000000..a636cfa --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/UM-DeepOutbreak.yml @@ -0,0 +1,33 @@ +team_name: "University of Michigan, Computer Science and Engineering" +team_abbr: "UM" +model_name: "DeepOutbreak" +model_abbr: "DeepOutbreak" +model_version: "1.0" +model_contributors: [ + { + "name": "Ruipu Li", + "affiliation": "University of Michigan", + "email": "liruipu@umich.edu", + }, + { + "name": "Hersh Vora", + "affiliation": "University of Michigan", + "email": "hershvo@umich.edu", + }, + { + "name": "Alexander Rodríguez", + "affiliation": "University of Michigan", + "email": "alrodri@umich.edu", + "orcid": "0000-0002-4313-9913" + } +] +website_url: "https://alrodri.engin.umich.edu/" +license: "CC-BY-4.0" +designated_model: true +data_inputs: "Flu hospitalizations from HHS, Google's symptom search data, and CDC's ILI data." +methods: "Deep neural network model with conformal predictions." +methods_long: "Deep neural network model with conformal predictions. The neural network architecture is a sequence-to-sequence model based on recurrent units and self-attention modules. It is trained in a multi-task setting where each region is considered a task. The uncertainty quantification is conducted post hoc with conformal predictions that follows adaptive conformal inference to adapt to distribution shifts. Spatial correlation is not considered." +citation: "Rodriguez, A., Tabassum, A., Cui, J., Xie, J., Ho, J., Agarwal, P., Adhikari, B. and Prakash, B.A., 2021, May. Deepcovid: An operational deep learning-driven framework for explainable real-time covid-19 forecasting. In Proceedings of the AAAI Conference on Artificial Intelligence (Vol. 35, No. 17, pp. 15393-15400). https://ojs.aaai.org/index.php/AAAI/article/view/17808" +ensemble_of_models: false +ensemble_of_hub_models: false +team_funding: "Start-up funds from the University of Michigan." diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/UMass-flusion.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/UMass-flusion.yml new file mode 100644 index 0000000..eb57ce9 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/UMass-flusion.yml @@ -0,0 +1,26 @@ +team_name: "UMass-Amherst" +team_abbr: "UMass" +model_name: "Ensemble of time series models" +model_abbr: "flusion" +model_version: "1.0" +model_contributors: [ + { + "name": "Evan Ray", + "affiliation": "UMass Amherst", + "email": "elray@umass.edu" + } +] +website_url: "https://github.com/reichlab/flusion" +license: "CC-BY-4.0" +citation: "citation" +team_funding: "funding" +designated_model: true +methods: "Ensemble of statistical and machine learning time series models." +data_inputs: "Weekly incident flu hospitalizations from HHS Protect, hospitalizations from FluSurv-NET, and ILI+ obtained by combining ILINet outpatient ILI rates with WHO/NREVSS test positivity rates." +methods_long: " + 2023-10-14: A linear pool of two sub-ensembles, one fit with all available data and one fit dropping the last data point which may be subject to revisions. Each sub-ensemble is an equally-weighted Vincent median ensemble of three components: 1) an AR(8) model fit only to HHS data on a fourth root scale; 2) a gradient boosting model fit to data from all three sources with uncertainty based on bootstrapping out-of-bag prediction errors; 3) a gradient boosting model fit to all three data sources with quantile regression separately for each quantile level, with predictions sorted to address quantile crossing. + 2023-10-21: Fit only to the data release including data up through the week of 2023-10-14. Dropped bootstrap method, so submission is a quantile averaged ensemble of AR(8) and the gradient boosting model with quantile regression for all quantiles. + 2023-11-25: Data for AK are suspect; low counts, including a reported 0 for Sat Nov 18. Roughly 18-20 reporting facilities per day rather than historical values of 21 or 22. I don't have a principled way to deal with this, so the submission for AK is a linear pool of predictions with and without the last weekly observation. + 2023-12-02: Added a third component model, which is a gradient boosted quantile regression model that does not use any features that are a measure of the level of recent incidence." +ensemble_of_models: true +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/UMass-trends_ensemble.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/UMass-trends_ensemble.yml new file mode 100644 index 0000000..6ef96fd --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/UMass-trends_ensemble.yml @@ -0,0 +1,37 @@ +team_name: "UMass-Amherst" +team_abbr: "UMass" +model_name: "Ensemble of baseline models with trends" +model_abbr: "trends_ensemble" +model_version: "1.0" +model_contributors: [ + { + "name": "Nutcha Wattanachit", + "affiliation": "UMass Amherst", + "email": "nwattanachit@schoolph.umass.edu" + }, + { + "name": "Aaron Gerding", + "affiliation": "UMass Amherst", + "email": "agerding@umass.edu" + }, + { + "name": "Nick Reich", + "affiliation": "UMass Amherst", + "email": "nick@umass.edu" + }, + { + "name": "Evan Ray", + "affiliation": "UMass Amherst", + "email": "elray@umass.edu" + } +] +website_url: "https://github.com/reichlab/flu-hosp-models-2021-2022" +license: "CC-BY-4.0" +citation: "citation" +team_funding: "funding" +designated_model: true +methods: "Equally weighted ensemble of simple time-series baseline models." +data_inputs: "Daily and weekly incident flu hospitalizations, queried through covidData" +methods_long: "Equally weighted ensemble of simple time-series baseline models. Each baseline model calculates first differences of incidence in recent weeks. These differences are sampled and then added to the most recently observed incidence. Variations on this method include (a) including the first differences and the negative of these differences to enforce symmetry, resulting in a flat-line forecast, (b) generating predictions by working on the daily scale and then aggregating to weekly predictions, or by working directly with weekly data; (c) varying the number of time-units in the past for computing the first differences (14 or 21 days, or 3 or 4 weeks) to focus on capturing recent trends, and (d) using the original time-series or a variance-stabilizing transformation of it, e.g. square-root. Additionally, the resulting predictive distributions are truncated so that any predicted samples computed to be less than zero are truncated to be zero." +ensemble_of_models: true +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/UNC_IDD-InfluPaint.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/UNC_IDD-InfluPaint.yml new file mode 100644 index 0000000..138a3df --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/UNC_IDD-InfluPaint.yml @@ -0,0 +1,27 @@ +team_name: "UNC Infectious Disease Dynamics" +team_abbr: "UNC_IDD" +model_name: "InfluPaint" +model_abbr: "InfluPaint" +model_version: "0.4" +model_contributors: [ + { + "name": "Joseph Lemaitre", + "affiliation": "University of North Carolina at Chapel Hill", + "email": "jo.lemaitresamra@gmail.com" + }, + { + "name": "Justin Lessler", + "affiliation": "University of North Carolina at Chapel Hill", + "email": "jlessler@unc.edu" + }, +] +website_url: "https://github.com/jcblemai/inpainting-idforecasts" +repo_url: "https://github.com/jcblemai/inpainting-idforecasts" +license: "CC-BY_SA-4.0" +designated_model: true +team_funding: "J.L. and J.C.L. disclose support from the National Institutes of Health (NIH 5R01AI102939)." +methods: "A generative denoising diffusion probabilistic model of Influenza dynamics generates synthetic epidemic trajectories, conditioned on past ground-truth data using an inpainting algorithm." +data_inputs: "inference on the target data only, but the machine learning model trained on Fluview and FlepiMoP projections for the US Flu Scenario modeling hub" +methods_long: "We use a generative AI method, denoising diffusion probabilistic model (DDPMs), to generate epidemic trajectories (we treat influenza epidemic curves as images (axes are time and location, a pixel value is e.g incident hospitalization and stay close to the image generation literature). So our model account characterizes the uncertainty that exists in the space of possible flu trajectories. Spatial correlation is considered as all the features of the full images should be reproduced in our generated images. Then, we use the COPaint algorithm to inpaint the future from ground-truth data." +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/UVAFluX-Ensemble.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/UVAFluX-Ensemble.yml new file mode 100644 index 0000000..48e21c1 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/UVAFluX-Ensemble.yml @@ -0,0 +1,50 @@ +team_name: "UVAFluX-Ensemble" +team_abbr: "UVAFluX" +model_name: "Ensemble of statistical and deep learning models" +model_abbr: "Ensemble" +model_version: "1.0" +model_contributors: [ + { + "name": "Aniruddha Adiga", + "affiliation": "UVA", + "email": "aniruddha@virginia.edu" + }, + { + "name": "Gursharn Kaur", + "affiliation": "UVA", + "email": "gursharn@virginia.edu" + }, + { + "name": "Bryan Lewis", + "affiliation": "UVA", + "email": "bl4zc@virginia.edu" + }, + { + "name": "Madhav Marathe", + "affiliation": "UVA", + "email": "marathe@virginia.edu" + }, + { + "name": "Srinivasan Venkatramanan", + "affiliation": "UVA", + "email": "srini@virginia.edu" + }, + { + "name": "Brian Klahn", + "affiliation": "UVA", + "email": "briandk@virginia.edu" + } +] +website_url: "https://biocomplexity.virginia.edu/institute/divisions/network-systems-science-and-advanced-computing" +license: "CC-BY-4.0" +citation: "citation" +team_funding: "funding" +designated_model: true +methods: "An ensemble of multiple statistical and deep learning models." +data_inputs: "Daily and weekly incident flu hospitalizations from HHS" +methods_long: "This is a US national and state-level multi-method model + forecasting the new influenza hospitalizations. Multiple methods include + ARIMA (optimized for length of observed data), exponential smoothing (optimized for length of observed data), LSTM model, Kalman filter, and + piecewise-linear approximation model for trend projection. The model forecasts are combined using Bayesian model averaging, mean-, or median-based ensembling strategies." +ensemble_of_models: true +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/VTSanghani-Ensemble.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/VTSanghani-Ensemble.yml new file mode 100644 index 0000000..6b17fac --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/VTSanghani-Ensemble.yml @@ -0,0 +1,28 @@ +team_name: "VTSanghani" +team_abbr: "VTSanghani" +model_name: "Ensemble of deep learning models combined with traditonal statistical models" +model_abbr: "Ensemble" +model_version: "1.0" +model_contributors: [ + { + "name": "Patrick Butler", + "affiliation": "Virginia Tech", + "email": "pabutler@vt.edu" + }, + { + "name": "Naren Ramakrishnan", + "affiliation": "Virginia Tech", + "email": "naren@cs.vt.edu" + }, +] +website_url: "https://sanghani.cs.vt.edu/" +license: "CC-BY-4.0" +citation: "citation" +team_funding: "funding" +designated_model: true +methods: "An ensemble of deep learning models as well as traditonal statistical methods." +data_inputs: "Daily and weekly incident flu hospitalizations from HHS, Air Quality Index from EPA, Demographic data" +methods_long: "This ensemble forecasts US national- and state-level influenza hospitalizations. Included models focus + on novel deep learning techniques augmented with traditional statistical predictions" +ensemble_of_models: true +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/cfa-flumech.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/cfa-flumech.yml new file mode 100644 index 0000000..37abe02 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/cfa-flumech.yml @@ -0,0 +1,24 @@ +team_name: "Center for Forecasting and Outbreak Analytics (CFA/CDC); SEIR model team" +team_abbr: "cfa" +model_name: "flu-mechanistic" +model_abbr: "flumech" +model_version: "1.0" +model_contributors: [ + { + "name": "Sam Brand", + "affiliation": "CFA/CDC", + "email": "usi1@cdc.gov" + }, + { + "name": "Trevor Martin", + "affiliation": "CFA/CDC", + "email": "upx3@cdc.gov" + } +] +license: "CC-BY-4.0" +designated_model: true +methods: "SEIR mechanistic model for each US state with Bayesian inference." +data_inputs: "Daily incident flu hospitalizations, queried through covidData, weekly NREVSS queried through `cdcfluview`." +methods_long: "A mechanistic model generates infections for each day in season for three age groups (children, adults and 65+ elderly) and three flu subtypes (H1N1pdm09, H3N2, B), some of these infections lead to observed data (with delays). The observed data used to calculate the log-likelihood is daily HHS incident hospitalisations per day, weekly flu A/B test positives from clinical labs, and total test distribution across ages and flu subtypes from Public Health labs." +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/cfarenewal-cfaepimlight.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/cfarenewal-cfaepimlight.yml new file mode 100644 index 0000000..1d1c4d4 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/cfarenewal-cfaepimlight.yml @@ -0,0 +1,26 @@ +team_name: "Center for Forecasting and Outbreak Analytics (CFA/CDC); renewal model team" +team_abbr: "cfarenewal" +model_name: "cfa-flu-epidemia-light" +model_abbr: "cfaepimlight" +model_version: "0.1" +model_contributors: [ + { + "name": "Catherine M. Herzog", + "affiliation": "CFA/CDC", + "email": "uvw5@cdc.gov" + }, + + { + "name": "Dylan H. Morris", + "affiliation": "CFA/CDC", + "email": "dylan@dylanhmorris.com" + } +] + +license: "CC-BY-4.0" +designated_model: true +methods: "Simple renewal model fit to daily incidence data using the Epidemia package" +data_inputs: "Daily incident flu hospitalizations via HealthData.gov" +methods_long: "Semi-mechanistic renewal equation model with day-of-week observation effects and a simple population size adjustment. R(t) extrapolation via a weekly random walk. Model developed and fit using the Epidemia package for R and Stan; see documentation: https://imperialcollegelondon.github.io/epidemia/articles/model-description.html" +ensemble_of_models: false +ensemble_of_hub_models: false diff --git a/tests/hubs/FluSight-forecast-hub/model-metadata/fjordhest-ensemble.yml b/tests/hubs/FluSight-forecast-hub/model-metadata/fjordhest-ensemble.yml new file mode 100644 index 0000000..fb63d91 --- /dev/null +++ b/tests/hubs/FluSight-forecast-hub/model-metadata/fjordhest-ensemble.yml @@ -0,0 +1,30 @@ +team_name: Fjordhest +team_abbr: fjordhest +model_name: fjordhest-ensemble +model_abbr: ensemble +model_contributors: [ +{ +"name": "Sasi Kandula", +"affiliation": "Norwegian Institute of Public Health", +"email": "sasikiran.kandula@fhi.no" +}, +{ +"name": "Alfonso Diz-Lois Palomares", +"affiliation": "Norwegian Institute of Public Health; University of Oslo", +"email": "Alfonso.Diz-LoisPalomares@fhi.no" +}, +{ +"name": "Birgitte De Blasio", +"affiliation": "Norwegian Institute of Public Health; University of Oslo", +"email": "Birgitte.Freiesleben.DeBlasio@fhi.no" +} +] +license: CC-BY_SA-4.0 +designated_model: true +data_inputs: US Department of Health and Human Services, COVID-19 Reported Patient Impact and Hospital Capacity by State Timeseries +methods: An inverse-WIS weighted ensemble of 3 component models - a compartmental model, a quantile AR model, and a baseline model of random walk with drift. +methods_long: The mechanistic model is a stochastic SEIR model, with a sequential Monte Carlo based inference. Quantile autoregression model uses quantgen R package, and is similar to CMU-Timeseries model from 2022/23 season. Random walk model uses fable R package. To build an ensemble, the quantile distributions of the component models are weighted (location- and target-specific) by the mean of inverse-WIS scores over the last 3 weeks that could be evaluated. The estimates are solely the responsibility of the contributors and do not represent, nor are endorsed by, the Norwegian Institute of Public Health. +ensemble_of_models: true +ensemble_of_hub_models: false +team_funding: none +citation: "Geir Storvik, Alfonso Diz-Lois Palomares, Solveig Engebretsen, et al. (2023) A sequential Monte Carlo approach to estimate a time varying reproduction number in infectious disease models: the COVID-19 case., Journal of the Royal Statistical Society Series A: Statistics in Society, 2023; https://doi.org/10.1093/jrsssa/qnad043; Tibshirani R, Brooks L (2020). quantgen: Tools for generalized quantile modeling; https://github.com/cmu-delphi/flu-hosp-forecast/; O'Hara-Wild M, Hyndman R, Wang E (2022). fable: Forecasting Models for Tidy Time Series. R package version 0.3.2."