From 7b63c802768d17c88ff5b45aff2beddd174f895e Mon Sep 17 00:00:00 2001 From: nikosbosse Date: Tue, 3 Dec 2024 13:24:30 +0100 Subject: [PATCH] prepare automated RSV submission --- .github/workflows/submit-rsv-forecasts.yaml | 130 ++++++++++++++++++++ rsv/Metaculus-cp.yaml | 28 +++++ 2 files changed, 158 insertions(+) create mode 100644 .github/workflows/submit-rsv-forecasts.yaml create mode 100644 rsv/Metaculus-cp.yaml diff --git a/.github/workflows/submit-rsv-forecasts.yaml b/.github/workflows/submit-rsv-forecasts.yaml new file mode 100644 index 0000000..c805ec0 --- /dev/null +++ b/.github/workflows/submit-rsv-forecasts.yaml @@ -0,0 +1,130 @@ +name: Make RSV Forecast Submission + +on: + schedule: + # Every Tuesday at 6pm CET (17:00 UTC) + - cron: '0 17 * * 2' + workflow_dispatch: # Allow manual triggering + +jobs: + submit-forecast: + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests pandas numpy + + - name: Create required directories + run: | + mkdir -p rsv/submissions + + - name: Run rsv forecasting script + run: python rsv/run-rsv-forecasts.py + env: + PYTHONPATH: ${{ github.workspace }} + + - name: Commit forecasts to repository + run: | + # Configure git + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Pull latest changes + git pull origin main --rebase + + # Add and commit the new forecasts + git add rsv/submissions/ + + # Only commit if there are changes + if git diff --staged --quiet; then + echo "No new forecasts to commit" + else + git commit -m "Store rsv forecasts $(date +%Y-%m-%d)" + # Try to push, if it fails due to conflicts, pull and push again + git push || (git pull --rebase origin main && git push) + fi + + - name: Fork and sync target repository + run: | + # Install GitHub CLI + gh auth login --with-token <<< "${{ secrets.PRIVATE_ACCESS_TOKEN }}" + + # Fork the repository (if not already forked) + gh repo fork cdcepi/FluSight-forecast-hub --clone=true || true + + # Sync the fork with upstream + pushd FluSight-forecast-hub + + # Add upstream remote if it doesn't exist + git remote add upstream https://github.com/HopkinsIDD/rsv-forecast-hub.git 2>/dev/null || true + + # Set the origin remote with authentication token + git remote set-url origin "https://${{ secrets.PRIVATE_ACCESS_TOKEN }}@github.com/${{ github.actor }}/rsv-forecast-hub.git" + + # Fetch and sync with upstream + git fetch upstream + git checkout main + git reset --hard upstream/main + git push origin main --force + + popd + + - name: Copy forecast files + run: | + # Create target directory if it doesn't exist + mkdir -p rsv-forecast-hub/model-output/Metaculus-cp + + # Copy new forecasts + cp -r rsv/submissions/* rsv-forecast-hub/model-output/Metaculus-cp/ + + - name: Create Pull Request + run: | + cd rsv-forecast-hub + + # Setup git config + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Create a new branch with timestamp to ensure uniqueness + BRANCH_NAME="rsv-forecast-update-$(date +%Y%m%d-%H%M%S)" + git checkout -b $BRANCH_NAME + + # Add and commit changes + git add model-output/Metaculus-cp/ + + # Only commit if there are changes + if git diff --staged --quiet; then + echo "No changes to commit" + exit 0 + else + git commit -m "Update rsv forecasts $(date +%Y-%m-%d)" + + # Set the remote URL with authentication token + git remote set-url origin "https://${{ secrets.PRIVATE_ACCESS_TOKEN }}@github.com/${{ github.actor }}/rsv-forecast-hub.git" + + # Force push to fork with the new unique branch + git push -f origin $BRANCH_NAME + + # Create PR + gh pr create \ + --title "Submission Metaculus forecasts $(date +%Y-%m-%d)" \ + --body "Automated rsv forecast submission from Metaculus" \ + --repo HopkinsIDD/rsv-forecast-hub \ + --base main \ + --head "${{ github.actor }}:$BRANCH_NAME" + fi + env: + PRIVATE_ACCESS_TOKEN: ${{ secrets.PRIVATE_ACCESS_TOKEN }} \ No newline at end of file diff --git a/rsv/Metaculus-cp.yaml b/rsv/Metaculus-cp.yaml new file mode 100644 index 0000000..49ef3d0 --- /dev/null +++ b/rsv/Metaculus-cp.yaml @@ -0,0 +1,28 @@ +team_name: "Metaculus" +team_abbr: "Metaculus" +model_name: "Metaculus Community Prediction" +model_abbr: "cp" +model_version: "1.0" +model_contributors: + [ + { + "name": "Ryan Beck", + "affiliation": "Metaculus", + "email": "ryan@metaculus.com", + }, + { + "name": "Nikos Bosse", + "affiliation": "Metaculus", + "email": "nikos@metaculus.com", + }, + ] +website_url: "https://www.metaculus.com/questions/30048/us-rsv-hospitalization-forecasts-2024-25/" +repo_url: "https://github.com/Metaculus/respiratory-diseases" +license: "CC-BY-4.0" +designated_model: true +team_funding: "This project is supported by the National Science Foundation under Award No. 2438211. Any opinions, findings and conclusions or recommendations expressed in this project are those of Metaculus and our forecasters, and do not necessarily reflect the views of the National Science Foundation." +methods: "A recency-weighted average of predictions made by forecasters on the Metaculus prediction platform. Missing forecasts are linearly interpolated." +data_inputs: "Users are allowed to make use of any data they choose. The recency-weighted average takes only the numeric forecasts made by forecasters on the platform into account. We only launch new questions every 2 weeks. This means that alternatingly, only the forecasts 1, 3, 5 or 0, 2, 4 will actually be available. Missing forecasts are linearly interpolated to always produce a set for horizons 1, 2, 3, 4." +methods_long: "The Metaculus Community Prediction is a consensus of recent forecaster predictions. It is designed to respond to big changes in forecaster opinion while still being fairly insensitive to outliers. For every forecaster, on ly their most recent prediction is kept. Predictions are assigned a number n, from oldest to newest (oldest is 1). Every prediction is weighted proportional to exp(sqrt(n)). Predictions are then aggregated by creating a mixture distribution of all available weighted forecasts." +ensemble_of_models: true +ensemble_of_hub_models: false \ No newline at end of file