test.yml

name: Automatic Tests
'on':
  push:
    branches:
      - main
    paths:
      - lenskit/**.py
      - '**pyproject.toml'
      - requirements*.txt
      - data/**
      - scripts/coverage-log.tcl
      - .github/workflows/test.yml
  pull_request:
    paths:
      - lenskit/**.py
      - '**pyproject.toml'
      - requirements*.txt
      - data/**
      - scripts/coverage-log.tcl
      - .github/workflows/test.yml
concurrency:
  group: test-${{github.ref}}
  cancel-in-progress: true
jobs:
  conda:
    name: Conda Python ${{matrix.python}} on ${{matrix.platform}}
    runs-on: ${{matrix.platform}}
    timeout-minutes: 30
    defaults:
      run:
        shell: bash -el {0}
    strategy:
      fail-fast: false
      matrix:
        python:
          - '3.10'
          - '3.11'
          - '3.12'
        platform:
          - ubuntu-latest
          - macos-latest
          - windows-latest
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Setup bootstrap Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: 👢 Generate Conda environment file
        run: |
          pip install -e .
          python -m lkdev.conda -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml
      - id: setup
        name: 📦 Set up Conda environment
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: ci-environment.yml
          environment-name: lkpy
          cache-environment: true
          init-shell: bash
      - name: 🍱 Install LensKit packages
        run: |
          pip install --no-deps -e lenskit
      - name: 🔍 Inspect environment
        run: |
          python -m lenskit.util.envcheck
      - name: 🏃🏻‍➡️ Test LKPY
        run: |
          python -m pytest --verbose --log-file=test.log --durations=25 --cov=lenskit/lenskit lenskit/tests
      - name: 📐 Coverage results
        run: |
          python utils/fix-coverage-paths.py
          coverage xml
          coverage report
      - name: 📤 Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-conda-${{matrix.platform}}-py${{matrix.python}}
          path: |
            test*.log
            .coverage
            coverage.xml
  vanilla:
    name: Vanilla Python ${{matrix.python}} on ${{matrix.platform}}
    runs-on: ${{matrix.platform}}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        python:
          - '3.10'
          - '3.11'
          - '3.12'
        platform:
          - ubuntu-latest
          - macos-latest
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Set up Python
        id: install-python
        uses: actions/setup-python@v5
        with:
          python-version: ${{matrix.python}}
          cache: pip
          cache-dependency-path: |
            requirements*.txt
            */pyproject.toml
      - name: 🕶️ Set up uv
        run: |
          pip install -U 'uv>=0.1.15'
      - name: 📦 Set up Python dependencies
        id: install-deps
        run: |
          uv pip install --python $PYTHON -r requirements-test.txt -e lenskit
        shell: bash
        env:
          PYTHON: ${{steps.install-python.outputs.python-path}}
          UV_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu
          UV_INDEX_STRATEGY: unsafe-first-match
      - name: 🔍 Inspect environment
        run: |
          python -m lenskit.util.envcheck
      - name: 🏃🏻‍➡️ Test LKPY
        run: |
          python -m pytest --verbose --log-file=test.log --durations=25 --cov=lenskit/lenskit lenskit/tests
      - name: 📐 Coverage results
        run: |
          python utils/fix-coverage-paths.py
          coverage xml
          coverage report
      - name: 📤 Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-vanilla-${{matrix.platform}}-py${{matrix.python}}
          path: |
            test*.log
            .coverage
            coverage.xml
  nojit:
    name: Non-JIT test coverage
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Set up Python
        id: install-python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
          cache: pip
          cache-dependency-path: |
            requirements*.txt
            */pyproject.toml
      - name: 🕶️ Set up uv
        run: |
          pip install -U 'uv>=0.1.15'
      - name: 📦 Set up Python dependencies
        id: install-deps
        run: |
          uv pip install --python $PYTHON -r requirements-test.txt -e lenskit -e lenskit-funksvd
        shell: bash
        env:
          PYTHON: ${{steps.install-python.outputs.python-path}}
          UV_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu
          UV_INDEX_STRATEGY: unsafe-first-match
      - name: 🔍 Inspect environment
        run: |
          python -m lenskit.util.envcheck
      - name: 🏃🏻‍➡️ Test LKPY
        run: |
          python -m pytest --verbose --log-file=test.log --durations=25 -m 'not slow' --cov=lenskit/lenskit --cov=lenskit-funksvd/lenskit lenskit/tests lenskit-funksvd/tests
        env:
          NUMBA_DISABLE_JIT: 1
          PYTORCH_JIT: 0
      - name: 📐 Coverage results
        run: |
          python utils/fix-coverage-paths.py
          coverage xml
          coverage report
      - name: 📤 Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-nojit
          path: |
            test*.log
            .coverage
            coverage.xml
  mindep:
    name: Minimal dependency tests
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Set up Python
        id: install-python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
          cache: pip
          cache-dependency-path: |
            requirements*.txt
            */pyproject.toml
      - name: 🕶️ Set up uv
        run: |
          pip install -U 'uv>=0.1.15'
      - name: 📦 Set up Python dependencies
        id: install-deps
        run: |
          uv pip install --python $PYTHON -r requirements-test.txt -e lenskit --resolution=lowest-direct
        shell: bash
        env:
          PYTHON: ${{steps.install-python.outputs.python-path}}
          UV_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu
          UV_INDEX_STRATEGY: unsafe-first-match
      - name: 🔍 Inspect environment
        run: |
          python -m lenskit.util.envcheck
      - name: 🏃🏻‍➡️ Test LKPY
        run: |
          python -m pytest --verbose --log-file=test.log --durations=25 --cov=lenskit/lenskit lenskit/tests
      - name: 📐 Coverage results
        run: |
          python utils/fix-coverage-paths.py
          coverage xml
          coverage report
      - name: 📤 Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-mindep
          path: |
            test*.log
            .coverage
            coverage.xml
  funksvd:
    name: FunkSVD tests on Python ${{matrix.python}}
    runs-on: ubuntu-latest
    timeout-minutes: 30
    defaults:
      run:
        shell: bash -el {0}
    strategy:
      fail-fast: false
      matrix:
        python:
          - '3.10'
          - '3.11'
          - '3.12'
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Setup bootstrap Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: 👢 Generate Conda environment file
        run: |
          pip install -e .
          python -m lkdev.conda -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml lenskit-funksvd/pyproject.toml
      - id: setup
        name: 📦 Set up Conda environment
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: ci-environment.yml
          environment-name: lkpy
          cache-environment: true
          init-shell: bash
      - name: 🍱 Install LensKit packages
        run: |
          pip install --no-deps -e lenskit -e lenskit-funksvd
      - name: 🔍 Inspect environment
        run: |
          python -m lenskit.util.envcheck
      - name: 🏃🏻‍➡️ Test LKPY
        run: |
          python -m pytest --verbose --log-file=test.log --durations=25 --cov=lenskit-funksvd/lenskit lenskit-funksvd/tests
      - name: 📐 Coverage results
        run: |
          python utils/fix-coverage-paths.py
          coverage xml
          coverage report
      - name: 📤 Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-funksvd-py${{matrix.python}}
          path: |
            test*.log
            .coverage
            coverage.xml
  funksvd-mindep:
    name: Minimal dependency tests for FunkSVD
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Set up Python
        id: install-python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
          cache: pip
          cache-dependency-path: |
            requirements*.txt
            */pyproject.toml
      - name: 🕶️ Set up uv
        run: |
          pip install -U 'uv>=0.1.15'
      - name: 📦 Set up Python dependencies
        id: install-deps
        run: |
          uv pip install --python $PYTHON -r requirements-test.txt -e lenskit -e lenskit-funksvd --resolution=lowest-direct
        shell: bash
        env:
          PYTHON: ${{steps.install-python.outputs.python-path}}
          UV_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu
          UV_INDEX_STRATEGY: unsafe-first-match
      - name: 🔍 Inspect environment
        run: |
          python -m lenskit.util.envcheck
      - name: 🏃🏻‍➡️ Test LKPY
        run: |
          python -m pytest --verbose --log-file=test.log --durations=25 --cov=lenskit-funksvd/lenskit lenskit-funksvd/tests
      - name: 📐 Coverage results
        run: |
          python utils/fix-coverage-paths.py
          coverage xml
          coverage report
      - name: 📤 Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-mindep-funksvd
          path: |
            test*.log
            .coverage
            coverage.xml
  implicit:
    name: Implicit bridge tests on Python ${{matrix.python}}
    runs-on: ubuntu-latest
    timeout-minutes: 30
    defaults:
      run:
        shell: bash -el {0}
    strategy:
      fail-fast: false
      matrix:
        python:
          - '3.10'
          - '3.11'
          - '3.12'
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Setup bootstrap Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: 👢 Generate Conda environment file
        run: |
          pip install -e .
          python -m lkdev.conda -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml lenskit-implicit/pyproject.toml
      - id: setup
        name: 📦 Set up Conda environment
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: ci-environment.yml
          environment-name: lkpy
          cache-environment: true
          init-shell: bash
      - name: 🍱 Install LensKit packages
        run: |
          pip install --no-deps -e lenskit -e lenskit-implicit
      - name: 🔍 Inspect environment
        run: |
          python -m lenskit.util.envcheck
      - name: 🏃🏻‍➡️ Test LKPY
        run: |
          python -m pytest --verbose --log-file=test.log --durations=25 --cov=lenskit-implicit/lenskit lenskit-implicit/tests
      - name: 📐 Coverage results
        run: |
          python utils/fix-coverage-paths.py
          coverage xml
          coverage report
      - name: 📤 Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-implicit-py${{matrix.python}}
          path: |
            test*.log
            .coverage
            coverage.xml
  implicit-mindep:
    name: Minimal dependency tests for Implicit
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Set up Python
        id: install-python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
          cache: pip
          cache-dependency-path: |
            requirements*.txt
            */pyproject.toml
      - name: 🕶️ Set up uv
        run: |
          pip install -U 'uv>=0.1.15'
      - name: 📦 Set up Python dependencies
        id: install-deps
        run: |
          uv pip install --python $PYTHON -r requirements-test.txt -e lenskit -e lenskit-implicit --resolution=lowest-direct
        shell: bash
        env:
          PYTHON: ${{steps.install-python.outputs.python-path}}
          UV_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu
          UV_INDEX_STRATEGY: unsafe-first-match
      - name: 🔍 Inspect environment
        run: |
          python -m lenskit.util.envcheck
      - name: 🏃🏻‍➡️ Test LKPY
        run: |
          python -m pytest --verbose --log-file=test.log --durations=25 --cov=lenskit-implicit/lenskit lenskit-implicit/tests
      - name: 📐 Coverage results
        run: |
          python utils/fix-coverage-paths.py
          coverage xml
          coverage report
      - name: 📤 Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-mindep-implicit
          path: |
            test*.log
            .coverage
            coverage.xml
  hpf:
    name: HPF bridge tests on Python ${{matrix.python}}
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        python:
          - '3.10'
          - '3.11'
          - '3.12'
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Set up Python
        id: install-python
        uses: actions/setup-python@v5
        with:
          python-version: ${{matrix.python}}
          cache: pip
          cache-dependency-path: |
            requirements*.txt
            */pyproject.toml
      - name: 🕶️ Set up uv
        run: |
          pip install -U 'uv>=0.1.15'
      - name: 📦 Set up Python dependencies
        id: install-deps
        run: |
          uv pip install --python $PYTHON -r requirements-test.txt -e lenskit -e lenskit-hpf
        shell: bash
        env:
          PYTHON: ${{steps.install-python.outputs.python-path}}
          UV_EXTRA_INDEX_URL: https://download.pytorch.org/whl/cpu
          UV_INDEX_STRATEGY: unsafe-first-match
      - name: 🔍 Inspect environment
        run: |
          python -m lenskit.util.envcheck
      - name: 🏃🏻‍➡️ Test LKPY
        run: |
          python -m pytest --verbose --log-file=test.log --durations=25 --cov=lenskit-hpf/lenskit lenskit-hpf/tests
      - name: 📐 Coverage results
        run: |
          python utils/fix-coverage-paths.py
          coverage xml
          coverage report
      - name: 📤 Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-hpf-py${{matrix.python}}
          path: |
            test*.log
            .coverage
            coverage.xml
  eval-tests:
    name: Evaluation-based tests
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Setup bootstrap Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: 👢 Generate Conda environment file
        run: |
          pip install -e .
          python -m lkdev.conda -o ci-environment.yml -e all requirements-test.txt lenskit/pyproject.toml lenskit-funksvd/pyproject.toml lenskit-implicit/pyproject.toml
      - id: setup
        name: 📦 Set up Conda environment
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: ci-environment.yml
          environment-name: lkpy
          cache-environment: true
          init-shell: bash
      - name: 🍱 Install LensKit packages
        run: |
          pip install --no-deps -e lenskit -e lenskit-funksvd -e lenskit-implicit
      - name: Cache ML data
        uses: actions/cache@v4
        with:
          path: |
            data
            !data/*.zip
          key: test-mldata-000-2ee919d5c0eef34d5a4f40bcf0480c1bf0310417db6921e3a2575c48991f379c2f4ad179f8514390133795614a96fa5b4ece55906c68a90af07c09670b2c3c5b
      - name: Download ML data
        run: |
          python -m lenskit.datasets.fetch ml-100k ml-20m
      - name: Run Eval Tests
        run: |
          python -m pytest --cov=lenskit/lenskit --cov=lenskit-funksvd/lenskit --cov=lenskit-implicit/lenskit -m 'eval or realdata' --log-file test-eval.log */tests
      - name: 📐 Coverage results
        run: |
          python utils/fix-coverage-paths.py
          coverage xml
          coverage report
      - name: 📤 Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-eval-tests
          path: |
            test*.log
            .coverage
            coverage.xml
  doc-tests:
    name: Demos, examples, and docs
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: 🐍 Setup bootstrap Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: 👢 Generate Conda environment file
        run: |
          pip install -e .
          python -m lkdev.conda -o ci-environment.yml -e all requirements-test.txt requirements-demo.txt lenskit/pyproject.toml lenskit-funksvd/pyproject.toml lenskit-implicit/pyproject.toml
      - id: setup
        name: 📦 Set up Conda environment
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: ci-environment.yml
          environment-name: lkpy
          cache-environment: true
          init-shell: bash
      - name: 🍱 Install LensKit packages
        run: |
          pip install --no-deps -e lenskit -e lenskit-funksvd -e lenskit-implicit
      - name: Cache ML data
        uses: actions/cache@v4
        with:
          path: |
            data
            !data/*.zip
          key: test-mldata-000-cd26f1c44a6962b0936346b346a9b418a3ed04b01a2892269fccd24a6387e943dba6d5e64ab2f8feb1823475601d65c2e6ebbeeeca0c2c210f0d37c00aabf2e9
      - name: Download ML data
        run: |
          python -m lenskit.datasets.fetch ml-100k ml-1m ml-10m ml-20m
      - name: 📕 Validate documentation examples
        run: |
          python -m pytest --cov=lenskit/lenskit --cov=lenskit-funksvd/lenskit --cov=lenskit-implicit/lenskit --nbval-lax --log-file test-docs.log docs */lenskit
      - name: 📐 Coverage results
        run: |
          python utils/fix-coverage-paths.py
          coverage xml
          coverage report
      - name: 📤 Upload test results
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: test-examples
          path: |
            test*.log
            .coverage
            coverage.xml
  results:
    name: Test suite results
    runs-on: ubuntu-latest
    needs:
      - conda
      - vanilla
      - nojit
      - mindep
      - funksvd
      - funksvd-mindep
      - implicit
      - implicit-mindep
      - hpf
      - eval-tests
      - doc-tests
    steps:
      - name: 🛒 Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Add upstream remote
        run: |
          git remote add upstream https://github.com/lenskit/lkpy.git
          git fetch upstream
      - name: 🐍 Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: 📦 Install reporting packages
        run: python -m pip install -r requirements-reporting.txt
      - name: 📥 Download test artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: test-*
          path: test-logs
      - name: 📋 List log files
        run: ls -lR test-logs
      - name: ⛙ Merge and report
        run: |
          coverage combine test-logs/*/.coverage
          coverage xml
          coverage html -d lenskit-coverage
          coverage report --format=markdown >coverage.md
      - name: Analyze diff coverage
        if: github.event_name == 'pull_request'
        run: |
          diff-cover --json-report diff-cover.json --markdown-report diff-cover.md \
              coverage.xml |tee diff-cover.txt
      - name: ± Measure and report coverage
        run: |
          echo $PR_NUMBER > ./lenskit-coverage/pr-number
          tclsh ./utils/coverage-log.tcl
          cat lenskit-coverage/report.md >$GITHUB_STEP_SUMMARY
      - name: 📤 Upload coverage report
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: coverage-report
          path: lenskit-coverage/
      - name: 🚫 Fail if coverage is too low
        run: coverage report --fail-under=90