test_linux_multigpu.yml

name: test (multi-GPU)

on:
  push:
    branches: [main, "[0-9]+.[0-9]+.x"] #this is new
  pull_request:
    branches: [main, "[0-9]+.[0-9]+.x"]
    types: [labeled, synchronize, opened]
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  test:
    # if PR has label "multiGPU tests" or "all tests" or if scheduled or manually triggered or on push
    if: >-
      (
        contains(github.event.pull_request.labels.*.name, 'multiGPU tests') ||
        contains(github.event.pull_request.labels.*.name, 'all tests') ||
        contains(github.event_name, 'schedule') ||
        contains(github.event_name, 'workflow_dispatch')
      )

    runs-on: [self-hosted, Linux, X64, CUDA]

    defaults:
      run:
        shell: bash -e {0} # -e to fail on error

    container:
      image: ghcr.io/scverse/scvi-tools:py3.12-cu12-base
      options: --user root --gpus all --pull always

    name: integration

    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python }}

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: "**/pyproject.toml"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip wheel uv
          python -m uv pip install --system "scvi-tools[tests] @ ."
          python -m pip install jax[cuda]
          python -m pip install nvidia-nccl-cu12

      - name: Run pytest
        env:
          MPLBACKEND: agg
          PLATFORM: ${{ matrix.os }}
          DISPLAY: :42
          COLUMNS: 120
        run: |
          coverage run -m pytest -v --color=yes --multigpu-tests --accelerator cuda --devices auto
          coverage report

      - uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}