From c827e94da08bff143b8f364b53fbea998855ea0a Mon Sep 17 00:00:00 2001 From: Sayak Paul Date: Wed, 10 Apr 2024 08:34:56 +0530 Subject: [PATCH] [Workflows] remove installation of `libsndfile1-dev` and `libgl1` from workflows (#7543) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * remove libsndfile1-dev and libgl1 from workflows and ensure that re present in the respective dockerfiles. * change to self-hosted runner; let's see 🤞 * add libsndfile1-dev libgl1 for now * use self-hosted runners for building and push too. --- .github/workflows/benchmark.yml | 1 - .github/workflows/build_docker_images.yml | 4 ++-- .github/workflows/nightly_tests.yml | 4 ---- .github/workflows/pr_test_fetcher.yml | 3 --- .github/workflows/pr_test_peft_backend.yml | 1 - .github/workflows/pr_tests.yml | 3 +-- .github/workflows/push_tests.yml | 10 +++++----- .github/workflows/push_tests_fast.yml | 1 - docker/diffusers-flax-cpu/Dockerfile | 1 + docker/diffusers-flax-tpu/Dockerfile | 1 + docker/diffusers-onnxruntime-cpu/Dockerfile | 1 + docker/diffusers-onnxruntime-cuda/Dockerfile | 1 + 12 files changed, 12 insertions(+), 19 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c0ae87521fdc..718c67731bd5 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -31,7 +31,6 @@ jobs: nvidia-smi - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] python -m uv pip install pandas peft diff --git a/.github/workflows/build_docker_images.yml b/.github/workflows/build_docker_images.yml index 3d061f88c241..51abd3483ad5 100644 --- a/.github/workflows/build_docker_images.yml +++ b/.github/workflows/build_docker_images.yml @@ -20,7 +20,7 @@ env: jobs: test-build-docker-images: - runs-on: ubuntu-latest + runs-on: [ self-hosted, intel-cpu, 8-cpu, ci ] if: github.event_name == 'pull_request' steps: - name: Set up Docker Buildx @@ -50,7 +50,7 @@ jobs: if: steps.file_changes.outputs.all != '' build-and-push-docker-images: - runs-on: ubuntu-latest + runs-on: [ self-hosted, intel-cpu, 8-cpu, ci ] if: github.event_name != 'pull_request' permissions: diff --git a/.github/workflows/nightly_tests.yml b/.github/workflows/nightly_tests.yml index d489da8e48eb..2f73c66de829 100644 --- a/.github/workflows/nightly_tests.yml +++ b/.github/workflows/nightly_tests.yml @@ -70,7 +70,6 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git @@ -131,7 +130,6 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git @@ -202,7 +200,6 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git @@ -262,7 +259,6 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git diff --git a/.github/workflows/pr_test_fetcher.yml b/.github/workflows/pr_test_fetcher.yml index 6a7da2cb73d0..4dbb118c6092 100644 --- a/.github/workflows/pr_test_fetcher.yml +++ b/.github/workflows/pr_test_fetcher.yml @@ -32,7 +32,6 @@ jobs: fetch-depth: 0 - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] - name: Environment @@ -89,7 +88,6 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pip install -e [quality,test] python -m pip install accelerate @@ -147,7 +145,6 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pip install -e [quality,test] diff --git a/.github/workflows/pr_test_peft_backend.yml b/.github/workflows/pr_test_peft_backend.yml index 506c5ad649f4..ca91535f0274 100644 --- a/.github/workflows/pr_test_peft_backend.yml +++ b/.github/workflows/pr_test_peft_backend.yml @@ -89,7 +89,6 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] if [ "${{ matrix.lib-versions }}" == "main" ]; then diff --git a/.github/workflows/pr_tests.yml b/.github/workflows/pr_tests.yml index ce92e32c76a0..aa4afebb9cc1 100644 --- a/.github/workflows/pr_tests.yml +++ b/.github/workflows/pr_tests.yml @@ -116,7 +116,6 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] python -m uv pip install accelerate @@ -147,6 +146,7 @@ jobs: - name: Run fast Flax TPU tests if: ${{ matrix.config.framework == 'flax' }} run: | + apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m pytest -n 4 --max-worker-restart=0 --dist=loadfile \ -s -v -k "Flax" \ @@ -205,7 +205,6 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 0a316c90dfed..36f011407901 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -71,7 +71,7 @@ jobs: nvidia-smi - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git @@ -121,7 +121,7 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git @@ -171,7 +171,7 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git @@ -222,7 +222,7 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git @@ -270,7 +270,7 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y + python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate.git diff --git a/.github/workflows/push_tests_fast.yml b/.github/workflows/push_tests_fast.yml index 9f0c44457d8a..7c50da7b5c34 100644 --- a/.github/workflows/push_tests_fast.yml +++ b/.github/workflows/push_tests_fast.yml @@ -68,7 +68,6 @@ jobs: - name: Install dependencies run: | - apt-get update && apt-get install libsndfile1-dev libgl1 -y python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH" python -m uv pip install -e [quality,test] diff --git a/docker/diffusers-flax-cpu/Dockerfile b/docker/diffusers-flax-cpu/Dockerfile index 36d036e34e5f..005c0f9caacf 100644 --- a/docker/diffusers-flax-cpu/Dockerfile +++ b/docker/diffusers-flax-cpu/Dockerfile @@ -12,6 +12,7 @@ RUN apt update && \ curl \ ca-certificates \ libsndfile1-dev \ + libgl1 \ python3.8 \ python3-pip \ python3.8-venv && \ diff --git a/docker/diffusers-flax-tpu/Dockerfile b/docker/diffusers-flax-tpu/Dockerfile index 78d5f972a753..05ea22488ab9 100644 --- a/docker/diffusers-flax-tpu/Dockerfile +++ b/docker/diffusers-flax-tpu/Dockerfile @@ -12,6 +12,7 @@ RUN apt update && \ curl \ ca-certificates \ libsndfile1-dev \ + libgl1 \ python3.8 \ python3-pip \ python3.8-venv && \ diff --git a/docker/diffusers-onnxruntime-cpu/Dockerfile b/docker/diffusers-onnxruntime-cpu/Dockerfile index 0d032d91e5eb..b60b467b7485 100644 --- a/docker/diffusers-onnxruntime-cpu/Dockerfile +++ b/docker/diffusers-onnxruntime-cpu/Dockerfile @@ -12,6 +12,7 @@ RUN apt update && \ curl \ ca-certificates \ libsndfile1-dev \ + libgl1 \ python3.8 \ python3-pip \ python3.8-venv && \ diff --git a/docker/diffusers-onnxruntime-cuda/Dockerfile b/docker/diffusers-onnxruntime-cuda/Dockerfile index 34e611df7257..16a0d76460f4 100644 --- a/docker/diffusers-onnxruntime-cuda/Dockerfile +++ b/docker/diffusers-onnxruntime-cuda/Dockerfile @@ -12,6 +12,7 @@ RUN apt update && \ curl \ ca-certificates \ libsndfile1-dev \ + libgl1 \ python3.8 \ python3-pip \ python3.8-venv && \