From e574d6ead04944d5975261bfd8d383d2c16bf318 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Mon, 26 Aug 2024 13:39:01 -0400 Subject: [PATCH 1/7] github: Add an actionlint github workflow Add a new github actions workflow that will perform linting on github actions workflow changes. It will only run if github workflows are modified. Signed-off-by: Russell Bryant --- .github/workflows/actionlint.dockerfile | 3 ++ .github/workflows/actionlint.yml | 42 ++++++++++++++++++++++ .github/workflows/matchers/actionlint.json | 17 +++++++++ 3 files changed, 62 insertions(+) create mode 100644 .github/workflows/actionlint.dockerfile create mode 100644 .github/workflows/actionlint.yml create mode 100644 .github/workflows/matchers/actionlint.json diff --git a/.github/workflows/actionlint.dockerfile b/.github/workflows/actionlint.dockerfile new file mode 100644 index 000000000000..01ac68066d44 --- /dev/null +++ b/.github/workflows/actionlint.dockerfile @@ -0,0 +1,3 @@ +# Since dependabot cannot update workflows using docker, +# we use this indirection since dependabot can update this file. +FROM docker.io/rhysd/actionlint:1.7.1@sha256:435ecdb63b1169e80ca3e136290072548c07fc4d76a044cf5541021712f8f344 diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml new file mode 100644 index 000000000000..be8964dbe699 --- /dev/null +++ b/.github/workflows/actionlint.yml @@ -0,0 +1,42 @@ +name: Lint GitHub Actions workflows +on: + push: + branches: + - "main" + paths: + - '.github/workflows/*.ya?ml' + - '.github/workflows/actionlint.*' + pull_request: + branches: + - "main" + paths: + - '.github/workflows/*.ya?ml' + - '.github/workflows/actionlint.*' + +env: + LC_ALL: en_US.UTF-8 + +defaults: + run: + shell: bash + +permissions: + contents: read + +jobs: + actionlint: + runs-on: ubuntu-latest + steps: + - name: "Checkout" + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + with: + fetch-depth: 0 + + - name: "Download actionlint" + run: | + docker build --tag actionlint - < .github/workflows/actionlint.dockerfile + + - name: "Check workflow files" + run: | + echo "::add-matcher::.github/workflows/matchers/actionlint.json" + docker run --volume="${PWD}:/repo" --workdir=/repo actionlint -color diff --git a/.github/workflows/matchers/actionlint.json b/.github/workflows/matchers/actionlint.json new file mode 100644 index 000000000000..4613e1617bfe --- /dev/null +++ b/.github/workflows/matchers/actionlint.json @@ -0,0 +1,17 @@ +{ + "problemMatcher": [ + { + "owner": "actionlint", + "pattern": [ + { + "regexp": "^(?:\\x1b\\[\\d+m)?(.+?)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*:(?:\\x1b\\[\\d+m)*(\\d+)(?:\\x1b\\[\\d+m)*: (?:\\x1b\\[\\d+m)*(.+?)(?:\\x1b\\[\\d+m)* \\[(.+?)\\]$", + "file": 1, + "line": 2, + "column": 3, + "message": 4, + "code": 5 + } + ] + } + ] +} From f60a00f2cc262d304568c5daa5e49640ba20825a Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Mon, 26 Aug 2024 13:30:03 -0400 Subject: [PATCH 2/7] github: Quote variables Avoid accidental globbing or argument splitting by quoting the `$GITHUB_ENV` variable. In practice, this would probably never be a problem since github sets the value, but it follows best shell practices and makes the shell linter happy. Signed-off-by: Russell Bryant --- .github/workflows/publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index aeeaf6efab04..352a41a271d7 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -26,7 +26,7 @@ jobs: - name: Extract branch info shell: bash run: | - echo "release_tag=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + echo "release_tag=${GITHUB_REF#refs/*/}" >> "$GITHUB_ENV" - name: Create Release id: create_release @@ -88,8 +88,8 @@ jobs: bash -x .github/workflows/scripts/build.sh ${{ matrix.python-version }} ${{ matrix.cuda-version }} wheel_name=$(ls dist/*whl | xargs -n 1 basename) asset_name=${wheel_name//"linux"/"manylinux1"} - echo "wheel_name=${wheel_name}" >> $GITHUB_ENV - echo "asset_name=${asset_name}" >> $GITHUB_ENV + echo "wheel_name=${wheel_name}" >> "$GITHUB_ENV" + echo "asset_name=${asset_name}" >> "$GITHUB_ENV" - name: Upload Release Asset uses: actions/upload-release-asset@v1 From 0faedcb835b72065bfad93c0a8d7d5525c5e86b9 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Mon, 26 Aug 2024 13:34:03 -0400 Subject: [PATCH 3/7] github: Handle filenames with special characters Resolve the following linter warning: ``` .github/workflows/publish.yml:87:9: shellcheck reported issue in this script: SC2011:warning:2:14: Use 'find .. -print0 | xargs -0 ..' or 'find .. -exec .. +' to allow non-alphanumeric filenames [shellcheck] ``` I don't expect this was a problem in practice, but it follows best practice as indicated by the linter. Signed-off-by: Russell Bryant --- .github/workflows/publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 352a41a271d7..4cbe32bdf33b 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -86,7 +86,7 @@ jobs: CMAKE_BUILD_TYPE: Release # do not compile with debug symbol to reduce wheel size run: | bash -x .github/workflows/scripts/build.sh ${{ matrix.python-version }} ${{ matrix.cuda-version }} - wheel_name=$(ls dist/*whl | xargs -n 1 basename) + wheel_name=$(find dist -name "*whl" -print0 | xargs -0 -n 1 basename) asset_name=${wheel_name//"linux"/"manylinux1"} echo "wheel_name=${wheel_name}" >> "$GITHUB_ENV" echo "asset_name=${asset_name}" >> "$GITHUB_ENV" From b12274de97ca7b093f84b10106c6f9d9841a8b87 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Mon, 26 Aug 2024 14:27:30 -0400 Subject: [PATCH 4/7] github: Update versions of various github actions These were raised by the action linter. Signed-off-by: Russell Bryant --- .github/workflows/add_label_automerge.yml | 2 +- .github/workflows/clang-format.yml | 4 ++-- .github/workflows/mypy.yaml | 4 ++-- .github/workflows/ruff.yml | 4 ++-- .github/workflows/yapf.yml | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/add_label_automerge.yml b/.github/workflows/add_label_automerge.yml index cd53b764c720..761cae8e33fb 100644 --- a/.github/workflows/add_label_automerge.yml +++ b/.github/workflows/add_label_automerge.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Add label - uses: actions/github-script@v5 + uses: actions/github-script@v6 with: script: | github.rest.issues.addLabels({ diff --git a/.github/workflows/clang-format.yml b/.github/workflows/clang-format.yml index d5f37396e69d..4eec72b96622 100644 --- a/.github/workflows/clang-format.yml +++ b/.github/workflows/clang-format.yml @@ -17,9 +17,9 @@ jobs: matrix: python-version: ["3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/mypy.yaml b/.github/workflows/mypy.yaml index ea767f4c3e26..24f58f88361c 100644 --- a/.github/workflows/mypy.yaml +++ b/.github/workflows/mypy.yaml @@ -17,9 +17,9 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/ruff.yml b/.github/workflows/ruff.yml index 90735d6e2bbf..73ce56e9e6a2 100644 --- a/.github/workflows/ruff.yml +++ b/.github/workflows/ruff.yml @@ -17,9 +17,9 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/yapf.yml b/.github/workflows/yapf.yml index c89f82dfaaaf..5f24b5b90b51 100644 --- a/.github/workflows/yapf.yml +++ b/.github/workflows/yapf.yml @@ -16,9 +16,9 @@ jobs: matrix: python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From ee987903a1bde087a034c367f0b797460fdf0523 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Tue, 27 Aug 2024 10:08:31 -0400 Subject: [PATCH 5/7] format.sh: Avoid exit when no files changed This script is set to exit any time a command fails. In the case of this line, the script unexpectedly errors out on this check to see if any files have changed that are relevant for the `clang-format` tool. The `grep` specifically will return non-zero in this case. The workaround here is to catch the error and run `echo -e` instead, which we know will succeed and the end result is what we want (the list of changed files will be empty). This allows the script to continue to completion, even when `clang-format` has nothing to do. Signed-off-by: Russell Bryant --- format.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/format.sh b/format.sh index 6563d89b192e..46f7a4653e66 100755 --- a/format.sh +++ b/format.sh @@ -263,7 +263,7 @@ clang_format_changed() { MERGEBASE="$(git merge-base origin/main HEAD)" # Get the list of changed files, excluding the specified ones - changed_files=$(git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.h' '*.cpp' '*.cu' '*.cuh' | grep -vFf <(printf "%s\n" "${CLANG_FORMAT_EXCLUDES[@]}")) + changed_files=$(git diff --name-only --diff-filter=ACM "$MERGEBASE" -- '*.h' '*.cpp' '*.cu' '*.cuh' | (grep -vFf <(printf "%s\n" "${CLANG_FORMAT_EXCLUDES[@]}") || echo -e)) if [ -n "$changed_files" ]; then echo "$changed_files" | xargs -P 5 clang-format -i fi From cd2f73b62587de9596eb56c10b3a8d85f54a8558 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Tue, 27 Aug 2024 11:08:50 -0400 Subject: [PATCH 6/7] format.sh: lint github ci configuration Update `format.sh` to run the `actionlint` tool to check for errors in GitHub CI configuration. Both this script and the workflow to run this tool in CI run it using docker to ensure that the same version is in use in both places. Signed-off-by: Russell Bryant --- format.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/format.sh b/format.sh index 46f7a4653e66..7769548c2f65 100755 --- a/format.sh +++ b/format.sh @@ -286,6 +286,29 @@ else fi echo 'vLLM clang-format: Done' +echo 'vLLM actionlint:' + +DOCKER="" +if command -v "docker" &>/dev/null; then + DOCKER="docker" +elif command -v "podman" &>/dev/null; then + DOCKER="podman" +else + echo "Docker or Podman are not installed. Please install one if you want to lint GitHub CI configuration." +fi + +actionlint() { + if [ -z "$DOCKER" ]; then + return + fi + # Ensure we use the same version of actionlint as CI + IMAGE="vllm/actionlint:latest" + ${DOCKER} build --tag "${IMAGE}" - < .github/workflows/actionlint.dockerfile + ${DOCKER} run --volume="${PWD}:/repo:z" --workdir=/repo "${IMAGE}" -color +} + +actionlint +echo 'vLLM actionlint: Done' if ! git diff --quiet &>/dev/null; then echo 'Reformatted files. Please review and stage the changes.' From 103f1dd78c9ea4f946401883879672090c680ff5 Mon Sep 17 00:00:00 2001 From: Russell Bryant Date: Mon, 7 Oct 2024 19:40:02 +0000 Subject: [PATCH 7/7] actionlint: Switch to prebuilt binary instead of docker Previously `format.sh` and the github workflow used docker or podman to run actionlint. This now will use an installed version if found, and will otherwise download and run a prebuilt binary from a github release. Signed-off-by: Russell Bryant --- .github/workflows/actionlint.dockerfile | 3 --- .github/workflows/actionlint.yml | 9 ++------- .gitignore | 3 +++ format.sh | 22 +--------------------- tools/actionlint.sh | 13 +++++++++++++ 5 files changed, 19 insertions(+), 31 deletions(-) delete mode 100644 .github/workflows/actionlint.dockerfile create mode 100755 tools/actionlint.sh diff --git a/.github/workflows/actionlint.dockerfile b/.github/workflows/actionlint.dockerfile deleted file mode 100644 index 01ac68066d44..000000000000 --- a/.github/workflows/actionlint.dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -# Since dependabot cannot update workflows using docker, -# we use this indirection since dependabot can update this file. -FROM docker.io/rhysd/actionlint:1.7.1@sha256:435ecdb63b1169e80ca3e136290072548c07fc4d76a044cf5541021712f8f344 diff --git a/.github/workflows/actionlint.yml b/.github/workflows/actionlint.yml index be8964dbe699..38e23651eefe 100644 --- a/.github/workflows/actionlint.yml +++ b/.github/workflows/actionlint.yml @@ -32,11 +32,6 @@ jobs: with: fetch-depth: 0 - - name: "Download actionlint" + - name: "Run actionlint" run: | - docker build --tag actionlint - < .github/workflows/actionlint.dockerfile - - - name: "Check workflow files" - run: | - echo "::add-matcher::.github/workflows/matchers/actionlint.json" - docker run --volume="${PWD}:/repo" --workdir=/repo actionlint -color + tools/actionlint.sh -color diff --git a/.gitignore b/.gitignore index 5367ece83489..1ea6e3419db2 100644 --- a/.gitignore +++ b/.gitignore @@ -199,3 +199,6 @@ hip_compat.h # Benchmark dataset benchmarks/*.json + +# Linting +actionlint diff --git a/format.sh b/format.sh index 7769548c2f65..a0df92b35013 100755 --- a/format.sh +++ b/format.sh @@ -287,27 +287,7 @@ fi echo 'vLLM clang-format: Done' echo 'vLLM actionlint:' - -DOCKER="" -if command -v "docker" &>/dev/null; then - DOCKER="docker" -elif command -v "podman" &>/dev/null; then - DOCKER="podman" -else - echo "Docker or Podman are not installed. Please install one if you want to lint GitHub CI configuration." -fi - -actionlint() { - if [ -z "$DOCKER" ]; then - return - fi - # Ensure we use the same version of actionlint as CI - IMAGE="vllm/actionlint:latest" - ${DOCKER} build --tag "${IMAGE}" - < .github/workflows/actionlint.dockerfile - ${DOCKER} run --volume="${PWD}:/repo:z" --workdir=/repo "${IMAGE}" -color -} - -actionlint +tools/actionlint.sh -color echo 'vLLM actionlint: Done' if ! git diff --quiet &>/dev/null; then diff --git a/tools/actionlint.sh b/tools/actionlint.sh new file mode 100755 index 000000000000..f6a8b5e83a2d --- /dev/null +++ b/tools/actionlint.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +if command -v actionlint &> /dev/null; then + actionlint "$@" + exit 0 +elif [ -x ./actionlint ]; then + ./actionlint "$@" + exit 0 +fi + +# download a binary to the current directory - v1.7.3 +bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/aa0a7be8e566b096e64a5df8ff290ec24fa58fbc/scripts/download-actionlint.bash) +./actionlint "$@"