Skip to content

Commit

Permalink
CI: Add action for Python regression tests (#3820)
Browse files Browse the repository at this point in the history
* CI: Add action for Python regression tests

* WIP: Fix Ubuntu ARM builds

* Fix Windows run

* WIP: Check self-hosted Ubuntu ARM runners

* WIP

* WIP: Check own self-hosted Ubuntu ARM runners

* Revert WIP

* Fix boolean usage

* Reduce download logging

* Fix

* Debug inputs

* Reduce upload logging

* Remove obsolete parameter

* Fix debug inputs

* Fix Fedora run

* Format
  • Loading branch information
oitel authored Dec 25, 2024
1 parent 8a0546d commit 9a85c96
Show file tree
Hide file tree
Showing 8 changed files with 137 additions and 163 deletions.
97 changes: 97 additions & 0 deletions .github/actions/python-regression-tests/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
name: Python regression tests

inputs:
autotest_data_s3_url:
required: false
type: string
default: "s3://data-autotest/test_data_2024-12-19"
build_config:
description: "Build config (Release, Debug)"
required: true
type: string
mrbind:
required: true
type: boolean
pytest_args:
description: "Additional pytest arguments"
required: false
type: string
default: ""
smoke:
required: true
type: boolean
test_artifacts_path:
description: "S3 path for test artifacts. Make sure it is unique for different jobs"
required: true
type: string
ubuntu20_arm_workaround:
description: "Disable specific tests incompatible with Ubuntu 20.04 ARM configuration"
required: false
type: boolean
default: false
upload_test_artifacts:
required: true
type: boolean

runs:
using: composite
steps:
- name: Debug inputs
shell: bash
run: |
echo << EOF
autotest_data_s3_url: ${{ inputs.autotest_data_s3_url }}
build_config: ${{ inputs.build_config }}
mrbind: ${{ inputs.mrbind }}
pytest_args: ${{ inputs.pytest_args }}
smoke: ${{ inputs.smoke }}
test_artifacts_path: ${{ inputs.test_artifacts_path }}
ubuntu20_arm_workaround: ${{ inputs.ubuntu20_arm_workaround }}
upload_test_artifacts: ${{ inputs.upload_test_artifacts }}
EOF
# cache management: https://github.com/MeshInspector/MeshLib/actions/caches
- name: Cache autotest data from S3 Tests
uses: actions/cache@v4
with:
path: test_data
key: ${{ inputs.autotest_data_s3_url }}

- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::259351611210:role/GitHubMeshLibAwsCredsRole
aws-region: us-east-1

- name: Copy autotest data from S3 Tests
shell: bash
run: aws s3 sync ${{ inputs.autotest_data_s3_url }} test_data --delete --no-sign-request --size-only --quiet

- name: Python Regression Tests
if: ${{ runner.os == 'Windows' }}
shell: powershell
working-directory: source\x64\${{ inputs.build_config }}
run: >
py -3 ..\..\..\scripts\run_python_test_script.py
-d '..\test_regression'
-s ${{ inputs.smoke }}
-bv ${{ fromJSON( inputs.mrbind ) && '3' || '2' }}
-a='${{ inputs.pytest_args }}'
- name: Python Regression Tests
if: ${{ runner.os != 'Windows' }}
shell: bash
working-directory: build/${{ inputs.build_config }}/bin
env:
MR_REGRESSION_TESTS_UBUNTUARM2004_MRBIND_ABI_ISSUES: ${{ fromJSON( inputs.ubuntu20_arm_workaround ) && '1' || '0' }}
run: >
python3 ../../../scripts/run_python_test_script.py
-d '../test_regression'
-s ${{ inputs.smoke }}
-bv ${{ fromJSON( inputs.mrbind ) && '3' || '2' }}
-a='${{ inputs.pytest_args }}'
- name: Copy test artifacts to S3
if: ${{ fromJSON( inputs.upload_test_artifacts ) && !cancelled() }}
shell: bash
run: aws s3 cp pytest_temp 's3://test-artifacts-git/tests/${{ github.run_id }}/${{ inputs.test_artifacts_path }}' --recursive --quiet
5 changes: 0 additions & 5 deletions .github/workflows/build-test-distribute.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ jobs:
needs: [ config, prepare-image ]
uses: ./.github/workflows/build-test-windows.yml
with:
autotest_data_s3_url: ${{ needs.config.outputs.autotest_data_s3_url }}
full_config_build: ${{ needs.config.outputs.full_config_build == 'true' }}
internal_build: ${{ needs.config.outputs.internal_build == 'true' }}
upload_artifacts: ${{ needs.config.outputs.upload_artifacts == 'true' }}
Expand All @@ -67,7 +66,6 @@ jobs:
uses: ./.github/workflows/build-test-ubuntu-arm64.yml
with:
app_version: ${{ needs.config.outputs.app_version }}
autotest_data_s3_url: ${{ needs.config.outputs.autotest_data_s3_url }}
docker_image_tag: ${{ needs.config.outputs.docker_image_tag }}
full_config_build: ${{ needs.config.outputs.full_config_build == 'true' }}
internal_build: ${{ needs.config.outputs.internal_build == 'true' }}
Expand All @@ -80,7 +78,6 @@ jobs:
uses: ./.github/workflows/build-test-ubuntu-x64.yml
with:
app_version: ${{ needs.config.outputs.app_version }}
autotest_data_s3_url: ${{ needs.config.outputs.autotest_data_s3_url }}
config_matrix: ${{ needs.config.outputs.ubuntu_x64_config_matrix }}
docker_image_tag: ${{ needs.config.outputs.docker_image_tag }}
full_config_build: ${{ needs.config.outputs.full_config_build == 'true' }}
Expand All @@ -94,7 +91,6 @@ jobs:
uses: ./.github/workflows/build-test-fedora.yml
with:
app_version: ${{ needs.config.outputs.app_version }}
autotest_data_s3_url: ${{ needs.config.outputs.autotest_data_s3_url }}
docker_image_tag: ${{ needs.config.outputs.docker_image_tag }}
full_config_build: ${{ needs.config.outputs.full_config_build == 'true' }}
internal_build: ${{ needs.config.outputs.internal_build == 'true' }}
Expand All @@ -113,7 +109,6 @@ jobs:
uses: ./.github/workflows/build-test-macos.yml
with:
app_version: ${{ needs.config.outputs.app_version }}
autotest_data_s3_url: ${{ needs.config.outputs.autotest_data_s3_url }}
full_config_build: ${{ needs.config.outputs.full_config_build == 'true' }}
internal_build: ${{ needs.config.outputs.internal_build == 'true' }}
upload_artifacts: ${{ needs.config.outputs.upload_artifacts == 'true' }}
Expand Down
39 changes: 7 additions & 32 deletions .github/workflows/build-test-fedora.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ on:
app_version:
required: true
type: string
autotest_data_s3_url:
required: true
type: string
docker_image_tag:
required: true
type: string
Expand Down Expand Up @@ -128,37 +125,15 @@ jobs:
working-directory: ./build/${{ matrix.config }}/bin
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'

- name: Configure AWS Credentials
if: ${{ inputs.upload_test_artifacts && inputs.internal_build }}
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::259351611210:role/GitHubMeshLibAwsCredsRole
aws-region: us-east-1

# cache managment: https://github.com/MeshInspector/MeshLib/actions/caches
- name: Cache autotest data from S3 Tests
if: ${{ inputs.internal_build }}
id: cache-autotest
uses: actions/cache@v4
with:
path: ./test_data
key: ${{ inputs.autotest_data_s3_url }}

- name: Copy autotest data from S3 Tests
if: ${{ inputs.internal_build }}
run: aws s3 sync ${{ inputs.autotest_data_s3_url }} ./test_data --delete --no-sign-request --size-only

- name: Python Regression Tests
env:
SMOKE: ${{ inputs.internal_build && !inputs.full_config_build && matrix.config == 'Debug' }}
BINDS_V: ${{ fromJSON('[2,3]')[inputs.mrbind || 0] }}
if: ${{ inputs.internal_build }}
working-directory: ./build/${{ matrix.config }}/bin
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_regression' -s ${{env.SMOKE}} -bv ${{env.BINDS_V}}

- name: Copy test artifacts to S3
if: ${{ inputs.upload_test_artifacts && inputs.internal_build && !cancelled() }}
run: aws s3 cp ./pytest_temp 's3://test-artifacts-git/tests/${{github.run_id}}/fedora/${{matrix.os}}/${{matrix.config}}' --recursive
uses: ./.github/actions/python-regression-tests
with:
build_config: ${{ matrix.config }}
mrbind: false
smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }}
test_artifacts_path: fedora/${{ matrix.os }}/${{ matrix.config }}
upload_test_artifacts: ${{ inputs.upload_test_artifacts }}

- name: Create RPM
if: ${{ matrix.config == 'Release' && matrix.compiler == 'GCC 13' }}
Expand Down
39 changes: 7 additions & 32 deletions .github/workflows/build-test-macos.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ on:
app_version:
required: true
type: string
autotest_data_s3_url:
required: true
type: string
full_config_build:
default: false
required: false
Expand Down Expand Up @@ -135,37 +132,15 @@ jobs:
working-directory: ./build/${{ matrix.config }}/bin
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'

- name: Configure AWS Credentials
if: ${{ inputs.upload_test_artifacts && inputs.internal_build }}
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::259351611210:role/GitHubMeshLibAwsCredsRole
aws-region: us-east-1

# cache managment: https://github.com/MeshInspector/MeshLib/actions/caches
- name: Cache autotest data from S3 Tests
if: ${{ inputs.internal_build }}
id: cache-autotest
uses: actions/cache@v4
with:
path: ./test_data
key: ${{ inputs.autotest_data_s3_url }}

- name: Copy autotest data from S3 Tests
if: ${{ inputs.internal_build }}
run: aws s3 sync ${{ inputs.autotest_data_s3_url }} ./test_data --delete --no-sign-request --size-only

- name: Python Regression Tests
env:
SMOKE: ${{ inputs.internal_build && !inputs.full_config_build && matrix.config == 'Debug' }}
BINDS_V: ${{ fromJSON('[2,3]')[inputs.mrbind || 0] }}
if: ${{ inputs.internal_build }}
working-directory: ./build/${{ matrix.config }}/bin
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_regression' -s ${{env.SMOKE}} -bv ${{env.BINDS_V}}

- name: Copy test artifacts to S3
if: ${{ inputs.upload_test_artifacts && inputs.internal_build && !cancelled() }}
run: aws s3 cp ./pytest_temp 's3://test-artifacts-git/tests/${{github.run_id}}/macos/${{matrix.os}}' --recursive
uses: ./.github/actions/python-regression-tests
with:
build_config: ${{ matrix.config }}
mrbind: ${{ inputs.mrbind }}
smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }}
test_artifacts_path: macos/${{ matrix.os }}
upload_test_artifacts: ${{ inputs.upload_test_artifacts }}

- name: Create Pkg
if: ${{ inputs.upload_artifacts && matrix.config == 'Release' }}
Expand Down
42 changes: 9 additions & 33 deletions .github/workflows/build-test-ubuntu-arm64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ on:
app_version:
required: true
type: string
autotest_data_s3_url:
required: true
type: string
docker_image_tag:
required: true
type: string
Expand Down Expand Up @@ -166,38 +163,17 @@ jobs:
working-directory: ./build/${{ matrix.config }}/bin
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'

- name: Configure AWS Credentials
if: ${{ inputs.upload_test_artifacts && inputs.internal_build }}
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::259351611210:role/GitHubMeshLibAwsCredsRole
aws-region: us-east-1

# cache managment: https://github.com/MeshInspector/MeshLib/actions/caches
- name: Cache autotest data from S3 Tests
if: ${{ inputs.internal_build }}
id: cache-autotest
uses: actions/cache@v4
with:
path: ./test_data
key: ${{ inputs.autotest_data_s3_url }}

- name: Copy autotest data from S3 Tests
if: ${{ inputs.internal_build }}
run: aws s3 sync ${{ inputs.autotest_data_s3_url }} ./test_data --delete --no-sign-request --size-only

- name: Python Regression Tests
env:
SMOKE: ${{ inputs.internal_build && !inputs.full_config_build && matrix.config == 'Debug' }}
MR_REGRESSION_TESTS_UBUNTUARM2004_MRBIND_ABI_ISSUES: ${{ fromJSON('[0,1]')[matrix.os == 'ubuntu20' && inputs.mrbind] }}
BINDS_V: ${{ fromJSON('[2,3]')[inputs.mrbind || 0] }}
if: ${{ inputs.internal_build }}
working-directory: ./build/${{ matrix.config }}/bin
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_regression' -s ${{env.SMOKE}} -bv ${{env.BINDS_V}} -a="--run-cuda=negative"

- name: Copy test artifacts to S3
if: ${{ inputs.upload_test_artifacts && inputs.internal_build && !cancelled() }}
run: aws s3 cp ./pytest_temp 's3://test-artifacts-git/tests/${{github.run_id}}/ubuntu_arm64//${{matrix.os}}' --recursive
uses: ./.github/actions/python-regression-tests
with:
build_config: ${{ matrix.config }}
mrbind: ${{ inputs.mrbind }}
pytest_args: "--run-cuda=negative"
smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }}
test_artifacts_path: ubuntu_arm64/${{ matrix.os }}
ubuntu20_arm_workaround: ${{ inputs.mrbind && matrix.os == 'ubuntu20' }}
upload_test_artifacts: ${{ inputs.upload_test_artifacts }}

- name: Create Deb
env:
Expand Down
40 changes: 8 additions & 32 deletions .github/workflows/build-test-ubuntu-x64.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,6 @@ on:
app_version:
required: true
type: string
autotest_data_s3_url:
required: true
type: string
config_matrix:
required: true
type: string
Expand Down Expand Up @@ -132,37 +129,16 @@ jobs:
working-directory: ./build/${{ matrix.config }}/bin
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_python'

- name: Configure AWS Credentials
if: ${{ inputs.upload_test_artifacts && inputs.internal_build }}
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: arn:aws:iam::259351611210:role/GitHubMeshLibAwsCredsRole
aws-region: us-east-1

# cache managment: https://github.com/MeshInspector/MeshLib/actions/caches
- name: Cache autotest data from S3 Tests
if: ${{ inputs.internal_build }}
id: cache-autotest
uses: actions/cache@v4
with:
path: ./test_data
key: ${{ inputs.autotest_data_s3_url }}

- name: Copy autotest data from S3 Tests
if: ${{ inputs.internal_build }}
run: aws s3 sync ${{ inputs.autotest_data_s3_url }} ./test_data --delete --no-sign-request --size-only

- name: Python Regression Tests
env:
SMOKE: ${{ inputs.internal_build && !inputs.full_config_build && matrix.config == 'Debug' }}
BINDS_V: ${{ fromJSON('[2,3]')[inputs.mrbind || 0] }}
if: ${{ inputs.internal_build }}
working-directory: ./build/${{ matrix.config }}/bin
run: python3 ./../../../scripts/run_python_test_script.py -d '../test_regression' -s ${{env.SMOKE}} -bv ${{env.BINDS_V}} -a="--run-cuda=negative"

- name: Copy test artifacts to S3
if: ${{ inputs.upload_test_artifacts && inputs.internal_build && !cancelled() }}
run: aws s3 cp ./pytest_temp 's3://test-artifacts-git/tests/${{github.run_id}}/${{matrix.os}}/${{matrix.config}}/${{matrix.compiler}}' --recursive
uses: ./.github/actions/python-regression-tests
with:
build_config: ${{ matrix.config }}
mrbind: ${{ inputs.mrbind }}
pytest_args: "--run-cuda=negative"
smoke: ${{ !inputs.full_config_build && matrix.config == 'Debug' }}
test_artifacts_path: ${{ matrix.os }}/${{matrix.config}}/${{matrix.compiler}}
upload_test_artifacts: ${{ inputs.upload_test_artifacts }}

- name: Create Deb
if: ${{ matrix.compiler == 'GCC' && matrix.config == 'Release' }}
Expand Down
Loading

0 comments on commit 9a85c96

Please sign in to comment.