diff --git a/.github/workflows/_build_test_upload.yml b/.github/workflows/_build_test_upload.yml index 5a2ed591e..874b911cf 100644 --- a/.github/workflows/_build_test_upload.yml +++ b/.github/workflows/_build_test_upload.yml @@ -12,13 +12,6 @@ on: pytorch_version: required: true type: string - do-upload: - required: false - default: true - type: boolean - secrets: - PYPI_TOKEN: - required: false permissions: id-token: write @@ -47,173 +40,9 @@ jobs: echo "type=$RELEASE_TYPE" >> $GITHUB_OUTPUT id: get_release_type - wheel_build_test: - needs: get_release_type - runs-on: ${{ matrix.os }} - container: ${{ startsWith( matrix.os, 'ubuntu' ) && 'pytorch/manylinux-cpu' || null }} - strategy: - fail-fast: false - matrix: - os: - - windows-latest - python-version: - - 3.9 - - "3.10" - - "3.11" - - "3.12" - - "3.13" - - pure - exclude: - - os: windows-latest - python-version: pure - steps: - - name: Checkout Source Repository - uses: actions/checkout@v4 - with: - ref: ${{ inputs.branch }} - submodules: recursive - - name: Setup Python ${{ matrix.python-version }} for Windows - if: ${{ startsWith( matrix.os, 'windows' ) }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Add temp runner environment variables - if: ${{ startsWith( matrix.os, 'macos' ) }} - shell: bash -l {0} - run: | - echo "MINICONDA_INSTALL_PATH_MACOS=${RUNNER_TEMP}/miniconda" >> "${GITHUB_ENV}" - - name: Setup msbuild on Windows - if: startsWith( matrix.os, 'windows' ) - uses: microsoft/setup-msbuild@v1.1 - - name: Set up Visual Studio shell - if: startsWith( matrix.os, 'windows' ) - uses: egor-tensin/vs-shell@v2 - with: - arch: x64 - - name: Install Build Dependency - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - pip install cmake ninja setuptools - echo "/home/runner/.local/bin" >> $GITHUB_PATH - - name: Install PyTorch and Build TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - PYTORCH_VERSION: ${{ inputs.pytorch_version }} - BUILD_S3: 0 - run: | - set -ex - packaging/build_wheel.sh - - name: Validate TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - pip3 install pkginfo - for pkg in dist/torchdata*.whl; do - echo "PkgInfo of $pkg:" - pkginfo $pkg - if ${{ startsWith( matrix.os, 'ubuntu' ) && matrix.python-version != 'pure' }}; then - auditwheel show $pkg - auditwheel repair $pkg --plat manylinux2014_x86_64 -w wheelhouse - fi - done - - name: Install TorchData Wheel - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - pip3 install dist/torchdata*.whl - - name: Run Smoke Tests - shell: bash -l {0} - env: - PYTHON_VERSION: ${{ matrix.python-version }} - run: | - if ${{ matrix.python-version == 'pure' }}; then - python test/smoke_test/smoke_test.py --no-s3 - else - python test/smoke_test/smoke_test.py - fi - - - name: Upload Wheels to Github - if: always() - uses: actions/upload-artifact@v4 - with: - name: torchdata-artifact-${{ matrix.os }}-${{ matrix.python-version }} - path: dist/torchdata*.whl - - wheel_upload: - if: always() && inputs.branch != '' && inputs.do-upload == true - needs: [get_release_type, wheel_build_test] - runs-on: ubuntu-latest - environment: pytorchbot-env - outputs: - upload: ${{ steps.trigger_upload.outputs.value }} - steps: - - name: Configure aws credentials (pytorch account) - if: ${{ needs.get_release_type.outputs.type == 'nightly' }} - uses: aws-actions/configure-aws-credentials@v3 - with: - role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_nightly_build_wheels - aws-region: us-east-1 - - - name: Configure aws credentials (pytorch account) - if: ${{ needs.get_release_type.outputs.type == 'test' }} - uses: aws-actions/configure-aws-credentials@v3 - with: - role-to-assume: arn:aws:iam::749337293305:role/gha_workflow_test_build_wheels - aws-region: us-east-1 - - name: Download Artifacts from Github - continue-on-error: true - uses: actions/download-artifact@v4 - with: - pattern: torchdata-artifact-* - merge-multiple: true - - name: Determine if Wheel Uploading is needed - run: | - upload=false - for txt in torchdata*.whl; do - upload=true - break - done - echo "value=$upload" >> $GITHUB_OUTPUT - id: trigger_upload - - name: Display All TorchData Wheels - if: steps.trigger_upload.outputs.value == 'true' - run: ls -lh torchdata*.whl - - name: Upload Wheels to S3 Storage - if: steps.trigger_upload.outputs.value == 'true' - run: | - if [[ ${{ inputs.branch }} == 'main' ]]; then - S3_PATH=s3://pytorch/whl/nightly/ - else - S3_PATH=s3://pytorch/whl/test/ - fi - pip3 install awscli - set -x - for pkg in torchdata*.whl; do - aws s3 cp "$pkg" "$S3_PATH" --acl public-read - done - - name: Upload Official Wheels to PYPI - if: | - steps.trigger_upload.outputs.value == 'true' && - needs.get_release_type.outputs.type == 'official' - env: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: | - pip3 install twine - python -m twine upload \ - --username __token__ \ - --password "$PYPI_TOKEN" \ - torchdata*.whl - build_docs: - if: | - always() && inputs.branch != '' && - ( needs.wheel_upload.outputs.upload == 'true') - needs: [get_release_type, wheel_upload] + if: always() && inputs.branch != '' + needs: get_release_type runs-on: ubuntu-latest steps: - name: Setup Python 3.9 diff --git a/.github/workflows/build-wheels-m1.yml b/.github/workflows/build-wheels-m1.yml deleted file mode 100644 index 05be60b51..000000000 --- a/.github/workflows/build-wheels-m1.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Build M1 Wheels - -on: - pull_request: - push: - branches: - - nightly - - main - - release/* - tags: - # NOTE: Binary build pipelines should only get triggered on release candidate builds - # Release candidate tags look like: v1.11.0-rc1 - - v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+ - workflow_dispatch: - -permissions: - id-token: write - contents: read - -jobs: - generate-matrix: - uses: pytorch/test-infra/.github/workflows/generate_binary_build_matrix.yml@main - with: - package-type: wheel - os: macos-arm64 - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build: - needs: generate-matrix - name: pytorch/data - uses: pytorch/test-infra/.github/workflows/build_wheels_macos.yml@main - with: - repository: pytorch/data - ref: "" - test-infra-repository: pytorch/test-infra - test-infra-ref: main - build-matrix: ${{ needs.generate-matrix.outputs.matrix }} - pre-script: "" - post-script: "" - package-name: torchdata - env-var-script: packaging/env-var-script.txt - runner-type: macos-m1-stable - smoke-test-script: test/smoke_test/smoke_test.py - trigger-event: ${{ github.event_name }} diff --git a/.github/workflows/build_wheels_linux.yml b/.github/workflows/build_wheels_linux.yml index 1f15f8d40..bbaccb9ff 100644 --- a/.github/workflows/build_wheels_linux.yml +++ b/.github/workflows/build_wheels_linux.yml @@ -27,6 +27,7 @@ jobs: test-infra-ref: main with-cuda: disable with-rocm: disable + python-versions: '["3.9"]' build: needs: generate-matrix strategy: