add workflow to build and test

.github/workflows/build-test.yml

@@ -0,0 +1,68 @@
+name: build-test
+on:
+
+  # makes workflow reusable
+  workflow_call:
+    inputs:
+      wf_category:
+        description: "workflow category: NIGHTLY, RELEASE"
+        type: string
+        default: NIGHTLY
+      push_to_pypi:
+        description: "When set to true, built whl and tar.gz will be pushed to public pypi if all tests pass"
+        type: boolean
+        default: false
+      gitref:
+        description: "git commit hash or tag name"
+        type: string
+        default: main
+
+      # build related parameters
+      build_label:
+        description: "requested runner label for build (specifies instance)"
+        type: string
+        default: ubuntu-20.04
+
+      # test related parameters
+      test_configs:
+        description: "python, label, timeout"
+        type: string
+        required: true
+
+jobs:
+
+    BUILD:
+        uses: ./.github/workflows/build.yml
+        with:
+            wf_category: ${{ inputs.wf_category }}
+            build_label: ${{ inputs.build_label }}
+            gitref: ${{ inputs.gitref }}
+            timeout: 20
+        secrets: inherit
+
+#    TEST-NIGHTLY:
+#        needs: [BUILD]
+#        strategy:
+#            fail-fast: false
+#            matrix:
+#                test_config: ${{ fromJson(inputs.test_configs) }}
+#        uses: ./.github/workflows/test-nightly.yml
+#        with:
+#            gitref: ${{ inputs.gitref }}
+#            test_label: ${{ matrix.test_config.label }}
+#            python: ${{ matrix.test_config.python }}
+#            timeout: ${{ matrix.test_config.timeout }}
+#            whl: ${{ needs.BUILD.outputs.whl }}
+#            testmo_run_id: ${{ needs.BUILD.outputs.testmo_run_id }}
+#        secrets: inherit
+
+#    UPLOAD:
+#        needs: TEST-NIGHTLY
+#        uses: ./.github/workflows/upload.yml
+#        with:
+#            label: k8s-util
+#            timeout: 40
+#            run_id: ${{ github.run_id }}
+#            push_to_pypi: ${{ inputs.push_to_pypi }}
+#            testmo_run_id: ${{ needs.BUILD.outputs.testmo_run_id }}
+#        secrets: inherit

.github/workflows/build.yml

@@ -0,0 +1,164 @@
+name: build
+on:
+  # makes workflow reusable
+  workflow_call:
+    inputs:
+      wf_category:
+        description: "categories: NIGHTLY, RELEASE"
+        type: string
+        default: NIGHTLY
+      build_label:
+        description: "requested runner label (specifies instance)"
+        type: string
+        required: true
+      timeout:
+        description: "time limit for run in minutes "
+        type: string
+        default: 20
+      gitref:
+        description: "git commit hash or branch name"
+        type: string
+        default: main
+    outputs:
+      whl:
+        description: 'basename for generated whl'
+        value: ${{ jobs.BUILD.outputs.whl }}
+      testmo_run_id:
+        description: 'testmo run id'
+        value: ${{ jobs.BUILD.outputs.testmo_run_id }}
+
+  # makes workflow manually callable
+  workflow_dispatch:
+    inputs:
+      wf_category:
+        description: "categories: NIGHTLY, RELEASE"
+        type: string
+        default: NIGHTLY
+      build_label:
+        description: "requested runner label (specifies instance)"
+        type: string
+        required: true
+      timeout:
+        description: "time limit for run in minutes "
+        type: string
+        default: 20
+      gitref:
+        description: "git commit hash or branch name"
+        type: string
+        default: main
+
+jobs:
+
+    BUILD:
+
+        runs-on: ${{ inputs.build_label }}
+        timeout-minutes: ${{ fromJson(inputs.timeout) }}
+        env:
+            HF_TOKEN: ${{ secrets.NM_HF_TOKEN }}
+        permissions:
+            contents: 'read'
+            id-token: 'write'
+
+        outputs:
+            assets_location: gs://llm-compressor/assets/${{ github.run_id }}
+            run_id: ${{ github.run_id }}
+            whl: ${{ steps.build.outputs.whl }}
+            tarfile: ${{ steps.build.outputs.tarfile }}
+            testmo_run_id: ${{ steps.create_testmo_run.outputs.id }}
+
+        steps:
+
+            - name: set python
+              id: set_python
+              uses: actions/setup-python@v4
+              with:
+              python-version: '3.10'
+
+            - name: checkout code
+              id: checkout
+              uses: actions/checkout@v4
+              with:
+                  ref: ${{ inputs.gitref }}
+
+            - name: install testmo
+              uses: neuralmagic/nm-actions/actions/install-testmo@v1.0.0
+
+            - name: create testmo run
+              id: create_testmo_run
+              uses: neuralmagic/nm-actions/actions/testmo-run-create@newadd
+              if: success()
+              with:
+                testmo_url: https://neuralmagic.testmo.net
+                testmo_token: ${{ secrets.TESTMO_TEST_TOKEN }}
+                source: 'build-test'
+                run_name: compressedtensors-${{ inputs.wf_category }}-${{ inputs.gitref }}-${GITHUB_ACTOR}
+
+            - name: build
+              id: build
+              uses: neuralmagic/nm-actions/actions/build-ml-whl@newadd
+              with:
+                  dev: false
+                  release: ${{ inputs.wf_category == 'RELEASE' }}
+
+            # GCP
+            - name: 'Authenticate to Google Cloud'
+              id: auth
+              uses: google-github-actions/auth@v2.1.3
+              with:
+                  project_id: ${{ secrets.GCP_PROJECT }}
+                  workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+                  service_account: ${{ secrets.GCP_GHA_SA }}
+
+            - name: 'Set up Cloud SDK'
+              uses: 'google-github-actions/setup-gcloud@v2'
+              with:
+                  version: '>= 473.0.0'
+
+            - name: copy whl and source distribution
+              run: |
+                gcloud storage cp dist/${{ steps.build.outputs.whlname }} gs://llm-compressor/assets/${{ github.run_id }}/${{ steps.build.outputs.whlname }}
+                gcloud storage cp dist/${{ steps.build.outputs.tarname }} gs://llm-compressor/assets/${{ github.run_id }}/${{ steps.build.outputs.tarname }}
+
+            - name: upload whl
+              uses: actions/upload-artifact@v4
+              if: success() || failure()
+              with:
+                name: ${{ steps.build.outputs.whlname }}
+                path: dist/${{ steps.build.outputs.whlname }}
+                retention-days: 5
+
+            - name: upload tar.gz
+              uses: actions/upload-artifact@v4
+              if: success() || failure()
+              with:
+                name: ${{ steps.build.outputs.tarname }}
+                path: dist/${{ steps.build.outputs.tarname }}
+                retention-days: 5
+
+            - name: summary
+              uses: neuralmagic/nm-actions/actions/summary-build@newadd
+              if: success() || failure()
+              with:
+                label: ${{ inputs.build_label }}
+                gitref: ${{ inputs.gitref }}
+                whl_status: ${{ steps.build.outputs.whl_status }}
+
+            - name: report build status to testmo
+              id: report_build
+              uses: neuralmagic/nm-actions/actions/testmo-run-submit-thread@newadd
+              if: (success() || failure()) && ${{ inputs.testmo_run_id != '' }}
+              with:
+                testmo_url: https://neuralmagic.testmo.net
+                testmo_token: ${{ secrets.TESTMO_TEST_TOKEN }}
+                testmo_run_id: ${{ steps.create_testmo_run.outputs.id }}
+                results: build-results
+                step_status: ${{ steps.build.outputs.whl_status }}
+
+            - name: run status
+              id: run_status
+              if: success() || failure()
+              env:
+                WHL_STATUS: ${{ steps.build.outputs.whl_status }}
+              run: |
+                echo "build status: ${WHL_STATUS}"
+                if [ -z "${WHL_STATUS}" ] || [ "${WHL_STATUS}" -ne "0" ]; then exit 1; fi

.github/workflows/trigger-all.yml

@@ -0,0 +1,44 @@
+name: build and test jobs for nightly and release
+run-name: ${{ github.actor }} triggered nightly on ${{ github.ref }}
+on:
+    schedule:
+      # * is a special character in YAML so you have to quote this string
+      - cron: '30 0 * * *'  # nightly run
+
+    push:
+      branches:
+        - auto
+
+    workflow_dispatch:
+        inputs:
+            wf_category:
+                description: "workflow category, default is NIGHTLY"
+                type: choice
+                options:
+                    - NIGHTLY
+                    - RELEASE
+                default: NIGHTLY
+            push_to_pypi:
+                description: "only set it to true for RELEASE; when set and tests pass, then '.whl' & '.tar.gz' will be pushed to public pypi"
+                type: boolean
+                default: false
+            gitref:
+                description: "git commit hash or tag name" 
+                type: string
+                default: 'main'
+
+jobs:
+
+    BUILD-TEST:
+        uses: ./.github/workflows/build-test.yml
+        name: ${{ inputs.wf_category || (github.event.schedule == '30 0 * * *' && 'NIGHTLY') }}
+        with:
+            #wf_category: ${{ inputs.wf_category || (github.event.schedule == '30 0 * * *' && 'NIGHTLY') }}
+            wf_category: ${{ 'NIGHTLY' || (github.event.schedule == '30 0 * * *' && 'NIGHTLY') }}
+            gitref: ${{ inputs.gitref || 'auto' }}
+            push_to_pypi: ${{ inputs.push_to_pypi || false }}
+
+            # nightly and weekly tests share the same configs
+            test_configs: '[{"python":"3.10.12","label":"ubuntu-20.04","timeout":"60"}]'
+
+        secrets: inherit