Merge pull request #4 from neuralmagic/build-out-benchmarking-workflow

Fill in benchmark results workflow

.github/actions/nm-github-action-benchmark/action.yaml

@@ -0,0 +1,46 @@
+name: nm github-action benchmark
+description: 'Use github-action-benchmark to visualize input json'
+inputs:
+  gh_action_benchmark_name:
+    description: "Name of the benchmark. Metrics are grouped by benchmark names. github_action_benchmark alert-trigger looks for the previous benchmark value in the benchmark-name group on the previous commit"
+  gh_action_benchmark_json_file_path:
+    description: "Path to the benchmark json file to upload (Note that this JSON should be in a `github-action-benchmark` consumable format - This is typically the output of neuralmagic/benchmarks/scripts/logging/gha_benchmark_logging.py)"
+    required: true
+  gh_action_benchmark_tool:
+    description: "A string that is input to the `tool` argument of `github-action-benchmark`. This should be either `customBiggerIsBetter` or `customSmallerIsBetter`"
+    required: true
+  gh_pages_branch:
+    description: "Github branch where the `github-action-benchmark` creates its index.html and data.js"
+    required: true
+  auto_push:
+    description: "When set to true, pushes the benchmark results to the `nm-gh-pages` branch."
+    required: true
+  github_token:
+    description: "secrets.GITHUB_TOKEN from the caller"
+    required: true
+
+runs:
+  using: composite
+  steps:
+    # A previous invocation of this action may have left the github pages branch in an
+    # inconsistent state.
+    - name: reset github pages branch
+      run: |
+        git update-ref refs/heads/${{ inputs.gh_pages_branch }} origin/${{ inputs.gh_pages_branch }}
+      shell: bash
+
+    - name: push to gh pages
+      uses: benchmark-action/github-action-benchmark@v1
+      with:
+        name: ${{ inputs.gh_action_benchmark_name }}
+        output-file-path: ${{ inputs.gh_action_benchmark_json_file_path }}
+        tool: ${{ inputs.gh_action_benchmark_tool }}
+        gh-pages-branch: ${{ inputs.gh_pages_branch }}
+        # Token required for pushing to nm-gh-pages branch
+        github-token: ${{ inputs.github_token }}
+        # Push and deploy to Github pages automatically
+        auto-push: ${{ inputs.auto_push == 'true' }}
+        # Create an alert when some value has regressed more than 10%
+        alert-threshold: "110%"
+        # TODO (varun): Is this a reasonable number ?
+        max-items-in-chart: 50

.github/workflows/process-benchmark-results.yaml

@@ -6,11 +6,89 @@ on:
 
 jobs:
   PROCESS_RESULTS:
+    permissions:
+      contents: 'write'
+      id-token: 'write'
     runs-on: ubuntu-latest
     steps:
-      - name: parse request
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Parse triggering request
         shell: bash
         run: |
           echo "asset: ${{ github.event.client_payload.asset }}"
-          echo "event: ${{ github.event }}"
-          echo "client_payload: ${{ github.event.client_payload }}"
+          echo "(upstream) run_id: ${{ github.event.client_payload.run_id }}"
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v2.1.3
+        with:
+            project_id: ${{ secrets.GCP_PROJECT }}
+            workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
+            service_account: ${{ secrets.GCP_GHA_SA }}
+
+      - name: Set up Google Cloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+        with:
+            version: '>= 473.0.0'
+
+      - name: Download GHA-formatted JSON files
+        shell: bash
+        run: |
+          base_path="${{ secrets.GCP_BUILD_ASSETS }}"
+          run_id="${{ github.event.client_payload.run_id }}"
+          asset="${{ github.event.client_payload.asset }}"
+          mkdir downloads
+          gcloud storage cp \
+            "$base_path/$run_id/$asset/*.json" \
+            downloads
+          ls -RA downloads
+
+
+      - name: nm-github-action-benchmark(bigger_is_better.json)
+        # Absence of the file indicates that there were no "bigger_is_better" metrics
+        if: hashFiles('downloads/bigger_is_better.json') != ''
+        uses: ./.github/actions/nm-github-action-benchmark
+        with:
+          gh_action_benchmark_name: "bigger_is_better"
+          gh_action_benchmark_json_file_path: "downloads/bigger_is_better.json"
+          gh_action_benchmark_tool: "customBiggerIsBetter"
+          gh_pages_branch: "nm-gh-pages"
+          auto_push: true
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: nm-github-action-benchmark(smaller_is_better.json)
+        # Absence of the file indicates that there were no "smaller_is_better" metrics
+        if: hashFiles('downloads/smaller_is_better.json') != ''
+        uses: ./.github/actions/nm-github-action-benchmark
+        with:
+          gh_action_benchmark_name: "smaller_is_better"
+          gh_action_benchmark_json_file_path: "downloads/smaller_is_better.json"
+          gh_action_benchmark_tool: "customSmallerIsBetter"
+          gh_pages_branch: "nm-gh-pages"
+          auto_push: true
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: nm-github-action-benchmark(observation_metrics.json)
+        # Absence of the file indicates that there were no "observation" metrics
+        if: hashFiles('downloads/observation_metrics.json') != ''
+        uses: ./.github/actions/nm-github-action-benchmark
+        with:
+          gh_action_benchmark_name: "observation_metrics"
+          gh_action_benchmark_json_file_path: "downloads/observation_metrics.json"
+          # `github-action-benchmark` expects a tool name that is either
+          # "customBiggerIsBetter" or "customSmallerIsBetter". This is a hack to
+          # work around that. Since we mark the action to not report failures, this
+          # is fine.
+          gh_action_benchmark_tool: "customBiggerIsBetter"
+          gh_pages_branch: "nm-gh-pages"
+          auto_push: true
+          github_token: ${{ secrets.GITHUB_TOKEN }}