Merge branch 'master' of github.com:ctuning/mlcommons-ck

ctuning · Feb 1, 2025 · 476f869 · 476f869
2 parents 3fec8a5 + 4fda64c
commit 476f869
Show file tree

Hide file tree

Showing 10 changed files with 199 additions and 12 deletions.
diff --git a/.github/workflows/cla.yml b/.github/workflows/cla.yml
@@ -4,10 +4,11 @@ on:
   issue_comment:
     types: [created]
   pull_request_target:
-    types: [opened,closed,synchronize]
+    types: [opened,closed,synchronize]  
 
 jobs:
   cla-check:
+    if: github.repository_owner == 'mlcommons'
     runs-on: ubuntu-latest
     steps:
       - name: "MLCommons CLA bot check"
@@ -22,7 +23,7 @@ jobs:
           path-to-signatures: 'cla-bot/v1/cla.json'
           # branch should not be protected
           branch: 'main'
-          allowlist: user1,bot*
+          allowlist: user1,mlcommons-bot,bot*
           remote-organization-name: mlcommons
           remote-repository-name: systems
 

diff --git a/.github/workflows/test-cmx-image-classification-onnx.yml b/.github/workflows/test-cmx-image-classification-onnx.yml
@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: 'CMX: image classification with ONNX'
+name: 'Image classification with ONNX via CMX'
 
 on:
   pull_request:
@@ -28,8 +28,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python3 -m pip install cmind
-        cmx pull repo --url=https://github.com/mlcommons/cm4mlops
+        python3 -m pip install cmx4mlops
         cmx test core
     - name: Test image classification with ONNX
       run: |

diff --git a/.github/workflows/test-cmx-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml b/.github/workflows/test-cmx-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml
@@ -0,0 +1,66 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: MLPerf inference bert (deepsparse, tf, onnxruntime, pytorch)
+
+on:
+  pull_request_target:
+    branches: [ "main", "master", "dev"]
+    paths:
+      - '.github/workflows/test-mlperf-inference-bert-deepsparse-tf-onnxruntime-pytorch.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # 3.12 didn't work on 20240305 - need to check
+        python-version: [ "3.11" ]
+        backend: [ "deepsparse", "tf", "onnxruntime", "pytorch" ]
+        precision: [ "int8", "fp32" ]
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        exclude:
+          - backend: tf
+          - backend: pytorch
+          - backend: onnxruntime
+          - precision: fp32
+          - os: windows-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Configure git longpaths (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        git config --system core.longpaths true
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmx4mlops
+    - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }}
+      if: matrix.os == 'windows-latest'
+      run: |
+        cmx run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }} --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --precision=${{ matrix.precision }} --target_qps=1 --v --quiet
+    - name: Test MLPerf Inference Bert ${{ matrix.backend }} on ${{ matrix.os }}
+      if: matrix.os != 'windows-latest'
+      run: |
+        cmx run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=bert-99 --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=5 --precision=${{ matrix.precision }} --target_qps=1 --v --quiet
+    - name: Push Test MLPerf Results (only for cTuning dev branches)
+      if: github.repository_owner == 'ctuning'
+      env:
+          USER: "GitHub Action"
+          EMAIL: "admin@cTuning.org"
+          GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}
+      run: |
+        git config --global user.name "${{ env.USER }}"
+        git config --global user.email "${{ env.EMAIL }}"
+        git config --global credential.https://github.com.helper ""
+        git config --global credential.https://github.com.helper "!gh auth git-credential"
+        git config --global credential.https://gist.github.com.helper ""
+        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+        cmx run script --tags=push,github,mlperf,inference,submission  --repo_url="https://github.com/ctuning/test_mlperf_inference_submissions"  --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
diff --git a/.github/workflows/test-cmx-mlperf-inference-resnet50.yml b/.github/workflows/test-cmx-mlperf-inference-resnet50.yml
@@ -1,11 +1,11 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: MLPerf inference ResNet50 via CMX
+name: 'MLPerf inference ResNet50 via CMX'
 
 on:
   pull_request_target:
-    branches: [ "main", "master", "dev", "mlperf-inference" ]
+    branches: [ "main", "master", "dev"]
     paths:
       - '.github/workflows/test-cmx-mlperf-inference-resnet50.yml'
       - '**'
@@ -41,16 +41,16 @@ jobs:
         git config --system core.longpaths true
     - name: Install dependencies
       run: |
-        pip install cmx4mlops
+        python3 -m pip install cmx4mlops
     - name: Test MLPerf Inference ResNet50 (Windows)
       if: matrix.os == 'windows-latest'
       run: |
-        cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet 
+        cmx run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --adr.loadgen.tags=_from-pip --pip_loadgen=yes --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1  --quiet 
     - name: Test MLPerf Inference ResNet50 (Linux/macOS)
       if: matrix.os != 'windows-latest'
       run: |
-        cm run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 -v --quiet 
-    - name: Push Test MLPerf Results when on cTuning test branch
+        cmx run script --tags=run-mlperf,inference,_submission,_short --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=resnet50 --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --target_qps=1 --v --quiet 
+    - name: Push Test MLPerf Results (only for cTuning dev branches)
       if: github.repository_owner == 'ctuning'
       env:
           USER: "GitHub Action"
@@ -63,4 +63,4 @@ jobs:
         git config --global credential.https://github.com.helper "!gh auth git-credential"
         git config --global credential.https://gist.github.com.helper ""
         git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
-        cm run script --tags=push,github,mlperf,inference,submission --repo_url=https://github.com/ctuning/test_mlperf_inference_submissions --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
+        cmx run script --tags=push,github,mlperf,inference,submission  --repo_url="https://github.com/ctuning/test_mlperf_inference_submissions"  --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
diff --git a/.github/workflows/test-cmx-mlperf-inference-rgat.yml b/.github/workflows/test-cmx-mlperf-inference-rgat.yml
@@ -0,0 +1,51 @@
+name: MLPerf inference R-GAT via CMX
+
+on:
+  pull_request:
+    branches: [ "main", "master", "dev" ]
+    paths:
+      - '.github/workflows/test-mlperf-inference-rgat.yml'
+      - '**'
+      - '!**.md'
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        python-version: [ "3.12" ]
+        backend: [ "pytorch" ]
+        implementation: [ "python" ]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Configure git longpaths (Windows)
+      if: matrix.os == 'windows-latest'
+      run: |
+        git config --system core.longpaths true
+    - name: Install dependencies
+      run: |
+        python3 -m pip install cmx4mlops
+    - name: Test MLPerf Inference R-GAT using ${{ matrix.backend }} on ${{ matrix.os }}
+      run: |
+        cmx run script --tags=run,mlperf,inference,generate-run-cmds,_submission,_short --submitter="MLCommons" --adr.inference-src.tags=_branch.dev --pull_changes=yes --pull_inference_changes=yes  --submitter="MLCommons" --hw_name=gh_${{ matrix.os }}_x86 --model=rgat --implementation=${{ matrix.implementation }} --backend=${{ matrix.backend }} --device=cpu --scenario=Offline --test_query_count=500 --adr.compiler.tags=gcc --category=datacenter --quiet  --v --target_qps=1
+    - name: Push Test MLPerf Results (only for cTuning dev branches)
+      if: github.repository_owner == 'ctuning'
+      env:
+          USER: "GitHub Action"
+          EMAIL: "admin@cTuning.org"
+          GITHUB_TOKEN: ${{ secrets.TEST_RESULTS_GITHUB_TOKEN }}
+      run: |
+        git config --global user.name "${{ env.USER }}"
+        git config --global user.email "${{ env.EMAIL }}"
+        git config --global credential.https://github.com.helper ""
+        git config --global credential.https://github.com.helper "!gh auth git-credential"
+        git config --global credential.https://gist.github.com.helper ""
+        git config --global credential.https://gist.github.com.helper "!gh auth git-credential"
+        cmx run script --tags=push,github,mlperf,inference,submission  --repo_url="https://github.com/ctuning/test_mlperf_inference_submissions"  --repo_branch=main --commit_message="Results from R50 GH action on ${{ matrix.os }}" --quiet
diff --git a/.github/workflows/test-cmx.yml b/.github/workflows/test-cmx.yml
@@ -0,0 +1,52 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: CMX test
+
+on:
+  pull_request:
+    branches: [ "master", "main", "dev" ]
+    paths:
+      - '.github/workflows/test-cm.yml'
+      - 'cm/**'
+      - '!cm/**.md'
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        on: [ubuntu-latest, windows-latest, macos-latest]
+        exclude:
+          - python-version: "3.7"
+            on: "macos-latest"
+    runs-on: "${{ matrix.on }}"
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies and test cm pull repo
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        pip install -r requirements.txt
+        python -m pip install --ignore-installed --verbose pip setuptools
+        python -m pip install cmx4mlops
+        python -m cmind
+        cm init
+    - name: Test
+      run: |
+        python tests/test_cm.py
+        cd && mkdir tmp_cm_repo
+        cd tmp_cm_repo && cm init repo
+        cmx add script tmp_cm_repo:my-test-script --tags=test,script
+        cmx add script .:my-test-script2 --tags=test2,script
+        cd && cmx add repo my-test-repo
+        cmx add script my-test-repo:my-test-script --tags=test,script
+        cd $HOME/CM/repos/my-test-repo && cmx add script .:my-test-script2 --tags=test2,script
+
diff --git a/README.md b/README.md
@@ -140,6 +140,11 @@ Copyright (c) 2014-2021 cTuning foundation
 
 * [Grigori Fursin](https://cKnowledge.org/gfursin) (FlexAI, cTuning)
 
+## Maintainers
+
+* CM, CM4MLOps and MLPerf automations: MLCommons
+* CMX (the next generation of CM): Grigori Fursin
+
 ## Long-term vision
 
 To learn more about the motivation behind CK and CM technology, please explore the following presentations:

diff --git a/cmx4mlops/README.md b/cmx4mlops/README.md
@@ -0,0 +1 @@
+TBD
diff --git a/cmx4mlops/cmx4mlops/repo/automation/cmx-demo/modulex.py b/cmx4mlops/cmx4mlops/repo/automation/cmx-demo/modulex.py
@@ -81,3 +81,14 @@ def test(self, i):
 
         return {'return':0}
 
+
+    ############################################################
+    def run(self, i):
+
+        import json
+        print (json.dumps(i, indent=2))
+
+        v = i.get('test', 'default')
+        v2 = i.get('test2', 'default')
+
+        return {'return':0, 'new_key':v, 'new_key2':v2}
diff --git a/docs/cmx/README.md b/docs/cmx/README.md
@@ -0,0 +1 @@
+TBD