rebase master

.github/workflows/ci.yml

@@ -46,6 +46,41 @@ jobs:
         run: |
           bash scripts/format.sh
 
+  Benchmarks:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Install python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install Ray Beam Runner
+        run: |
+          pip install -e .[test]
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements_dev.txt
+          pip install -U "ray @ https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl"
+      - name: Run Benchmark
+        run: |
+          pytest benchmark/basic.py --benchmark-only --benchmark-json bm-result.json
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Python Benchmark with pytest-benchmark
+          tool: 'pytest'
+          output-file-path: bm-result.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: false
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true
+
   Tests:
     runs-on: ubuntu-latest
     strategy:

benchmark/simple.py → benchmark/basic.py

@@ -24,7 +24,7 @@ def run_pipeline():
     return None
 
 
-def test_simple_benchmark(benchmark):
+def test_basic_benchmark(benchmark):
     ray.init()
 
     res = benchmark(tools.run_pipeline)