rebase master

first commit

bug fix

update script

update script

update benchmark parameter

update script

update script

.github/workflows/benchmarks.yml

@@ -43,7 +43,10 @@ jobs:
           pip install -U "ray @ https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl"
       - name: Run Benchmark
         run: |
-          pytest benchmark/simple.py --benchmark-only --benchmark-json bm-result.json
+          ray start --head
+          pytest -v benchmark/basic.py --benchmark-only --benchmark-json bm-result.json
+          cat bm-result.json
+          ray stop
       - name: Store benchmark result
         uses: benchmark-action/github-action-benchmark@v1
         with:

.github/workflows/ci.yml

@@ -45,6 +45,44 @@ jobs:
         run: |
           bash scripts/format.sh
 
+  Benchmarks:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Install python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install Ray Beam Runner
+        run: |
+          pip install -e .[test]
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements_dev.txt
+          pip install -U "ray @ https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp38-cp38-manylinux2014_x86_64.whl"
+      - name: Run Benchmark
+        run: |
+          ray start --head
+          pytest -v benchmark/basic.py --benchmark-only --benchmark-json bm-result.json
+          cat bm-result.json
+          ray stop
+      - name: Store benchmark result
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Python Benchmark with pytest-benchmark
+          tool: 'pytest'
+          output-file-path: bm-result.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: false
+          # Show alert with commit comment on detecting possible performance regression
+          alert-threshold: '200%'
+          comment-on-alert: true
+          fail-on-alert: true
+
   Tests:
     runs-on: ubuntu-latest
     strategy:

benchmark/simple.py → benchmark/basic.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -16,13 +17,15 @@
 #
 
 import ray
+from apache_beam.tools import fn_api_runner_microbenchmark as bm
 
 
 def run_pipeline():
+    bm.run_benchmark(1, 10, 100, True)
     return None
 
 
-def test_simple_benchmark(benchmark):
+def test_basic_benchmark(benchmark):
     ray.init()
 
     res = benchmark(run_pipeline)