From 97334e144adc4dcdb34e4ec5fbd6bbcd151f6ff8 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Sat, 12 Feb 2022 14:55:24 -0800 Subject: [PATCH 01/43] added continuous benchmarking workflow file --- .github/workflows/continuous-benchmark.yml | 70 ++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/workflows/continuous-benchmark.yml diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml new file mode 100644 index 000000000..936a290c3 --- /dev/null +++ b/.github/workflows/continuous-benchmark.yml @@ -0,0 +1,70 @@ +name: Continuous Benchmarking + + +on: + push: + branches: [continuous-benchmarking] + + +permissions: + contents: write + deployments: write + + +jobs: + benchmark: + name: Run C benchmark and record results + runs-on: ubuntu-latest + + steps: + - name: Setup Java JDK + uses: actions/setup-java@v1.4.3 + with: + java-version: 14 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Checkout Lingua Franca repository + uses: actions/checkout@v2 + with: + repository: LDeng0205/lingua-franca + submodules: true + + - name: Checkout reactor-c repository + uses: actions/checkout@v2 + with: + repository: lf-lang/reactor-c + path: org.lflang/src/lib/c/reactor-c + + - name: Install Python dependencies + run: pip3 install -r benchmark/runner/requirements.txt + + - name: Build lfc + run: | + ./gradlew buildLfc + + # LF BENCHMARK RUN + - name: Set LF_PATH + run: | + echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV + - name: Run C micro pingpong benchmark + run: | + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark=savina_micro_pingpong target=lf-c + + # Use continuous benchmark action to store result + - name: Store benchmark result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Lingua Franca C target Benchmark + tool: customSmallerIsBetter + output-file-path: ./benchmark/runner/benchmark_result.json + # github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '120%' + comment-on-alert: true + fail-on-alert: true + \ No newline at end of file From 0a98ccc6b92d134a3fcc4ed5748edcd3caaebed2 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Sat, 12 Feb 2022 16:02:58 -0800 Subject: [PATCH 02/43] udpated output file path --- .github/workflows/api-docs.yml | 56 +++++++++++----------- .github/workflows/ci.yml | 2 +- .github/workflows/continuous-benchmark.yml | 2 +- .github/workflows/unit-tests.yml | 48 +++++++++---------- 4 files changed, 54 insertions(+), 54 deletions(-) diff --git a/.github/workflows/api-docs.yml b/.github/workflows/api-docs.yml index c63befc80..e784f7dfc 100644 --- a/.github/workflows/api-docs.yml +++ b/.github/workflows/api-docs.yml @@ -1,32 +1,32 @@ -name: API docs +# name: API docs -on: - push: - branches: [ main ] +# on: +# push: +# branches: [ main ] -jobs: - build: +# jobs: +# build: - runs-on: macos-latest +# runs-on: macos-latest - steps: - - name: Requirements - run: brew install doxygen - && brew install sphinx-doc - && pip3 install sphinx-rtd-theme - && pip3 install breathe - && pip3 install sphinx-sitemap - && pip3 install exhale - - name: Checkout repo - uses: actions/checkout@1.0.0 - - name: Build docs - run: cd docs - && make doxygen - && cd _build/html - && touch .nojekyll - - name: Deploy - uses: JamesIves/github-pages-deploy-action@releases/v3 - with: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - BRANCH: gh-pages # The branch the action should deploy to. - FOLDER: docs/_build/html # The folder the action should deploy. +# steps: +# - name: Requirements +# run: brew install doxygen +# && brew install sphinx-doc +# && pip3 install sphinx-rtd-theme +# && pip3 install breathe +# && pip3 install sphinx-sitemap +# && pip3 install exhale +# - name: Checkout repo +# uses: actions/checkout@1.0.0 +# - name: Build docs +# run: cd docs +# && make doxygen +# && cd _build/html +# && touch .nojekyll +# - name: Deploy +# uses: JamesIves/github-pages-deploy-action@releases/v3 +# with: +# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} +# BRANCH: gh-pages # The branch the action should deploy to. +# FOLDER: docs/_build/html # The folder the action should deploy. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 88659485a..95686c417 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: CI +# name: CI on: push: diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 936a290c3..b39599789 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -60,7 +60,7 @@ jobs: with: name: Lingua Franca C target Benchmark tool: customSmallerIsBetter - output-file-path: ./benchmark/runner/benchmark_result.json + output-file-path: benchmark/runner/benchmark_result.json # github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true # Show alert with commit comment on detecting possible performance regression diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 5f74ef6f9..adce001fd 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,27 +1,27 @@ -name: Unit tests +# name: Unit tests -on: - workflow_call: - inputs: - cmake-args: - description: Arguments to pass to cmake - type: string - required: false -jobs: - run: - strategy: - matrix: - platform: [ubuntu-latest, macos-latest, windows-latest] - runs-on: ${{ matrix.platform }} +# on: +# workflow_call: +# inputs: +# cmake-args: +# description: Arguments to pass to cmake +# type: string +# required: false +# jobs: +# run: +# strategy: +# matrix: +# platform: [ubuntu-latest, macos-latest, windows-latest] +# runs-on: ${{ matrix.platform }} - steps: - - name: Check out reactor-c repository - uses: actions/checkout@v2 +# steps: +# - name: Check out reactor-c repository +# uses: actions/checkout@v2 - - name: Build and run unit tests ${{ inputs.cmake-args }} - run: | - mkdir build - cd build - cmake .. ${{ inputs.cmake-args }} - cmake --build . - make test +# - name: Build and run unit tests ${{ inputs.cmake-args }} +# run: | +# mkdir build +# cd build +# cmake .. ${{ inputs.cmake-args }} +# cmake --build . +# make test From 063c9210076ef62289ae6a110289fdcdd3755985 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Sun, 13 Feb 2022 16:57:10 -0800 Subject: [PATCH 03/43] modifiede checkout file path --- .github/workflows/api-docs.yml | 56 +++++++++++----------- .github/workflows/ci.yml | 2 +- .github/workflows/continuous-benchmark.yml | 12 ++--- .github/workflows/unit-tests.yml | 48 +++++++++---------- 4 files changed, 59 insertions(+), 59 deletions(-) diff --git a/.github/workflows/api-docs.yml b/.github/workflows/api-docs.yml index e784f7dfc..c63befc80 100644 --- a/.github/workflows/api-docs.yml +++ b/.github/workflows/api-docs.yml @@ -1,32 +1,32 @@ -# name: API docs +name: API docs -# on: -# push: -# branches: [ main ] +on: + push: + branches: [ main ] -# jobs: -# build: +jobs: + build: -# runs-on: macos-latest + runs-on: macos-latest -# steps: -# - name: Requirements -# run: brew install doxygen -# && brew install sphinx-doc -# && pip3 install sphinx-rtd-theme -# && pip3 install breathe -# && pip3 install sphinx-sitemap -# && pip3 install exhale -# - name: Checkout repo -# uses: actions/checkout@1.0.0 -# - name: Build docs -# run: cd docs -# && make doxygen -# && cd _build/html -# && touch .nojekyll -# - name: Deploy -# uses: JamesIves/github-pages-deploy-action@releases/v3 -# with: -# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# BRANCH: gh-pages # The branch the action should deploy to. -# FOLDER: docs/_build/html # The folder the action should deploy. + steps: + - name: Requirements + run: brew install doxygen + && brew install sphinx-doc + && pip3 install sphinx-rtd-theme + && pip3 install breathe + && pip3 install sphinx-sitemap + && pip3 install exhale + - name: Checkout repo + uses: actions/checkout@1.0.0 + - name: Build docs + run: cd docs + && make doxygen + && cd _build/html + && touch .nojekyll + - name: Deploy + uses: JamesIves/github-pages-deploy-action@releases/v3 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BRANCH: gh-pages # The branch the action should deploy to. + FOLDER: docs/_build/html # The folder the action should deploy. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95686c417..88659485a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -# name: CI +name: CI on: push: diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index b39599789..d6234e317 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -33,11 +33,11 @@ jobs: repository: LDeng0205/lingua-franca submodules: true - - name: Checkout reactor-c repository - uses: actions/checkout@v2 - with: - repository: lf-lang/reactor-c - path: org.lflang/src/lib/c/reactor-c + # - name: Checkout reactor-c repository + # uses: actions/checkout@v2 + # with: + # repository: lf-lang/reactor-c + # path: org.lflang/src/lib/c/reactor-c - name: Install Python dependencies run: pip3 install -r benchmark/runner/requirements.txt @@ -60,7 +60,7 @@ jobs: with: name: Lingua Franca C target Benchmark tool: customSmallerIsBetter - output-file-path: benchmark/runner/benchmark_result.json + output-file-path: /benchmark/runner/benchmark_result.json # github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true # Show alert with commit comment on detecting possible performance regression diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index adce001fd..5f74ef6f9 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -1,27 +1,27 @@ -# name: Unit tests +name: Unit tests -# on: -# workflow_call: -# inputs: -# cmake-args: -# description: Arguments to pass to cmake -# type: string -# required: false -# jobs: -# run: -# strategy: -# matrix: -# platform: [ubuntu-latest, macos-latest, windows-latest] -# runs-on: ${{ matrix.platform }} +on: + workflow_call: + inputs: + cmake-args: + description: Arguments to pass to cmake + type: string + required: false +jobs: + run: + strategy: + matrix: + platform: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.platform }} -# steps: -# - name: Check out reactor-c repository -# uses: actions/checkout@v2 + steps: + - name: Check out reactor-c repository + uses: actions/checkout@v2 -# - name: Build and run unit tests ${{ inputs.cmake-args }} -# run: | -# mkdir build -# cd build -# cmake .. ${{ inputs.cmake-args }} -# cmake --build . -# make test + - name: Build and run unit tests ${{ inputs.cmake-args }} + run: | + mkdir build + cd build + cmake .. ${{ inputs.cmake-args }} + cmake --build . + make test From 0df54bf900039f31914657ec03e037a39a18ca07 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Sun, 13 Feb 2022 23:48:59 -0800 Subject: [PATCH 04/43] modified file path to match updated result generation script --- .github/workflows/continuous-benchmark.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index d6234e317..0afb2ed61 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -50,9 +50,9 @@ jobs: - name: Set LF_PATH run: | echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - - name: Run C micro pingpong benchmark + - name: Run C micro pingpong and count benchmark run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark=savina_micro_pingpong target=lf-c + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark=savina_micro_pingpong,savina_micro_count target=lf-c # Use continuous benchmark action to store result - name: Store benchmark result @@ -60,11 +60,11 @@ jobs: with: name: Lingua Franca C target Benchmark tool: customSmallerIsBetter - output-file-path: /benchmark/runner/benchmark_result.json + output-file-path: multirun/benchmark_result.json # github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: true + auto-push: false # Show alert with commit comment on detecting possible performance regression alert-threshold: '120%' - comment-on-alert: true - fail-on-alert: true + comment-on-alert: false + fail-on-alert: false \ No newline at end of file From c5211ded8b11d92ef193b9b2a4806c0f4f3b8c59 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Mon, 14 Feb 2022 10:28:03 -0800 Subject: [PATCH 05/43] adjusted workflow settings --- .github/workflows/continuous-benchmark.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 0afb2ed61..c68799d06 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -33,12 +33,6 @@ jobs: repository: LDeng0205/lingua-franca submodules: true - # - name: Checkout reactor-c repository - # uses: actions/checkout@v2 - # with: - # repository: lf-lang/reactor-c - # path: org.lflang/src/lib/c/reactor-c - - name: Install Python dependencies run: pip3 install -r benchmark/runner/requirements.txt @@ -61,8 +55,8 @@ jobs: name: Lingua Franca C target Benchmark tool: customSmallerIsBetter output-file-path: multirun/benchmark_result.json - # github-token: ${{ secrets.GITHUB_TOKEN }} - auto-push: false + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true # Show alert with commit comment on detecting possible performance regression alert-threshold: '120%' comment-on-alert: false From 85f7845a14e4604f4a1fa249b93955ebbbd8c896 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Mon, 14 Feb 2022 10:41:15 -0800 Subject: [PATCH 06/43] running all benchmarks --- .github/workflows/continuous-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index c68799d06..5726a8d03 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -46,7 +46,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C micro pingpong and count benchmark run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark=savina_micro_pingpong,savina_micro_count target=lf-c + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark="glob(*)" target=lf-c # Use continuous benchmark action to store result - name: Store benchmark result From adb55e53774d12cbbc5d48b07ba7369b28780c5e Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Mon, 14 Feb 2022 20:18:53 -0800 Subject: [PATCH 07/43] tweaked workflow file --- .github/workflows/continuous-benchmark.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 5726a8d03..58a27e45e 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -13,7 +13,7 @@ permissions: jobs: benchmark: - name: Run C benchmark and record results + name: Run C Benchmarks runs-on: ubuntu-latest steps: @@ -44,12 +44,12 @@ jobs: - name: Set LF_PATH run: | echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - - name: Run C micro pingpong and count benchmark + - name: Run C Benchmarks run: | python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark="glob(*)" target=lf-c # Use continuous benchmark action to store result - - name: Store benchmark result + - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: name: Lingua Franca C target Benchmark @@ -57,8 +57,9 @@ jobs: output-file-path: multirun/benchmark_result.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true + # Show alert with commit comment on detecting possible performance regression - alert-threshold: '120%' - comment-on-alert: false + alert-threshold: '150%' + comment-on-alert: true fail-on-alert: false \ No newline at end of file From 7398aa447392d82a36dab7897823450aa97f8f3f Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 15 Feb 2022 10:06:21 -0800 Subject: [PATCH 08/43] test --- .github/workflows/continuous-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 58a27e45e..2e245063e 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -59,7 +59,7 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '150%' - comment-on-alert: true + alert-threshold: '200%' + comment-on-alert: false fail-on-alert: false \ No newline at end of file From 45f337be7259221c0d8185557c8dfe8b5200369a Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 15 Feb 2022 20:41:50 -0800 Subject: [PATCH 09/43] changed iteration number --- .github/workflows/continuous-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 2e245063e..fc435c53f 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -46,7 +46,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=1 benchmark="glob(*)" target=lf-c + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c # Use continuous benchmark action to store result - name: Store Benchmark Result From cd2f1d424eb2c00205b12d574f75cbf292f696d5 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 1 Mar 2022 15:19:37 -0800 Subject: [PATCH 10/43] continue on error --- .github/workflows/continuous-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index fc435c53f..8e5c4a7be 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -46,7 +46,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True # Use continuous benchmark action to store result - name: Store Benchmark Result From b8568b15c09c85763e827b4341411c8003b1fed8 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 8 Mar 2022 11:25:59 -0800 Subject: [PATCH 11/43] changed action and runner to russell --- .github/workflows/continuous-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 8e5c4a7be..0b248a56c 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -14,7 +14,7 @@ permissions: jobs: benchmark: name: Run C Benchmarks - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Setup Java JDK @@ -46,7 +46,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=12 benchmark="glob(*)" target=lf-c continue_on_error=True # Use continuous benchmark action to store result - name: Store Benchmark Result From 4201ce176424835b61b25a2b665945384889dbeb Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 8 Mar 2022 12:47:05 -0800 Subject: [PATCH 12/43] checking out current version of reactor-c --- .github/workflows/continuous-benchmark.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 0b248a56c..a2e2f98c2 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -2,9 +2,12 @@ name: Continuous Benchmarking on: + # pull_request: + # types: + # - opened push: branches: [continuous-benchmarking] - + permissions: contents: write @@ -14,7 +17,7 @@ permissions: jobs: benchmark: name: Run C Benchmarks - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Setup Java JDK @@ -31,7 +34,10 @@ jobs: uses: actions/checkout@v2 with: repository: LDeng0205/lingua-franca - submodules: true + - name: Checkout current branch of reactor-c + uses: actions/checkout@v2 + with: + path: org.lflang/src/lib/c/reactor-c - name: Install Python dependencies run: pip3 install -r benchmark/runner/requirements.txt @@ -59,7 +65,7 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' + alert-threshold: '120%' comment-on-alert: false fail-on-alert: false \ No newline at end of file From c30ac24753e59e549e35850574f8f341d88bedd0 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 8 Mar 2022 12:58:51 -0800 Subject: [PATCH 13/43] checking out lf from lf-lang --- .github/workflows/continuous-benchmark.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index a2e2f98c2..b906d1cce 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -2,9 +2,9 @@ name: Continuous Benchmarking on: - # pull_request: - # types: - # - opened + pull_request: + types: + - opened push: branches: [continuous-benchmarking] @@ -33,7 +33,8 @@ jobs: - name: Checkout Lingua Franca repository uses: actions/checkout@v2 with: - repository: LDeng0205/lingua-franca + repository: lf-lang/lingua-franca + ref: continuous-benchmark - name: Checkout current branch of reactor-c uses: actions/checkout@v2 with: @@ -52,7 +53,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=12 benchmark="glob(*)" target=lf-c continue_on_error=True + python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True # Use continuous benchmark action to store result - name: Store Benchmark Result @@ -65,7 +66,7 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '120%' - comment-on-alert: false + alert-threshold: '200%' + comment-on-alert: true fail-on-alert: false \ No newline at end of file From 3d4e91e72f68b7e1e9a8bdee93870577c765e7a1 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Tue, 8 Mar 2022 13:08:22 -0800 Subject: [PATCH 14/43] fixed file name --- .github/workflows/continuous-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index b906d1cce..07b67eb85 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -53,7 +53,7 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark_json.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True + python3 benchmark/runner/run_benchmark.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True json=True # Use continuous benchmark action to store result - name: Store Benchmark Result From 87c2811779022e0a18c257a466539e3e910be381 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Thu, 10 Mar 2022 18:16:25 -0800 Subject: [PATCH 15/43] modified to use benchmark repository --- .github/workflows/continuous-benchmark.yml | 37 ++++++++++++++-------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 07b67eb85..0372884d1 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -14,46 +14,57 @@ permissions: deployments: write -jobs: + +jobs: benchmark: name: Run C Benchmarks - runs-on: ubuntu-latest + runs-on: self-hosted steps: - name: Setup Java JDK uses: actions/setup-java@v1.4.3 with: java-version: 14 - + - name: Setup Python uses: actions/setup-python@v2 with: python-version: 3.8 + + - name: Checkout benchmark repository + uses: actions/checkout@v2 + with: + repository: lf-lang/benchmarks-lingua-franca + ref: continuous-benchmark + - name: Checkout Lingua Franca repository uses: actions/checkout@v2 with: repository: lf-lang/lingua-franca - ref: continuous-benchmark - - name: Checkout current branch of reactor-c + path: lf + + - name: Checkout current version of reactor-c uses: actions/checkout@v2 with: - path: org.lflang/src/lib/c/reactor-c - + path: lf/org.lflang/src/lib/c/reactor-c + - name: Install Python dependencies - run: pip3 install -r benchmark/runner/requirements.txt + run: pip3 install -r runner/requirements.txt - name: Build lfc run: | + cd lf ./gradlew buildLfc - # LF BENCHMARK RUN - - name: Set LF_PATH + - name: Set LF_PATH and LF_BENCHMARKS_PATH environmental variable run: | - echo "LF_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV + echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV + echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV + - name: Run C Benchmarks run: | - python3 benchmark/runner/run_benchmark.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c continue_on_error=True json=True + python3 runner/run_benchmark.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c json=True # Use continuous benchmark action to store result - name: Store Benchmark Result @@ -67,6 +78,6 @@ jobs: # Show alert with commit comment on detecting possible performance regression alert-threshold: '200%' - comment-on-alert: true + comment-on-alert: false fail-on-alert: false \ No newline at end of file From aa71a558fcc5ce7189c0e4670dfedcc395587452 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Thu, 10 Mar 2022 18:17:45 -0800 Subject: [PATCH 16/43] testing on github machine --- .github/workflows/continuous-benchmark.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 0372884d1..8a14988ff 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -18,7 +18,7 @@ permissions: jobs: benchmark: name: Run C Benchmarks - runs-on: self-hosted + runs-on: ubuntu-latest steps: - name: Setup Java JDK From 9105e37c52fd5b40e13221db73ff7f9380f45af6 Mon Sep 17 00:00:00 2001 From: LDeng0205 Date: Thu, 10 Mar 2022 21:18:24 -0800 Subject: [PATCH 17/43] added comments --- .github/workflows/continuous-benchmark.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 8a14988ff..9d0eb6c46 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -18,7 +18,7 @@ permissions: jobs: benchmark: name: Run C Benchmarks - runs-on: ubuntu-latest + runs-on: ubuntu-latest #FIXME: change to self-hosted after russel is set up. steps: - name: Setup Java JDK @@ -36,7 +36,7 @@ jobs: uses: actions/checkout@v2 with: repository: lf-lang/benchmarks-lingua-franca - ref: continuous-benchmark + ref: continuous-benchmark # FIXME: delete this line after merge - name: Checkout Lingua Franca repository uses: actions/checkout@v2 @@ -64,7 +64,7 @@ jobs: - name: Run C Benchmarks run: | - python3 runner/run_benchmark.py -m test_mode=False iterations=5 benchmark="glob(*)" target=lf-c json=True + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 benchmark="glob(*)" target=lf-c json=True # Use continuous benchmark action to store result - name: Store Benchmark Result @@ -77,7 +77,7 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' + alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true comment-on-alert: false fail-on-alert: false \ No newline at end of file From d740d3378501a84fd74d5efbc4150045ccaf9fdf Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Sat, 9 Jul 2022 16:51:50 -0700 Subject: [PATCH 18/43] [benchmark] Update cb workflow. --- .github/workflows/continuous-benchmark.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 9d0eb6c46..3148284d7 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -2,11 +2,8 @@ name: Continuous Benchmarking on: - pull_request: - types: - - opened - push: - branches: [continuous-benchmarking] + pull_request: + workflow_dispatch: permissions: From ceab5942ba49cc75552b20ed37930a1377af2395 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Sat, 9 Jul 2022 19:15:34 -0700 Subject: [PATCH 19/43] [benchmarks] Update workflow. --- .github/workflows/continuous-benchmark.yml | 32 ++++++++++------------ 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 3148284d7..d4d497206 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -4,36 +4,29 @@ name: Continuous Benchmarking on: pull_request: workflow_dispatch: - - + + permissions: contents: write deployments: write - -jobs: +jobs: benchmark: name: Run C Benchmarks runs-on: ubuntu-latest #FIXME: change to self-hosted after russel is set up. steps: - - name: Setup Java JDK - uses: actions/setup-java@v1.4.3 - with: - java-version: 14 - - name: Setup Python uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Checkout benchmark repository uses: actions/checkout@v2 with: repository: lf-lang/benchmarks-lingua-franca - ref: continuous-benchmark # FIXME: delete this line after merge + ref: cb # FIXME: delete this line after merge - name: Checkout Lingua Franca repository uses: actions/checkout@v2 @@ -41,13 +34,16 @@ jobs: repository: lf-lang/lingua-franca path: lf + - name: Prepare LF build environment + uses: ./lf/.github/actions/prepare-build-env + - name: Checkout current version of reactor-c uses: actions/checkout@v2 with: path: lf/org.lflang/src/lib/c/reactor-c - name: Install Python dependencies - run: pip3 install -r runner/requirements.txt + run: pip3 install -r runner/requirements.txt - name: Build lfc run: | @@ -58,18 +54,21 @@ jobs: run: | echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - + - name: Run C Benchmarks run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 benchmark="glob(*)" target=lf-c json=True - + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 benchmark="glob(*)" target=lf-c + + - name: Collect results + run: python3 runner/collect_results.py latest continuous_benchmarking_results.json + # Use continuous benchmark action to store result - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: name: Lingua Franca C target Benchmark tool: customSmallerIsBetter - output-file-path: multirun/benchmark_result.json + output-file-path: continuous_benchmarking_results.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true @@ -77,4 +76,3 @@ jobs: alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true comment-on-alert: false fail-on-alert: false - \ No newline at end of file From 3ac8ecf1711f8610615a70400f52f4ac8c8b01bc Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Wed, 13 Jul 2022 14:13:28 -0700 Subject: [PATCH 20/43] REVERT ME --- .github/workflows/continuous-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index d4d497206..858d79096 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -73,6 +73,6 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true - comment-on-alert: false + alert-threshold: '100%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + comment-on-alert: true fail-on-alert: false From 4ca49de79e2ecf57a836ebb67e36f932996ab44e Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Wed, 13 Jul 2022 15:09:59 -0700 Subject: [PATCH 21/43] [benchmarks] Benchmark different runtime versions. --- .github/workflows/continuous-benchmark.yml | 28 ++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 858d79096..c2a2ce125 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -55,9 +55,33 @@ jobs: echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - - name: Run C Benchmarks + - name: Run C Benchmarks (multithreaded) run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 benchmark="glob(*)" target=lf-c + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + benchmark="glob(*)" target=lf-c target.params.scheduler=GEDF_NP,NP,adaptive threads=0 + + - name: Collect results + run: python3 runner/collect_results.py latest continuous_benchmarking_results.json + + # Use continuous benchmark action to store result + - name: Store Benchmark Result + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Lingua Franca C target Benchmark + tool: customSmallerIsBetter + output-file-path: continuous_benchmarking_results.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + + # Show alert with commit comment on detecting possible performance regression + alert-threshold: '100%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + comment-on-alert: true + fail-on-alert: false + + - name: Run C Benchmarks (unthreaded) + run: | + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + benchmark="glob(*)" target=lf-c-unthreaded - name: Collect results run: python3 runner/collect_results.py latest continuous_benchmarking_results.json From dd98802221b55602b1cd06a202368da2b755b242 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Thu, 14 Jul 2022 15:17:51 -0700 Subject: [PATCH 22/43] [benchmarks] Use size=fast. --- .github/workflows/continuous-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index c2a2ce125..6f759dac4 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -57,7 +57,7 @@ jobs: - name: Run C Benchmarks (multithreaded) run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 size=fast \ benchmark="glob(*)" target=lf-c target.params.scheduler=GEDF_NP,NP,adaptive threads=0 - name: Collect results @@ -80,7 +80,7 @@ jobs: - name: Run C Benchmarks (unthreaded) run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 size=fast \ benchmark="glob(*)" target=lf-c-unthreaded - name: Collect results From b9f341af731bd5502b6a6a0fb2174dba2bd224db Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Fri, 15 Jul 2022 10:43:21 -0700 Subject: [PATCH 23/43] Update continuous-benchmark.yml. --- .github/workflows/continuous-benchmark.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 6f759dac4..879798e72 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -61,15 +61,15 @@ jobs: benchmark="glob(*)" target=lf-c target.params.scheduler=GEDF_NP,NP,adaptive threads=0 - name: Collect results - run: python3 runner/collect_results.py latest continuous_benchmarking_results.json + run: python3 runner/collect_results.py latest continuous-benchmarking-results-multi-threaded.json # Use continuous benchmark action to store result - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: - name: Lingua Franca C target Benchmark + name: Lingua Franca C Benchmark -- Multithreaded tool: customSmallerIsBetter - output-file-path: continuous_benchmarking_results.json + output-file-path: continuous-benchmarking-results-multi-threaded.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true @@ -84,15 +84,15 @@ jobs: benchmark="glob(*)" target=lf-c-unthreaded - name: Collect results - run: python3 runner/collect_results.py latest continuous_benchmarking_results.json + run: python3 runner/collect_results.py latest continuous-benchmarking-results-single-threaded.json # Use continuous benchmark action to store result - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: - name: Lingua Franca C target Benchmark + name: Lingua Franca C Benchmark -- Single-Threaded tool: customSmallerIsBetter - output-file-path: continuous_benchmarking_results.json + output-file-path: continuous-benchmarking-results-single-threaded.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true From 5f89f934587511b73edbfa4d2a8122684c51e2c3 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Fri, 15 Jul 2022 16:25:56 -0700 Subject: [PATCH 24/43] [benchmarks] Adjust threshold for warning. --- .github/workflows/continuous-benchmark.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 879798e72..c1fa4c46f 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -72,9 +72,7 @@ jobs: output-file-path: continuous-benchmarking-results-multi-threaded.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true - - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '100%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true comment-on-alert: true fail-on-alert: false @@ -97,6 +95,6 @@ jobs: auto-push: true # Show alert with commit comment on detecting possible performance regression - alert-threshold: '100%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true comment-on-alert: true fail-on-alert: false From 3812932d03996bb127f55445189072dfca20ac56 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Fri, 15 Jul 2022 16:45:17 -0700 Subject: [PATCH 25/43] [benchmarks] Rename a step; update comments. --- .github/workflows/continuous-benchmark.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index c1fa4c46f..654241696 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest #FIXME: change to self-hosted after russel is set up. steps: - - name: Setup Python + - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.8 @@ -26,7 +26,7 @@ jobs: uses: actions/checkout@v2 with: repository: lf-lang/benchmarks-lingua-franca - ref: cb # FIXME: delete this line after merge + ref: cb # FIXME: delete this line after `cb` is merged. - name: Checkout Lingua Franca repository uses: actions/checkout@v2 @@ -63,7 +63,6 @@ jobs: - name: Collect results run: python3 runner/collect_results.py latest continuous-benchmarking-results-multi-threaded.json - # Use continuous benchmark action to store result - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: @@ -72,7 +71,7 @@ jobs: output-file-path: continuous-benchmarking-results-multi-threaded.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true - alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold comment-on-alert: true fail-on-alert: false @@ -84,7 +83,6 @@ jobs: - name: Collect results run: python3 runner/collect_results.py latest continuous-benchmarking-results-single-threaded.json - # Use continuous benchmark action to store result - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 with: @@ -93,8 +91,6 @@ jobs: output-file-path: continuous-benchmarking-results-single-threaded.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true - - # Show alert with commit comment on detecting possible performance regression - alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold and set alert/comment to true + alert-threshold: '200%' # FIXME: After russel is set up, lower the threshold comment-on-alert: true fail-on-alert: false From 72edc94e505b396bbd6ef6ce21345b25edf234ec Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Wed, 20 Jul 2022 16:54:15 -0700 Subject: [PATCH 26/43] [benchmarks] Update to match collect_results.py. --- .github/workflows/continuous-benchmark.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 654241696..87cc4b36a 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -57,11 +57,11 @@ jobs: - name: Run C Benchmarks (multithreaded) run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 size=fast \ + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 problem_size=small \ benchmark="glob(*)" target=lf-c target.params.scheduler=GEDF_NP,NP,adaptive threads=0 - name: Collect results - run: python3 runner/collect_results.py latest continuous-benchmarking-results-multi-threaded.json + run: python3 runner/collect_results.py continuous-benchmarking-results-multi-threaded.json - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 @@ -77,11 +77,11 @@ jobs: - name: Run C Benchmarks (unthreaded) run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 size=fast \ + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 problem_size=small \ benchmark="glob(*)" target=lf-c-unthreaded - name: Collect results - run: python3 runner/collect_results.py latest continuous-benchmarking-results-single-threaded.json + run: python3 runner/collect_results.py continuous-benchmarking-results-single-threaded.json - name: Store Benchmark Result uses: benchmark-action/github-action-benchmark@v1 From 8153c63883d5b1dd328ae3ff081001f9d84bec97 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Tue, 12 Jul 2022 15:50:43 -0700 Subject: [PATCH 27/43] [benchmarks] Create second benchmark workflow. --- .github/actions/set-up-workspace/action.yml | 40 ++++++++++++++ .github/workflows/continuous-benchmark.yml | 40 +++----------- .../workflows/user-requested-benchmark.yml | 53 +++++++++++++++++++ 3 files changed, 100 insertions(+), 33 deletions(-) create mode 100644 .github/actions/set-up-workspace/action.yml create mode 100644 .github/workflows/user-requested-benchmark.yml diff --git a/.github/actions/set-up-workspace/action.yml b/.github/actions/set-up-workspace/action.yml new file mode 100644 index 000000000..47b9ffbd8 --- /dev/null +++ b/.github/actions/set-up-workspace/action.yml @@ -0,0 +1,40 @@ + +name: Set up workspace +description: Check out repositories, download software, install LFC, etc. +runs: + using: "composite" + steps: + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: 3.8 + + - name: Checkout Lingua Franca repository + uses: actions/checkout@v2 + with: + repository: lf-lang/lingua-franca + path: lf + + - name: Prepare LF build environment + uses: ./lf/.github/actions/prepare-build-env + + - name: Checkout current version of reactor-c + uses: actions/checkout@v2 + with: + path: lf/org.lflang/src/lib/c/reactor-c + + - name: Install Python dependencies + run: pip3 install -r runner/requirements.txt + shell: bash + + - name: Build lfc + run: | + cd lf + ./gradlew buildLfc + shell: bash + + - name: Set LF_PATH and LF_BENCHMARKS_PATH environmental variable + run: | + echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV + echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV + shell: bash \ No newline at end of file diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 87cc4b36a..0fbca965b 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -2,6 +2,8 @@ name: Continuous Benchmarking on: + push: + branches: [continuous-benchmarking] #FIXME: delete this line before merge pull_request: workflow_dispatch: @@ -17,48 +19,20 @@ jobs: runs-on: ubuntu-latest #FIXME: change to self-hosted after russel is set up. steps: - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: 3.8 - - name: Checkout benchmark repository uses: actions/checkout@v2 with: repository: lf-lang/benchmarks-lingua-franca ref: cb # FIXME: delete this line after `cb` is merged. - - name: Checkout Lingua Franca repository - uses: actions/checkout@v2 - with: - repository: lf-lang/lingua-franca - path: lf - - - name: Prepare LF build environment - uses: ./lf/.github/actions/prepare-build-env - - - name: Checkout current version of reactor-c - uses: actions/checkout@v2 - with: - path: lf/org.lflang/src/lib/c/reactor-c - - - name: Install Python dependencies - run: pip3 install -r runner/requirements.txt - - - name: Build lfc - run: | - cd lf - ./gradlew buildLfc - - - name: Set LF_PATH and LF_BENCHMARKS_PATH environmental variable - run: | - echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV - echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV + - name: Set up workspace + uses: ./.github/actions/set-up-workspace - name: Run C Benchmarks (multithreaded) run: | - python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 problem_size=small \ - benchmark="glob(*)" target=lf-c target.params.scheduler=GEDF_NP,NP,adaptive threads=0 + python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + benchmark="glob(*)" target=lf-c problem_size=small \ + target.params.scheduler=GEDF_NP,NP,adaptive threads=0 - name: Collect results run: python3 runner/collect_results.py continuous-benchmarking-results-multi-threaded.json diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml new file mode 100644 index 000000000..dbfe0b0a5 --- /dev/null +++ b/.github/workflows/user-requested-benchmark.yml @@ -0,0 +1,53 @@ +name: User-requested benchmarks +on: + issue_comment: + types: [created] + push: # TODO: Remove this when this is in the default branch. + workflow_dispatch: + inputs: + pr_number: + required: true + type: number +jobs: + run_benchmarks: + runs-on: ubuntu-latest + name: Post benchmark results in a PR. + steps: + - name: Checkout benchmark repository + uses: actions/checkout@v2 + with: + repository: lf-lang/benchmarks-lingua-franca + ref: cb # FIXME: delete this line after merge + + - name: Set up workspace + uses: lf-lang/reactor-c/.github/actions/set-up-workspace@continuous-benchmarking + + - name: Run C Benchmarks + # run: | + # ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + # benchmark="glob(*)" target=lf-c size=fast \ + # target.params.scheduler=GEDF_NP,NP,adaptive threads=1,3,6,12,24 + run: | # DEBUG + ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + benchmark="savina_parallelism_recmatmul" target=lf-c size=fast \ + target.params.scheduler=NP,adaptive threads=1,3,0 + shell: bash + + - name: Collect benchmark results + run: | + ./runner/collect_results.py latest latest-benchmark-results.csv + shell: bash + + - name: Make figure + run: | + ./runner/make-graphics.py latest-benchmark-results.csv latest-benchmark-results.png --uri + shell: bash + id: figure + + - name: Comment PR + uses: thollander/actions-comment-pull-request@v1 + with: + message: | + ![Visualization of the benchmark results.](${{ steps.figure.outputs.DATA_URI }}) + pr_number: 51 # ${{ inputs.pr_number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c361b8e5d3029758515eb33606bc2632ca3cbbf5 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Tue, 12 Jul 2022 22:08:20 -0700 Subject: [PATCH 28/43] [benchmarks] Try another way of saving images. --- .../workflows/user-requested-benchmark.yml | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml index dbfe0b0a5..20a5b6b75 100644 --- a/.github/workflows/user-requested-benchmark.yml +++ b/.github/workflows/user-requested-benchmark.yml @@ -38,16 +38,43 @@ jobs: ./runner/collect_results.py latest latest-benchmark-results.csv shell: bash + - name: Checkout repository for saving graphics + uses: actions/checkout@v2 + with: + repository: lf-lang/benchmarks-lingua-franca + ref: graphics + path: graphics-repo + fetch-depth: 0 # History is required for push + - name: Make figure run: | - ./runner/make-graphics.py latest-benchmark-results.csv latest-benchmark-results.png --uri + OUTPUT_DIR=runner/images/$(date -I) + SHA=${{ github.sha }} + OUTPUT_FILE=$OUTPUT_DIR/${SHA:0:7}.png + mkdir -p graphics-repo/$OUTPUT_DIR + ./runner/make-graphics.py latest-benchmark-results.csv graphics-repo/$OUTPUT_FILE + echo "::set-output name=output_file::$OUTPUT_FILE" shell: bash id: figure + - name: Commit figure + run: | + cd graphics-repo + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git add ${{ steps.figure.outputs.OUTPUT_FILE }} + git commit -m "benchmark ${{ github.sha }}" + + - name: Push figure + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: graphics + - name: Comment PR uses: thollander/actions-comment-pull-request@v1 with: message: | - ![Visualization of the benchmark results.](${{ steps.figure.outputs.DATA_URI }}) + ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/benchmarks-lingua-franca/graphics/${{ steps.figure.outputs.OUTPUT_FILE }}) pr_number: 51 # ${{ inputs.pr_number }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From dcd20bb8355f6edc718e765bc2be092aa6c0b3c5 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Wed, 13 Jul 2022 17:35:51 -0700 Subject: [PATCH 29/43] [benchmarks] First working PR comment. --- .github/workflows/continuous-benchmark.yml | 2 +- .../workflows/user-requested-benchmark.yml | 54 +++++++++++-------- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 0fbca965b..80e2734cb 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -23,7 +23,7 @@ jobs: uses: actions/checkout@v2 with: repository: lf-lang/benchmarks-lingua-franca - ref: cb # FIXME: delete this line after `cb` is merged. + ref: automated-full-benchmark # FIXME: delete this line after merge - name: Set up workspace uses: ./.github/actions/set-up-workspace diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml index 20a5b6b75..d8bf22a3b 100644 --- a/.github/workflows/user-requested-benchmark.yml +++ b/.github/workflows/user-requested-benchmark.yml @@ -1,4 +1,5 @@ name: User-requested benchmarks + on: issue_comment: types: [created] @@ -8,6 +9,11 @@ on: pr_number: required: true type: number + +permissions: + contents: write + pull-requests: write + jobs: run_benchmarks: runs-on: ubuntu-latest @@ -17,10 +23,21 @@ jobs: uses: actions/checkout@v2 with: repository: lf-lang/benchmarks-lingua-franca - ref: cb # FIXME: delete this line after merge + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 1 + ref: automated-full-benchmark # FIXME: delete this line after merge + + - name: Checkout reactor-c + uses: actions/checkout@v2 + with: + repository: lf-lang/reactor-c + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 0 # It will be necessary to push to this repo + ref: automated-full-benchmark # FIXME: delete this line after merge + path: reactor-c - name: Set up workspace - uses: lf-lang/reactor-c/.github/actions/set-up-workspace@continuous-benchmarking + uses: lf-lang/reactor-c/.github/actions/set-up-workspace@automated-full-benchmark - name: Run C Benchmarks # run: | @@ -38,43 +55,36 @@ jobs: ./runner/collect_results.py latest latest-benchmark-results.csv shell: bash - - name: Checkout repository for saving graphics - uses: actions/checkout@v2 - with: - repository: lf-lang/benchmarks-lingua-franca - ref: graphics - path: graphics-repo - fetch-depth: 0 # History is required for push - - name: Make figure run: | - OUTPUT_DIR=runner/images/$(date -I) + OUTPUT_DIR=images/$(date -I) SHA=${{ github.sha }} OUTPUT_FILE=$OUTPUT_DIR/${SHA:0:7}.png - mkdir -p graphics-repo/$OUTPUT_DIR - ./runner/make-graphics.py latest-benchmark-results.csv graphics-repo/$OUTPUT_FILE + mkdir -p reactor-c/$OUTPUT_DIR + ./runner/make-graphics.py latest-benchmark-results.csv reactor-c/$OUTPUT_FILE echo "::set-output name=output_file::$OUTPUT_FILE" shell: bash id: figure - name: Commit figure run: | - cd graphics-repo + cd reactor-c git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" + git fetch + git switch graphics git add ${{ steps.figure.outputs.OUTPUT_FILE }} git commit -m "benchmark ${{ github.sha }}" - - - name: Push figure - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branch: graphics + git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | \ + xargs -L1 git config --unset-all + git push https://token:${{ secrets.GITHUB_TOKEN }}@github.com/lf-lang/reactor-c.git + cd .. + shell: bash - name: Comment PR uses: thollander/actions-comment-pull-request@v1 with: message: | - ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/benchmarks-lingua-franca/graphics/${{ steps.figure.outputs.OUTPUT_FILE }}) - pr_number: 51 # ${{ inputs.pr_number }} + ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/reactor-c/graphics/${{ steps.figure.outputs.OUTPUT_FILE }}) + pr_number: 91 # ${{ inputs.pr_number }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 4208bbc81d6aa0d3cf99ed1879d0d05fca424c5a Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Wed, 13 Jul 2022 23:25:40 -0700 Subject: [PATCH 30/43] [benchmarks] Separate main and feature benchmarks. --- .github/actions/run-benchmarks/action.yml | 56 +++++++++++++++++++ .github/actions/set-up-workspace/action.yml | 15 +---- .../workflows/user-requested-benchmark.yml | 53 +++++++++--------- 3 files changed, 86 insertions(+), 38 deletions(-) create mode 100644 .github/actions/run-benchmarks/action.yml diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml new file mode 100644 index 000000000..87ab6bc52 --- /dev/null +++ b/.github/actions/run-benchmarks/action.yml @@ -0,0 +1,56 @@ +name: Run benchmarks +description: Run benchmarks and collect results + +inputs: + scheduler: + type: string # A comma-separated list (no spaces!) is allowed. + required: true + runtime-ref: + type: string + required: true + + +outputs: + output-file: + description: "The CSV file summarizing all benchmark results" + value: ${{ steps.collect-results.outputs.output-file }} + + +runs: + using: "composite" + steps: + - name: Update LFC + run: | + echo $(ls) + echo "Checking out ${{ inputs.runtime-ref }}" + cd lf/org.lflang/src/lib/c/reactor-c && \ + git checkout ${{ inputs.runtime-ref }} && cd ../../../../.. + git checkout $(cat org.lflang/src/lib/c/reactor-c/lingua-franca-ref.txt) + echo $(ls) + ./gradlew buildLfc + cd .. + shell: bash + + - name: Run benchmarks + # TODO: Run all benchmarks with several numbers of threads + # run: | + # ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + # benchmark="glob(*)" target=lf-c size=fast \ + # target.params.scheduler=GEDF_NP,NP,adaptive threads=1,3,6,12,24 + run: | # DEBUG + ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + benchmark="savina_concurrency_cigsmok" target=lf-c size=fast \ + target.params.scheduler=${{ inputs.scheduler }} threads=1,3 + shell: bash + + - name: Collect benchmark results + id: collect-results + run: | + SHA=$(cd lf/org.lflang/src/lib/c/reactor-c && \ + git rev-parse --short=7 HEAD && cd ../../../../..) + BRANCH=$(git rev-parse --abbrev-ref HEAD) + OUTPUT_FILE=${{ inputs.scheduler }}-$BRANCH-$SHA.csv + echo "The collected benchmark results will be saved to $OUTPUT_FILE" + ./runner/collect_results.py latest $OUTPUT_FILE + echo "::set-output name=output-file::$OUTPUT_FILE" + shell: bash diff --git a/.github/actions/set-up-workspace/action.yml b/.github/actions/set-up-workspace/action.yml index 47b9ffbd8..4dd632398 100644 --- a/.github/actions/set-up-workspace/action.yml +++ b/.github/actions/set-up-workspace/action.yml @@ -14,27 +14,18 @@ runs: with: repository: lf-lang/lingua-franca path: lf + fetch-depth: 0 # We may need to check out older refs + submodules: recursive - name: Prepare LF build environment uses: ./lf/.github/actions/prepare-build-env - - name: Checkout current version of reactor-c - uses: actions/checkout@v2 - with: - path: lf/org.lflang/src/lib/c/reactor-c - - name: Install Python dependencies run: pip3 install -r runner/requirements.txt shell: bash - - name: Build lfc - run: | - cd lf - ./gradlew buildLfc - shell: bash - - name: Set LF_PATH and LF_BENCHMARKS_PATH environmental variable run: | echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - shell: bash \ No newline at end of file + shell: bash diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml index d8bf22a3b..cc799c0a4 100644 --- a/.github/workflows/user-requested-benchmark.yml +++ b/.github/workflows/user-requested-benchmark.yml @@ -5,10 +5,11 @@ on: types: [created] push: # TODO: Remove this when this is in the default branch. workflow_dispatch: - inputs: - pr_number: - required: true - type: number + +env: # TODO: Do not hard-code these! + PR_NUMBER: 91 + SCHEDULER: NP + FEATURE_BRANCH: automated-full-benchmark permissions: contents: write @@ -33,36 +34,37 @@ jobs: repository: lf-lang/reactor-c token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 # It will be necessary to push to this repo - ref: automated-full-benchmark # FIXME: delete this line after merge + ref: graphics path: reactor-c - name: Set up workspace uses: lf-lang/reactor-c/.github/actions/set-up-workspace@automated-full-benchmark - - name: Run C Benchmarks - # run: | - # ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ - # benchmark="glob(*)" target=lf-c size=fast \ - # target.params.scheduler=GEDF_NP,NP,adaptive threads=1,3,6,12,24 - run: | # DEBUG - ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ - benchmark="savina_parallelism_recmatmul" target=lf-c size=fast \ - target.params.scheduler=NP,adaptive threads=1,3,0 - shell: bash + - name: Run C Benchmarks (feature branch) + id: benchmarks-feature + uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark + with: + scheduler: ${{ env.SCHEDULER }} + runtime-ref: ${{ env.FEATURE_BRANCH }} - - name: Collect benchmark results - run: | - ./runner/collect_results.py latest latest-benchmark-results.csv - shell: bash + - name: Run C Benchmarks (main branch) + id: benchmarks-main + uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark + with: + scheduler: GEDF_NP,NP,adaptive + runtime-ref: main - name: Make figure run: | OUTPUT_DIR=images/$(date -I) + echo "Figure will be created in $OUTPUT_DIR" SHA=${{ github.sha }} - OUTPUT_FILE=$OUTPUT_DIR/${SHA:0:7}.png + OUTPUT_FILE=$OUTPUT_DIR/$SCHEDULER-$FEATURE_BRANCH-${SHA:0:7}.png + echo "Figure will be named $OUTPUT_FILE" mkdir -p reactor-c/$OUTPUT_DIR - ./runner/make-graphics.py latest-benchmark-results.csv reactor-c/$OUTPUT_FILE - echo "::set-output name=output_file::$OUTPUT_FILE" + ./runner/make-graphics.py ${{ steps.benchmarks-feature.outputs.output-file }} \ + ${{ steps.benchmarks-main.outputs.output-file }} reactor-c/$OUTPUT_FILE + echo "::set-output name=output-file::$OUTPUT_FILE" shell: bash id: figure @@ -72,8 +74,7 @@ jobs: git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git fetch - git switch graphics - git add ${{ steps.figure.outputs.OUTPUT_FILE }} + git add ${{ steps.figure.outputs.output-file }} git commit -m "benchmark ${{ github.sha }}" git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | \ xargs -L1 git config --unset-all @@ -85,6 +86,6 @@ jobs: uses: thollander/actions-comment-pull-request@v1 with: message: | - ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/reactor-c/graphics/${{ steps.figure.outputs.OUTPUT_FILE }}) - pr_number: 91 # ${{ inputs.pr_number }} + ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/reactor-c/graphics/${{ steps.figure.outputs.output-file }}) + pr_number: ${{ env.PR_NUMBER }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From c4d5a508426749c99b3cf5bb299526a06aeb33bd Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Thu, 14 Jul 2022 16:21:25 -0700 Subject: [PATCH 31/43] [benchmarks] Avoid redundant runs. --- .github/actions/run-benchmarks/action.yml | 41 ++++++++++++++----- .../workflows/user-requested-benchmark.yml | 10 +++-- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml index 87ab6bc52..7ee971712 100644 --- a/.github/actions/run-benchmarks/action.yml +++ b/.github/actions/run-benchmarks/action.yml @@ -13,23 +13,46 @@ inputs: outputs: output-file: description: "The CSV file summarizing all benchmark results" - value: ${{ steps.collect-results.outputs.output-file }} - + value: ${{ steps.compute-output-file.outputs.output-file }} + runs: using: "composite" steps: - - name: Update LFC + - name: Check out requested branches run: | echo $(ls) echo "Checking out ${{ inputs.runtime-ref }}" cd lf/org.lflang/src/lib/c/reactor-c && \ git checkout ${{ inputs.runtime-ref }} && cd ../../../../.. git checkout $(cat org.lflang/src/lib/c/reactor-c/lingua-franca-ref.txt) - echo $(ls) + cd .. + shell: bash + - name: Compute output file + id: compute-output-file + run: | + SHA=$(cd lf/org.lflang/src/lib/c/reactor-c && \ + git rev-parse --short=7 HEAD && cd ../../../../../..) + BRANCH=${{ inputs.runtime-ref }} + OUTPUT_DIR=csvs/$(date -I) + mkdir -p reactor-c/$OUTPUT_DIR + OUTPUT_FILE=$OUTPUT_DIR/${{ inputs.scheduler }}-$BRANCH-${SHA:0:7}.csv + echo "::set-output name=output-file::$OUTPUT_FILE" + if [ -f reactor-c/$OUTPUT_FILE ]; then + echo "The benchmark will not be run because results for that commit are already saved at $OUTPUT_FILE." + echo "::set-output name=file-already-exists::true" + else + echo "The collected benchmark results will be saved to $OUTPUT_FILE." + fi + shell: bash + + - name: Update LFC + run: | + cd lf ./gradlew buildLfc cd .. shell: bash + if: ${{ ! steps.compute-output-file.outputs.file-already-exists }} - name: Run benchmarks # TODO: Run all benchmarks with several numbers of threads @@ -42,15 +65,11 @@ runs: benchmark="savina_concurrency_cigsmok" target=lf-c size=fast \ target.params.scheduler=${{ inputs.scheduler }} threads=1,3 shell: bash + if: ${{ ! steps.compute-output-file.outputs.file-already-exists }} - name: Collect benchmark results id: collect-results run: | - SHA=$(cd lf/org.lflang/src/lib/c/reactor-c && \ - git rev-parse --short=7 HEAD && cd ../../../../..) - BRANCH=$(git rev-parse --abbrev-ref HEAD) - OUTPUT_FILE=${{ inputs.scheduler }}-$BRANCH-$SHA.csv - echo "The collected benchmark results will be saved to $OUTPUT_FILE" - ./runner/collect_results.py latest $OUTPUT_FILE - echo "::set-output name=output-file::$OUTPUT_FILE" + ./runner/collect_results.py latest reactor-c/${{ steps.compute-output-file.outputs.output-file }} shell: bash + if: ${{ ! steps.compute-output-file.outputs.file-already-exists }} diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml index cc799c0a4..8330dcd2c 100644 --- a/.github/workflows/user-requested-benchmark.yml +++ b/.github/workflows/user-requested-benchmark.yml @@ -26,7 +26,7 @@ jobs: repository: lf-lang/benchmarks-lingua-franca token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 1 - ref: automated-full-benchmark # FIXME: delete this line after merge + ref: automated-full-benchmark # FIXME: delete this line after merge - name: Checkout reactor-c uses: actions/checkout@v2 @@ -62,19 +62,21 @@ jobs: OUTPUT_FILE=$OUTPUT_DIR/$SCHEDULER-$FEATURE_BRANCH-${SHA:0:7}.png echo "Figure will be named $OUTPUT_FILE" mkdir -p reactor-c/$OUTPUT_DIR - ./runner/make-graphics.py ${{ steps.benchmarks-feature.outputs.output-file }} \ - ${{ steps.benchmarks-main.outputs.output-file }} reactor-c/$OUTPUT_FILE + ./runner/make-graphics.py reactor-c/${{ steps.benchmarks-feature.outputs.output-file }} \ + reactor-c/${{ steps.benchmarks-main.outputs.output-file }} reactor-c/$OUTPUT_FILE echo "::set-output name=output-file::$OUTPUT_FILE" shell: bash id: figure - - name: Commit figure + - name: Commit figure and output files run: | cd reactor-c git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git fetch git add ${{ steps.figure.outputs.output-file }} + git add ${{ steps.benchmarks-feature.outputs.output-file }} + git add ${{ steps.benchmarks-main.outputs.output-file }} git commit -m "benchmark ${{ github.sha }}" git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | \ xargs -L1 git config --unset-all From d5bb7e92d326a3dcd3d8b3c60542752b793985f0 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Thu, 14 Jul 2022 18:25:18 -0700 Subject: [PATCH 32/43] [benchmarks] Integrate C++ benchmark runs. --- .github/actions/run-benchmarks/action.yml | 16 +++++++++++----- .github/workflows/user-requested-benchmark.yml | 16 ++++++++++++++-- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml index 7ee971712..e870a6729 100644 --- a/.github/actions/run-benchmarks/action.yml +++ b/.github/actions/run-benchmarks/action.yml @@ -8,6 +8,9 @@ inputs: runtime-ref: type: string required: true + target: + type: string + default: c # As in, "reactor-c" outputs: @@ -20,18 +23,21 @@ runs: using: "composite" steps: - name: Check out requested branches + id: check-out run: | echo $(ls) echo "Checking out ${{ inputs.runtime-ref }}" - cd lf/org.lflang/src/lib/c/reactor-c && \ + SUBMODULE_DIR=org.lflang/src/lib/${{ inputs.target }}/reactor-${{ inputs.target }} + cd lf/$SUBMODULE_DIR && \ git checkout ${{ inputs.runtime-ref }} && cd ../../../../.. - git checkout $(cat org.lflang/src/lib/c/reactor-c/lingua-franca-ref.txt) + git checkout $(cat $SUBMODULE_DIR/lingua-franca-ref.txt) + echo "::set-output name=submodule-dir::$SUBMODULE_DIR" cd .. shell: bash - name: Compute output file id: compute-output-file run: | - SHA=$(cd lf/org.lflang/src/lib/c/reactor-c && \ + SHA=$(cd lf/${{ steps.check-out.outputs.submodule-dir }} && \ git rev-parse --short=7 HEAD && cd ../../../../../..) BRANCH=${{ inputs.runtime-ref }} OUTPUT_DIR=csvs/$(date -I) @@ -62,8 +68,8 @@ runs: # target.params.scheduler=GEDF_NP,NP,adaptive threads=1,3,6,12,24 run: | # DEBUG ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ - benchmark="savina_concurrency_cigsmok" target=lf-c size=fast \ - target.params.scheduler=${{ inputs.scheduler }} threads=1,3 + benchmark="savina_concurrency_cigsmok" target=lf-${{ inputs.target }} size=fast \ + ++target.params.scheduler=${{ inputs.scheduler }} threads=1,3 shell: bash if: ${{ ! steps.compute-output-file.outputs.file-already-exists }} diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml index 8330dcd2c..c5b7685e1 100644 --- a/.github/workflows/user-requested-benchmark.yml +++ b/.github/workflows/user-requested-benchmark.yml @@ -54,6 +54,14 @@ jobs: scheduler: GEDF_NP,NP,adaptive runtime-ref: main + - name: Run C++ Benchmarks (main branch) + id: benchmarks-main-cpp + uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark + with: + scheduler: default + runtime-ref: master + target: cpp + - name: Make figure run: | OUTPUT_DIR=images/$(date -I) @@ -62,8 +70,11 @@ jobs: OUTPUT_FILE=$OUTPUT_DIR/$SCHEDULER-$FEATURE_BRANCH-${SHA:0:7}.png echo "Figure will be named $OUTPUT_FILE" mkdir -p reactor-c/$OUTPUT_DIR - ./runner/make-graphics.py reactor-c/${{ steps.benchmarks-feature.outputs.output-file }} \ - reactor-c/${{ steps.benchmarks-main.outputs.output-file }} reactor-c/$OUTPUT_FILE + ./runner/make-graphics.py \ + reactor-c/${{ steps.benchmarks-feature.outputs.output-file }} \ + reactor-c/${{ steps.benchmarks-main.outputs.output-file }} \ + reactor-c/${{ steps.benchmarks-main-cpp.outputs.output-file }} \ + reactor-c/$OUTPUT_FILE echo "::set-output name=output-file::$OUTPUT_FILE" shell: bash id: figure @@ -77,6 +88,7 @@ jobs: git add ${{ steps.figure.outputs.output-file }} git add ${{ steps.benchmarks-feature.outputs.output-file }} git add ${{ steps.benchmarks-main.outputs.output-file }} + git add ${{ steps.benchmarks-main-cpp.outputs.output-file }} git commit -m "benchmark ${{ github.sha }}" git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | \ xargs -L1 git config --unset-all From 409d9eeef97dc4af8ec2f06d3d17f31ea2d41c86 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Thu, 14 Jul 2022 20:29:27 -0700 Subject: [PATCH 33/43] [benchmarks] Update workflow. --- .github/workflows/continuous-benchmark.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 80e2734cb..789c5c803 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -23,10 +23,10 @@ jobs: uses: actions/checkout@v2 with: repository: lf-lang/benchmarks-lingua-franca - ref: automated-full-benchmark # FIXME: delete this line after merge + ref: automated-full-benchmark # FIXME: delete this line before merge - name: Set up workspace - uses: ./.github/actions/set-up-workspace + uses: lf-lang/reactor-c/.github/actions/set-up-workspace@automated-full-benchmark - name: Run C Benchmarks (multithreaded) run: | From 13da0ae0346f2a9c25c02904165480ae85e87ebf Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Thu, 14 Jul 2022 21:29:04 -0700 Subject: [PATCH 34/43] [benchmarks] Refactor workflow. --- .../action.yml | 50 +++++++++++++++++++ .github/actions/run-benchmarks/action.yml | 1 + .github/workflows/continuous-benchmark.yml | 7 +++ .../workflows/user-requested-benchmark.yml | 37 ++++---------- 4 files changed, 67 insertions(+), 28 deletions(-) create mode 100644 .github/actions/run-benchmarks-all-runtime-versions/action.yml diff --git a/.github/actions/run-benchmarks-all-runtime-versions/action.yml b/.github/actions/run-benchmarks-all-runtime-versions/action.yml new file mode 100644 index 000000000..e09ca01e6 --- /dev/null +++ b/.github/actions/run-benchmarks-all-runtime-versions/action.yml @@ -0,0 +1,50 @@ +name: Run benchmarks for all runtime versions + +inputs: + scheduler: + type: string # A comma-separated list (no spaces!) is allowed. + required: true + feature-branch: + type: string + required: true + + +outputs: + output-files: + description: "The CSV files summarizing all benchmark results, separated by spaces" + value: ${{ steps.list-output-files.outputs.output-files }} + + +runs: + using: "composite" + steps: + - name: Run C Benchmarks (feature branch) + id: benchmarks-feature + uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark + with: + scheduler: ${{ inputs.scheduler }} + runtime-ref: ${{ inputs.feature-branch }} + + - name: Run C Benchmarks (main branch) + id: benchmarks-main + uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark + with: + scheduler: GEDF_NP,NP,adaptive + runtime-ref: main + + - name: Run C++ Benchmarks (main branch) + id: benchmarks-main-cpp + uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark + with: + scheduler: default + runtime-ref: master + target: cpp + + - name: List Output Files + id: list-output-files + run: | + echo "::set-output name=output-files::\ + ${{ steps.benchmarks-feature.outputs.output-file }} \ + ${{ steps.benchmarks-main.outputs.output-file }} \ + ${{ steps.benchmarks-main-cpp.outputs.output-file }}" + shell: bash diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml index e870a6729..9bf00ccb3 100644 --- a/.github/actions/run-benchmarks/action.yml +++ b/.github/actions/run-benchmarks/action.yml @@ -34,6 +34,7 @@ runs: echo "::set-output name=submodule-dir::$SUBMODULE_DIR" cd .. shell: bash + - name: Compute output file id: compute-output-file run: | diff --git a/.github/workflows/continuous-benchmark.yml b/.github/workflows/continuous-benchmark.yml index 789c5c803..997d8e595 100644 --- a/.github/workflows/continuous-benchmark.yml +++ b/.github/workflows/continuous-benchmark.yml @@ -28,6 +28,13 @@ jobs: - name: Set up workspace uses: lf-lang/reactor-c/.github/actions/set-up-workspace@automated-full-benchmark + - name: Update LFC + run: | + cd lf + ./gradlew buildLfc + cd .. + shell: bash + - name: Run C Benchmarks (multithreaded) run: | python3 runner/run_benchmark.py -m continue_on_error=True iterations=12 \ diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml index c5b7685e1..3f3d23c73 100644 --- a/.github/workflows/user-requested-benchmark.yml +++ b/.github/workflows/user-requested-benchmark.yml @@ -40,27 +40,12 @@ jobs: - name: Set up workspace uses: lf-lang/reactor-c/.github/actions/set-up-workspace@automated-full-benchmark - - name: Run C Benchmarks (feature branch) - id: benchmarks-feature - uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark + - name: Run Benchmarks + id: run-benchmarks + uses: lf-lang/reactor-c/.github/actions/run-benchmarks-all-runtime-versions@automated-full-benchmark with: scheduler: ${{ env.SCHEDULER }} - runtime-ref: ${{ env.FEATURE_BRANCH }} - - - name: Run C Benchmarks (main branch) - id: benchmarks-main - uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark - with: - scheduler: GEDF_NP,NP,adaptive - runtime-ref: main - - - name: Run C++ Benchmarks (main branch) - id: benchmarks-main-cpp - uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark - with: - scheduler: default - runtime-ref: master - target: cpp + feature-branch: ${{ env.FEATURE_BRANCH }} - name: Make figure run: | @@ -70,11 +55,9 @@ jobs: OUTPUT_FILE=$OUTPUT_DIR/$SCHEDULER-$FEATURE_BRANCH-${SHA:0:7}.png echo "Figure will be named $OUTPUT_FILE" mkdir -p reactor-c/$OUTPUT_DIR - ./runner/make-graphics.py \ - reactor-c/${{ steps.benchmarks-feature.outputs.output-file }} \ - reactor-c/${{ steps.benchmarks-main.outputs.output-file }} \ - reactor-c/${{ steps.benchmarks-main-cpp.outputs.output-file }} \ - reactor-c/$OUTPUT_FILE + cd reactor-c + ../runner/make-graphics.py ${{ steps.run-benchmarks.outputs.output-files }} $OUTPUT_FILE + cd .. echo "::set-output name=output-file::$OUTPUT_FILE" shell: bash id: figure @@ -85,10 +68,8 @@ jobs: git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git fetch - git add ${{ steps.figure.outputs.output-file }} - git add ${{ steps.benchmarks-feature.outputs.output-file }} - git add ${{ steps.benchmarks-main.outputs.output-file }} - git add ${{ steps.benchmarks-main-cpp.outputs.output-file }} + git add ${{ steps.run-benchmarks.outputs.output-files }} \ + ${{ steps.figure.outputs.output-file }} git commit -m "benchmark ${{ github.sha }}" git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | \ xargs -L1 git config --unset-all From 5bc22535318859732b34cf9eb65ff7277742b4de Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Thu, 14 Jul 2022 22:13:23 -0700 Subject: [PATCH 35/43] [benchmarks] Refactor workflow. --- .../actions/visualize-save-upload/action.yml | 50 +++++++++++++++++++ .../workflows/user-requested-benchmark.yml | 40 ++------------- 2 files changed, 54 insertions(+), 36 deletions(-) create mode 100644 .github/actions/visualize-save-upload/action.yml diff --git a/.github/actions/visualize-save-upload/action.yml b/.github/actions/visualize-save-upload/action.yml new file mode 100644 index 000000000..16901468d --- /dev/null +++ b/.github/actions/visualize-save-upload/action.yml @@ -0,0 +1,50 @@ +name: Visualize, Save, and Upload Benchmark Results + +inputs: + csv-files: + type: string + required: true + token: + type: string + description: Personal access token # FIXME: This should be truly a PAT, not secrets.GITHUB_TOKEN + required: true + +runs: + using: "composite" + steps: + - name: Make figure + run: | + OUTPUT_DIR=images/$(date -I) + echo "Figure will be created in $OUTPUT_DIR" + SHA=${{ github.sha }} + OUTPUT_FILE=$OUTPUT_DIR/$SCHEDULER-$FEATURE_BRANCH-${SHA:0:7}.png + echo "Figure will be named $OUTPUT_FILE" + mkdir -p reactor-c/$OUTPUT_DIR + cd reactor-c + ../runner/make-graphics.py ${{ inputs.csv-files }} $OUTPUT_FILE + cd .. + echo "::set-output name=output-file::$OUTPUT_FILE" + shell: bash + id: figure + + - name: Commit figure and output files + run: | + cd reactor-c + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --global user.name "github-actions[bot]" + git fetch + git add ${{ inputs.csv-files }} ${{ steps.figure.outputs.output-file }} + git commit -m "benchmark ${{ github.sha }}" + git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | \ + xargs -L1 git config --unset-all + git push https://token:${{ inputs.token }}@github.com/lf-lang/reactor-c.git + cd .. + shell: bash + + - name: Comment PR + uses: thollander/actions-comment-pull-request@v1 + with: + message: | + ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/reactor-c/graphics/${{ steps.figure.outputs.output-file }}) + pr_number: ${{ env.PR_NUMBER }} + GITHUB_TOKEN: ${{ inputs.token }} diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml index 3f3d23c73..754097f58 100644 --- a/.github/workflows/user-requested-benchmark.yml +++ b/.github/workflows/user-requested-benchmark.yml @@ -47,40 +47,8 @@ jobs: scheduler: ${{ env.SCHEDULER }} feature-branch: ${{ env.FEATURE_BRANCH }} - - name: Make figure - run: | - OUTPUT_DIR=images/$(date -I) - echo "Figure will be created in $OUTPUT_DIR" - SHA=${{ github.sha }} - OUTPUT_FILE=$OUTPUT_DIR/$SCHEDULER-$FEATURE_BRANCH-${SHA:0:7}.png - echo "Figure will be named $OUTPUT_FILE" - mkdir -p reactor-c/$OUTPUT_DIR - cd reactor-c - ../runner/make-graphics.py ${{ steps.run-benchmarks.outputs.output-files }} $OUTPUT_FILE - cd .. - echo "::set-output name=output-file::$OUTPUT_FILE" - shell: bash - id: figure - - - name: Commit figure and output files - run: | - cd reactor-c - git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --global user.name "github-actions[bot]" - git fetch - git add ${{ steps.run-benchmarks.outputs.output-files }} \ - ${{ steps.figure.outputs.output-file }} - git commit -m "benchmark ${{ github.sha }}" - git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | \ - xargs -L1 git config --unset-all - git push https://token:${{ secrets.GITHUB_TOKEN }}@github.com/lf-lang/reactor-c.git - cd .. - shell: bash - - - name: Comment PR - uses: thollander/actions-comment-pull-request@v1 + - name: Visualize, Save, and Upload + uses: lf-lang/reactor-c/.github/actions/visualize-save-upload@automated-full-benchmark with: - message: | - ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/reactor-c/graphics/${{ steps.figure.outputs.output-file }}) - pr_number: ${{ env.PR_NUMBER }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + csv-files: ${{ steps.run-benchmarks.outputs.output-files }} + token: ${{ secrets.GITHUB_TOKEN }} # FIXME: Should be a PAT From c4c0e0be755bb4c48ccc27576a18e28bf8c45e5a Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Thu, 14 Jul 2022 22:35:47 -0700 Subject: [PATCH 36/43] [benchmarks] Report partial benchmark progress. --- .../action.yml | 9 +++++ .github/actions/run-benchmarks/action.yml | 18 ++++----- .../actions/visualize-save-upload/action.yml | 30 ++++++++++++-- .../workflows/user-requested-benchmark.yml | 39 +++++++++++++++++-- 4 files changed, 80 insertions(+), 16 deletions(-) diff --git a/.github/actions/run-benchmarks-all-runtime-versions/action.yml b/.github/actions/run-benchmarks-all-runtime-versions/action.yml index e09ca01e6..d3b9cded7 100644 --- a/.github/actions/run-benchmarks-all-runtime-versions/action.yml +++ b/.github/actions/run-benchmarks-all-runtime-versions/action.yml @@ -7,6 +7,10 @@ inputs: feature-branch: type: string required: true + num-workers: + type: string # Provide a comma-separated list without spaces. + description: The number of workers to be used by the scheduler. + required: true outputs: @@ -24,6 +28,8 @@ runs: with: scheduler: ${{ inputs.scheduler }} runtime-ref: ${{ inputs.feature-branch }} + target: c + num-workers: ${{ inputs.num-workers }} - name: Run C Benchmarks (main branch) id: benchmarks-main @@ -31,6 +37,8 @@ runs: with: scheduler: GEDF_NP,NP,adaptive runtime-ref: main + target: c + num-workers: ${{ inputs.num-workers }} - name: Run C++ Benchmarks (main branch) id: benchmarks-main-cpp @@ -39,6 +47,7 @@ runs: scheduler: default runtime-ref: master target: cpp + num-workers: ${{ inputs.num-workers }} - name: List Output Files id: list-output-files diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml index 9bf00ccb3..7cdb38904 100644 --- a/.github/actions/run-benchmarks/action.yml +++ b/.github/actions/run-benchmarks/action.yml @@ -10,7 +10,10 @@ inputs: required: true target: type: string - default: c # As in, "reactor-c" + num-workers: + type: string # Provide a comma-separated list without spaces. + description: The number of workers to be used by the scheduler. + required: true outputs: @@ -43,7 +46,7 @@ runs: BRANCH=${{ inputs.runtime-ref }} OUTPUT_DIR=csvs/$(date -I) mkdir -p reactor-c/$OUTPUT_DIR - OUTPUT_FILE=$OUTPUT_DIR/${{ inputs.scheduler }}-$BRANCH-${SHA:0:7}.csv + OUTPUT_FILE=$OUTPUT_DIR/${{ inputs.scheduler }}-${{ inputs.num-workers }}-$BRANCH-${SHA:0:7}.csv echo "::set-output name=output-file::$OUTPUT_FILE" if [ -f reactor-c/$OUTPUT_FILE ]; then echo "The benchmark will not be run because results for that commit are already saved at $OUTPUT_FILE." @@ -62,15 +65,10 @@ runs: if: ${{ ! steps.compute-output-file.outputs.file-already-exists }} - name: Run benchmarks - # TODO: Run all benchmarks with several numbers of threads - # run: | - # ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ - # benchmark="glob(*)" target=lf-c size=fast \ - # target.params.scheduler=GEDF_NP,NP,adaptive threads=1,3,6,12,24 - run: | # DEBUG + run: | ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ - benchmark="savina_concurrency_cigsmok" target=lf-${{ inputs.target }} size=fast \ - ++target.params.scheduler=${{ inputs.scheduler }} threads=1,3 + benchmark="glob(*)" target=lf-${{ inputs.target }} size=fast \ + ++target.params.scheduler=${{ inputs.scheduler }} threads=${{ inputs.num-workers }} shell: bash if: ${{ ! steps.compute-output-file.outputs.file-already-exists }} diff --git a/.github/actions/visualize-save-upload/action.yml b/.github/actions/visualize-save-upload/action.yml index 16901468d..21b0058f1 100644 --- a/.github/actions/visualize-save-upload/action.yml +++ b/.github/actions/visualize-save-upload/action.yml @@ -8,6 +8,14 @@ inputs: type: string description: Personal access token # FIXME: This should be truly a PAT, not secrets.GITHUB_TOKEN required: true + id: + type: string + description: String that can be used in a file name to distinguish this from other similar runs. + required: true + finalize: + description: Whether this is the last run of this workflow to edit the same comment. + type: boolean + default: false runs: using: "composite" @@ -17,7 +25,7 @@ runs: OUTPUT_DIR=images/$(date -I) echo "Figure will be created in $OUTPUT_DIR" SHA=${{ github.sha }} - OUTPUT_FILE=$OUTPUT_DIR/$SCHEDULER-$FEATURE_BRANCH-${SHA:0:7}.png + OUTPUT_FILE=$OUTPUT_DIR/$SCHEDULER-$FEATURE_BRANCH-${{ inputs.id }}-${SHA:0:7}.png echo "Figure will be named $OUTPUT_FILE" mkdir -p reactor-c/$OUTPUT_DIR cd reactor-c @@ -35,8 +43,6 @@ runs: git fetch git add ${{ inputs.csv-files }} ${{ steps.figure.outputs.output-file }} git commit -m "benchmark ${{ github.sha }}" - git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | \ - xargs -L1 git config --unset-all git push https://token:${{ inputs.token }}@github.com/lf-lang/reactor-c.git cd .. shell: bash @@ -45,6 +51,24 @@ runs: uses: thollander/actions-comment-pull-request@v1 with: message: | + Running benchmarks from commit ${{ github.sha }}... + + Results so far: + + ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/reactor-c/graphics/${{ steps.figure.outputs.output-file }}) + comment_includes: ${{ github.sha }} + pr_number: ${{ env.PR_NUMBER }} + GITHUB_TOKEN: ${{ inputs.token }} + if: ${{ inputs.finalize == 'false' }} + + - name: Finalize Comment + uses: thollander/actions-comment-pull-request@v1 + with: + message: | + :heavy_check_mark: Finished running benchmarks from ${{ github.sha }}. + ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/reactor-c/graphics/${{ steps.figure.outputs.output-file }}) + comment_includes: ${{ github.sha }} pr_number: ${{ env.PR_NUMBER }} GITHUB_TOKEN: ${{ inputs.token }} + if: ${{ inputs.finalize == 'true' }} diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml index 754097f58..e97757a56 100644 --- a/.github/workflows/user-requested-benchmark.yml +++ b/.github/workflows/user-requested-benchmark.yml @@ -40,15 +40,48 @@ jobs: - name: Set up workspace uses: lf-lang/reactor-c/.github/actions/set-up-workspace@automated-full-benchmark - - name: Run Benchmarks - id: run-benchmarks + - name: Run Benchmarks Part 1 + id: run-benchmarks-1 uses: lf-lang/reactor-c/.github/actions/run-benchmarks-all-runtime-versions@automated-full-benchmark with: scheduler: ${{ env.SCHEDULER }} feature-branch: ${{ env.FEATURE_BRANCH }} + num-workers: 1,12 - name: Visualize, Save, and Upload uses: lf-lang/reactor-c/.github/actions/visualize-save-upload@automated-full-benchmark with: - csv-files: ${{ steps.run-benchmarks.outputs.output-files }} + csv-files: ${{ steps.run-benchmarks-1.outputs.output-files }} token: ${{ secrets.GITHUB_TOKEN }} # FIXME: Should be a PAT + id: 1 + + - name: Run Benchmarks Part 2 + id: run-benchmarks-2 + uses: lf-lang/reactor-c/.github/actions/run-benchmarks-all-runtime-versions@automated-full-benchmark + with: + scheduler: ${{ env.SCHEDULER }} + feature-branch: ${{ env.FEATURE_BRANCH }} + num-workers: 3,6,24 + + - name: Visualize, Save, and Upload + uses: lf-lang/reactor-c/.github/actions/visualize-save-upload@automated-full-benchmark + with: + csv-files: "${{ steps.run-benchmarks-1.outputs.output-files }} ${{ steps.run-benchmarks-2.outputs.output-files }}" + token: ${{ secrets.GITHUB_TOKEN }} # FIXME: Should be a PAT + id: 2 + + - name: Run Benchmarks Part 3 + id: run-benchmarks-3 + uses: lf-lang/reactor-c/.github/actions/run-benchmarks-all-runtime-versions@automated-full-benchmark + with: + scheduler: ${{ env.SCHEDULER }} + feature-branch: ${{ env.FEATURE_BRANCH }} + num-workers: 2,4,9,18 + + - name: Visualize, Save, and Upload + uses: lf-lang/reactor-c/.github/actions/visualize-save-upload@automated-full-benchmark + with: + csv-files: "${{ steps.run-benchmarks-1.outputs.output-files }} ${{ steps.run-benchmarks-2.outputs.output-files }} ${{ steps.run-benchmarks-3.outputs.output-files }}" + token: ${{ secrets.GITHUB_TOKEN }} # FIXME: Should be a PAT + id: 3 + finalize: true From a3a753e40153447ab33084f421d8a791cd2851b7 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Fri, 15 Jul 2022 13:37:38 -0700 Subject: [PATCH 37/43] [benchmarks] Allow re-running of workflow. --- .github/actions/visualize-save-upload/action.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/visualize-save-upload/action.yml b/.github/actions/visualize-save-upload/action.yml index 21b0058f1..008add353 100644 --- a/.github/actions/visualize-save-upload/action.yml +++ b/.github/actions/visualize-save-upload/action.yml @@ -42,8 +42,10 @@ runs: git config --global user.name "github-actions[bot]" git fetch git add ${{ inputs.csv-files }} ${{ steps.figure.outputs.output-file }} - git commit -m "benchmark ${{ github.sha }}" - git push https://token:${{ inputs.token }}@github.com/lf-lang/reactor-c.git + if [[ $(git status) != *"nothing to commit"* ]]; then + git commit -m "benchmark ${{ github.sha }}" + git push https://token:${{ inputs.token }}@github.com/lf-lang/reactor-c.git + fi cd .. shell: bash From 9400549d9948126ce8b8210677836c2defb1106a Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Fri, 15 Jul 2022 14:53:34 -0700 Subject: [PATCH 38/43] [benchmarks] Set timeout. This timeout is _much_ lower than what we previously had to use. This is because of the size=fast parameter which cuts down the problem size for benchmarks that take unreasonably long to execute (e.g., SortedLinkedList). It is also an order of magnitude greater than the max execution time using the unthreaded C runtime, so any implementation that times out is unacceptably slow. --- .github/actions/run-benchmarks/action.yml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml index 7cdb38904..8c062fac2 100644 --- a/.github/actions/run-benchmarks/action.yml +++ b/.github/actions/run-benchmarks/action.yml @@ -14,6 +14,11 @@ inputs: type: string # Provide a comma-separated list without spaces. description: The number of workers to be used by the scheduler. required: true + iterations: + type: number + description: The number of iterations with which to run each benchmark. + required: false + default: 12 outputs: @@ -66,9 +71,17 @@ runs: - name: Run benchmarks run: | - ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ - benchmark="glob(*)" target=lf-${{ inputs.target }} size=fast \ - ++target.params.scheduler=${{ inputs.scheduler }} threads=${{ inputs.num-workers }} + # Allow at most 1 second per iteration. This will result in timeouts on runtime versions + # that are an order of magnitude slower than the unthreaded C runtime. + ./runner/run_benchmark.py -m \ + continue_on_error=True \ + timeout=${{ inputs.iterations }} \ + iterations=${{ inputs.iterations }} \ + benchmark="glob(*)" \ + target=lf-${{ inputs.target }} \ + size=fast \ + ++target.params.scheduler=${{ inputs.scheduler }} \ + threads=${{ inputs.num-workers }} shell: bash if: ${{ ! steps.compute-output-file.outputs.file-already-exists }} From 9c79b1150c3fa2d59ef11af2078a4e4b87efa442 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Fri, 15 Jul 2022 15:39:34 -0700 Subject: [PATCH 39/43] [benchmarks] Push to benchmarks repo. Do not push to reactor-c. --- .../actions/visualize-save-upload/action.yml | 19 ++++++++++++------- .../workflows/user-requested-benchmark.yml | 16 ++++++++-------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/actions/visualize-save-upload/action.yml b/.github/actions/visualize-save-upload/action.yml index 008add353..677e4fce4 100644 --- a/.github/actions/visualize-save-upload/action.yml +++ b/.github/actions/visualize-save-upload/action.yml @@ -6,7 +6,7 @@ inputs: required: true token: type: string - description: Personal access token # FIXME: This should be truly a PAT, not secrets.GITHUB_TOKEN + description: Personal access token required: true id: type: string @@ -16,6 +16,11 @@ inputs: description: Whether this is the last run of this workflow to edit the same comment. type: boolean default: false + repo-for-saving-data: + description: | + The directory name of the repository where graphs and CSVs from benchmarks should be saved. + type: string + default: benchmarks-lingua-franca runs: using: "composite" @@ -27,8 +32,8 @@ runs: SHA=${{ github.sha }} OUTPUT_FILE=$OUTPUT_DIR/$SCHEDULER-$FEATURE_BRANCH-${{ inputs.id }}-${SHA:0:7}.png echo "Figure will be named $OUTPUT_FILE" - mkdir -p reactor-c/$OUTPUT_DIR - cd reactor-c + mkdir -p ${{ inputs.repo-for-saving-data }}/$OUTPUT_DIR + cd ${{ inputs.repo-for-saving-data }} ../runner/make-graphics.py ${{ inputs.csv-files }} $OUTPUT_FILE cd .. echo "::set-output name=output-file::$OUTPUT_FILE" @@ -37,14 +42,14 @@ runs: - name: Commit figure and output files run: | - cd reactor-c + cd ${{ inputs.repo-for-saving-data }} git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git fetch git add ${{ inputs.csv-files }} ${{ steps.figure.outputs.output-file }} if [[ $(git status) != *"nothing to commit"* ]]; then git commit -m "benchmark ${{ github.sha }}" - git push https://token:${{ inputs.token }}@github.com/lf-lang/reactor-c.git + git push https://token:${{ inputs.token }}@github.com/lf-lang/${{ inputs.repo-for-saving-data }}.git fi cd .. shell: bash @@ -57,7 +62,7 @@ runs: Results so far: - ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/reactor-c/graphics/${{ steps.figure.outputs.output-file }}) + ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/${{ inputs.repo-for-saving-data }}/saved-benchmark-results/${{ steps.figure.outputs.output-file }}) comment_includes: ${{ github.sha }} pr_number: ${{ env.PR_NUMBER }} GITHUB_TOKEN: ${{ inputs.token }} @@ -69,7 +74,7 @@ runs: message: | :heavy_check_mark: Finished running benchmarks from ${{ github.sha }}. - ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/reactor-c/graphics/${{ steps.figure.outputs.output-file }}) + ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/${{ inputs.repo-for-saving-data }}/saved-benchmark-results/${{ steps.figure.outputs.output-file }}) comment_includes: ${{ github.sha }} pr_number: ${{ env.PR_NUMBER }} GITHUB_TOKEN: ${{ inputs.token }} diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml index e97757a56..976f31944 100644 --- a/.github/workflows/user-requested-benchmark.yml +++ b/.github/workflows/user-requested-benchmark.yml @@ -28,14 +28,14 @@ jobs: fetch-depth: 1 ref: automated-full-benchmark # FIXME: delete this line after merge - - name: Checkout reactor-c + - name: Checkout benchmarks-lingua-franca uses: actions/checkout@v2 with: - repository: lf-lang/reactor-c - token: ${{ secrets.GITHUB_TOKEN }} + repository: lf-lang/benchmarks-lingua-franca + token: ${{ secrets.PAT_FOR_PUSHING_BENCHMARK_RESULTS }} fetch-depth: 0 # It will be necessary to push to this repo - ref: graphics - path: reactor-c + ref: saved-benchmark-results + path: benchmarks-lingua-franca - name: Set up workspace uses: lf-lang/reactor-c/.github/actions/set-up-workspace@automated-full-benchmark @@ -52,7 +52,7 @@ jobs: uses: lf-lang/reactor-c/.github/actions/visualize-save-upload@automated-full-benchmark with: csv-files: ${{ steps.run-benchmarks-1.outputs.output-files }} - token: ${{ secrets.GITHUB_TOKEN }} # FIXME: Should be a PAT + token: ${{ secrets.PAT_FOR_PUSHING_BENCHMARK_RESULTS }} id: 1 - name: Run Benchmarks Part 2 @@ -67,7 +67,7 @@ jobs: uses: lf-lang/reactor-c/.github/actions/visualize-save-upload@automated-full-benchmark with: csv-files: "${{ steps.run-benchmarks-1.outputs.output-files }} ${{ steps.run-benchmarks-2.outputs.output-files }}" - token: ${{ secrets.GITHUB_TOKEN }} # FIXME: Should be a PAT + token: ${{ secrets.PAT_FOR_PUSHING_BENCHMARK_RESULTS }} id: 2 - name: Run Benchmarks Part 3 @@ -82,6 +82,6 @@ jobs: uses: lf-lang/reactor-c/.github/actions/visualize-save-upload@automated-full-benchmark with: csv-files: "${{ steps.run-benchmarks-1.outputs.output-files }} ${{ steps.run-benchmarks-2.outputs.output-files }} ${{ steps.run-benchmarks-3.outputs.output-files }}" - token: ${{ secrets.GITHUB_TOKEN }} # FIXME: Should be a PAT + token: ${{ secrets.PAT_FOR_PUSHING_BENCHMARK_RESULTS }} id: 3 finalize: true From 504924f749c450bd8f43a3827003a876a26f72b2 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Fri, 15 Jul 2022 16:00:14 -0700 Subject: [PATCH 40/43] [benchmarks] Re-use old data that is not outdated. --- .github/actions/run-benchmarks/action.yml | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml index 8c062fac2..d6089fb7a 100644 --- a/.github/actions/run-benchmarks/action.yml +++ b/.github/actions/run-benchmarks/action.yml @@ -19,6 +19,11 @@ inputs: description: The number of iterations with which to run each benchmark. required: false default: 12 + repo-for-saving-data: + description: | + The directory name of the repository where graphs and CSVs from benchmarks should be saved. + type: string + default: benchmarks-lingua-franca outputs: @@ -48,16 +53,18 @@ runs: run: | SHA=$(cd lf/${{ steps.check-out.outputs.submodule-dir }} && \ git rev-parse --short=7 HEAD && cd ../../../../../..) - BRANCH=${{ inputs.runtime-ref }} OUTPUT_DIR=csvs/$(date -I) - mkdir -p reactor-c/$OUTPUT_DIR - OUTPUT_FILE=$OUTPUT_DIR/${{ inputs.scheduler }}-${{ inputs.num-workers }}-$BRANCH-${SHA:0:7}.csv - echo "::set-output name=output-file::$OUTPUT_FILE" - if [ -f reactor-c/$OUTPUT_FILE ]; then + mkdir -p ${{ inputs.repo-for-saving-data }}/csvs + OUTPUT_FILENAME=${{ inputs.scheduler }}-${{ inputs.num-workers }}-${{ inputs.runtime-ref }}-${SHA:0:7}.csv + EXISTING_FILE=$(find ${{ inputs.repo-for-saving-data }}/csvs -name $OUTPUT_FILENAME | head -n 1) + if [[ -z $EXISTING_FILE ]]; then + mkdir -p ${{ inputs.repo-for-saving-data }}/$OUTPUT_DIR + echo "The collected benchmark results will be saved to $OUTPUT_FILE." + echo "::set-output name=output-file::$OUTPUT_DIR/$OUTPUT_FILENAME" + else echo "The benchmark will not be run because results for that commit are already saved at $OUTPUT_FILE." echo "::set-output name=file-already-exists::true" - else - echo "The collected benchmark results will be saved to $OUTPUT_FILE." + echo "::set-output name=output-file::$EXISTING_FILE" fi shell: bash @@ -88,6 +95,6 @@ runs: - name: Collect benchmark results id: collect-results run: | - ./runner/collect_results.py latest reactor-c/${{ steps.compute-output-file.outputs.output-file }} + ./runner/collect_results.py latest ${{ inputs.repo-for-saving-data }}/${{ steps.compute-output-file.outputs.output-file }} shell: bash if: ${{ ! steps.compute-output-file.outputs.file-already-exists }} From c58d9370dfcbf7823f0de8b18b4718ebc7684f66 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Sat, 16 Jul 2022 14:42:26 -0700 Subject: [PATCH 41/43] [benchmarks] Include a stack trace on timeout. --- .github/actions/run-benchmarks/action.yml | 15 ++++++++++++--- .github/actions/set-up-workspace/action.yml | 6 ++++++ .github/actions/visualize-save-upload/action.yml | 15 ++++++++++++++- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml index d6089fb7a..a7b541355 100644 --- a/.github/actions/run-benchmarks/action.yml +++ b/.github/actions/run-benchmarks/action.yml @@ -24,6 +24,11 @@ inputs: The directory name of the repository where graphs and CSVs from benchmarks should be saved. type: string default: benchmarks-lingua-franca + errors-file: + description: | + The file in which to write any error messages. + type: string + default: errors.txt outputs: @@ -56,13 +61,15 @@ runs: OUTPUT_DIR=csvs/$(date -I) mkdir -p ${{ inputs.repo-for-saving-data }}/csvs OUTPUT_FILENAME=${{ inputs.scheduler }}-${{ inputs.num-workers }}-${{ inputs.runtime-ref }}-${SHA:0:7}.csv - EXISTING_FILE=$(find ${{ inputs.repo-for-saving-data }}/csvs -name $OUTPUT_FILENAME | head -n 1) + cd ${{ inputs.repo-for-saving-data }} + EXISTING_FILE=$(find ./csvs -name $OUTPUT_FILENAME | head -n 1) + cd .. if [[ -z $EXISTING_FILE ]]; then mkdir -p ${{ inputs.repo-for-saving-data }}/$OUTPUT_DIR echo "The collected benchmark results will be saved to $OUTPUT_FILE." echo "::set-output name=output-file::$OUTPUT_DIR/$OUTPUT_FILENAME" else - echo "The benchmark will not be run because results for that commit are already saved at $OUTPUT_FILE." + echo "The benchmark will not be run because results for that commit are already saved at $EXISTING_FILE." echo "::set-output name=file-already-exists::true" echo "::set-output name=output-file::$EXISTING_FILE" fi @@ -88,7 +95,9 @@ runs: target=lf-${{ inputs.target }} \ size=fast \ ++target.params.scheduler=${{ inputs.scheduler }} \ - threads=${{ inputs.num-workers }} + threads=${{ inputs.num-workers }} \ + | tee >(grep "\[ERROR\]" >> ${{ inputs.errors-file }}) + echo "::set-output name=output-file::$EXISTING_FILE" shell: bash if: ${{ ! steps.compute-output-file.outputs.file-already-exists }} diff --git a/.github/actions/set-up-workspace/action.yml b/.github/actions/set-up-workspace/action.yml index 4dd632398..69b2d4036 100644 --- a/.github/actions/set-up-workspace/action.yml +++ b/.github/actions/set-up-workspace/action.yml @@ -29,3 +29,9 @@ runs: echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV shell: bash + + - name: Install eu-stack + run: | + sudo apt-get update + sudo apt-get install elfutils + shell: bash diff --git a/.github/actions/visualize-save-upload/action.yml b/.github/actions/visualize-save-upload/action.yml index 677e4fce4..8eb4b7c94 100644 --- a/.github/actions/visualize-save-upload/action.yml +++ b/.github/actions/visualize-save-upload/action.yml @@ -46,7 +46,7 @@ runs: git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git fetch - git add ${{ inputs.csv-files }} ${{ steps.figure.outputs.output-file }} + git add -f ${{ inputs.csv-files }} ${{ steps.figure.outputs.output-file }} if [[ $(git status) != *"nothing to commit"* ]]; then git commit -m "benchmark ${{ github.sha }}" git push https://token:${{ inputs.token }}@github.com/lf-lang/${{ inputs.repo-for-saving-data }}.git @@ -54,11 +54,23 @@ runs: cd .. shell: bash + - name: Compute error messages + id: compute-error-messages + run: | + echo $(ls) + if [ -f errors.txt ]; then + ERROR_MESSAGES=$(cat errors.txt) + echo "The benchmark runner reported the following errors:\n$ERROR_MESSAGES" + echo "::set-output name=error-messages::Error messages: $ERROR_MESSAGES" + fi + shell: bash + - name: Comment PR uses: thollander/actions-comment-pull-request@v1 with: message: | Running benchmarks from commit ${{ github.sha }}... + ${{ steps.compute-error-messages.outputs.error-messages }} Results so far: @@ -73,6 +85,7 @@ runs: with: message: | :heavy_check_mark: Finished running benchmarks from ${{ github.sha }}. + ${{ steps.compute-error-messages.outputs.error-messages }} ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/${{ inputs.repo-for-saving-data }}/saved-benchmark-results/${{ steps.figure.outputs.output-file }}) comment_includes: ${{ github.sha }} From 7772a04f2292997822270350f51044b885372469 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Mon, 18 Jul 2022 13:01:11 -0700 Subject: [PATCH 42/43] [benchmarks] Minor repairs. --- .github/actions/run-benchmarks/action.yml | 2 +- .github/actions/visualize-save-upload/action.yml | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml index a7b541355..ca9638fe5 100644 --- a/.github/actions/run-benchmarks/action.yml +++ b/.github/actions/run-benchmarks/action.yml @@ -66,7 +66,7 @@ runs: cd .. if [[ -z $EXISTING_FILE ]]; then mkdir -p ${{ inputs.repo-for-saving-data }}/$OUTPUT_DIR - echo "The collected benchmark results will be saved to $OUTPUT_FILE." + echo "The collected benchmark results will be saved to $OUTPUT_DIR/$OUTPUT_FILENAME." echo "::set-output name=output-file::$OUTPUT_DIR/$OUTPUT_FILENAME" else echo "The benchmark will not be run because results for that commit are already saved at $EXISTING_FILE." diff --git a/.github/actions/visualize-save-upload/action.yml b/.github/actions/visualize-save-upload/action.yml index 8eb4b7c94..51a6a5670 100644 --- a/.github/actions/visualize-save-upload/action.yml +++ b/.github/actions/visualize-save-upload/action.yml @@ -59,9 +59,11 @@ runs: run: | echo $(ls) if [ -f errors.txt ]; then - ERROR_MESSAGES=$(cat errors.txt) - echo "The benchmark runner reported the following errors:\n$ERROR_MESSAGES" - echo "::set-output name=error-messages::Error messages: $ERROR_MESSAGES" + ERROR_MESSAGES="The benchmark runner reported the following errors:"$'\n'"$(cat errors.txt)" + echo "$ERROR_MESSAGES" + echo 'ERROR_MESSAGES<> $GITHUB_ENV + echo "$ERROR_MESSAGES" >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV fi shell: bash @@ -70,7 +72,7 @@ runs: with: message: | Running benchmarks from commit ${{ github.sha }}... - ${{ steps.compute-error-messages.outputs.error-messages }} + ${{ env.ERROR_MESSAGES }} Results so far: @@ -85,7 +87,7 @@ runs: with: message: | :heavy_check_mark: Finished running benchmarks from ${{ github.sha }}. - ${{ steps.compute-error-messages.outputs.error-messages }} + ${{ env.ERROR_MESSAGES }} ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/${{ inputs.repo-for-saving-data }}/saved-benchmark-results/${{ steps.figure.outputs.output-file }}) comment_includes: ${{ github.sha }} From 56d9d0a8d92a600f771fdd3cac82c6f29c509213 Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Mon, 18 Jul 2022 15:02:19 -0700 Subject: [PATCH 43/43] Disable stack trace by default. --- .github/actions/run-benchmarks/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml index ca9638fe5..7904a1fca 100644 --- a/.github/actions/run-benchmarks/action.yml +++ b/.github/actions/run-benchmarks/action.yml @@ -95,6 +95,7 @@ runs: target=lf-${{ inputs.target }} \ size=fast \ ++target.params.scheduler=${{ inputs.scheduler }} \ + ++stacktrace=True \ threads=${{ inputs.num-workers }} \ | tee >(grep "\[ERROR\]" >> ${{ inputs.errors-file }}) echo "::set-output name=output-file::$EXISTING_FILE"