From 7bc8dfc37fa003950b237af52acb1243f7b1b72a Mon Sep 17 00:00:00 2001 From: Peter Donovan Date: Wed, 13 Jul 2022 23:25:40 -0700 Subject: [PATCH] [benchmarks] Separate main and feature benchmarks. --- .github/actions/run-benchmarks/action.yml | 54 ++++++++++++++++++ .github/actions/set-up-workspace/action.yml | 4 +- .../workflows/user-requested-benchmark.yml | 56 ++++++++++++------- 3 files changed, 91 insertions(+), 23 deletions(-) create mode 100644 .github/actions/run-benchmarks/action.yml diff --git a/.github/actions/run-benchmarks/action.yml b/.github/actions/run-benchmarks/action.yml new file mode 100644 index 0000000000..f92cde88ba --- /dev/null +++ b/.github/actions/run-benchmarks/action.yml @@ -0,0 +1,54 @@ +name: Run benchmarks +description: Run benchmarks and collect results + +inputs: + scheduler: + type: string # A comma-separated list (no spaces!) is allowed. + required: true + runtime-ref: + type: string + required: true + + +outputs: + output-file: + description: "The CSV file summarizing all benchmark results" + value: ${{ steps.collect-results.outputs.output-file }} + + +runs: + using: "composite" + steps: + - name: Update LFC + run: | + echo $(ls) + cd lf/org.lflang/src/lib/c/reactor-c && \ + git switch ${{ inputs.runtime-ref }} && cd ../../../../.. + echo $(ls) + ./gradlew buildLfc + cd .. + shell: bash + + - name: Run benchmarks + # TODO: Run all benchmarks with several numbers of threads + # run: | + # ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + # benchmark="glob(*)" target=lf-c size=fast \ + # target.params.scheduler=GEDF_NP,NP,adaptive threads=1,3,6,12,24 + run: | # DEBUG + ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ + benchmark="savina_concurrency_cigsmok" target=lf-c size=fast \ + target.params.scheduler=${{ github.event.inputs.scheduler }} threads=1,3 + shell: bash + + - name: Collect benchmark results + id: collect-results + run: | + SHA=$(cd lf/org.lflang/src/lib/c/reactor-c && \ + git rev-parse --short=7 HEAD && cd ../../../../..) + BRANCH=$(git rev-parse --abbrev-ref HEAD) + OUTPUT_FILE=${{ inputs.scheduler }}-$BRANCH-$SHA.csv + echo "The collected benchmark results will be saved to $OUTPUT_FILE" + ./runner/collect_results.py latest $OUTPUT_FILE + echo "::set-output name=output-file::$OUTPUT_FILE" + shell: bash diff --git a/.github/actions/set-up-workspace/action.yml b/.github/actions/set-up-workspace/action.yml index 47b9ffbd82..ba405224d4 100644 --- a/.github/actions/set-up-workspace/action.yml +++ b/.github/actions/set-up-workspace/action.yml @@ -27,7 +27,7 @@ runs: run: pip3 install -r runner/requirements.txt shell: bash - - name: Build lfc + - name: Build lfc # FIXME: Redundant with run-benchmarks run: | cd lf ./gradlew buildLfc @@ -37,4 +37,4 @@ runs: run: | echo "LF_PATH=$GITHUB_WORKSPACE/lf" >> $GITHUB_ENV echo "LF_BENCHMARKS_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV - shell: bash \ No newline at end of file + shell: bash diff --git a/.github/workflows/user-requested-benchmark.yml b/.github/workflows/user-requested-benchmark.yml index d8bf22a3b8..a18cf71f21 100644 --- a/.github/workflows/user-requested-benchmark.yml +++ b/.github/workflows/user-requested-benchmark.yml @@ -7,8 +7,14 @@ on: workflow_dispatch: inputs: pr_number: - required: true + default: 91 # TODO: remove this line and uncomment required: true + # required: true type: number + scheduler: + default: NP + feature_branch: + default: automated_full_benchmark # TODO (same as above) + # required: true permissions: contents: write @@ -33,36 +39,45 @@ jobs: repository: lf-lang/reactor-c token: ${{ secrets.GITHUB_TOKEN }} fetch-depth: 0 # It will be necessary to push to this repo - ref: automated-full-benchmark # FIXME: delete this line after merge + ref: graphics path: reactor-c - name: Set up workspace uses: lf-lang/reactor-c/.github/actions/set-up-workspace@automated-full-benchmark - - name: Run C Benchmarks - # run: | - # ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ - # benchmark="glob(*)" target=lf-c size=fast \ - # target.params.scheduler=GEDF_NP,NP,adaptive threads=1,3,6,12,24 - run: | # DEBUG - ./runner/run_benchmark.py -m continue_on_error=True iterations=12 \ - benchmark="savina_parallelism_recmatmul" target=lf-c size=fast \ - target.params.scheduler=NP,adaptive threads=1,3,0 - shell: bash + - name: Checkout benchmark repository + uses: actions/checkout@v2 + with: + repository: lf-lang/benchmarks-lingua-franca + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 1 + ref: main - - name: Collect benchmark results - run: | - ./runner/collect_results.py latest latest-benchmark-results.csv - shell: bash + - name: Run C Benchmarks (feature branch) + id: benchmarks-feature + uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark + with: + scheduler: ${{ github.event.inputs.scheduler }} + runtime-ref: ${{ github.event.inputs.branch }} + + - name: Run C Benchmarks (main branch) + id: benchmarks-main + uses: lf-lang/reactor-c/.github/actions/run-benchmarks@automated-full-benchmark + with: + scheduler: GEDF_NP,NP,adaptive + runtime-ref: main - name: Make figure run: | OUTPUT_DIR=images/$(date -I) + echo "Figure will be created in $OUTPUT_DIR" SHA=${{ github.sha }} OUTPUT_FILE=$OUTPUT_DIR/${SHA:0:7}.png + echo "Figure will be named $OUTPUT_FILE" mkdir -p reactor-c/$OUTPUT_DIR - ./runner/make-graphics.py latest-benchmark-results.csv reactor-c/$OUTPUT_FILE - echo "::set-output name=output_file::$OUTPUT_FILE" + ./runner/make-graphics.py ${{ steps.benchmarks-feature.outputs.output-file }} \ + ${{ steps.benchmarks-main.outputs.output-file }} reactor-c/$OUTPUT_FILE + echo "::set-output name=output-file::$OUTPUT_FILE" shell: bash id: figure @@ -72,8 +87,7 @@ jobs: git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" git config --global user.name "github-actions[bot]" git fetch - git switch graphics - git add ${{ steps.figure.outputs.OUTPUT_FILE }} + git add ${{ steps.figure.outputs.output-file }} git commit -m "benchmark ${{ github.sha }}" git config -l | grep 'http\..*\.extraheader' | cut -d= -f1 | \ xargs -L1 git config --unset-all @@ -86,5 +100,5 @@ jobs: with: message: | ![Visualization of the benchmark results.](https://raw.githubusercontent.com/lf-lang/reactor-c/graphics/${{ steps.figure.outputs.OUTPUT_FILE }}) - pr_number: 91 # ${{ inputs.pr_number }} + pr_number: ${{ inputs.pr_number }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}