From 907701f64d275c7bb0bb594caaa1fb60f268adbd Mon Sep 17 00:00:00 2001 From: Michael Ekstrand Date: Wed, 24 Jul 2024 10:51:10 -0400 Subject: [PATCH] expand coverage reporting for diffs --- .github/workflows/test.yml | 39 ++++++++++++++++++--------------- lkdev/workflows/test.py | 44 ++++++++++++++++++++++++-------------- utils/coverage-log.tcl | 37 +++++++++++++++++++++++++++++--- 3 files changed, 84 insertions(+), 36 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a0e965590..78f896975 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -654,12 +654,12 @@ jobs: uses: actions/checkout@v4 with: fetch-depth: 0 - - name: list remotes - run: | - git remote add upstream https://github.com/lenskit/lkpy.git - git fetch upstream - - name: 🐍 Setup coverage - run: pipx install 'coverage[toml]' + - name: 🐍 Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + - name: 📦 Install reporting packages + run: python -m pip -r requirements-reporting.txt - name: 📥 Download test artifacts uses: actions/download-artifact@v4 with: @@ -667,21 +667,26 @@ jobs: path: test-logs - name: 📋 List log files run: ls -lR test-logs - - name: ⛙ Merge coverage reports + - name: ⛙ Merge and report run: | coverage combine test-logs/*/.coverage - - name: ± Compute change in coverage - run: tclsh ./utils/coverage-log.tcl - - name: 📃 Produce coverage reports - run: | + coverage xml coverage html -d lenskit-coverage - echo '
' >>"$GITHUB_STEP_SUMMARY" - echo 'Full Coverage Report' >>"$GITHUB_STEP_SUMMARY" - echo '' >>"$GITHUB_STEP_SUMMARY" - coverage report --format=markdown --fail-under=90 >>"$GITHUB_STEP_SUMMARY" - echo '' >>"$GITHUB_STEP_SUMMARY" - echo '
' >>"$GITHUB_STEP_SUMMARY" + coverage report --format=markdown >coverage.md + - name: Analyze diff coverage + if: github.event_name == 'pull_request' + run: | + diff-cover --json-report diff-cover.json --markdown-report diff-cover.md \ + coverage.xml |tee diff-cover.txt + - name: ± Measure and report coverage + run: | + tclsh ./utils/coverage-log.tcl + cat lenskit-coverage/report.md >$GITHUB_STEP_SUMMARY - name: 📤 Upload coverage report uses: actions/upload-artifact@v4 + if: always() with: + name: coverage-report path: lenskit-coverage/ + - name: 🚫 Fail if coverage is too low + run: coverage report --fail-under=90 diff --git a/lkdev/workflows/test.py b/lkdev/workflows/test.py index a33f9f2c5..20360838d 100644 --- a/lkdev/workflows/test.py +++ b/lkdev/workflows/test.py @@ -447,13 +447,14 @@ def jobs_result(deps: list[str]) -> GHJob: "steps": [ step_checkout(), { - "name": "list remotes", - "run": script(""" - git remote add upstream https://github.com/lenskit/lkpy.git - git fetch upstream - """), + "name": "🐍 Set up Python", + "uses": "actions/setup-python@v5", + "with": {"python-version": META_PYTHON}, + }, + { + "name": "📦 Install reporting packages", + "run": "python -m pip -r requirements-reporting.txt", }, - {"name": "🐍 Setup coverage", "run": "pipx install 'coverage[toml]'"}, { "name": "📥 Download test artifacts", "uses": "actions/download-artifact@v4", @@ -468,30 +469,41 @@ def jobs_result(deps: list[str]) -> GHJob: }, # inspired by https://hynek.me/articles/ditch-codecov-python/ { - "name": "⛙ Merge coverage reports", + "name": "⛙ Merge and report", "run": script(""" coverage combine test-logs/*/.coverage + coverage xml + coverage html -d lenskit-coverage + coverage report --format=markdown >coverage.md """), }, - {"name": "± Compute change in coverage", "run": "tclsh ./utils/coverage-log.tcl"}, { - "name": "📃 Produce coverage reports", + "name": "Analyze diff coverage", + "if": "github.event_name == 'pull_request'", "run": script(""" - coverage html -d lenskit-coverage - echo '
' >>"$GITHUB_STEP_SUMMARY" - echo 'Full Coverage Report' >>"$GITHUB_STEP_SUMMARY" - echo '' >>"$GITHUB_STEP_SUMMARY" - coverage report --format=markdown --fail-under=90 >>"$GITHUB_STEP_SUMMARY" - echo '' >>"$GITHUB_STEP_SUMMARY" - echo '
' >>"$GITHUB_STEP_SUMMARY" + diff-cover --json-report diff-cover.json --markdown-report diff-cover.md \\ + coverage.xml |tee diff-cover.txt + """), + }, + { + "name": "± Measure and report coverage", + "run": script(""" + tclsh ./utils/coverage-log.tcl + cat lenskit-coverage/report.md >$GITHUB_STEP_SUMMARY """), }, { "name": "📤 Upload coverage report", "uses": "actions/upload-artifact@v4", + "if": "always()", "with": { + "name": "coverage-report", "path": "lenskit-coverage/", }, }, + { + "name": "🚫 Fail if coverage is too low", + "run": "coverage report --fail-under=90", + }, ], } diff --git a/utils/coverage-log.tcl b/utils/coverage-log.tcl index d50b09b03..c7244e308 100755 --- a/utils/coverage-log.tcl +++ b/utils/coverage-log.tcl @@ -39,14 +39,45 @@ if {[ev GITHUB_BASE_REF base]} { exit 2 } + set diff_lines [exec jq .total_num_lines diff-coverage.json] + set diff_bad [exec jq .total_num_violations diff-coverage.json] + set diff_cov [eval {1.0 - ($diff_bad / $diff_lines)}] + set prev_cov [exec jq .totals.percent_covered <<$prev_data 2>@stderr] set cur_cov [exec jq .totals.percent_covered coverage.json 2>&stderr] set cov_change [expr {$cur_cov - $prev_cov}] - set sumh [open $env(GITHUB_STEP_SUMMARY) a] - puts $sumh [format "Coverage change **%.2f%%** (from %.2f%% to %.2f%%).\n" $cov_change $prev_cov $cur_cov] - close $sumh + # write the coverage report + set reph [open lenskit-coverage/report.md w] + puts $reph [format + "Covered **%.2f%%** of diff (coverage changed **%.2f%%** from %.2f%% to %.2f%%).\n" + $diff_cov $cov_change $prev_cov $cur_cov + ] + + set dsh [open diff-cover.md r] + while {[gets $dsh line] >= 0} { + if {[regexp {^# Diff} $line]} { + # do nothing, first header + } elseif {[regexp {^## Diff: (.*)} $line -> label]} { + puts $reph "
" + puts $reph "$label\n" + } elseif {[regexp {^## lenskit/} $line]} { + # done + break + } else { + puts $reph $line + } + } + puts $reph "\n
" + close $dsh + + puts $reph "
\n" + puts $reph "Source Coverage Report\n" + exec coverage report --format=markdown 2>@stderr >@$reph + puts $reph "\n
" + + close $reph } elseif {[ev GITHUB_EVENT_NAME] && [ev GITHUB_TOKEN]} { puts stderr "saving coverage data" set data [jq "{meta: .meta, totals: .totals}" coverage.json 2>@stderr]