expand coverage reporting for diffs

lenskit · Jul 24, 2024 · 907701f · 907701f
1 parent ec4eb59
commit 907701f
Show file tree

Hide file tree

Showing 3 changed files with 84 additions and 36 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -654,34 +654,39 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-      - name: list remotes
-        run: |
-          git remote add upstream https://github.com/lenskit/lkpy.git
-          git fetch upstream
-      - name: 🐍 Setup coverage
-        run: pipx install 'coverage[toml]'
+      - name: 🐍 Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+      - name: 📦 Install reporting packages
+        run: python -m pip -r requirements-reporting.txt
       - name: 📥 Download test artifacts
         uses: actions/download-artifact@v4
         with:
           pattern: test-*
           path: test-logs
       - name: 📋 List log files
         run: ls -lR test-logs
-      - name: ⛙ Merge coverage reports
+      - name: ⛙ Merge and report
         run: |
           coverage combine test-logs/*/.coverage
-      - name: ± Compute change in coverage
-        run: tclsh ./utils/coverage-log.tcl
-      - name: 📃 Produce coverage reports
-        run: |
+          coverage xml
           coverage html -d lenskit-coverage
-          echo '<details>' >>"$GITHUB_STEP_SUMMARY"
-          echo '<summary>Full Coverage Report</summary>' >>"$GITHUB_STEP_SUMMARY"
-          echo '' >>"$GITHUB_STEP_SUMMARY"
-          coverage report --format=markdown --fail-under=90 >>"$GITHUB_STEP_SUMMARY"
-          echo '' >>"$GITHUB_STEP_SUMMARY"
-          echo '</details>' >>"$GITHUB_STEP_SUMMARY"
+          coverage report --format=markdown >coverage.md
+      - name: Analyze diff coverage
+        if: github.event_name == 'pull_request'
+        run: |
+          diff-cover --json-report diff-cover.json --markdown-report diff-cover.md \
+              coverage.xml |tee diff-cover.txt
+      - name: ± Measure and report coverage
+        run: |
+          tclsh ./utils/coverage-log.tcl
+          cat lenskit-coverage/report.md >$GITHUB_STEP_SUMMARY
       - name: 📤 Upload coverage report
         uses: actions/upload-artifact@v4
+        if: always()
         with:
+          name: coverage-report
           path: lenskit-coverage/
+      - name: 🚫 Fail if coverage is too low
+        run: coverage report --fail-under=90
diff --git a/lkdev/workflows/test.py b/lkdev/workflows/test.py
@@ -447,13 +447,14 @@ def jobs_result(deps: list[str]) -> GHJob:
         "steps": [
             step_checkout(),
             {
-                "name": "list remotes",
-                "run": script("""
-                    git remote add upstream https://github.com/lenskit/lkpy.git
-                    git fetch upstream
-                """),
+                "name": "🐍 Set up Python",
+                "uses": "actions/setup-python@v5",
+                "with": {"python-version": META_PYTHON},
+            },
+            {
+                "name": "📦 Install reporting packages",
+                "run": "python -m pip -r requirements-reporting.txt",
             },
-            {"name": "🐍 Setup coverage", "run": "pipx install 'coverage[toml]'"},
             {
                 "name": "📥 Download test artifacts",
                 "uses": "actions/download-artifact@v4",
@@ -468,30 +469,41 @@ def jobs_result(deps: list[str]) -> GHJob:
             },
             # inspired by https://hynek.me/articles/ditch-codecov-python/
             {
-                "name": "⛙ Merge coverage reports",
+                "name": "⛙ Merge and report",
                 "run": script("""
                     coverage combine test-logs/*/.coverage
+                    coverage xml
+                    coverage html -d lenskit-coverage
+                    coverage report --format=markdown >coverage.md
                 """),
             },
-            {"name": "± Compute change in coverage", "run": "tclsh ./utils/coverage-log.tcl"},
             {
-                "name": "📃 Produce coverage reports",
+                "name": "Analyze diff coverage",
+                "if": "github.event_name == 'pull_request'",
                 "run": script("""
-                    coverage html -d lenskit-coverage
-                    echo '<details>' >>"$GITHUB_STEP_SUMMARY"
-                    echo '<summary>Full Coverage Report</summary>' >>"$GITHUB_STEP_SUMMARY"
-                    echo '' >>"$GITHUB_STEP_SUMMARY"
-                    coverage report --format=markdown --fail-under=90 >>"$GITHUB_STEP_SUMMARY"
-                    echo '' >>"$GITHUB_STEP_SUMMARY"
-                    echo '</details>' >>"$GITHUB_STEP_SUMMARY"
+                    diff-cover --json-report diff-cover.json --markdown-report diff-cover.md \\
+                        coverage.xml |tee diff-cover.txt
+                """),
+            },
+            {
+                "name": "± Measure and report coverage",
+                "run": script("""
+                    tclsh ./utils/coverage-log.tcl
+                    cat lenskit-coverage/report.md >$GITHUB_STEP_SUMMARY
                 """),
             },
             {
                 "name": "📤 Upload coverage report",
                 "uses": "actions/upload-artifact@v4",
+                "if": "always()",
                 "with": {
+                    "name": "coverage-report",
                     "path": "lenskit-coverage/",
                 },
             },
+            {
+                "name": "🚫 Fail if coverage is too low",
+                "run": "coverage report --fail-under=90",
+            },
         ],
     }
diff --git a/utils/coverage-log.tcl b/utils/coverage-log.tcl
@@ -39,14 +39,45 @@ if {[ev GITHUB_BASE_REF base]} {
         exit 2
     }
 
+    set diff_lines [exec jq .total_num_lines diff-coverage.json]
+    set diff_bad [exec jq .total_num_violations diff-coverage.json]
+    set diff_cov [eval {1.0 - ($diff_bad / $diff_lines)}]
+
     set prev_cov [exec jq .totals.percent_covered <<$prev_data 2>@stderr]
     set cur_cov [exec jq .totals.percent_covered coverage.json 2>&stderr]
 
     set cov_change [expr {$cur_cov - $prev_cov}]
 
-    set sumh [open $env(GITHUB_STEP_SUMMARY) a]
-    puts $sumh [format "Coverage change **%.2f%%** (from %.2f%% to %.2f%%).\n" $cov_change $prev_cov $cur_cov]
-    close $sumh
+    # write the coverage report
+    set reph [open lenskit-coverage/report.md w]
+    puts $reph [format
+        "Covered **%.2f%%** of diff (coverage changed **%.2f%%** from %.2f%% to %.2f%%).\n"
+        $diff_cov $cov_change $prev_cov $cur_cov
+    ]
+
+    set dsh [open diff-cover.md r]
+    while {[gets $dsh line] >= 0} {
+        if {[regexp {^# Diff} $line]} {
+            # do nothing, first header
+        } elseif {[regexp {^## Diff: (.*)} $line -> label]} {
+            puts $reph "<details>"
+            puts $reph "<summary>$label</summary>\n"
+        } elseif {[regexp {^## lenskit/} $line]} {
+            # done
+            break
+        } else {
+            puts $reph $line
+        }
+    }
+    puts $reph "\n</details>"
+    close $dsh
+
+    puts $reph "<details>\n"
+    puts $reph "<summary>Source Coverage Report</summary>\n"
+    exec coverage report --format=markdown 2>@stderr >@$reph
+    puts $reph "\n</details>"
+
+    close $reph
 } elseif {[ev GITHUB_EVENT_NAME] && [ev GITHUB_TOKEN]} {
     puts stderr "saving coverage data"
     set data [jq "{meta: .meta, totals: .totals}" coverage.json 2>@stderr]