From ce35f15d28a7ed77f312735a072712ddca5e7af5 Mon Sep 17 00:00:00 2001 From: Brad Jorsch Date: Tue, 5 Nov 2024 16:51:17 -0500 Subject: [PATCH] ci: Initial code to process coverage data (#40016) To be able to inform the author about the coverage on their PR, we'll need to extract a summary of coverage changes from the raw data we're collecting. This is a first step towards that, combining the raw coverage data into combined files and producing a summary of covered lines in each file. Also, since it annoyed me, I added `WordPress.WP.GlobalVariablesOverride` to our `Jetpack-NoWP` ruleset so non-WordPress code doesn't get useless complaints about common variable names like `$path`. Also, fix Boost coverage that was missing some config settings. --- .github/files/coverage-munger/.gitignore | 2 + .../files/coverage-munger/.phan/config.php | 11 +++ .github/files/coverage-munger/composer.json | 6 ++ .../extract-php-summary-data.php | 36 ++++++++++ .github/files/coverage-munger/package.json | 8 +++ .../files/coverage-munger/process-coverage.sh | 44 ++++++++++++ .github/files/generate-ci-matrix.php | 2 - .github/files/process-coverage.sh | 15 ---- .github/workflows/tests.yml | 2 +- .phan/monorepo-pseudo-projects.jsonc | 1 + pnpm-lock.yaml | 68 +++++++++++++++++++ pnpm-workspace.yaml | 1 + .../codesniffer/Jetpack-NoWP/ruleset.xml | 3 + .../changelog/add-coverage-processing | 4 ++ .../boost/changelog/add-coverage-processing | 5 ++ projects/plugins/boost/tests/jest.config.cjs | 1 + tools/check-changelogger-use.php | 2 - tools/cli/commands/dependencies.js | 5 +- tools/docker/config/wp-tests-config.php | 2 +- 19 files changed, 196 insertions(+), 22 deletions(-) create mode 100644 .github/files/coverage-munger/.gitignore create mode 100644 .github/files/coverage-munger/.phan/config.php create mode 100644 .github/files/coverage-munger/composer.json create mode 100755 .github/files/coverage-munger/extract-php-summary-data.php create mode 100644 .github/files/coverage-munger/package.json create mode 100755 .github/files/coverage-munger/process-coverage.sh delete mode 100755 .github/files/process-coverage.sh create mode 100644 projects/packages/codesniffer/changelog/add-coverage-processing create mode 100644 projects/plugins/boost/changelog/add-coverage-processing diff --git a/.github/files/coverage-munger/.gitignore b/.github/files/coverage-munger/.gitignore new file mode 100644 index 0000000000000..bed28b15a24cc --- /dev/null +++ b/.github/files/coverage-munger/.gitignore @@ -0,0 +1,2 @@ +/composer.lock +/node_modules diff --git a/.github/files/coverage-munger/.phan/config.php b/.github/files/coverage-munger/.phan/config.php new file mode 100644 index 0000000000000..6c8c8fab21221 --- /dev/null +++ b/.github/files/coverage-munger/.phan/config.php @@ -0,0 +1,11 @@ + array() ) ); diff --git a/.github/files/coverage-munger/composer.json b/.github/files/coverage-munger/composer.json new file mode 100644 index 0000000000000..cd0795298f797 --- /dev/null +++ b/.github/files/coverage-munger/composer.json @@ -0,0 +1,6 @@ +{ + "require-dev": { + "yoast/phpunit-polyfills": "^1.1.1", + "phpunit/phpcov": "^8.2" + } +} diff --git a/.github/files/coverage-munger/extract-php-summary-data.php b/.github/files/coverage-munger/extract-php-summary-data.php new file mode 100755 index 0000000000000..f1e36c4a0011e --- /dev/null +++ b/.github/files/coverage-munger/extract-php-summary-data.php @@ -0,0 +1,36 @@ +#!/usr/bin/env php + \n" ); + exit( 1 ); +} + +require __DIR__ . '/vendor/autoload.php'; +$cov = require $argv[1]; +$report = $cov->getReport(); + +foreach ( $report as $item ) { + if ( ! $item instanceof File ) { + continue; + } + + $path = $item->pathAsString(); + + fputcsv( + STDOUT, + array( $path, $item->numberOfExecutableLines() + $item->numberOfExecutableBranches(), $item->numberOfExecutedLines() + $item->numberOfExecutedBranches() ), + "\t", + '"', + '' + ); +} diff --git a/.github/files/coverage-munger/package.json b/.github/files/coverage-munger/package.json new file mode 100644 index 0000000000000..4b62d05803239 --- /dev/null +++ b/.github/files/coverage-munger/package.json @@ -0,0 +1,8 @@ +{ + "private": true, + "name": "jetpack-gh-config-munger", + "devDependencies": { + "istanbul-merge": "^2.0.0", + "nyc": "^17.1.0" + } +} diff --git a/.github/files/coverage-munger/process-coverage.sh b/.github/files/coverage-munger/process-coverage.sh new file mode 100755 index 0000000000000..0c15eb5cfdf2e --- /dev/null +++ b/.github/files/coverage-munger/process-coverage.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -eo pipefail + +BASE=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) + +[[ -d coverage ]] && find coverage -type d -empty -delete +if [[ ! -d coverage ]]; then + echo 'No coverage was generated.' + exit 0 +fi + +echo '::group::Copy coverage into artifacts' +tar --owner=0 --group=0 --xz -cvvf artifacts/coverage.tar.xz coverage +echo '::endgroup::' + +TMP_DIR=$( mktemp -d ) +trap 'rm -rf "$TMP_DIR"' exit + +echo "::group::Combining PHP coverage" +composer --working-dir="$BASE" update +"$BASE"/vendor/bin/phpcov merge --php artifacts/php-combined.cov coverage +perl -i -pwe 'BEGIN { $prefix = shift; $prefix=~s!/*$!/!; $re = qr/\Q$prefix\E/; $l = length( $prefix ); } s!s:(\d+):"$re! sprintf( qq(s:%d:"), $1 - $l ) !ge' "$GITHUB_WORKSPACE" artifacts/php-combined.cov +echo '::endgroup::' + +echo "::group::Combining JS coverage" +pnpm --filter=jetpack-gh-config-munger exec istanbul-merge --out "$PWD"/artifacts/js-combined.json $( find "$PWD/coverage" -name '*.json' ) +perl -i -pwe 'BEGIN { $prefix = shift; $prefix=~s!/*$!/!; $re = qr/\Q$prefix\E/; } s!"$re!"!g' "$GITHUB_WORKSPACE" artifacts/js-combined.json +echo '::endgroup::' + +echo "::group::Creating PHP coverage summary" +"$BASE"/extract-php-summary-data.php artifacts/php-combined.cov > "$TMP_DIR/php-summary.tsv" +echo '::endgroup::' + +echo "::group::Creating JS coverage summary" +mkdir "$TMP_DIR/js" +cp artifacts/js-combined.json "$TMP_DIR/js" +pnpm --filter=jetpack-gh-config-munger exec nyc report --no-exclude-after-remap --report-dir="$TMP_DIR" --temp-dir="$TMP_DIR/js" --reporter=json-summary +jq -r 'to_entries[] | select( .key != "total" ) | [ .key, .value.lines.total, .value.lines.covered ] | @tsv' "$TMP_DIR/coverage-summary.json" > "$TMP_DIR/js-summary.tsv" +echo '::endgroup::' + +echo "::group::Combining coverage summaries" +sort "$TMP_DIR/php-summary.tsv" "$TMP_DIR/js-summary.tsv" > artifacts/summary.tsv +echo '::endgroup::' diff --git a/.github/files/generate-ci-matrix.php b/.github/files/generate-ci-matrix.php index 1e5ccd9168914..4fa03fb9c8d48 100755 --- a/.github/files/generate-ci-matrix.php +++ b/.github/files/generate-ci-matrix.php @@ -6,8 +6,6 @@ * @package automattic/jetpack */ -// phpcs:disable WordPress.WP.GlobalVariablesOverride - chdir( __DIR__ . '/../../' ); // Default versions for PHP and Node. diff --git a/.github/files/process-coverage.sh b/.github/files/process-coverage.sh deleted file mode 100755 index a30fcc75cdbab..0000000000000 --- a/.github/files/process-coverage.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -EXIT=0 - -[[ -d coverage ]] && find coverage -type d -empty -delete -if [[ ! -d coverage ]]; then - echo 'No coverage was generated.' - exit $EXIT -fi - -echo '::group::Copy coverage into artifacts' -tar --owner=0 --group=0 --xz -cvvf artifacts/coverage.tar.xz coverage -echo '::endgroup::' diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c05c67942a56c..cfd8b156fc24a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -252,7 +252,7 @@ jobs: env: CHANGED: ${{ steps.changed.outputs.projects }} if: matrix.script == 'test-coverage' - run: .github/files/process-coverage.sh + run: .github/files/coverage-munger/process-coverage.sh - name: Check for artifacts id: check-artifacts diff --git a/.phan/monorepo-pseudo-projects.jsonc b/.phan/monorepo-pseudo-projects.jsonc index 657b04634aff1..e24e0336b48a9 100644 --- a/.phan/monorepo-pseudo-projects.jsonc +++ b/.phan/monorepo-pseudo-projects.jsonc @@ -1,6 +1,7 @@ // Monorepo pseudo-projects. These are things that have their own composer.json but aren't actual projects. { "monorepo/actions-tool-setup-composer-plugin": ".github/actions/tool-setup/composer-plugin/", + "monorepo/coverage-munger": ".github/files/coverage-munger/", "monorepo/cli-doc-parser": "tools/cli/helpers/doc-parser/", "monorepo/e2e-commons": "tools/e2e-commons/" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e4824b4e0869c..bfae1c596b853 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,15 @@ importers: specifier: workspace:* version: link:tools/js-tools + .github/files/coverage-munger: + devDependencies: + istanbul-merge: + specifier: ^2.0.0 + version: 2.0.0 + nyc: + specifier: ^17.1.0 + version: 17.1.0 + projects/github-actions/repo-gardening: dependencies: '@actions/core': @@ -10691,6 +10700,11 @@ packages: resolution: {integrity: sha512-n3s8EwkdFIJCG3BPKBYvskgXGoy88ARzvegkitk60NxRdwltLOTaH7CUiMRXvwYorl0Q712iEjcWB+fK/MrWVw==} engines: {node: '>=10'} + istanbul-merge@2.0.0: + resolution: {integrity: sha512-Y812/uTdnF5Qc2qWxA7jQOTkqpFLEr7BHy8mzUQFRJstTjPigNS1Bh3q06AbOhBZ7tZqrI4MZdMgG34KVnUn6w==} + engines: {node: '>= 8'} + hasBin: true + istanbul-reports@3.1.7: resolution: {integrity: sha512-BewmUXImeuRk2YY0PVbxgKAysvhRPUQE0h5QRM++nVWyubKGV0l8qQ5op8+B2DOmwSe63Jivj0BjkPQVf8fP5g==} engines: {node: '>=8'} @@ -11482,6 +11496,10 @@ packages: mitt@3.0.1: resolution: {integrity: sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==} + mkdirp@0.5.6: + resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} + hasBin: true + mkdirp@1.0.4: resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} engines: {node: '>=10'} @@ -11605,6 +11623,11 @@ packages: engines: {node: '>=8.9'} hasBin: true + nyc@17.1.0: + resolution: {integrity: sha512-U42vQ4czpKa0QdI1hu950XuNhYqgoM+ZF1HT+VuUHL9hPfDPVvNQyltmMqdE9bUHMVa+8yNbc3QKTj8zQhlVxQ==} + engines: {node: '>=18'} + hasBin: true + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} @@ -22644,6 +22667,15 @@ snapshots: transitivePeerDependencies: - supports-color + istanbul-merge@2.0.0: + dependencies: + array.prototype.flatmap: 1.3.2 + for-each: 0.3.3 + glob: 7.2.3 + istanbul-lib-coverage: 3.2.2 + mkdirp: 0.5.6 + yargs: 15.4.1 + istanbul-reports@3.1.7: dependencies: html-escaper: 2.0.2 @@ -23905,6 +23937,10 @@ snapshots: mitt@3.0.1: {} + mkdirp@0.5.6: + dependencies: + minimist: 1.2.8 + mkdirp@1.0.4: {} mock-xmlhttprequest@8.3.0: {} @@ -24020,6 +24056,38 @@ snapshots: transitivePeerDependencies: - supports-color + nyc@17.1.0: + dependencies: + '@istanbuljs/load-nyc-config': 1.1.0 + '@istanbuljs/schema': 0.1.3 + caching-transform: 4.0.0 + convert-source-map: 1.9.0 + decamelize: 1.2.0 + find-cache-dir: 3.3.2 + find-up: 4.1.0 + foreground-child: 3.3.0 + get-package-type: 0.1.0 + glob: 7.2.3 + istanbul-lib-coverage: 3.2.2 + istanbul-lib-hook: 3.0.0 + istanbul-lib-instrument: 6.0.3 + istanbul-lib-processinfo: 2.0.3 + istanbul-lib-report: 3.0.1 + istanbul-lib-source-maps: 4.0.1 + istanbul-reports: 3.1.7 + make-dir: 3.1.0 + node-preload: 0.2.1 + p-map: 3.0.0 + process-on-spawn: 1.0.0 + resolve-from: 5.0.0 + rimraf: 3.0.2 + signal-exit: 3.0.7 + spawn-wrap: 2.0.0 + test-exclude: 6.0.0 + yargs: 15.4.1 + transitivePeerDependencies: + - supports-color + object-assign@4.1.1: {} object-inspect@1.13.2: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index a90bfe09301b9..6efa0c329a846 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -4,3 +4,4 @@ packages: - 'tools/cli' - 'tools/e2e-commons' - 'tools/js-tools' + - '.github/files/coverage-munger' diff --git a/projects/packages/codesniffer/Jetpack-NoWP/ruleset.xml b/projects/packages/codesniffer/Jetpack-NoWP/ruleset.xml index 508f467df75ad..58adfe9b494ad 100644 --- a/projects/packages/codesniffer/Jetpack-NoWP/ruleset.xml +++ b/projects/packages/codesniffer/Jetpack-NoWP/ruleset.xml @@ -25,5 +25,8 @@ 0 + + 0 + diff --git a/projects/packages/codesniffer/changelog/add-coverage-processing b/projects/packages/codesniffer/changelog/add-coverage-processing new file mode 100644 index 0000000000000..516c38c461a00 --- /dev/null +++ b/projects/packages/codesniffer/changelog/add-coverage-processing @@ -0,0 +1,4 @@ +Significance: patch +Type: changed + +Add `WordPress.WP.GlobalVariablesOverride` to `Jetpack-NoWP` ruleset. diff --git a/projects/plugins/boost/changelog/add-coverage-processing b/projects/plugins/boost/changelog/add-coverage-processing new file mode 100644 index 0000000000000..82e4b68f811fc --- /dev/null +++ b/projects/plugins/boost/changelog/add-coverage-processing @@ -0,0 +1,5 @@ +Significance: patch +Type: fixed +Comment: Fix Boost coverage + + diff --git a/projects/plugins/boost/tests/jest.config.cjs b/projects/plugins/boost/tests/jest.config.cjs index ee682abe68aa7..63fbc82fd7424 100644 --- a/projects/plugins/boost/tests/jest.config.cjs +++ b/projects/plugins/boost/tests/jest.config.cjs @@ -3,6 +3,7 @@ const path = require( 'path' ); const coverageConfig = require( 'jetpack-js-tools/jest/config.coverage.js' ); module.exports = { + ...coverageConfig, rootDir: path.join( __dirname, '..' ), testEnvironment: 'jsdom', collectCoverageFrom: [ diff --git a/tools/check-changelogger-use.php b/tools/check-changelogger-use.php index 01cffd655667a..5bdcb2fbb3d64 100755 --- a/tools/check-changelogger-use.php +++ b/tools/check-changelogger-use.php @@ -6,8 +6,6 @@ * @package automattic/jetpack */ -// phpcs:disable WordPress.WP.GlobalVariablesOverride - chdir( __DIR__ . '/../' ); /** diff --git a/tools/cli/commands/dependencies.js b/tools/cli/commands/dependencies.js index 751526a9f7def..3d5202039d091 100644 --- a/tools/cli/commands/dependencies.js +++ b/tools/cli/commands/dependencies.js @@ -20,7 +20,10 @@ infrastructureFileSets.base = new Set( [ infrastructureFileSets.test = new Set( [ ...infrastructureFileSets.base, '.github/files/generate-ci-matrix.php', - '.github/files/process-coverage.sh', + '.github/files/coverage-munger/composer.json', + '.github/files/coverage-munger/package.json', + '.github/files/coverage-munger/extract-php-summary-data.php', + '.github/files/coverage-munger/process-coverage.sh', '.github/files/setup-wordpress-env.sh', '.github/workflows/tests.yml', ] ); diff --git a/tools/docker/config/wp-tests-config.php b/tools/docker/config/wp-tests-config.php index 086643fb010ef..5f2428f49d7b9 100644 --- a/tools/docker/config/wp-tests-config.php +++ b/tools/docker/config/wp-tests-config.php @@ -73,7 +73,7 @@ define( 'LOGGED_IN_SALT', 'put your unique phrase here' ); define( 'NONCE_SALT', 'put your unique phrase here' ); -// phpcs:ignore WordPress.WP.GlobalVariablesOverride.Prohibited, VariableAnalysis.CodeAnalysis.VariableAnalysis.UnusedVariable +// phpcs:ignore VariableAnalysis.CodeAnalysis.VariableAnalysis.UnusedVariable $table_prefix = 'wptests_'; // Only numbers, letters, and underscores please! define( 'WP_TESTS_DOMAIN', 'example.org' );