diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index fc29f0c015ee1..27339c1eefc41 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -33,6 +33,7 @@ on: # Confirm any changes to relevant workflow files. - '.github/workflows/performance.yml' - '.github/workflows/reusable-performance.yml' + - '.github/workflows/reusable-performance-*.yml' workflow_dispatch: # Cancels all previous workflow runs for pull requests that have not completed. @@ -47,21 +48,82 @@ concurrency: permissions: {} jobs: + determine-matrix: + name: Determine Matrix + runs-on: ubuntu-24.04 + if: ${{ ( github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' ) && ! contains( github.event.before, '00000000' ) }} + permissions: {} + env: + TARGET_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }} + outputs: + subjects: ${{ steps.set-subjects.outputs.result }} + target_sha: ${{ env.TARGET_SHA }} + steps: + # The `workflow_dispatch` event is the only one missing the needed SHA to target. + - name: Retrieve previous commit SHA (if necessary) + if: ${{ github.event_name == 'workflow_dispatch' }} + run: echo "TARGET_SHA=$(git rev-parse HEAD^1)" >> "$GITHUB_ENV" + + - name: Set subjects + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + id: set-subjects + with: + script: | + const artifacts = await github.rest.actions.listArtifactsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'wordpress-build-' + process.env.TARGET_SHA, + }); + const has_previous_build = !! artifacts.data.artifacts[0]; + + const subjects = [ + 'current', + ]; + + if ( context.eventName === 'push' && context.ref === 'refs/heads/trunk' ) { + subjects.push( 'base' ); + } else if ( has_previous_build ) { + subjects.push( 'before' ); + } + + return subjects; + # Runs the performance test suite. performance: - name: ${{ matrix.multisite && 'Multisite' || 'Single site' }} - uses: ./.github/workflows/reusable-performance.yml + name: ${{ matrix.multisite && 'Multisite' || 'Single Site' }} ${{ matrix.memcached && 'Memcached' || 'Default' }} + uses: ./.github/workflows/reusable-performance-test-v2.yml + needs: [ determine-matrix ] + permissions: + contents: read + strategy: + fail-fast: false + matrix: + memcached: [ true, false ] + multisite: [ true, false ] + subject: ${{ fromJson( needs.determine-matrix.outputs.subjects ) }} + with: + memcached: ${{ matrix.memcached }} + multisite: ${{ matrix.multisite }} + subject: ${{ matrix.subject }} + TARGET_SHA: ${{ needs.determine-matrix.outputs.target_sha }} + + compare: + name: ${{ matrix.label }} + uses: ./.github/workflows/reusable-performance-report-v2.yml + needs: [ determine-matrix, performance ] permissions: contents: read - if: ${{ ( github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' ) }} strategy: fail-fast: false matrix: memcached: [ true, false ] multisite: [ true, false ] + label: [ Compare ] with: memcached: ${{ matrix.memcached }} multisite: ${{ matrix.multisite }} + BASE_TAG: ${{ needs.performance.outputs.BASE_TAG }} + publish: ${{ contains( fromJson( needs.determine-matrix.outputs.subjects ), 'base' ) && ! matrix.memcached && ! matrix.multisite }} secrets: CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} diff --git a/.github/workflows/reusable-performance-report-v2.yml b/.github/workflows/reusable-performance-report-v2.yml new file mode 100644 index 0000000000000..880945541e4e8 --- /dev/null +++ b/.github/workflows/reusable-performance-report-v2.yml @@ -0,0 +1,114 @@ +## +# A reusable workflow that compares and publishes the performance tests. +## +name: Compare and publish performance Tests + +on: + workflow_call: + inputs: + BASE_TAG: + description: 'The version being used for baseline measurements.' + required: true + type: string + memcached: + description: 'Whether to enable memcached.' + required: false + type: boolean + default: false + multisite: + description: 'Whether to use Multisite.' + required: false + type: boolean + default: false + publish: + description: 'Whether to publish the results to Code Vitals.' + required: false + type: boolean + default: false + secrets: + CODEVITALS_PROJECT_TOKEN: + description: 'The authorization token for https://www.codevitals.run/project/wordpress.' + required: false + +env: + BASE_TAG: ${{ inputs.BASE_TAG }} + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Performs the following steps: + # - Checkout repository. + # - Set up Node.js. + # - Download the relevant performance test artifacts. + # - List the downloaded files for debugging. + # - Compare results. + # - Add workflow summary. + # - Determine the sha of the baseline tag if necessary. + # - Publish performance results if necessary. + compare: + name: ${{ inputs.multisite && 'Multisite' || 'Single Site' }} ${{ inputs.memcached && 'Memcached' || 'Default' }} ${{ inputs.publish && '(Publishes)' || '' }} + runs-on: ubuntu-24.04 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + fetch-depth: ${{ github.event_name == 'workflow_dispatch' && '2' || '1' }} + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + node-version-file: '.nvmrc' + cache: npm + + - name: Download artifacts + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + pattern: performance-${{ inputs.multisite && 'multisite' || 'single' }}-${{ inputs.memcached && 'memcached' || 'default' }}-* + path: artifacts + merge-multiple: true + + - name: List files + run: tree artifacts + + - name: Compare results + run: node ./tests/performance/compare-results.js "${RUNNER_TEMP}/summary.md" + + - name: Add workflow summary + run: cat "${RUNNER_TEMP}/summary.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Set the base sha + # Only needed when publishing results. + if: ${{ inputs.publish }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + id: base-sha + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + result-encoding: string + script: | + const baseRef = await github.rest.git.getRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: 'tags/' + process.env.BASE_TAG, + }); + return baseRef.data.object.sha; + + - name: Publish performance results + if: ${{ inputs.publish }} + env: + BASE_SHA: ${{ steps.base-sha.outputs.result }} + CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} + HOST_NAME: www.codevitals.run + run: | + if [ -z "$CODEVITALS_PROJECT_TOKEN" ]; then + echo "Performance results could not be published. 'CODEVITALS_PROJECT_TOKEN' is not set" + exit 1 + fi + COMMITTED_AT="$(git show -s "$GITHUB_SHA" --format='%cI')" + node ./tests/performance/log-results.js "$CODEVITALS_PROJECT_TOKEN" trunk "$GITHUB_SHA" "$BASE_SHA" "$COMMITTED_AT" "$HOST_NAME" diff --git a/.github/workflows/reusable-performance-test-v2.yml b/.github/workflows/reusable-performance-test-v2.yml new file mode 100644 index 0000000000000..421bb129ebe12 --- /dev/null +++ b/.github/workflows/reusable-performance-test-v2.yml @@ -0,0 +1,270 @@ +## +# A reusable workflow that runs the performance test suite. +## +name: Run performance Tests + +on: + workflow_call: + inputs: + subject: + description: Subject of the test. One of `current`, `before`, or `base`. + required: true + type: string + LOCAL_DIR: + description: 'Where to run WordPress from.' + required: false + type: 'string' + default: 'build' + BASE_TAG: + description: 'The version being used for baseline measurements.' + required: false + type: 'string' + default: '6.7.0' + TARGET_SHA: + description: 'SHA hash of the target commit used for "before" measurements.' + required: true + type: 'string' + php-version: + description: 'The PHP version to use.' + required: false + type: 'string' + default: 'latest' + memcached: + description: 'Whether to enable memcached.' + required: false + type: 'boolean' + default: false + multisite: + description: 'Whether to use Multisite.' + required: false + type: 'boolean' + default: false + outputs: + BASE_TAG: + description: 'The version being used for baseline measurements.' + value: ${{ inputs.BASE_TAG }} + secrets: + CODEVITALS_PROJECT_TOKEN: + description: 'The authorization token for https://www.codevitals.run/project/wordpress.' + required: false + +env: + PUPPETEER_SKIP_DOWNLOAD: ${{ true }} + + # Prevent wp-scripts from downloading extra Playwright browsers, + # since Chromium will be installed in its dedicated step already. + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: true + + # Performance testing should be performed in an environment reflecting a standard production environment. + LOCAL_WP_DEBUG: false + LOCAL_SCRIPT_DEBUG: false + LOCAL_SAVEQUERIES: false + LOCAL_WP_DEVELOPMENT_MODE: "''" + + BASE_TAG: ${{ inputs.BASE_TAG }} + TARGET_SHA: ${{ inputs.TARGET_SHA }} + + LOCAL_DIR: ${{ inputs.LOCAL_DIR }} + LOCAL_PHP_MEMCACHED: ${{ inputs.memcached }} + LOCAL_PHP: ${{ inputs.php-version }}${{ 'latest' != inputs.php-version && '-fpm' || '' }} + LOCAL_MULTISITE: ${{ inputs.multisite }} + +# Disable permissions for all available scopes by default. +# Any needed permissions should be configured at the job level. +permissions: {} + +jobs: + # Performs the following steps: + # - Configure environment variables. + # - Checkout repository. + # - Set up Node.js. + # - Log debug information. + # - Install npm dependencies. + # - Install Playwright browsers. + # - Build WordPress. + # - Start Docker environment. + # - Put the baseline or target version of WordPress in place if necessary. + # - Install object cache drop-in. + # - Log running Docker containers. + # - Docker debug information. + # - Install WordPress. + # - WordPress debug information. + # - Enable themes on Multisite. + # - Install WordPress Importer plugin. + # - Import mock data. + # - Deactivate WordPress Importer plugin. + # - Update permalink structure. + # - Install additional languages. + # - Disable external HTTP requests. + # - Disable cron. + # - List defined constants. + # - Install MU plugin. + # - Run performance tests. + # - Archive artifacts. + # - Ensure version-controlled files are not modified or deleted. + performance: + name: Test ${{ inputs.subject == 'base' && inputs.BASE_TAG || inputs.subject }} + runs-on: ubuntu-24.04 + permissions: + contents: read + + steps: + - name: Configure environment variables + run: | + echo "PHP_FPM_UID=$(id -u)" >> "$GITHUB_ENV" + echo "PHP_FPM_GID=$(id -g)" >> "$GITHUB_ENV" + + - name: Checkout repository + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + show-progress: ${{ runner.debug == '1' && 'true' || 'false' }} + fetch-depth: ${{ github.event_name == 'workflow_dispatch' && '2' || '1' }} + persist-credentials: false + + - name: Set up Node.js + uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0 + with: + node-version-file: '.nvmrc' + cache: npm + + - name: Log debug information + run: | + npm --version + node --version + curl --version + git --version + locale -a + + - name: Install npm dependencies + run: npm ci + + - name: Install Playwright browsers + run: npx playwright install --with-deps chromium + + - name: Start Docker environment + run: npm run env:start + + - name: Build WordPress + run: npm run build + + - name: Download previous build artifact (target branch or previous commit) + if: ${{ inputs.subject == 'before' }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + id: get-previous-build + with: + script: | + const artifacts = await github.rest.actions.listArtifactsForRepo({ + owner: context.repo.owner, + repo: context.repo.repo, + name: 'wordpress-build-' + process.env.TARGET_SHA, + }); + const matchArtifact = artifacts.data.artifacts[0]; + if ( ! matchArtifact ) { + core.setFailed( 'No artifact found!' ); + return false; + } + const download = await github.rest.actions.downloadArtifact( { + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + } ); + const fs = require( 'fs' ); + fs.writeFileSync( process.env.GITHUB_WORKSPACE + '/before.zip', Buffer.from( download.data ) ) + return true; + + - name: Unzip the previous build + if: ${{ inputs.subject == 'before' }} + run: | + unzip "${GITHUB_WORKSPACE}/before.zip" + unzip -o "${GITHUB_WORKSPACE}/wordpress.zip" + + - name: Set the environment to the baseline version + if: ${{ inputs.subject == 'base' }} + run: | + VERSION="${BASE_TAG%.0}" + npm run env:cli -- core download --version="$VERSION" --force --path="/var/www/${LOCAL_DIR}" + + - name: Install object cache drop-in + if: ${{ inputs.memcached }} + run: cp src/wp-content/object-cache.php build/wp-content/object-cache.php + + - name: Log running Docker containers + run: docker ps -a + + - name: Docker debug information + run: | + docker -v + docker compose run --rm mysql mysql --version + docker compose run --rm php php --version + docker compose run --rm php php -m + docker compose run --rm php php -i + docker compose run --rm php locale -a + + - name: Install WordPress + run: npm run env:install + + - name: Check version number + run: npm run env:cli -- core version --path="/var/www/${LOCAL_DIR}" + + - name: Enable themes on Multisite + if: ${{ inputs.multisite }} + run: | + npm run env:cli -- theme enable twentytwentyone --network --path="/var/www/${LOCAL_DIR}" + npm run env:cli -- theme enable twentytwentythree --network --path="/var/www/${LOCAL_DIR}" + npm run env:cli -- theme enable twentytwentyfour --network --path="/var/www/${LOCAL_DIR}" + npm run env:cli -- theme enable twentytwentyfive --network --path="/var/www/${LOCAL_DIR}" + + - name: Install WordPress Importer plugin + run: npm run env:cli -- plugin install wordpress-importer --activate --path="/var/www/${LOCAL_DIR}" + + - name: Import mock data + run: | + curl -O https://raw.githubusercontent.com/WordPress/theme-test-data/b9752e0533a5acbb876951a8cbb5bcc69a56474c/themeunittestdata.wordpress.xml + npm run env:cli -- import themeunittestdata.wordpress.xml --authors=create --path="/var/www/${LOCAL_DIR}" + rm themeunittestdata.wordpress.xml + + - name: Deactivate WordPress Importer plugin + run: npm run env:cli -- plugin deactivate wordpress-importer --path="/var/www/${LOCAL_DIR}" + + - name: Update permalink structure + run: npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path="/var/www/${LOCAL_DIR}" + + - name: Install additional languages + run: | + npm run env:cli -- language core install de_DE --path="/var/www/${LOCAL_DIR}" + npm run env:cli -- language plugin install de_DE --all --path="/var/www/${LOCAL_DIR}" + npm run env:cli -- language theme install de_DE --all --path="/var/www/${LOCAL_DIR}" + + # Prevent background update checks from impacting test stability. + - name: Disable external HTTP requests + run: npm run env:cli -- config set WP_HTTP_BLOCK_EXTERNAL true --raw --type=constant --path="/var/www/${LOCAL_DIR}" + + # Prevent background tasks from impacting test stability. + - name: Disable cron + run: npm run env:cli -- config set DISABLE_WP_CRON true --raw --type=constant --path="/var/www/${LOCAL_DIR}" + + - name: List defined constants + run: npm run env:cli -- config list --path="/var/www/${LOCAL_DIR}" + + - name: Install MU plugin + run: | + mkdir "./${LOCAL_DIR}/wp-content/mu-plugins" + cp ./tests/performance/wp-content/mu-plugins/server-timing.php "./${LOCAL_DIR}/wp-content/mu-plugins/server-timing.php" + + - name: Run performance tests + run: npm run test:performance + env: + TEST_RESULTS_PREFIX: ${{ inputs.subject != 'current' && inputs.subject || '' }} + + - name: Archive artifacts + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + if: always() + with: + name: performance-${{ inputs.multisite && 'multisite' || 'single' }}-${{ inputs.memcached && 'memcached' || 'default' }}-${{ inputs.subject }} + path: artifacts + if-no-files-found: error + include-hidden-files: true + + - name: Ensure version-controlled files are not modified or deleted + run: git diff --exit-code