From 81c4a74f5163f9804243ffa323db170127946c8d Mon Sep 17 00:00:00 2001 From: Paul Mucur Date: Fri, 26 Apr 2024 20:00:25 +0100 Subject: [PATCH] Pare down cross-building configuration In an attempt to make the precompilation of gems easier to maintain, try to make it as concise as possible. Specifically: * Switch the downloading of Abseil and RE2's archives to Rake file tasks as they will automatically be required by the default gem task as long as they are added to the gem specification. * Switch from invoking Docker directly in CI to using the gem:$platform tasks we define in the Rakefile. These will automatically use the correct version of rake-compiler-docker's Docker images for us and use rake-compiler's native:$platform tasks and the gem package task to build the correct gem. * Remove the test-gem-build and build-gems tasks in favour of explicitly using the appropriate Rake tasks (rake gem:$platform for precompiled gems and rake gem for the C Ruby gem). * We keep the test-gem-install script as it is easier to pass to Docker and the FreeBSD VM as a single entrypoint though it now only installs the gem and its required dependencies (reusing the Bundler cache created by setup-ruby where possible) before running the test suite in the gem install directory. --- .github/workflows/precompile-gem.yml | 15 ++-- .github/workflows/tests.yml | 123 +++++++++++++++++---------- .gitignore | 1 + Rakefile | 115 +++++++------------------ scripts/build-gems | 30 ------- scripts/test-gem-build | 30 ------- scripts/test-gem-install | 42 ++------- 7 files changed, 121 insertions(+), 235 deletions(-) delete mode 100755 scripts/build-gems delete mode 100755 scripts/test-gem-build diff --git a/.github/workflows/precompile-gem.yml b/.github/workflows/precompile-gem.yml index 475b81d2..02353bfd 100644 --- a/.github/workflows/precompile-gem.yml +++ b/.github/workflows/precompile-gem.yml @@ -12,15 +12,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.3" + bundler-cache: true - uses: actions/cache@v4 with: path: ports/archives - key: archives-ubuntu-${{hashFiles('ext/re2/extconf.rb')}} - - run: | - docker run --rm -v "$(pwd):/re2" -w /re2 \ - "ghcr.io/rake-compiler/rake-compiler-dock-image:1.5.0-mri-${{ inputs.platform }}" \ - ./scripts/test-gem-build gems ${{inputs.platform}} + key: archives-ubuntu-${{ hashFiles('ext/re2/extconf.rb') }} + - run: bundle exec rake gem:${{ inputs.platform }} - uses: actions/upload-artifact@v4 with: - name: "cruby-${{inputs.platform}}-gem" - path: gems + name: "cruby-${{ inputs.platform }}-gem" + path: pkg/*.gem diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bf5c8e82..c06b01c9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,7 @@ name: re2 Tests concurrency: - group: "${{github.workflow}}-${{github.sha}}" + group: "${{ github.workflow }}-${{ github.sha }}" cancel-in-progress: true on: @@ -22,16 +22,16 @@ jobs: - uses: actions/cache@v4 with: path: ports/archives - key: archives-ubuntu-${{hashFiles('ext/re2/extconf.rb')}} + key: archives-ubuntu-${{ hashFiles('ext/re2/extconf.rb') }} - uses: ruby/setup-ruby@v1 with: ruby-version: "3.3" bundler-cache: true - - run: ./scripts/test-gem-build gems ruby + - run: bundle exec rake gem - uses: actions/upload-artifact@v4 with: name: cruby-gem - path: gems + path: pkg/*.gem precompile-aarch64-linux: uses: ./.github/workflows/precompile-gem.yml @@ -124,8 +124,10 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems - - run: ./scripts/test-gem-install gems --enable-system-libraries + path: pkg + - run: ./scripts/test-gem-install --enable-system-libraries + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-ubuntu: needs: "build-cruby-gem" @@ -141,12 +143,14 @@ jobs: with: ruby-version: ${{ matrix.ruby }} apt-get: libre2-dev + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems - - run: ./scripts/test-gem-install gems --${{ matrix.sys }}-system-libraries - shell: bash + path: pkg + - run: ./scripts/test-gem-install --${{ matrix.sys }}-system-libraries + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-macos: needs: "build-cruby-gem" @@ -162,12 +166,14 @@ jobs: with: ruby-version: ${{ matrix.ruby }} brew: re2 + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems - - run: ./scripts/test-gem-install gems --${{ matrix.sys }}-system-libraries - shell: bash + path: pkg + - run: ./scripts/test-gem-install --${{ matrix.sys }}-system-libraries + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-windows-2019: needs: "build-cruby-gem" @@ -183,12 +189,15 @@ jobs: with: ruby-version: ${{ matrix.ruby }} mingw: re2 + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems - - run: ./scripts/test-gem-install gems --${{ matrix.sys }}-system-libraries + path: pkg + - run: ./scripts/test-gem-install --${{ matrix.sys }}-system-libraries shell: bash + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-windows-2022: needs: "build-cruby-gem" @@ -204,12 +213,15 @@ jobs: with: ruby-version: ${{ matrix.ruby }} mingw: re2 + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems - - run: ./scripts/test-gem-install gems --${{ matrix.sys }}-system-libraries + path: pkg + - run: ./scripts/test-gem-install --${{ matrix.sys }}-system-libraries shell: bash + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-freebsd: needs: "build-cruby-gem" @@ -223,13 +235,13 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems + path: pkg - uses: vmactions/freebsd-vm@v1 with: usesh: true copyback: false prepare: pkg install -y ruby devel/ruby-gems sysutils/rubygem-bundler devel/pkgconf devel/cmake shells/bash devel/re2 - run: ./scripts/test-gem-install gems --${{ matrix.sys }}-system-libraries + run: ./scripts/test-gem-install --${{ matrix.sys }}-system-libraries test-vendored-and-system: needs: "build-cruby-gem" @@ -246,10 +258,12 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems + path: pkg - name: "Link libre2 into Ruby's lib directory" run: ln -s /usr/lib/x86_64-linux-gnu/libre2.so ${{ steps.setup-ruby.outputs.ruby-prefix }}/lib/libre2.so - - run: ./scripts/test-gem-install gems + - run: ./scripts/test-gem-install + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-aarch64-linux: needs: "precompile-aarch64-linux" @@ -263,14 +277,14 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-aarch64-linux-gem - path: gems + path: pkg - name: Enable execution of multi-architecture containers by QEMU run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - run: | docker run --rm -v "$(pwd):/re2" -w /re2 \ --platform=linux/arm64/v8 \ - ruby:${{matrix.ruby}} \ - ./scripts/test-gem-install ./gems + ruby:${{ matrix.ruby }} \ + ./scripts/test-gem-install test-precompiled-arm-linux: needs: "precompile-arm-linux" @@ -284,14 +298,14 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-arm-linux-gem - path: gems + path: pkg - name: enable execution of multi-architecture containers by qemu run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - run: | docker run --rm -v "$(pwd):/re2" -w /re2 \ --platform=linux/arm/v7 \ - ruby:${{matrix.ruby}} \ - ./scripts/test-gem-install ./gems + ruby:${{ matrix.ruby }} \ + ./scripts/test-gem-install test-precompiled-x86-linux: needs: "precompile-x86-linux" @@ -305,14 +319,14 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-x86-linux-gem - path: gems + path: pkg - name: Enable execution of multi-architecture containers by QEMU run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - run: | docker run --rm -v "$(pwd):/re2" -w /re2 \ --platform=linux/386 \ - ruby:${{matrix.ruby}} \ - ./scripts/test-gem-install ./gems + ruby:${{ matrix.ruby }} \ + ./scripts/test-gem-install test-precompiled-x86_64-linux: needs: "precompile-x86_64-linux" @@ -325,12 +339,15 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-x86_64-linux-gem - path: gems - - run: ./scripts/test-gem-install gems + path: pkg + - run: ./scripts/test-gem-install + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-arm64-darwin: needs: "precompile-arm64-darwin" @@ -343,12 +360,15 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-arm64-darwin-gem - path: gems - - run: ./scripts/test-gem-install gems + path: pkg + - run: ./scripts/test-gem-install + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-x86_64-darwin: needs: "precompile-x86_64-darwin" @@ -361,12 +381,15 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-x86_64-darwin-gem - path: gems - - run: ./scripts/test-gem-install gems + path: pkg + - run: ./scripts/test-gem-install + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-x64-mingw-ucrt: needs: "precompile-x64-mingw-ucrt" @@ -379,13 +402,16 @@ jobs: - uses: actions/checkout@v4 - uses: MSP-Greg/setup-ruby-pkgs@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-x64-mingw-ucrt-gem - path: gems - - run: ./scripts/test-gem-install gems + path: pkg + - run: ./scripts/test-gem-install shell: bash + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-x64-mingw32: needs: "precompile-x64-mingw32" @@ -398,13 +424,16 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-x64-mingw32-gem - path: gems - - run: ./scripts/test-gem-install gems + path: pkg + - run: ./scripts/test-gem-install shell: bash + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-x86_64-alpine: needs: "precompile-x86_64-linux" @@ -414,12 +443,12 @@ jobs: ruby: ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3"] runs-on: ubuntu-latest container: - image: "ruby:${{matrix.ruby}}-alpine" + image: "ruby:${{ matrix.ruby }}-alpine" steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: name: cruby-x86_64-linux-gem - path: gems + path: pkg - run: apk add bash libstdc++ gcompat - - run: ./scripts/test-gem-install gems + - run: ./scripts/test-gem-install diff --git a/.gitignore b/.gitignore index 38822ee8..5b96cbaf 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ tmp doc Gemfile.lock ports/ +pkg/ diff --git a/Rakefile b/Rakefile index a80ec2f0..918589eb 100644 --- a/Rakefile +++ b/Rakefile @@ -1,40 +1,23 @@ # frozen_string_literal: true require 'rake/extensiontask' -require 'rspec/core/rake_task' require 'rake_compiler_dock' -require 'yaml' +require 'rspec/core/rake_task' require_relative 'ext/re2/recipes' -CLEAN.include FileList['**/*{.o,.so,.dylib,.bundle}'], - FileList['**/extconf.h'], - FileList['**/Makefile'], - FileList['pkg/'] - -CLOBBER.include FileList['**/tmp'], - FileList['**/*.log'], - FileList['doc/**'], - FileList['tmp/'] -CLOBBER.add("ports/*").exclude(%r{ports/archives$}) +re2_gemspec = Gem::Specification.load('re2.gemspec') +abseil_recipe, re2_recipe = load_recipes -RE2_GEM_SPEC = Gem::Specification.load('re2.gemspec') - -task :prepare do - puts "Preparing project for gem building..." - recipes = load_recipes - recipes.each { |recipe| recipe.download } -end +# Add Abseil and RE2's latest archives to the gem files. (Note these will be +# removed from the precompiled native gems.) +abseil_archive = File.join("ports/archives", File.basename(abseil_recipe.files[0][:url])) +re2_archive = File.join("ports/archives", File.basename(re2_recipe.files[0][:url])) -task gem: :prepare +re2_gemspec.files << abseil_archive +re2_gemspec.files << re2_archive -Gem::PackageTask.new(RE2_GEM_SPEC) do |p| - p.need_zip = false - p.need_tar = false -end - -CROSS_RUBY_VERSIONS = %w[3.3.0 3.2.0 3.1.0 3.0.0 2.7.0 2.6.0].join(':') -CROSS_RUBY_PLATFORMS = %w[ +cross_platforms = %w[ aarch64-linux arm-linux arm64-darwin @@ -46,13 +29,15 @@ CROSS_RUBY_PLATFORMS = %w[ x86_64-linux ].freeze -ENV['RUBY_CC_VERSION'] = CROSS_RUBY_VERSIONS +ENV['RUBY_CC_VERSION'] = %w[3.3.0 3.2.0 3.1.0 3.0.0 2.7.0 2.6.0].join(':') -Rake::ExtensionTask.new('re2', RE2_GEM_SPEC) do |e| +Gem::PackageTask.new(re2_gemspec).define + +Rake::ExtensionTask.new('re2', re2_gemspec) do |e| e.cross_compile = true e.cross_config_options << '--enable-cross-build' e.config_options << '--disable-system-libraries' - e.cross_platform = CROSS_RUBY_PLATFORMS + e.cross_platform = cross_platforms e.cross_compiling do |spec| spec.files.reject! { |path| File.fnmatch?('ports/*', path) } spec.dependencies.reject! { |dep| dep.name == 'mini_portile2' } @@ -61,72 +46,34 @@ end RSpec::Core::RakeTask.new(:spec) -namespace 'gem' do - def gem_builder(platform) - # use Task#invoke because the pkg/*gem task is defined at runtime - Rake::Task["native:#{platform}"].invoke - Rake::Task["pkg/#{RE2_GEM_SPEC.full_name}-#{Gem::Platform.new(platform)}.gem"].invoke - end +namespace :gem do + cross_platforms.each do |platform| - CROSS_RUBY_PLATFORMS.each do |platform| - # The Linux x86 image (ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-x86_64-linux) - # is based on CentOS 7 and has two versions of cmake installed: - # a 2.8 version in /usr/bin and a 3.25 in /usr/local/bin. The latter is needed by abseil. - cmake = - case platform - when 'x86_64-linux', 'x86-linux' - '/usr/local/bin/cmake' - else - 'cmake' - end - - desc "build native gem for #{platform} platform" + # Compile each platform's native gem, packaging up the result. Note we add + # /usr/local/bin to the PATH as it contains the newest version of CMake in + # the rake-compiler-dock images. + desc "Compile and build native gem for #{platform} platform" task platform do RakeCompilerDock.sh <<~SCRIPT, platform: platform, verbose: true gem install bundler --no-document && bundle && - bundle exec rake gem:#{platform}:builder CMAKE=#{cmake} + bundle exec rake native:#{platform} pkg/#{re2_gemspec.full_name}-#{Gem::Platform.new(platform)}.gem PATH="/usr/local/bin:$PATH" SCRIPT end - - namespace platform do - desc "build native gem for #{platform} platform (guest container)" - task 'builder' do - gem_builder(platform) - end - end end - - desc 'build all native gems' - multitask 'native' => CROSS_RUBY_PLATFORMS end -def add_file_to_gem(relative_source_path) - dest_path = File.join(gem_build_path, relative_source_path) - dest_dir = File.dirname(dest_path) - - mkdir_p dest_dir unless Dir.exist?(dest_dir) - rm_f dest_path if File.exist?(dest_path) - safe_ln relative_source_path, dest_path - - RE2_GEM_SPEC.files << relative_source_path +# Set up file tasks for Abseil and RE2's archives so they are automatically +# downloaded when required by the gem task. +file abseil_archive do + abseil_recipe.download end -def gem_build_path - File.join 'pkg', RE2_GEM_SPEC.full_name +file re2_archive do + re2_recipe.download end -def add_vendored_libraries - dependencies = YAML.load_file(File.join(File.dirname(__FILE__), 'dependencies.yml')) - abseil_archive = File.join('ports', 'archives', "#{dependencies['abseil']['version']}.tar.gz") - libre2_archive = File.join('ports', 'archives', "re2-#{dependencies['libre2']['version']}.tar.gz") - - add_file_to_gem(abseil_archive) - add_file_to_gem(libre2_archive) -end - -task gem_build_path do - add_vendored_libraries -end +task default: :spec -task default: [:compile, :spec] +CLEAN.add("lib/**/*.{o,so,bundle}", "pkg") +CLOBBER.add("ports") diff --git a/scripts/build-gems b/scripts/build-gems deleted file mode 100755 index 1e8d2b16..00000000 --- a/scripts/build-gems +++ /dev/null @@ -1,30 +0,0 @@ -#! /usr/bin/env bash -# -# script to build gems for all relevant platforms -# -set -o errexit -set -o nounset -set -x - -rm -rf tmp pkg gems -mkdir -p gems - -# prelude: let's check that things work -bundle update - -bundle exec rake clean clobber -bundle exec rake compile -bundle exec rake spec - -# MRI et al (standard gem) -bundle exec rake clean clobber -bundle exec rake gem -cp -v pkg/re2*.gem gems - -# precompiled native gems ("fat binary") -bundle exec rake gem:native -cp -v pkg/re2*.gem gems - -pushd gems - ls *.gem | sort | xargs sha256sum -popd diff --git a/scripts/test-gem-build b/scripts/test-gem-build deleted file mode 100755 index 8f998c4a..00000000 --- a/scripts/test-gem-build +++ /dev/null @@ -1,30 +0,0 @@ -#! /usr/bin/env bash -# -# run as part of CI -# -if [[ $# -lt 2 ]] ; then - echo "usage: $(basename $0) " - exit 1 -fi - -set -e -u - -OUTPUT_DIR=$1 -BUILD_NATIVE_GEM=$2 - -test -e /etc/os-release && cat /etc/os-release - -set -x - -bundle install --local || bundle install - -if [[ "${BUILD_NATIVE_GEM}" == "ruby" ]] ; then - bundle exec rake clean - bundle exec rake gem -else - bundle exec rake gem:${BUILD_NATIVE_GEM}:builder -fi - -mkdir -p ${OUTPUT_DIR} -cp -v pkg/*.gem ${OUTPUT_DIR} -ls -l ${OUTPUT_DIR}/* diff --git a/scripts/test-gem-install b/scripts/test-gem-install index d6bfd82c..62dda80a 100755 --- a/scripts/test-gem-install +++ b/scripts/test-gem-install @@ -1,40 +1,8 @@ -#! /usr/bin/env bash -# -# run as part of CI -# -if [[ $# -lt 1 ]] ; then - echo "usage: $(basename $0) [install_flags]" - exit 1 -fi +#!/usr/bin/env bash -GEMS_DIR=$1 -shift -INSTALL_FLAGS=$* +set -eu -test -e /etc/os-release && cat /etc/os-release - -set -e -x -u - -pushd $GEMS_DIR - gemfile=$(ls *.gem | head -n1) - ls -l ${gemfile} - gem install --no-document ${gemfile} -- ${INSTALL_FLAGS} - gem list -d re2 -popd - -if [ -n "${BUNDLE_APP_CONFIG:-}" ] ; then - export BUNDLE_CACHE_PATH="${BUNDLE_APP_CONFIG}/cache" -fi - -bundle install --local || bundle install - -# Use the cached vendored gems directory if available. -# This is easier than trying to retrieve the value of `bundle config get path`. -cached_gems="$(pwd)/vendor/bundle" -if [ -d "$cached_gems" ]; then - export BUNDLE_PATH=$cached_gems -fi - -cd "$(dirname "$(gem which re2)")" -cd .. +gem install --no-document pkg/*.gem -- "$@" +cd "$(dirname "$(gem which re2)")/.." +bundle bundle exec rake spec