diff --git a/.github/workflows/precompile-gem.yml b/.github/workflows/precompile-gem.yml index 475b81d..02353bf 100644 --- a/.github/workflows/precompile-gem.yml +++ b/.github/workflows/precompile-gem.yml @@ -12,15 +12,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.3" + bundler-cache: true - uses: actions/cache@v4 with: path: ports/archives - key: archives-ubuntu-${{hashFiles('ext/re2/extconf.rb')}} - - run: | - docker run --rm -v "$(pwd):/re2" -w /re2 \ - "ghcr.io/rake-compiler/rake-compiler-dock-image:1.5.0-mri-${{ inputs.platform }}" \ - ./scripts/test-gem-build gems ${{inputs.platform}} + key: archives-ubuntu-${{ hashFiles('ext/re2/extconf.rb') }} + - run: bundle exec rake gem:${{ inputs.platform }} - uses: actions/upload-artifact@v4 with: - name: "cruby-${{inputs.platform}}-gem" - path: gems + name: "cruby-${{ inputs.platform }}-gem" + path: pkg/*.gem diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bf5c8e8..c06b01c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,7 +1,7 @@ name: re2 Tests concurrency: - group: "${{github.workflow}}-${{github.sha}}" + group: "${{ github.workflow }}-${{ github.sha }}" cancel-in-progress: true on: @@ -22,16 +22,16 @@ jobs: - uses: actions/cache@v4 with: path: ports/archives - key: archives-ubuntu-${{hashFiles('ext/re2/extconf.rb')}} + key: archives-ubuntu-${{ hashFiles('ext/re2/extconf.rb') }} - uses: ruby/setup-ruby@v1 with: ruby-version: "3.3" bundler-cache: true - - run: ./scripts/test-gem-build gems ruby + - run: bundle exec rake gem - uses: actions/upload-artifact@v4 with: name: cruby-gem - path: gems + path: pkg/*.gem precompile-aarch64-linux: uses: ./.github/workflows/precompile-gem.yml @@ -124,8 +124,10 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems - - run: ./scripts/test-gem-install gems --enable-system-libraries + path: pkg + - run: ./scripts/test-gem-install --enable-system-libraries + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-ubuntu: needs: "build-cruby-gem" @@ -141,12 +143,14 @@ jobs: with: ruby-version: ${{ matrix.ruby }} apt-get: libre2-dev + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems - - run: ./scripts/test-gem-install gems --${{ matrix.sys }}-system-libraries - shell: bash + path: pkg + - run: ./scripts/test-gem-install --${{ matrix.sys }}-system-libraries + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-macos: needs: "build-cruby-gem" @@ -162,12 +166,14 @@ jobs: with: ruby-version: ${{ matrix.ruby }} brew: re2 + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems - - run: ./scripts/test-gem-install gems --${{ matrix.sys }}-system-libraries - shell: bash + path: pkg + - run: ./scripts/test-gem-install --${{ matrix.sys }}-system-libraries + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-windows-2019: needs: "build-cruby-gem" @@ -183,12 +189,15 @@ jobs: with: ruby-version: ${{ matrix.ruby }} mingw: re2 + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems - - run: ./scripts/test-gem-install gems --${{ matrix.sys }}-system-libraries + path: pkg + - run: ./scripts/test-gem-install --${{ matrix.sys }}-system-libraries shell: bash + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-windows-2022: needs: "build-cruby-gem" @@ -204,12 +213,15 @@ jobs: with: ruby-version: ${{ matrix.ruby }} mingw: re2 + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems - - run: ./scripts/test-gem-install gems --${{ matrix.sys }}-system-libraries + path: pkg + - run: ./scripts/test-gem-install --${{ matrix.sys }}-system-libraries shell: bash + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-freebsd: needs: "build-cruby-gem" @@ -223,13 +235,13 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems + path: pkg - uses: vmactions/freebsd-vm@v1 with: usesh: true copyback: false prepare: pkg install -y ruby devel/ruby-gems sysutils/rubygem-bundler devel/pkgconf devel/cmake shells/bash devel/re2 - run: ./scripts/test-gem-install gems --${{ matrix.sys }}-system-libraries + run: ./scripts/test-gem-install --${{ matrix.sys }}-system-libraries test-vendored-and-system: needs: "build-cruby-gem" @@ -246,10 +258,12 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-gem - path: gems + path: pkg - name: "Link libre2 into Ruby's lib directory" run: ln -s /usr/lib/x86_64-linux-gnu/libre2.so ${{ steps.setup-ruby.outputs.ruby-prefix }}/lib/libre2.so - - run: ./scripts/test-gem-install gems + - run: ./scripts/test-gem-install + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-aarch64-linux: needs: "precompile-aarch64-linux" @@ -263,14 +277,14 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-aarch64-linux-gem - path: gems + path: pkg - name: Enable execution of multi-architecture containers by QEMU run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - run: | docker run --rm -v "$(pwd):/re2" -w /re2 \ --platform=linux/arm64/v8 \ - ruby:${{matrix.ruby}} \ - ./scripts/test-gem-install ./gems + ruby:${{ matrix.ruby }} \ + ./scripts/test-gem-install test-precompiled-arm-linux: needs: "precompile-arm-linux" @@ -284,14 +298,14 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-arm-linux-gem - path: gems + path: pkg - name: enable execution of multi-architecture containers by qemu run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - run: | docker run --rm -v "$(pwd):/re2" -w /re2 \ --platform=linux/arm/v7 \ - ruby:${{matrix.ruby}} \ - ./scripts/test-gem-install ./gems + ruby:${{ matrix.ruby }} \ + ./scripts/test-gem-install test-precompiled-x86-linux: needs: "precompile-x86-linux" @@ -305,14 +319,14 @@ jobs: - uses: actions/download-artifact@v4 with: name: cruby-x86-linux-gem - path: gems + path: pkg - name: Enable execution of multi-architecture containers by QEMU run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - run: | docker run --rm -v "$(pwd):/re2" -w /re2 \ --platform=linux/386 \ - ruby:${{matrix.ruby}} \ - ./scripts/test-gem-install ./gems + ruby:${{ matrix.ruby }} \ + ./scripts/test-gem-install test-precompiled-x86_64-linux: needs: "precompile-x86_64-linux" @@ -325,12 +339,15 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-x86_64-linux-gem - path: gems - - run: ./scripts/test-gem-install gems + path: pkg + - run: ./scripts/test-gem-install + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-arm64-darwin: needs: "precompile-arm64-darwin" @@ -343,12 +360,15 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-arm64-darwin-gem - path: gems - - run: ./scripts/test-gem-install gems + path: pkg + - run: ./scripts/test-gem-install + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-x86_64-darwin: needs: "precompile-x86_64-darwin" @@ -361,12 +381,15 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-x86_64-darwin-gem - path: gems - - run: ./scripts/test-gem-install gems + path: pkg + - run: ./scripts/test-gem-install + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-x64-mingw-ucrt: needs: "precompile-x64-mingw-ucrt" @@ -379,13 +402,16 @@ jobs: - uses: actions/checkout@v4 - uses: MSP-Greg/setup-ruby-pkgs@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-x64-mingw-ucrt-gem - path: gems - - run: ./scripts/test-gem-install gems + path: pkg + - run: ./scripts/test-gem-install shell: bash + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-x64-mingw32: needs: "precompile-x64-mingw32" @@ -398,13 +424,16 @@ jobs: - uses: actions/checkout@v4 - uses: ruby/setup-ruby@v1 with: - ruby-version: "${{matrix.ruby}}" + ruby-version: "${{ matrix.ruby }}" + bundler-cache: true - uses: actions/download-artifact@v4 with: name: cruby-x64-mingw32-gem - path: gems - - run: ./scripts/test-gem-install gems + path: pkg + - run: ./scripts/test-gem-install shell: bash + env: + BUNDLE_PATH: ${{ github.workspace }}/vendor/bundle test-precompiled-x86_64-alpine: needs: "precompile-x86_64-linux" @@ -414,12 +443,12 @@ jobs: ruby: ["2.6", "2.7", "3.0", "3.1", "3.2", "3.3"] runs-on: ubuntu-latest container: - image: "ruby:${{matrix.ruby}}-alpine" + image: "ruby:${{ matrix.ruby }}-alpine" steps: - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 with: name: cruby-x86_64-linux-gem - path: gems + path: pkg - run: apk add bash libstdc++ gcompat - - run: ./scripts/test-gem-install gems + - run: ./scripts/test-gem-install diff --git a/.gitignore b/.gitignore index 38822ee..5b96cba 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ tmp doc Gemfile.lock ports/ +pkg/ diff --git a/Rakefile b/Rakefile index a80ec2f..918589e 100644 --- a/Rakefile +++ b/Rakefile @@ -1,40 +1,23 @@ # frozen_string_literal: true require 'rake/extensiontask' -require 'rspec/core/rake_task' require 'rake_compiler_dock' -require 'yaml' +require 'rspec/core/rake_task' require_relative 'ext/re2/recipes' -CLEAN.include FileList['**/*{.o,.so,.dylib,.bundle}'], - FileList['**/extconf.h'], - FileList['**/Makefile'], - FileList['pkg/'] - -CLOBBER.include FileList['**/tmp'], - FileList['**/*.log'], - FileList['doc/**'], - FileList['tmp/'] -CLOBBER.add("ports/*").exclude(%r{ports/archives$}) +re2_gemspec = Gem::Specification.load('re2.gemspec') +abseil_recipe, re2_recipe = load_recipes -RE2_GEM_SPEC = Gem::Specification.load('re2.gemspec') - -task :prepare do - puts "Preparing project for gem building..." - recipes = load_recipes - recipes.each { |recipe| recipe.download } -end +# Add Abseil and RE2's latest archives to the gem files. (Note these will be +# removed from the precompiled native gems.) +abseil_archive = File.join("ports/archives", File.basename(abseil_recipe.files[0][:url])) +re2_archive = File.join("ports/archives", File.basename(re2_recipe.files[0][:url])) -task gem: :prepare +re2_gemspec.files << abseil_archive +re2_gemspec.files << re2_archive -Gem::PackageTask.new(RE2_GEM_SPEC) do |p| - p.need_zip = false - p.need_tar = false -end - -CROSS_RUBY_VERSIONS = %w[3.3.0 3.2.0 3.1.0 3.0.0 2.7.0 2.6.0].join(':') -CROSS_RUBY_PLATFORMS = %w[ +cross_platforms = %w[ aarch64-linux arm-linux arm64-darwin @@ -46,13 +29,15 @@ CROSS_RUBY_PLATFORMS = %w[ x86_64-linux ].freeze -ENV['RUBY_CC_VERSION'] = CROSS_RUBY_VERSIONS +ENV['RUBY_CC_VERSION'] = %w[3.3.0 3.2.0 3.1.0 3.0.0 2.7.0 2.6.0].join(':') -Rake::ExtensionTask.new('re2', RE2_GEM_SPEC) do |e| +Gem::PackageTask.new(re2_gemspec).define + +Rake::ExtensionTask.new('re2', re2_gemspec) do |e| e.cross_compile = true e.cross_config_options << '--enable-cross-build' e.config_options << '--disable-system-libraries' - e.cross_platform = CROSS_RUBY_PLATFORMS + e.cross_platform = cross_platforms e.cross_compiling do |spec| spec.files.reject! { |path| File.fnmatch?('ports/*', path) } spec.dependencies.reject! { |dep| dep.name == 'mini_portile2' } @@ -61,72 +46,34 @@ end RSpec::Core::RakeTask.new(:spec) -namespace 'gem' do - def gem_builder(platform) - # use Task#invoke because the pkg/*gem task is defined at runtime - Rake::Task["native:#{platform}"].invoke - Rake::Task["pkg/#{RE2_GEM_SPEC.full_name}-#{Gem::Platform.new(platform)}.gem"].invoke - end +namespace :gem do + cross_platforms.each do |platform| - CROSS_RUBY_PLATFORMS.each do |platform| - # The Linux x86 image (ghcr.io/rake-compiler/rake-compiler-dock-image:1.3.0-mri-x86_64-linux) - # is based on CentOS 7 and has two versions of cmake installed: - # a 2.8 version in /usr/bin and a 3.25 in /usr/local/bin. The latter is needed by abseil. - cmake = - case platform - when 'x86_64-linux', 'x86-linux' - '/usr/local/bin/cmake' - else - 'cmake' - end - - desc "build native gem for #{platform} platform" + # Compile each platform's native gem, packaging up the result. Note we add + # /usr/local/bin to the PATH as it contains the newest version of CMake in + # the rake-compiler-dock images. + desc "Compile and build native gem for #{platform} platform" task platform do RakeCompilerDock.sh <<~SCRIPT, platform: platform, verbose: true gem install bundler --no-document && bundle && - bundle exec rake gem:#{platform}:builder CMAKE=#{cmake} + bundle exec rake native:#{platform} pkg/#{re2_gemspec.full_name}-#{Gem::Platform.new(platform)}.gem PATH="/usr/local/bin:$PATH" SCRIPT end - - namespace platform do - desc "build native gem for #{platform} platform (guest container)" - task 'builder' do - gem_builder(platform) - end - end end - - desc 'build all native gems' - multitask 'native' => CROSS_RUBY_PLATFORMS end -def add_file_to_gem(relative_source_path) - dest_path = File.join(gem_build_path, relative_source_path) - dest_dir = File.dirname(dest_path) - - mkdir_p dest_dir unless Dir.exist?(dest_dir) - rm_f dest_path if File.exist?(dest_path) - safe_ln relative_source_path, dest_path - - RE2_GEM_SPEC.files << relative_source_path +# Set up file tasks for Abseil and RE2's archives so they are automatically +# downloaded when required by the gem task. +file abseil_archive do + abseil_recipe.download end -def gem_build_path - File.join 'pkg', RE2_GEM_SPEC.full_name +file re2_archive do + re2_recipe.download end -def add_vendored_libraries - dependencies = YAML.load_file(File.join(File.dirname(__FILE__), 'dependencies.yml')) - abseil_archive = File.join('ports', 'archives', "#{dependencies['abseil']['version']}.tar.gz") - libre2_archive = File.join('ports', 'archives', "re2-#{dependencies['libre2']['version']}.tar.gz") - - add_file_to_gem(abseil_archive) - add_file_to_gem(libre2_archive) -end - -task gem_build_path do - add_vendored_libraries -end +task default: :spec -task default: [:compile, :spec] +CLEAN.add("lib/**/*.{o,so,bundle}", "pkg") +CLOBBER.add("ports") diff --git a/scripts/build-gems b/scripts/build-gems deleted file mode 100755 index 1e8d2b1..0000000 --- a/scripts/build-gems +++ /dev/null @@ -1,30 +0,0 @@ -#! /usr/bin/env bash -# -# script to build gems for all relevant platforms -# -set -o errexit -set -o nounset -set -x - -rm -rf tmp pkg gems -mkdir -p gems - -# prelude: let's check that things work -bundle update - -bundle exec rake clean clobber -bundle exec rake compile -bundle exec rake spec - -# MRI et al (standard gem) -bundle exec rake clean clobber -bundle exec rake gem -cp -v pkg/re2*.gem gems - -# precompiled native gems ("fat binary") -bundle exec rake gem:native -cp -v pkg/re2*.gem gems - -pushd gems - ls *.gem | sort | xargs sha256sum -popd diff --git a/scripts/test-gem-build b/scripts/test-gem-build deleted file mode 100755 index 8f998c4..0000000 --- a/scripts/test-gem-build +++ /dev/null @@ -1,30 +0,0 @@ -#! /usr/bin/env bash -# -# run as part of CI -# -if [[ $# -lt 2 ]] ; then - echo "usage: $(basename $0) " - exit 1 -fi - -set -e -u - -OUTPUT_DIR=$1 -BUILD_NATIVE_GEM=$2 - -test -e /etc/os-release && cat /etc/os-release - -set -x - -bundle install --local || bundle install - -if [[ "${BUILD_NATIVE_GEM}" == "ruby" ]] ; then - bundle exec rake clean - bundle exec rake gem -else - bundle exec rake gem:${BUILD_NATIVE_GEM}:builder -fi - -mkdir -p ${OUTPUT_DIR} -cp -v pkg/*.gem ${OUTPUT_DIR} -ls -l ${OUTPUT_DIR}/* diff --git a/scripts/test-gem-install b/scripts/test-gem-install index d6bfd82..62dda80 100755 --- a/scripts/test-gem-install +++ b/scripts/test-gem-install @@ -1,40 +1,8 @@ -#! /usr/bin/env bash -# -# run as part of CI -# -if [[ $# -lt 1 ]] ; then - echo "usage: $(basename $0) [install_flags]" - exit 1 -fi +#!/usr/bin/env bash -GEMS_DIR=$1 -shift -INSTALL_FLAGS=$* +set -eu -test -e /etc/os-release && cat /etc/os-release - -set -e -x -u - -pushd $GEMS_DIR - gemfile=$(ls *.gem | head -n1) - ls -l ${gemfile} - gem install --no-document ${gemfile} -- ${INSTALL_FLAGS} - gem list -d re2 -popd - -if [ -n "${BUNDLE_APP_CONFIG:-}" ] ; then - export BUNDLE_CACHE_PATH="${BUNDLE_APP_CONFIG}/cache" -fi - -bundle install --local || bundle install - -# Use the cached vendored gems directory if available. -# This is easier than trying to retrieve the value of `bundle config get path`. -cached_gems="$(pwd)/vendor/bundle" -if [ -d "$cached_gems" ]; then - export BUNDLE_PATH=$cached_gems -fi - -cd "$(dirname "$(gem which re2)")" -cd .. +gem install --no-document pkg/*.gem -- "$@" +cd "$(dirname "$(gem which re2)")/.." +bundle bundle exec rake spec