From 358f9a5cbcaad5a67f0aef3bfc58766e40e62e80 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Mon, 15 Jan 2024 21:52:38 -0500 Subject: [PATCH 01/10] ci: add x86_64-linux alpine to the multiarch matrix --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43af38f..1478f1a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -296,6 +296,9 @@ jobs: - from_image: arm32v6/alpine platform: arm-linux dockerfile: alpine + - from_image: alpine + platform: x86_64-linux + dockerfile: alpine runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 From f700f7dec54cb9adc5f2085716b52b72d5cd2d34 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 19 Dec 2023 17:40:57 -0500 Subject: [PATCH 02/10] test: reproduce an alpine symbol issue see https://github.com/sparklemotion/sqlite3-ruby/issues/434 --- test/rcd_test/ext/mri/rcd_test_ext.c | 14 ++++++++++++++ test/rcd_test/ext/mri/rcd_test_ext.h | 5 +++++ test/rcd_test/test/test_basic.rb | 7 +++++++ 3 files changed, 26 insertions(+) diff --git a/test/rcd_test/ext/mri/rcd_test_ext.c b/test/rcd_test/ext/mri/rcd_test_ext.c index b85a47d..1a06eb5 100644 --- a/test/rcd_test/ext/mri/rcd_test_ext.c +++ b/test/rcd_test/ext/mri/rcd_test_ext.c @@ -40,6 +40,19 @@ rcdt_darwin_builtin_available_eh(VALUE self) #endif } +static VALUE +rcdt_largefile_op_removed_from_musl(VALUE self) +{ + // Reference a symbol that was removed in Musl 1.24 🙄 + // https://github.com/sparklemotion/sqlite3-ruby/issues/434 +#ifdef __linux__ + posix_fallocate(STDERR_FILENO, 0, 0); + return Qtrue; +#else + return Qfalse; +#endif +} + void Init_rcd_test_ext(void) { @@ -48,4 +61,5 @@ Init_rcd_test_ext(void) rb_define_singleton_method(rb_mRcdTest, "darwin_builtin_available?", rcdt_darwin_builtin_available_eh, 0); rb_define_singleton_method(rb_mRcdTest, "isinf?", rcdt_isinf_eh, 1); rb_define_singleton_method(rb_mRcdTest, "isnan?", rcdt_isnan_eh, 1); + rb_define_singleton_method(rb_mRcdTest, "largefile_op_removed_from_musl", rcdt_largefile_op_removed_from_musl, 0); } diff --git a/test/rcd_test/ext/mri/rcd_test_ext.h b/test/rcd_test/ext/mri/rcd_test_ext.h index 14e8d9e..0b3c1a6 100644 --- a/test/rcd_test/ext/mri/rcd_test_ext.h +++ b/test/rcd_test/ext/mri/rcd_test_ext.h @@ -1,6 +1,11 @@ #ifndef RCD_TEST_EXT_H #define RCD_TEST_EXT_H 1 +// see rcdt_largefile_op_removed_from_musl +#define _LARGEFILE_SOURCE 1 +#define _FILE_OFFSET_BITS 64 +#include + #include "ruby.h" #endif /* RCD_TEST_EXT_H */ diff --git a/test/rcd_test/test/test_basic.rb b/test/rcd_test/test/test_basic.rb index 9f3659a..ddbc437 100644 --- a/test/rcd_test/test/test_basic.rb +++ b/test/rcd_test/test/test_basic.rb @@ -22,10 +22,17 @@ def test_check_darwin_compiler_rt_symbol_resolution def test_floating_point_classification_macros skip("jruby should not run libc-specific tests") if RUBY_ENGINE == "jruby" + refute(RcdTest.isinf?(42.0)) assert(RcdTest.isinf?(Float::INFINITY)) refute(RcdTest.isnan?(42.0)) assert(RcdTest.isnan?(Float::NAN)) end + def test_largefile_op_removed_from_musl + skip("jruby should not run libc-specific tests") if RUBY_ENGINE == "jruby" + + is_linux = RUBY_PLATFORM.include?("linux") + assert_equal(is_linux, RcdTest.largefile_op_removed_from_musl) + end end From 49ecf3d4c898b6b71764d6bd1e819c13bb5af186 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Wed, 20 Dec 2023 14:13:31 -0500 Subject: [PATCH 03/10] change `-linux` to `-linux-gnu` everywhere Test gems built with the `-linux-gnu` suffix only on ruby 3.1 and later, which are the ones that ship with rubygems 3.2.33 or later (the version that first recognizes the `-gnu` suffix). Also make sure we tag the `-linux-gnu` docker images as `-linux` for backwards compatibility, and make sure we test that workflow. --- .github/workflows/ci.yml | 48 ++++++++++++++++++++++------ .github/workflows/publish-images.yml | 12 +++---- .gitignore | 9 +++--- Dockerfile.mri.erb | 8 ++--- History.md | 3 ++ README.md | 33 ++++++++++++------- Rakefile | 18 +++++++---- lib/rake_compiler_dock.rb | 3 +- test/rcd_test/Rakefile | 16 +++++++++- test/test_starter.rb | 2 +- 10 files changed, 106 insertions(+), 46 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1478f1a..e1e370b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,18 +45,22 @@ jobs: fail-fast: false matrix: include: - - platform: aarch64-linux - - platform: arm-linux + - platform: aarch64-linux-gnu + alias: aarch64-linux + - platform: arm-linux-gnu + alias: arm-linux - platform: arm64-darwin - platform: jruby - platform: x64-mingw-ucrt static: true - platform: x64-mingw32 static: true - - platform: x86-linux + - platform: x86-linux-gnu + alias: x86-linux - platform: x86-mingw32 - platform: x86_64-darwin - - platform: x86_64-linux + - platform: x86_64-linux-gnu + alias: x86_64-linux runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -114,6 +118,21 @@ jobs: name: gem-${{ matrix.platform }}-static path: test/rcd_test/pkg/*-*-*.gem + - if: matrix.alias + name: Build native gem ${{ matrix.alias }} + run: | + cd test/rcd_test/ + bundle install + bundle exec rake clean clobber + bundle exec rake gem:${{ matrix.alias }} + + - if: matrix.alias + name: Upload native gem ${{ matrix.alias }} + uses: actions/upload-artifact@v3 + with: + name: gem-${{ matrix.alias }} + path: test/rcd_test/pkg/*-*-*.gem + test_source_gem: name: test source needs: build_source_gem @@ -172,6 +191,15 @@ jobs: platform: x86_64-darwin - os: ubuntu platform: x86_64-linux + - os: ubuntu # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix + ruby: "3.1" + platform: x86_64-linux-gnu + - os: ubuntu + ruby: "3.2" + platform: x86_64-linux-gnu + - os: ubuntu + ruby: "3.3" + platform: x86_64-linux-gnu - os: ubuntu ruby: jruby platform: jruby @@ -282,22 +310,22 @@ jobs: matrix: include: - from_image: amd64/centos - platform: x86_64-linux + platform: x86_64-linux # centos-8 ships ruby 2.5, rubygems won't recognize -gnu suffix dockerfile: centos - from_image: navikey/raspbian-bullseye - platform: arm-linux + platform: arm-linux # bullseye ships ruby 2.7, rubygems won't recognize -gnu suffix dockerfile: debian - from_image: arm64v8/ubuntu - platform: aarch64-linux + platform: aarch64-linux # arm64v8 ships ruby 3.0, rubygems won't recognize -gnu suffix dockerfile: debian - from_image: i386/alpine - platform: x86-linux + platform: x86-linux-gnu dockerfile: alpine - from_image: arm32v6/alpine - platform: arm-linux + platform: arm-linux-gnu dockerfile: alpine - from_image: alpine - platform: x86_64-linux + platform: x86_64-linux-gnu dockerfile: alpine runs-on: ubuntu-latest steps: diff --git a/.github/workflows/publish-images.yml b/.github/workflows/publish-images.yml index ca5ff38..6bacffb 100644 --- a/.github/workflows/publish-images.yml +++ b/.github/workflows/publish-images.yml @@ -13,15 +13,15 @@ jobs: fail-fast: false matrix: platform: - - x86-mingw32 + - aarch64-linux-gnu + - arm-linux-gnu + - arm64-darwin - x64-mingw-ucrt - x64-mingw32 - - x86-linux - - x86_64-linux + - x86-linux-gnu + - x86-mingw32 - x86_64-darwin - - arm64-darwin - - arm-linux - - aarch64-linux + - x86_64-linux-gnu - jruby runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index 09c077a..e9bdd19 100644 --- a/.gitignore +++ b/.gitignore @@ -1,15 +1,14 @@ /.bundle/ /.yardoc -/Dockerfile.mri.aarch64-linux -/Dockerfile.mri.arm-linux +/Dockerfile.mri.aarch64-linux-gnu +/Dockerfile.mri.arm-linux-gnu /Dockerfile.mri.arm64-darwin -/Dockerfile.mri.arm64-linux /Dockerfile.mri.x64-mingw-ucrt /Dockerfile.mri.x64-mingw32 -/Dockerfile.mri.x86-linux +/Dockerfile.mri.x86-linux-gnu /Dockerfile.mri.x86-mingw32 /Dockerfile.mri.x86_64-darwin -/Dockerfile.mri.x86_64-linux +/Dockerfile.mri.x86_64-linux-gnu /Gemfile.lock /_yardoc/ /cache/ diff --git a/Dockerfile.mri.erb b/Dockerfile.mri.erb index 7a6d1e9..137b317 100644 --- a/Dockerfile.mri.erb +++ b/Dockerfile.mri.erb @@ -1,7 +1,7 @@ <% image = case platform - when /x86_64-linux/ then "quay.io/pypa/manylinux2014_x86_64" - when /x86-linux/ then "quay.io/pypa/manylinux2014_i686" + when /x86_64-linux-gnu/ then "quay.io/pypa/manylinux2014_x86_64" + when /x86-linux-gnu/ then "quay.io/pypa/manylinux2014_i686" else "ubuntu:20.04" end manylinux = !!(image =~ /manylinux/) @@ -68,8 +68,8 @@ RUN dpkg -i /debs/*.deb RUN apt-get -y update && \ apt-get install -y <% if platform =~ /darwin/ %> clang python lzma-dev libxml2-dev libssl-dev libc++-10-dev <% end %><% -if platform =~ /aarch64-linux/ %> gcc-aarch64-linux-gnu g++-aarch64-linux-gnu <% end %><% -if platform =~ /arm-linux/ %> gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf <% end %><% +if platform =~ /aarch64-linux-gnu/ %> gcc-aarch64-linux-gnu g++-aarch64-linux-gnu <% end %><% +if platform =~ /arm-linux-gnu/ %> gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf <% end %><% if platform =~ /x86-mingw32/ %> gcc-mingw-w64-i686 g++-mingw-w64-i686 <% end %><% if platform =~ /x64-mingw32/ %> gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 <% end %> && \ rm -rf /var/lib/apt/lists/* diff --git a/History.md b/History.md index 8dab9ee..608b0d7 100644 --- a/History.md +++ b/History.md @@ -1,6 +1,9 @@ next / unreleased ----------------- +* Linux builds and their images are now fully qualified with the libc flavor. So, `x86_64-linux` + should now be referred to as `x86_64-linux-gnu`, and the generated files are also named with the + libc name. * Replace `rvm` with `rbenv` and `ruby-build` - `rvm` has been replaced by `rbenv` and `ruby-build` - no longer applying sendfile patches to bootstrap rubies diff --git a/README.md b/README.md index c28b8de..a0ac8c7 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Your Rakefile should enable cross compilation like so: ```ruby exttask = Rake::ExtensionTask.new('my_extension', my_gem_spec) do |ext| ext.cross_compile = true - ext.cross_platform = %w[x86-mingw32 x64-mingw-ucrt x64-mingw32 x86-linux x86_64-linux x86_64-darwin arm64-darwin] + ext.cross_platform = %w[x86-mingw32 x64-mingw-ucrt x64-mingw32 x86-linux-gnu x86_64-linux-gnu x86_64-darwin arm64-darwin] end ``` @@ -44,7 +44,7 @@ Additionally it may also be used to build ffi based binary gems like [libusb](ht ### Interactive Usage Rake-compiler-dock offers the shell command `rake-compiler-dock` and a [ruby API](http://www.rubydoc.info/gems/rake-compiler-dock/RakeCompilerDock) for issuing commands within the docker image, described below. -There are dedicated images for `x86-mingw32`, `x64-mingw-ucrt`, `x64-mingw32`, `x86-linux`, `x86_64-linux`, `x86_64-darwin`, `arm64-darwin` and `jruby` targets. +There are dedicated images for targets: `aarch64-linux-gnu`, `arm-linux-gnu`, `arm64-darwin`, `x64-mingw-ucrt`, `x64-mingw32`, `x86-linux-gnu`, `x86-mingw32`, `x86_64-darwin`, `x86_64-linux-gnu`, and `jruby`. The images contain all supported cross ruby versions, with the exception of `x64-mingw32`, which has versions before 3.1 only, and `x64-mingw-ucrt`, which has only ruby-3.1+. This is to match the [changed platform of RubyInstaller-3.1](https://rubyinstaller.org/2021/12/31/rubyinstaller-3.1.0-1-released.html). @@ -68,12 +68,12 @@ To build x86 Windows and x86_64 Linux binary gems interactively, it can be calle user@host:$ ls pkg/*.gem your-gem-1.0.0.gem your-gem-1.0.0-x86-mingw32.gem - user@host:$ RCD_PLATFORM=x86_64-linux rake-compiler-dock # this enters a container for amd64 Linux target + user@host:$ RCD_PLATFORM=x86_64-linux-gnu rake-compiler-dock # this enters a container for amd64 Linux target user@adc55b2b92a9:$ bundle user@adc55b2b92a9:$ rake cross native gem user@adc55b2b92a9:$ exit user@host:$ ls pkg/*.gem - your-gem-1.0.0.gem your-gem-1.0.0-x86_64-linux.gem + your-gem-1.0.0.gem your-gem-1.0.0-x86_64-linux-gnu.gem Or non-interactive: @@ -118,10 +118,21 @@ To make the build process reproducible for other parties, it is recommended to a This can be done like this: ```ruby +PLATFORMS = %w[ + aarch64-linux-gnu + arm-linux-gnu + arm64-darwin + x64-mingw-ucrt + x64-mingw32 + x86-linux-gnu + x86-mingw32 + x86_64-darwin + x86_64-linux-gnu +] task 'gem:native' do require 'rake_compiler_dock' sh "bundle package --all" # Avoid repeated downloads of gems by using gem files from the host. - %w[ x86-mingw32 x64-mingw-ucrt x64-mingw32 x86-linux x86_64-linux arm-linux aarch64-linux x86_64-darwin arm64-darwin ].each do |plat| + PLATFORMS.each do |plat| RakeCompilerDock.sh "bundle --local && rake native:#{plat} gem", platform: plat end RakeCompilerDock.sh "bundle --local && rake java gem", rubyvm: :jruby @@ -187,10 +198,10 @@ jobs: name: "native-gem" runs-on: ubuntu-latest container: - image: "ghcr.io/rake-compiler/rake-compiler-dock-image:1.2.2-mri-x86_64-linux" + image: "ghcr.io/rake-compiler/rake-compiler-dock-image:1.2.2-mri-x86_64-linux-gnu" steps: - uses: actions/checkout@v2 - - run: bundle install && bundle exec rake gem:x86_64-linux:rcd + - run: bundle install && bundle exec rake gem:x86_64-linux-gnu:rcd - uses: actions/upload-artifact@v2 with: name: native-gem @@ -198,10 +209,10 @@ jobs: retention-days: 1 ``` -Where the referenced rake task might be defined by: +Where the referenced rake task might be defined by something like: ``` ruby -cross_platforms = ["x64-mingw32", "x86_64-linux", "x86_64-darwin", "arm64-darwin"] +cross_platforms = ["x64-mingw32", "x86_64-linux-gnu", "x86_64-darwin", "arm64-darwin"] namespace "gem" do cross_platforms.each do |platform| @@ -223,7 +234,7 @@ For an example of rake tasks that support this style of invocation, visit https: OCI images snapshotted from `main` are published weekly to Github Container Registry with the string "snapshot" in place of the version number in the tag name, e.g.: -- `ghcr.io/rake-compiler/rake-compiler-dock-image:snapshot-mri-x86_64-linux` +- `ghcr.io/rake-compiler/rake-compiler-dock-image:snapshot-mri-x86_64-linux-gnu` These images are intended for integration testing. They may not work properly and should not be considered production ready. @@ -237,7 +248,7 @@ The following variables are recognized by rake-compiler-dock: * `RCD_RUBYVM` - The ruby VM and toolchain to be used. Must be one of `mri`, `jruby`. * `RCD_PLATFORM` - The target rubygems platform. - Must be a space separated list out of `x86-mingw32`, `x64-mingw-ucrt`, `x64-mingw32`, `x86-linux`, `x86_64-linux`, `arm-linux`, `aarch64-linux`, `x86_64-darwin` and `arm64-darwin`. + Must be a space separated list out of `aarch64-linux-gnu`, `arm-linux-gnu`, `arm64-darwin`, `x64-mingw-ucrt`, `x64-mingw32`, `x86-linux-gnu`, `x86-mingw32`, `x86_64-darwin`, `x86_64-linux-gnu`. It is ignored when `rubyvm` is set to `:jruby`. * `RCD_IMAGE` - The docker image that is downloaded and started. Defaults to "ghcr.io/rake-compiler/rake-compiler-dock-image:IMAGE_VERSION-PLATFORM" with an image version that is determined by the gem version. diff --git a/Rakefile b/Rakefile index f79bbea..c420370 100644 --- a/Rakefile +++ b/Rakefile @@ -9,15 +9,16 @@ CLEAN.include("tmp") RakeCompilerDock::GemHelper.install_tasks platforms = [ - ["x86-mingw32", "i686-w64-mingw32"], - ["x64-mingw32", "x86_64-w64-mingw32"], + # tuple is [platform, target] + ["aarch64-linux-gnu", "aarch64-linux-gnu"], + ["arm-linux-gnu", "arm-linux-gnueabihf"], + ["arm64-darwin", "aarch64-apple-darwin"], ["x64-mingw-ucrt", "x86_64-w64-mingw32"], - ["x86-linux", "i686-redhat-linux"], - ["x86_64-linux", "x86_64-redhat-linux"], + ["x64-mingw32", "x86_64-w64-mingw32"], + ["x86-linux-gnu", "i686-redhat-linux-gnu"], + ["x86-mingw32", "i686-w64-mingw32"], ["x86_64-darwin", "x86_64-apple-darwin"], - ["arm64-darwin", "aarch64-apple-darwin"], - ["arm-linux", "arm-linux-gnueabihf"], - ["aarch64-linux", "aarch64-linux-gnu"], + ["x86_64-linux-gnu", "x86_64-redhat-linux-gnu"], ] namespace :build do @@ -30,6 +31,9 @@ namespace :build do task sdf do image_name = RakeCompilerDock::Starter.container_image_name(platform: platform) sh(*RakeCompilerDock.docker_build_cmd(platform), "-t", image_name, "-f", "Dockerfile.mri.#{platform}", ".") + if image_name.include?("linux-gnu") + sh("docker", "tag", image_name, image_name.sub("linux-gnu", "linux")) + end end df = ERB.new(File.read("Dockerfile.mri.erb"), trim_mode: ">").result(binding) diff --git a/lib/rake_compiler_dock.rb b/lib/rake_compiler_dock.rb index 3147864..f73d7e2 100644 --- a/lib/rake_compiler_dock.rb +++ b/lib/rake_compiler_dock.rb @@ -26,7 +26,8 @@ module RakeCompilerDock # # Option +:platform+ can be set to a list of space separated values. # It selects the docker image(s) with an appropriate toolchain. - # Allowed values are +:x86-mingw32+, +x64-mingw32+, +x86-linux+ or +x86_64-linux+. + # Allowed values are +aarch64-linux-gnu+, +arm-linux-gnu+, +arm64-darwin+, +x64-mingw-ucrt+, + # +x64-mingw32+, +x86-linux-gnu+, +x86-mingw32+, +x86_64-darwin+, +x86_64-linux-gnu+. # If the list contains multiple values, +cmd+ is consecutively executed in each of the docker images, # Option +:platform+ is ignored when +:rubyvm+ is set to +:jruby+. # Default is "x86-mingw32 x64-mingw32" . diff --git a/test/rcd_test/Rakefile b/test/rcd_test/Rakefile index 97b798a..e4b0d63 100644 --- a/test/rcd_test/Rakefile +++ b/test/rcd_test/Rakefile @@ -18,7 +18,21 @@ else ext.ext_dir = 'ext/mri' ext.lib_dir = 'lib/rcd_test' ext.cross_compile = true - ext.cross_platform = %w[x86-mingw32 x64-mingw-ucrt x64-mingw32 x86-linux x86_64-linux aarch64-linux arm-linux x86_64-darwin arm64-darwin] + ext.cross_platform = %w[ + aarch64-linux + aarch64-linux-gnu + arm-linux + arm-linux-gnu + arm64-darwin + x64-mingw-ucrt + x64-mingw32 + x86-linux + x86-linux-gnu + x86-mingw32 + x86_64-darwin + x86_64-linux + x86_64-linux-gnu + ] end end diff --git a/test/test_starter.rb b/test/test_starter.rb index b3ad9ae..70ac29e 100644 --- a/test/test_starter.rb +++ b/test/test_starter.rb @@ -113,7 +113,7 @@ def test_container_jrubyvm? assert(Starter.container_jrubyvm?({:rubyvm => "jruby"})) assert(Starter.container_jrubyvm?({:platform => "jruby"})) refute(Starter.container_jrubyvm?({:rubyvm => "mri"})) - refute(Starter.container_jrubyvm?({:platform => "x86_64-linux"})) + refute(Starter.container_jrubyvm?({:platform => "x86_64-linux-gnu"})) end def test_platforms From da4268d5c63cf63fdf77a942114378e7170f81dd Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 16 Jan 2024 14:45:37 -0500 Subject: [PATCH 04/10] feat: add x86_64-linux-musl support ci: extract x86_64-linux-gnu testing into two matrixed jobs, one for testing against setup-ruby, another for testing against official ruby docker images. ci: add a matrix of x86_64-linux-musl test rcd_test.gemspec: remove dependency on `git` so we can run in basic containers rake-compiler-dock.gemspec: handle missing `git` so we can run in basic containers --- .github/workflows/ci.yml | 128 ++++++++++++++++++++++++++------- .gitignore | 1 + Dockerfile.mri.erb | 4 ++ Rakefile | 1 + build/mk_musl_cross.sh | 37 ++++++++++ rake-compiler-dock.gemspec | 8 ++- test/rcd_test/Rakefile | 1 + test/rcd_test/rcd_test.gemspec | 17 ++--- 8 files changed, 161 insertions(+), 36 deletions(-) create mode 100755 build/mk_musl_cross.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1e370b..61df976 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: - platform: x86_64-darwin - platform: x86_64-linux-gnu alias: x86_64-linux + - platform: x86_64-linux-musl runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -134,22 +135,13 @@ jobs: path: test/rcd_test/pkg/*-*-*.gem test_source_gem: - name: test source + name: source gem needs: build_source_gem strategy: fail-fast: false matrix: - os: - - ubuntu - ruby: - - "3.3" - - "3.2" - - "3.1" - - "3.0" - - "2.7" - - "2.6" - - "2.5" - - "2.4" + os: [ubuntu] + ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] runs-on: ${{ matrix.os }}-latest steps: - uses: actions/checkout@v3 @@ -168,6 +160,100 @@ jobs: bundle install ruby -rrcd_test -S rake test + test_x86_64-linux-gnu-in-setup-ruby: + name: x86_64-linux-gnu setup-ruby + needs: build_native_gem + strategy: + fail-fast: false + matrix: + ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] + platform: [x86_64-linux] + include: + # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix + - ruby: "3.3" + platform: x86_64-linux-gnu + - ruby: "3.2" + platform: x86_64-linux-gnu + - ruby: "3.1" + platform: x86_64-linux-gnu + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + - name: Download gem-${{ matrix.platform }} + uses: actions/download-artifact@v3 + with: + name: gem-${{ matrix.platform }} + - name: Install gem-${{ matrix.platform }} + run: gem install --local *.gem --verbose + - name: Run tests + run: | + cd test/rcd_test/ + bundle install + ruby -rrcd_test -S rake test + + test_x86_64-linux-gnu-in-docker: + name: x86_64-linux-gnu docker + needs: build_native_gem + strategy: + fail-fast: false + matrix: + ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] + platform: [x86_64-linux] + include: + # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix + - ruby: "3.3" + platform: x86_64-linux-gnu + - ruby: "3.2" + platform: x86_64-linux-gnu + - ruby: "3.1" + platform: x86_64-linux-gnu + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Download gem-${{ matrix.platform }} + uses: actions/download-artifact@v3 + with: + name: gem-${{ matrix.platform }} + - name: Run tests + run: | + docker run --rm -v $PWD:/work -w /work \ + ruby:${{ matrix.ruby }} \ + bash -c " + gem install --local *.gem --verbose && + cd test/rcd_test/ && + bundle install && + ruby -rrcd_test -S rake test + " + + test_x86_64-linux-musl: + name: x86_64-linux-musl + needs: build_native_gem + strategy: + fail-fast: false + matrix: + ruby: ["3.3", "3.2", "3.1", "3.0", "2.7"] # ruby:2.6-alpine and earlier ship with rubygems that doesn't recognize the -musl suffix + platform: [x86_64-linux-musl] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Download gem-${{ matrix.platform }} + uses: actions/download-artifact@v3 + with: + name: gem-${{ matrix.platform }} + - name: Run tests + run: | + docker run --rm -v $PWD:/work -w /work \ + ruby:${{ matrix.ruby }}-alpine \ + sh -c " + gem install --local *.gem --verbose && + cd test/rcd_test/ && + bundle install && + ruby -rrcd_test -S rake test + " + test_native_gem: name: test native needs: build_native_gem @@ -176,7 +262,6 @@ jobs: matrix: os: - macos - - ubuntu ruby: - "3.3" - "3.2" @@ -189,17 +274,6 @@ jobs: include: - os: macos platform: x86_64-darwin - - os: ubuntu - platform: x86_64-linux - - os: ubuntu # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix - ruby: "3.1" - platform: x86_64-linux-gnu - - os: ubuntu - ruby: "3.2" - platform: x86_64-linux-gnu - - os: ubuntu - ruby: "3.3" - platform: x86_64-linux-gnu - os: ubuntu ruby: jruby platform: jruby @@ -319,13 +393,13 @@ jobs: platform: aarch64-linux # arm64v8 ships ruby 3.0, rubygems won't recognize -gnu suffix dockerfile: debian - from_image: i386/alpine - platform: x86-linux-gnu + platform: x86-linux-musl dockerfile: alpine - from_image: arm32v6/alpine - platform: arm-linux-gnu + platform: arm-linux-musl dockerfile: alpine - from_image: alpine - platform: x86_64-linux-gnu + platform: x86_64-linux-musl dockerfile: alpine runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index e9bdd19..5c3988f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ /Dockerfile.mri.x86-mingw32 /Dockerfile.mri.x86_64-darwin /Dockerfile.mri.x86_64-linux-gnu +/Dockerfile.mri.x86_64-linux-musl /Gemfile.lock /_yardoc/ /cache/ diff --git a/Dockerfile.mri.erb b/Dockerfile.mri.erb index 137b317..361c212 100644 --- a/Dockerfile.mri.erb +++ b/Dockerfile.mri.erb @@ -64,6 +64,10 @@ COPY --from=larskanis/mingw64-ucrt:20.04 \ /debs/ RUN dpkg -i /debs/*.deb +<% elsif platform =~ /linux-musl/ %> +COPY build/mk_musl_cross.sh /tmp +RUN /tmp/mk_musl_cross.sh <%= target %> + <% elsif !manylinux %> RUN apt-get -y update && \ apt-get install -y <% diff --git a/Rakefile b/Rakefile index c420370..017d5c9 100644 --- a/Rakefile +++ b/Rakefile @@ -19,6 +19,7 @@ platforms = [ ["x86-mingw32", "i686-w64-mingw32"], ["x86_64-darwin", "x86_64-apple-darwin"], ["x86_64-linux-gnu", "x86_64-redhat-linux-gnu"], + ["x86_64-linux-musl", "x86_64-unknown-linux-musl"], ] namespace :build do diff --git a/build/mk_musl_cross.sh b/build/mk_musl_cross.sh new file mode 100755 index 0000000..e2dfce9 --- /dev/null +++ b/build/mk_musl_cross.sh @@ -0,0 +1,37 @@ +#! /usr/bin/env bash + +set -o errexit +set -o pipefail +set -x + +# check if target is provided as the first argument, and exit if not +if [ -z "$1" ]; then + echo "No target provided" + exit 1 +fi +TARGET=$1 + +git clone --depth 1 https://github.com/richfelker/musl-cross-make.git +pushd musl-cross-make + +cat > config.mak < e + warn "WARNING: Could not discover files for gemspec: #{e}" + [] + end + spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) spec.require_paths = ["lib"] diff --git a/test/rcd_test/Rakefile b/test/rcd_test/Rakefile index e4b0d63..53131bc 100644 --- a/test/rcd_test/Rakefile +++ b/test/rcd_test/Rakefile @@ -32,6 +32,7 @@ else x86_64-darwin x86_64-linux x86_64-linux-gnu + x86_64-linux-musl ] end end diff --git a/test/rcd_test/rcd_test.gemspec b/test/rcd_test/rcd_test.gemspec index e7bc78b..a1b1278 100644 --- a/test/rcd_test/rcd_test.gemspec +++ b/test/rcd_test/rcd_test.gemspec @@ -12,16 +12,17 @@ Gem::Specification.new do |spec| spec.required_ruby_version = ">= 2.0.0" spec.license = "MIT" - # Specify which files should be added to the gem when it is released. - # The `git ls-files -z` loads the files in the RubyGem that have been added into git. - spec.files = Dir.chdir(File.expand_path(__dir__)) do - `git ls-files -z`.split("\x0").reject { |f| f.match(%r{\A(?:test|spec|features)/}) } - end + spec.files = [ + "ext/java/RcdTestExtService.java", + "ext/java/RubyRcdTest.java", + "ext/mri/extconf.rb", + "ext/mri/rcd_test_ext.c", + "ext/mri/rcd_test_ext.h", + "lib/rcd_test.rb", + ] + spec.bindir = "exe" spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } spec.require_paths = ["lib"] spec.extensions = ["ext/mri/extconf.rb"] - - # Uncomment to register a new dependency of your gem - # spec.add_dependency "example-gem", "~> 1.0" end From ee2db7940d653e96728eb374874c9875631d2449 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 16 Jan 2024 15:26:55 -0500 Subject: [PATCH 05/10] feat: add x86-linux-musl ci: extract testing matrices for x86-linux gnu and musl --- .github/workflows/ci.yml | 63 ++++++++++++++++++++++++++++++++++++++++ .gitignore | 1 + Rakefile | 1 + test/rcd_test/Rakefile | 1 + 4 files changed, 66 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 61df976..e330642 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,6 +57,7 @@ jobs: static: true - platform: x86-linux-gnu alias: x86-linux + - platform: x86-linux-musl - platform: x86-mingw32 - platform: x86_64-darwin - platform: x86_64-linux-gnu @@ -254,6 +255,68 @@ jobs: ruby -rrcd_test -S rake test " + test_x86-linux-gnu: + name: x86-linux-gnu + needs: build_native_gem + strategy: + fail-fast: false + matrix: + ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] + platform: [x86-linux] + include: + # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix + - ruby: "3.3" + platform: x86-linux-gnu + - ruby: "3.2" + platform: x86-linux-gnu + - ruby: "3.1" + platform: x86-linux-gnu + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Download gem-${{ matrix.platform }} + uses: actions/download-artifact@v3 + with: + name: gem-${{ matrix.platform }} + - name: Run tests + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker run --rm -v $PWD:/work -w /work \ + --platform=linux/386 ruby:${{ matrix.ruby }} \ + sh -c " + gem install --local *.gem --verbose && + cd test/rcd_test/ && + bundle install && + ruby -rrcd_test -S rake test + " + + test_x86-linux-musl: + name: x86-linux-musl + needs: build_native_gem + strategy: + fail-fast: false + matrix: + ruby: ["3.3", "3.2", "3.1", "3.0", "2.7"] # ruby:2.6-alpine and earlier ship with rubygems that doesn't recognize the -musl suffix + platform: [x86-linux-musl] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Download gem-${{ matrix.platform }} + uses: actions/download-artifact@v3 + with: + name: gem-${{ matrix.platform }} + - name: Run tests + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker run --rm -v $PWD:/work -w /work \ + --platform=linux/386 ruby:${{ matrix.ruby }}-alpine \ + sh -c " + gem install --local *.gem --verbose && + cd test/rcd_test/ && + bundle install && + ruby -rrcd_test -S rake test + " + test_native_gem: name: test native needs: build_native_gem diff --git a/.gitignore b/.gitignore index 5c3988f..dc3c6a8 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ /Dockerfile.mri.x64-mingw-ucrt /Dockerfile.mri.x64-mingw32 /Dockerfile.mri.x86-linux-gnu +/Dockerfile.mri.x86-linux-musl /Dockerfile.mri.x86-mingw32 /Dockerfile.mri.x86_64-darwin /Dockerfile.mri.x86_64-linux-gnu diff --git a/Rakefile b/Rakefile index 017d5c9..8cc2310 100644 --- a/Rakefile +++ b/Rakefile @@ -16,6 +16,7 @@ platforms = [ ["x64-mingw-ucrt", "x86_64-w64-mingw32"], ["x64-mingw32", "x86_64-w64-mingw32"], ["x86-linux-gnu", "i686-redhat-linux-gnu"], + ["x86-linux-musl", "i686-unknown-linux-musl"], ["x86-mingw32", "i686-w64-mingw32"], ["x86_64-darwin", "x86_64-apple-darwin"], ["x86_64-linux-gnu", "x86_64-redhat-linux-gnu"], diff --git a/test/rcd_test/Rakefile b/test/rcd_test/Rakefile index 53131bc..d4a806e 100644 --- a/test/rcd_test/Rakefile +++ b/test/rcd_test/Rakefile @@ -28,6 +28,7 @@ else x64-mingw32 x86-linux x86-linux-gnu + x86-linux-musl x86-mingw32 x86_64-darwin x86_64-linux From 46dbb80a178b4d39e9d5fb671adc83839673ae10 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 16 Jan 2024 16:02:31 -0500 Subject: [PATCH 06/10] feat: add arm-linux-musl note that we have to patch rake-compiler to build ruby 2.7 on 32-byte arm because of coroutine issues that weren't fixed until 3.0. ci: extract testing matrices for arm-linux gnu and musl --- .github/workflows/ci.yml | 63 +++++++++++++++++++ .gitignore | 1 + Dockerfile.mri.erb | 8 +-- Rakefile | 1 + .../0004-Enable-build-of-static-libruby.patch | 10 ++- test/rcd_test/Rakefile | 1 + 6 files changed, 76 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e330642..8dd298f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,7 @@ jobs: alias: aarch64-linux - platform: arm-linux-gnu alias: arm-linux + - platform: arm-linux-musl - platform: arm64-darwin - platform: jruby - platform: x64-mingw-ucrt @@ -317,6 +318,68 @@ jobs: ruby -rrcd_test -S rake test " + test_arm-linux-gnu: + name: arm-linux-gnu + needs: build_native_gem + strategy: + fail-fast: false + matrix: + ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] + platform: [arm-linux] + include: + # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix + - ruby: "3.3" + platform: arm-linux-gnu + - ruby: "3.2" + platform: arm-linux-gnu + - ruby: "3.1" + platform: arm-linux-gnu + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Download gem-${{ matrix.platform }} + uses: actions/download-artifact@v3 + with: + name: gem-${{ matrix.platform }} + - name: Run tests + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker run --rm -v $PWD:/work -w /work \ + --platform=linux/arm/v7 ruby:${{ matrix.ruby }} \ + sh -c " + gem install --local *.gem --verbose && + cd test/rcd_test/ && + bundle install && + ruby -rrcd_test -S rake test + " + + test_arm-linux-musl: + name: arm-linux-musl + needs: build_native_gem + strategy: + fail-fast: false + matrix: + ruby: ["3.3", "3.2", "3.1"] # TODO can we update rubygems and get earlier images working? + platform: [arm-linux-musl] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Download gem-${{ matrix.platform }} + uses: actions/download-artifact@v3 + with: + name: gem-${{ matrix.platform }} + - name: Run tests + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker run --rm -v $PWD:/work -w /work \ + --platform=linux/arm/v7 ruby:${{ matrix.ruby }}-alpine \ + sh -c " + gem install --local *.gem --verbose && + cd test/rcd_test/ && + bundle install && + ruby -rrcd_test -S rake test + " + test_native_gem: name: test native needs: build_native_gem diff --git a/.gitignore b/.gitignore index dc3c6a8..1bdf461 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /.yardoc /Dockerfile.mri.aarch64-linux-gnu /Dockerfile.mri.arm-linux-gnu +/Dockerfile.mri.arm-linux-musl /Dockerfile.mri.arm64-darwin /Dockerfile.mri.x64-mingw-ucrt /Dockerfile.mri.x64-mingw32 diff --git a/Dockerfile.mri.erb b/Dockerfile.mri.erb index 361c212..fab2b8d 100644 --- a/Dockerfile.mri.erb +++ b/Dockerfile.mri.erb @@ -148,11 +148,9 @@ else ] end -xrubies_build_plan.each do |xrubies, bootstrap_ruby_version| %> -ENV XRUBIES <%= xrubies %> +strip = '-s' if platform !~ /darwin/ -<% strip = '-s' if platform !~ /darwin/ %> -# Build xruby versions, then cleanup all build artifacts +xrubies_build_plan.each do |xrubies, bootstrap_ruby_version| %> RUN bash -c " \ rbenv shell <%= bootstrap_ruby_version %> && \ export CPPFLAGS='<%= "-D__USE_MINGW_ANSI_STDIO=1" if platform =~ /x64-mingw-ucrt/ %>' && \ @@ -161,7 +159,7 @@ RUN bash -c " \ <%= "export LIBS='-l:libssp.a' &&" if platform =~ /mingw/ %> \ <%= "export CC=#{target}-clang &&" if platform =~ /darwin/ %> \ export MAKE='make V=1' && \ - rake-compiler cross-ruby VERSION=$XRUBIES HOST=<%= target %> && \ + rake-compiler cross-ruby VERSION=<%= xrubies %> HOST=<%= target %> && \ rm -rf ~/.rake-compiler/builds ~/.rake-compiler/sources \ " <% end %> diff --git a/Rakefile b/Rakefile index 8cc2310..dc61908 100644 --- a/Rakefile +++ b/Rakefile @@ -12,6 +12,7 @@ platforms = [ # tuple is [platform, target] ["aarch64-linux-gnu", "aarch64-linux-gnu"], ["arm-linux-gnu", "arm-linux-gnueabihf"], + ["arm-linux-musl", "arm-linux-musleabihf"], ["arm64-darwin", "aarch64-apple-darwin"], ["x64-mingw-ucrt", "x86_64-w64-mingw32"], ["x64-mingw32", "x86_64-w64-mingw32"], diff --git a/build/patches2/rake-compiler-1.2.5/0004-Enable-build-of-static-libruby.patch b/build/patches2/rake-compiler-1.2.5/0004-Enable-build-of-static-libruby.patch index 1258814..a48c31c 100644 --- a/build/patches2/rake-compiler-1.2.5/0004-Enable-build-of-static-libruby.patch +++ b/build/patches2/rake-compiler-1.2.5/0004-Enable-build-of-static-libruby.patch @@ -2,7 +2,7 @@ diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake index 8317a2a..5a1b3ea 100644 --- a/tasks/bin/cross-ruby.rake +++ b/tasks/bin/cross-ruby.rake -@@ -116,10 +116,22 @@ RUBY_CC_VERSIONS.split(":").each do |ruby_cc_version| +@@ -116,10 +116,26 @@ RUBY_CC_VERSIONS.split(":").each do |ruby_cc_version| "--host=#{mingw_host}", "--target=#{mingw_target}", "--build=#{RUBY_BUILD}", @@ -12,7 +12,8 @@ index 8317a2a..5a1b3ea 100644 '--disable-install-doc', '--with-ext=', ] -+ if mingw_host=~/darwin/ + ++ if mingw_host =~ /darwin/ + options += [ + '--enable-static', + '--disable-shared', @@ -23,6 +24,9 @@ index 8317a2a..5a1b3ea 100644 + '--enable-shared', + ] + end - ++ ++ # https://github.com/docker-library/ruby/issues/308 ++ options << "--with-coroutine=arm32" if major == "2.7" && mingw_target =~ /arm-linux-musl/ ++ # Force Winsock2 for Ruby 1.8, 1.9 defaults to it options << "--with-winsock2" if major == "1.8" diff --git a/test/rcd_test/Rakefile b/test/rcd_test/Rakefile index d4a806e..cba8f54 100644 --- a/test/rcd_test/Rakefile +++ b/test/rcd_test/Rakefile @@ -23,6 +23,7 @@ else aarch64-linux-gnu arm-linux arm-linux-gnu + arm-linux-musl arm64-darwin x64-mingw-ucrt x64-mingw32 From 317158251b01d16f8224dadc834aa216fec8278e Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Thu, 18 Jan 2024 09:28:56 -0500 Subject: [PATCH 07/10] feat: add aarch64-linux-musl ci: extract testing matrices for aarch64-linux gnu and musl --- .github/workflows/ci.yml | 63 ++++++++++++++++++++++++++++++++++++++++ .gitignore | 1 + Rakefile | 1 + test/rcd_test/Rakefile | 1 + 4 files changed, 66 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8dd298f..65517c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,7 @@ jobs: include: - platform: aarch64-linux-gnu alias: aarch64-linux + - platform: aarch64-linux-musl - platform: arm-linux-gnu alias: arm-linux - platform: arm-linux-musl @@ -380,6 +381,68 @@ jobs: ruby -rrcd_test -S rake test " + test_aarch64-linux-gnu: + name: aarch64-linux-gnu + needs: build_native_gem + strategy: + fail-fast: false + matrix: + ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] + platform: [aarch64-linux] + include: + # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix + - ruby: "3.3" + platform: aarch64-linux-gnu + - ruby: "3.2" + platform: aarch64-linux-gnu + - ruby: "3.1" + platform: aarch64-linux-gnu + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Download gem-${{ matrix.platform }} + uses: actions/download-artifact@v3 + with: + name: gem-${{ matrix.platform }} + - name: Run tests + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker run --rm -v $PWD:/work -w /work \ + --platform=linux/arm64 ruby:${{ matrix.ruby }} \ + sh -c " + gem install --local *.gem --verbose && + cd test/rcd_test/ && + bundle install && + ruby -rrcd_test -S rake test + " + + test_aarch64-linux-musl: + name: aarch64-linux-musl + needs: build_native_gem + strategy: + fail-fast: false + matrix: + ruby: ["3.3", "3.2", "3.1", "3.0", "2.7"] # ruby:2.6-alpine and earlier ship with rubygems that doesn't recognize the -musl suffix + platform: [aarch64-linux-musl] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Download gem-${{ matrix.platform }} + uses: actions/download-artifact@v3 + with: + name: gem-${{ matrix.platform }} + - name: Run tests + run: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + docker run --rm -v $PWD:/work -w /work \ + --platform=linux/arm64 ruby:${{ matrix.ruby }}-alpine \ + sh -c " + gem install --local *.gem --verbose && + cd test/rcd_test/ && + bundle install && + ruby -rrcd_test -S rake test + " + test_native_gem: name: test native needs: build_native_gem diff --git a/.gitignore b/.gitignore index 1bdf461..6213d23 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /.bundle/ /.yardoc /Dockerfile.mri.aarch64-linux-gnu +/Dockerfile.mri.aarch64-linux-musl /Dockerfile.mri.arm-linux-gnu /Dockerfile.mri.arm-linux-musl /Dockerfile.mri.arm64-darwin diff --git a/Rakefile b/Rakefile index dc61908..369e524 100644 --- a/Rakefile +++ b/Rakefile @@ -11,6 +11,7 @@ RakeCompilerDock::GemHelper.install_tasks platforms = [ # tuple is [platform, target] ["aarch64-linux-gnu", "aarch64-linux-gnu"], + ["aarch64-linux-musl", "aarch64-linux-musl"], ["arm-linux-gnu", "arm-linux-gnueabihf"], ["arm-linux-musl", "arm-linux-musleabihf"], ["arm64-darwin", "aarch64-apple-darwin"], diff --git a/test/rcd_test/Rakefile b/test/rcd_test/Rakefile index cba8f54..252f94c 100644 --- a/test/rcd_test/Rakefile +++ b/test/rcd_test/Rakefile @@ -21,6 +21,7 @@ else ext.cross_platform = %w[ aarch64-linux aarch64-linux-gnu + aarch64-linux-musl arm-linux arm-linux-gnu arm-linux-musl From 1baaf03084157a842e4244d5c1c801fabeba12f7 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Thu, 18 Jan 2024 13:38:38 -0500 Subject: [PATCH 08/10] ci: collapse linux matrices, update older images' rubygems because we're now updating rubygems on older images, we're able to have a full test matrix. --- .github/workflows/ci.yml | 319 ++++++++------------------------------- 1 file changed, 60 insertions(+), 259 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65517c1..9b67a1c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -155,30 +155,27 @@ jobs: uses: actions/download-artifact@v3 with: name: gem-ruby - - name: Install source gem - run: gem install --local *.gem --verbose - name: Test source gem run: | + gem install --local *.gem --verbose cd test/rcd_test/ bundle install ruby -rrcd_test -S rake test - test_x86_64-linux-gnu-in-setup-ruby: - name: x86_64-linux-gnu setup-ruby + test-x86_64-linux-setup-ruby: + name: "${{ matrix.platform }} setup-ruby(${{ matrix.ruby }})" needs: build_native_gem strategy: fail-fast: false matrix: + platform: [x86_64-linux, x86_64-linux-gnu] ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] - platform: [x86_64-linux] include: - # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix - - ruby: "3.3" - platform: x86_64-linux-gnu - - ruby: "3.2" - platform: x86_64-linux-gnu - - ruby: "3.1" - platform: x86_64-linux-gnu + # declare rubygems for each ruby version + - { ruby: "2.7", rubygems: "3.4.22" } + - { ruby: "2.6", rubygems: "3.4.22" } + - { ruby: "2.5", rubygems: "3.3.26" } + - { ruby: "2.4", rubygems: "3.3.26" } runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -189,117 +186,55 @@ jobs: uses: actions/download-artifact@v3 with: name: gem-${{ matrix.platform }} - - name: Install gem-${{ matrix.platform }} - run: gem install --local *.gem --verbose - - name: Run tests + - name: Test gem-${{ matrix.platform }} run: | + gem update --system ${{ matrix.rubygems }} + gem install --local *.gem --verbose cd test/rcd_test/ bundle install ruby -rrcd_test -S rake test - test_x86_64-linux-gnu-in-docker: - name: x86_64-linux-gnu docker - needs: build_native_gem - strategy: - fail-fast: false - matrix: - ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] - platform: [x86_64-linux] - include: - # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix - - ruby: "3.3" - platform: x86_64-linux-gnu - - ruby: "3.2" - platform: x86_64-linux-gnu - - ruby: "3.1" - platform: x86_64-linux-gnu - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Download gem-${{ matrix.platform }} - uses: actions/download-artifact@v3 - with: - name: gem-${{ matrix.platform }} - - name: Run tests - run: | - docker run --rm -v $PWD:/work -w /work \ - ruby:${{ matrix.ruby }} \ - bash -c " - gem install --local *.gem --verbose && - cd test/rcd_test/ && - bundle install && - ruby -rrcd_test -S rake test - " - - test_x86_64-linux-musl: - name: x86_64-linux-musl - needs: build_native_gem - strategy: - fail-fast: false - matrix: - ruby: ["3.3", "3.2", "3.1", "3.0", "2.7"] # ruby:2.6-alpine and earlier ship with rubygems that doesn't recognize the -musl suffix - platform: [x86_64-linux-musl] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Download gem-${{ matrix.platform }} - uses: actions/download-artifact@v3 - with: - name: gem-${{ matrix.platform }} - - name: Run tests - run: | - docker run --rm -v $PWD:/work -w /work \ - ruby:${{ matrix.ruby }}-alpine \ - sh -c " - gem install --local *.gem --verbose && - cd test/rcd_test/ && - bundle install && - ruby -rrcd_test -S rake test - " - - test_x86-linux-gnu: - name: x86-linux-gnu + test_architecture_matrix: + name: "${{ matrix.platform }} ${{ matrix.ruby }}" needs: build_native_gem strategy: fail-fast: false matrix: + platform: + - aarch64-linux + - aarch64-linux-gnu + - aarch64-linux-musl + - arm-linux + - arm-linux-gnu + - arm-linux-musl + - x86-linux + - x86-linux-gnu + - x86-linux-musl + - x86_64-linux + - x86_64-linux-gnu + - x86_64-linux-musl ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] - platform: [x86-linux] include: - # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix - - ruby: "3.3" - platform: x86-linux-gnu - - ruby: "3.2" - platform: x86-linux-gnu - - ruby: "3.1" - platform: x86-linux-gnu - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Download gem-${{ matrix.platform }} - uses: actions/download-artifact@v3 - with: - name: gem-${{ matrix.platform }} - - name: Run tests - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker run --rm -v $PWD:/work -w /work \ - --platform=linux/386 ruby:${{ matrix.ruby }} \ - sh -c " - gem install --local *.gem --verbose && - cd test/rcd_test/ && - bundle install && - ruby -rrcd_test -S rake test - " - - test_x86-linux-musl: - name: x86-linux-musl - needs: build_native_gem - strategy: - fail-fast: false - matrix: - ruby: ["3.3", "3.2", "3.1", "3.0", "2.7"] # ruby:2.6-alpine and earlier ship with rubygems that doesn't recognize the -musl suffix - platform: [x86-linux-musl] + # declare rubygems for each ruby version + - { ruby: "2.7", rubygems: "3.4.22" } + - { ruby: "2.6", rubygems: "3.4.22" } + - { ruby: "2.5", rubygems: "3.3.26" } + - { ruby: "2.4", rubygems: "3.3.26" } + # declare docker image for each platform + - { platform: aarch64-linux-musl, docker_tag: "-alpine" } + - { platform: arm-linux-musl, docker_tag: "-alpine" } + - { platform: x86-linux-musl, docker_tag: "-alpine" } + - { platform: x86_64-linux-musl, docker_tag: "-alpine" } + # declare docker platform for each platform + - { platform: aarch64-linux, docker_platform: "--platform=linux/arm64" } + - { platform: aarch64-linux-gnu, docker_platform: "--platform=linux/arm64" } + - { platform: aarch64-linux-musl, docker_platform: "--platform=linux/arm64" } + - { platform: arm-linux, docker_platform: "--platform=linux/arm/v7" } + - { platform: arm-linux-gnu, docker_platform: "--platform=linux/arm/v7" } + - { platform: arm-linux-musl, docker_platform: "--platform=linux/arm/v7" } + - { platform: x86-linux, docker_platform: "--platform=linux/386" } + - { platform: x86-linux-gnu, docker_platform: "--platform=linux/386" } + - { platform: x86-linux-musl, docker_platform: "--platform=linux/386" } runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -311,155 +246,23 @@ jobs: run: | docker run --rm --privileged multiarch/qemu-user-static --reset -p yes docker run --rm -v $PWD:/work -w /work \ - --platform=linux/386 ruby:${{ matrix.ruby }}-alpine \ + ${{ matrix.docker_platform}} ruby:${{ matrix.ruby }}${{ matrix.docker_tag }} \ sh -c " + gem update --system ${{ matrix.rubygems }} && gem install --local *.gem --verbose && cd test/rcd_test/ && bundle install && ruby -rrcd_test -S rake test " - test_arm-linux-gnu: - name: arm-linux-gnu + test_the_rest: + name: "${{ matrix.platform }} ${{ matrix.ruby }}" needs: build_native_gem strategy: fail-fast: false matrix: + os: [macos] ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] - platform: [arm-linux] - include: - # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix - - ruby: "3.3" - platform: arm-linux-gnu - - ruby: "3.2" - platform: arm-linux-gnu - - ruby: "3.1" - platform: arm-linux-gnu - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Download gem-${{ matrix.platform }} - uses: actions/download-artifact@v3 - with: - name: gem-${{ matrix.platform }} - - name: Run tests - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker run --rm -v $PWD:/work -w /work \ - --platform=linux/arm/v7 ruby:${{ matrix.ruby }} \ - sh -c " - gem install --local *.gem --verbose && - cd test/rcd_test/ && - bundle install && - ruby -rrcd_test -S rake test - " - - test_arm-linux-musl: - name: arm-linux-musl - needs: build_native_gem - strategy: - fail-fast: false - matrix: - ruby: ["3.3", "3.2", "3.1"] # TODO can we update rubygems and get earlier images working? - platform: [arm-linux-musl] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Download gem-${{ matrix.platform }} - uses: actions/download-artifact@v3 - with: - name: gem-${{ matrix.platform }} - - name: Run tests - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker run --rm -v $PWD:/work -w /work \ - --platform=linux/arm/v7 ruby:${{ matrix.ruby }}-alpine \ - sh -c " - gem install --local *.gem --verbose && - cd test/rcd_test/ && - bundle install && - ruby -rrcd_test -S rake test - " - - test_aarch64-linux-gnu: - name: aarch64-linux-gnu - needs: build_native_gem - strategy: - fail-fast: false - matrix: - ruby: ["3.3", "3.2", "3.1", "3.0", "2.7", "2.6", "2.5", "2.4"] - platform: [aarch64-linux] - include: - # ruby 3.0 and earlier ship rubygems < 3.2.33, so can't recognize the -gnu suffix - - ruby: "3.3" - platform: aarch64-linux-gnu - - ruby: "3.2" - platform: aarch64-linux-gnu - - ruby: "3.1" - platform: aarch64-linux-gnu - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Download gem-${{ matrix.platform }} - uses: actions/download-artifact@v3 - with: - name: gem-${{ matrix.platform }} - - name: Run tests - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker run --rm -v $PWD:/work -w /work \ - --platform=linux/arm64 ruby:${{ matrix.ruby }} \ - sh -c " - gem install --local *.gem --verbose && - cd test/rcd_test/ && - bundle install && - ruby -rrcd_test -S rake test - " - - test_aarch64-linux-musl: - name: aarch64-linux-musl - needs: build_native_gem - strategy: - fail-fast: false - matrix: - ruby: ["3.3", "3.2", "3.1", "3.0", "2.7"] # ruby:2.6-alpine and earlier ship with rubygems that doesn't recognize the -musl suffix - platform: [aarch64-linux-musl] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Download gem-${{ matrix.platform }} - uses: actions/download-artifact@v3 - with: - name: gem-${{ matrix.platform }} - - name: Run tests - run: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - docker run --rm -v $PWD:/work -w /work \ - --platform=linux/arm64 ruby:${{ matrix.ruby }}-alpine \ - sh -c " - gem install --local *.gem --verbose && - cd test/rcd_test/ && - bundle install && - ruby -rrcd_test -S rake test - " - - test_native_gem: - name: test native - needs: build_native_gem - strategy: - fail-fast: false - matrix: - os: - - macos - ruby: - - "3.3" - - "3.2" - - "3.1" - - "3.0" - - "2.7" - - "2.6" - - "2.5" - - "2.4" include: - os: macos platform: x86_64-darwin @@ -504,16 +307,15 @@ jobs: uses: actions/download-artifact@v3 with: name: gem-${{ matrix.platform }} - - name: Install gem-${{ matrix.platform }} - run: gem install --local *.gem --verbose - - name: Run tests + - name: Test gem-${{ matrix.platform }} run: | + gem install --local *.gem --verbose cd test/rcd_test/ bundle install ruby -rrcd_test -S rake test - test_static_native_gem: - name: test static + test_windows_static: + name: "static ${{ matrix.platform }} ${{ matrix.ruby }}" needs: build_native_gem strategy: fail-fast: false @@ -557,16 +359,15 @@ jobs: uses: actions/download-artifact@v3 with: name: gem-${{ matrix.platform }}-static - - name: Install gem-${{ matrix.platform }}-static - run: gem install --local *.gem --verbose - - name: Run tests + - name: Test gem-${{ matrix.platform }}-static run: | + gem install --local *.gem --verbose cd test/rcd_test/ bundle install ruby -rrcd_test -S rake test - test_native_gem_multiarch: - name: ${{ matrix.platform }} on ${{ matrix.from_image }} + test_ad_hoc: + name: "${{ matrix.platform }} on ${{ matrix.from_image }}" needs: build_native_gem strategy: fail-fast: false From cfe45734732e650f0fb70ba562625b416a800fad Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Fri, 19 Jan 2024 08:42:11 -0500 Subject: [PATCH 09/10] doc: update README with GNU/Musl description. --- README.md | 54 +++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 47 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index a0ac8c7..058c2e5 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,43 @@ This is kind of successor of [rake-compiler-dev-box](https://github.com/tjschuck It is wrapped as a gem for easier setup, usage and integration and is based on lightweight Docker containers. It is also more reliable, since the underlying docker images are versioned and immutable. +## Supported platforms + +The following platforms are supported for cross-compilation by rake-compiler-dock: + +- `aarch64-linux` and `aarch64-linux-gnu` +- `aarch64-linux-musl` +- `arm-linux` and `arm-linux-gnu` +- `arm-linux-musl` +- `arm64-darwin` +- `jruby` +- `x64-mingw-ucrt` +- `x64-mingw32` +- `x86-linux` and `x86-linux-gnu` +- `x86-linux-musl` +- `x86-mingw32` +- `x86_64-darwin` +- `x86_64-linux` and `x86_64-linux-gnu` +- `x86_64-linux-musl` + +### Windows + +`x64-mingw-ucrt` should be used for Ruby 3.1 and later on windows. `x64-mingw32` should be used for Ruby 3.0 and earlier. This is to match the [changed platform of RubyInstaller-3.1](https://rubyinstaller.org/2021/12/31/rubyinstaller-3.1.0-1-released.html). + +### GNU and Musl + +Platform names with a `*-linux` suffix are aliases for `*-linux-gnu`, since the Rubygems default is to assume `gnu` if no libc is specified. + +Some C extensions may not require separate GNU and Musl builds, in which case it's acceptable to ship a single `*-linux` gem to cover both platforms. + +The `*-linux-gnu` and `*-linux-musl` platform name suffixes require Rubygems 3.3.22 or later (or Bundler 2.3.21 or later) at installation time. Ruby version 3.0 and later ship with a sufficient Rubygems version, but versions compatible with earlier Rubies are: + +- ruby: "2.7", rubygems: "3.4.22" +- ruby: "2.6", rubygems: "3.4.22" +- ruby: "2.5", rubygems: "3.3.26" +- ruby: "2.4", rubygems: "3.3.26" + + ## Installation Install docker [following the instructions on the docker website](https://docs.docker.com/engine/install/) ... or install [docker-toolbox for Windows and OSX](https://github.com/docker/toolbox/releases) or boot2docker on [Windows](https://github.com/boot2docker/windows-installer/releases) or [OS X](https://github.com/boot2docker/osx-installer/releases) . @@ -33,10 +70,12 @@ Your Rakefile should enable cross compilation like so: ```ruby exttask = Rake::ExtensionTask.new('my_extension', my_gem_spec) do |ext| ext.cross_compile = true - ext.cross_platform = %w[x86-mingw32 x64-mingw-ucrt x64-mingw32 x86-linux-gnu x86_64-linux-gnu x86_64-darwin arm64-darwin] + ext.cross_platform = %w[x86-mingw32 x64-mingw-ucrt x64-mingw32 x86-linux x86_64-linux x86_64-darwin arm64-darwin] end ``` +where you should choose your platforms from the list in the "Supported platforms" section. + See below, how to invoke cross builds in your Rakefile. Additionally it may also be used to build ffi based binary gems like [libusb](https://github.com/larskanis/libusb), but currently doesn't provide any additional build helpers for this use case, beyond docker invocation and cross compilers. @@ -44,9 +83,6 @@ Additionally it may also be used to build ffi based binary gems like [libusb](ht ### Interactive Usage Rake-compiler-dock offers the shell command `rake-compiler-dock` and a [ruby API](http://www.rubydoc.info/gems/rake-compiler-dock/RakeCompilerDock) for issuing commands within the docker image, described below. -There are dedicated images for targets: `aarch64-linux-gnu`, `arm-linux-gnu`, `arm64-darwin`, `x64-mingw-ucrt`, `x64-mingw32`, `x86-linux-gnu`, `x86-mingw32`, `x86_64-darwin`, `x86_64-linux-gnu`, and `jruby`. -The images contain all supported cross ruby versions, with the exception of `x64-mingw32`, which has versions before 3.1 only, and `x64-mingw-ucrt`, which has only ruby-3.1+. -This is to match the [changed platform of RubyInstaller-3.1](https://rubyinstaller.org/2021/12/31/rubyinstaller-3.1.0-1-released.html). `rake-compiler-dock` without arguments starts an interactive shell session. This is best suited to try out and debug a build. @@ -68,7 +104,7 @@ To build x86 Windows and x86_64 Linux binary gems interactively, it can be calle user@host:$ ls pkg/*.gem your-gem-1.0.0.gem your-gem-1.0.0-x86-mingw32.gem - user@host:$ RCD_PLATFORM=x86_64-linux-gnu rake-compiler-dock # this enters a container for amd64 Linux target + user@host:$ RCD_PLATFORM=x86_64-linux-gnu rake-compiler-dock # this enters a container for amd64 Linux GNU target user@adc55b2b92a9:$ bundle user@adc55b2b92a9:$ rake cross native gem user@adc55b2b92a9:$ exit @@ -120,14 +156,18 @@ This can be done like this: ```ruby PLATFORMS = %w[ aarch64-linux-gnu + aarch64-linux-musl arm-linux-gnu + arm-linux-musl arm64-darwin x64-mingw-ucrt x64-mingw32 x86-linux-gnu + x86-linux-musl x86-mingw32 x86_64-darwin x86_64-linux-gnu + x86_64-linux-musl ] task 'gem:native' do require 'rake_compiler_dock' @@ -212,7 +252,7 @@ jobs: Where the referenced rake task might be defined by something like: ``` ruby -cross_platforms = ["x64-mingw32", "x86_64-linux-gnu", "x86_64-darwin", "arm64-darwin"] +cross_platforms = ["x64-mingw32", "x86_64-linux", "x86_64-darwin", "arm64-darwin"] namespace "gem" do cross_platforms.each do |platform| @@ -248,7 +288,7 @@ The following variables are recognized by rake-compiler-dock: * `RCD_RUBYVM` - The ruby VM and toolchain to be used. Must be one of `mri`, `jruby`. * `RCD_PLATFORM` - The target rubygems platform. - Must be a space separated list out of `aarch64-linux-gnu`, `arm-linux-gnu`, `arm64-darwin`, `x64-mingw-ucrt`, `x64-mingw32`, `x86-linux-gnu`, `x86-mingw32`, `x86_64-darwin`, `x86_64-linux-gnu`. + Must be a space separated list out of the platforms listed under "Supported platforms" above. It is ignored when `rubyvm` is set to `:jruby`. * `RCD_IMAGE` - The docker image that is downloaded and started. Defaults to "ghcr.io/rake-compiler/rake-compiler-dock-image:IMAGE_VERSION-PLATFORM" with an image version that is determined by the gem version. From 93fce127c7c07c5ca6b7249e7d0202203a58e006 Mon Sep 17 00:00:00 2001 From: Mike Dalessio Date: Tue, 23 Jan 2024 08:08:02 -0500 Subject: [PATCH 10/10] update rake-compiler patch to apply cleanly to 1.2.5 --- .../0004-Enable-build-of-static-libruby.patch | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/build/patches2/rake-compiler-1.2.5/0004-Enable-build-of-static-libruby.patch b/build/patches2/rake-compiler-1.2.5/0004-Enable-build-of-static-libruby.patch index a48c31c..b591792 100644 --- a/build/patches2/rake-compiler-1.2.5/0004-Enable-build-of-static-libruby.patch +++ b/build/patches2/rake-compiler-1.2.5/0004-Enable-build-of-static-libruby.patch @@ -1,8 +1,8 @@ diff --git a/tasks/bin/cross-ruby.rake b/tasks/bin/cross-ruby.rake -index 8317a2a..5a1b3ea 100644 +index 8317a2a3..8ed21718 100644 --- a/tasks/bin/cross-ruby.rake +++ b/tasks/bin/cross-ruby.rake -@@ -116,10 +116,26 @@ RUBY_CC_VERSIONS.split(":").each do |ruby_cc_version| +@@ -116,11 +116,27 @@ "--host=#{mingw_host}", "--target=#{mingw_target}", "--build=#{RUBY_BUILD}", @@ -12,7 +12,7 @@ index 8317a2a..5a1b3ea 100644 '--disable-install-doc', '--with-ext=', ] - + + if mingw_host =~ /darwin/ + options += [ + '--enable-static', @@ -30,3 +30,4 @@ index 8317a2a..5a1b3ea 100644 + # Force Winsock2 for Ruby 1.8, 1.9 defaults to it options << "--with-winsock2" if major == "1.8" + options << "--prefix=#{install_dir}"