diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f17a12cdbb..5c255c0dd13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,6 +46,7 @@ jobs: - uses: actions/checkout@v1 # v1 because of https://github.com/actions/checkout/issues/334 with: submodules: true + - run: apt install -y autoconf - run: bundle install --local || bundle install - run: bundle exec rake compile -- --${{matrix.sys}}-system-libraries - run: bundle exec rake test diff --git a/ext/nokogiri/extconf.rb b/ext/nokogiri/extconf.rb index 8ea793d2a43..5fd9d7c0e0d 100644 --- a/ext/nokogiri/extconf.rb +++ b/ext/nokogiri/extconf.rb @@ -1055,50 +1055,8 @@ def configure $VPATH << "$(srcdir)/../../gumbo-parser/src" find_header("nokogiri_gumbo.h") || abort("nokogiri_gumbo.h not found") else - libgumbo_recipe = process_recipe("libgumbo", "1.0.0-nokogiri", static_p, cross_build_p, false) do |recipe| - recipe.configure_options = [] - - class << recipe - def downloaded? - true - end - - def extract - target = File.join(tmp_path, "gumbo-parser") - output("Copying gumbo-parser files into #{target}...") - FileUtils.mkdir_p(target) - FileUtils.cp(Dir.glob(File.join(PACKAGE_ROOT_DIR, "gumbo-parser/src/*")), target) - end - - def configured? - true - end - - def install - lib_dir = File.join(port_path, "lib") - inc_dir = File.join(port_path, "include") - FileUtils.mkdir_p([lib_dir, inc_dir]) - FileUtils.cp(File.join(work_path, "libgumbo.a"), lib_dir) - FileUtils.cp(Dir.glob(File.join(work_path, "*.h")), inc_dir) - end - - def compile - cflags = concat_flags(ENV["CFLAGS"], "-fPIC", "-O2", "-g") - - env = { "CC" => gcc_cmd, "CFLAGS" => cflags } - if config_cross_build? - if host.include?("darwin") - env["AR"] = "#{host}-libtool" - env["ARFLAGS"] = "-o" - else - env["AR"] = "#{host}-ar" - end - env["RANLIB"] = "#{host}-ranlib" - end - - execute("compile", make_cmd, { env: env }) - end - end + libgumbo_recipe = process_recipe("libgumbo", "1.0.0-nokogiri", true, cross_build_p, false) do |recipe| + recipe.source_directory = File.join(PACKAGE_ROOT_DIR, "gumbo-parser") end append_cppflags("-I#{File.join(libgumbo_recipe.path, "include")}") $libs = $libs + " " + File.join(libgumbo_recipe.path, "lib", "libgumbo.a") @@ -1142,6 +1100,7 @@ def compile File.open("Makefile", "at") do |mk| mk.print(<<~EOF) + .PHONY: clean-ports all: clean-ports clean-ports: $(DLLIB) \t-$(Q)$(RUBY) $(srcdir)/extconf.rb --clean --#{static_p ? "enable" : "disable"}-static diff --git a/gumbo-parser/.gitignore b/gumbo-parser/.gitignore index 37a46fc2b3d..afe2b1da42c 100644 --- a/gumbo-parser/.gitignore +++ b/gumbo-parser/.gitignore @@ -1,5 +1,10 @@ -build -googletest -src/*.o -fuzzer/build -src/libgumbo.a \ No newline at end of file +/build +/googletest +/src/*.o +/fuzzer/build +/src/libgumbo.a +/aclocal.m4 +/autom4te.cache/ +/configure +/configure.in +Makefile.in diff --git a/gumbo-parser/Makefile.am b/gumbo-parser/Makefile.am new file mode 100644 index 00000000000..591ff85c08a --- /dev/null +++ b/gumbo-parser/Makefile.am @@ -0,0 +1,5 @@ +if HAS_TESTS + SUBDIRS = src test +else + SUBDIRS = src +endif diff --git a/gumbo-parser/Makefile b/gumbo-parser/Makefile.old similarity index 100% rename from gumbo-parser/Makefile rename to gumbo-parser/Makefile.old diff --git a/gumbo-parser/build-aux/.gitignore b/gumbo-parser/build-aux/.gitignore new file mode 100644 index 00000000000..72fd7022e87 --- /dev/null +++ b/gumbo-parser/build-aux/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory except for this file. +# Credit: https://stackoverflow.com/a/932982 +* +!/.gitignore diff --git a/gumbo-parser/configure.ac b/gumbo-parser/configure.ac new file mode 100644 index 00000000000..ba8608597df --- /dev/null +++ b/gumbo-parser/configure.ac @@ -0,0 +1,11 @@ +AC_INIT([Gumbo], [1.0]) +AC_CONFIG_SRCDIR([src/nokogiri_gumbo.h]) +AC_CONFIG_AUX_DIR([build-aux]) +AM_INIT_AUTOMAKE([subdir-objects foreign serial-tests no-dist no-installinfo no-installman -Wall]) +AC_PROG_RANLIB +AC_PROG_CC +AC_PROG_CXX +AM_PROG_AR +AM_CONDITIONAL([HAS_TESTS], [test -d "${srcdir}/test"]) +AC_CONFIG_FILES([Makefile src/Makefile test/Makefile]) +AC_OUTPUT diff --git a/gumbo-parser/src/Makefile b/gumbo-parser/src/Makefile deleted file mode 100644 index 6bd4a18fbed..00000000000 --- a/gumbo-parser/src/Makefile +++ /dev/null @@ -1,34 +0,0 @@ -# this Makefile is used by ext/nokogiri/extconf.rb -# to enable a mini_portile2 recipe to build the gumbo parser -.PHONY: clean - -CFLAGS += -std=c99 -Wall - -# allow the ENV var to override this -RANLIB ?= ranlib - -gumbo_objs := \ - ascii.o \ - attribute.o \ - char_ref.o \ - error.o \ - foreign_attrs.o \ - parser.o \ - string_buffer.o \ - string_piece.o \ - svg_attrs.o \ - svg_tags.o \ - tag.o \ - tag_lookup.o \ - token_buffer.o \ - tokenizer.o \ - utf8.o \ - util.o \ - vector.o - -libgumbo.a: $(gumbo_objs) - $(AR) $(ARFLAGS) $@ $(gumbo_objs) - - ($(RANLIB) $@ || true) >/dev/null 2>&1 - -clean: - rm -f $(gumbo_objs) libgumbo.a diff --git a/gumbo-parser/src/Makefile.am b/gumbo-parser/src/Makefile.am new file mode 100644 index 00000000000..f41055b415f --- /dev/null +++ b/gumbo-parser/src/Makefile.am @@ -0,0 +1,20 @@ +lib_LIBRARIES = libgumbo.a +libgumbo_a_SOURCES = \ + ascii.c \ + attribute.c \ + char_ref.c \ + error.c \ + foreign_attrs.c \ + parser.c \ + string_buffer.c \ + string_piece.c \ + svg_attrs.c \ + svg_tags.c \ + tag.c \ + tag_lookup.c \ + token_buffer.c \ + tokenizer.c \ + utf8.c \ + util.c \ + vector.c +include_HEADERS = nokogiri_gumbo.h diff --git a/gumbo-parser/test/Makefile.am b/gumbo-parser/test/Makefile.am new file mode 100644 index 00000000000..e5ad1de2b70 --- /dev/null +++ b/gumbo-parser/test/Makefile.am @@ -0,0 +1,21 @@ +check_LIBRARIES = libgtest_main.a +libgtest_main_a_SOURCES = ../googletest/src/gtest-all.cc ../googletest/src/gtest_main.cc +libgtest_main_a_CPPFLAGS = -I$(top_srcdir)/googletest/include -I$(top_srcdir)/googletest +libgtest_main_a_CXXFLAGS = -pthread + +check_PROGRAMS = gumbotest +gumbotest_SOURCES = attribute.cc \ + parser.cc \ + string_buffer.cc \ + string_piece.cc \ + test_utils.cc \ + token_buffer.cc \ + tokenizer.cc \ + utf8.cc \ + vector.cc + +gumbotest_LDADD = libgtest_main.a ../src/libgumbo.a +gumbotest_LDFLAGS = -pthread +gumbotest_CPPFLAGS = -I$(top_srcdir)/googletest/include -I$(top_srcdir)/src + +TESTS = gumbotest diff --git a/nokogiri.gemspec b/nokogiri.gemspec index 44322172004..da30f102230 100644 --- a/nokogiri.gemspec +++ b/nokogiri.gemspec @@ -182,9 +182,13 @@ Gem::Specification.new do |spec| "ext/nokogiri/xml_xpath_context.c", "ext/nokogiri/xslt_stylesheet.c", "gumbo-parser/CHANGES.md", - "gumbo-parser/Makefile", + "gumbo-parser/Makefile.am", + "gumbo-parser/Makefile.in", "gumbo-parser/THANKS", - "gumbo-parser/src/Makefile", + "gumbo-parser/configure", + "gumbo-parser/configure.ac", + "gumbo-parser/src/Makefile.am", + "gumbo-parser/src/Makefile.in", "gumbo-parser/src/README.md", "gumbo-parser/src/ascii.c", "gumbo-parser/src/ascii.h", diff --git a/rakelib/check-manifest.rake b/rakelib/check-manifest.rake index 50ca2d53eb5..53e391b9923 100644 --- a/rakelib/check-manifest.rake +++ b/rakelib/check-manifest.rake @@ -56,7 +56,12 @@ task :check_manifest, [:verbose] do |_, args| [0-9]* appveyor.yml **/compile_commands.json + gumbo-parser/Makefile.old + gumbo-parser/aclocal.m4 + gumbo-parser/autom4te.cache/* + gumbo-parser/build-aux/* gumbo-parser/fuzzer/* + gumbo-parser/googletest/* gumbo-parser/test/* gumbo-parser/gperf-filter.sed lib/nokogiri/**/nokogiri.{jar,so} diff --git a/rakelib/extensions.rake b/rakelib/extensions.rake index 66867f407f2..0cc974f82ec 100644 --- a/rakelib/extensions.rake +++ b/rakelib/extensions.rake @@ -444,7 +444,7 @@ else end Rake::ExtensionTask.new("nokogiri", NOKOGIRI_SPEC.dup) do |ext| - ext.source_pattern = "*.{c,cc,cpp,h}" + ext.source_pattern = "{.,../../gumbo-parser/src}/*.{c,cc,cpp,h}" ext.gem_spec.files.reject! { |path| File.fnmatch?("**/*.{java,jar}", path, File::FNM_EXTGLOB) } ext.lib_dir = File.join(*["lib", "nokogiri", ENV["FAT_DIR"]].compact) diff --git a/rakelib/gumbo.rake b/rakelib/gumbo.rake index f574c2697c8..4ced6621278 100644 --- a/rakelib/gumbo.rake +++ b/rakelib/gumbo.rake @@ -1,13 +1,18 @@ # frozen_string_literal: true +# We want to run the gumbo test suite using exactly the same compiled gumbo-parser +# that Nokogiri uses. +# +# To that end, we first need to get the Rake ExtensionTask to run extconf.rb which will +# run the gumbo-parser's configure script. We don't want to compile the extension +# at this point, so we make `gumbo:test` depend on the Nokogiri Makefile. namespace "gumbo" do - gtest_pkg = "gumbo-parser/googletest" - gtest_lib = File.join(gtest_pkg, "make/gtest_main.a") - - file gtest_lib => gtest_pkg do - sh("make -C gumbo-parser/googletest/make gtest_main.a") - end + host = RbConfig::CONFIG["host"] + nokogiri_makefile = File.join("tmp/#{RUBY_PLATFORM}/nokogiri/#{RUBY_VERSION}/Makefile") + gumbotest_builddir = "tmp/#{RUBY_PLATFORM}/nokogiri/#{RUBY_VERSION}/tmp/#{host}/ports/libgumbo/1.0.0-nokogiri/libgumbo-1.0.0-nokogiri" + gumbotest_configure = File.absolute_path("gumbo-parser/configure") + gtest_pkg = "gumbo-parser/googletest" file gtest_pkg do sh(<<~EOF) curl -L https://github.com/google/googletest/archive/release-1.8.0.tar.gz | \ @@ -16,19 +21,26 @@ namespace "gumbo" do end desc "Run the gumbo parser test suite" - task "test" => gtest_lib do - sh("make -j2 -C gumbo-parser") + task "test" => [nokogiri_makefile, gtest_pkg] do + sh("make", "-j2", "-C", gumbotest_builddir, "check") end - desc "Clean up after the gumbo parser test suite" - task "clean" do - sh("make -j2 -C gumbo-parser clean") + file gumbotest_configure do + sh("autoreconf", "-fiv", chdir: "gumbo-parser") end + # Make sure the libgumbo configure script is created before trying to compile the extension. + file nokogiri_makefile => gumbotest_configure + CLOBBER.add(gtest_pkg) + CLOBBER.add(gumbotest_configure) + CLOBBER.add("gumbo-parser/Makefile.in") + CLOBBER.add("gumbo-parser/src/Makefile.in") + CLOBBER.add("gumbo-parser/test/Makefile.in") + CLOBBER.add("gumbo-parser/build-aux/*") + CLOBBER.add("gumbo-parser/autom4te.cache") + CLOBBER.add("gumbo-parser/aclocal.m4") end desc "Run the gumbo parser test suite" task "gumbo" => "gumbo:test" - -task "clean" => "gumbo:clean" # rubocop:disable Rake/Desc