From 356a15ad31b1dbec0ef0b2a9b2cc4f1930558c97 Mon Sep 17 00:00:00 2001 From: odenix Date: Thu, 5 Dec 2024 20:28:24 -0800 Subject: [PATCH 1/2] Support customizing SymbolLookup for tree-sitter native library Motivation: - Enable clients to customize the SymbolLookup used for the tree-sitter native library. For example, clients can use this capability to embed the tree-sitter native library in their JAR, which is a common way to solve the native library distribution problem. Changes: - Add interface NativeLibraryLookup. - Add class ChainedLibraryLookup, which loads NativeLibraryLookup implementations using java.util.ServiceLoader and chains the SymbolLookup's returned by them. - Change TreeSitter.java patch to delegate to ChainedLibraryLookup. - Update build instructions in README. - Fix execution of jextract.ps1 script in Windows build. Result: - Clients that need to customize the SymbolLookup used for the tree-sitter native library no longer need to patch and build java-tree-sitter from source. --- README.md | 3 ++ pom.xml | 3 +- scripts/TreeSitter_java.patch | 21 ++++---------- .../jtreesitter/NativeLibraryLookup.java | 21 ++++++++++++++ .../internal/ChainedLibraryLookup.java | 29 +++++++++++++++++++ 5 files changed, 61 insertions(+), 16 deletions(-) create mode 100644 src/main/java/io/github/treesitter/jtreesitter/NativeLibraryLookup.java create mode 100644 src/main/java/io/github/treesitter/jtreesitter/internal/ChainedLibraryLookup.java diff --git a/README.md b/README.md index 03e37dd..aaec60f 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,9 @@ Java bindings to the [tree-sitter] parsing library. git clone https://github.com/tree-sitter/java-tree-sitter cd java-tree-sitter git submodule init +git submodule update +# build tree-sitter and tree-sitter-java native libraries, +# for example by replicating the steps in .github/workflows/ci.yml mvn test ``` diff --git a/pom.xml b/pom.xml index ab8107c..63f6e14 100644 --- a/pom.xml +++ b/pom.xml @@ -81,7 +81,8 @@ ${jextract.skip} Generating sources using jextract - + + diff --git a/scripts/TreeSitter_java.patch b/scripts/TreeSitter_java.patch index c1a4350..4919bea 100644 --- a/scripts/TreeSitter_java.patch +++ b/scripts/TreeSitter_java.patch @@ -1,22 +1,13 @@ ---- a/generated-sources/jextract/io/github/treesitter/jtreesitter/internal/TreeSitter.java -+++ b/generated-sources/jextract/io/github/treesitter/jtreesitter/internal/TreeSitter.java -@@ -55,9 +55,16 @@ public class TreeSitter { +--- a/target/generated-sources/jextract/io/github/treesitter/jtreesitter/internal/TreeSitter.java ++++ b/target/generated-sources/jextract/io/github/treesitter/jtreesitter/internal/TreeSitter.java +@@ -55,9 +55,7 @@ }; } - + - static final SymbolLookup SYMBOL_LOOKUP = SymbolLookup.libraryLookup(System.mapLibraryName("tree-sitter"), LIBRARY_ARENA) - .or(SymbolLookup.loaderLookup()) - .or(Linker.nativeLinker().defaultLookup()); -+ static final SymbolLookup SYMBOL_LOOKUP = findLibrary().or(Linker.nativeLinker().defaultLookup()); -+ -+ private static final SymbolLookup findLibrary() { -+ try { -+ String library = System.mapLibraryName("tree-sitter"); -+ return SymbolLookup.libraryLookup(library, LIBRARY_ARENA); -+ } catch (IllegalArgumentException e) { -+ return SymbolLookup.loaderLookup(); -+ } -+ } - ++ static final SymbolLookup SYMBOL_LOOKUP = new ChainedLibraryLookup().get(LIBRARY_ARENA); + public static final ValueLayout.OfBoolean C_BOOL = ValueLayout.JAVA_BOOLEAN; public static final ValueLayout.OfByte C_CHAR = ValueLayout.JAVA_BYTE; diff --git a/src/main/java/io/github/treesitter/jtreesitter/NativeLibraryLookup.java b/src/main/java/io/github/treesitter/jtreesitter/NativeLibraryLookup.java new file mode 100644 index 0000000..12baa36 --- /dev/null +++ b/src/main/java/io/github/treesitter/jtreesitter/NativeLibraryLookup.java @@ -0,0 +1,21 @@ +package io.github.treesitter.jtreesitter; + +import java.lang.foreign.Arena; +import java.lang.foreign.SymbolLookup; + +/** + * An interface implemented by clients that wish to customize the {@link SymbolLookup} used for the tree-sitter + * native library. Implementation classes must be registered by listing their fully qualified class name + * in a resource file named {@code META-INF/services/io.github.treesitter.jtreesitter.NativeLibraryLookup}. + * + * @see java.util.ServiceLoader + */ +public interface NativeLibraryLookup { + /** + * Returns the {@link SymbolLookup} to be used for the tree-sitter native library. + * + * @param arena an arena for the symbol lookup + * @return the {@link SymbolLookup} to be used for the tree-sitter native library + */ + SymbolLookup get(Arena arena); +} diff --git a/src/main/java/io/github/treesitter/jtreesitter/internal/ChainedLibraryLookup.java b/src/main/java/io/github/treesitter/jtreesitter/internal/ChainedLibraryLookup.java new file mode 100644 index 0000000..27986ce --- /dev/null +++ b/src/main/java/io/github/treesitter/jtreesitter/internal/ChainedLibraryLookup.java @@ -0,0 +1,29 @@ +package io.github.treesitter.jtreesitter.internal; + +import io.github.treesitter.jtreesitter.NativeLibraryLookup; +import java.lang.foreign.Arena; +import java.lang.foreign.Linker; +import java.lang.foreign.SymbolLookup; +import java.util.Optional; +import java.util.ServiceLoader; + +final class ChainedLibraryLookup implements NativeLibraryLookup { + @Override + public SymbolLookup get(Arena arena) { + var serviceLoader = ServiceLoader.load(NativeLibraryLookup.class); + SymbolLookup lookup = (name) -> Optional.empty(); + for (var libraryLookup : serviceLoader) { + lookup = lookup.or(libraryLookup.get(arena)); + } + return lookup.or(findLibrary(arena)).or(Linker.nativeLinker().defaultLookup()); + } + + private SymbolLookup findLibrary(Arena arena) { + try { + String library = System.mapLibraryName("tree-sitter"); + return SymbolLookup.libraryLookup(library, arena); + } catch (IllegalArgumentException e) { + return SymbolLookup.loaderLookup(); + } + } +} From 3846ad4cca574bb1a627620e7baaa13691281c38 Mon Sep 17 00:00:00 2001 From: odenix Date: Fri, 6 Dec 2024 12:02:35 -0800 Subject: [PATCH 2/2] Incorporate review feedback --- README.md | 6 ++---- pom.xml | 3 +-- scripts/TreeSitter_java.patch | 4 ++-- .../github/treesitter/jtreesitter/NativeLibraryLookup.java | 1 + .../jtreesitter/internal/ChainedLibraryLookup.java | 5 +++-- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index aaec60f..95ccc0a 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,12 @@ Java bindings to the [tree-sitter] parsing library. - Install JDK 22 and set `JAVA_HOME` to it - Download [jextract] and add it to your `PATH` +- Install the `tree-sitter` & `tree-sitter-java` libraries ```bash git clone https://github.com/tree-sitter/java-tree-sitter cd java-tree-sitter -git submodule init -git submodule update -# build tree-sitter and tree-sitter-java native libraries, -# for example by replicating the steps in .github/workflows/ci.yml +git submodule update --init mvn test ``` diff --git a/pom.xml b/pom.xml index 63f6e14..ab8107c 100644 --- a/pom.xml +++ b/pom.xml @@ -81,8 +81,7 @@ ${jextract.skip} Generating sources using jextract - - + diff --git a/scripts/TreeSitter_java.patch b/scripts/TreeSitter_java.patch index 4919bea..981f902 100644 --- a/scripts/TreeSitter_java.patch +++ b/scripts/TreeSitter_java.patch @@ -1,5 +1,5 @@ ---- a/target/generated-sources/jextract/io/github/treesitter/jtreesitter/internal/TreeSitter.java -+++ b/target/generated-sources/jextract/io/github/treesitter/jtreesitter/internal/TreeSitter.java +--- a/generated-sources/jextract/io/github/treesitter/jtreesitter/internal/TreeSitter.java ++++ b/generated-sources/jextract/io/github/treesitter/jtreesitter/internal/TreeSitter.java @@ -55,9 +55,7 @@ }; } diff --git a/src/main/java/io/github/treesitter/jtreesitter/NativeLibraryLookup.java b/src/main/java/io/github/treesitter/jtreesitter/NativeLibraryLookup.java index 12baa36..af0c04b 100644 --- a/src/main/java/io/github/treesitter/jtreesitter/NativeLibraryLookup.java +++ b/src/main/java/io/github/treesitter/jtreesitter/NativeLibraryLookup.java @@ -10,6 +10,7 @@ * * @see java.util.ServiceLoader */ +@FunctionalInterface public interface NativeLibraryLookup { /** * Returns the {@link SymbolLookup} to be used for the tree-sitter native library. diff --git a/src/main/java/io/github/treesitter/jtreesitter/internal/ChainedLibraryLookup.java b/src/main/java/io/github/treesitter/jtreesitter/internal/ChainedLibraryLookup.java index 27986ce..6b72ffc 100644 --- a/src/main/java/io/github/treesitter/jtreesitter/internal/ChainedLibraryLookup.java +++ b/src/main/java/io/github/treesitter/jtreesitter/internal/ChainedLibraryLookup.java @@ -11,6 +11,7 @@ final class ChainedLibraryLookup implements NativeLibraryLookup { @Override public SymbolLookup get(Arena arena) { var serviceLoader = ServiceLoader.load(NativeLibraryLookup.class); + // NOTE: can't use _ because of palantir/palantir-java-format#934 SymbolLookup lookup = (name) -> Optional.empty(); for (var libraryLookup : serviceLoader) { lookup = lookup.or(libraryLookup.get(arena)); @@ -18,9 +19,9 @@ public SymbolLookup get(Arena arena) { return lookup.or(findLibrary(arena)).or(Linker.nativeLinker().defaultLookup()); } - private SymbolLookup findLibrary(Arena arena) { + private static SymbolLookup findLibrary(Arena arena) { try { - String library = System.mapLibraryName("tree-sitter"); + var library = System.mapLibraryName("tree-sitter"); return SymbolLookup.libraryLookup(library, arena); } catch (IllegalArgumentException e) { return SymbolLookup.loaderLookup();