From 1327419123a4636980ac14e2be0ccbe584d6a920 Mon Sep 17 00:00:00 2001 From: Tamas Balog Date: Wed, 18 Sep 2024 14:05:13 +0200 Subject: [PATCH 1/3] Update the project to use the IntelliJ Platform Gradle Plugin 2.0 and Java 21 --- .github/workflows/build.yml | 4 +- .gitignore | 1 + CHANGELOG.md | 8 +- NOTICE.txt | 4 +- README.md | 14 + build.gradle.kts | 110 ++- gradle.properties | 11 +- gradle/libs.versions.toml | 12 +- .../menubar/CreateIndexDialogFactory.java | 20 +- .../apache/lucene/store/MMapDirectory.java | 469 ++++++++++++ .../store/MemorySegmentAccessInput.java | 39 + .../lucene/store/MemorySegmentIndexInput.java | 675 ++++++++++++++++++ .../MemorySegmentIndexInputProvider.java | 137 ++++ .../org/apache/lucene/store/NativeAccess.java | 39 + .../lucene/store/PosixNativeAccess.java | 153 ++++ 15 files changed, 1642 insertions(+), 54 deletions(-) create mode 100644 src/main/java/org/apache/lucene/store/MMapDirectory.java create mode 100644 src/main/java/org/apache/lucene/store/MemorySegmentAccessInput.java create mode 100644 src/main/java/org/apache/lucene/store/MemorySegmentIndexInput.java create mode 100644 src/main/java/org/apache/lucene/store/MemorySegmentIndexInputProvider.java create mode 100644 src/main/java/org/apache/lucene/store/NativeAccess.java create mode 100644 src/main/java/org/apache/lucene/store/PosixNativeAccess.java diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 51c3e12..b4ca805 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -71,8 +71,6 @@ jobs: echo "$CHANGELOG" >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - ./gradlew listProductsReleases # prepare list of IDEs for Plugin Verifier - # Build plugin - name: Build plugin run: ./gradlew buildPlugin @@ -171,7 +169,7 @@ jobs: # Run Verify Plugin task and IntelliJ Plugin Verifier tool - name: Run Plugin Verification tasks - run: ./gradlew runPluginVerifier -Dplugin.verifier.home.dir=${{ needs.build.outputs.pluginVerifierHomeDir }} + run: ./gradlew verifyPlugin -Dplugin.verifier.home.dir=${{ needs.build.outputs.pluginVerifierHomeDir }} # Collect Plugin Verifier Result - name: Collect Plugin Verifier Result diff --git a/.gitignore b/.gitignore index e2e5d94..f7b1125 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .idea .qodana build +.intellijPlatform diff --git a/CHANGELOG.md b/CHANGELOG.md index cfa7a76..4b2fc7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ ## [Unreleased] +## [0.4.0] +### Changed +- New supported IDE version range: 2024.2 - 2024.3.*. +- Updated the project to use the IntelliJ Platform Gradle Plugin 2.0. +- Updated the project to use JDK 21. + ## [0.3.0] ### Changed - Updated Lucene to 9.11.1. @@ -13,7 +19,7 @@ - Updated Lucene to 9.11.0. - Changed the file chooser dialog in the *Open Index*, *Create Index* dialogs and the *Custom Analyzer* panel to an IntelliJ one to provide a smoother user experience. -- Replaced the previous GIF loading icon to a sharper SVG one. +- Replaced the previous GIF loading icon with a sharper SVG one. ## [0.1.0] ### Added diff --git a/NOTICE.txt b/NOTICE.txt index 45b7ebc..6f95284 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -2,5 +2,5 @@ This product includes software developed at The Apache Software Foundation (http://www.apache.org/). Many classes are reused from the Luke subproject of the Apache Lucene (https://github.com/apache/lucene) project. -Those classes are placed under the 'com.picimako.org.apache.lucene.luke...' package structure corresponding -to their original packages. +Those classes are placed either under the 'com.picimako.org.apache.lucene.luke...' package structure corresponding +to their original packages, or in their original packages when no other option is available. diff --git a/README.md b/README.md index bdd1cf9..b288865 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,20 @@ The tool can be opened with the menu action at `Tools` / `Luke - Lucene Toolbox` feature with the same name, and it opens on a new editor tab. Only one Luke editor tab can be open at once, calling the action again focuses on the already open Luke editor. +## IntelliJ 2024.2+ and JDK 21 support + +Starting from Lucas v0.4.0, the plugin is built on JDK 21 as required by the IntelliJ Platform. + +Since some parts of Lucene use preview features from Java 21, namely the Foreign Function and Memory API, +and lucene-core includes Java 19, 20 and 21 variants of some classes using that API, +there are a few workarounds in the plugin to make sure those classes are available, and work with Java 21 properly. + +For users of Lucas, make sure you add the `--enable-preview` JVM argument to the IDE's custom VM options +under `Help` > `Edit Custom VM Options...`, and then restart your IDE. This may or may not have side effects +to how your IDE operates. + +## Differences to Luke + Main differences between Lucas and Luke: - Opening the Luke tab in the IDE doesn't open up the **Open Index** dialog automatically. A plugin setting might be added in a future release to toggle this. - Luke's themes are removed. The plugin uses the theme currently set in the IDE. diff --git a/build.gradle.kts b/build.gradle.kts index 690f0f4..af7b1aa 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,27 +1,39 @@ import org.apache.tools.ant.filters.* import org.jetbrains.changelog.Changelog import org.jetbrains.changelog.markdownToHTML - -fun properties(key: String) = providers.gradleProperty(key) -fun environment(key: String) = providers.environmentVariable(key) +import org.jetbrains.intellij.platform.gradle.tasks.RunIdeTask plugins { id("java") // Java support alias(libs.plugins.kotlin) // Kotlin support - alias(libs.plugins.gradleIntelliJPlugin) // Gradle IntelliJ Plugin + alias(libs.plugins.intelliJPlatform) // IntelliJ Platform Gradle Plugin alias(libs.plugins.changelog) // Gradle Changelog Plugin } -group = properties("pluginGroup").get() -version = properties("pluginVersion").get() +group = providers.gradleProperty("pluginGroup").get() +version = providers.gradleProperty("pluginVersion").get() + +// Set the JVM language level used to build the project. Use Java 11 for 2020.3+, and Java 17 for 2022.2+. +kotlin { + jvmToolchain(21) +} // Configure project's dependencies repositories { mavenCentral() + // IntelliJ Platform Gradle Plugin Repositories Extension - read more: https://plugins.jetbrains.com/docs/intellij/tools-intellij-platform-gradle-plugin-repositories-extension.html + intellijPlatform { + defaultRepositories() + } } // Dependencies are managed with Gradle version catalog - read more: https://docs.gradle.org/current/userguide/platforms.html#sub:version-catalog dependencies { + //Lucene + + //In case one wants to open in index using AssertingCodec in that index, + // add implementation("org.apache.lucene:lucene-test-framework:$luceneVersion") + val luceneVersion = "9.11.1" implementation("org.apache.lucene:lucene-core:$luceneVersion") @@ -46,47 +58,34 @@ dependencies { runtimeOnly("org.apache.lucene:lucene-analysis-stempel:$luceneVersion") runtimeOnly("org.apache.lucene:lucene-suggest:$luceneVersion") - //In case one wants to open in index using AssertingCodec in that index, - // add implementation("org.apache.lucene:lucene-test-framework:$luceneVersion") -} + // IntelliJ Platform Gradle Plugin Dependencies Extension - read more: https://plugins.jetbrains.com/docs/intellij/tools-intellij-platform-gradle-plugin-dependencies-extension.html -// Set the JVM language level used to build the project. Use Java 11 for 2020.3+, and Java 17 for 2022.2+. -kotlin { - jvmToolchain(17) -} + intellijPlatform { + create(providers.gradleProperty("platformType"), providers.gradleProperty("platformVersion")) -// Configure Gradle IntelliJ Plugin - read more: https://plugins.jetbrains.com/docs/intellij/tools-gradle-intellij-plugin.html -intellij { - pluginName = properties("pluginName") - version = properties("platformVersion") - type = properties("platformType") + // Plugin Dependencies. Uses `platformBundledPlugins` property from the gradle.properties file for bundled IntelliJ Platform plugins. + bundledPlugins(providers.gradleProperty("platformBundledPlugins").map { it.split(',') }) - // Plugin Dependencies. Uses `platformPlugins` property from the gradle.properties file. - plugins = properties("platformPlugins").map { it.split(',').map(String::trim).filter(String::isNotEmpty) } -} + // Plugin Dependencies. Uses `platformPlugins` property from the gradle.properties file for plugin from JetBrains Marketplace. + plugins(providers.gradleProperty("platformPlugins").map { it.split(',') }) -// Configure Gradle Changelog Plugin - read more: https://github.com/JetBrains/gradle-changelog-plugin -changelog { - groups.empty() - repositoryUrl = properties("pluginRepositoryUrl") -} - -tasks { - wrapper { - gradleVersion = properties("gradleVersion").get() + instrumentationTools() + pluginVerifier() + zipSigner() } +} - patchPluginXml { - version = properties("pluginVersion") - sinceBuild = properties("pluginSinceBuild") - untilBuild = properties("pluginUntilBuild") +// Configure IntelliJ Platform Gradle Plugin - read more: https://plugins.jetbrains.com/docs/intellij/tools-intellij-platform-gradle-plugin-extension.html +intellijPlatform { + pluginConfiguration { + version = providers.gradleProperty("pluginVersion") // Extract the section from README.md and provide for the plugin's manifest - pluginDescription = providers.fileContents(layout.projectDirectory.file("README.md")).asText.map { + description = providers.fileContents(layout.projectDirectory.file("README.md")).asText.map { val start = "" val end = "" - with (it.lines()) { + with(it.lines()) { if (!containsAll(listOf(start, end))) { throw GradleException("Plugin description section not found in README.md:\n$start ... $end") } @@ -96,7 +95,7 @@ tasks { val changelog = project.changelog // local variable for configuration cache compatibility // Get the latest available change notes from the changelog file - changeNotes = properties("pluginVersion").map { pluginVersion -> + changeNotes = providers.gradleProperty("pluginVersion").map { pluginVersion -> with(changelog) { renderItem( (getOrNull(pluginVersion) ?: getUnreleased()) @@ -106,6 +105,43 @@ tasks { ) } } + + ideaVersion { + sinceBuild = providers.gradleProperty("pluginSinceBuild") + untilBuild = providers.gradleProperty("pluginUntilBuild") + } + } + + pluginVerification { + ides { + recommended() + } + } +} + +//Required for classes in 'org.apache.lucene.store' because they use JDK 21 Preview features. +// Without this option, finding those classes would fail during the IDE's run. +tasks.named("runIde") { + jvmArgumentProviders += CommandLineArgumentProvider { + listOf("--enable-preview") + } +} + +// Configure Gradle Changelog Plugin - read more: https://github.com/JetBrains/gradle-changelog-plugin +changelog { + groups.empty() + repositoryUrl = providers.gradleProperty("pluginRepositoryUrl") +} + +tasks { + wrapper { + gradleVersion = providers.gradleProperty("gradleVersion").get() + } + + //Required for classes in 'org.apache.lucene.store' because they use JDK 21 Preview features. + // Without this option, the build would fail. + compileJava { + options.compilerArgs.add("--enable-preview") } // Process UTF8 property files to unicode escapes. diff --git a/gradle.properties b/gradle.properties index 14b0c9f..9a37c21 100644 --- a/gradle.properties +++ b/gradle.properties @@ -4,20 +4,23 @@ pluginGroup = com.picimako.lucas pluginName = Lucas pluginRepositoryUrl = https://github.com/picimako/lucas # SemVer format -> https://semver.org -pluginVersion = 0.3.0 +pluginVersion = 0.4.0 # Supported build number ranges and IntelliJ Platform versions -> https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html -pluginSinceBuild = 241 -pluginUntilBuild = 242.* +pluginSinceBuild = 242 +pluginUntilBuild = 243.* # IntelliJ Platform Properties -> https://plugins.jetbrains.com/docs/intellij/tools-gradle-intellij-plugin.html#configuration-intellij-extension platformType = IC -platformVersion = 2024.1 +platformVersion = 2024.2 # Plugin Dependencies -> https://plugins.jetbrains.com/docs/intellij/plugin-dependencies.html # Example: platformPlugins = com.intellij.java, com.jetbrains.php:203.4449.22 platformPlugins = +# Example: platformBundledPlugins = com.intellij.java +platformBundledPlugins = + # Gradle Releases -> https://github.com/gradle/gradle/releases gradleVersion = 8.8 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index a36d15c..46a4d89 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,16 +1,16 @@ [versions] # libraries -annotations = "24.1.0" +junit = "4.13.2" # plugins -kotlin = "1.9.24" -changelog = "2.2.0" -gradleIntelliJPlugin = "1.17.4" +kotlin = "1.9.25" +changelog = "2.2.1" +intelliJPlatform = "2.0.1" [libraries] -annotations = { group = "org.jetbrains", name = "annotations", version.ref = "annotations" } +junit = { group = "junit", name = "junit", version.ref = "junit" } [plugins] changelog = { id = "org.jetbrains.changelog", version.ref = "changelog" } -gradleIntelliJPlugin = { id = "org.jetbrains.intellij", version.ref = "gradleIntelliJPlugin" } +intelliJPlatform = { id = "org.jetbrains.intellij.platform", version.ref = "intelliJPlatform" } kotlin = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } diff --git a/src/main/java/com/picimako/org/apache/lucene/luke/app/desktop/components/dialog/menubar/CreateIndexDialogFactory.java b/src/main/java/com/picimako/org/apache/lucene/luke/app/desktop/components/dialog/menubar/CreateIndexDialogFactory.java index 4fa5a01..aaed552 100644 --- a/src/main/java/com/picimako/org/apache/lucene/luke/app/desktop/components/dialog/menubar/CreateIndexDialogFactory.java +++ b/src/main/java/com/picimako/org/apache/lucene/luke/app/desktop/components/dialog/menubar/CreateIndexDialogFactory.java @@ -37,6 +37,11 @@ import org.apache.lucene.luke.util.LoggerFactory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FSLockFactory; +import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.MMapDirectory; +import org.apache.lucene.store.NIOFSDirectory; +import org.apache.lucene.util.Constants; import org.apache.lucene.util.NamedThreadFactory; import org.jetbrains.annotations.Nullable; @@ -282,7 +287,7 @@ protected Void doInBackground() throws Exception { setOKActionEnabled(false); try { - Directory dir = FSDirectory.open(path); + Directory dir = open(path, FSLockFactory.getDefault()); IndexTools toolsModel = new IndexToolsFactory().newInstance(dir); if (dataDirTF.getText().isEmpty()) { @@ -355,5 +360,18 @@ protected void done() { private void clearDataDir(ActionEvent e) { dataDirTF.setText(""); } + + /** + * Just like {@link FSDirectory#open(Path)}, but allows you to also specify a custom {@link LockFactory}. + *

+ * LICENSE NOTE: This is copied from {@link FSDirectory} to be able to create a custom {@code MMapDirectory} instance. + */ + private static FSDirectory open(Path path, LockFactory lockFactory) throws IOException { + if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) { + return new MMapDirectory(path, lockFactory); + } else { + return new NIOFSDirectory(path, lockFactory); + } + } } } diff --git a/src/main/java/org/apache/lucene/store/MMapDirectory.java b/src/main/java/org/apache/lucene/store/MMapDirectory.java new file mode 100644 index 0000000..6196a67 --- /dev/null +++ b/src/main/java/org/apache/lucene/store/MMapDirectory.java @@ -0,0 +1,469 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.store; + +import java.io.IOException; +import java.nio.channels.ClosedChannelException; // javadoc @link +import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Locale; +import java.util.Optional; +import java.util.concurrent.Future; +import java.util.function.BiPredicate; +import java.util.logging.Logger; +import org.apache.lucene.util.Constants; +import org.apache.lucene.util.SuppressForbidden; + +/** + * File-based {@link Directory} implementation that uses mmap for reading, and {@link + * FSDirectory.FSIndexOutput} for writing. + * + *

NOTE: memory mapping uses up a portion of the virtual memory address space in your + * process equal to the size of the file being mapped. Before using this class, be sure your have + * plenty of virtual address space, e.g. by using a 64 bit JRE, or a 32 bit JRE with indexes that + * are guaranteed to fit within the address space. On 32 bit platforms also consult {@link + * #MMapDirectory(Path, LockFactory, long)} if you have problems with mmap failing because of + * fragmented address space. If you get an OutOfMemoryException, it is recommended to reduce the + * chunk size, until it works. + * + *

This class supports preloading files into physical memory upon opening. This can help improve + * performance of searches on a cold page cache at the expense of slowing down opening an index. See + * {@link #setPreload(BiPredicate)} for more details. + * + *

Due to this bug in OpenJDK, + * MMapDirectory's {@link IndexInput#close} is unable to close the underlying OS file handle. Only + * when GC finally collects the underlying objects, which could be quite some time later, will the + * file handle be closed. + * + *

This will consume additional transient disk usage: on Windows, attempts to delete or overwrite + * the files will result in an exception; on other platforms, which typically have a "delete on + * last close" semantics, while such operations will succeed, the bytes are still consuming + * space on disk. For many applications this limitation is not a problem (e.g. if you have plenty of + * disk space, and you don't rely on overwriting files on Windows) but it's still an important + * limitation to be aware of. + * + *

This class supplies the workaround mentioned in the bug report, which may fail on + * non-Oracle/OpenJDK JVMs. It forcefully unmaps the buffer on close by using an undocumented + * internal cleanup functionality. If {@link #UNMAP_SUPPORTED} is true, the workaround + * will be automatically enabled (with no guarantees; if you discover any problems, you can disable + * it by using system property {@link #ENABLE_UNMAP_HACK_SYSPROP}). + * + *

For the hack to work correct, the following requirements need to be fulfilled: The used JVM + * must be at least Oracle Java / OpenJDK. In addition, the following permissions need to be granted + * to {@code lucene-core.jar} in your policy + * file: + * + *

+ * + *

Starting with Java 19 this class will use the modern {@code MemorySegment} API which + * allows to safely unmap (if you discover any problems with this preview API, you can disable it by + * using system property {@link #ENABLE_MEMORY_SEGMENTS_SYSPROP}). + * + *

Starting with Java 21 on some platforms like Linux and MacOS X, this class will invoke + * the syscall {@code madvise()} to advise how OS kernel should handle paging after opening a file. + * For this to work, Java code must be able to call native code. If this is not allowed, a warning + * is logged. To enable native access for Lucene in a modularized application, pass {@code + * --enable-native-access=org.apache.lucene.core} to the Java command line. If Lucene is running in + * a classpath-based application, use {@code --enable-native-access=ALL-UNNAMED}. + * + *

NOTE: Accessing this class either directly or indirectly from a thread while it's + * interrupted can close the underlying channel immediately if at the same time the thread is + * blocked on IO. The channel will remain closed and subsequent access to {@link MMapDirectory} will + * throw a {@link ClosedChannelException}. If your application uses either {@link + * Thread#interrupt()} or {@link Future#cancel(boolean)} you should use the legacy {@code + * RAFDirectory} from the Lucene {@code misc} module in favor of {@link MMapDirectory}. + * + *

NOTE: If your application requires external synchronization, you should not + * synchronize on the MMapDirectory instance as this may cause deadlock; use your own + * (non-Lucene) objects instead. + *

+ *

LICENCE NOTE

+ * This is a modified version of the original {@code org.apache.lucene.store.MMapDirectory} + * with the following changes: + * + * This class remains in the original package because {@link MappedByteBufferIndexInputProvider} and + * {@link MemorySegmentIndexInputProvider} are package-private. + * + * @see Blog post + * about MMapDirectory + */ +public class MMapDirectory extends FSDirectory { + + private static final Logger LOG = Logger.getLogger(MMapDirectory.class.getName()); + + /** + * Argument for {@link #setPreload(BiPredicate)} that configures all files to be preloaded upon + * opening them. + */ + public static final BiPredicate ALL_FILES = (filename, context) -> true; + + /** + * Argument for {@link #setPreload(BiPredicate)} that configures no files to be preloaded upon + * opening them. + */ + public static final BiPredicate NO_FILES = (filename, context) -> false; + + /** + * Argument for {@link #setPreload(BiPredicate)} that configures files to be preloaded upon + * opening them if they use the {@link IOContext#LOAD} I/O context. + */ + public static final BiPredicate BASED_ON_LOAD_IO_CONTEXT = + (filename, context) -> context.load; + + private BiPredicate preload = NO_FILES; + + /** + * Default max chunk size: + * + *
    + *
  • 16 GiBytes for 64 bit Java 19 / 20 / 21 JVMs + *
  • 1 GiBytes for other 64 bit JVMs + *
  • 256 MiBytes for 32 bit JVMs + *
+ */ + public static final long DEFAULT_MAX_CHUNK_SIZE; + + /** + * This sysprop allows to control the workaround/hack for unmapping the buffers from address space + * after closing {@link IndexInput}. By default it is enabled; set to {@code false} to disable the + * unmap hack globally. On command line pass {@code + * -Dorg.apache.lucene.store.MMapDirectory.enableUnmapHack=false} to disable. + * + * @lucene.internal + */ + public static final String ENABLE_UNMAP_HACK_SYSPROP = + "org.apache.lucene.store.MMapDirectory.enableUnmapHack"; + + /** + * This sysprop allows to control if {@code MemorySegment} API should be used on supported Java + * versions. By default it is enabled; set to {@code false} to use legacy {@code ByteBuffer} + * implementation. On command line pass {@code + * -Dorg.apache.lucene.store.MMapDirectory.enableMemorySegments=false} to disable. + * + * @lucene.internal + */ + public static final String ENABLE_MEMORY_SEGMENTS_SYSPROP = + "org.apache.lucene.store.MMapDirectory.enableMemorySegments"; + + final int chunkSizePower; + + /** + * Create a new MMapDirectory for the named location. The directory is created at the named + * location if it does not yet exist. + * + * @param path the path of the directory + * @param lockFactory the lock factory to use + * @throws IOException if there is a low-level I/O error + */ + public MMapDirectory(Path path, LockFactory lockFactory) throws IOException { + this(path, lockFactory, DEFAULT_MAX_CHUNK_SIZE); + } + + /** + * Create a new MMapDirectory for the named location and {@link FSLockFactory#getDefault()}. The + * directory is created at the named location if it does not yet exist. + * + * @param path the path of the directory + * @throws IOException if there is a low-level I/O error + */ + public MMapDirectory(Path path) throws IOException { + this(path, FSLockFactory.getDefault()); + } + + /** + * Create a new MMapDirectory for the named location and {@link FSLockFactory#getDefault()}. The + * directory is created at the named location if it does not yet exist. + * + * @deprecated use {@link #MMapDirectory(Path, long)} instead. + */ + @Deprecated + public MMapDirectory(Path path, int maxChunkSize) throws IOException { + this(path, (long) maxChunkSize); + } + + /** + * Create a new MMapDirectory for the named location and {@link FSLockFactory#getDefault()}. The + * directory is created at the named location if it does not yet exist. + * + * @param path the path of the directory + * @param maxChunkSize maximum chunk size (for default see {@link #DEFAULT_MAX_CHUNK_SIZE}) used + * for memory mapping. + * @throws IOException if there is a low-level I/O error + */ + public MMapDirectory(Path path, long maxChunkSize) throws IOException { + this(path, FSLockFactory.getDefault(), maxChunkSize); + } + + /** + * Create a new MMapDirectory for the named location and {@link FSLockFactory#getDefault()}. The + * directory is created at the named location if it does not yet exist. + * + * @deprecated use {@link #MMapDirectory(Path, LockFactory, long)} instead. + */ + @Deprecated + public MMapDirectory(Path path, LockFactory lockFactory, int maxChunkSize) throws IOException { + this(path, lockFactory, (long) maxChunkSize); + } + + /** + * Create a new MMapDirectory for the named location, specifying the maximum chunk size used for + * memory mapping. The directory is created at the named location if it does not yet exist. + * + *

Especially on 32 bit platform, the address space can be very fragmented, so large index + * files cannot be mapped. Using a lower chunk size makes the directory implementation a little + * bit slower (as the correct chunk may be resolved on lots of seeks) but the chance is higher + * that mmap does not fail. On 64 bit Java platforms, this parameter should always be large (like + * 1 GiBytes, or even larger with recent Java versions), as the address space is big enough. If it + * is larger, fragmentation of address space increases, but number of file handles and mappings is + * lower for huge installations with many open indexes. + * + *

Please note: The chunk size is always rounded down to a power of 2. + * + * @param path the path of the directory + * @param lockFactory the lock factory to use, or null for the default ({@link + * NativeFSLockFactory}); + * @param maxChunkSize maximum chunk size (for default see {@link #DEFAULT_MAX_CHUNK_SIZE}) used + * for memory mapping. + * @throws IOException if there is a low-level I/O error + */ + public MMapDirectory(Path path, LockFactory lockFactory, long maxChunkSize) throws IOException { + super(path, lockFactory); + if (maxChunkSize <= 0L) { + throw new IllegalArgumentException("Maximum chunk size for mmap must be >0"); + } + this.chunkSizePower = Long.SIZE - 1 - Long.numberOfLeadingZeros(maxChunkSize); + assert (1L << chunkSizePower) <= maxChunkSize; + assert (1L << chunkSizePower) > (maxChunkSize / 2); + } + + /** + * This method is retired, see deprecation notice! + * + * @throws UnsupportedOperationException as setting cannot be changed + * @deprecated Please use new system property {@link #ENABLE_UNMAP_HACK_SYSPROP} instead + */ + @Deprecated(forRemoval = true) + public void setUseUnmap(final boolean useUnmapHack) { + if (useUnmapHack != UNMAP_SUPPORTED) { + throw new UnsupportedOperationException( + "It is no longer possible configure unmap hack for directory instances. Please use the global system property: " + + ENABLE_UNMAP_HACK_SYSPROP); + } + } + + /** + * Returns true, if the unmap workaround is enabled. + * + * @see #setUseUnmap + * @deprecated use {@link #UNMAP_SUPPORTED} + */ + @Deprecated + public boolean getUseUnmap() { + return UNMAP_SUPPORTED; + } + + /** + * Configure which files to preload in physical memory upon opening. The default implementation + * does not preload anything. The behavior is best effort and operating system-dependent. + * + * @param preload a {@link BiPredicate} whose first argument is the file name, and second argument + * is the {@link IOContext} used to open the file + * @see #ALL_FILES + * @see #NO_FILES + */ + public void setPreload(BiPredicate preload) { + this.preload = preload; + } + + /** + * Configure whether to preload files on this {@link MMapDirectory} into physical memory upon + * opening. The behavior is best effort and operating system-dependent. + * + * @deprecated Use {@link #setPreload(BiPredicate)} instead which provides more granular control. + */ + @Deprecated + public void setPreload(boolean preload) { + this.preload = preload ? ALL_FILES : NO_FILES; + } + + /** + * Return whether files are loaded into physical memory upon opening. + * + * @deprecated This information is no longer reliable now that preloading is more granularly + * configured via a predicate. + * @see #setPreload(BiPredicate) + */ + @Deprecated + public boolean getPreload() { + return preload == ALL_FILES; + } + + /** + * Returns the current mmap chunk size. + * + * @see #MMapDirectory(Path, LockFactory, long) + */ + public final long getMaxChunkSize() { + return 1L << chunkSizePower; + } + + /** Creates an IndexInput for the file with the given name. */ + @Override + public IndexInput openInput(String name, IOContext context) throws IOException { + ensureOpen(); + ensureCanRead(name); + Path path = directory.resolve(name); + return PROVIDER.openInput(path, context, chunkSizePower, preload.test(name, context)); + } + + // visible for tests: + static final MMapIndexInputProvider PROVIDER; + + /** true, if this platform supports unmapping mmapped files. */ + public static final boolean UNMAP_SUPPORTED; + + /** + * if {@link #UNMAP_SUPPORTED} is {@code false}, this contains the reason why unmapping is not + * supported. + */ + public static final String UNMAP_NOT_SUPPORTED_REASON; + + interface MMapIndexInputProvider { + IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload) + throws IOException; + + long getDefaultMaxChunkSize(); + + boolean isUnmapSupported(); + + String getUnmapNotSupportedReason(); + + boolean supportsMadvise(); + + default IOException convertMapFailedIOException( + IOException ioe, String resourceDescription, long bufSize) { + final String originalMessage; + final Throwable originalCause; + if (ioe.getCause() instanceof OutOfMemoryError) { + // nested OOM confuses users, because it's "incorrect", just print a plain message: + originalMessage = "Map failed"; + originalCause = null; + } else { + originalMessage = ioe.getMessage(); + originalCause = ioe.getCause(); + } + final String moreInfo; + if (!Constants.JRE_IS_64BIT) { + moreInfo = + "MMapDirectory should only be used on 64bit platforms, because the address space on 32bit operating systems is too small. "; + } else if (Constants.WINDOWS) { + moreInfo = + "Windows is unfortunately very limited on virtual address space. If your index size is several hundred Gigabytes, consider changing to Linux. "; + } else if (Constants.LINUX) { + moreInfo = + "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'), and 'sysctl vm.max_map_count'. "; + } else { + moreInfo = "Please review 'ulimit -v', 'ulimit -m' (both should return 'unlimited'). "; + } + final IOException newIoe = + new IOException( + String.format( + Locale.ENGLISH, + "%s: %s [this may be caused by lack of enough unfragmented virtual address space " + + "or too restrictive virtual memory limits enforced by the operating system, " + + "preventing us to map a chunk of %d bytes. %sMore information: " + + "https://blog.thetaphi.de/2012/07/use-lucenes-mmapdirectory-on-64bit.html]", + originalMessage, + resourceDescription, + bufSize, + moreInfo), + originalCause); + newIoe.setStackTrace(ioe.getStackTrace()); + return newIoe; + } + } + + // Extracted to a method to be able to apply the SuppressForbidden annotation + @SuppressWarnings("removal") + @SuppressForbidden(reason = "security manager") + private static T doPrivileged(PrivilegedAction action) { + return AccessController.doPrivileged(action); + } + + private static boolean checkMemorySegmentsSysprop() { + try { + return Optional.ofNullable(System.getProperty(ENABLE_MEMORY_SEGMENTS_SYSPROP)) + .map(Boolean::valueOf) + .orElse(Boolean.TRUE); + } catch ( + @SuppressWarnings("unused") + SecurityException ignored) { + LOG.warning( + "Cannot read sysprop " + + ENABLE_MEMORY_SEGMENTS_SYSPROP + + ", so MemorySegments will be enabled by default, if possible."); + return true; + } + } + + private static MMapIndexInputProvider lookupProvider() { + if (checkMemorySegmentsSysprop() == false) { + return new MappedByteBufferIndexInputProvider(); + } + final int runtimeVersion = Runtime.version().feature(); + if (runtimeVersion >= 19) { + try { + return new MemorySegmentIndexInputProvider(); + } catch (RuntimeException | Error e) { + throw e; + } catch (Throwable th) { + throw new AssertionError(th); + } + } + return new MappedByteBufferIndexInputProvider(); + } + + /** + * Returns true, if MMapDirectory uses the platform's {@code madvise()} syscall to advise how OS + * kernel should handle paging after opening a file. + */ + public static boolean supportsMadvise() { + return PROVIDER.supportsMadvise(); + } + + static { + PROVIDER = doPrivileged(MMapDirectory::lookupProvider); + DEFAULT_MAX_CHUNK_SIZE = PROVIDER.getDefaultMaxChunkSize(); + UNMAP_SUPPORTED = PROVIDER.isUnmapSupported(); + UNMAP_NOT_SUPPORTED_REASON = PROVIDER.getUnmapNotSupportedReason(); + } +} diff --git a/src/main/java/org/apache/lucene/store/MemorySegmentAccessInput.java b/src/main/java/org/apache/lucene/store/MemorySegmentAccessInput.java new file mode 100644 index 0000000..f15f62d --- /dev/null +++ b/src/main/java/org/apache/lucene/store/MemorySegmentAccessInput.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.store; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; + +/** + * Provides access to the backing memory segment. + * + *

Expert API, allows access to the backing memory. + */ +public interface MemorySegmentAccessInput extends RandomAccessInput, Cloneable { + + /** Returns the memory segment for a given position and length, or null. */ + MemorySegment segmentSliceOrNull(long pos, int len) throws IOException; + + default void readBytes(long pos, byte[] bytes, int offset, int length) throws IOException { + for (int i = 0; i < length; i++) { + bytes[offset + i] = readByte(pos + i); + } + } + + MemorySegmentAccessInput clone(); +} diff --git a/src/main/java/org/apache/lucene/store/MemorySegmentIndexInput.java b/src/main/java/org/apache/lucene/store/MemorySegmentIndexInput.java new file mode 100644 index 0000000..3eead55 --- /dev/null +++ b/src/main/java/org/apache/lucene/store/MemorySegmentIndexInput.java @@ -0,0 +1,675 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.store; + +import java.io.EOFException; +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.ValueLayout; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.Objects; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.GroupVIntUtil; + +/** + * Base IndexInput implementation that uses an array of MemorySegments to represent a file. + * + *

For efficiency, this class requires that the segment size are a power-of-two ( + * chunkSizePower). + */ +@SuppressWarnings("preview") +abstract class MemorySegmentIndexInput extends IndexInput + implements RandomAccessInput, MemorySegmentAccessInput { + static final ValueLayout.OfByte LAYOUT_BYTE = ValueLayout.JAVA_BYTE; + static final ValueLayout.OfShort LAYOUT_LE_SHORT = + ValueLayout.JAVA_SHORT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + static final ValueLayout.OfInt LAYOUT_LE_INT = + ValueLayout.JAVA_INT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + static final ValueLayout.OfLong LAYOUT_LE_LONG = + ValueLayout.JAVA_LONG_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + static final ValueLayout.OfFloat LAYOUT_LE_FLOAT = + ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN); + + final long length; + final long chunkSizeMask; + final int chunkSizePower; + final Arena arena; + final MemorySegment[] segments; + + int curSegmentIndex = -1; + MemorySegment + curSegment; // redundant for speed: segments[curSegmentIndex], also marker if closed! + long curPosition; // relative to curSegment, not globally + + public static MemorySegmentIndexInput newInstance( + String resourceDescription, + Arena arena, + MemorySegment[] segments, + long length, + int chunkSizePower) { + assert Arrays.stream(segments).map(MemorySegment::scope).allMatch(arena.scope()::equals); + if (segments.length == 1) { + return new SingleSegmentImpl(resourceDescription, arena, segments[0], length, chunkSizePower); + } else { + return new MultiSegmentImpl(resourceDescription, arena, segments, 0, length, chunkSizePower); + } + } + + private MemorySegmentIndexInput( + String resourceDescription, + Arena arena, + MemorySegment[] segments, + long length, + int chunkSizePower) { + super(resourceDescription); + this.arena = arena; + this.segments = segments; + this.length = length; + this.chunkSizePower = chunkSizePower; + this.chunkSizeMask = (1L << chunkSizePower) - 1L; + this.curSegment = segments[0]; + } + + void ensureOpen() { + if (curSegment == null) { + throw alreadyClosed(null); + } + } + + // the unused parameter is just to silence javac about unused variables + RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos) + throws IOException { + if (pos < 0L) { + return new IllegalArgumentException(action + " negative position (pos=" + pos + "): " + this); + } else { + throw new EOFException(action + " past EOF (pos=" + pos + "): " + this); + } + } + + AlreadyClosedException alreadyClosed(RuntimeException e) { + // we use NPE to signal if this input is closed (to not have checks everywhere). If NPE happens, + // we check the "is closed" condition explicitly by checking that our "curSegment" is null. + // Care must be taken to not leak the NPE to code outside MemorySegmentIndexInput! + if (this.curSegment == null) { + return new AlreadyClosedException("Already closed: " + this); + } + // in Java 22 or later we can check the isAlive status of all segments + // (see https://bugs.openjdk.org/browse/JDK-8310644): + if (Arrays.stream(segments).allMatch(s -> s.scope().isAlive()) == false) { + return new AlreadyClosedException("Already closed: " + this); + } + // fallback for Java 21: ISE can be thrown by MemorySegment and contains "closed" in message: + if (e instanceof IllegalStateException + && e.getMessage() != null + && e.getMessage().contains("closed")) { + // the check is on message only, so preserve original cause for debugging: + return new AlreadyClosedException("Already closed: " + this, e); + } + // otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to + // the IndexInput method): + throw e; + } + + @Override + public final byte readByte() throws IOException { + try { + final byte v = curSegment.get(LAYOUT_BYTE, curPosition); + curPosition++; + return v; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException e) { + do { + curSegmentIndex++; + if (curSegmentIndex >= segments.length) { + throw new EOFException("read past EOF: " + this); + } + curSegment = segments[curSegmentIndex]; + curPosition = 0L; + } while (curSegment.byteSize() == 0L); + final byte v = curSegment.get(LAYOUT_BYTE, curPosition); + curPosition++; + return v; + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public final void readBytes(byte[] b, int offset, int len) throws IOException { + try { + MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len); + curPosition += len; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException e) { + readBytesBoundary(b, offset, len); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + private void readBytesBoundary(byte[] b, int offset, int len) throws IOException { + try { + long curAvail = curSegment.byteSize() - curPosition; + while (len > curAvail) { + MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, (int) curAvail); + len -= curAvail; + offset += curAvail; + curSegmentIndex++; + if (curSegmentIndex >= segments.length) { + throw new EOFException("read past EOF: " + this); + } + curSegment = segments[curSegmentIndex]; + curPosition = 0L; + curAvail = curSegment.byteSize(); + } + MemorySegment.copy(curSegment, LAYOUT_BYTE, curPosition, b, offset, len); + curPosition += len; + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public void readInts(int[] dst, int offset, int length) throws IOException { + try { + MemorySegment.copy(curSegment, LAYOUT_LE_INT, curPosition, dst, offset, length); + curPosition += Integer.BYTES * (long) length; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException iobe) { + super.readInts(dst, offset, length); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public void readLongs(long[] dst, int offset, int length) throws IOException { + try { + MemorySegment.copy(curSegment, LAYOUT_LE_LONG, curPosition, dst, offset, length); + curPosition += Long.BYTES * (long) length; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException iobe) { + super.readLongs(dst, offset, length); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public void readFloats(float[] dst, int offset, int length) throws IOException { + try { + MemorySegment.copy(curSegment, LAYOUT_LE_FLOAT, curPosition, dst, offset, length); + curPosition += Float.BYTES * (long) length; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException iobe) { + super.readFloats(dst, offset, length); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public final short readShort() throws IOException { + try { + final short v = curSegment.get(LAYOUT_LE_SHORT, curPosition); + curPosition += Short.BYTES; + return v; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException e) { + return super.readShort(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public final int readInt() throws IOException { + try { + final int v = curSegment.get(LAYOUT_LE_INT, curPosition); + curPosition += Integer.BYTES; + return v; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException e) { + return super.readInt(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public final int readVInt() throws IOException { + // this can make JVM less confused (see LUCENE-10366) + return super.readVInt(); + } + + @Override + public final long readVLong() throws IOException { + // this can make JVM less confused (see LUCENE-10366) + return super.readVLong(); + } + + @Override + public final long readLong() throws IOException { + try { + final long v = curSegment.get(LAYOUT_LE_LONG, curPosition); + curPosition += Long.BYTES; + return v; + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException e) { + return super.readLong(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public long getFilePointer() { + ensureOpen(); + return (((long) curSegmentIndex) << chunkSizePower) + curPosition; + } + + @Override + public void seek(long pos) throws IOException { + ensureOpen(); + // we use >> here to preserve negative, so we will catch AIOOBE, + // in case pos + offset overflows. + final int si = (int) (pos >> chunkSizePower); + try { + if (si != curSegmentIndex) { + final MemorySegment seg = segments[si]; + // write values, on exception all is unchanged + this.curSegmentIndex = si; + this.curSegment = seg; + } + this.curPosition = Objects.checkIndex(pos & chunkSizeMask, curSegment.byteSize() + 1); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "seek", pos); + } + } + + @Override + public byte readByte(long pos) throws IOException { + try { + final int si = (int) (pos >> chunkSizePower); + return segments[si].get(LAYOUT_BYTE, pos & chunkSizeMask); + } catch (IndexOutOfBoundsException ioobe) { + throw handlePositionalIOOBE(ioobe, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + protected void readGroupVInt(long[] dst, int offset) throws IOException { + try { + final int len = + GroupVIntUtil.readGroupVInt( + this, + curSegment.byteSize() - curPosition, + p -> curSegment.get(LAYOUT_LE_INT, p), + curPosition, + dst, + offset); + curPosition += len; + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + // used only by random access methods to handle reads across boundaries + private void setPos(long pos, int si) throws IOException { + try { + final MemorySegment seg = segments[si]; + // write values, on exception above all is unchanged + this.curPosition = pos & chunkSizeMask; + this.curSegmentIndex = si; + this.curSegment = seg; + } catch (IndexOutOfBoundsException ioobe) { + throw handlePositionalIOOBE(ioobe, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public short readShort(long pos) throws IOException { + final int si = (int) (pos >> chunkSizePower); + try { + return segments[si].get(LAYOUT_LE_SHORT, pos & chunkSizeMask); + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException ioobe) { + // either it's a boundary, or read past EOF, fall back: + setPos(pos, si); + return readShort(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public int readInt(long pos) throws IOException { + final int si = (int) (pos >> chunkSizePower); + try { + return segments[si].get(LAYOUT_LE_INT, pos & chunkSizeMask); + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException ioobe) { + // either it's a boundary, or read past EOF, fall back: + setPos(pos, si); + return readInt(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public long readLong(long pos) throws IOException { + final int si = (int) (pos >> chunkSizePower); + try { + return segments[si].get(LAYOUT_LE_LONG, pos & chunkSizeMask); + } catch ( + @SuppressWarnings("unused") + IndexOutOfBoundsException ioobe) { + // either it's a boundary, or read past EOF, fall back: + setPos(pos, si); + return readLong(); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public final long length() { + return length; + } + + @Override + public final MemorySegmentIndexInput clone() { + final MemorySegmentIndexInput clone = buildSlice((String) null, 0L, this.length); + try { + clone.seek(getFilePointer()); + } catch (IOException ioe) { + throw new AssertionError(ioe); + } + + return clone; + } + + /** + * Creates a slice of this index input, with the given description, offset, and length. The slice + * is seeked to the beginning. + */ + @Override + public final MemorySegmentIndexInput slice(String sliceDescription, long offset, long length) { + if (offset < 0 || length < 0 || offset + length > this.length) { + throw new IllegalArgumentException( + "slice() " + + sliceDescription + + " out of bounds: offset=" + + offset + + ",length=" + + length + + ",fileLength=" + + this.length + + ": " + + this); + } + + return buildSlice(sliceDescription, offset, length); + } + + /** Builds the actual sliced IndexInput (may apply extra offset in subclasses). * */ + MemorySegmentIndexInput buildSlice(String sliceDescription, long offset, long length) { + ensureOpen(); + + final long sliceEnd = offset + length; + final int startIndex = (int) (offset >>> chunkSizePower); + final int endIndex = (int) (sliceEnd >>> chunkSizePower); + + // we always allocate one more slice, the last one may be a 0 byte one after truncating with + // asSlice(): + final MemorySegment slices[] = ArrayUtil.copyOfSubArray(segments, startIndex, endIndex + 1); + + // set the last segment's limit for the sliced view. + slices[slices.length - 1] = slices[slices.length - 1].asSlice(0L, sliceEnd & chunkSizeMask); + + offset = offset & chunkSizeMask; + + final String newResourceDescription = getFullSliceDescription(sliceDescription); + if (slices.length == 1) { + return new SingleSegmentImpl( + newResourceDescription, + null, // clones don't have an Arena, as they can't close) + slices[0].asSlice(offset, length), + length, + chunkSizePower); + } else { + return new MultiSegmentImpl( + newResourceDescription, + null, // clones don't have an Arena, as they can't close) + slices, + offset, + length, + chunkSizePower); + } + } + + static boolean checkIndex(long index, long length) { + return index >= 0 && index < length; + } + + @Override + public final void close() throws IOException { + if (curSegment == null) { + return; + } + + // the master IndexInput has an Arena and is able + // to release all resources (unmap segments) - a + // side effect is that other threads still using clones + // will throw IllegalStateException + if (arena != null) { + while (arena.scope().isAlive()) { + try { + arena.close(); + break; + } catch ( + @SuppressWarnings("unused") + IllegalStateException e) { + Thread.onSpinWait(); + } + } + } + + // make sure all accesses to this IndexInput instance throw NPE: + curSegment = null; + Arrays.fill(segments, null); + } + + /** Optimization of MemorySegmentIndexInput for when there is only one segment. */ + static final class SingleSegmentImpl extends MemorySegmentIndexInput { + + SingleSegmentImpl( + String resourceDescription, + Arena arena, + MemorySegment segment, + long length, + int chunkSizePower) { + super(resourceDescription, arena, new MemorySegment[] {segment}, length, chunkSizePower); + this.curSegmentIndex = 0; + } + + @Override + public void seek(long pos) throws IOException { + ensureOpen(); + try { + curPosition = Objects.checkIndex(pos, length + 1); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "seek", pos); + } + } + + @Override + public long getFilePointer() { + ensureOpen(); + return curPosition; + } + + @Override + public byte readByte(long pos) throws IOException { + try { + return curSegment.get(LAYOUT_BYTE, pos); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public short readShort(long pos) throws IOException { + try { + return curSegment.get(LAYOUT_LE_SHORT, pos); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public int readInt(long pos) throws IOException { + try { + return curSegment.get(LAYOUT_LE_INT, pos); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public long readLong(long pos) throws IOException { + try { + return curSegment.get(LAYOUT_LE_LONG, pos); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "read", pos); + } catch (NullPointerException | IllegalStateException e) { + throw alreadyClosed(e); + } + } + + @Override + public MemorySegment segmentSliceOrNull(long pos, int len) throws IOException { + try { + Objects.checkIndex(pos + len, this.length + 1); + return curSegment.asSlice(pos, len); + } catch (IndexOutOfBoundsException e) { + throw handlePositionalIOOBE(e, "segmentSliceOrNull", pos); + } + } + } + + /** This class adds offset support to MemorySegmentIndexInput, which is needed for slices. */ + static final class MultiSegmentImpl extends MemorySegmentIndexInput { + private final long offset; + + MultiSegmentImpl( + String resourceDescription, + Arena arena, + MemorySegment[] segments, + long offset, + long length, + int chunkSizePower) { + super(resourceDescription, arena, segments, length, chunkSizePower); + this.offset = offset; + try { + seek(0L); + } catch (IOException ioe) { + throw new AssertionError(ioe); + } + assert curSegment != null && curSegmentIndex >= 0; + } + + @Override + RuntimeException handlePositionalIOOBE(RuntimeException unused, String action, long pos) + throws IOException { + return super.handlePositionalIOOBE(unused, action, pos - offset); + } + + @Override + public void seek(long pos) throws IOException { + assert pos >= 0L : "negative position"; + super.seek(pos + offset); + } + + @Override + public long getFilePointer() { + return super.getFilePointer() - offset; + } + + @Override + public byte readByte(long pos) throws IOException { + return super.readByte(pos + offset); + } + + @Override + public short readShort(long pos) throws IOException { + return super.readShort(pos + offset); + } + + @Override + public int readInt(long pos) throws IOException { + return super.readInt(pos + offset); + } + + @Override + public long readLong(long pos) throws IOException { + return super.readLong(pos + offset); + } + + public MemorySegment segmentSliceOrNull(long pos, int len) throws IOException { + if (pos + len > length) { + throw handlePositionalIOOBE(null, "segmentSliceOrNull", pos); + } + pos = pos + offset; + final int si = (int) (pos >> chunkSizePower); + final MemorySegment seg = segments[si]; + final long segOffset = pos & chunkSizeMask; + if (checkIndex(segOffset + len, seg.byteSize() + 1)) { + return seg.asSlice(segOffset, len); + } + return null; + } + + @Override + MemorySegmentIndexInput buildSlice(String sliceDescription, long ofs, long length) { + return super.buildSlice(sliceDescription, this.offset + ofs, length); + } + } +} diff --git a/src/main/java/org/apache/lucene/store/MemorySegmentIndexInputProvider.java b/src/main/java/org/apache/lucene/store/MemorySegmentIndexInputProvider.java new file mode 100644 index 0000000..846963e --- /dev/null +++ b/src/main/java/org/apache/lucene/store/MemorySegmentIndexInputProvider.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.store; + +import java.io.IOException; +import java.lang.foreign.Arena; +import java.lang.foreign.MemorySegment; +import java.nio.channels.FileChannel; +import java.nio.channels.FileChannel.MapMode; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Locale; +import java.util.Optional; +import java.util.logging.Logger; +import org.apache.lucene.util.Constants; +import org.apache.lucene.util.Unwrappable; + +@SuppressWarnings("preview") +final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexInputProvider { + + private final Optional nativeAccess; + + public MemorySegmentIndexInputProvider() { + this.nativeAccess = NativeAccess.getImplementation(); + var log = Logger.getLogger(getClass().getName()); + log.info( + String.format( + Locale.ENGLISH, + "Using MemorySegmentIndexInput%s with Java 21 or later; to disable start with -D%s=false", + nativeAccess.map(n -> " and native madvise support").orElse(""), + MMapDirectory.ENABLE_MEMORY_SEGMENTS_SYSPROP)); + } + + @Override + public IndexInput openInput(Path path, IOContext context, int chunkSizePower, boolean preload) + throws IOException { + final String resourceDescription = "MemorySegmentIndexInput(path=\"" + path.toString() + "\")"; + + // Work around for JDK-8259028: we need to unwrap our test-only file system layers + path = Unwrappable.unwrapAll(path); + + boolean success = false; + final Arena arena = Arena.ofShared(); + try (var fc = FileChannel.open(path, StandardOpenOption.READ)) { + final long fileSize = fc.size(); + final IndexInput in = + MemorySegmentIndexInput.newInstance( + resourceDescription, + arena, + map(arena, resourceDescription, fc, context, chunkSizePower, preload, fileSize), + fileSize, + chunkSizePower); + success = true; + return in; + } finally { + if (success == false) { + arena.close(); + } + } + } + + @Override + public long getDefaultMaxChunkSize() { + return Constants.JRE_IS_64BIT ? (1L << 34) : (1L << 28); + } + + @Override + public boolean isUnmapSupported() { + return true; + } + + @Override + public String getUnmapNotSupportedReason() { + return null; + } + + @Override + public boolean supportsMadvise() { + return nativeAccess.isPresent(); + } + + private final MemorySegment[] map( + Arena arena, + String resourceDescription, + FileChannel fc, + IOContext context, + int chunkSizePower, + boolean preload, + long length) + throws IOException { + if ((length >>> chunkSizePower) >= Integer.MAX_VALUE) + throw new IllegalArgumentException("File too big for chunk size: " + resourceDescription); + + final long chunkSize = 1L << chunkSizePower; + + // we always allocate one more segments, the last one may be a 0 byte one + final int nrSegments = (int) (length >>> chunkSizePower) + 1; + + final MemorySegment[] segments = new MemorySegment[nrSegments]; + + long startOffset = 0L; + for (int segNr = 0; segNr < nrSegments; segNr++) { + final long segSize = + (length > (startOffset + chunkSize)) ? chunkSize : (length - startOffset); + final MemorySegment segment; + try { + segment = fc.map(MapMode.READ_ONLY, startOffset, segSize, arena); + } catch (IOException ioe) { + throw convertMapFailedIOException(ioe, resourceDescription, segSize); + } + // if preload apply it without madvise. + // if chunk size is too small (2 MiB), disable madvise support (incorrect alignment) + if (preload) { + segment.load(); + } else if (nativeAccess.isPresent() && chunkSizePower >= 21) { + nativeAccess.get().madvise(segment, context); + } + segments[segNr] = segment; + startOffset += segSize; + } + return segments; + } +} diff --git a/src/main/java/org/apache/lucene/store/NativeAccess.java b/src/main/java/org/apache/lucene/store/NativeAccess.java new file mode 100644 index 0000000..30c3790 --- /dev/null +++ b/src/main/java/org/apache/lucene/store/NativeAccess.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.store; + +import java.io.IOException; +import java.lang.foreign.MemorySegment; +import java.util.Optional; +import org.apache.lucene.util.Constants; + +@SuppressWarnings("preview") +abstract class NativeAccess { + + /** Invoke the {@code madvise} call for the given {@link MemorySegment}. */ + public abstract void madvise(MemorySegment segment, IOContext context) throws IOException; + + /** + * Return the NativeAccess instance for this platform. At moment we only support Linux and MacOS + */ + public static Optional getImplementation() { + if (Constants.LINUX || Constants.MAC_OS_X) { + return PosixNativeAccess.getInstance(); + } + return Optional.empty(); + } +} diff --git a/src/main/java/org/apache/lucene/store/PosixNativeAccess.java b/src/main/java/org/apache/lucene/store/PosixNativeAccess.java new file mode 100644 index 0000000..bd11237 --- /dev/null +++ b/src/main/java/org/apache/lucene/store/PosixNativeAccess.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.store; + +import java.io.IOException; +import java.lang.foreign.FunctionDescriptor; +import java.lang.foreign.Linker; +import java.lang.foreign.MemorySegment; +import java.lang.foreign.SymbolLookup; +import java.lang.foreign.ValueLayout; +import java.lang.invoke.MethodHandle; +import java.util.Locale; +import java.util.Optional; +import java.util.logging.Logger; +import org.apache.lucene.store.IOContext.Context; + +@SuppressWarnings("preview") +final class PosixNativeAccess extends NativeAccess { + + private static final Logger LOG = Logger.getLogger(PosixNativeAccess.class.getName()); + + // these constants were extracted from glibc and macos header files - luckily they are the same: + + /** No further special treatment. */ + public static final int POSIX_MADV_NORMAL = 0; + + /** Expect random page references. */ + public static final int POSIX_MADV_RANDOM = 1; + + /** Expect sequential page references. */ + public static final int POSIX_MADV_SEQUENTIAL = 2; + + /** Will need these pages. */ + public static final int POSIX_MADV_WILLNEED = 3; + + /** Don't need these pages. */ + public static final int POSIX_MADV_DONTNEED = 4; + + private static final MethodHandle MH$posix_madvise; + + private static final Optional INSTANCE; + + private PosixNativeAccess() {} + + static Optional getInstance() { + return INSTANCE; + } + + static { + MethodHandle adviseHandle = null; + PosixNativeAccess instance = null; + try { + adviseHandle = lookupMadvise(); + instance = new PosixNativeAccess(); + } catch (UnsupportedOperationException uoe) { + LOG.warning(uoe.getMessage()); + } catch ( + @SuppressWarnings("unused") + IllegalCallerException ice) { + LOG.warning( + String.format( + Locale.ENGLISH, + "Lucene has no access to native functions. To enable access to native functions, " + + "pass the following on command line: --enable-native-access=%s", + Optional.ofNullable(PosixNativeAccess.class.getModule().getName()) + .orElse("ALL-UNNAMED"))); + } + MH$posix_madvise = adviseHandle; + INSTANCE = Optional.ofNullable(instance); + } + + private static MethodHandle lookupMadvise() { + final Linker linker = Linker.nativeLinker(); + final SymbolLookup stdlib = linker.defaultLookup(); + return findFunction( + linker, + stdlib, + "posix_madvise", + FunctionDescriptor.of( + ValueLayout.JAVA_INT, + ValueLayout.ADDRESS, + ValueLayout.JAVA_LONG, + ValueLayout.JAVA_INT)); + } + + private static MethodHandle findFunction( + Linker linker, SymbolLookup lookup, String name, FunctionDescriptor desc) { + final MemorySegment symbol = + lookup + .find(name) + .orElseThrow( + () -> + new UnsupportedOperationException( + "Platform has no symbol for '" + name + "' in libc.")); + return linker.downcallHandle(symbol, desc); + } + + @Override + public void madvise(MemorySegment segment, IOContext context) throws IOException { + // Note: madvise is bypassed if the segment should be preloaded via MemorySegment#load. + if (segment.byteSize() == 0L) { + return; // empty segments should be excluded, because they may have no address at all + } + final Integer advice = mapIOContext(context); + if (advice == null) { + return; // do nothing + } + final int ret; + try { + ret = (int) MH$posix_madvise.invokeExact(segment, segment.byteSize(), advice.intValue()); + } catch (Throwable th) { + throw new AssertionError(th); + } + if (ret != 0) { + throw new IOException( + String.format( + Locale.ENGLISH, + "Call to posix_madvise with address=0x%08X and byteSize=%d failed with return code %d.", + segment.address(), + segment.byteSize(), + ret)); + } + } + + private Integer mapIOContext(IOContext ctx) { + // Merging always wins and implies sequential access, because kernel is advised to free pages + // after use: + if (ctx.context == Context.MERGE) { + return POSIX_MADV_SEQUENTIAL; + } + if (ctx.randomAccess) { + return POSIX_MADV_RANDOM; + } + if (ctx.readOnce) { + return POSIX_MADV_SEQUENTIAL; + } + return null; + } +} From 0c875d93cb23d403bc3344f61c68093b531a892f Mon Sep 17 00:00:00 2001 From: Tamas Balog Date: Wed, 18 Sep 2024 14:05:35 +0200 Subject: [PATCH 2/3] Update gradle wrapper to 8.9 --- gradle.properties | 2 +- gradle/wrapper/gradle-wrapper.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 9a37c21..6d6fddc 100644 --- a/gradle.properties +++ b/gradle.properties @@ -22,7 +22,7 @@ platformPlugins = platformBundledPlugins = # Gradle Releases -> https://github.com/gradle/gradle/releases -gradleVersion = 8.8 +gradleVersion = 8.9 # Opt-out flag for bundling Kotlin standard library -> https://jb.gg/intellij-platform-kotlin-stdlib kotlin.stdlib.default.dependency = false diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index a441313..09523c0 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-bin.zip networkTimeout=10000 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME From 89138a330e0bcb8af727aafc8e23fdd8ae606d2b Mon Sep 17 00:00:00 2001 From: Tamas Balog Date: Thu, 19 Sep 2024 08:04:33 +0200 Subject: [PATCH 3/3] Add notification popup for JDK 21 configuration --- .../lucas/ProjectStartupActivity.java | 42 +++++++++++++++++++ src/main/resources/META-INF/plugin.xml | 2 + 2 files changed, 44 insertions(+) create mode 100644 src/main/java/com/picimako/lucas/ProjectStartupActivity.java diff --git a/src/main/java/com/picimako/lucas/ProjectStartupActivity.java b/src/main/java/com/picimako/lucas/ProjectStartupActivity.java new file mode 100644 index 0000000..e2affa8 --- /dev/null +++ b/src/main/java/com/picimako/lucas/ProjectStartupActivity.java @@ -0,0 +1,42 @@ +//Copyright 2024 Tamás Balog. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. +package com.picimako.lucas; + +import com.intellij.ide.BrowserUtil; +import com.intellij.ide.util.RunOnceUtil; +import com.intellij.notification.NotificationGroupManager; +import com.intellij.notification.NotificationType; +import com.intellij.notification.Notifications; +import com.intellij.openapi.actionSystem.AnAction; +import com.intellij.openapi.actionSystem.AnActionEvent; +import com.intellij.openapi.project.Project; +import com.intellij.openapi.startup.ProjectActivity; +import kotlin.Unit; +import kotlin.coroutines.Continuation; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * Performs actions on project startup. + */ +public final class ProjectStartupActivity implements ProjectActivity { + + @Override + public @Nullable Object execute(@NotNull Project project, @NotNull Continuation continuation) { + RunOnceUtil.runOnceForApp("Lucas v0.4.0 - JDK 21", () -> { + var notification = NotificationGroupManager.getInstance().getNotificationGroup("Lucas JDK 21 Configuration") + .createNotification("Lucas v0.4.0 is built on JDK 21, and it requires manual configuration which is detailed in the documentation.", NotificationType.WARNING) + .setSubtitle("Lucas JDK 21 Configuration") + .addAction(new AnAction("Open Documentation") { + + @Override + public void actionPerformed(@NotNull AnActionEvent e) { + BrowserUtil.browse("https://github.com/picimako/lucas/blob/main/README.md#intellij-20242-and-jdk-21-support"); + } + }); + + Notifications.Bus.notify(notification, project); + }); + + return Unit.INSTANCE; + } +} diff --git a/src/main/resources/META-INF/plugin.xml b/src/main/resources/META-INF/plugin.xml index 2dd3861..97c36c9 100644 --- a/src/main/resources/META-INF/plugin.xml +++ b/src/main/resources/META-INF/plugin.xml @@ -8,6 +8,8 @@ com.intellij.modules.platform + +