Merge branch '7.12'

at055612 · at055612 · commit 98764beb4dcf · 2026-05-22T10:02:48.000+01:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -13,17 +13,16 @@ DO NOT ADD CHANGES HERE - ADD THEM USING log_change.sh
 ~~~
 
 
-## [v7.12.4] - 2026-05-14
+* Bug : Fix typo in query snippet name (`Eval first first value` => `Eval first value`).
+
+* Bug **#5549** : Make jffi extract its native library to the same dir as the LMDB native library file. Add the config prop `providedJffiLibraryPath` to allow for a provided jffi lib.
 
 * Refactor : Improve logging and system info output for Data Feed Keys.
 
 * Feature **#5526** : Add event logging to ask stroom AI.
 
 * Bug **#5544** : Fix DocRef bug.
 
-
-## [v7.12.3] - 2026-05-11
-
 * Feature **#5183** : Add title to dashboard link function.
 
 * Bug **#5518** : Fix expression editor bug.
@@ -36,21 +35,12 @@ DO NOT ADD CHANGES HERE - ADD THEM USING log_change.sh
 
 * Bug : Fix permission exceptions when getting volume system info.
 
-
-## [v7.12.2] - 2026-05-05
-
 * Bug **#5532** : Relax Data Feed Identities validation so salt is optional.
 
-
-## [v7.12.1] - 2026-04-27
-
 * Bug **#5520** : Fix annotation decoration in queries/dashboards not working if the EventId/StreamId columns are not longs.
 
 * Feature : Add config prop `stroom.annotation.eventLinkCacheSizeLimit` (default 1,000,000) to protect Stroom from caching too many annotation to event links. If this limit is exceeded, the query will error.
 
-
-## [v7.12.0] - 2026-04-27
-
 * Feature : Change the annotation caching to invalidate on a field basis rather than the whole annotation.
 
 * Bug : Fix annotation dash/query columns not updating when status/label/collection names are deleted/modified.
diff --git a/stroom-config/stroom-config-app/src/test/resources/stroom/config/app/expected.yaml b/stroom-config/stroom-config-app/src/test/resources/stroom/config/app/expected.yaml
@@ -588,6 +588,7 @@ appConfig:
   lifecycle:
     enabled: true
   lmdbLibrary:
+    providedJffiLibraryPath: null
     providedSystemLibraryPath: null
     systemLibraryExtractDir: "lmdb_library"
   logging:
diff --git a/stroom-core-client-widget/src/main/java/edu/ycp/cs/dh/acegwt/public/ace/snippets/stroom_query.js b/stroom-core-client-widget/src/main/java/edu/ycp/cs/dh/acegwt/public/ace/snippets/stroom_query.js
@@ -36,7 +36,7 @@ eval EventId = first(EventId)
     },
     {
         "tabTrigger": "first",
-        "name": "Eval first first value",
+        "name": "Eval first value",
         "content": "eval ${1:field_name} = first(${1})\n$0",
     },
 ];
diff --git a/stroom-lmdb/src/main/java/stroom/lmdb/LmdbLibrary.java b/stroom-lmdb/src/main/java/stroom/lmdb/LmdbLibrary.java
@@ -21,6 +21,7 @@
 import stroom.util.logging.LambdaLogger;
 import stroom.util.logging.LambdaLoggerFactory;
 import stroom.util.logging.LogUtil;
+import stroom.util.shared.NullSafe;
 
 import jakarta.inject.Inject;
 import jakarta.inject.Provider;
@@ -31,7 +32,6 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.Objects;
-import java.util.Optional;
 
 @Singleton // The LMDB lib is dealt with statically by LMDB java so only want to initialise it once
 public class LmdbLibrary {
@@ -67,49 +67,107 @@ private synchronized void configureLibraryUnderLock(final LmdbLibraryConfig lmdb
         if (!HAS_LIBRARY_BEEN_CONFIGURED) {
             LOGGER.info("Configuring LMDB system library");
 
-            final Path lmdbSystemLibraryPath = Optional.ofNullable(lmdbLibraryConfig.getProvidedSystemLibraryPath())
-                    .map(pathCreator::toAppPath)
-                    .orElse(null);
+            LOGGER.debug(() -> LogUtil.message("""
+                            configureLibraryUnderLock() - lmdbLibraryConfig: {},
+                              {},
+                              {},
+                              {},
+                              {}""",
+                    lmdbLibraryConfig,
+                    dumpSystemProp(LmdbLibraryConfig.LMDB_EXTRACT_DIR_PROP),
+                    dumpSystemProp(LmdbLibraryConfig.LMDB_NATIVE_LIB_PROP),
+                    dumpSystemProp(LmdbLibraryConfig.JFFI_EXTRACT_DIR_PROP),
+                    dumpSystemProp(LmdbLibraryConfig.JFFI_NATIVE_LIB_PROP)));
+
+            final Path lmdbSystemLibraryPath = NullSafe.get(
+                    lmdbLibraryConfig.getProvidedSystemLibraryPath(),
+                    pathCreator::toAppPath);
+            final Path jffiNativeLibraryPath = NullSafe.get(
+                    lmdbLibraryConfig.getProvidedJffiLibraryPath(),
+                    pathCreator::toAppPath);
+
+            boolean extractLmdb = false;
+            boolean extractJffi = false;
 
             if (lmdbSystemLibraryPath != null) {
                 if (!Files.isReadable(lmdbSystemLibraryPath)) {
-                    throw new RuntimeException("Unable to read LMDB system library at " +
-                            lmdbSystemLibraryPath.toAbsolutePath().normalize());
+                    throw new RuntimeException("Unable to read LMDB system library at " + lmdbSystemLibraryPath);
                 }
                 // jakarta.validation should ensure the path is valid if set
                 final String lmdbNativeLibProp = LmdbLibraryConfig.LMDB_NATIVE_LIB_PROP;
-                System.setProperty(lmdbNativeLibProp, lmdbSystemLibraryPath.toAbsolutePath().normalize().toString());
-                LOGGER.info("Using provided LMDB system library file. Setting prop {} to '{}'",
+                System.setProperty(lmdbNativeLibProp, lmdbSystemLibraryPath.toString());
+                LOGGER.info("Using provided LMDB native library file. Setting prop {} to '{}'",
                         lmdbNativeLibProp, lmdbSystemLibraryPath);
             } else {
-                final Path systemLibraryExtractDir = getLibraryExtractDir(lmdbLibraryConfig);
+                extractLmdb = true;
+            }
+
+            final String jffiNativeLibProp = LmdbLibraryConfig.JFFI_NATIVE_LIB_PROP;
+            final String jffiPropVal = System.getenv(jffiNativeLibProp);
+            if (NullSafe.isNonBlankString(jffiPropVal)) {
+                LOGGER.info("Using provided JFFI native library file. Property {} already set to '{}'",
+                        jffiNativeLibProp, jffiPropVal);
+            } else if (jffiNativeLibraryPath != null) {
+                if (!Files.isReadable(jffiNativeLibraryPath)) {
+                    throw new RuntimeException("Unable to read JFFI native library at " + jffiNativeLibraryPath);
+                }
+                // jakarta.validation should ensure the path is valid if set
+                System.setProperty(jffiNativeLibProp, jffiNativeLibraryPath.toString());
+                LOGGER.info("Using provided JFFI native library file. Setting prop {} to '{}'",
+                        jffiNativeLibProp, lmdbSystemLibraryPath);
+            } else {
+                extractJffi = true;
+            }
 
-                // LMDB extracts the lib on boot to a unique temp file and should delete on JVM exit,
+            LOGGER.debug("configureLibraryUnderLock() - extractLmdb: {}, extractJffi: {}", extractLmdb, extractJffi);
+            if (extractLmdb || extractJffi) {
+                final Path systemLibraryExtractDir = getLibraryExtractDir(lmdbLibraryConfig);
+                // LMDB/JFFI extract the lib on boot to a unique temp file and should delete on JVM exit,
                 // but just in case clear out any old ones.
                 cleanUpExtractDir(systemLibraryExtractDir);
 
+                // Extract them both to the same place as both will need a non 'noexec' mount.
+
                 // Set the location to extract the bundled LMDB binary to
-                final String lmdbExtractDirProp = LmdbLibraryConfig.LMDB_EXTRACT_DIR_PROP;
-                System.setProperty(
-                        lmdbExtractDirProp,
-                        systemLibraryExtractDir.toAbsolutePath().normalize().toString());
-                LOGGER.info("Bundled LMDB system library binary will be extracted. Setting prop {} to '{}'",
-                        lmdbExtractDirProp, systemLibraryExtractDir);
-                HAS_LIBRARY_BEEN_CONFIGURED = true;
+                if (extractLmdb) {
+                    final String lmdbExtractDirProp = LmdbLibraryConfig.LMDB_EXTRACT_DIR_PROP;
+                    System.setProperty(lmdbExtractDirProp, systemLibraryExtractDir.toString());
+                    LOGGER.info("Bundled LMDB native library binary will be extracted and used. " +
+                                "Setting system prop '{}' to '{}'",
+                            lmdbExtractDirProp, systemLibraryExtractDir);
+                }
+
+                // Set the location to extract the bundled Jffi binary to
+                if (extractJffi) {
+                    final String jffiExtractDirProp = LmdbLibraryConfig.JFFI_EXTRACT_DIR_PROP;
+                    if (System.getProperty(jffiExtractDirProp) == null) {
+                        // Allow a jvm arg to override our config
+                        System.setProperty(jffiExtractDirProp, systemLibraryExtractDir.toString());
+                        LOGGER.info("Bundled JFFI native library binary will be extracted and used. " +
+                                    "Setting system prop '{}' to '{}'",
+                                jffiExtractDirProp, systemLibraryExtractDir);
+                    }
+                }
             }
+            HAS_LIBRARY_BEEN_CONFIGURED = true;
         } else {
             LOGGER.debug("Another thread beat us to it");
         }
     }
 
+    private String dumpSystemProp(final String propName) {
+        return "'" + propName + "': '" + System.getProperty(propName) + "'";
+    }
+
     private Path getLibraryExtractDir(final LmdbLibraryConfig lmdbLibraryConfig) {
         final String extractDirStr = lmdbLibraryConfig.getSystemLibraryExtractDir();
 
         final String extractDirPropName = lmdbLibraryConfig.getFullPathStr(LmdbLibraryConfig.EXTRACT_DIR_PROP_NAME);
 
         Path extractDir;
         if (extractDirStr == null) {
-            LOGGER.warn("LMDB system library extract dir is not set ({}), falling back to temporary directory {}.",
+            LOGGER.warn("LMDB system library extract dir is not set ({}), falling back to temporary directory {}. " +
+                        "If the temporary directory is mounted with 'noexec' any use of LMDB will error.",
                     extractDirPropName,
                     tempDirProvider.get());
 
@@ -119,6 +177,7 @@ private Path getLibraryExtractDir(final LmdbLibraryConfig lmdbLibraryConfig) {
         } else {
             extractDir = pathCreator.toAppPath(extractDirStr);
         }
+        extractDir = extractDir.toAbsolutePath().normalize();
 
         try {
             if (LOGGER.isDebugEnabled()) {
@@ -152,21 +211,27 @@ private Path getLibraryExtractDir(final LmdbLibraryConfig lmdbLibraryConfig) {
     }
 
     private void cleanUpExtractDir(final Path extractDir) {
-        try (final DirectoryStream<Path> directoryStream = Files.newDirectoryStream(
-                extractDir, "lmdbjava-native-library-*.so")) {
+        cleanUpExtractDir(extractDir, "lmdbjava-native-library-*.so", "LMDB native library");
+        // The jffi file should not really ever be there because it is deleted once loaded, but just in case
+        // See com.kenai.jffi.internal.StubLoader
+        cleanUpExtractDir(extractDir, "jffi*.so", "JFFI native library");
+    }
 
-            for (final Path redundantLibraryFilePath : directoryStream) {
-                LOGGER.info("Deleting redundant LMDB library file "
-                        + redundantLibraryFilePath.toAbsolutePath().normalize());
+    private static void cleanUpExtractDir(final Path extractDir, final String glob, final String name) {
+        try (final DirectoryStream<Path> directoryStream = Files.newDirectoryStream(extractDir, glob)) {
+            for (final Path file : directoryStream) {
+                LOGGER.info("Deleting redundant {} file {}", name, file.toAbsolutePath().normalize());
                 try {
-                    Files.deleteIfExists(redundantLibraryFilePath);
+                    Files.deleteIfExists(file);
                 } catch (final IOException e) {
-                    LOGGER.error("Unable to delete file " + extractDir.toAbsolutePath().normalize(), e);
+                    LOGGER.error(() -> LogUtil.message("Unable to delete {} file {}",
+                            name, file.toAbsolutePath().normalize()), e);
                     // swallow error as these old files don't matter really
                 }
             }
         } catch (final IOException e) {
-            throw new RuntimeException("Error listing contents of " + extractDir.toAbsolutePath().normalize());
+            throw new RuntimeException(LogUtil.message("Error listing contents of {} with glob '{}'",
+                    extractDir.toAbsolutePath().normalize(), glob));
         }
     }
 }
diff --git a/stroom-lmdb/src/main/java/stroom/lmdb/LmdbLibraryConfig.java b/stroom-lmdb/src/main/java/stroom/lmdb/LmdbLibraryConfig.java
@@ -32,59 +32,89 @@
 public class LmdbLibraryConfig extends AbstractConfig implements IsStroomConfig {
 
     static final String SYSTEM_LIBRARY_PATH_PROP_NAME = "providedSystemLibraryPath";
+    static final String JFFI_LIBRARY_PATH_PROP_NAME = "providedJffiLibraryPath";
     static final String EXTRACT_DIR_PROP_NAME = "systemLibraryExtractDir";
     static final String DEFAULT_LIBRARY_EXTRACT_SUB_DIR_NAME = "lmdb_library";
 
     // These are dups of org.lmdbjava.Library.LMDB_* but that class is pkg private for some reason.
     public static final String LMDB_EXTRACT_DIR_PROP = "lmdbjava.extract.dir";
     public static final String LMDB_NATIVE_LIB_PROP = "lmdbjava.native.lib";
+    public static final String JFFI_EXTRACT_DIR_PROP = "jffi.extract.dir";
+    public static final String JFFI_NATIVE_LIB_PROP = "jffi.boot.library.path";
 
     private final String providedSystemLibraryPath;
+    private final String providedJffiLibraryPath;
     private final String systemLibraryExtractDir;
 
     public LmdbLibraryConfig() {
         providedSystemLibraryPath = null;
+        providedJffiLibraryPath = null;
         systemLibraryExtractDir = DEFAULT_LIBRARY_EXTRACT_SUB_DIR_NAME;
     }
 
     @SuppressWarnings("unused")
     @JsonCreator
     public LmdbLibraryConfig(@JsonProperty(SYSTEM_LIBRARY_PATH_PROP_NAME) final String providedSystemLibraryPath,
+                             @JsonProperty(JFFI_LIBRARY_PATH_PROP_NAME) final String providedJffiLibraryPath,
                              @JsonProperty(EXTRACT_DIR_PROP_NAME) final String systemLibraryExtractDir) {
         this.providedSystemLibraryPath = providedSystemLibraryPath;
+        this.providedJffiLibraryPath = providedJffiLibraryPath;
         this.systemLibraryExtractDir = systemLibraryExtractDir;
     }
 
     @ValidFilePath
     @RequiresRestart(RestartScope.SYSTEM)
     @JsonProperty(SYSTEM_LIBRARY_PATH_PROP_NAME)
-    @JsonPropertyDescription("The path to a provided LMDB system library file. If unset the LMDB binary " +
+    @JsonPropertyDescription(
+            "The path to a provided LMDB native library file. If unset the LMDB binary " +
             "bundled with Stroom will be extracted to 'systemLibraryExtractDir'. This property can be used if " +
             "you already have LMDB installed or want to make use of a package manager provided instance. " +
-            "If you set this property care needs  to be taken over version compatibility between the version " +
-            "of LMDBJava (that Stroom uses to interact with LMDB) and the version of the LMDB binary.")
+            "If you set this property care needs to be taken over version compatibility between the version " +
+            "of LMDBJava (that Stroom uses to interact with LMDB) and the version of the LMDB binary. " +
+            "By default this is unset.")
     public String getProvidedSystemLibraryPath() {
         return providedSystemLibraryPath;
     }
 
+    @ValidFilePath
+    @RequiresRestart(RestartScope.SYSTEM)
+    @JsonProperty(JFFI_LIBRARY_PATH_PROP_NAME)
+    @JsonPropertyDescription(
+            "The path to a provided JFFI native library file, which is used by LMDBJava. If unset the JFFI binary " +
+            "bundled with Stroom will be extracted to 'systemLibraryExtractDir'. This property can be used if " +
+            "you provide your own version of JFFI. " +
+            "If you set this property care needs to be taken over version compatibility between the version " +
+            "of JFFI and the version of the JFFI binary. " +
+            "By default this is unset.")
+    public String getProvidedJffiLibraryPath() {
+        return providedJffiLibraryPath;
+    }
+
     @RequiresRestart(RestartScope.SYSTEM)
     @JsonProperty(EXTRACT_DIR_PROP_NAME)
-    @JsonPropertyDescription("The directory to extract the bundled LMDB system library to. Only used if " +
-            "property providedSystemLibraryPath is not set. On boot Stroom will extract the LMDB binary to this " +
-            "location. It will also delete old copies of the LMDB system library if found.")
+    @JsonPropertyDescription(
+            "The directory to extract the bundled LMDB and JFFI native libraries to. Only used if " +
+            "property providedSystemLibraryPath or property providedJffiLibraryPath are not set. " +
+            "On boot Stroom will clear this directory and then extract the native libraries into it. " +
+            "IMPORTANT: This directory must not have the 'noexec' mount option, or all LMDB use will error. " +
+            "Also, the contents are ephemeral, so should ideally be mounted using tmpfs or similar.")
     public String getSystemLibraryExtractDir() {
         return systemLibraryExtractDir;
     }
 
     @Override
     public String toString() {
         return "LmdbLibraryConfig{" +
-                "providedSystemLibraryPath='" + providedSystemLibraryPath + '\'' +
-                ", systemLibraryExtractDir='" + systemLibraryExtractDir + '\'' +
-                '}';
+               "providedSystemLibraryPath='" + providedSystemLibraryPath + '\'' +
+               ", providedJffiLibraryPath='" + providedJffiLibraryPath + '\'' +
+               ", systemLibraryExtractDir='" + systemLibraryExtractDir + '\'' +
+               '}';
     }
 
     public LmdbLibraryConfig withSystemLibraryExtractDir(final String systemLibraryExtractDir) {
-        return new LmdbLibraryConfig(providedSystemLibraryPath, systemLibraryExtractDir);
+        return new LmdbLibraryConfig(
+                providedSystemLibraryPath,
+                providedJffiLibraryPath,
+                systemLibraryExtractDir);
     }
 }

Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ eval EventId = first(EventId)`
`36`	`36`	`},`
`37`	`37`	`{`
`38`	`38`	`"tabTrigger": "first",`
`39`		`- "name": "Eval first first value",`
	`39`	`+ "name": "Eval first value",`
`40`	`40`	`"content": "eval ${1:field_name} = first(${1})\n$0",`
`41`	`41`	`},`
`42`	`42`	`];`