Merge branch 'master' into gh-5279-s3-zstd

Author: at055612

.sdkmanrc

@@ -1,4 +1,4 @@
 # Enable auto-env through the sdkman_auto_env config
 # Add key=value pairs of SDKs to use below
-java=25.0.2-tem
+java=25.0.3-tem
 

CHANGELOG.md

@@ -13,6 +13,169 @@ DO NOT ADD CHANGES HERE - ADD THEM USING log_change.sh
 ~~~
 
 
+## [v7.12.4] - 2026-05-14
+
+* Refactor : Improve logging and system info output for Data Feed Keys.
+
+* Feature **#5526** : Add event logging to ask stroom AI.
+
+* Bug **#5544** : Fix DocRef bug.
+
+
+## [v7.12.3] - 2026-05-11
+
+* Feature **#5183** : Add title to dashboard link function.
+
+* Bug **#5518** : Fix expression editor bug.
+
+* Bug **#5507** : Improve dashboard column filtering.
+
+* Bug **#5516** : Allow event links with only feed USE permission.
+
+* Feature : Add System Info Admin Servlet. Add menu of all admin servlets at `<admin port>/<admin path>/menu`.
+
+* Bug : Fix permission exceptions when getting volume system info.
+
+
+## [v7.12.2] - 2026-05-05
+
+* Bug **#5532** : Relax Data Feed Identities validation so salt is optional.
+
+
+## [v7.12.1] - 2026-04-27
+
+* Bug **#5520** : Fix annotation decoration in queries/dashboards not working if the EventId/StreamId columns are not longs.
+
+* Feature : Add config prop `stroom.annotation.eventLinkCacheSizeLimit` (default 1,000,000) to protect Stroom from caching too many annotation to event links. If this limit is exceeded, the query will error.
+
+
+## [v7.12.0] - 2026-04-27
+
+* Feature : Change the annotation caching to invalidate on a field basis rather than the whole annotation.
+
+* Bug : Fix annotation dash/query columns not updating when status/label/collection names are deleted/modified.
+
+* Bug : Fix annotation decorations in dash/query columns showing deleted annotation entries.
+
+* Bug : Fix annotation Comment and History columns not updating when entries are added/updated/deleted.
+
+* Bug **#5508** : Improve annotation decoration performance.
+
+* Bug : Fix Annotation -> Events screen not refreshing the linked events list when an event link is added/removed.
+
+* Bug : Add missing directories (data_feed_identities, git_repo, lmdb_library, planb, reference_staging_data) as volumes in the docker image.
+
+
+## [v7.13-beta.2] - 2026-04-15
+
+* Build : Fix CI build failure due to missing Docker container prefix.
+
+
+## [v7.13-beta.1] - 2026-04-13
+
+* Feature **#5282** : Add pipeline scheduling.
+
+* Feature **#5346** : Choose which external document changes to save when saving a pipeline.
+
+* Issue **#5366** : Add level and status to rules.
+
+* Feature **#5377** : Embedded pipeline docs.
+
+* Issue **#5366** : Create new doc object DataGen.
+
+* Issue **#5366** : Add Execution tab to DataGen.
+
+* Feature **#5387** : Allow pipeline stepping across multiple streams.
+
+* Issue **#5366** : Implement job scheduling for DataGen.
+
+* Feature **#3103** : Allow multiple dashboard instances.
+
+* Refactor : Refactor the feedKey locking in PreAggregator and make AttributeMapUtil#readKeys() more lenient when reading `.meta `files.
+
+* Dependency : Uplift org.apache.commons:commons-pool2 from 2.12.1 to 2.13.1.
+
+* Dependency : Uplift org.flywaydb:flyway-core from 11.20.0 to 12.0.0.
+
+* Dependency : Uplift org.eclipse.jgit:org.eclipse.jgit from 7.3.0.202506031305-r to 7.5.0.202512021534-r.
+
+* Dependency : Uplift org.apache.solr:solr-solrj from 9.8.0 to 9.10.1.
+
+* Dependency : Uplift swagger from 2.2.41 to 2.2.42.
+
+* Bug : Fix output of the manage_users --listPermissions command.
+
+* Bug : Fix missing arg validation on reset_password CLI command. Obfuscate password in logging.
+
+* Refactor : Remove HBase statistics (may require change to default config).
+
+* Refactor : Remove ScyllaDB based state store.
+
+* Feature : Editing items in the UI now indicates save required only when changes are made.
+
+* Issue **#5366** : Refactor schedulers.
+
+* Feature **#3206** : User tab sessions.
+
+* Issue **#5366** : Refactor schedulers to interface and cleanup.
+
+* Dependency : Uplift gwt from 2.12.2 to 2.13.0.
+
+* Dependency : Uplift co.elastic.clients:elasticsearch-java from 9.2.1 to 9.3.2.
+
+* Build : Uplift gradle-wrapper from 9.3.0 to 9.3.1.
+
+* Dependency : Uplift org.yaml:snakeyaml from 2.2 to 2.6.
+
+* Refactor : Replace NullSafe.requireNonNullElse() with Objects.requireNonNullElse().
+
+* Refactor : Replace NullSafe.requireNonNullElseGet() with Objects.requireNonNullElseGet().
+
+* Issue **#5366** : Fix typo.
+
+* Feature **#5366** : Rebase on master.
+
+* Feature **#5366** : Cleanup.
+
+* Feature **#5366** : Checkstyle.
+
+* Feature **#5232** : Add standard annotation comments. MIGRATION: Any comments previously configured in standardComments property will need to saved as annotation comments.
+
+* Feature **#5366** : Change icon for DataGen.
+
+* Feature **#5366** : Add feed name to rule detection.
+
+* Feature : Improve the ProgressMonitor task creation logging output to included counts of errored and skipped filters. Change task creation to re-test filter enabled/deleted state just prior to creating tasks. Add validation to ProcessorConfig to enforce a minimum value of `1` on some properties.
+
+* Feature **#5366** : Fix DataGen destination feed not firing dirty event.
+
+* Feature **#5366** : Checkstyle.
+
+* Bug **#5503** : Fix search rerank LLM processing and add debug logging.
+
+* Dependency : Uplift docker images to `eclipse-temurin:25.0.2_10-jdk-alpine-3.23`. Change Java docker build to use 25.0.2. Change ERD build to use a fixed docker image.
+
+* Bug **#5484** : Fix stream browser Retention column and Info pane showing wrong retention period/rule.
+
+* Feature **#5490** : Add rerank capability to dense vector search.
+
+* Feature **#5492** : Change Git Repo folder colour as red looked like an error.
+
+* Bug **#5486** : Fix SQL to physically delete annotations.
+
+* Bug **#5488** : Fix `forwardFileDestinations[n].atomicMoveEnabled` property not being used.
+
+* Feature **#5480** : Group some annotation history items and make them expandable.
+
+* Bug **#5485** : Fix Plan B snapshot race condition bug.
+
+* Bug **#5495** : Fix query column order bug.
+
+* Bug **#5495** : Add option to reset query table customisation.
+
+* Feature **#5161** : Make dashboard tab menu available from settings icon.
+
+
 ## [v7.12-beta.1] - 2026-03-24
 
 * Feature **#5427** : Change the Data Feed Key authentication mechanism to support authentication by X509 certificate DN. Add a new allowed type of `CERTIFICATE_IDENTITY` to `.receive.enabledAuthenticationTypes`. Rename property `.receive.dataFeedKeysDir` to  `.receive.dataFeedIdentitiesDir` and change the structure of the files in it, all files will have to be replaced. Rename property `.receive.dataFeedKeyOwnerMetaKey` to `.receive.dataFeedOwnerMetaKey`. Change the default value of `.receive.dataFeedIdentitiesDir` from `data_feed_keys` to `data_feed_identities`.
@@ -2080,7 +2243,9 @@ DO NOT ADD CHANGES HERE - ADD THEM USING log_change.sh
 * Issue **#3830** : Add S3 data storage option.
 
 
-[Unreleased]: https://github.com/gchq/stroom/compare/v7.12-beta.1...HEAD
+[Unreleased]: https://github.com/gchq/stroom/compare/v7.13-beta.2...HEAD
+[v7.13-beta.2]: https://github.com/gchq/stroom/compare/v7.13-beta.1...v7.13-beta.2
+[v7.13-beta.1]: https://github.com/gchq/stroom/compare/v7.12-beta.1...v7.13-beta.1
 [v7.12-beta.1]: https://github.com/gchq/stroom/compare/v7.11.6...v7.12-beta.1
 [v7.11.6]: https://github.com/gchq/stroom/compare/v7.11.5...v7.11.6
 [v7.11.5]: https://github.com/gchq/stroom/compare/v7.11.4...v7.11.5

build.gradle

@@ -197,6 +197,8 @@ subprojects {
             implementation platform(libs.dropwizard.dependencies)
             implementation platform(libs.dropwizard.bom)
             implementation platform(libs.aws.bom)
+            implementation platform(libs.jackson.bom)
+            implementation platform(libs.jackson2.bom)
 
             testImplementation platform(libs.junit.bom)
         }

ci_build.sh

@@ -107,8 +107,9 @@ create_file_hash() {
 stop_and_clear_down_stroom_all_dbs() {
   # clear down stroom-all-dbs container and volumes so we have a blank slate
   echo -e "${GREEN}Clearing down stroom-all-dbs${NC}"
-  docker ps -q -f=name='stroom-all-dbs' | xargs -r docker stop --time 0
-  docker ps -a -q -f=name='stroom-all-dbs' | xargs -r docker rm
+  # bouncit.sh prefixes the volume and container names with bounceit_
+  docker ps -q -f=name='bounceit_stroom-all-dbs' | xargs -r docker stop --time 0
+  docker ps -a -q -f=name='bounceit_stroom-all-dbs' | xargs -r docker rm
   docker volume ls -q -f=name='bounceit_stroom-all-dbs*' | xargs -r docker volume rm
 }
 
@@ -139,6 +140,9 @@ start_databases() {
     -x \
     "${dbs_to_start[@]}"
 
+  echo -e "${GREEN}Running Docker containers:${NC}"
+  docker ps --format '{{.Names}}'
+
   popd > /dev/null
 }
 
@@ -148,6 +152,7 @@ generate_ddl_dump() {
 
   stop_and_clear_down_stroom_all_dbs
 
+  # The service name is not prefixed with bounceit_ at this point
   start_databases stroom-all-dbs
 
   # Run the db migration against the empty db to give us a vanilla
@@ -159,7 +164,7 @@ generate_ddl_dump() {
   echo -e "${GREEN}Dumping the database DDL${NC}"
   # Produce the dump file
   docker exec \
-    stroom-all-dbs \
+    bounceit_stroom-all-dbs \
     mysqldump \
       -d \
       -p"my-secret-pw" \

container_build/docker_java/Dockerfile

@@ -18,7 +18,7 @@
 # It also contains plantuml for plant uml image generation
 # Using 'openjdk' on Alpine is not fully supported so using Eclipse Temurin JDK to ensure we have a known jdk version
 # See https://github.com/docker-library/docs/blob/master/openjdk/README.md#openjdkversion-alpine
-FROM eclipse-temurin:25.0.2_10-jdk-alpine-3.23@sha256:da683f4f02f9427597d8fa162b73b8222fe08596dcebaf23e4399576ff8b037e
+FROM eclipse-temurin:25.0.3_9-jdk-alpine-3.23@sha256:30d9f87d702c2c1c601ed0d31e0c88ea1ea474ee7676cda7b7a59e759181c4dd
 
 # Work from the shared git repo dir
 WORKDIR /builder/shared

gradle/libs.versions.toml

@@ -6,7 +6,7 @@ stroomStats = "1.0-alpha.6"
 # ------------3rd-party------------
 aws = "2.40.5"
 commons-io = "2.21.0"
-dropwizard = "4.0.14" # used to set the dropwizard-bom version, that controls lots of dependency versions
+dropwizard = "5.0.1" # used to set the dropwizard-bom version, that controls lots of dependency versions
 elasticsearch = "9.2.1"
 flyway = "12.0.0"
 guice = "7.0.0"
@@ -93,16 +93,30 @@ hikari = { module = "com.zaxxer:HikariCP", version = "7.0.2" }
 http-client = { module = "org.apache.httpcomponents.client5:httpclient5" } # version controlled by dropwizard-dependencies
 jBCrypt = { module = "de.svenkubiak:jBCrypt", version = "0.4.3" }
 jspecify = { module = "org.jspecify:jspecify", version = "1.0.0" }
-jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations" } # version controlled by dropwizard-dependencies
-#jackson-core = { module = "com.fasterxml.jackson.core:jackson-core", version = "2.9.10" } # version controlled by dropwizard-dependencies
-jackson-core = { module = "com.fasterxml.jackson.core:jackson-core" } # version controlled by dropwizard-dependencies
-jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind" } # version controlled by dropwizard-dependencies # files('libs/jackson-databind-2.10.5.1.jar'),#
-jackson-dataformat-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" } # version controlled by dropwizard-dependencies
-jackson-datatype-jdk8 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jdk8" } # version controlled by dropwizard-dependencies
-jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" } # version controlled by dropwizard-dependencies
+
+# Dropwiz 5.0.1 still pulls in jackson 2 but they have different manven coords
+# and package names so can be used together.
+# -jdk8 and -jsr310 are now included in -databind in v3
+# https://github.com/FasterXML/jackson/blob/main/jackson3/MIGRATING_TO_JACKSON_3.md
+jackson-bom = { module = "tools.jackson:jackson-bom", version = "3.1.2" }
+# jackson-annotations is an exception as it still uses the old v2 maven coords
+jackson-annotations = { module = "com.fasterxml.jackson.core:jackson-annotations" } # version controlled by jackson-bom
+jackson-core = { module = "tools.jackson.core:jackson-core" } # version controlled by jackson-bom
+jackson-databind = { module = "tools.jackson.core:jackson-databind" } # version controlled by jackson-bom
+jackson-dataformat-yaml = { module = "tools.jackson.dataformat:jackson-dataformat-yaml" } # version controlled by jackson-bom
+
+# This is legacy Jackson v2 for anything that needs the old version, e.g. mbknor-jackson-jsonSchema
+# jackson-annotations is the same for both and is included above
+jackson2-bom = { module = "com.fasterxml.jackson:jackson-bom", version = "2.21.2" }
+jackson2-core = { module = "com.fasterxml.jackson.core:jackson-core" } # version controlled by jackson-bom
+jackson2-databind = { module = "com.fasterxml.jackson.core:jackson-databind" } # version controlled by jackson-bom
+jackson2-dataformat-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" } # version controlled by jackson-bom
+jackson2-datatype-jdk8 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jdk8" } # version controlled by jackson-bom
+jackson2-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" } # version controlled by jackson-bom
+
 jakarta-activation = { module = "jakarta.activation:jakarta.activation-api" } # version controlled by dropwizard-dependencies
 jakarta-annotation-api = { module = "jakarta.annotation:jakarta.annotation-api" } # version controlled by dropwizard-dependencies
-jakarta-el = { module = "org.glassfish:jakarta.el" } # version controlled by dropwizard-dependencies
+jakarta-el = { module = "jakarta.el:jakarta.el-api", version = "5.0.1" }
 jakarta-json-api = { module = "jakarta.json:jakarta.json-api", version = "2.0.1" }
 jakarta-servlet-api = { module = "jakarta.servlet:jakarta.servlet-api" } # version controlled by dropwizard-dependencies
 jakarta-validation-api = { module = "jakarta.validation:jakarta.validation-api" } # version controlled by dropwizard-dependencies
@@ -125,8 +139,9 @@ jersey-server = { module = "org.glassfish.jersey.core:jersey-server" } # version
 jersey-test-framework-grizzly2 = { module = "org.glassfish.jersey.test-framework.providers:jersey-test-framework-provider-grizzly2" } # version controlled by dropwizard-dependencies
 jetty-http = { module = "org.eclipse.jetty:jetty-http" } # version controlled by dropwizard-dependencies
 jetty-server = { module = "org.eclipse.jetty:jetty-server" } # version controlled by dropwizard-dependencies
-jetty-servlet = { module = "org.eclipse.jetty:jetty-servlet" } # version controlled by dropwizard-dependencies
-jetty-servlets = { module = "org.eclipse.jetty:jetty-servlets" } # version controlled by dropwizard-dependencies
+# The ee10 part needs to correspond to the version pulled in by dropwizard-jetty
+jetty-servlet = { module = "org.eclipse.jetty.ee10:jetty-ee10-servlet" } # version controlled by dropwizard-dependencies
+jetty-servlets = { module = "org.eclipse.jetty.ee10:jetty-ee10-servlets" } # version controlled by dropwizard-dependencies
 jgit = { module = "org.eclipse.jgit:org.eclipse.jgit", version.ref = "jgit" }
 jgitssh = { module = "org.eclipse.jgit:org.eclipse.jgit.ssh.apache", version.ref = "jgit" }
 jinjava = { module = "com.hubspot.jinjava:jinjava", version = "2.8.2" }
@@ -181,11 +196,13 @@ swagger-annotations = { module = "io.swagger.core.v3:swagger-annotations-jakarta
 swagger-core = { module = "io.swagger.core.v3:swagger-core-jakarta", version.ref = "swagger" }
 swagger-jaxrs = { module = "io.swagger.core.v3:swagger-jaxrs2-jakarta", version.ref = "swagger" }
 vavr = { module = "io.vavr:vavr", version = "0.10.4" }
-wiremock = { module = "org.wiremock:wiremock", version = "3.3.1" }
+# Wiremock 3.13.2 works with jetty 11 by default, but DW 5.0.1 is using Jetty 12, so we have to use
+# wiremock-jetty12
+wiremock = { module = "org.wiremock:wiremock-jetty12", version = "3.13.2" }
 ws-rs-api = { module = "jakarta.ws.rs:jakarta.ws.rs-api" } # version controlled by dropwizard-dependencies
 ws-rs-api-gwt = { module = "javax.ws.rs:javax.ws.rs-api", version = "2.1.1" } # For GWT compilation
 xerces = { module = "xerces:xercesImpl", version = "2.12.2" }
-zero-allocation-hashing = { module = "net.openhft:zero-allocation-hashing", version = "0.16" }
+zero-allocation-hashing = { module = "net.openhft:zero-allocation-hashing", version = "2026.0" }
 zstd-jni = { module =  "com.github.luben:zstd-jni", version = "1.5.7-6" }
 
 [plugins]
@@ -239,3 +256,4 @@ common-test-runtime = [
     "log4j-over-slf4j",
     "logback-classic",
     "logback-core"]
+

scripts/v07_11_db_pre_migration_checks.sql

@@ -0,0 +1,25 @@
+-- A set of SQL queries to run before migrating from v6 to v7
+-- See https://gchq.github.io/stroom-docs/releases/v07.11/upgrade-notes/
+-- 
+-- Run with the mysql --table arg to get formatted output
+-- e.g.
+-- mysql --force --table -h"localhost" -P"3306" -u"stroomuser" -p"stroompassword1" stroom < v07_11_db_pre_migration_checks.sql > v07_11_db_pre_migration_checks.out 2>&1
+-- docker exec -i stroom-all-dbs mysql --force --table -h"localhost" -P"3307" -u"stroomuser" -p"stroompassword1" stroom < v07_11_db_pre_migration_checks.sql > v07_11_db_pre_migration_checks.out 2>&1
+
+
+\! echo 'Find annotation entries with no assigned user. No action required if this returns nothing.';
+
+SELECT a.id, a.title, ae.id AS entry_id
+FROM annotation a
+INNER JOIN annotation_entry ae ON a.id = ae.fk_annotation_id
+WHERE ae.entry_user_uuid IS NULL OR ae.entry_user_uuid = '';
+
+\! echo 'Listing all enabled Stroom users for reference';
+
+SELECT uuid, name, display_name, full_name
+FROM stroom_user su
+WHERE is_group = false
+AND enabled = true
+ORDER by name;
+
+\! echo 'Finished';