Merge branch 'folio-org:master' into deployment

Author: nielserik

.editorconfig

@@ -6,11 +6,10 @@ root = true
 [*]
 trim_trailing_whitespace = true
 insert_final_newline = true
-max_line_length = 80
+max_line_length = 120
 indent_style = space
 indent_size = 2
 
 # 4 space indentation
 [*.py]
-indent_style = space
 indent_size = 4

.github/workflows/api-doc.yml

@@ -33,13 +33,14 @@ on:
     paths:
       - 'ramls/**'
     tags: '[vV][0-9]+.[0-9]+.[0-9]+*'
+  workflow_dispatch:
 
 jobs:
   api-doc:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.REF }}
           submodules: recursive

.github/workflows/api-lint.yml

@@ -33,13 +33,14 @@ on:
   pull_request:
     paths:
       - 'ramls/**'
+  workflow_dispatch:
 
 jobs:
   api-lint:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Prepare folio-tools

.github/workflows/api-schema-lint.yml

@@ -23,13 +23,14 @@ on:
   pull_request:
     paths:
       - 'ramls/**'
+  workflow_dispatch:
 
 jobs:
   api-schema-lint:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Prepare folio-tools

.github/workflows/mac.yml

@@ -15,7 +15,7 @@ jobs:
     - uses: actions/setup-java@v3
       with:
         distribution: 'temurin'
-        java-version: '17'
+        java-version: '21'
         cache: maven
     - run: mvn clean install -DskipTests
     - run: brew install ${{ matrix.postgres }}

Dockerfile

@@ -1,4 +1,4 @@
-FROM folioci/alpine-jre-openjdk17:latest
+FROM folioci/alpine-jre-openjdk21:latest
 
 # Install latest patch versions of packages: https://pythonspeed.com/articles/security-updates-in-docker/
 USER root

Jenkinsfile

@@ -5,7 +5,7 @@ buildMvn {
   mvnDeploy = 'yes'
   doKubeDeploy = true
   publishPreview = false
-  buildNode = 'jenkins-agent-java17'
+  buildNode = 'jenkins-agent-java21'
 
   doDocker = {
     buildJavaDocker {

NEWS.md

@@ -1,17 +1,107 @@
-## v27.2.0 In progress
+## v29.1.0 YYYY-mm-DD
+### Breaking changes
+* Description ([ISSUE](https://folio-org.atlassian.net/browse/ISSUE))
+
+### New APIs versions
+* Provides `item-storage 10.2`
+
+### Features
+* Make max.request.size configurable for reindex holdings/items producers ([MODINVSTOR-1372](https://folio-org.atlassian.net/browse/MODINVSTOR-1372))
+* Create index for discoverySuppress and source fields to improve performance of queries which include mentioned fields ([MODINVSTOR-1377](https://folio-org.atlassian.net/browse/MODINVSTOR-1377))
+* Introduce new post retrieve api to fetch inventory item ([MODINVSTOR-1381](https://folio-org.atlassian.net/browse/MODINVSTOR-1381))
+* Optimize reindex instances database fetch query ([MODINVSTOR-1395](https://folio-org.atlassian.net/browse/MODINVSTOR-1395))
+
+### Bug fixes
+* Fix ordering of electronic access items for inventory-hierarchy, oai-pmh-view ([MODINVSTOR-1224](https://folio-org.atlassian.net/browse/MODINVSTOR-1224))
+* Fix error serialization when writing to files uploaded to S3 ([MODINVSTOR-1389](https://folio-org.atlassian.net/browse/MODINVSTOR-1389))
+
+### Tech Dept
+* Description ([ISSUE](https://folio-org.atlassian.net/browse/ISSUE))
+
+### Dependencies
+* Bump `LIB_NAME` from `OLD_VERSION` to `NEW_VERSION`
+* Add `LIB_NAME VERSION`
+* Remove `LIB_NAME`
+
+---
+
+## v29.0.0 2025-03-13
+### Breaking changes
+* Upgrade to Java 21 ([MODINVSTOR-1364](https://folio-org.atlassian.net/browse/MODINVSTOR-1364))
+* Remove deprecated batch API `/instance-storage/batch/instances`  ([MODINVSTOR-1182](https://folio-org.atlassian.net/browse/MODINVSTOR-1182))
+
+### New APIs versions
+* Provides `inventory-hierarchy 0.6`
+* Provides `instance-storage v11.1`
+* Disables `instance-storage-batch`
+
+### Features
+* Add new location property locationName that points to location name. ([MODINVSTOR-1362](https://folio-org.atlassian.net/browse/MODINVSTOR-1362))
+* Update rollback mechanism for instances when linking/unlinking with subjects is failed. ([MODINVSTOR-1299](https://folio-org.atlassian.net/browse/MODINVSTOR-1299))
+* Unable to delete local Subject types/sources when they are linked to an Instance ([MODINVSTOR-1284](https://folio-org.atlassian.net/browse/MODINVSTOR-1284))
+* Modify endpoint for bulk instances upsert with publish events flag ([MODINVSTOR-1283](https://folio-org.atlassian.net/browse/MODINVSTOR-1283))
+* Change Kafka event publishing keys for holdings and items ([MODINVSTOR-1281](https://folio-org.atlassian.net/browse/MODINVSTOR-1281))
+* Merge custom ECS TLR feature branch into master ([MODINVSTOR-1262](https://folio-org.atlassian.net/browse/MODINVSTOR-1262))
+* Service points synchronization: implement processors ([MODINVSTOR-1246](https://folio-org.atlassian.net/browse/MODINVSTOR-1246))
+* Service points synchronization: create a verticle ([MODINVSTOR-1245](https://folio-org.atlassian.net/browse/MODINVSTOR-1245))
+* Do not return routing service points by default ([MODINVSTOR-1219](https://folio-org.atlassian.net/browse/MODINVSTOR-1219))
+* Implement Kafka Event Publishing for Call-Number Type CRUD Operations ([MODINVSTOR-1275](https://folio-org.atlassian.net/browse/MODINVSTOR-1275))
+* Extend domain events with eventId and eventTs ([MODINVSTOR-1322](https://folio-org.atlassian.net/browse/MODINVSTOR-1322))
+* Revert the publication period migration changes ([MODINVSTOR-1280](https://folio-org.atlassian.net/browse/MODINVSTOR-1280))
+* Optimize check for existing shadow Instance before creating it ([MODINVSTOR-1354](https://folio-org.atlassian.net/browse/MODINVSTOR-1354))
+* Add "deleted" field to Instance schema ([MODINVSTOR-1342](https://folio-org.atlassian.net/browse/MODINVSTOR-1342))
+* Perform bulk in transaction to prevent Optimistic Locking ([MODINVSTOR-1369](https://folio-org.atlassian.net/browse/MODINVSTOR-1369))
+* Prevent update of instance/item/holdings in case there are no changes ([MODINVSTOR-1363](https://folio-org.atlassian.net/browse/MODINVSTOR-1363))
+
+### Bug fixes
+* Add item barcode right truncation search index ([MODINVSTOR-1292](https://folio-org.atlassian.net/browse/MODINVSTOR-1292))
+* Sort holdings by location name in instanceId query ([MODINVSTOR-1343](https://folio-org.atlassian.net/browse/MODINVSTOR-1343))
+
+### Tech Dept
+* Delete 3 unused instance database indexes ([MODINVSTOR-1277](https://folio-org.atlassian.net/browse/MODINVSTOR-1277))
+
+### Dependencies
+* Bump `raml-module-builder` from `35.3.0` to `35.4.0`
+* Bump `folio-kafka-wrapper` from `3.2.0` to `3.3.1`
+* Bump `folio-s3-client` from `2.2.0` to `2.3.0`
+* Bump `vertx` from `4.5.10` to `4.5.13`
+* Bump `caffeine` from `3.1.8` to `3.2.0`
+* Bump `lombok` from `1.18.34` to `1.18.36`
+* Bump `log4j` from `2.24.1` to `2.24.3`
+* Remove `google-code-gson`
 
+---
+
+## v28.0.0 2024-11-01
 ### Breaking changes
+* Migrate "publicationPeriod" data to the Dates object and remove it from the Instance schema ([MODINVSTOR-1232](https://folio-org.atlassian.net/browse/MODINVSTOR-1232))
+* Delete deprecated `shelf-locations` API ([MODINVSTOR-1183](https://folio-org.atlassian.net/browse/MODINVSTOR-1183))
 * Required sourceId field in holdings record ([MODINVSTOR-1161](https://folio-org.atlassian.net/browse/MODINVSTOR-1161))
 
 ### New APIs versions
+* Provides `instance-storage 11.0`
+* Provides `instance-storage-batch 3.0`
+* Provides `instance-storage-batch-sync 3.0`
+* Provides `instance-storage-batch-sync-unsafe 3.0`
+* Provides `inventory-view 3.0`
+* Provides `inventory-view-instance-set 3.0`
+* Provides `instance-iteration 1.0`
+* Provides `holdings-storage 8.0`
+* Provides `bound-with-parts-storage 2.0`
+* Provides `async-migration 1.0`
 * Provides `subject-source 1.0`
 * Provides `subject-types 1.0`
+* Provides `instance-storage-bulk 1.0`
 * Provides `instance-date-types 1.0`
-* Provides `instance-storage 10.2`
-* Requires `holdings-storage 6.1`
+* Provides `locations 3.1`
+
+### Removed APIs
+* `shelf-locations`
 
 ### Features
+* Add floating collection flag in location schema ([MODINVSTOR-1250](https://issues.folio.org/browse/MODINVSTOR-1250))
 * Implement domain event production for location create/update/delete ([MODINVSTOR-1181](https://issues.folio.org/browse/MODINVSTOR-1181))
+* Add a new boolean field ecsRequestRouting to the service point schema ([MODINVSTOR-1179](https://issues.folio.org/browse/MODINVSTOR-1179))
 * Implement domain event production for library create/update/delete ([MODINVSTOR-1216](https://issues.folio.org/browse/MODINVSTOR-1216))
 * Implement domain event production for campus create/update/delete ([MODINVSTOR-1217](https://issues.folio.org/browse/MODINVSTOR-1217))
 * Implement domain event production for institution create/update/delete ([MODINVSTOR-1218](https://issues.folio.org/browse/MODINVSTOR-1218))
@@ -23,25 +113,26 @@
 * Info, not warn, about expected 403 from /user-tenants ([MODINVSTOR-1237](https://folio-org.atlassian.net/browse/MODINVSTOR-1237))
 * Implement Subject sources management ([MODINVSTOR-1222](https://folio-org.atlassian.net/browse/MODINVSTOR-1222))
 * Add new date type fields to Instance schema ([MODINVSTOR-1188](https://folio-org.atlassian.net/browse/MODINVSTOR-1188))
-
+* Implement endpoint for bulk instances upsert from external file ([MODINVSTOR-1225](https://folio-org.atlassian.net/browse/MODINVSTOR-1225))
+* Add Subject source and Subject type to schema ([MODINVSTOR-1205](https://folio-org.atlassian.net/browse/MODINVSTOR-1205))
+* Add codes to Subject sources ([MODINVSTOR-1264](https://folio-org.atlassian.net/browse/MODINVSTOR-1264))
+* Implement publication period migration, create new InstanceWithoutPubPeriod schema for request/response API ([MODINVSTOR-1271](https://folio-org.atlassian.net/browse/MODINVSTOR-1271))
 
 ### Bug fixes
 * Unintended update of instance records \_version (optimistic locking) whenever any of its holdings or items are created, updated or deleted. ([MODINVSTOR-1186](https://folio-org.atlassian.net/browse/MODINVSTOR-1186))
 * Deserialization of Date from long (MODINVSTOR-1198)[https://folio-org.atlassian.net/browse/MODINVSTOR-1198]
 * Do not delete Kafka topics on postTenant if collection topics is enabled ([MODINVSTOR-1192](https://folio-org.atlassian.net/browse/MODINVSTOR-1192))
 * Identifier types: change Cancelled LCCN to Canceled LCCN ([MODINVSTOR-1212](https://folio-org.atlassian.net/browse/MODINVSTOR-1212))
+* Add user-tenants.collection.get to all ECS APIs ([MODINVSTOR-1253](https://folio-org.atlassian.net/browse/MODINVSTOR-1253))
+* Add user-tenants.collection.get to POST /\_/tenant API ([MODINVSTOR-1260](https://folio-org.atlassian.net/browse/MODINVSTOR-1260))
+* Update "BC" name in GET instance-date-type to "B.C." ([MODINVSTOR-1255](https://folio-org.atlassian.net/browse/MODINVSTOR-1255))
 
 ### Tech Dept
+* Upgrade localstack from 0.11.3 to s3-latest (=3.8.0) ([MODINVSTOR-1272](https://folio-org.atlassian.net/browse/MODINVSTOR-1272))
+* Add module descriptor validator plugin and fix the permission names ([MODINVSTOR-1247](https://folio-org.atlassian.net/browse/MODINVSTOR-1247))
 * Kafka testcontainers: kafka.KafkaContainer, apache/kafka-native:3.8.0, KafkaTopicsExistsTest fix ([MODINVSTOR-1251](https://folio-org.atlassian.net/browse/MODINVSTOR-1251))
 
-### Dependencies
-* Bump `LIB_NAME` from `OLD_VERSION` to `NEW_VERSION`
-* Bump `domain-models-runtime` from `35.2.0` to `35.2.2`
-* Bump `holdings-storage` from `6.0` to `7.0`
-* Bump `holdings-storage-batch-sync` from `1.1` to `2.0`
-* Bump `holdings-storage-batch-sync-unsafe` from `1.0` to `2.0`
-* Add `LIB_NAME` `2.7.4`
-* Remove `LIB_NAME`
+---
 
 ## v27.1.0 2024-03-19
 ### New APIs versions
@@ -64,7 +155,6 @@
 * Create base for reference data APIs integration tests ([MODINVSTOR-1164](https://issues.folio.org/browse/MODINVSTOR-1164))
 * Make response message more informative for hrid exceptions ([MODINVSTOR-1100](https://issues.folio.org/browse/MODINVSTOR-1100))
 
-
 ### Dependencies
 * Bump `vertx` from `4.3.5` to `4.5.5`
 * Bump `log4j` from `2.17.2` to `2.23.1`

README.MD

@@ -40,7 +40,6 @@ Version 2.0. See the file "[LICENSE](LICENSE)" for more information.
 * [Appendix 1 - Docker Information](#appendix-1---docker-information)
   * [When Using the Modules as Docker Containers](#when-using-the-modules-as-docker-containers)
     * [Finding a Routable Address](#finding-a-routable-address)
-* [Batch interface](#batch-interface)
 * [HRID Management](#hrid-management)
 * [Inventory view endpoint](#inventory-view-endpoint)
 * [Domain event pattern](#domain-event-pattern)
@@ -106,6 +105,7 @@ These properties can be changed by setting env variable.
 
 * `KAFKA_DOMAIN_TOPIC_NUM_PARTITIONS` Default value - `50`
 * `KAFKA_CLASSIFICATION_TYPE_TOPIC_NUM_PARTITIONS` Default value - `1`
+* `KAFKA_CALL_NUMBER_TYPE_TOPIC_NUM_PARTITIONS` Default value - `1`
 * `KAFKA_LOCATION_TOPIC_NUM_PARTITIONS` Default value - `1`
 * `KAFKA_LIBRARY_TOPIC_NUM_PARTITIONS` Default value - `1`
 * `KAFKA_CAMPUS_TOPIC_NUM_PARTITIONS` Default value - `1`
@@ -114,6 +114,20 @@ These properties can be changed by setting env variable.
 * `KAFKA_REINDEX_RECORDS_TOPIC_NUM_PARTITIONS` Default value - `16`
 * `KAFKA_SUBJECT_SOURCE_TOPIC_NUM_PARTITIONS` Default value - `1`
 
+There is also possibility for customizing through properties the Kafka Topic
+`message retention` (in milliseconds) and `maximum message size` (in bytes). The default values of these configurations
+for any topic are 604800000 milliseconds (or 1 week) and 1048576 bytes (or 1 MB) respectively
+
+These are the defined topic properties for `message retention` and `maximum message size` for `reindex-records` topic
+
+* `KAFKA_REINDEX_RECORDS_TOPIC_MESSAGE_RETENTION` Default value - `86400000` (1 day)
+* `KAFKA_REINDEX_RECORDS_TOPIC_MAX_MESSAGE_SIZE` Default value - `10485760` (10 MB)
+
+in case, any of these topic properties are changed for `reindex-records` the topic needs to be recreated and module needs to be reinstalled.
+
+Changing maximum message size for kafka producer:
+* `KAFKA_REINDEX_PRODUCER_MAX_REQUEST_SIZE_BYTES` Default value - `10485760` (10 MB)
+
 # Building
 
 run `mvn install` from the root directory.
@@ -128,20 +142,24 @@ These environment variables configure Kafka, for details see [Kafka](#kafka):
 * `REPLICATION_FACTOR`
 * `KAFKA_DOMAIN_TOPIC_NUM_PARTITIONS`
 * `KAFKA_CLASSIFICATION_TYPE_TOPIC_NUM_PARTITIONS`
+* `KAFKA_CALL_NUMBER_TYPE_TOPIC_NUM_PARTITIONS`
 * `KAFKA_SUBJECT_TYPE_TOPIC_NUM_PARTITIONS`
 * `KAFKA_REINDEX_RECORDS_TOPIC_NUM_PARTITIONS`
 * `KAFKA_SUBJECT_SOURCE_TOPIC_NUM_PARTITIONS`
 
 
 These environment variables configure Kafka topic for specific business-related topics
 * `KAFKA_CLASSIFICATION_TYPE_TOPIC_NUM_PARTITIONS`
+* `KAFKA_CALL_NUMBER_TYPE_TOPIC_NUM_PARTITIONS`
 * `KAFKA_LOCATION_TOPIC_NUM_PARTITIONS`
 * `KAFKA_LIBRARY_TOPIC_NUM_PARTITIONS`
 * `KAFKA_CAMPUS_TOPIC_NUM_PARTITIONS`
 * `KAFKA_INSTITUTION_TOPIC_NUM_PARTITIONS`
 * `KAFKA_SUBJECT_TYPE_TOPIC_NUM_PARTITIONS`
 * `KAFKA_REINDEX_RECORDS_TOPIC_NUM_PARTITIONS`
 * `KAFKA_SUBJECT_SOURCE_TOPIC_NUM_PARTITIONS`
+* `KAFKA_REINDEX_RECORDS_TOPIC_MESSAGE_RETENTION`
+* `KAFKA_REINDEX_RECORDS_TOPIC_MAX_MESSAGE_SIZE`
 
 mod-inventory-storage also supports all Raml Module Builder (RMB) environment variables,
 for details see [RMB](https://github.com/folio-org/raml-module-builder#environment-variables):
@@ -163,6 +181,15 @@ for details see [RMB](https://github.com/folio-org/raml-module-builder#environme
 * `DB_EXPLAIN_QUERY_THRESHOLD`
 * `DB_ALLOW_SUPPRESS_OPTIMISTIC_LOCKING`
 
+These environment variables configure the module interaction with S3-compatible storage (AWS S3, Minio Server):
+* `S3_URL` (default value - `http://127.0.0.1:9000`)
+* `S3_REGION`
+* `S3_BUCKET` (default value - `marc-migrations`)
+* `S3_ACCESS_KEY_ID`
+* `S3_SECRET_ACCESS_KEY`
+* `S3_IS_AWS` (default value - `false`)
+* `ECS_TLR_FEATURE_ENABLED` (default value - `false`)
+
 # Local Deployment using Docker
 
 ## Preparation
@@ -315,27 +342,6 @@ On Linux, `ifconfig docker0 | grep 'inet addr:'` should give output similar to `
 
 On Mac OS X (using Docker Native), `ifconfig en0 | grep 'inet '` should give output similar to `inet 192.168.X.X netmask 0xffffff00 broadcast 192.168.X.X`, the first IP address is usually routable from within containers.
 
-# Batch interface
-
-Batch interface was introduced for processing a collection of entities in bulk. It is not transactional, each entity is processed separately, response contains combined results of processing.
-
-###Design for batch save Instances endpoint
-_Method_: POST
-
-_Resource_: /instance-storage/batch/instances (interface "instance-storage-batch")
-
-_Body_: collection of Instances and total number of Instances
-
-_Returns_: collection of successfully created Instances, error messages for failure Instances, total number of created Instances:
-
-- If at least one Instance is successfully saved - returns 201 response with saved instances ("instances" section), "errorMessages" array contains errors for failed Instances (empty if all Instances are successfully saved).
-
-- If save for all Instances failed - returns 500 response with "errorMessages" array, explaining a reason of why Instances were processed with failures (one error message per one Instance). Instances array is empty.
-
-Regardless the batch size, number of parallel connections to the db is limited to 4 by default. To override the default number of concurrent db connections - "inventory.storage.parallel.db.connections.limit" program argument should be specified on module deployment.
-
-`java -Dport=%p -jar ../mod-source-record-storage/mod-source-record-storage-server/target/mod-source-record-storage-server-fat.jar -Dhttp.port=%p embed_postgres=true inventory.storage.parallel.db.connections.limit=10`
-
 # HRID Management
 
 When instances, holdings records and items are added to inventory, they will be assigned a human