Allow custom S3 endpoint, remove old legacy commented code

kshepherd · kshepherd · commit dcf3d2fb2fa6 · 2025-10-08T16:46:50.000+02:00
diff --git a/dspace-api/src/main/java/org/dspace/storage/bitstore/S3BitStoreService.java b/dspace-api/src/main/java/org/dspace/storage/bitstore/S3BitStoreService.java
@@ -11,6 +11,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.security.DigestInputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
@@ -40,6 +41,10 @@
 import org.dspace.storage.bitstore.service.BitstreamStorageService;
 import org.dspace.util.FunctionalUtils;
 import org.springframework.beans.factory.annotation.Autowired;
+
+import jakarta.annotation.Nullable;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.NotNull;
 import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
 import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
 import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
@@ -95,6 +100,13 @@ public class S3BitStoreService extends BaseBitStoreService {
 
     private String awsAccessKey;
     private String awsSecretKey;
+
+    /**
+     * Custom S3 endpoint URI which will override the Amazon defaults
+     * if present.
+     */
+    private String endpoint = null;
+
     private String awsRegionName;
     private boolean useRelativePath;
     private double targetThroughputGbps = 10.0;
@@ -123,6 +135,7 @@ public class S3BitStoreService extends BaseBitStoreService {
     /**
      * Utility method for generate AmazonS3 builder
      *
+     * @param endpoint optional custom s3 endpoint
      * @param regions wanted regions in client
      * @param awsCredentials credentials of the client
      * @param targetThroughput target throughput in Gbps
@@ -131,8 +144,9 @@ public class S3BitStoreService extends BaseBitStoreService {
      * @return builder with the specified parameters
      */
     protected static Supplier<S3AsyncClient> amazonClientBuilderBy(
-            Region region,
-            AwsCredentialsProvider credentialsProvider,
+            @Nullable String endpoint,
+            @NotNull Region region,
+            @NotNull AwsCredentialsProvider credentialsProvider,
             double targetThroughput,
             long minPartSize,
             Integer maxConcurrency
@@ -152,6 +166,11 @@ protected static Supplier<S3AsyncClient> amazonClientBuilderBy(
                 crtBuilder.maxConcurrency(maxConcurrency);
             }
 
+            if (StringUtils.isNotBlank(endpoint)) {
+                crtBuilder.endpointOverride(URI.create(endpoint));
+                crtBuilder.forcePathStyle(true);
+            }
+
             return crtBuilder.targetThroughputInGbps(targetThroughput).minimumPartSizeInBytes(minPartSize).build();
         };
     }
@@ -181,10 +200,13 @@ public boolean isEnabled() {
      */
     @Override
     public void init() throws IOException {
+
         if (this.isInitialized() || !this.isEnabled()) {
             return;
         }
 
+
+
         try {
             if (StringUtils.isNotBlank(getAwsAccessKey()) && StringUtils.isNotBlank(getAwsSecretKey())) {
                 log.warn("Use local defined S3 credentials");
@@ -202,6 +224,7 @@ public void init() throws IOException {
                 s3AsyncClient = FunctionalUtils.getDefaultOrBuild(
                         this.s3AsyncClient,
                         amazonClientBuilderBy(
+                                endpoint,
                                 region,
                                 StaticCredentialsProvider.create(AwsBasicCredentials.create(getAwsAccessKey(),
                                         getAwsSecretKey())), targetThroughputGbps, minPartSizeBytes, maxConcurrency)
@@ -211,7 +234,7 @@ public void init() throws IOException {
                 log.info("Using a IAM role or aws environment credentials");
                 s3AsyncClient = FunctionalUtils.getDefaultOrBuild(
                         this.s3AsyncClient,
-                        amazonClientBuilderBy(null, null , targetThroughputGbps, minPartSizeBytes, maxConcurrency));
+                        amazonClientBuilderBy(endpoint, null, null , targetThroughputGbps, minPartSizeBytes, maxConcurrency));
             }
 
             // bucket name
@@ -301,6 +324,9 @@ public InputStream get(Bitstream bitstream) throws IOException {
      */
     @Override
     public void put(Bitstream bitstream, InputStream in) throws IOException {
+        log.error(getEndpoint());
+        log.error(getAwsAccessKey());
+        log.error(getAwsSecretKey());
         String key = getFullKey(bitstream.getInternalId());
 
         try (DigestInputStream dis = new DigestInputStream(in, MessageDigest.getInstance(CSA))) {
@@ -478,6 +504,15 @@ public void setAwsSecretKey(String awsSecretKey) {
         this.awsSecretKey = awsSecretKey;
     }
 
+    public String getEndpoint() {
+        return endpoint;
+    }
+
+    @Autowired
+    public void setEndpoint(String endpoint) {
+        this.endpoint = endpoint;
+    }
+
     public String getAwsRegionName() {
         return awsRegionName;
     }
@@ -596,58 +631,6 @@ public static void main(String[] args) throws Exception {
         store.bucketName = DEFAULT_BUCKET_PREFIX + hostname + ".s3test";
         store.s3AsyncClient.createBucket(r -> r.bucket(store.bucketName)).join();
 
-        /* Broken in DSpace 6 TODO Refactor
-        // time everything, todo, switch to caliper
-        long start = Instant.now().toEpochMilli();
-        // Case 1: store a file
-        String id = store.generateId();
-        System.out.print("put() file " + assetFile + " under ID " + id + ": ");
-        FileInputStream fis = new FileInputStream(assetFile);
-        //TODO create bitstream for assetfile...
-        Map attrs = store.put(fis, id);
-        long now =  Instant.now().toEpochMilli();
-        System.out.println((now - start) + " msecs");
-        start = now;
-        // examine the metadata returned
-        Iterator iter = attrs.keySet().iterator();
-        System.out.println("Metadata after put():");
-        while (iter.hasNext())
-        {
-            String key = (String)iter.next();
-            System.out.println( key + ": " + (String)attrs.get(key) );
-        }
-        // Case 2: get metadata and compare
-        System.out.print("about() file with ID " + id + ": ");
-        Map attrs2 = store.about(id, attrs);
-        now =  Instant.now().toEpochMilli();
-        System.out.println((now - start) + " msecs");
-        start = now;
-        iter = attrs2.keySet().iterator();
-        System.out.println("Metadata after about():");
-        while (iter.hasNext())
-        {
-            String key = (String)iter.next();
-            System.out.println( key + ": " + (String)attrs.get(key) );
-        }
-        // Case 3: retrieve asset and compare bits
-        System.out.print("get() file with ID " + id + ": ");
-        java.io.FileOutputStream fos = new java.io.FileOutputStream(assetFile+".echo");
-        InputStream in = store.get(id);
-        Utils.bufferedCopy(in, fos);
-        fos.close();
-        in.close();
-        now =  Instant.now().toEpochMilli();
-        System.out.println((now - start) + " msecs");
-        start = now;
-        // Case 4: remove asset
-        System.out.print("remove() file with ID: " + id + ": ");
-        store.remove(id);
-        now =  Instant.now().toEpochMilli();
-        System.out.println((now - start) + " msecs");
-        System.out.flush();
-        // should get nothing back now - will throw exception
-        store.get(id);
-*/
     }
 
     /**
diff --git a/dspace/config/modules/assetstore.cfg b/dspace/config/modules/assetstore.cfg
@@ -55,6 +55,9 @@ assetstore.s3.subfolder =
 assetstore.s3.awsAccessKey =
 assetstore.s3.awsSecretKey =
 
+# Optional custom S3 endpoint URI. Leave this blank / commented to use the Amazon default
+#assetstore.s3.endpoint =
+
 # If the credentials are left empty,
 # then this setting is ignored and the default AWS region will be used.
 assetstore.s3.awsRegionName =
@@ -118,4 +121,4 @@ assetstore.s3.s3ChecksumAlgorithm = CRC32
 
 # The maximum counter value for S3 operations.
 # Default: -1 (unlimited) connection is never manualy closed in BitstoreService
-# assetstore.jcloud.maxCounter = -1
+# assetstore.jcloud.maxCounter = -1
diff --git a/dspace/config/spring/api/bitstore.xml b/dspace/config/spring/api/bitstore.xml
@@ -28,6 +28,9 @@
         <!-- S3 bucket name to store assets in. example: longsight-dspace-auk -->
         <property name="bucketName" value="${assetstore.s3.bucketName}"/>
 
+        <!-- S3 endpoint URI if not using Amazon defaults, example: minio.mysite.org -->
+        <property name="endpoint" value="${assetstore.s3.endpoint:}"/>
+
         <!-- AWS S3 Region to use: {us-east-1, us-west-1, eu-west-1, eu-central-1, ap-southeast-1, ... } -->
         <!-- Optional, sdk default is us-east-1 -->
         <property name="awsRegionName" value="${assetstore.s3.awsRegionName}"/>
