Create Readme.md for s3-compatible

Better descriptions typo & comments
Refacoring with skipCredentialSubscopingIndirection -> finaly removed
Rebase with AWS updates from main branch adding roleArn, camelCase refactoring, typo, cleaning
Add default AWS credentials provider for STS
Error Co-authored-by: Gerrit-K <[email protected]>
Rebase from quarkus and keep only sts with some suggestions from code review
helm unit test

helm/polaris/tests/configmap_test.yaml

@@ -183,159 +183,141 @@ tests:
     set:
       logging: { file: { enabled: true, json: true }, console: { enabled: true, json: true } }
     asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.file.enable=true" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.console.enable=true" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.file.json=true" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.console.json=true" }
-
-  - it: should include logging categories
-    set:
-      logging:
-        categories:
-          # compact style
-          org.acme: DEBUG
-          # expanded style
-          org:
-            acme:
-              service: INFO
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.category.\"org.acme\".level=DEBUG" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.category.\"org.acme.service\".level=INFO" }
-
-  - it: should include MDC context
-    set:
-      logging:
-        mdc:
-          # compact style
-          org.acme: foo
-          # expanded style
-          org:
-            acme:
-              service: foo
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.log.mdc.\"org.acme\"=foo" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.log.mdc.\"org.acme.service\"=foo" }
-
-  - it: should include telemetry configuration
-    set:
-      tracing: { enabled: true, endpoint: http://custom:4317, attributes: { service.name: custom, foo: bar } }
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.exporter.otlp.endpoint=http://custom:4317" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.resource.attributes\\[\\d\\]=service.name=custom" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.resource.attributes\\[\\d\\]=foo=bar" }
-
-  - it: should include set sample rate numeric
-    set:
-      tracing: { enabled: true, sample: "0.123" }
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.traces.sampler=parentbased_traceidratio" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.traces.sampler.arg=0.123" }
-
-  - it: should include set sample rate "all"
-    set:
-      tracing: { enabled: true, sample: "all" }
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.traces.sampler=parentbased_always_on" }
-
-  - it: should include set sample rate "none"
-    set:
-      tracing: { enabled: true, sample: "none" }
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.traces.sampler=always_off" }
-
-  - it: should disable tracing by default
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.sdk.disabled=true" }
-
-  - it: should disable tracing
-    set:
-      tracing: { enabled: false }
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.sdk.disabled=true" }
-
-  - it: should include custom metrics
-    set:
-      metrics: { enabled: true, tags: { app: custom, foo: bar } }
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.metrics.tags.app=custom" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.metrics.tags.foo=bar" }
-
-  - it: should disable metrics
-    set:
-      metrics: { enabled: false }
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.micrometer.enabled=false" }
-
-  - it: should include advanced configuration
-    set:
-        advancedConfig:
-          # compact style
-          quarkus.compact.custom: true
-          # expanded style
-          quarkus:
-            expanded:
-              custom: foo
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.compact.custom=true" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.expanded.custom=foo" }
-
-  - it: should not include CORS configuration by default
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors" }
-        not: true
-
-  - it: should include CORS configuration if defined
-    set:
-      cors: { allowedOrigins: [ "http://localhost:3000", "https://localhost:4000" ], allowedMethods: [ "GET", "POST" ], allowedHeaders: [ "X-Custom1", "X-Custom2" ], exposedHeaders: [ "X-Exposed-Custom1", "X-Exposed-Custom2" ], accessControlMaxAge: "PT1H", accessControlAllowCredentials: false }
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.origins=http://localhost:3000,https://localhost:4000" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.methods=GET,POST" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.headers=X-Custom1,X-Custom2" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.exposed-headers=X-Exposed-Custom1,X-Exposed-Custom2" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.access-control-max-age=PT1H" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.access-control-allow-credentials=false" }
-
-  - it: should configure rate-limiter with default values
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.filter.type=no-op" }
-
-  - it: should configure rate-limiter no-op
-    set:
-      rateLimiter.type: no-op
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.filter.type=no-op" }
-
-  - it: should configure rate-limiter with default token bucket values
-    set:
-      rateLimiter.type: default
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.filter.type=default" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.type=default" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.requests-per-second=9999" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.window=PT10S" }
-
-  - it: should configure rate-limiter with custom token bucket values
-    set:
-      rateLimiter:
-        type: custom
-        tokenBucket:
-          type: custom
-          requestsPerSecond: 1234
-          window: PT5S
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.filter.type=custom" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.type=custom" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.requests-per-second=1234" }
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.window=PT5S" }
-
-  - it: should not include tasks configuration by default
-    asserts:
-      - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.tasks" }
-        not: true
-
-  - it: should include tasks configuration if defined
+      - equal:
+          path: data
+          value:
+            polaris-server.yml: |-
+              authenticator:
+                class: org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator
+              callContextResolver:
+                type: default
+              cors:
+                allowed-credentials: true
+                allowed-headers:
+                - '*'
+                allowed-methods:
+                - PATCH
+                - POST
+                - DELETE
+                - GET
+                - PUT
+                allowed-origins:
+                - http://localhost:8080
+                allowed-timing-origins:
+                - http://localhost:8080
+                exposed-headers:
+                - '*'
+                preflight-max-age: 600
+              defaultRealms:
+              - default-realm
+              featureConfiguration:
+                ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: false
+                SUPPORTED_CATALOG_STORAGE_TYPES:
+                - S3
+                - S3_COMPATIBLE
+                - GCS
+                - AZURE
+                - FILE
+              io:
+                factoryType: default
+              logging:
+                appenders:
+                - logFormat: '%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}]
+                    %c{30}: %m %kvp%n%ex'
+                  threshold: ALL
+                  type: console
+                level: INFO
+                loggers:
+                  org.apache.iceberg.rest: DEBUG
+                  org.apache.polaris: DEBUG
+              maxRequestBodyBytes: -1
+              metaStoreManager:
+                type: in-memory
+              oauth2:
+                type: test
+              rateLimiter:
+                type: no-op
+              realmContextResolver:
+                type: default
+              server:
+                adminConnectors:
+                - port: 8182
+                  type: http
+                applicationConnectors:
+                - port: 8181
+                  type: http
+                maxThreads: 200
+                minThreads: 10
+                requestLog:
+                  appenders:
+                  - type: console
+  - it: should set config map data (auto sorted)
     set:
-      tasks: { maxConcurrentTasks: 10, maxQueuedTasks: 20 }
+      polarisServerConfig:
+        server:
+          maxThreads: 200
+          minThreads: 10
+          applicationConnectors:
+          - type: http
+            port: 8181
+          adminConnectors:
+          - type: http
+            port: 8182
+          requestLog:
+            appenders:
+            - type: console
+        featureConfiguration:
+          ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: false
+          SUPPORTED_CATALOG_STORAGE_TYPES:
+          - S3
+        callContextResolver:
+          type: default
+        realmContextResolver:
+          type: default
+        defaultRealms:
+        - default-realm
+        metaStoreManager:
+          type: eclipse-link
+          persistence-unit: polaris
+          conf-file: /eclipselink-config/conf.jar!/persistence.xml
+        io:
+          factoryType: default
+        oauth2:
+          type: default
+        tokenBroker:
+          type: symmetric-key
+          secret: polaris
+        authenticator:
+          class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator
+        cors:
+          allowed-origins:
+          - http://localhost:8080
+          allowed-timing-origins:
+          - http://localhost:8080
+          allowed-methods:
+          - PATCH
+          - POST
+          - DELETE
+          - GET
+          - PUT
+          allowed-headers:
+          - "*"
+          exposed-headers:
+          - "*"
+          preflight-max-age: 600
+          allowed-credentials: true
+        logging:
+          level: INFO
+          loggers:
+            org.apache.iceberg.rest: INFO
+            org.apache.polaris: INFO
+          appenders:
+          - type: console
+            threshold: ALL
+            logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c{30}: %m %kvp%n%ex"
+        maxRequestBodyBytes: -1
+        rateLimiter:
+          type: no-op
     asserts:
       - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.tasks.max-concurrent-tasks=10" }
       - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.tasks.max-queued-tasks=20" }

polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java

@@ -38,14 +38,14 @@
 import org.apache.polaris.core.admin.model.FileStorageConfigInfo;
 import org.apache.polaris.core.admin.model.GcpStorageConfigInfo;
 import org.apache.polaris.core.admin.model.PolarisCatalog;
-import org.apache.polaris.core.admin.model.S3StorageConfigInfo;
+import org.apache.polaris.core.admin.model.S3CompatibleStorageConfigInfo;
 import org.apache.polaris.core.admin.model.StorageConfigInfo;
 import org.apache.polaris.core.storage.FileStorageConfigurationInfo;
 import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo;
 import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo;
 import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo;
 import org.apache.polaris.core.storage.gcp.GcpStorageConfigurationInfo;
-import org.apache.polaris.core.storage.s3.S3StorageConfigurationInfo;
+import org.apache.polaris.core.storage.s3compatible.S3CompatibleStorageConfigurationInfo;
 
 /**
  * Catalog specific subclass of the {@link PolarisEntity} that handles conversion from the {@link
@@ -143,30 +143,19 @@ private StorageConfigInfo getStorageInfo(Map<String, String> internalProperties)
             .setRegion(awsConfig.getRegion())
             .build();
       }
-      if (configInfo instanceof S3StorageConfigurationInfo) {
-        S3StorageConfigurationInfo s3Config = (S3StorageConfigurationInfo) configInfo;
-        return S3StorageConfigInfo.builder()
+      if (configInfo instanceof S3CompatibleStorageConfigurationInfo) {
+        S3CompatibleStorageConfigurationInfo s3Config =
+            (S3CompatibleStorageConfigurationInfo) configInfo;
+        return S3CompatibleStorageConfigInfo.builder()
             .setStorageType(StorageConfigInfo.StorageTypeEnum.S3_COMPATIBLE)
             .setS3Endpoint(s3Config.getS3Endpoint())
             .setS3PathStyleAccess(s3Config.getS3PathStyleAccess())
-            .setCredsVendingStrategy(
-                org.apache.polaris.core.admin.model.S3StorageConfigInfo.CredsVendingStrategyEnum
-                    .valueOf(
-                        org.apache.polaris.core.admin.model.S3StorageConfigInfo
-                            .CredsVendingStrategyEnum.class,
-                        s3Config.getCredsVendingStrategy().name()))
-            .setCredsCatalogAndClientStrategy(
-                org.apache.polaris.core.admin.model.S3StorageConfigInfo
-                    .CredsCatalogAndClientStrategyEnum.valueOf(
-                    org.apache.polaris.core.admin.model.S3StorageConfigInfo
-                        .CredsCatalogAndClientStrategyEnum.class,
-                    s3Config.getCredsCatalogAndClientStrategy().name()))
             .setAllowedLocations(s3Config.getAllowedLocations())
-            .setS3CredentialsCatalogAccessKeyId(s3Config.getS3CredentialsCatalogAccessKeyId())
-            .setS3CredentialsCatalogSecretAccessKey(
+            .setS3CredentialsCatalogAccessKeyEnvVar(s3Config.getS3CredentialsCatalogAccessKeyId())
+            .setS3CredentialsCatalogSecretAccessKeyEnvVar(
                 s3Config.getS3CredentialsCatalogSecretAccessKey())
-            .setS3CredentialsClientAccessKeyId(s3Config.getS3CredentialsClientSecretAccessKey())
-            .setS3CredentialsClientSecretAccessKey(s3Config.getS3CredentialsClientAccessKeyId())
+            .setS3Region(s3Config.getS3Region())
+            .setS3RoleArn(s3Config.getS3RoleArn())
             .build();
       }
       if (configInfo instanceof AzureStorageConfigurationInfo) {
@@ -280,24 +269,17 @@ public Builder setStorageConfigurationInfo(
             break;
 
           case S3_COMPATIBLE:
-            S3StorageConfigInfo s3ConfigModel = (S3StorageConfigInfo) storageConfigModel;
+            S3CompatibleStorageConfigInfo s3ConfigModel =
+                (S3CompatibleStorageConfigInfo) storageConfigModel;
             config =
-                new S3StorageConfigurationInfo(
+                new S3CompatibleStorageConfigurationInfo(
                     PolarisStorageConfigurationInfo.StorageType.S3_COMPATIBLE,
-                    S3StorageConfigInfo.CredsVendingStrategyEnum.valueOf(
-                        org.apache.polaris.core.storage.s3.S3StorageConfigurationInfo
-                            .CredsVendingStrategyEnum.class,
-                        s3ConfigModel.getCredsVendingStrategy().name()),
-                    S3StorageConfigInfo.CredsCatalogAndClientStrategyEnum.valueOf(
-                        org.apache.polaris.core.storage.s3.S3StorageConfigurationInfo
-                            .CredsCatalogAndClientStrategyEnum.class,
-                        s3ConfigModel.getCredsCatalogAndClientStrategy().name()),
                     s3ConfigModel.getS3Endpoint(),
-                    s3ConfigModel.getS3CredentialsCatalogAccessKeyId(),
-                    s3ConfigModel.getS3CredentialsCatalogSecretAccessKey(),
-                    s3ConfigModel.getS3CredentialsClientAccessKeyId(),
-                    s3ConfigModel.getS3CredentialsClientSecretAccessKey(),
+                    s3ConfigModel.getS3CredentialsCatalogAccessKeyEnvVar(),
+                    s3ConfigModel.getS3CredentialsCatalogSecretAccessKeyEnvVar(),
                     s3ConfigModel.getS3PathStyleAccess(),
+                    s3ConfigModel.getS3Region(),
+                    s3ConfigModel.getS3RoleArn(),
                     new ArrayList<>(allowedLocations));
             break;
           case AZURE:

polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisCredentialProperty.java

@@ -24,7 +24,8 @@ public enum PolarisCredentialProperty {
   AWS_SECRET_KEY(String.class, "s3.secret-access-key", "the aws access key secret"),
   AWS_TOKEN(String.class, "s3.session-token", "the aws scoped access token"),
   AWS_ENDPOINT(String.class, "s3.endpoint", "the aws s3 endpoint"),
-  AWS_PATH_STYLE_ACCESS(Boolean.class, "s3.path-style-access", "the aws s3 path style access"),
+  AWS_PATH_STYLE_ACCESS(
+      Boolean.class, "s3.path-style-access", "whether or not to use path-style access"),
   CLIENT_REGION(
       String.class, "client.region", "region to configure client for making requests to AWS"),