diff --git a/helm/polaris/tests/configmap_test.yaml b/helm/polaris/tests/configmap_test.yaml index ef725ec4f..e070bf0dc 100644 --- a/helm/polaris/tests/configmap_test.yaml +++ b/helm/polaris/tests/configmap_test.yaml @@ -183,159 +183,141 @@ tests: set: logging: { file: { enabled: true, json: true }, console: { enabled: true, json: true } } asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.file.enable=true" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.console.enable=true" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.file.json=true" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.console.json=true" } - - - it: should include logging categories - set: - logging: - categories: - # compact style - org.acme: DEBUG - # expanded style - org: - acme: - service: INFO - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.category.\"org.acme\".level=DEBUG" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.log.category.\"org.acme.service\".level=INFO" } - - - it: should include MDC context - set: - logging: - mdc: - # compact style - org.acme: foo - # expanded style - org: - acme: - service: foo - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.log.mdc.\"org.acme\"=foo" } - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.log.mdc.\"org.acme.service\"=foo" } - - - it: should include telemetry configuration - set: - tracing: { enabled: true, endpoint: http://custom:4317, attributes: { service.name: custom, foo: bar } } - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.exporter.otlp.endpoint=http://custom:4317" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.resource.attributes\\[\\d\\]=service.name=custom" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.resource.attributes\\[\\d\\]=foo=bar" } - - - it: should include set sample rate numeric - set: - tracing: { enabled: true, sample: "0.123" } - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.traces.sampler=parentbased_traceidratio" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.traces.sampler.arg=0.123" } - - - it: should include set sample rate "all" - set: - tracing: { enabled: true, sample: "all" } - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.traces.sampler=parentbased_always_on" } - - - it: should include set sample rate "none" - set: - tracing: { enabled: true, sample: "none" } - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.traces.sampler=always_off" } - - - it: should disable tracing by default - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.sdk.disabled=true" } - - - it: should disable tracing - set: - tracing: { enabled: false } - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.otel.sdk.disabled=true" } - - - it: should include custom metrics - set: - metrics: { enabled: true, tags: { app: custom, foo: bar } } - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.metrics.tags.app=custom" } - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.metrics.tags.foo=bar" } - - - it: should disable metrics - set: - metrics: { enabled: false } - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.micrometer.enabled=false" } - - - it: should include advanced configuration - set: - advancedConfig: - # compact style - quarkus.compact.custom: true - # expanded style - quarkus: - expanded: - custom: foo - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.compact.custom=true" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.expanded.custom=foo" } - - - it: should not include CORS configuration by default - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors" } - not: true - - - it: should include CORS configuration if defined - set: - cors: { allowedOrigins: [ "http://localhost:3000", "https://localhost:4000" ], allowedMethods: [ "GET", "POST" ], allowedHeaders: [ "X-Custom1", "X-Custom2" ], exposedHeaders: [ "X-Exposed-Custom1", "X-Exposed-Custom2" ], accessControlMaxAge: "PT1H", accessControlAllowCredentials: false } - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.origins=http://localhost:3000,https://localhost:4000" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.methods=GET,POST" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.headers=X-Custom1,X-Custom2" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.exposed-headers=X-Exposed-Custom1,X-Exposed-Custom2" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.access-control-max-age=PT1H" } - - matchRegex: { path: 'data["application.properties"]', pattern: "quarkus.http.cors.access-control-allow-credentials=false" } - - - it: should configure rate-limiter with default values - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.filter.type=no-op" } - - - it: should configure rate-limiter no-op - set: - rateLimiter.type: no-op - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.filter.type=no-op" } - - - it: should configure rate-limiter with default token bucket values - set: - rateLimiter.type: default - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.filter.type=default" } - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.type=default" } - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.requests-per-second=9999" } - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.window=PT10S" } - - - it: should configure rate-limiter with custom token bucket values - set: - rateLimiter: - type: custom - tokenBucket: - type: custom - requestsPerSecond: 1234 - window: PT5S - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.filter.type=custom" } - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.type=custom" } - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.requests-per-second=1234" } - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.rate-limiter.token-bucket.window=PT5S" } - - - it: should not include tasks configuration by default - asserts: - - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.tasks" } - not: true - - - it: should include tasks configuration if defined + - equal: + path: data + value: + polaris-server.yml: |- + authenticator: + class: org.apache.polaris.service.auth.TestInlineBearerTokenPolarisAuthenticator + callContextResolver: + type: default + cors: + allowed-credentials: true + allowed-headers: + - '*' + allowed-methods: + - PATCH + - POST + - DELETE + - GET + - PUT + allowed-origins: + - http://localhost:8080 + allowed-timing-origins: + - http://localhost:8080 + exposed-headers: + - '*' + preflight-max-age: 600 + defaultRealms: + - default-realm + featureConfiguration: + ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: false + SUPPORTED_CATALOG_STORAGE_TYPES: + - S3 + - S3_COMPATIBLE + - GCS + - AZURE + - FILE + io: + factoryType: default + logging: + appenders: + - logFormat: '%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] + %c{30}: %m %kvp%n%ex' + threshold: ALL + type: console + level: INFO + loggers: + org.apache.iceberg.rest: DEBUG + org.apache.polaris: DEBUG + maxRequestBodyBytes: -1 + metaStoreManager: + type: in-memory + oauth2: + type: test + rateLimiter: + type: no-op + realmContextResolver: + type: default + server: + adminConnectors: + - port: 8182 + type: http + applicationConnectors: + - port: 8181 + type: http + maxThreads: 200 + minThreads: 10 + requestLog: + appenders: + - type: console + - it: should set config map data (auto sorted) set: - tasks: { maxConcurrentTasks: 10, maxQueuedTasks: 20 } + polarisServerConfig: + server: + maxThreads: 200 + minThreads: 10 + applicationConnectors: + - type: http + port: 8181 + adminConnectors: + - type: http + port: 8182 + requestLog: + appenders: + - type: console + featureConfiguration: + ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING: false + SUPPORTED_CATALOG_STORAGE_TYPES: + - S3 + callContextResolver: + type: default + realmContextResolver: + type: default + defaultRealms: + - default-realm + metaStoreManager: + type: eclipse-link + persistence-unit: polaris + conf-file: /eclipselink-config/conf.jar!/persistence.xml + io: + factoryType: default + oauth2: + type: default + tokenBroker: + type: symmetric-key + secret: polaris + authenticator: + class: org.apache.polaris.service.auth.DefaultPolarisAuthenticator + cors: + allowed-origins: + - http://localhost:8080 + allowed-timing-origins: + - http://localhost:8080 + allowed-methods: + - PATCH + - POST + - DELETE + - GET + - PUT + allowed-headers: + - "*" + exposed-headers: + - "*" + preflight-max-age: 600 + allowed-credentials: true + logging: + level: INFO + loggers: + org.apache.iceberg.rest: INFO + org.apache.polaris: INFO + appenders: + - type: console + threshold: ALL + logFormat: "%-5p [%d{ISO8601} - %-6r] [%t] [%X{aid}%X{sid}%X{tid}%X{wid}%X{oid}%X{srv}%X{job}%X{rid}] %c{30}: %m %kvp%n%ex" + maxRequestBodyBytes: -1 + rateLimiter: + type: no-op asserts: - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.tasks.max-concurrent-tasks=10" } - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.tasks.max-queued-tasks=20" } diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index f8a37dd6f..ab70b9b49 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -38,14 +38,14 @@ import org.apache.polaris.core.admin.model.FileStorageConfigInfo; import org.apache.polaris.core.admin.model.GcpStorageConfigInfo; import org.apache.polaris.core.admin.model.PolarisCatalog; -import org.apache.polaris.core.admin.model.S3StorageConfigInfo; +import org.apache.polaris.core.admin.model.S3CompatibleStorageConfigInfo; import org.apache.polaris.core.admin.model.StorageConfigInfo; import org.apache.polaris.core.storage.FileStorageConfigurationInfo; import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; import org.apache.polaris.core.storage.gcp.GcpStorageConfigurationInfo; -import org.apache.polaris.core.storage.s3.S3StorageConfigurationInfo; +import org.apache.polaris.core.storage.s3compatible.S3CompatibleStorageConfigurationInfo; /** * Catalog specific subclass of the {@link PolarisEntity} that handles conversion from the {@link @@ -143,30 +143,19 @@ private StorageConfigInfo getStorageInfo(Map internalProperties) .setRegion(awsConfig.getRegion()) .build(); } - if (configInfo instanceof S3StorageConfigurationInfo) { - S3StorageConfigurationInfo s3Config = (S3StorageConfigurationInfo) configInfo; - return S3StorageConfigInfo.builder() + if (configInfo instanceof S3CompatibleStorageConfigurationInfo) { + S3CompatibleStorageConfigurationInfo s3Config = + (S3CompatibleStorageConfigurationInfo) configInfo; + return S3CompatibleStorageConfigInfo.builder() .setStorageType(StorageConfigInfo.StorageTypeEnum.S3_COMPATIBLE) .setS3Endpoint(s3Config.getS3Endpoint()) .setS3PathStyleAccess(s3Config.getS3PathStyleAccess()) - .setCredsVendingStrategy( - org.apache.polaris.core.admin.model.S3StorageConfigInfo.CredsVendingStrategyEnum - .valueOf( - org.apache.polaris.core.admin.model.S3StorageConfigInfo - .CredsVendingStrategyEnum.class, - s3Config.getCredsVendingStrategy().name())) - .setCredsCatalogAndClientStrategy( - org.apache.polaris.core.admin.model.S3StorageConfigInfo - .CredsCatalogAndClientStrategyEnum.valueOf( - org.apache.polaris.core.admin.model.S3StorageConfigInfo - .CredsCatalogAndClientStrategyEnum.class, - s3Config.getCredsCatalogAndClientStrategy().name())) .setAllowedLocations(s3Config.getAllowedLocations()) - .setS3CredentialsCatalogAccessKeyId(s3Config.getS3CredentialsCatalogAccessKeyId()) - .setS3CredentialsCatalogSecretAccessKey( + .setS3CredentialsCatalogAccessKeyEnvVar(s3Config.getS3CredentialsCatalogAccessKeyId()) + .setS3CredentialsCatalogSecretAccessKeyEnvVar( s3Config.getS3CredentialsCatalogSecretAccessKey()) - .setS3CredentialsClientAccessKeyId(s3Config.getS3CredentialsClientSecretAccessKey()) - .setS3CredentialsClientSecretAccessKey(s3Config.getS3CredentialsClientAccessKeyId()) + .setS3Region(s3Config.getS3Region()) + .setS3RoleArn(s3Config.getS3RoleArn()) .build(); } if (configInfo instanceof AzureStorageConfigurationInfo) { @@ -280,24 +269,17 @@ public Builder setStorageConfigurationInfo( break; case S3_COMPATIBLE: - S3StorageConfigInfo s3ConfigModel = (S3StorageConfigInfo) storageConfigModel; + S3CompatibleStorageConfigInfo s3ConfigModel = + (S3CompatibleStorageConfigInfo) storageConfigModel; config = - new S3StorageConfigurationInfo( + new S3CompatibleStorageConfigurationInfo( PolarisStorageConfigurationInfo.StorageType.S3_COMPATIBLE, - S3StorageConfigInfo.CredsVendingStrategyEnum.valueOf( - org.apache.polaris.core.storage.s3.S3StorageConfigurationInfo - .CredsVendingStrategyEnum.class, - s3ConfigModel.getCredsVendingStrategy().name()), - S3StorageConfigInfo.CredsCatalogAndClientStrategyEnum.valueOf( - org.apache.polaris.core.storage.s3.S3StorageConfigurationInfo - .CredsCatalogAndClientStrategyEnum.class, - s3ConfigModel.getCredsCatalogAndClientStrategy().name()), s3ConfigModel.getS3Endpoint(), - s3ConfigModel.getS3CredentialsCatalogAccessKeyId(), - s3ConfigModel.getS3CredentialsCatalogSecretAccessKey(), - s3ConfigModel.getS3CredentialsClientAccessKeyId(), - s3ConfigModel.getS3CredentialsClientSecretAccessKey(), + s3ConfigModel.getS3CredentialsCatalogAccessKeyEnvVar(), + s3ConfigModel.getS3CredentialsCatalogSecretAccessKeyEnvVar(), s3ConfigModel.getS3PathStyleAccess(), + s3ConfigModel.getS3Region(), + s3ConfigModel.getS3RoleArn(), new ArrayList<>(allowedLocations)); break; case AZURE: diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisCredentialProperty.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisCredentialProperty.java index 13838e6af..b7f1a9808 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisCredentialProperty.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisCredentialProperty.java @@ -24,7 +24,8 @@ public enum PolarisCredentialProperty { AWS_SECRET_KEY(String.class, "s3.secret-access-key", "the aws access key secret"), AWS_TOKEN(String.class, "s3.session-token", "the aws scoped access token"), AWS_ENDPOINT(String.class, "s3.endpoint", "the aws s3 endpoint"), - AWS_PATH_STYLE_ACCESS(Boolean.class, "s3.path-style-access", "the aws s3 path style access"), + AWS_PATH_STYLE_ACCESS( + Boolean.class, "s3.path-style-access", "whether or not to use path-style access"), CLIENT_REGION( String.class, "client.region", "region to configure client for making requests to AWS"), diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java index 4d4bdc871..99c08c299 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/PolarisStorageConfigurationInfo.java @@ -48,7 +48,7 @@ import org.apache.polaris.core.storage.aws.AwsStorageConfigurationInfo; import org.apache.polaris.core.storage.azure.AzureStorageConfigurationInfo; import org.apache.polaris.core.storage.gcp.GcpStorageConfigurationInfo; -import org.apache.polaris.core.storage.s3.S3StorageConfigurationInfo; +import org.apache.polaris.core.storage.s3compatible.S3CompatibleStorageConfigurationInfo; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,7 +64,7 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.NAME) @JsonSubTypes({ @JsonSubTypes.Type(value = AwsStorageConfigurationInfo.class), - @JsonSubTypes.Type(value = S3StorageConfigurationInfo.class), + @JsonSubTypes.Type(value = S3CompatibleStorageConfigurationInfo.class), @JsonSubTypes.Type(value = AzureStorageConfigurationInfo.class), @JsonSubTypes.Type(value = GcpStorageConfigurationInfo.class), @JsonSubTypes.Type(value = FileStorageConfigurationInfo.class), diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3/S3CredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3/S3CredentialsStorageIntegration.java deleted file mode 100644 index 5fdbbdf37..000000000 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3/S3CredentialsStorageIntegration.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.core.storage.s3; - -import java.net.URI; -import java.util.EnumMap; -import java.util.Set; -import org.apache.polaris.core.PolarisDiagnostics; -import org.apache.polaris.core.storage.InMemoryStorageIntegration; -import org.apache.polaris.core.storage.PolarisCredentialProperty; -import org.jetbrains.annotations.NotNull; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.sts.StsClient; -import software.amazon.awssdk.services.sts.StsClientBuilder; -import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; -import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; - -/** Credential vendor that supports generating */ -public class S3CredentialsStorageIntegration - extends InMemoryStorageIntegration { - - private static final Logger LOGGER = - LoggerFactory.getLogger(S3CredentialsStorageIntegration.class); - - private StsClient stsClient; - - // Constructor - public S3CredentialsStorageIntegration() { - super(S3CredentialsStorageIntegration.class.getName()); - } - - public void createStsClient(S3StorageConfigurationInfo s3storageConfig) { - - LOGGER.debug("S3Compatible - createStsClient()"); - - LOGGER.info( - "S3Compatible - AWS STS endpoint is unique and different from the S3 Endpoint. AWS SDK need to be overided with dedicated Endpoint from S3Compatible, otherwise the AWS STS url is targeted"); - - StsClientBuilder stsBuilder = software.amazon.awssdk.services.sts.StsClient.builder(); - - stsBuilder.region( - Region - .US_WEST_1); // default region to avoid bug, because most (all?) S3 compatible softwares - // do not care about regions - stsBuilder.endpointOverride(URI.create(s3storageConfig.getS3Endpoint())); - stsBuilder.credentialsProvider( - StaticCredentialsProvider.create( - AwsBasicCredentials.create( - s3storageConfig.getS3CredentialsCatalogAccessKeyId(), - s3storageConfig.getS3CredentialsCatalogSecretAccessKey()))); - - this.stsClient = stsBuilder.build(); - LOGGER.debug("S3Compatible - stsClient successfully built"); - } - - /** {@inheritDoc} */ - @Override - public EnumMap getSubscopedCreds( - @NotNull PolarisDiagnostics diagnostics, - @NotNull S3StorageConfigurationInfo storageConfig, - boolean allowListOperation, - @NotNull Set allowedReadLocations, - @NotNull Set allowedWriteLocations) { - - LOGGER.debug("S3Compatible - getSubscopedCreds - applying credential strategy"); - - EnumMap propertiesMap = - new EnumMap<>(PolarisCredentialProperty.class); - propertiesMap.put(PolarisCredentialProperty.AWS_ENDPOINT, storageConfig.getS3Endpoint()); - propertiesMap.put( - PolarisCredentialProperty.AWS_PATH_STYLE_ACCESS, - storageConfig.getS3PathStyleAccess().toString()); - - switch (storageConfig.getCredsVendingStrategy()) { - case KEYS_SAME_AS_CATALOG: - propertiesMap.put( - PolarisCredentialProperty.AWS_KEY_ID, - storageConfig.getS3CredentialsCatalogAccessKeyId()); - propertiesMap.put( - PolarisCredentialProperty.AWS_SECRET_KEY, - storageConfig.getS3CredentialsCatalogSecretAccessKey()); - break; - - case KEYS_DEDICATED_TO_CLIENT: - propertiesMap.put( - PolarisCredentialProperty.AWS_KEY_ID, - storageConfig.getS3CredentialsClientAccessKeyId()); - propertiesMap.put( - PolarisCredentialProperty.AWS_SECRET_KEY, - storageConfig.getS3CredentialsClientSecretAccessKey()); - break; - - case TOKEN_WITH_ASSUME_ROLE: - if (this.stsClient == null) { - createStsClient(storageConfig); - } - LOGGER.debug("S3Compatible - assumeRole !"); - AssumeRoleResponse response = - stsClient.assumeRole( - AssumeRoleRequest.builder().roleSessionName("PolarisCredentialsSTS").build()); - - propertiesMap.put( - PolarisCredentialProperty.AWS_KEY_ID, response.credentials().accessKeyId()); - propertiesMap.put( - PolarisCredentialProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); - propertiesMap.put( - PolarisCredentialProperty.AWS_TOKEN, response.credentials().sessionToken()); - break; - - // @TODO implement the MinIO external OpenID Connect - - // https://min.io/docs/minio/linux/developers/security-token-service.html?ref=docs-redirect#id1 - // case TOKEN_WITH_ASSUME_ROLE_WITH_WEB_IDENTITY: - // break; - } - - return propertiesMap; - } -} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3/S3StorageConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3/S3StorageConfigurationInfo.java deleted file mode 100644 index c66deeff7..000000000 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3/S3StorageConfigurationInfo.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.core.storage.s3; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.base.MoreObjects; -import java.util.List; -import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; - -/** Polaris Storage Configuration information for an S3 Compatible solution, MinIO, Dell ECS... */ -public class S3StorageConfigurationInfo extends PolarisStorageConfigurationInfo { - - // 5 is the approximate max allowed locations for the size of AccessPolicy when LIST is required - // for allowed read and write locations for subscoping creds. - @JsonIgnore private static final int MAX_ALLOWED_LOCATIONS = 5; - private @NotNull CredsVendingStrategyEnum credsVendingStrategy; - private @NotNull CredsCatalogAndClientStrategyEnum credsCatalogAndClientStrategy; - private @NotNull String s3endpoint; - private @NotNull Boolean s3pathStyleAccess; - private @NotNull String s3CredentialsCatalogAccessKeyId; - private @NotNull String s3CredentialsCatalogSecretAccessKey; - private @Nullable String s3CredentialsClientAccessKeyId; - private @Nullable String s3CredentialsClientSecretAccessKey; - - // Define how and what the catalog client will receive as credentials - public static enum CredsVendingStrategyEnum { - KEYS_SAME_AS_CATALOG, - KEYS_DEDICATED_TO_CLIENT, - TOKEN_WITH_ASSUME_ROLE; - }; - - // Define how the access and secret keys will be receive during the catalo creation, if - // ENV_VAR_NAME, the variable must exist in the Polaris running environement - it is more secured, - // but less dynamic - public static enum CredsCatalogAndClientStrategyEnum { - VALUE, - ENV_VAR_NAME; - }; - - // Constructor - @JsonCreator - public S3StorageConfigurationInfo( - @JsonProperty(value = "storageType", required = true) @NotNull StorageType storageType, - @JsonProperty(value = "credsVendingStrategy", required = true) @NotNull - CredsVendingStrategyEnum credsVendingStrategy, - @JsonProperty(value = "credsCatalogAndClientStrategy", required = true) @NotNull - CredsCatalogAndClientStrategyEnum credsCatalogAndClientStrategy, - @JsonProperty(value = "s3Endpoint", required = true) @NotNull String s3Endpoint, - @JsonProperty(value = "s3CredentialsCatalogAccessKeyId", required = true) @NotNull - String s3CredentialsCatalogAccessKeyId, - @JsonProperty(value = "s3CredentialsCatalogSecretAccessKey", required = true) @NotNull - String s3CredentialsCatalogSecretAccessKey, - @JsonProperty(value = "s3CredentialsClientAccessKeyId", required = false) @Nullable - String s3CredentialsClientAccessKeyId, - @JsonProperty(value = "s3CredentialsClientSecretAccessKey", required = false) @Nullable - String s3CredentialsClientSecretAccessKey, - @JsonProperty(value = "s3PathStyleAccess", required = false) @NotNull - Boolean s3PathStyleAccess, - @JsonProperty(value = "allowedLocations", required = true) @NotNull - List allowedLocations) { - - // Classic super and constructor stuff storing data in private internal properties - super(storageType, allowedLocations); - validateMaxAllowedLocations(MAX_ALLOWED_LOCATIONS); - this.credsVendingStrategy = - CredsVendingStrategyEnum.valueOf( - CredsVendingStrategyEnum.class, credsVendingStrategy.name()); - this.credsCatalogAndClientStrategy = - CredsCatalogAndClientStrategyEnum.valueOf( - CredsCatalogAndClientStrategyEnum.class, credsCatalogAndClientStrategy.name()); - this.s3pathStyleAccess = s3PathStyleAccess; - this.s3endpoint = s3Endpoint; - - // The constructor is called multiple time during catalog life - // to do substitution only once, there is a basic if null test, otherwise affect the data from - // the "Polaris cache storage" - // this way the first time the value is retrived from the name of the variable - // next time the getenv will try to retrive a variable but is using the value as a nome, it will - // be null, we affect the value provided by "Polaris cache storage" - if (CredsCatalogAndClientStrategyEnum.ENV_VAR_NAME.equals(credsCatalogAndClientStrategy)) { - String cai = System.getenv(s3CredentialsCatalogAccessKeyId); - String cas = System.getenv(s3CredentialsCatalogSecretAccessKey); - String cli = System.getenv(s3CredentialsClientAccessKeyId); - String cls = System.getenv(s3CredentialsClientSecretAccessKey); - this.s3CredentialsCatalogAccessKeyId = (cai != null) ? cai : s3CredentialsCatalogAccessKeyId; - this.s3CredentialsCatalogSecretAccessKey = - (cas != null) ? cas : s3CredentialsCatalogSecretAccessKey; - this.s3CredentialsClientAccessKeyId = (cli != null) ? cli : s3CredentialsClientAccessKeyId; - this.s3CredentialsClientSecretAccessKey = - (cls != null) ? cls : s3CredentialsClientSecretAccessKey; - } else { - this.s3CredentialsCatalogAccessKeyId = s3CredentialsCatalogAccessKeyId; - this.s3CredentialsCatalogSecretAccessKey = s3CredentialsCatalogSecretAccessKey; - this.s3CredentialsClientAccessKeyId = s3CredentialsClientAccessKeyId; - this.s3CredentialsClientSecretAccessKey = s3CredentialsClientSecretAccessKey; - } - } - - public @NotNull CredsVendingStrategyEnum getCredsVendingStrategy() { - return this.credsVendingStrategy; - } - - public @NotNull CredsCatalogAndClientStrategyEnum getCredsCatalogAndClientStrategy() { - return this.credsCatalogAndClientStrategy; - } - - public @NotNull String getS3Endpoint() { - return this.s3endpoint; - } - - public @NotNull Boolean getS3PathStyleAccess() { - return this.s3pathStyleAccess; - } - - public @NotNull String getS3CredentialsCatalogAccessKeyId() { - return this.s3CredentialsCatalogAccessKeyId; - } - - public @NotNull String getS3CredentialsCatalogSecretAccessKey() { - return this.s3CredentialsCatalogSecretAccessKey; - } - - public @Nullable String getS3CredentialsClientAccessKeyId() { - return this.s3CredentialsClientAccessKeyId; - } - - public @Nullable String getS3CredentialsClientSecretAccessKey() { - return this.s3CredentialsClientSecretAccessKey; - } - - @Override - public String toString() { - return MoreObjects.toStringHelper(this) - .add("storageType", getStorageType()) - .add("storageType", getStorageType().name()) - .add("allowedLocation", getAllowedLocations()) - .toString(); - } - - @Override - public String getFileIoImplClassName() { - return "org.apache.iceberg.aws.s3.S3FileIO"; - } -} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java new file mode 100644 index 000000000..3dfb03814 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleCredentialsStorageIntegration.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import static org.apache.polaris.core.PolarisConfiguration.STORAGE_CREDENTIAL_DURATION_SECONDS; + +import jakarta.annotation.Nonnull; +import java.net.URI; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import org.apache.polaris.core.PolarisConfigurationStore; +import org.apache.polaris.core.PolarisDiagnostics; +import org.apache.polaris.core.context.RealmContext; +import org.apache.polaris.core.storage.InMemoryStorageIntegration; +import org.apache.polaris.core.storage.PolarisCredentialProperty; +import org.apache.polaris.core.storage.StorageUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.policybuilder.iam.IamConditionOperator; +import software.amazon.awssdk.policybuilder.iam.IamEffect; +import software.amazon.awssdk.policybuilder.iam.IamPolicy; +import software.amazon.awssdk.policybuilder.iam.IamResource; +import software.amazon.awssdk.policybuilder.iam.IamStatement; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.StsClientBuilder; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; +import software.amazon.awssdk.services.sts.model.AssumeRoleResponse; + +/** S3 compatible implementation of PolarisStorageIntegration */ +public class S3CompatibleCredentialsStorageIntegration + extends InMemoryStorageIntegration { + + private static final Logger LOGGER = + LoggerFactory.getLogger(S3CompatibleCredentialsStorageIntegration.class); + private final PolarisConfigurationStore configurationStore; + + public S3CompatibleCredentialsStorageIntegration(PolarisConfigurationStore configurationStore) { + super(configurationStore, S3CompatibleCredentialsStorageIntegration.class.getName()); + this.configurationStore = configurationStore; + } + + /** {@inheritDoc} */ + @Override + public EnumMap getSubscopedCreds( + @Nonnull RealmContext realmContext, + @Nonnull PolarisDiagnostics diagnostics, + @Nonnull S3CompatibleStorageConfigurationInfo storageConfig, + boolean allowListOperation, + @Nonnull Set allowedReadLocations, + @Nonnull Set allowedWriteLocations) { + + StsClient stsClient; + String caI = System.getenv(storageConfig.getS3CredentialsCatalogAccessKeyId()); + String caS = System.getenv(storageConfig.getS3CredentialsCatalogSecretAccessKey()); + + EnumMap propertiesMap = + new EnumMap<>(PolarisCredentialProperty.class); + propertiesMap.put(PolarisCredentialProperty.AWS_ENDPOINT, storageConfig.getS3Endpoint()); + propertiesMap.put( + PolarisCredentialProperty.AWS_PATH_STYLE_ACCESS, + storageConfig.getS3PathStyleAccess().toString()); + if (storageConfig.getS3Region() != null) { + propertiesMap.put(PolarisCredentialProperty.CLIENT_REGION, storageConfig.getS3Region()); + } + + LOGGER.debug("S3Compatible - createStsClient()"); + try { + StsClientBuilder stsBuilder = software.amazon.awssdk.services.sts.StsClient.builder(); + stsBuilder.endpointOverride(URI.create(storageConfig.getS3Endpoint())); + if (caI != null && caS != null) { + // else default provider build credentials from profile or standard AWS env var + stsBuilder.credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(caI, caS))); + LOGGER.debug( + "S3Compatible - stsClient using keys from catalog settings - overiding default constructor"); + } + stsClient = stsBuilder.build(); + LOGGER.debug("S3Compatible - stsClient successfully built"); + AssumeRoleResponse response = + stsClient.assumeRole( + AssumeRoleRequest.builder() + .roleSessionName("PolarisCredentialsSTS") + .roleArn( + (storageConfig.getS3RoleArn() == null) ? "" : storageConfig.getS3RoleArn()) + .policy( + policyString(allowListOperation, allowedReadLocations, allowedWriteLocations) + .toJson()) + .durationSeconds( + configurationStore.getConfiguration( + realmContext, STORAGE_CREDENTIAL_DURATION_SECONDS)) + .build()); + propertiesMap.put(PolarisCredentialProperty.AWS_KEY_ID, response.credentials().accessKeyId()); + propertiesMap.put( + PolarisCredentialProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); + propertiesMap.put(PolarisCredentialProperty.AWS_TOKEN, response.credentials().sessionToken()); + LOGGER.debug( + "S3Compatible - assumeRole - Token Expiration at : {}", + response.credentials().expiration().toString()); + + } catch (Exception e) { + System.err.println("S3Compatible - stsClient - build failure : " + e.getMessage()); + } + + return propertiesMap; + } + + /* + * function from AwsCredentialsStorageIntegration but without roleArn parameter + */ + private IamPolicy policyString( + boolean allowList, Set readLocations, Set writeLocations) { + IamPolicy.Builder policyBuilder = IamPolicy.builder(); + IamStatement.Builder allowGetObjectStatementBuilder = + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:GetObject") + .addAction("s3:GetObjectVersion"); + Map bucketListStatementBuilder = new HashMap<>(); + Map bucketGetLocationStatementBuilder = new HashMap<>(); + + String arnPrefix = "arn:aws:s3:::"; + Stream.concat(readLocations.stream(), writeLocations.stream()) + .distinct() + .forEach( + location -> { + URI uri = URI.create(location); + allowGetObjectStatementBuilder.addResource( + IamResource.create( + arnPrefix + StorageUtil.concatFilePrefixes(parseS3Path(uri), "*", "/"))); + final var bucket = arnPrefix + StorageUtil.getBucket(uri); + if (allowList) { + bucketListStatementBuilder + .computeIfAbsent( + bucket, + (String key) -> + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:ListBucket") + .addResource(key)) + .addCondition( + IamConditionOperator.STRING_LIKE, + "s3:prefix", + StorageUtil.concatFilePrefixes(trimLeadingSlash(uri.getPath()), "*", "/")); + } + bucketGetLocationStatementBuilder.computeIfAbsent( + bucket, + key -> + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:GetBucketLocation") + .addResource(key)); + }); + + if (!writeLocations.isEmpty()) { + IamStatement.Builder allowPutObjectStatementBuilder = + IamStatement.builder() + .effect(IamEffect.ALLOW) + .addAction("s3:PutObject") + .addAction("s3:DeleteObject"); + writeLocations.forEach( + location -> { + URI uri = URI.create(location); + allowPutObjectStatementBuilder.addResource( + IamResource.create( + arnPrefix + StorageUtil.concatFilePrefixes(parseS3Path(uri), "*", "/"))); + }); + policyBuilder.addStatement(allowPutObjectStatementBuilder.build()); + } + if (!bucketListStatementBuilder.isEmpty()) { + bucketListStatementBuilder + .values() + .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build())); + } else if (allowList) { + // add list privilege with 0 resources + policyBuilder.addStatement( + IamStatement.builder().effect(IamEffect.ALLOW).addAction("s3:ListBucket").build()); + } + + bucketGetLocationStatementBuilder + .values() + .forEach(statementBuilder -> policyBuilder.addStatement(statementBuilder.build())); + return policyBuilder.addStatement(allowGetObjectStatementBuilder.build()).build(); + } + + /* function from AwsCredentialsStorageIntegration */ + private static @Nonnull String parseS3Path(URI uri) { + String bucket = StorageUtil.getBucket(uri); + String path = trimLeadingSlash(uri.getPath()); + return String.join("/", bucket, path); + } + + /* function from AwsCredentialsStorageIntegration */ + private static @Nonnull String trimLeadingSlash(String path) { + if (path.startsWith("/")) { + path = path.substring(1); + } + return path; + } +} diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java new file mode 100644 index 000000000..776279546 --- /dev/null +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/s3compatible/S3CompatibleStorageConfigurationInfo.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.core.storage.s3compatible; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.base.MoreObjects; +import java.util.List; +import org.apache.polaris.core.storage.PolarisStorageConfigurationInfo; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +/** + * S3-Compatible Storage Configuration. This class holds the parameters needed to connect to + * S3-compatible storage services such as MinIO, Ceph, Dell ECS, etc. + */ +public class S3CompatibleStorageConfigurationInfo extends PolarisStorageConfigurationInfo { + + // 5 is the approximate max allowed locations for the size of AccessPolicy when LIST is required + // for allowed read and write locations for subscoping creds. + @JsonIgnore private static final int MAX_ALLOWED_LOCATIONS = 5; + private final @NotNull String s3Endpoint; + private final @Nullable String s3CredentialsCatalogAccessKeyId; + private final @Nullable String s3CredentialsCatalogSecretAccessKey; + private final @NotNull Boolean s3PathStyleAccess; + private final @Nullable String s3Region; + private final @Nullable String s3RoleArn; + + @JsonCreator + public S3CompatibleStorageConfigurationInfo( + @JsonProperty(value = "storageType", required = true) @NotNull StorageType storageType, + @JsonProperty(value = "s3Endpoint", required = true) @NotNull String s3Endpoint, + @JsonProperty(value = "s3CredentialsCatalogAccessKeyId", required = true) @Nullable + String s3CredentialsCatalogAccessKeyId, + @JsonProperty(value = "s3CredentialsCatalogSecretAccessKey", required = true) @Nullable + String s3CredentialsCatalogSecretAccessKey, + @JsonProperty(value = "s3PathStyleAccess", required = false) @NotNull + Boolean s3PathStyleAccess, + @JsonProperty(value = "s3Region", required = false) @Nullable String s3Region, + @JsonProperty(value = "s3RoleArn", required = false) @Nullable String s3RoleArn, + @JsonProperty(value = "allowedLocations", required = true) @Nullable + List allowedLocations) { + + super(StorageType.S3_COMPATIBLE, allowedLocations); + validateMaxAllowedLocations(MAX_ALLOWED_LOCATIONS); + this.s3PathStyleAccess = s3PathStyleAccess; + this.s3Endpoint = s3Endpoint; + this.s3CredentialsCatalogAccessKeyId = + (s3CredentialsCatalogAccessKeyId == null) ? "" : s3CredentialsCatalogAccessKeyId; + this.s3CredentialsCatalogSecretAccessKey = + (s3CredentialsCatalogSecretAccessKey == null) ? "" : s3CredentialsCatalogSecretAccessKey; + this.s3Region = s3Region; + this.s3RoleArn = s3RoleArn; + } + + public @NotNull String getS3Endpoint() { + return this.s3Endpoint; + } + + public @NotNull Boolean getS3PathStyleAccess() { + return this.s3PathStyleAccess; + } + + public @Nullable String getS3CredentialsCatalogAccessKeyId() { + return this.s3CredentialsCatalogAccessKeyId; + } + + public @Nullable String getS3CredentialsCatalogSecretAccessKey() { + return this.s3CredentialsCatalogSecretAccessKey; + } + + public @Nullable String getS3RoleArn() { + return this.s3RoleArn; + } + + public @Nullable String getS3Region() { + return this.s3Region; + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("storageType", getStorageType().name()) + .add("allowedLocation", getAllowedLocations()) + .add("s3Region", getS3Region()) + .add("s3RoleArn", getS3RoleArn()) + .add("s3PathStyleAccess", getS3PathStyleAccess()) + .add("s3Endpoint", getS3Endpoint()) + .toString(); + } + + @Override + public String getFileIoImplClassName() { + return "org.apache.iceberg.aws.s3.S3FileIO"; + } +} diff --git a/quarkus/defaults/src/main/resources/application-it.properties b/quarkus/defaults/src/main/resources/application-it.properties index 5f46d203f..e4ad1a6e0 100644 --- a/quarkus/defaults/src/main/resources/application-it.properties +++ b/quarkus/defaults/src/main/resources/application-it.properties @@ -35,7 +35,7 @@ polaris.features.defaults."ALLOW_WILDCARD_LOCATION"=true polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=true polaris.features.defaults."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_it"=true polaris.features.defaults."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true -polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","GCS","AZURE"] +polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["FILE","S3","S3_COMPATIBLE","GCS","AZURE"] polaris.realm-context.realms=POLARIS,OTHER diff --git a/quarkus/defaults/src/main/resources/application.properties b/quarkus/defaults/src/main/resources/application.properties index 8b25cbcd5..68e228abd 100644 --- a/quarkus/defaults/src/main/resources/application.properties +++ b/quarkus/defaults/src/main/resources/application.properties @@ -85,7 +85,7 @@ polaris.realm-context.header-name=Polaris-Realm polaris.realm-context.require-header=false polaris.features.defaults."ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING"=false -polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","GCS","AZURE","FILE"] +polaris.features.defaults."SUPPORTED_CATALOG_STORAGE_TYPES"=["S3","S3_COMPATIBLE","GCS","AZURE","FILE"] # realm overrides # polaris.features.realm-overrides."my-realm"."INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST"=true # polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true diff --git a/regtests/minio/Readme.md b/regtests/minio/Readme.md index 08089f56f..afa54e0b2 100644 --- a/regtests/minio/Readme.md +++ b/regtests/minio/Readme.md @@ -18,22 +18,21 @@ --> # MiniIO Secured -## Minio and secured buckets with TLS self-signed / custom AC +## Minio and secured buckets with TLS self-signed / custom Certificate Authority -To be able to test Polaris with buckets in TLS under custom AC or self-signed certificate +To be able to test Polaris with buckets in TLS under custom Certificate Authority or self-signed certificate ## MiniIO generate self-signed certificates designed for docker-compose setup - Download minio certificate generator : https://github.com/minio/certgen -- ```./certgen -host "localhost,minio,*"``` -- put them in ./certs and ./certs/CAs -- they will be mounted in default minio container placeholder +- Generate certifications: ```./certgen -host "localhost,minio,*"``` +- put them in ./certs and ./certs/CAs. They will be mounted in the default MinIO container placeholder. ## Test minIO secured TLS buckets from self-signed certificate with AWS CLI - ```aws s3 ls s3:// --recursive --endpoint-url=https://localhost:9000 --no-verify-ssl``` - ```aws s3 ls s3:// --recursive --endpoint-url=https://localhost:9000 --ca-bundle=./certs/public.crt``` -## add to java cacerts only the public.crt as an AC +## add to java cacerts only the public.crt as an Certificate Authority - ```sudo keytool -import -trustcacerts -cacerts -storepass changeit -noprompt -alias minio -file ./certs/public.crt``` - ```keytool -list -cacerts -alias minio -storepass changeit``` diff --git a/regtests/minio/docker-compose.yml b/regtests/minio/docker-compose.yml index b61ca6537..ff6a5c0a7 100644 --- a/regtests/minio/docker-compose.yml +++ b/regtests/minio/docker-compose.yml @@ -54,14 +54,10 @@ services: until (/usr/bin/mc config host add minio https://minio:9000 admin password) do echo '...waiting...' && sleep 1; done; /usr/bin/mc rm -r --force --quiet minio/warehouse; /usr/bin/mc mb --ignore-existing minio/warehouse; - /usr/bin/mc policy set readwrite minio/warehouse; /usr/bin/mc rm -r --force --quiet minio/warehouse2; /usr/bin/mc mb --ignore-existing minio/warehouse2; - /usr/bin/mc policy set readwrite minio/warehouse2; /usr/bin/mc admin user add minio minio-user-catalog 12345678-minio-catalog; - /usr/bin/mc admin user add minio minio-user-client 12345678-minio-client; /usr/bin/mc admin policy attach minio readwrite --user minio-user-catalog; - /usr/bin/mc admin policy attach minio readwrite --user minio-user-client; tail -f /dev/null " networks: diff --git a/regtests/minio/miniodata/Readme.md b/regtests/minio/miniodata/Readme.md new file mode 100644 index 000000000..d65c6f472 --- /dev/null +++ b/regtests/minio/miniodata/Readme.md @@ -0,0 +1 @@ +# Folder for MinIO data container volume diff --git a/regtests/run_spark_sql_s3compatible.sh b/regtests/run_spark_sql_s3compatible.sh index fc16fa542..172488b7b 100755 --- a/regtests/run_spark_sql_s3compatible.sh +++ b/regtests/run_spark_sql_s3compatible.sh @@ -47,7 +47,6 @@ if [ $# -ne 0 ] && [ $# -ne 1 ]; then fi # Init -SPARK_BEARER_TOKEN="${REGTEST_ROOT_BEARER_TOKEN:-principal:root;realm:default-realm}" REGTEST_HOME=$(dirname $(realpath $0)) cd ${REGTEST_HOME} @@ -65,6 +64,20 @@ fi S3_LOCATION_2="s3://warehouse2/polaris/" +# SPARK_BEARER_TOKEN +if ! output=$(curl -s -X POST -H "Polaris-Realm: POLARIS" "http://${POLARIS_HOST:-localhost}:8181/api/catalog/v1/oauth/tokens" \ + -d "grant_type=client_credentials" \ + -d "client_id=root" \ + -d "client_secret=secret" \ + -d "scope=PRINCIPAL_ROLE:ALL"); then + echo "Error: Failed to retrieve bearer token" + exit 1 +fi +SPARK_BEARER_TOKEN=$(echo "$output" | awk -F\" '{print $4}') +if [ "SPARK_BEARER_TOKEN" == "unauthorized_client" ]; then + echo "Error: Failed to retrieve bearer token" + exit 1 +fi # check if Polaris is running polaris_http_code=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs --output /dev/null) @@ -73,6 +86,7 @@ if [ $polaris_http_code -eq 000 ] && [ $polaris_http_code -ne 200 ]; then exit 1 fi + # check if cacerts contain MinIO certificate cert_response=$(keytool -list -cacerts -alias minio -storepass changeit | grep trustedCertEntry) echo $cert_response @@ -108,14 +122,15 @@ fi # creation of catalog +echo """ +These environnement variables have to be available to Polaris service : +CATALOG_S3_KEY_ID = minio-user-catalog +CATALOG_S3_KEY_SECRET = 12345678-minio-catalog +export CATALOG_S3_KEY_ID=minio-user-catalog +export CATALOG_S3_KEY_SECRET=12345678-minio-catalog +""" -# if "credsCatalogAndClientStrategy"=="ENV_VAR_NAME" and not "VALUE", then the following environnement variables have to be available to Polaris -# CATALOG_ID=minio-user-catalog -# CATALOG_SECRET=12345678-minio-catalog -# CLIENT_ID=minio-user-client -# CLIENT_SECRET=12345678-minio-client - -echo -e "\n----\nCREATE Catalog\n" +echo -e "\n----\nCREATE Catalog with few parameters \n" response_catalog=$(curl --output /dev/null -w "%{http_code}" -s -i -X POST -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \ -H 'Accept: application/json' \ -H 'Content-Type: application/json' \ @@ -130,18 +145,12 @@ response_catalog=$(curl --output /dev/null -w "%{http_code}" -s -i -X POST -H " }, \"storageConfigInfo\": { \"storageType\": \"S3_COMPATIBLE\", - \"credsVendingStrategy\": \"TOKEN_WITH_ASSUME_ROLE\", - \"credsCatalogAndClientStrategy\": \"VALUE\", \"allowedLocations\": [\"${S3_LOCATION}/\"], - \"s3.path-style-access\": true, - \"s3.endpoint\": \"https://localhost:9000\", - \"s3.credentials.catalog.access-key-id\": \"minio-user-catalog\", - \"s3.credentials.catalog.secret-access-key\": \"12345678-minio-catalog\", - \"s3.credentials.client.access-key-id\": \"minio-user-client\", - \"s3.credentials.client.secret-access-key\": \"12345678-minio-client\" + \"s3.endpoint\": \"https://localhost:9000\" } }" ) + echo -e "Catalog creation - response API http code : $response_catalog \n" if [ $response_catalog -ne 201 ] && [ $response_catalog -ne 409 ]; then echo "Problem during catalog creation" @@ -149,16 +158,14 @@ if [ $response_catalog -ne 201 ] && [ $response_catalog -ne 409 ]; then fi - - echo -e "Get the catalog created : \n" curl -s -i -X GET -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \ -H 'Accept: application/json' \ -H 'Content-Type: application/json' \ http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs/manual_spark -# Try to update the catalog, - adding a second bucket in the alllowed locations -echo -e "\n----\nUPDATE the catalog, - adding a second bucket in the alllowed locations\n" +# Update the catalog +echo -e "\n----\nUPDATE the catalog v1, - adding a second bucket in the alllowed locations\n" curl -s -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \ -H 'Accept: application/json' \ -H 'Content-Type: application/json' \ @@ -170,26 +177,17 @@ curl -s -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \ }, \"storageConfigInfo\": { \"storageType\": \"S3_COMPATIBLE\", - \"credsVendingStrategy\": \"TOKEN_WITH_ASSUME_ROLE\", - \"credsCatalogAndClientStrategy\": \"VALUE\", \"allowedLocations\": [\"${S3_LOCATION}/\",\"${S3_LOCATION_2}/\"], - \"s3.path-style-access\": true, \"s3.endpoint\": \"https://localhost:9000\", - \"s3.credentials.catalog.access-key-id\": \"minio-user-catalog\", - \"s3.credentials.catalog.secret-access-key\": \"12345678-minio-catalog\", - \"s3.credentials.client.access-key-id\": \"minio-user-client\", - \"s3.credentials.client.secret-access-key\": \"12345678-minio-client\" + \"s3.region\": \"region-1\", + \"s3.pathStyleAccess\": true, + \"s3.credentials.catalog.accessKeyEnvVar\": \"CATALOG_S3_KEY_ID\", + \"s3.credentials.catalog.secretAccessKeyEnvVar\": \"CATALOG_S3_KEY_SECRET\", + \"s3.roleArn\": \"arn:xxx:xxx:xxx:xxxx\" } }" -echo -e "Get the catalog updated with second allowed location : \n" -curl -s -i -X GET -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" \ - -H 'Accept: application/json' \ - -H 'Content-Type: application/json' \ - http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs/manual_spark - - echo -e "\n----\nAdd TABLE_WRITE_DATA to the catalog's catalog_admin role since by default it can only manage access and metadata\n" curl -i -X PUT -H "Authorization: Bearer ${SPARK_BEARER_TOKEN}" -H 'Accept: application/json' -H 'Content-Type: application/json' \ http://${POLARIS_HOST:-localhost}:8181/api/management/v1/catalogs/manual_spark/catalog-roles/catalog_admin/grants \ @@ -212,9 +210,9 @@ ${SPARK_HOME}/bin/spark-sql --verbose \ -f "minio/queries-for-spark.sql" + echo -e "\n\n\nEnd of tests, a table and a view data with displayed should be visible in log above" echo "Minio stopping, bucket browser will be shutdown, volume data of the bucket remains in 'regtests/minio/miniodata'" echo ":-)" echo "" -docker-compose --progress quiet --project-name minio --project-directory minio/ -f minio/docker-compose.yml down - +docker-compose --progress quiet --project-name polaris-minio --project-directory minio/ -f minio/docker-compose.yml down diff --git a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java index 0161e5239..f4c033802 100644 --- a/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java +++ b/service/common/src/main/java/org/apache/polaris/service/storage/PolarisStorageIntegrationProviderImpl.java @@ -41,7 +41,7 @@ import org.apache.polaris.core.storage.aws.AwsCredentialsStorageIntegration; import org.apache.polaris.core.storage.azure.AzureCredentialsStorageIntegration; import org.apache.polaris.core.storage.gcp.GcpCredentialsStorageIntegration; -import org.apache.polaris.core.storage.s3.S3CredentialsStorageIntegration; +import org.apache.polaris.core.storage.s3compatible.S3CompatibleCredentialsStorageIntegration; import software.amazon.awssdk.services.sts.StsClient; @ApplicationScoped @@ -85,7 +85,9 @@ public PolarisStorageIntegrationProviderImpl( new AwsCredentialsStorageIntegration(configurationStore, stsClientSupplier.get()); break; case S3_COMPATIBLE: - storageIntegration = (PolarisStorageIntegration) new S3CredentialsStorageIntegration(); + storageIntegration = + (PolarisStorageIntegration) + new S3CompatibleCredentialsStorageIntegration(configurationStore); break; case GCS: storageIntegration = diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml index d4a1f44fb..370a62dc0 100644 --- a/spec/polaris-management-service.yml +++ b/spec/polaris-management-service.yml @@ -878,7 +878,7 @@ components: propertyName: storageType mapping: S3: "#/components/schemas/AwsStorageConfigInfo" - S3_COMPATIBLE: "#/components/schemas/S3StorageConfigInfo" + S3_COMPATIBLE: "#/components/schemas/S3CompatibleStorageConfigInfo" AZURE: "#/components/schemas/AzureStorageConfigInfo" GCS: "#/components/schemas/GcpStorageConfigInfo" FILE: "#/components/schemas/FileStorageConfigInfo" @@ -907,57 +907,38 @@ components: required: - roleArn - S3StorageConfigInfo: + S3CompatibleStorageConfigInfo: type: object - description: S3 compatible storage configuration info (MinIO, Dell ECS, Netapp StorageGRID, ...) + description: s3-compatible storage configuration info (MinIO, Ceph, Dell ECS, Netapp StorageGRID, ...) allOf: - $ref: '#/components/schemas/StorageConfigInfo' properties: - credsCatalogAndClientStrategy: - type: string - enum: - - VALUE - - ENV_VAR_NAME - default: ENV_VAR_NAME - example: "ACCESS_KEY" - description: When you send key VALUE directly via this command, they should apear in logs. By ENV_VAR_NAME without dollar, only a reference will appear in logs, but the value have to be available as environnement variable in the context where Polaris is running - credsVendingStrategy: - type: string - enum: - - TOKEN_WITH_ASSUME_ROLE - - KEYS_SAME_AS_CATALOG - - KEYS_DEDICATED_TO_CLIENT - default: TOKEN_WITH_ASSUME_ROLE - description: The catalog strategy to vend credentials to client. Options possible are same keys than catalog, keys dedicated to clients, or Tokens with STS methods 'assumeRole' for Dell ECS or NetApp StorageGrid solution, 'truc' for MinIo solution) - s3.path-style-access: - type: boolean - description: if true use path style - default: false s3.endpoint: type: string description: the S3 endpoint example: "http[s]://host:port" - s3.credentials.catalog.access-key-id: + s3.credentials.catalog.accessKeyEnvVar: type: string - description: The ACCESS_KEY_ID used y the catalog to communicate with S3 - example: "$AWS_ACCESS_KEY_ID" - s3.credentials.catalog.secret-access-key: + description: Default to AWS credentials, otherwise set the environment variable name for the 'ACCESS_KEY_ID' used by the catalog to communicate with S3 + example: "CATALOG_1_ACCESS_KEY_ENV_VARIABLE_NAME or AWS_ACCESS_KEY_ID" + s3.credentials.catalog.secretAccessKeyEnvVar: type: string - description: The SECRET_ACCESS_KEY used y the catalog to communicate with S3 - example: "$AWS_SECRET_ACCESS_KEY" - s3.credentials.client.access-key-id: + description: Default to AWS credentials, otherwise set the environment variable name for the 'SECRET_ACCESS_KEY' used by the catalog to communicate with S3 + example: "CATALOG_1_SECRET_KEY_ENV_VARIABLE_NAME or AWS_SECRET_ACCESS_KEY" + s3.pathStyleAccess: + type: boolean + description: Whether or not to use path-style access + default: false + s3.region: type: string - description: Optional - ACCESS_KEY_ID vended by catalog to the client in case of this CredentialVendedStrategy is selected - example: "$AWS_ACCESS_KEY_ID" - s3.credentials.client.secret-access-key: + description: Optional - the s3 region where data is stored + example: "rack-1 or us-east-1" + s3.roleArn: type: string - description: Optional - SECRET_ACCESS_KEY vended by catalog to the client in case of this CredentialVendedStrategy is selected - example: "$AWS_SECRET_ACCESS_KEY" + description: Optional - a s3 role arn + example: "arn:aws:iam::123456789001:principal/abc1-b-self1234" required: - - credsVendingStrategy - s3.endpoint - - s3.credentials.catalog.access-key-id - - s3.credentials.catalog.secret-access-key AzureStorageConfigInfo: type: object