diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 69a1516d0..0ab6a807b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,7 +19,7 @@ [versions] hadoop = "3.4.1" -iceberg = "1.7.1" +iceberg = "1.8.1" quarkus = "3.19.1" immutables = "2.10.1" picocli = "4.7.6" diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java index 4d0c987b2..7478b1ea2 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java @@ -34,6 +34,7 @@ public static RESTCatalog restCatalog( PolarisApiEndpoints endpoints, PrincipalWithCredentials credentials, String catalog, + String warehouse, Map extraProperties) { String authToken = client.obtainToken(credentials); SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); @@ -53,11 +54,11 @@ public static RESTCatalog restCatalog( .put( org.apache.iceberg.CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO") - .put("warehouse", catalog) + .put("warehouse", warehouse) .put("header." + endpoints.realmHeaderName(), endpoints.realmId()) .putAll(extraProperties); - restCatalog.initialize("polaris", propertiesBuilder.buildKeepingLast()); + restCatalog.initialize(catalog, propertiesBuilder.buildKeepingLast()); return restCatalog; } } diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index 48dbce059..dd6ccb63d 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -459,8 +459,7 @@ public void testIcebergRegisterTableInExternalCatalog() throws IOException { .assignUUID() .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) - .addSchema( - new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())), 1) + .addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get()))) .build(); TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation)); @@ -503,7 +502,7 @@ public void testIcebergUpdateTableInExternalCatalog() throws IOException { .assignUUID() .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) - .addSchema(new Schema(col1), 1) + .addSchema(new Schema(col1)) .build(); TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation)); @@ -551,8 +550,7 @@ public void testIcebergDropTableInExternalCatalog() throws IOException { .assignUUID() .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) - .addSchema( - new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())), 1) + .addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get()))) .build(); TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation)); diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index 2a31f2f11..dfe16c807 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -90,6 +90,7 @@ import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -117,16 +118,16 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests private static URI externalCatalogBase; protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; - private static String principalRoleName; private static ClientCredentials adminCredentials; - private static PrincipalWithCredentials principalCredentials; private static PolarisApiEndpoints endpoints; private static PolarisClient client; private static ManagementApi managementApi; - private static CatalogApi catalogApi; + private PrincipalWithCredentials principalCredentials; + private CatalogApi catalogApi; private RESTCatalog restCatalog; private String currentCatalogName; + private TestInfo testInfo; private final String catalogBaseLocation = s3BucketBase + "/" + System.getenv("USER") + "/path/to/data"; @@ -158,10 +159,6 @@ static void setup( endpoints = apiEndpoints; client = polarisClient(endpoints); managementApi = client.managementApi(credentials); - String principalName = client.newEntityName("snowman-rest"); - principalRoleName = client.newEntityName("rest-admin"); - principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); - catalogApi = client.catalogApi(principalCredentials); URI testRootUri = IntegrationTestsHelper.getTemporaryDirectory(tempDir); s3BucketBase = testRootUri.resolve("my-bucket"); externalCatalogBase = testRootUri.resolve("external-catalog"); @@ -174,10 +171,10 @@ static void close() throws Exception { @BeforeEach public void before(TestInfo testInfo) { + this.testInfo = testInfo; String principalName = "snowman-rest-" + UUID.randomUUID(); - principalRoleName = "rest-admin-" + UUID.randomUUID(); - PrincipalWithCredentials principalCredentials = - managementApi.createPrincipalWithRole(principalName, principalRoleName); + String principalRoleName = "rest-admin-" + UUID.randomUUID(); + principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); catalogApi = client.catalogApi(principalCredentials); @@ -218,6 +215,21 @@ public void before(TestInfo testInfo) { managementApi.createCatalog(principalRoleName, catalog); + restCatalog = initCatalog(currentCatalogName, ImmutableMap.of()); + } + + @AfterEach + public void cleanUp() { + client.cleanUp(adminCredentials); + } + + @Override + protected RESTCatalog catalog() { + return restCatalog; + } + + @Override + protected RESTCatalog initCatalog(String catalogName, Map additionalProperties) { Optional restCatalogConfig = testInfo .getTestMethod() @@ -233,24 +245,14 @@ public void before(TestInfo testInfo) { extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]); } }); - - restCatalog = - IcebergHelper.restCatalog( - client, - endpoints, - principalCredentials, - currentCatalogName, - extraPropertiesBuilder.build()); - } - - @AfterEach - public void cleanUp() { - client.cleanUp(adminCredentials); - } - - @Override - protected RESTCatalog catalog() { - return restCatalog; + extraPropertiesBuilder.putAll(additionalProperties); + return IcebergHelper.restCatalog( + client, + endpoints, + principalCredentials, + catalogName, + currentCatalogName, + extraPropertiesBuilder.buildKeepingLast()); } @Override @@ -273,6 +275,33 @@ protected boolean overridesRequestedLocation() { return true; } + @Test + @Override + public void createAndDropEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.createAndDropEmptyNamespace(); + } + + @Test + @Override + public void namespacePropertiesOnEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.namespacePropertiesOnEmptyNamespace(); + } + + @Test + @Override + public void listTablesInEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.listTablesInEmptyNamespace(); + } + @Test public void testListGrantsOnCatalogObjectsToCatalogRoles() { restCatalog.createNamespace(Namespace.of("ns1")); diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java index 1fb24f5fe..c92c1b5ff 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java @@ -42,6 +42,7 @@ import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; @@ -113,7 +114,17 @@ public void before(TestInfo testInfo) { managementApi.createCatalog(principalRoleName, catalog); restCatalog = - IcebergHelper.restCatalog(client, endpoints, principalCredentials, catalogName, Map.of()); + IcebergHelper.restCatalog( + client, + endpoints, + principalCredentials, + catalogName, + catalogName, + Map.of( + org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", + "catalog-default-key1", + org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", + "catalog-default-key2")); } @AfterEach @@ -156,4 +167,13 @@ protected boolean supportsServerSideRetry() { protected boolean overridesRequestedLocation() { return true; } + + @Test + @Override + public void listViewsInEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ or Polaris supports empty namespaces. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.listViewsInEmptyNamespace(); + } } diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIT.java index 735973d8e..a38e41f1b 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIT.java @@ -20,21 +20,38 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import java.lang.reflect.Field; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; + import org.apache.iceberg.view.ViewCatalogTests; import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTest; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.AnnotatedElementContext; +import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.api.io.TempDirFactory; @QuarkusIntegrationTest -public class QuarkusRestCatalogViewFileIT - extends PolarisRestCatalogViewFileIntegrationTest { +public class QuarkusRestCatalogViewFileIT extends PolarisRestCatalogViewFileIntegrationTest { @BeforeEach - public void setUpTempDir(@TempDir Path tempDir) throws Exception { + public void setUpTempDir(@TempDir(factory = CustomTempDirFactory.class) Path tempDir) + throws Exception { // see https://github.com/quarkusio/quarkus/issues/13261 Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); field.setAccessible(true); field.set(this, tempDir); } + + private static class CustomTempDirFactory implements TempDirFactory { + @Override + public Path createTempDirectory( + AnnotatedElementContext elementContext, ExtensionContext extensionContext) + throws Exception { + Path basePath = Paths.get(BASE_LOCATION.replaceFirst("file://", "")); + Files.createDirectories(basePath); + return Files.createTempDirectory(basePath, null); + } + } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java index 84ac558b6..d66fbb7f9 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java @@ -61,7 +61,6 @@ import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.catalog.CatalogTests; import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.BadRequestException; @@ -153,6 +152,8 @@ public Map getConfigOverrides() { "true", "polaris.features.defaults.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", "true", + "polaris.features.defaults.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", + "true", "polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); } @@ -177,7 +178,6 @@ public Map getConfigOverrides() { private BasePolarisCatalog catalog; private CallContext callContext; private AwsStorageConfigInfo storageConfigModel; - private StsClient stsClient; private String realmName; private PolarisMetaStoreManager metaStoreManager; private PolarisCallContext polarisContext; @@ -241,32 +241,6 @@ public void before(TestInfo testInfo) { securityContext, new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); - String storageLocation = "s3://my-bucket/path/to/data"; - storageConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::012345678901:role/jdoe") - .setExternalId("externalId") - .setUserArn("aws::a:user:arn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of(storageLocation, "s3://externally-owned-bucket")) - .build(); - catalogEntity = - adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); - - PolarisPassthroughResolutionView passthroughView = - new PolarisPassthroughResolutionView( - callContext, entityManager, securityContext, CATALOG_NAME); - TaskExecutor taskExecutor = Mockito.mock(); RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(createMockMetaStoreManagerFactory()); this.fileIOFactory = @@ -289,19 +263,7 @@ public void before(TestInfo testInfo) { isA(AwsStorageConfigurationInfo.class))) .thenReturn((PolarisStorageIntegration) storageIntegration); - this.catalog = - new BasePolarisCatalog( - entityManager, - metaStoreManager, - callContext, - passthroughView, - securityContext, - taskExecutor, - fileIOFactory); - this.catalog.initialize( - CATALOG_NAME, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); + this.catalog = initCatalog(CATALOG_NAME, ImmutableMap.of()); } @AfterEach @@ -315,6 +277,52 @@ protected BasePolarisCatalog catalog() { return catalog; } + @Override + protected BasePolarisCatalog initCatalog( + String catalogName, Map additionalProperties) { + String storageLocation = "s3://my-bucket/path/to/data"; + storageConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn("arn:aws:iam::012345678901:role/jdoe") + .setExternalId("externalId") + .setUserArn("aws::a:user:arn") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of(storageLocation, "s3://externally-owned-bucket")) + .build(); + catalogEntity = + adminService.createCatalog( + new CatalogEntity.Builder() + .setName(catalogName) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty( + PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build()); + PolarisPassthroughResolutionView passthroughView = + new PolarisPassthroughResolutionView( + callContext, entityManager, securityContext, catalogName); + TaskExecutor taskExecutor = Mockito.mock(); + BasePolarisCatalog basePolarisCatalog = + new BasePolarisCatalog( + entityManager, + metaStoreManager, + callContext, + passthroughView, + securityContext, + taskExecutor, + fileIOFactory); + + ImmutableMap.Builder propertiesBuilder = + ImmutableMap.builder() + .put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO") + .putAll(additionalProperties); + basePolarisCatalog.initialize(catalogName, propertiesBuilder.buildKeepingLast()); + return basePolarisCatalog; + } + @Override protected boolean requiresNamespaceCreate() { return true; @@ -370,6 +378,42 @@ public Map purgeRealms(Iterable realms) { }; } + @Test + @Override + public void listNamespacesWithEmptyNamespace() { + // TODO: remove this override test once Polaris handles empty namespaces the same as the + // superclass expectation. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.listNamespacesWithEmptyNamespace(); + } + + @Test + @Override + public void createAndDropEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ or Polaris supports empty namespaces. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.createAndDropEmptyNamespace(); + } + + @Test + @Override + public void namespacePropertiesOnEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ or Polaris supports empty namespaces. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.namespacePropertiesOnEmptyNamespace(); + } + + @Test + @Override + public void listTablesInEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ or Polaris supports empty namespaces. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.listTablesInEmptyNamespace(); + } + @Test public void testRenameTableMissingDestinationNamespace() { Assumptions.assumeTrue( @@ -694,7 +738,7 @@ public void testCreateNotificationCreateTableInExternalLocation() { TableMetadata.buildFromEmpty() .assignUUID() .setLocation(anotherTableLocation) - .addSchema(SCHEMA, 4) + .addSchema(SCHEMA) .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) .build(); @@ -751,7 +795,7 @@ public void testCreateNotificationCreateTableOutsideOfMetadataLocation() { TableMetadata.buildFromEmpty() .assignUUID() .setLocation(anotherTableLocation) - .addSchema(SCHEMA, 4) + .addSchema(SCHEMA) .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) .build(); @@ -828,7 +872,7 @@ public void testUpdateNotificationCreateTableInExternalLocation() { TableMetadata.buildFromEmpty() .assignUUID() .setLocation(anotherTableLocation) - .addSchema(SCHEMA, 4) + .addSchema(SCHEMA) .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) .build(); @@ -1398,7 +1442,7 @@ public void testDropNotificationWhenTableExists() { @Override public void testDropTableWithPurge() { if (this.requiresNamespaceCreate()) { - ((SupportsNamespaces) catalog).createNamespace(NS); + catalog.createNamespace(NS); } Assertions.assertThatPredicate(catalog::tableExists) @@ -1491,7 +1535,7 @@ public void testDropTableWithPurgeDisabled() { CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); if (this.requiresNamespaceCreate()) { - ((SupportsNamespaces) noPurgeCatalog).createNamespace(NS); + noPurgeCatalog.createNamespace(NS); } Assertions.assertThatPredicate(noPurgeCatalog::tableExists) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java index b85443faf..79a925b9f 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java @@ -66,8 +66,10 @@ import org.apache.polaris.service.config.RealmEntityManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.io.TempDir; import org.mockito.Mockito; @@ -199,7 +201,12 @@ public void before(TestInfo testInfo) { this.catalog.initialize( CATALOG_NAME, ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); + CatalogProperties.FILE_IO_IMPL, + "org.apache.iceberg.inmemory.InMemoryFileIO", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", + "catalog-default-key1", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", + "catalog-default-key2")); } @AfterEach @@ -222,4 +229,13 @@ protected Catalog tableCatalog() { protected boolean requiresNamespaceCreate() { return true; } + + @Test + @Override + public void listViewsInEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ or Polaris supports empty namespaces. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.listViewsInEmptyNamespace(); + } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java index 29d0bd90c..5a9e30318 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java @@ -1721,7 +1721,7 @@ public Catalog createCallContextCatalog( FileIO fileIO = CatalogUtil.loadFileIO(fileIoImpl, Map.of(), new Configuration()); TableMetadata tableMetadata = TableMetadata.buildFromEmpty() - .addSchema(SCHEMA, SCHEMA.highestFieldId()) + .addSchema(SCHEMA) .setLocation( String.format("%s/bucket/table/metadata/v1.metadata.json", storageLocation)) .addPartitionSpec(PartitionSpec.unpartitioned()) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java index 8aaac080e..3239d3669 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java @@ -31,6 +31,7 @@ import org.apache.iceberg.rest.HTTPClient; import org.apache.iceberg.rest.RESTClient; import org.apache.iceberg.rest.auth.AuthConfig; +import org.apache.iceberg.rest.auth.AuthSession; import org.apache.iceberg.rest.auth.OAuth2Util; import org.apache.polaris.service.it.env.ClientCredentials; import org.apache.polaris.service.it.env.PolarisApiEndpoints; @@ -61,6 +62,7 @@ public void testIcebergRestApiRefreshToken( HTTPClient.builder(Map.of()) .withHeader(endpoints.realmHeaderName(), endpoints.realmId()) .uri(path) + .withAuthSession(AuthSession.EMPTY) .build()) { String credentialString = clientCredentials.clientId() + ":" + clientCredentials.clientSecret(); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java index 4444bcbee..97bd91b68 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java @@ -21,12 +21,17 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.QuarkusTestProfile; import java.lang.reflect.Field; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Map; import org.apache.iceberg.view.ViewCatalogTests; import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTest; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.AnnotatedElementContext; +import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.api.io.TempDirFactory; @QuarkusTest public class QuarkusRestCatalogViewFileIntegrationTest @@ -41,10 +46,22 @@ public Map getConfigOverrides() { } @BeforeEach - public void setUpTempDir(@TempDir Path tempDir) throws Exception { + public void setUpTempDir(@TempDir(factory = CustomTempDirFactory.class) Path tempDir) + throws Exception { // see https://github.com/quarkusio/quarkus/issues/13261 Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); field.setAccessible(true); field.set(this, tempDir); } + + private static class CustomTempDirFactory implements TempDirFactory { + @Override + public Path createTempDirectory( + AnnotatedElementContext elementContext, ExtensionContext extensionContext) + throws Exception { + Path basePath = Paths.get(BASE_LOCATION.replaceFirst("file://", "")); + Files.createDirectories(basePath); + return Files.createTempDirectory(basePath, null); + } + } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java index 9cd0ddd9e..9d47fe947 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java @@ -100,8 +100,7 @@ static TableMetadata writeTableMetadata( tmBuilder .setLocation("path/to/table") .addSchema( - new Schema(List.of(Types.NestedField.of(1, false, "field1", Types.StringType.get()))), - 1) + new Schema(List.of(Types.NestedField.of(1, false, "field1", Types.StringType.get())))) .addSortOrder(SortOrder.unsorted()) .assignUUID(UUID.randomUUID().toString()) .addPartitionSpec(PartitionSpec.unpartitioned()); diff --git a/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref b/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref index a23d1d941..4cd8ce0f8 100755 --- a/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref +++ b/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref @@ -1,4 +1,4 @@ -{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_basic_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]} +{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_basic_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","POST /v1/{prefix}/transactions/commit","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","HEAD /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]} Catalog created spark-sql (default)> use polaris; spark-sql ()> show namespaces; diff --git a/regtests/t_spark_sql/ref/spark_sql_views.sh.ref b/regtests/t_spark_sql/ref/spark_sql_views.sh.ref index 8e4bff270..853c736db 100755 --- a/regtests/t_spark_sql/ref/spark_sql_views.sh.ref +++ b/regtests/t_spark_sql/ref/spark_sql_views.sh.ref @@ -1,4 +1,4 @@ -{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_views_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]} +{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_views_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","POST /v1/{prefix}/transactions/commit","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","HEAD /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]} Catalog created spark-sql (default)> use polaris; spark-sql ()> show namespaces; diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index ca9771ef2..f82bf7ce7 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -215,6 +215,10 @@ public void initialize(String name, Map properties) { name, this.catalogName); + // Ensure catalogProperties is assigned before calling metricsReporter() for proper + // functionality. + catalogProperties = properties; + // Base location from catalogEntity is primary source of truth, otherwise fall through // to the same key from the properties map, and finally fall through to WAREHOUSE_LOCATION. String baseLocation = @@ -261,7 +265,6 @@ public void initialize(String name, Map properties) { closeableGroup.addCloseable(metricsReporter()); closeableGroup.setSuppressCloseFailure(true); - catalogProperties = properties; tableDefaultProperties = PropertyUtil.propertiesWithPrefix(properties, CatalogProperties.TABLE_DEFAULT_PREFIX); diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 968d9c5fd..12530570c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -92,23 +92,27 @@ public class IcebergCatalogAdapter ImmutableSet.builder() .add(Endpoint.V1_LIST_NAMESPACES) .add(Endpoint.V1_LOAD_NAMESPACE) + .add(Endpoint.V1_NAMESPACE_EXISTS) .add(Endpoint.V1_CREATE_NAMESPACE) .add(Endpoint.V1_UPDATE_NAMESPACE) .add(Endpoint.V1_DELETE_NAMESPACE) .add(Endpoint.V1_LIST_TABLES) .add(Endpoint.V1_LOAD_TABLE) + .add(Endpoint.V1_TABLE_EXISTS) .add(Endpoint.V1_CREATE_TABLE) .add(Endpoint.V1_UPDATE_TABLE) .add(Endpoint.V1_DELETE_TABLE) .add(Endpoint.V1_RENAME_TABLE) .add(Endpoint.V1_REGISTER_TABLE) .add(Endpoint.V1_REPORT_METRICS) + .add(Endpoint.V1_COMMIT_TRANSACTION) .build(); private static final Set VIEW_ENDPOINTS = ImmutableSet.builder() .add(Endpoint.V1_LIST_VIEWS) .add(Endpoint.V1_LOAD_VIEW) + .add(Endpoint.V1_VIEW_EXISTS) .add(Endpoint.V1_CREATE_VIEW) .add(Endpoint.V1_UPDATE_VIEW) .add(Endpoint.V1_DELETE_VIEW)