From 93a28938697113d11eb822047ee8acaeb6ada83a Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Mon, 17 Feb 2025 16:29:18 -0500 Subject: [PATCH 01/11] Upgrade Iceberg to 1.8.0 --- gradle/libs.versions.toml | 2 +- .../it/test/PolarisApplicationIntegrationTest.java | 8 +++----- .../quarkus/catalog/BasePolarisCatalogTest.java | 10 +++++----- .../catalog/PolarisCatalogHandlerWrapperAuthzTest.java | 2 +- .../polaris/service/quarkus/task/TaskTestUtils.java | 3 +-- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 69a1516d0..b331b2dbf 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,7 +19,7 @@ [versions] hadoop = "3.4.1" -iceberg = "1.7.1" +iceberg = "1.8.0" quarkus = "3.19.1" immutables = "2.10.1" picocli = "4.7.6" diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java index 48dbce059..dd6ccb63d 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java @@ -459,8 +459,7 @@ public void testIcebergRegisterTableInExternalCatalog() throws IOException { .assignUUID() .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) - .addSchema( - new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())), 1) + .addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get()))) .build(); TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation)); @@ -503,7 +502,7 @@ public void testIcebergUpdateTableInExternalCatalog() throws IOException { .assignUUID() .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) - .addSchema(new Schema(col1), 1) + .addSchema(new Schema(col1)) .build(); TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation)); @@ -551,8 +550,7 @@ public void testIcebergDropTableInExternalCatalog() throws IOException { .assignUUID() .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) - .addSchema( - new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())), 1) + .addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get()))) .build(); TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation)); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java index 84ac558b6..3f3d0bf40 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java @@ -694,7 +694,7 @@ public void testCreateNotificationCreateTableInExternalLocation() { TableMetadata.buildFromEmpty() .assignUUID() .setLocation(anotherTableLocation) - .addSchema(SCHEMA, 4) + .addSchema(SCHEMA) .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) .build(); @@ -751,7 +751,7 @@ public void testCreateNotificationCreateTableOutsideOfMetadataLocation() { TableMetadata.buildFromEmpty() .assignUUID() .setLocation(anotherTableLocation) - .addSchema(SCHEMA, 4) + .addSchema(SCHEMA) .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) .build(); @@ -828,7 +828,7 @@ public void testUpdateNotificationCreateTableInExternalLocation() { TableMetadata.buildFromEmpty() .assignUUID() .setLocation(anotherTableLocation) - .addSchema(SCHEMA, 4) + .addSchema(SCHEMA) .addPartitionSpec(PartitionSpec.unpartitioned()) .addSortOrder(SortOrder.unsorted()) .build(); @@ -1398,7 +1398,7 @@ public void testDropNotificationWhenTableExists() { @Override public void testDropTableWithPurge() { if (this.requiresNamespaceCreate()) { - ((SupportsNamespaces) catalog).createNamespace(NS); + catalog.createNamespace(NS); } Assertions.assertThatPredicate(catalog::tableExists) @@ -1491,7 +1491,7 @@ public void testDropTableWithPurgeDisabled() { CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); if (this.requiresNamespaceCreate()) { - ((SupportsNamespaces) noPurgeCatalog).createNamespace(NS); + noPurgeCatalog.createNamespace(NS); } Assertions.assertThatPredicate(noPurgeCatalog::tableExists) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java index 29d0bd90c..5a9e30318 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/PolarisCatalogHandlerWrapperAuthzTest.java @@ -1721,7 +1721,7 @@ public Catalog createCallContextCatalog( FileIO fileIO = CatalogUtil.loadFileIO(fileIoImpl, Map.of(), new Configuration()); TableMetadata tableMetadata = TableMetadata.buildFromEmpty() - .addSchema(SCHEMA, SCHEMA.highestFieldId()) + .addSchema(SCHEMA) .setLocation( String.format("%s/bucket/table/metadata/v1.metadata.json", storageLocation)) .addPartitionSpec(PartitionSpec.unpartitioned()) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java index 9cd0ddd9e..9d47fe947 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/task/TaskTestUtils.java @@ -100,8 +100,7 @@ static TableMetadata writeTableMetadata( tmBuilder .setLocation("path/to/table") .addSchema( - new Schema(List.of(Types.NestedField.of(1, false, "field1", Types.StringType.get()))), - 1) + new Schema(List.of(Types.NestedField.of(1, false, "field1", Types.StringType.get())))) .addSortOrder(SortOrder.unsorted()) .assignUUID(UUID.randomUUID().toString()) .addPartitionSpec(PartitionSpec.unpartitioned()); From f78df385bd30e7dc6efa0dc8396a9de13a228a43 Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Mon, 17 Feb 2025 16:55:07 -0500 Subject: [PATCH 02/11] Fix Endpoints and implement `initCatalog` --- .../PolarisRestCatalogIntegrationTest.java | 28 +++++-------- .../catalog/BasePolarisCatalogTest.java | 40 +++++++++++-------- .../catalog/IcebergCatalogAdapter.java | 4 ++ 3 files changed, 39 insertions(+), 33 deletions(-) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index 2a31f2f11..83a524301 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -117,13 +117,12 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests private static URI externalCatalogBase; protected static final String VIEW_QUERY = "select * from ns1.layer1_table"; - private static String principalRoleName; private static ClientCredentials adminCredentials; - private static PrincipalWithCredentials principalCredentials; private static PolarisApiEndpoints endpoints; private static PolarisClient client; private static ManagementApi managementApi; - private static CatalogApi catalogApi; + private PrincipalWithCredentials principalCredentials; + private CatalogApi catalogApi; private RESTCatalog restCatalog; private String currentCatalogName; @@ -158,10 +157,6 @@ static void setup( endpoints = apiEndpoints; client = polarisClient(endpoints); managementApi = client.managementApi(credentials); - String principalName = client.newEntityName("snowman-rest"); - principalRoleName = client.newEntityName("rest-admin"); - principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); - catalogApi = client.catalogApi(principalCredentials); URI testRootUri = IntegrationTestsHelper.getTemporaryDirectory(tempDir); s3BucketBase = testRootUri.resolve("my-bucket"); externalCatalogBase = testRootUri.resolve("external-catalog"); @@ -175,9 +170,8 @@ static void close() throws Exception { @BeforeEach public void before(TestInfo testInfo) { String principalName = "snowman-rest-" + UUID.randomUUID(); - principalRoleName = "rest-admin-" + UUID.randomUUID(); - PrincipalWithCredentials principalCredentials = - managementApi.createPrincipalWithRole(principalName, principalRoleName); + String principalRoleName = "rest-admin-" + UUID.randomUUID(); + principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); catalogApi = client.catalogApi(principalCredentials); @@ -234,13 +228,7 @@ public void before(TestInfo testInfo) { } }); - restCatalog = - IcebergHelper.restCatalog( - client, - endpoints, - principalCredentials, - currentCatalogName, - extraPropertiesBuilder.build()); + restCatalog = initCatalog(currentCatalogName, extraPropertiesBuilder.build()); } @AfterEach @@ -253,6 +241,12 @@ protected RESTCatalog catalog() { return restCatalog; } + @Override + protected RESTCatalog initCatalog(String catalogName, Map additionalProperties) { + return IcebergHelper.restCatalog( + client, endpoints, principalCredentials, catalogName, additionalProperties); + } + @Override protected boolean requiresNamespaceCreate() { return true; diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java index 3f3d0bf40..317210996 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java @@ -263,10 +263,6 @@ public void before(TestInfo testInfo) { .setStorageConfigurationInfo(storageConfigModel, storageLocation) .build()); - PolarisPassthroughResolutionView passthroughView = - new PolarisPassthroughResolutionView( - callContext, entityManager, securityContext, CATALOG_NAME); - TaskExecutor taskExecutor = Mockito.mock(); RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(createMockMetaStoreManagerFactory()); this.fileIOFactory = @@ -290,18 +286,10 @@ public void before(TestInfo testInfo) { .thenReturn((PolarisStorageIntegration) storageIntegration); this.catalog = - new BasePolarisCatalog( - entityManager, - metaStoreManager, - callContext, - passthroughView, - securityContext, - taskExecutor, - fileIOFactory); - this.catalog.initialize( - CATALOG_NAME, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); + initCatalog( + CATALOG_NAME, + ImmutableMap.of( + CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); } @AfterEach @@ -315,6 +303,26 @@ protected BasePolarisCatalog catalog() { return catalog; } + @Override + protected BasePolarisCatalog initCatalog( + String catalogName, Map additionalProperties) { + PolarisPassthroughResolutionView passthroughView = + new PolarisPassthroughResolutionView( + callContext, entityManager, securityContext, CATALOG_NAME); + TaskExecutor taskExecutor = Mockito.mock(); + BasePolarisCatalog basePolarisCatalog = + new BasePolarisCatalog( + entityManager, + metaStoreManager, + callContext, + passthroughView, + securityContext, + taskExecutor, + fileIOFactory); + basePolarisCatalog.initialize(CATALOG_NAME, additionalProperties); + return basePolarisCatalog; + } + @Override protected boolean requiresNamespaceCreate() { return true; diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java index 968d9c5fd..12530570c 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/IcebergCatalogAdapter.java @@ -92,23 +92,27 @@ public class IcebergCatalogAdapter ImmutableSet.builder() .add(Endpoint.V1_LIST_NAMESPACES) .add(Endpoint.V1_LOAD_NAMESPACE) + .add(Endpoint.V1_NAMESPACE_EXISTS) .add(Endpoint.V1_CREATE_NAMESPACE) .add(Endpoint.V1_UPDATE_NAMESPACE) .add(Endpoint.V1_DELETE_NAMESPACE) .add(Endpoint.V1_LIST_TABLES) .add(Endpoint.V1_LOAD_TABLE) + .add(Endpoint.V1_TABLE_EXISTS) .add(Endpoint.V1_CREATE_TABLE) .add(Endpoint.V1_UPDATE_TABLE) .add(Endpoint.V1_DELETE_TABLE) .add(Endpoint.V1_RENAME_TABLE) .add(Endpoint.V1_REGISTER_TABLE) .add(Endpoint.V1_REPORT_METRICS) + .add(Endpoint.V1_COMMIT_TRANSACTION) .build(); private static final Set VIEW_ENDPOINTS = ImmutableSet.builder() .add(Endpoint.V1_LIST_VIEWS) .add(Endpoint.V1_LOAD_VIEW) + .add(Endpoint.V1_VIEW_EXISTS) .add(Endpoint.V1_CREATE_VIEW) .add(Endpoint.V1_UPDATE_VIEW) .add(Endpoint.V1_DELETE_VIEW) From a6c0584ddc40da258538e1d30dbff4337749b148 Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Tue, 25 Feb 2025 21:00:52 -0500 Subject: [PATCH 03/11] Skip tests using Junit Assumptions --- .../PolarisRestCatalogIntegrationTest.java | 22 +++++++++++++++++++ .../catalog/BasePolarisCatalogTest.java | 21 ++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index 83a524301..a60d75e2f 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -90,6 +90,7 @@ import org.assertj.core.api.InstanceOfAssertFactories; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -267,6 +268,27 @@ protected boolean overridesRequestedLocation() { return true; } + @Test + @Override + public void createAndDropEmptyNamespace() { + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.createAndDropEmptyNamespace(); + } + + @Test + @Override + public void namespacePropertiesOnEmptyNamespace() { + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.namespacePropertiesOnEmptyNamespace(); + } + + @Test + @Override + public void listTablesInEmptyNamespace() { + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.listTablesInEmptyNamespace(); + } + @Test public void testListGrantsOnCatalogObjectsToCatalogRoles() { restCatalog.createNamespace(Namespace.of("ns1")); diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java index 317210996..0190a60bb 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java @@ -378,6 +378,27 @@ public Map purgeRealms(Iterable realms) { }; } + @Test + @Override + public void createAndDropEmptyNamespace() { + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.createAndDropEmptyNamespace(); + } + + @Test + @Override + public void namespacePropertiesOnEmptyNamespace() { + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.namespacePropertiesOnEmptyNamespace(); + } + + @Test + @Override + public void listTablesInEmptyNamespace() { + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.listTablesInEmptyNamespace(); + } + @Test public void testRenameTableMissingDestinationNamespace() { Assumptions.assumeTrue( From 7f34d34d690b15d9490b55336d3c5d2e9a6eaaae Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Tue, 25 Feb 2025 22:11:01 -0500 Subject: [PATCH 04/11] Fix `PolarisRestCatalogIntegrationTest` --- .../polaris/service/it/env/IcebergHelper.java | 5 ++- .../PolarisRestCatalogIntegrationTest.java | 43 +++++++++++-------- ...PolarisRestCatalogViewIntegrationBase.java | 3 +- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java index 4d0c987b2..7478b1ea2 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java @@ -34,6 +34,7 @@ public static RESTCatalog restCatalog( PolarisApiEndpoints endpoints, PrincipalWithCredentials credentials, String catalog, + String warehouse, Map extraProperties) { String authToken = client.obtainToken(credentials); SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty(); @@ -53,11 +54,11 @@ public static RESTCatalog restCatalog( .put( org.apache.iceberg.CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO") - .put("warehouse", catalog) + .put("warehouse", warehouse) .put("header." + endpoints.realmHeaderName(), endpoints.realmId()) .putAll(extraProperties); - restCatalog.initialize("polaris", propertiesBuilder.buildKeepingLast()); + restCatalog.initialize(catalog, propertiesBuilder.buildKeepingLast()); return restCatalog; } } diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index a60d75e2f..173b5f6fd 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -122,11 +122,12 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests private static PolarisApiEndpoints endpoints; private static PolarisClient client; private static ManagementApi managementApi; + private PrincipalWithCredentials principalCredentials; private CatalogApi catalogApi; - private RESTCatalog restCatalog; private String currentCatalogName; + private TestInfo testInfo; private final String catalogBaseLocation = s3BucketBase + "/" + System.getenv("USER") + "/path/to/data"; @@ -170,6 +171,7 @@ static void close() throws Exception { @BeforeEach public void before(TestInfo testInfo) { + this.testInfo = testInfo; String principalName = "snowman-rest-" + UUID.randomUUID(); String principalRoleName = "rest-admin-" + UUID.randomUUID(); principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName); @@ -213,6 +215,21 @@ public void before(TestInfo testInfo) { managementApi.createCatalog(principalRoleName, catalog); + restCatalog = initCatalog(currentCatalogName, ImmutableMap.of()); + } + + @AfterEach + public void cleanUp() { + client.cleanUp(adminCredentials); + } + + @Override + protected RESTCatalog catalog() { + return restCatalog; + } + + @Override + protected RESTCatalog initCatalog(String catalogName, Map additionalProperties) { Optional restCatalogConfig = testInfo .getTestMethod() @@ -228,24 +245,14 @@ public void before(TestInfo testInfo) { extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]); } }); - - restCatalog = initCatalog(currentCatalogName, extraPropertiesBuilder.build()); - } - - @AfterEach - public void cleanUp() { - client.cleanUp(adminCredentials); - } - - @Override - protected RESTCatalog catalog() { - return restCatalog; - } - - @Override - protected RESTCatalog initCatalog(String catalogName, Map additionalProperties) { + extraPropertiesBuilder.putAll(additionalProperties); return IcebergHelper.restCatalog( - client, endpoints, principalCredentials, catalogName, additionalProperties); + client, + endpoints, + principalCredentials, + catalogName, + currentCatalogName, + extraPropertiesBuilder.buildKeepingLast()); } @Override diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java index 1fb24f5fe..eb6f344a5 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java @@ -113,7 +113,8 @@ public void before(TestInfo testInfo) { managementApi.createCatalog(principalRoleName, catalog); restCatalog = - IcebergHelper.restCatalog(client, endpoints, principalCredentials, catalogName, Map.of()); + IcebergHelper.restCatalog( + client, endpoints, principalCredentials, catalogName, catalogName, Map.of()); } @AfterEach From facfeb18d2dd5dc9ffcc1859af430270f1e576a1 Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Wed, 26 Feb 2025 20:17:04 -0500 Subject: [PATCH 05/11] Fix `testCatalogWithCustomMetricsReporter` in `BasePolarisCatalogTest` --- .../catalog/BasePolarisCatalogTest.java | 60 ++++++++++--------- .../service/catalog/BasePolarisCatalog.java | 5 +- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java index 0190a60bb..70b45c479 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java @@ -153,6 +153,8 @@ public Map getConfigOverrides() { "true", "polaris.features.defaults.\"INITIALIZE_DEFAULT_CATALOG_FILEIO_FOR_TEST\"", "true", + "polaris.features.defaults.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", + "true", "polaris.features.defaults.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]"); } @@ -241,28 +243,6 @@ public void before(TestInfo testInfo) { securityContext, new PolarisAuthorizerImpl(new PolarisConfigurationStore() {})); - String storageLocation = "s3://my-bucket/path/to/data"; - storageConfigModel = - AwsStorageConfigInfo.builder() - .setRoleArn("arn:aws:iam::012345678901:role/jdoe") - .setExternalId("externalId") - .setUserArn("aws::a:user:arn") - .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) - .setAllowedLocations(List.of(storageLocation, "s3://externally-owned-bucket")) - .build(); - catalogEntity = - adminService.createCatalog( - new CatalogEntity.Builder() - .setName(CATALOG_NAME) - .setDefaultBaseLocation(storageLocation) - .setReplaceNewLocationPrefixWithCatalogDefault("file:") - .addProperty( - PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") - .addProperty( - PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") - .setStorageConfigurationInfo(storageConfigModel, storageLocation) - .build()); - RealmEntityManagerFactory realmEntityManagerFactory = new RealmEntityManagerFactory(createMockMetaStoreManagerFactory()); this.fileIOFactory = @@ -285,11 +265,7 @@ public void before(TestInfo testInfo) { isA(AwsStorageConfigurationInfo.class))) .thenReturn((PolarisStorageIntegration) storageIntegration); - this.catalog = - initCatalog( - CATALOG_NAME, - ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); + this.catalog = initCatalog(CATALOG_NAME, ImmutableMap.of()); } @AfterEach @@ -306,9 +282,30 @@ protected BasePolarisCatalog catalog() { @Override protected BasePolarisCatalog initCatalog( String catalogName, Map additionalProperties) { + String storageLocation = "s3://my-bucket/path/to/data"; + storageConfigModel = + AwsStorageConfigInfo.builder() + .setRoleArn("arn:aws:iam::012345678901:role/jdoe") + .setExternalId("externalId") + .setUserArn("aws::a:user:arn") + .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) + .setAllowedLocations(List.of(storageLocation, "s3://externally-owned-bucket")) + .build(); + catalogEntity = + adminService.createCatalog( + new CatalogEntity.Builder() + .setName(catalogName) + .setDefaultBaseLocation(storageLocation) + .setReplaceNewLocationPrefixWithCatalogDefault("file:") + .addProperty( + PolarisConfiguration.ALLOW_EXTERNAL_TABLE_LOCATION.catalogConfig(), "true") + .addProperty( + PolarisConfiguration.ALLOW_UNSTRUCTURED_TABLE_LOCATION.catalogConfig(), "true") + .setStorageConfigurationInfo(storageConfigModel, storageLocation) + .build()); PolarisPassthroughResolutionView passthroughView = new PolarisPassthroughResolutionView( - callContext, entityManager, securityContext, CATALOG_NAME); + callContext, entityManager, securityContext, catalogName); TaskExecutor taskExecutor = Mockito.mock(); BasePolarisCatalog basePolarisCatalog = new BasePolarisCatalog( @@ -319,7 +316,12 @@ protected BasePolarisCatalog initCatalog( securityContext, taskExecutor, fileIOFactory); - basePolarisCatalog.initialize(CATALOG_NAME, additionalProperties); + + ImmutableMap.Builder propertiesBuilder = + ImmutableMap.builder() + .put(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO") + .putAll(additionalProperties); + basePolarisCatalog.initialize(catalogName, propertiesBuilder.buildKeepingLast()); return basePolarisCatalog; } diff --git a/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java b/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java index ca9771ef2..f82bf7ce7 100644 --- a/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java +++ b/service/common/src/main/java/org/apache/polaris/service/catalog/BasePolarisCatalog.java @@ -215,6 +215,10 @@ public void initialize(String name, Map properties) { name, this.catalogName); + // Ensure catalogProperties is assigned before calling metricsReporter() for proper + // functionality. + catalogProperties = properties; + // Base location from catalogEntity is primary source of truth, otherwise fall through // to the same key from the properties map, and finally fall through to WAREHOUSE_LOCATION. String baseLocation = @@ -261,7 +265,6 @@ public void initialize(String name, Map properties) { closeableGroup.addCloseable(metricsReporter()); closeableGroup.setSuppressCloseFailure(true); - catalogProperties = properties; tableDefaultProperties = PropertyUtil.propertiesWithPrefix(properties, CatalogProperties.TABLE_DEFAULT_PREFIX); From 7881b90dcd6dd64d40034010ee8f257fa61b2e18 Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Thu, 27 Feb 2025 20:41:29 -0500 Subject: [PATCH 06/11] Skip `listNamespacesWithEmptyNamespace` and add comments --- .../test/PolarisRestCatalogIntegrationTest.java | 6 ++++++ .../quarkus/catalog/BasePolarisCatalogTest.java | 17 +++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java index 173b5f6fd..dfe16c807 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java @@ -278,6 +278,8 @@ protected boolean overridesRequestedLocation() { @Test @Override public void createAndDropEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ. Assumptions.assumeTrue(supportsEmptyNamespace()); super.createAndDropEmptyNamespace(); } @@ -285,6 +287,8 @@ public void createAndDropEmptyNamespace() { @Test @Override public void namespacePropertiesOnEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ. Assumptions.assumeTrue(supportsEmptyNamespace()); super.namespacePropertiesOnEmptyNamespace(); } @@ -292,6 +296,8 @@ public void namespacePropertiesOnEmptyNamespace() { @Test @Override public void listTablesInEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ. Assumptions.assumeTrue(supportsEmptyNamespace()); super.listTablesInEmptyNamespace(); } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java index 70b45c479..d66fbb7f9 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogTest.java @@ -61,7 +61,6 @@ import org.apache.iceberg.UpdateSchema; import org.apache.iceberg.catalog.CatalogTests; import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.SupportsNamespaces; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.BadRequestException; @@ -179,7 +178,6 @@ public Map getConfigOverrides() { private BasePolarisCatalog catalog; private CallContext callContext; private AwsStorageConfigInfo storageConfigModel; - private StsClient stsClient; private String realmName; private PolarisMetaStoreManager metaStoreManager; private PolarisCallContext polarisContext; @@ -380,9 +378,20 @@ public Map purgeRealms(Iterable realms) { }; } + @Test + @Override + public void listNamespacesWithEmptyNamespace() { + // TODO: remove this override test once Polaris handles empty namespaces the same as the + // superclass expectation. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.listNamespacesWithEmptyNamespace(); + } + @Test @Override public void createAndDropEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ or Polaris supports empty namespaces. Assumptions.assumeTrue(supportsEmptyNamespace()); super.createAndDropEmptyNamespace(); } @@ -390,6 +399,8 @@ public void createAndDropEmptyNamespace() { @Test @Override public void namespacePropertiesOnEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ or Polaris supports empty namespaces. Assumptions.assumeTrue(supportsEmptyNamespace()); super.namespacePropertiesOnEmptyNamespace(); } @@ -397,6 +408,8 @@ public void namespacePropertiesOnEmptyNamespace() { @Test @Override public void listTablesInEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ or Polaris supports empty namespaces. Assumptions.assumeTrue(supportsEmptyNamespace()); super.listTablesInEmptyNamespace(); } From 66fccd9e44c3317b46083cdb4ddff56e04e12ca6 Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Fri, 28 Feb 2025 16:56:28 -0500 Subject: [PATCH 07/11] Fix `defaultViewProperties` in `ViewCatalogTests` --- ...PolarisRestCatalogViewIntegrationBase.java | 21 ++++++++++++++++++- .../catalog/BasePolarisCatalogViewTest.java | 18 +++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java index eb6f344a5..c92c1b5ff 100644 --- a/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java +++ b/integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java @@ -42,6 +42,7 @@ import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.extension.ExtendWith; @@ -114,7 +115,16 @@ public void before(TestInfo testInfo) { restCatalog = IcebergHelper.restCatalog( - client, endpoints, principalCredentials, catalogName, catalogName, Map.of()); + client, + endpoints, + principalCredentials, + catalogName, + catalogName, + Map.of( + org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", + "catalog-default-key1", + org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", + "catalog-default-key2")); } @AfterEach @@ -157,4 +167,13 @@ protected boolean supportsServerSideRetry() { protected boolean overridesRequestedLocation() { return true; } + + @Test + @Override + public void listViewsInEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ or Polaris supports empty namespaces. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.listViewsInEmptyNamespace(); + } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java index b85443faf..79a925b9f 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/catalog/BasePolarisCatalogViewTest.java @@ -66,8 +66,10 @@ import org.apache.polaris.service.config.RealmEntityManagerFactory; import org.apache.polaris.service.storage.PolarisStorageIntegrationProviderImpl; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assumptions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; import org.junit.jupiter.api.io.TempDir; import org.mockito.Mockito; @@ -199,7 +201,12 @@ public void before(TestInfo testInfo) { this.catalog.initialize( CATALOG_NAME, ImmutableMap.of( - CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.inmemory.InMemoryFileIO")); + CatalogProperties.FILE_IO_IMPL, + "org.apache.iceberg.inmemory.InMemoryFileIO", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key1", + "catalog-default-key1", + CatalogProperties.VIEW_DEFAULT_PREFIX + "key2", + "catalog-default-key2")); } @AfterEach @@ -222,4 +229,13 @@ protected Catalog tableCatalog() { protected boolean requiresNamespaceCreate() { return true; } + + @Test + @Override + public void listViewsInEmptyNamespace() { + // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus. + // This test can be removed once Quarkus supports AssertJ or Polaris supports empty namespaces. + Assumptions.assumeTrue(supportsEmptyNamespace()); + super.listViewsInEmptyNamespace(); + } } From 53b0086e8a7f783fa7a8f1820b27d9874f63fa50 Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Tue, 4 Mar 2025 20:47:05 -0500 Subject: [PATCH 08/11] Fix `createViewWithCustomMetadataLocation` in `ViewCatalogTests` --- .../it/QuarkusRestCatalogViewFileIT.java | 23 ++++++++++++++++--- ...kusRestCatalogViewFileIntegrationTest.java | 19 ++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIT.java b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIT.java index 735973d8e..a38e41f1b 100644 --- a/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIT.java +++ b/quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIT.java @@ -20,21 +20,38 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import java.lang.reflect.Field; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; + import org.apache.iceberg.view.ViewCatalogTests; import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTest; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.AnnotatedElementContext; +import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.api.io.TempDirFactory; @QuarkusIntegrationTest -public class QuarkusRestCatalogViewFileIT - extends PolarisRestCatalogViewFileIntegrationTest { +public class QuarkusRestCatalogViewFileIT extends PolarisRestCatalogViewFileIntegrationTest { @BeforeEach - public void setUpTempDir(@TempDir Path tempDir) throws Exception { + public void setUpTempDir(@TempDir(factory = CustomTempDirFactory.class) Path tempDir) + throws Exception { // see https://github.com/quarkusio/quarkus/issues/13261 Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); field.setAccessible(true); field.set(this, tempDir); } + + private static class CustomTempDirFactory implements TempDirFactory { + @Override + public Path createTempDirectory( + AnnotatedElementContext elementContext, ExtensionContext extensionContext) + throws Exception { + Path basePath = Paths.get(BASE_LOCATION.replaceFirst("file://", "")); + Files.createDirectories(basePath); + return Files.createTempDirectory(basePath, null); + } + } } diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java index 4444bcbee..97bd91b68 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIntegrationTest.java @@ -21,12 +21,17 @@ import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.QuarkusTestProfile; import java.lang.reflect.Field; +import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Map; import org.apache.iceberg.view.ViewCatalogTests; import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTest; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.extension.AnnotatedElementContext; +import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.api.io.TempDirFactory; @QuarkusTest public class QuarkusRestCatalogViewFileIntegrationTest @@ -41,10 +46,22 @@ public Map getConfigOverrides() { } @BeforeEach - public void setUpTempDir(@TempDir Path tempDir) throws Exception { + public void setUpTempDir(@TempDir(factory = CustomTempDirFactory.class) Path tempDir) + throws Exception { // see https://github.com/quarkusio/quarkus/issues/13261 Field field = ViewCatalogTests.class.getDeclaredField("tempDir"); field.setAccessible(true); field.set(this, tempDir); } + + private static class CustomTempDirFactory implements TempDirFactory { + @Override + public Path createTempDirectory( + AnnotatedElementContext elementContext, ExtensionContext extensionContext) + throws Exception { + Path basePath = Paths.get(BASE_LOCATION.replaceFirst("file://", "")); + Files.createDirectories(basePath); + return Files.createTempDirectory(basePath, null); + } + } } From f43ad8dd3db9ea08d6dd90d0b9cc0bb8e6dc84b1 Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Wed, 5 Mar 2025 10:05:39 -0500 Subject: [PATCH 09/11] Fix `testIcebergRestApiRefreshToken` --- .../service/quarkus/it/QuarkusApplicationIntegrationTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java index 8aaac080e..3239d3669 100644 --- a/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java +++ b/quarkus/service/src/test/java/org/apache/polaris/service/quarkus/it/QuarkusApplicationIntegrationTest.java @@ -31,6 +31,7 @@ import org.apache.iceberg.rest.HTTPClient; import org.apache.iceberg.rest.RESTClient; import org.apache.iceberg.rest.auth.AuthConfig; +import org.apache.iceberg.rest.auth.AuthSession; import org.apache.iceberg.rest.auth.OAuth2Util; import org.apache.polaris.service.it.env.ClientCredentials; import org.apache.polaris.service.it.env.PolarisApiEndpoints; @@ -61,6 +62,7 @@ public void testIcebergRestApiRefreshToken( HTTPClient.builder(Map.of()) .withHeader(endpoints.realmHeaderName(), endpoints.realmId()) .uri(path) + .withAuthSession(AuthSession.EMPTY) .build()) { String credentialString = clientCredentials.clientId() + ":" + clientCredentials.clientSecret(); From 99535bd193791749c9c28cfb5ff341832785f7c1 Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Thu, 6 Mar 2025 10:09:28 -0500 Subject: [PATCH 10/11] Upgrade Iceberg to 1.8.1 --- gradle/libs.versions.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b331b2dbf..0ab6a807b 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,7 +19,7 @@ [versions] hadoop = "3.4.1" -iceberg = "1.8.0" +iceberg = "1.8.1" quarkus = "3.19.1" immutables = "2.10.1" picocli = "4.7.6" From 15cea732e74c25233ce52196070183f3ad6c4297 Mon Sep 17 00:00:00 2001 From: Liam Bao Date: Thu, 6 Mar 2025 10:18:51 -0500 Subject: [PATCH 11/11] Fix regtests --- regtests/t_spark_sql/ref/spark_sql_basic.sh.ref | 2 +- regtests/t_spark_sql/ref/spark_sql_views.sh.ref | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref b/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref index a23d1d941..4cd8ce0f8 100755 --- a/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref +++ b/regtests/t_spark_sql/ref/spark_sql_basic.sh.ref @@ -1,4 +1,4 @@ -{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_basic_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]} +{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_basic_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","POST /v1/{prefix}/transactions/commit","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","HEAD /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]} Catalog created spark-sql (default)> use polaris; spark-sql ()> show namespaces; diff --git a/regtests/t_spark_sql/ref/spark_sql_views.sh.ref b/regtests/t_spark_sql/ref/spark_sql_views.sh.ref index 8e4bff270..853c736db 100755 --- a/regtests/t_spark_sql/ref/spark_sql_views.sh.ref +++ b/regtests/t_spark_sql/ref/spark_sql_views.sh.ref @@ -1,4 +1,4 @@ -{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_views_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]} +{"defaults":{"default-base-location":"file:///tmp/spark_sql_s3_catalog"},"overrides":{"prefix":"spark_sql_views_catalog"},"endpoints":["GET /v1/{prefix}/namespaces","GET /v1/{prefix}/namespaces/{namespace}","HEAD /v1/{prefix}/namespaces/{namespace}","POST /v1/{prefix}/namespaces","POST /v1/{prefix}/namespaces/{namespace}/properties","DELETE /v1/{prefix}/namespaces/{namespace}","GET /v1/{prefix}/namespaces/{namespace}/tables","GET /v1/{prefix}/namespaces/{namespace}/tables/{table}","HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/namespaces/{namespace}/tables","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}","DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}","POST /v1/{prefix}/tables/rename","POST /v1/{prefix}/namespaces/{namespace}/register","POST /v1/{prefix}/namespaces/{namespace}/tables/{table}/metrics","POST /v1/{prefix}/transactions/commit","GET /v1/{prefix}/namespaces/{namespace}/views","GET /v1/{prefix}/namespaces/{namespace}/views/{view}","HEAD /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/namespaces/{namespace}/views","POST /v1/{prefix}/namespaces/{namespace}/views/{view}","DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}","POST /v1/{prefix}/views/rename","POST /v1/{prefix}/transactions/commit"]} Catalog created spark-sql (default)> use polaris; spark-sql ()> show namespaces;