Upgrade Iceberg to 1.8

gradle/libs.versions.toml

@@ -19,7 +19,7 @@
 
 [versions]
 hadoop = "3.4.1"
-iceberg = "1.7.1"
+iceberg = "1.8.1"
 quarkus = "3.19.1"
 immutables = "2.10.1"
 picocli = "4.7.6"

integration-tests/src/main/java/org/apache/polaris/service/it/env/IcebergHelper.java

@@ -34,6 +34,7 @@ public static RESTCatalog restCatalog(
       PolarisApiEndpoints endpoints,
       PrincipalWithCredentials credentials,
       String catalog,
+      String warehouse,
       Map<String, String> extraProperties) {
     String authToken = client.obtainToken(credentials);
     SessionCatalog.SessionContext context = SessionCatalog.SessionContext.createEmpty();
@@ -53,11 +54,11 @@ public static RESTCatalog restCatalog(
             .put(
                 org.apache.iceberg.CatalogProperties.FILE_IO_IMPL,
                 "org.apache.iceberg.inmemory.InMemoryFileIO")
-            .put("warehouse", catalog)
+            .put("warehouse", warehouse)
             .put("header." + endpoints.realmHeaderName(), endpoints.realmId())
             .putAll(extraProperties);
 
-    restCatalog.initialize("polaris", propertiesBuilder.buildKeepingLast());
+    restCatalog.initialize(catalog, propertiesBuilder.buildKeepingLast());
     return restCatalog;
   }
 }

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisApplicationIntegrationTest.java

@@ -459,8 +459,7 @@ public void testIcebergRegisterTableInExternalCatalog() throws IOException {
               .assignUUID()
               .addPartitionSpec(PartitionSpec.unpartitioned())
               .addSortOrder(SortOrder.unsorted())
-              .addSchema(
-                  new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())), 1)
+              .addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())))
               .build();
       TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation));
 
@@ -503,7 +502,7 @@ public void testIcebergUpdateTableInExternalCatalog() throws IOException {
               .assignUUID()
               .addPartitionSpec(PartitionSpec.unpartitioned())
               .addSortOrder(SortOrder.unsorted())
-              .addSchema(new Schema(col1), 1)
+              .addSchema(new Schema(col1))
               .build();
       TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation));
 
@@ -551,8 +550,7 @@ public void testIcebergDropTableInExternalCatalog() throws IOException {
               .assignUUID()
               .addPartitionSpec(PartitionSpec.unpartitioned())
               .addSortOrder(SortOrder.unsorted())
-              .addSchema(
-                  new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())), 1)
+              .addSchema(new Schema(Types.NestedField.of(1, false, "col1", Types.StringType.get())))
               .build();
       TableMetadataParser.write(tableMetadata, fileIo.newOutputFile(metadataLocation));
 

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java

@@ -90,6 +90,7 @@
 import org.assertj.core.api.InstanceOfAssertFactories;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -117,16 +118,16 @@ public class PolarisRestCatalogIntegrationTest extends CatalogTests<RESTCatalog>
   private static URI externalCatalogBase;
 
   protected static final String VIEW_QUERY = "select * from ns1.layer1_table";
-  private static String principalRoleName;
   private static ClientCredentials adminCredentials;
-  private static PrincipalWithCredentials principalCredentials;
   private static PolarisApiEndpoints endpoints;
   private static PolarisClient client;
   private static ManagementApi managementApi;
-  private static CatalogApi catalogApi;
 
+  private PrincipalWithCredentials principalCredentials;
+  private CatalogApi catalogApi;
   private RESTCatalog restCatalog;
   private String currentCatalogName;
+  private TestInfo testInfo;
 
   private final String catalogBaseLocation =
       s3BucketBase + "/" + System.getenv("USER") + "/path/to/data";
@@ -158,10 +159,6 @@ static void setup(
     endpoints = apiEndpoints;
     client = polarisClient(endpoints);
     managementApi = client.managementApi(credentials);
-    String principalName = client.newEntityName("snowman-rest");
-    principalRoleName = client.newEntityName("rest-admin");
-    principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName);
-    catalogApi = client.catalogApi(principalCredentials);
     URI testRootUri = IntegrationTestsHelper.getTemporaryDirectory(tempDir);
     s3BucketBase = testRootUri.resolve("my-bucket");
     externalCatalogBase = testRootUri.resolve("external-catalog");
@@ -174,10 +171,10 @@ static void close() throws Exception {
 
   @BeforeEach
   public void before(TestInfo testInfo) {
+    this.testInfo = testInfo;
     String principalName = "snowman-rest-" + UUID.randomUUID();
-    principalRoleName = "rest-admin-" + UUID.randomUUID();
-    PrincipalWithCredentials principalCredentials =
-        managementApi.createPrincipalWithRole(principalName, principalRoleName);
+    String principalRoleName = "rest-admin-" + UUID.randomUUID();
+    principalCredentials = managementApi.createPrincipalWithRole(principalName, principalRoleName);
 
     catalogApi = client.catalogApi(principalCredentials);
 
@@ -218,6 +215,21 @@ public void before(TestInfo testInfo) {
 
     managementApi.createCatalog(principalRoleName, catalog);
 
+    restCatalog = initCatalog(currentCatalogName, ImmutableMap.of());
+  }
+
+  @AfterEach
+  public void cleanUp() {
+    client.cleanUp(adminCredentials);
+  }
+
+  @Override
+  protected RESTCatalog catalog() {
+    return restCatalog;
+  }
+
+  @Override
+  protected RESTCatalog initCatalog(String catalogName, Map<String, String> additionalProperties) {
     Optional<PolarisRestCatalogIntegrationTest.RestCatalogConfig> restCatalogConfig =
         testInfo
             .getTestMethod()
@@ -233,24 +245,14 @@ public void before(TestInfo testInfo) {
             extraPropertiesBuilder.put(config.value()[i], config.value()[i + 1]);
           }
         });
-
-    restCatalog =
-        IcebergHelper.restCatalog(
-            client,
-            endpoints,
-            principalCredentials,
-            currentCatalogName,
-            extraPropertiesBuilder.build());
-  }
-
-  @AfterEach
-  public void cleanUp() {
-    client.cleanUp(adminCredentials);
-  }
-
-  @Override
-  protected RESTCatalog catalog() {
-    return restCatalog;
+    extraPropertiesBuilder.putAll(additionalProperties);
+    return IcebergHelper.restCatalog(
+        client,
+        endpoints,
+        principalCredentials,
+        catalogName,
+        currentCatalogName,
+        extraPropertiesBuilder.buildKeepingLast());
   }
 
   @Override
@@ -273,6 +275,33 @@ protected boolean overridesRequestedLocation() {
     return true;
   }
 
+  @Test
+  @Override
+  public void createAndDropEmptyNamespace() {
+    // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus.
+    // This test can be removed once Quarkus supports AssertJ.
+    Assumptions.assumeTrue(supportsEmptyNamespace());
+    super.createAndDropEmptyNamespace();
+  }
+
+  @Test
+  @Override
+  public void namespacePropertiesOnEmptyNamespace() {
+    // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus.
+    // This test can be removed once Quarkus supports AssertJ.
+    Assumptions.assumeTrue(supportsEmptyNamespace());
+    super.namespacePropertiesOnEmptyNamespace();
+  }
+
+  @Test
+  @Override
+  public void listTablesInEmptyNamespace() {
+    // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus.
+    // This test can be removed once Quarkus supports AssertJ.
+    Assumptions.assumeTrue(supportsEmptyNamespace());
+    super.listTablesInEmptyNamespace();
+  }
+
   @Test
   public void testListGrantsOnCatalogObjectsToCatalogRoles() {
     restCatalog.createNamespace(Namespace.of("ns1"));

integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogViewIntegrationBase.java

@@ -42,6 +42,7 @@
 import org.junit.jupiter.api.Assumptions;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInfo;
 import org.junit.jupiter.api.extension.ExtendWith;
 
@@ -113,7 +114,17 @@ public void before(TestInfo testInfo) {
     managementApi.createCatalog(principalRoleName, catalog);
 
     restCatalog =
-        IcebergHelper.restCatalog(client, endpoints, principalCredentials, catalogName, Map.of());
+        IcebergHelper.restCatalog(
+            client,
+            endpoints,
+            principalCredentials,
+            catalogName,
+            catalogName,
+            Map.of(
+                org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key1",
+                "catalog-default-key1",
+                org.apache.iceberg.CatalogProperties.VIEW_DEFAULT_PREFIX + "key2",
+                "catalog-default-key2"));
   }
 
   @AfterEach
@@ -156,4 +167,13 @@ protected boolean supportsServerSideRetry() {
   protected boolean overridesRequestedLocation() {
     return true;
   }
+
+  @Test
+  @Override
+  public void listViewsInEmptyNamespace() {
+    // Skip this test because AssertJ's Assumptions.assumeThat() is not compatible with Quarkus.
+    // This test can be removed once Quarkus supports AssertJ or Polaris supports empty namespaces.
+    Assumptions.assumeTrue(supportsEmptyNamespace());
+    super.listViewsInEmptyNamespace();
+  }
 }

quarkus/service/src/intTest/java/org/apache/polaris/service/quarkus/it/QuarkusRestCatalogViewFileIT.java

@@ -20,21 +20,38 @@
 
 import io.quarkus.test.junit.QuarkusIntegrationTest;
 import java.lang.reflect.Field;
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
+
 import org.apache.iceberg.view.ViewCatalogTests;
 import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTest;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.AnnotatedElementContext;
+import org.junit.jupiter.api.extension.ExtensionContext;
 import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.api.io.TempDirFactory;
 
 @QuarkusIntegrationTest
-public class QuarkusRestCatalogViewFileIT
-    extends PolarisRestCatalogViewFileIntegrationTest {
+public class QuarkusRestCatalogViewFileIT extends PolarisRestCatalogViewFileIntegrationTest {
 
   @BeforeEach
-  public void setUpTempDir(@TempDir Path tempDir) throws Exception {
+  public void setUpTempDir(@TempDir(factory = CustomTempDirFactory.class) Path tempDir)
+      throws Exception {
     // see https://github.com/quarkusio/quarkus/issues/13261
     Field field = ViewCatalogTests.class.getDeclaredField("tempDir");
     field.setAccessible(true);
     field.set(this, tempDir);
   }
+
+  private static class CustomTempDirFactory implements TempDirFactory {
+    @Override
+    public Path createTempDirectory(
+        AnnotatedElementContext elementContext, ExtensionContext extensionContext)
+        throws Exception {
+      Path basePath = Paths.get(BASE_LOCATION.replaceFirst("file://", ""));
+      Files.createDirectories(basePath);
+      return Files.createTempDirectory(basePath, null);
+    }
+  }
 }