WIP: Test Upcoming Iceberg and Nessie version

- Testing with Iceberg 1.5.0-SNAPSHOT and Nessie 0.76.0 - Remove deprecated usage
trinodb · Jan 9, 2024 · bc1337a · bc1337a
1 parent 6021f33
commit bc1337a
Show file tree

Hide file tree

Showing 7 changed files with 50 additions and 14 deletions.
diff --git a/plugin/trino-iceberg/pom.xml b/plugin/trino-iceberg/pom.xml
@@ -16,7 +16,7 @@
     <properties>
         <air.main.basedir>${project.parent.basedir}</air.main.basedir>
         <!-- Nessie version (matching to Iceberg release) must be bumped along with Iceberg version bump to avoid compatibility issues -->
-        <dep.nessie.version>0.71.0</dep.nessie.version>
+        <dep.nessie.version>0.76.0</dep.nessie.version>
     </properties>
 
     <dependencies>
@@ -239,12 +239,32 @@
             <groupId>org.projectnessie.nessie</groupId>
             <artifactId>nessie-client</artifactId>
             <version>${dep.nessie.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-annotations</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
             <groupId>org.projectnessie.nessie</groupId>
             <artifactId>nessie-model</artifactId>
             <version>${dep.nessie.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-annotations</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>com.fasterxml.jackson.core</groupId>
+                    <artifactId>jackson-databind</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
@@ -344,14 +364,14 @@
         <dependency>
             <groupId>org.apache.httpcomponents.client5</groupId>
             <artifactId>httpclient5</artifactId>
-            <version>5.2.1</version>
+            <version>5.3</version>
             <scope>runtime</scope>
         </dependency>
 
         <dependency>
             <groupId>org.apache.httpcomponents.core5</groupId>
             <artifactId>httpcore5</artifactId>
-            <version>5.2.1</version>
+            <version>5.2.4</version>
             <scope>runtime</scope>
         </dependency>
 

diff --git a/...berg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieCatalogModule.java b/...berg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieCatalogModule.java
@@ -22,9 +22,9 @@
 import io.trino.plugin.iceberg.catalog.IcebergTableOperationsProvider;
 import io.trino.plugin.iceberg.catalog.TrinoCatalogFactory;
 import org.apache.iceberg.nessie.NessieIcebergClient;
+import org.projectnessie.client.NessieClientBuilder;
 import org.projectnessie.client.api.NessieApiV1;
 import org.projectnessie.client.auth.BearerAuthenticationProvider;
-import org.projectnessie.client.http.HttpClientBuilder;
 
 import static io.airlift.configuration.ConfigBinder.configBinder;
 import static java.lang.Math.toIntExact;
@@ -47,7 +47,7 @@ protected void setup(Binder binder)
     @Singleton
     public static NessieIcebergClient createNessieIcebergClient(IcebergNessieCatalogConfig icebergNessieCatalogConfig)
     {
-        HttpClientBuilder builder = HttpClientBuilder.builder()
+        NessieClientBuilder builder = NessieClientBuilder.createClientBuilderFromSystemSettings()
                 .withUri(icebergNessieCatalogConfig.getServerUri())
                 .withDisableCompression(!icebergNessieCatalogConfig.isCompressionEnabled())
                 .withReadTimeout(toIntExact(icebergNessieCatalogConfig.getReadTimeout().toMillis()))

diff --git a/...rg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieTableOperations.java b/...rg/src/main/java/io/trino/plugin/iceberg/catalog/nessie/IcebergNessieTableOperations.java
@@ -95,7 +95,7 @@ protected void refreshFromMetadataLocation(String newLocation)
     @Override
     protected String getRefreshedLocation(boolean invalidateCaches)
     {
-        table = nessieClient.table(toIdentifier(new SchemaTableName(database, tableName)));
+        table = (IcebergTable) nessieClient.fetchContent(toIdentifier(new SchemaTableName(database, tableName)));
 
         if (table == null) {
             throw new TableNotFoundException(getSchemaTableName());
@@ -109,7 +109,8 @@ protected void commitNewTable(TableMetadata metadata)
     {
         verify(version.isEmpty(), "commitNewTable called on a table which already exists");
         try {
-            nessieClient.commitTable(null, metadata, writeNewMetadata(metadata, 0), table, toKey(new SchemaTableName(database, this.tableName)));
+            nessieClient.commitTable(null, metadata, writeNewMetadata(metadata, 0), table.getId(),
+                    toKey(new SchemaTableName(database, this.tableName)));
         }
         catch (NessieNotFoundException e) {
             throw new TrinoException(ICEBERG_COMMIT_ERROR, format("Cannot commit: ref '%s' no longer exists", nessieClient.refName()), e);
@@ -126,7 +127,8 @@ protected void commitToExistingTable(TableMetadata base, TableMetadata metadata)
     {
         verify(version.orElseThrow() >= 0, "commitToExistingTable called on a new table");
         try {
-            nessieClient.commitTable(base, metadata, writeNewMetadata(metadata, version.getAsInt() + 1), table, toKey(new SchemaTableName(database, this.tableName)));
+            nessieClient.commitTable(base, metadata, writeNewMetadata(metadata, version.getAsInt() + 1), table.getId(),
+                    toKey(new SchemaTableName(database, this.tableName)));
         }
         catch (NessieNotFoundException e) {
             throw new TrinoException(ICEBERG_COMMIT_ERROR, format("Cannot commit: ref '%s' no longer exists", nessieClient.refName()), e);

diff --git a/...-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java b/...-iceberg/src/test/java/io/trino/plugin/iceberg/catalog/nessie/TestTrinoNessieCatalog.java
@@ -36,8 +36,8 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.parallel.Execution;
+import org.projectnessie.client.NessieClientBuilder;
 import org.projectnessie.client.api.NessieApiV1;
-import org.projectnessie.client.http.HttpClientBuilder;
 
 import java.io.File;
 import java.io.IOException;
@@ -94,7 +94,7 @@ protected TrinoCatalog createTrinoCatalog(boolean useUniqueTableLocations)
         TrinoFileSystemFactory fileSystemFactory = new HdfsFileSystemFactory(HDFS_ENVIRONMENT, HDFS_FILE_SYSTEM_STATS);
         IcebergNessieCatalogConfig icebergNessieCatalogConfig = new IcebergNessieCatalogConfig()
                 .setServerUri(URI.create(nessieContainer.getRestApiUri()));
-        NessieApiV1 nessieApi = HttpClientBuilder.builder()
+        NessieApiV1 nessieApi = NessieClientBuilder.createClientBuilderFromSystemSettings()
                 .withUri(nessieContainer.getRestApiUri())
                 .build(NessieApiV1.class);
         NessieIcebergClient nessieClient = new NessieIcebergClient(nessieApi, icebergNessieCatalogConfig.getDefaultReferenceName(), null, ImmutableMap.of());
@@ -118,7 +118,7 @@ public void testDefaultLocation()
         IcebergNessieCatalogConfig icebergNessieCatalogConfig = new IcebergNessieCatalogConfig()
                 .setDefaultWarehouseDir(tmpDirectory.toAbsolutePath().toString())
                 .setServerUri(URI.create(nessieContainer.getRestApiUri()));
-        NessieApiV1 nessieApi = HttpClientBuilder.builder()
+        NessieApiV1 nessieApi = NessieClientBuilder.createClientBuilderFromSystemSettings()
                 .withUri(nessieContainer.getRestApiUri())
                 .build(NessieApiV1.class);
         NessieIcebergClient nessieClient = new NessieIcebergClient(nessieApi, icebergNessieCatalogConfig.getDefaultReferenceName(), null, ImmutableMap.of());

diff --git a/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java b/plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/containers/NessieContainer.java
@@ -28,7 +28,7 @@ public class NessieContainer
 {
     private static final Logger log = Logger.get(NessieContainer.class);
 
-    public static final String DEFAULT_IMAGE = "projectnessie/nessie:0.71.0";
+    public static final String DEFAULT_IMAGE = "projectnessie/nessie:0.76.0";
     public static final String DEFAULT_HOST_NAME = "nessie";
     public static final String VERSION_STORE_TYPE = "INMEMORY";
 

diff --git a/pom.xml b/pom.xml
@@ -185,7 +185,7 @@
         <dep.errorprone.version>2.23.0</dep.errorprone.version>
         <dep.flyway.version>10.4.1</dep.flyway.version>
         <dep.google.http.client.version>1.43.3</dep.google.http.client.version>
-        <dep.iceberg.version>1.4.3</dep.iceberg.version>
+        <dep.iceberg.version>1.5.0-SNAPSHOT</dep.iceberg.version>
         <dep.jna.version>5.14.0</dep.jna.version>
         <dep.joda.version>2.12.6</dep.joda.version>
         <dep.jsonwebtoken.version>0.12.3</dep.jsonwebtoken.version>
@@ -2113,6 +2113,20 @@
         </dependencies>
     </dependencyManagement>
 
+    <repositories>
+        <repository>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+            <id>apache.snapshots</id>
+            <name>Apache Development Snapshot Repository</name>
+            <url>https://repository.apache.org/content/repositories/snapshots/</url>
+        </repository>
+    </repositories>
+
     <build>
         <pluginManagement>
             <plugins>

diff --git a/...java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java b/...java/io/trino/tests/product/launcher/env/environment/EnvSinglenodeSparkIcebergNessie.java
@@ -37,7 +37,7 @@ public class EnvSinglenodeSparkIcebergNessie
 {
     private static final int SPARK_THRIFT_PORT = 10213;
     private static final int NESSIE_PORT = 19120;
-    private static final String NESSIE_VERSION = "0.71.0";
+    private static final String NESSIE_VERSION = "0.76.0";
     private static final String SPARK = "spark";
 
     private final DockerFiles dockerFiles;