Update supported clickhouse versions

docs/src/main/sphinx/connector/clickhouse.md

@@ -19,7 +19,7 @@ from different catalogs accessing ClickHouse or any other supported data source.
 
 To connect to a ClickHouse server, you need:
 
-- ClickHouse (version 23.8 or higher) or Altinity (version 21.8 or higher).
+- ClickHouse (version 24.3 or higher) or Altinity (version 22.3 or higher).
 - Network access from the Trino coordinator and workers to the ClickHouse
   server. Port 8123 is the default port.
 

plugin/trino-clickhouse/pom.xml

@@ -21,7 +21,6 @@
         <dependency>
             <groupId>com.clickhouse</groupId>
             <artifactId>clickhouse-jdbc</artifactId>
-            <classifier>all</classifier>
         </dependency>
 
         <dependency>
@@ -113,6 +112,14 @@
             <scope>runtime</scope>
         </dependency>
 
+        <dependency>
+            <!-- Clickhouse client will by default use LZ4 compression, which requires this dependency -->
+            <!-- https://github.com/ClickHouse/clickhouse-docs/pull/2408/files -->
+            <groupId>org.lz4</groupId>
+            <artifactId>lz4-java</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
         <dependency>
             <groupId>io.airlift</groupId>
             <artifactId>junit-extensions</artifactId>

plugin/trino-clickhouse/src/main/java/io/trino/plugin/clickhouse/ClickHouseClientModule.java

@@ -35,6 +35,7 @@
 import java.util.Properties;
 
 import static com.clickhouse.client.config.ClickHouseClientOption.USE_BINARY_STRING;
+import static com.clickhouse.jdbc.JdbcConfig.PROP_EXTERNAL_DATABASE;
 import static com.google.inject.multibindings.Multibinder.newSetBinder;
 import static io.airlift.configuration.ConfigBinder.configBinder;
 import static io.trino.plugin.clickhouse.ClickHouseClient.DEFAULT_DOMAIN_COMPACTION_THRESHOLD;
@@ -64,6 +65,11 @@ public static ConnectionFactory createConnectionFactory(BaseJdbcConfig config, C
         Properties properties = new Properties();
         // The connector expects byte array for FixedString and String types
         properties.setProperty(USE_BINARY_STRING.getKey(), "true");
+        // externalDatabase=false is needed because Schema listing fetch is extremely slow on Clickhouse-server 24.3+
+        // https://github.com/ClickHouse/clickhouse-java/issues/1245
+        // https://github.com/ClickHouse/clickhouse-java/issues/1584
+        // in Clickhouse itself it has been left `true` by default only for backward compatibility.
+        properties.setProperty(PROP_EXTERNAL_DATABASE, "false");
         return new ClickHouseConnectionFactory(DriverConnectionFactory.builder(new ClickHouseDriver(), config.getConnectionUrl(), credentialProvider)
                 .setConnectionProperties(properties)
                 .setOpenTelemetry(openTelemetry)

plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java

@@ -709,24 +709,11 @@ protected TestTable simpleTable()
         return new TestTable(onRemoteDatabase(), "tpch.simple_table", "(col BIGINT) Engine=Log", ImmutableList.of("1", "2"));
     }
 
-    @Test
     @Override
-    public void testCreateTableWithLongTableName()
+    protected void verifyTableNameLengthFailurePermissible(Throwable e)
     {
-        // Override because ClickHouse connector can create a table which can't be dropped
-        String baseTableName = "test_create_" + randomNameSuffix();
-        String validTableName = baseTableName + "z".repeat(maxTableNameLength().orElseThrow() - baseTableName.length());
-
-        assertUpdate("CREATE TABLE " + validTableName + " (a bigint)");
-        assertThat(getQueryRunner().tableExists(getSession(), validTableName)).isTrue();
-        assertThatThrownBy(() -> assertUpdate("DROP TABLE " + validTableName))
-                .hasMessageMatching("(?s).*(Bad path syntax|File name too long).*");
-
-        String invalidTableName = baseTableName + "z".repeat(maxTableNameLength().orElseThrow() - baseTableName.length() + 1);
-        assertThat(query("CREATE TABLE " + invalidTableName + " (a bigint)"))
-                .failure().hasMessageMatching("(?s).*(Cannot open file|File name too long).*");
-        // ClickHouse lefts a table even if the above statement failed
-        assertThat(getQueryRunner().tableExists(getSession(), validTableName)).isTrue();
+        assertThat(e).hasMessageMatching(".*The max length of table name for database tpch is %d, current length is [0-9]+.*\n"
+                .formatted(maxTableNameLength().orElseThrow()));
     }
 
     @Test
@@ -767,31 +754,6 @@ protected void verifySchemaNameLengthFailurePermissible(Throwable e)
         assertThat(e).hasMessageContaining("File name too long");
     }
 
-    @Test
-    @Override
-    public void testRenameTableToLongTableName()
-    {
-        // Override because ClickHouse connector can rename to a table which can't be dropped
-        String sourceTableName = "test_source_long_table_name_" + randomNameSuffix();
-        assertUpdate("CREATE TABLE " + sourceTableName + " AS SELECT 123 x", 1);
-
-        String baseTableName = "test_target_long_table_name_" + randomNameSuffix();
-        // The max length is different from CREATE TABLE case
-        String validTargetTableName = baseTableName + "z".repeat(255 - ".sql".length() - baseTableName.length());
-
-        assertUpdate("ALTER TABLE " + sourceTableName + " RENAME TO " + validTargetTableName);
-        assertThat(getQueryRunner().tableExists(getSession(), validTargetTableName)).isTrue();
-        assertQuery("SELECT x FROM " + validTargetTableName, "VALUES 123");
-        assertThatThrownBy(() -> assertUpdate("DROP TABLE " + validTargetTableName))
-                .hasMessageMatching("(?s).*(Bad path syntax|File name too long).*");
-
-        assertUpdate("CREATE TABLE " + sourceTableName + " AS SELECT 123 x", 1);
-        String invalidTargetTableName = validTargetTableName + "z";
-        assertThatThrownBy(() -> assertUpdate("ALTER TABLE " + sourceTableName + " RENAME TO " + invalidTargetTableName))
-                .hasMessageMatching("(?s).*(Cannot rename|File name too long).*");
-        assertThat(getQueryRunner().tableExists(getSession(), invalidTargetTableName)).isFalse();
-    }
-
     @Test
     @Override // Override because the failure message differs
     public void testNativeQueryIncorrectSyntax()
@@ -1179,7 +1141,7 @@ public void testExecuteProcedureWithInvalidQuery()
     protected OptionalInt maxTableNameLength()
     {
         // The numeric value depends on file system
-        return OptionalInt.of(255 - ".sql.detached".length());
+        return OptionalInt.of(209);
     }
 
     @Override

plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestingClickHouseServer.java

@@ -27,15 +27,30 @@
 public class TestingClickHouseServer
         implements Closeable
 {
+    /**
+     * <a href="https://clickhouse.com/docs/en/faq/operations/production#how-to-choose-between-clickhouse-releases">How to Choose Between ClickHouse Releases?</a>
+     * <p>
+     * stable is the kind of package we recommend by default.
+     * They are released roughly monthly (and thus provide new features with reasonable delay)
+     * and three latest stable releases are supported in terms of diagnostics and backporting of bugfixes.
+     * lts are released twice a year and are supported for a year after their initial release.
+     * <p>
+     * <a href="https://kb.altinity.com/altinity-kb-setup-and-maintenance/clickhouse-versions/">Versioning schema</a>
+     */
     private static final DockerImageName CLICKHOUSE_IMAGE = DockerImageName.parse("clickhouse/clickhouse-server");
-    public static final DockerImageName CLICKHOUSE_LATEST_IMAGE = CLICKHOUSE_IMAGE.withTag("24.1.8.22");   // EOL by Apr 2025
-    public static final DockerImageName CLICKHOUSE_DEFAULT_IMAGE = CLICKHOUSE_IMAGE.withTag("23.8.12.13"); // EOL by Jun 2024
+    // https://clickhouse.com/docs/en/whats-new/changelog#-clickhouse-release-2412-2024-12-19
+    public static final DockerImageName CLICKHOUSE_LATEST_IMAGE = CLICKHOUSE_IMAGE.withTag("24.12.1.1614");   // EOL in 3 releases after 2024-12-19
+    // https://clickhouse.com/docs/en/whats-new/changelog#-clickhouse-release-243-lts-2024-03-27
+    public static final DockerImageName CLICKHOUSE_DEFAULT_IMAGE = CLICKHOUSE_IMAGE.withTag("24.3.14.35"); // EOL in 1 year after 2024-03-27
 
-    // Altinity Stable Builds Life-Cycle Table https://docs.altinity.com/altinitystablebuilds/#altinity-stable-builds-life-cycle-table
-    // On Mac/arm try `21.8.12.29.altinitydev.arm` instead of the specified stable build
+    /**
+     * <a href="https://docs.altinity.com/altinitystablebuilds/#altinity-stable-builds-life-cycle-table">Altinity Stable Builds Life-Cycle Table</a>
+     * <p>
+     * On Mac/arm 23.3.13.7.altinitystable, 23.8.8.21.altinitystable and 22.8.15.25.altinitystable and later versions available on ARM.
+     */
     private static final DockerImageName ALTINITY_IMAGE = DockerImageName.parse("altinity/clickhouse-server").asCompatibleSubstituteFor("clickhouse/clickhouse-server");
-    public static final DockerImageName ALTINITY_LATEST_IMAGE = ALTINITY_IMAGE.withTag("23.8.8.21.altinitystable");  // EOL is 27 Dec 2026
-    public static final DockerImageName ALTINITY_DEFAULT_IMAGE = ALTINITY_IMAGE.withTag("21.8.15.15.altinitystable"); // EOL is 30 Aug 2024
+    public static final DockerImageName ALTINITY_LATEST_IMAGE = ALTINITY_IMAGE.withTag("24.3.12.76.altinitystable");  // EOL is 23 Jul 2027
+    public static final DockerImageName ALTINITY_DEFAULT_IMAGE = ALTINITY_IMAGE.withTag("22.3.15.34.altinitystable"); // EOL is 15 Jul 2025
 
     private final ClickHouseContainer dockerContainer;
 

pom.xml

@@ -462,8 +462,7 @@
             <dependency>
                 <groupId>com.clickhouse</groupId>
                 <artifactId>clickhouse-jdbc</artifactId>
-                <version>0.6.3</version>
-                <classifier>all</classifier>
+                <version>0.7.1-patch1</version>
             </dependency>
 
             <dependency>
@@ -2282,6 +2281,14 @@
                 </exclusions>
             </dependency>
 
+            <dependency>
+                <!-- Clickhouse client will by default use LZ4 compression, which requires this dependency -->
+                <!-- https://github.com/ClickHouse/clickhouse-docs/pull/2408/files -->
+                <groupId>org.lz4</groupId>
+                <artifactId>lz4-java</artifactId>
+                <version>1.8.0</version>
+            </dependency>
+
             <dependency>
                 <groupId>org.mariadb.jdbc</groupId>
                 <artifactId>mariadb-java-client</artifactId>
@@ -2602,6 +2609,13 @@
                     <groupId>org.basepom.maven</groupId>
                     <artifactId>duplicate-finder-maven-plugin</artifactId>
                     <configuration>
+                        <ignoredDependencies>
+                            <ignoredDependency>
+                                <!-- clickhouse-jdbc contains conflicting dependencies inside itself -->
+                                <groupId>com.clickhouse</groupId>
+                                <artifactId>clickhouse-jdbc</artifactId>
+                            </ignoredDependency>
+                        </ignoredDependencies>
                         <exceptions combine.children="append">
                             <exception>
                                 <conflictingDependencies>

testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeClickhouse.java

@@ -23,6 +23,7 @@
 import io.trino.tests.product.launcher.env.common.TestsEnvironment;
 import io.trino.tests.product.launcher.testcontainers.PortBinder;
 import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy;
+import org.testcontainers.utility.DockerImageName;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -47,6 +48,8 @@ public class EnvMultinodeClickhouse
 {
     private static final String ZOOKEEPER = "zookeeper";
 
+    private static final DockerImageName CLICKHOUSE_IMAGE = DockerImageName.parse("clickhouse/clickhouse-server"); // see TestingClickHouseServer for details
+    private static final DockerImageName CLICKHOUSE_DEFAULT_IMAGE = CLICKHOUSE_IMAGE.withTag("24.3.14.35"); // EOL in 1 year after 2024-03-27
     private static final String CLICKHOUSE = "clickhouse";
     private static final String CLICKHOUSE_NTH = CLICKHOUSE + "-";
     private static final String CONTAINER_CLICKHOUSE_CONFIG_DIR = "/etc/clickhouse-server/";
@@ -109,7 +112,7 @@ private static DockerContainer createClickHouse(int number, DockerFiles dockerFi
         int httpPort = CLICKHOUSE_DEFAULT_HTTP_PORT + number;
         int nativePort = CLICKHOUSE_DEFAULT_NATIVE_PORT + number;
 
-        DockerContainer container = new DockerContainer("yandex/clickhouse-server:21.3.2.5", logicalName(number))
+        DockerContainer container = new DockerContainer(CLICKHOUSE_DEFAULT_IMAGE.toString(), logicalName(number))
                 .withCopyFileToContainer(
                         forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-clickhouse/test.xml")),
                         CONTAINER_CLICKHOUSE_USERS_D + "test.xml")

testing/trino-product-tests/pom.xml

@@ -230,7 +230,6 @@
         <dependency>
             <groupId>com.clickhouse</groupId>
             <artifactId>clickhouse-jdbc</artifactId>
-            <classifier>all</classifier>
             <scope>runtime</scope>
         </dependency>
 
@@ -284,6 +283,14 @@
             <scope>runtime</scope>
         </dependency>
 
+        <dependency>
+            <!-- Clickhouse client will by default use LZ4 compression, which requires this dependency -->
+            <!-- https://github.com/ClickHouse/clickhouse-docs/pull/2408/files -->
+            <groupId>org.lz4</groupId>
+            <artifactId>lz4-java</artifactId>
+            <scope>runtime</scope>
+        </dependency>
+
         <dependency>
             <groupId>org.mariadb.jdbc</groupId>
             <artifactId>mariadb-java-client</artifactId>