From 6f3ed92d6ae707b82e2a3337cc2238c540d40f85 Mon Sep 17 00:00:00 2001 From: Sasha Sheikin Date: Wed, 18 Dec 2024 12:35:13 +0100 Subject: [PATCH] Update supported clickhouse versions --- docs/src/main/sphinx/connector/clickhouse.md | 2 +- plugin/trino-clickhouse/pom.xml | 9 +++- .../TestClickHouseConnectorTest.java | 46 ++----------------- .../clickhouse/TestingClickHouseServer.java | 23 +++++++--- pom.xml | 18 +++++++- .../environment/EnvMultinodeClickhouse.java | 5 +- testing/trino-product-tests/pom.xml | 9 +++- 7 files changed, 58 insertions(+), 54 deletions(-) diff --git a/docs/src/main/sphinx/connector/clickhouse.md b/docs/src/main/sphinx/connector/clickhouse.md index 0da7a863bb28..4164f1647f27 100644 --- a/docs/src/main/sphinx/connector/clickhouse.md +++ b/docs/src/main/sphinx/connector/clickhouse.md @@ -19,7 +19,7 @@ from different catalogs accessing ClickHouse or any other supported data source. To connect to a ClickHouse server, you need: -- ClickHouse (version 23.8 or higher) or Altinity (version 21.8 or higher). +- ClickHouse (version 24.3 or higher) or Altinity (version 22.3 or higher). - Network access from the Trino coordinator and workers to the ClickHouse server. Port 8123 is the default port. diff --git a/plugin/trino-clickhouse/pom.xml b/plugin/trino-clickhouse/pom.xml index 860d259d6fb0..7b2cebdc056c 100644 --- a/plugin/trino-clickhouse/pom.xml +++ b/plugin/trino-clickhouse/pom.xml @@ -21,7 +21,6 @@ com.clickhouse clickhouse-jdbc - all @@ -113,6 +112,14 @@ runtime + + + + org.lz4 + lz4-java + runtime + + io.airlift junit-extensions diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java index 4069feced7f8..44035e20c684 100644 --- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java +++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestClickHouseConnectorTest.java @@ -709,24 +709,11 @@ protected TestTable simpleTable() return new TestTable(onRemoteDatabase(), "tpch.simple_table", "(col BIGINT) Engine=Log", ImmutableList.of("1", "2")); } - @Test @Override - public void testCreateTableWithLongTableName() + protected void verifyTableNameLengthFailurePermissible(Throwable e) { - // Override because ClickHouse connector can create a table which can't be dropped - String baseTableName = "test_create_" + randomNameSuffix(); - String validTableName = baseTableName + "z".repeat(maxTableNameLength().orElseThrow() - baseTableName.length()); - - assertUpdate("CREATE TABLE " + validTableName + " (a bigint)"); - assertThat(getQueryRunner().tableExists(getSession(), validTableName)).isTrue(); - assertThatThrownBy(() -> assertUpdate("DROP TABLE " + validTableName)) - .hasMessageMatching("(?s).*(Bad path syntax|File name too long).*"); - - String invalidTableName = baseTableName + "z".repeat(maxTableNameLength().orElseThrow() - baseTableName.length() + 1); - assertThat(query("CREATE TABLE " + invalidTableName + " (a bigint)")) - .failure().hasMessageMatching("(?s).*(Cannot open file|File name too long).*"); - // ClickHouse lefts a table even if the above statement failed - assertThat(getQueryRunner().tableExists(getSession(), validTableName)).isTrue(); + assertThat(e).hasMessageContaining("The max length of table name for database tpch is %d, current length is %d" + .formatted(maxTableNameLength().orElseThrow(), maxTableNameLength().orElseThrow() + 1)); } @Test @@ -767,31 +754,6 @@ protected void verifySchemaNameLengthFailurePermissible(Throwable e) assertThat(e).hasMessageContaining("File name too long"); } - @Test - @Override - public void testRenameTableToLongTableName() - { - // Override because ClickHouse connector can rename to a table which can't be dropped - String sourceTableName = "test_source_long_table_name_" + randomNameSuffix(); - assertUpdate("CREATE TABLE " + sourceTableName + " AS SELECT 123 x", 1); - - String baseTableName = "test_target_long_table_name_" + randomNameSuffix(); - // The max length is different from CREATE TABLE case - String validTargetTableName = baseTableName + "z".repeat(255 - ".sql".length() - baseTableName.length()); - - assertUpdate("ALTER TABLE " + sourceTableName + " RENAME TO " + validTargetTableName); - assertThat(getQueryRunner().tableExists(getSession(), validTargetTableName)).isTrue(); - assertQuery("SELECT x FROM " + validTargetTableName, "VALUES 123"); - assertThatThrownBy(() -> assertUpdate("DROP TABLE " + validTargetTableName)) - .hasMessageMatching("(?s).*(Bad path syntax|File name too long).*"); - - assertUpdate("CREATE TABLE " + sourceTableName + " AS SELECT 123 x", 1); - String invalidTargetTableName = validTargetTableName + "z"; - assertThatThrownBy(() -> assertUpdate("ALTER TABLE " + sourceTableName + " RENAME TO " + invalidTargetTableName)) - .hasMessageMatching("(?s).*(Cannot rename|File name too long).*"); - assertThat(getQueryRunner().tableExists(getSession(), invalidTargetTableName)).isFalse(); - } - @Test @Override // Override because the failure message differs public void testNativeQueryIncorrectSyntax() @@ -1179,7 +1141,7 @@ public void testExecuteProcedureWithInvalidQuery() protected OptionalInt maxTableNameLength() { // The numeric value depends on file system - return OptionalInt.of(255 - ".sql.detached".length()); + return OptionalInt.of(209); } @Override diff --git a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestingClickHouseServer.java b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestingClickHouseServer.java index 1b1d843e1acf..4d017b2a8120 100644 --- a/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestingClickHouseServer.java +++ b/plugin/trino-clickhouse/src/test/java/io/trino/plugin/clickhouse/TestingClickHouseServer.java @@ -27,15 +27,23 @@ public class TestingClickHouseServer implements Closeable { + // https://clickhouse.com/docs/en/faq/operations/production#how-to-choose-between-clickhouse-releases + // * stable is the kind of package we recommend by default. + // They are released roughly monthly (and thus provide new features with reasonable delay) + // and three latest stable releases are supported in terms of diagnostics and backporting of bugfixes. + // * lts are released twice a year and are supported for a year after their initial release. + // versioning schema: https://kb.altinity.com/altinity-kb-setup-and-maintenance/clickhouse-versions/ private static final DockerImageName CLICKHOUSE_IMAGE = DockerImageName.parse("clickhouse/clickhouse-server"); - public static final DockerImageName CLICKHOUSE_LATEST_IMAGE = CLICKHOUSE_IMAGE.withTag("24.1.8.22"); // EOL by Apr 2025 - public static final DockerImageName CLICKHOUSE_DEFAULT_IMAGE = CLICKHOUSE_IMAGE.withTag("23.8.12.13"); // EOL by Jun 2024 + // https://clickhouse.com/docs/en/whats-new/changelog#-clickhouse-release-2412-2024-12-19 + public static final DockerImageName CLICKHOUSE_LATEST_IMAGE = CLICKHOUSE_IMAGE.withTag("24.12.1.1614"); // EOL in 3 releases after 2024-12-19 + // https://clickhouse.com/docs/en/whats-new/changelog#-clickhouse-release-243-lts-2024-03-27 + public static final DockerImageName CLICKHOUSE_DEFAULT_IMAGE = CLICKHOUSE_IMAGE.withTag("24.3.14.35"); // EOL in 1 year after 2024-03-27 // Altinity Stable Builds Life-Cycle Table https://docs.altinity.com/altinitystablebuilds/#altinity-stable-builds-life-cycle-table - // On Mac/arm try `21.8.12.29.altinitydev.arm` instead of the specified stable build + // On Mac/arm 23.3.13.7.altinitystable, 23.8.8.21.altinitystable and 22.8.15.25.altinitystable and later versions available on ARM. private static final DockerImageName ALTINITY_IMAGE = DockerImageName.parse("altinity/clickhouse-server").asCompatibleSubstituteFor("clickhouse/clickhouse-server"); - public static final DockerImageName ALTINITY_LATEST_IMAGE = ALTINITY_IMAGE.withTag("23.8.8.21.altinitystable"); // EOL is 27 Dec 2026 - public static final DockerImageName ALTINITY_DEFAULT_IMAGE = ALTINITY_IMAGE.withTag("21.8.15.15.altinitystable"); // EOL is 30 Aug 2024 + public static final DockerImageName ALTINITY_LATEST_IMAGE = ALTINITY_IMAGE.withTag("24.3.12.76.altinitystable"); // EOL is 23 Jul 2027 + public static final DockerImageName ALTINITY_DEFAULT_IMAGE = ALTINITY_IMAGE.withTag("22.3.15.34.altinitystable"); // EOL is 15 Jul 2025 private final ClickHouseContainer dockerContainer; @@ -66,7 +74,10 @@ public void execute(String sql) public String getJdbcUrl() { - return format("jdbc:clickhouse://%s:%s/", dockerContainer.getHost(), + // externalDatabase=false is needed because Schema listing fetch is extremely slow on Clickhouse-server 24.3+ + // https://github.com/ClickHouse/clickhouse-java/issues/1245 + // https://github.com/ClickHouse/clickhouse-java/issues/1584 + return format("jdbc:clickhouse://%s:%s/?externalDatabase=false", dockerContainer.getHost(), dockerContainer.getMappedPort(8123)); } diff --git a/pom.xml b/pom.xml index 492ba2be73c8..9bb7dae9c0a1 100644 --- a/pom.xml +++ b/pom.xml @@ -462,8 +462,7 @@ com.clickhouse clickhouse-jdbc - 0.6.3 - all + 0.7.1-patch1 @@ -2282,6 +2281,14 @@ + + + + org.lz4 + lz4-java + 1.8.0 + + org.mariadb.jdbc mariadb-java-client @@ -2602,6 +2609,13 @@ org.basepom.maven duplicate-finder-maven-plugin + + + + com.clickhouse + clickhouse-jdbc + + diff --git a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeClickhouse.java b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeClickhouse.java index 0eb754dc2002..04c4ceff174c 100644 --- a/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeClickhouse.java +++ b/testing/trino-product-tests-launcher/src/main/java/io/trino/tests/product/launcher/env/environment/EnvMultinodeClickhouse.java @@ -23,6 +23,7 @@ import io.trino.tests.product.launcher.env.common.TestsEnvironment; import io.trino.tests.product.launcher.testcontainers.PortBinder; import org.testcontainers.containers.startupcheck.IsRunningStartupCheckStrategy; +import org.testcontainers.utility.DockerImageName; import java.io.IOException; import java.io.UncheckedIOException; @@ -47,6 +48,8 @@ public class EnvMultinodeClickhouse { private static final String ZOOKEEPER = "zookeeper"; + private static final DockerImageName CLICKHOUSE_IMAGE = DockerImageName.parse("clickhouse/clickhouse-server"); // see TestingClickHouseServer for details + private static final DockerImageName CLICKHOUSE_DEFAULT_IMAGE = CLICKHOUSE_IMAGE.withTag("24.3.14.35"); // EOL in 1 year after 2024-03-27 private static final String CLICKHOUSE = "clickhouse"; private static final String CLICKHOUSE_NTH = CLICKHOUSE + "-"; private static final String CONTAINER_CLICKHOUSE_CONFIG_DIR = "/etc/clickhouse-server/"; @@ -109,7 +112,7 @@ private static DockerContainer createClickHouse(int number, DockerFiles dockerFi int httpPort = CLICKHOUSE_DEFAULT_HTTP_PORT + number; int nativePort = CLICKHOUSE_DEFAULT_NATIVE_PORT + number; - DockerContainer container = new DockerContainer("yandex/clickhouse-server:21.3.2.5", logicalName(number)) + DockerContainer container = new DockerContainer(CLICKHOUSE_DEFAULT_IMAGE.toString(), logicalName(number)) .withCopyFileToContainer( forHostPath(dockerFiles.getDockerFilesHostPath("conf/environment/multinode-clickhouse/test.xml")), CONTAINER_CLICKHOUSE_USERS_D + "test.xml") diff --git a/testing/trino-product-tests/pom.xml b/testing/trino-product-tests/pom.xml index b524d674de83..e7938546608a 100644 --- a/testing/trino-product-tests/pom.xml +++ b/testing/trino-product-tests/pom.xml @@ -230,7 +230,6 @@ com.clickhouse clickhouse-jdbc - all runtime @@ -284,6 +283,14 @@ runtime + + + + org.lz4 + lz4-java + runtime + + org.mariadb.jdbc mariadb-java-client