Fall back to non-UTF8 string functions with pre-23.8 ClickHouse

.docker/clickhouse/single_node_tls/Dockerfile

@@ -1,4 +1,4 @@
-FROM clickhouse/clickhouse-server:23.8-alpine
+FROM clickhouse/clickhouse-server:24.1-alpine
 COPY .docker/clickhouse/single_node_tls/certificates /etc/clickhouse-server/certs
 RUN chown clickhouse:clickhouse -R /etc/clickhouse-server/certs \
     && chmod 600 /etc/clickhouse-server/certs/* \

.github/workflows/check.yml

@@ -41,6 +41,7 @@ jobs:
           down-flags: "--volumes"
           services: |
             clickhouse
+            clickhouse_older_version
             clickhouse_tls
 
       - name: Install Clojure CLI
@@ -66,11 +67,23 @@ jobs:
         run: yarn build-static-viz
 
       # Use custom deps.edn containing "user/clickhouse" alias to include driver sources
-      - name: Run tests
+      - name: Prepare deps.edn
         run: |
           mkdir -p /home/runner/.config/clojure
           cat modules/drivers/clickhouse/.github/deps.edn | sed -e "s|PWD|$PWD|g" > /home/runner/.config/clojure/deps.edn
-          DRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse
+
+      - name: Run ClickHouse driver tests with 23.3
+        env:
+          DRIVERS: clickhouse
+          MB_CLICKHOUSE_TEST_PORT: 8124
+        run: |
+          clojure -X:dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test
+
+      - name: Run all tests with the latest ClickHouse version
+        env:
+          DRIVERS: clickhouse
+        run: |
+          clojure -X:dev:drivers:drivers-dev:test:user/clickhouse
 
       - name: Build ClickHouse driver
         run: |

CHANGELOG.md

@@ -1,3 +1,9 @@
+# 1.3.4
+
+### New features
+
+* If introspected ClickHouse version is lower than 23.8, the driver will not use [startsWithUTF8](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswithutf8) and fall back to its [non-UTF8 counterpart](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswith) instead. There is a drawback in this compatibility mode: potentially incorrect filtering results when working with non-latin strings. If your use case includes filtering by columns with such strings and you experience these issues, consider upgrading your ClickHouse server to 23.8+. ([#224](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/224))
+
 # 1.3.3
 
 ### Bug fixes

README.md

@@ -76,7 +76,7 @@ docker run -d -p 3000:3000 \
 | 0.46.x           | 1.1.7          |
 | 0.47.x           | 1.2.3          |
 | 0.47.7+          | 1.2.5          |
-| 0.48.x           | 1.3.2          |
+| 0.48.x           | 1.3.4          |
 
 ## Creating a Metabase Docker image with ClickHouse driver
 
@@ -142,6 +142,7 @@ The driver should work fine for many use cases. Please consider the following it
 
 * As the underlying JDBC driver version does not support columns with `AggregateFunction` type, these columns are excluded from the table metadata and data browser result sets to prevent sync or data browsing errors.
 * If the past month/week/quarter/year filter over a DateTime64 column is not working as intended, this is likely due to a [type conversion issue](https://github.com/ClickHouse/ClickHouse/pull/50280). See [this report](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/164) for more details. This issue was resolved as of ClickHouse 23.5.
+* If introspected ClickHouse version is lower than 23.8, the driver will not use [startsWithUTF8](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswithutf8) and fall back to its [non-UTF8 counterpart](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswith) instead. There is a drawback in this compatibility mode: potentially incorrect filtering results when working with non-latin strings. If your use case includes filtering by columns with such strings and you experience these issues, consider upgrading your ClickHouse server to 23.8+.
 
 ## Contributing
 

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.8'
 services:
   clickhouse:
-    image: 'clickhouse/clickhouse-server:23.8-alpine'
+    image: 'clickhouse/clickhouse-server:24.1-alpine'
     container_name: 'metabase-driver-clickhouse-server'
     ports:
       - '8123:8123'
@@ -14,6 +14,21 @@ services:
       - './.docker/clickhouse/single_node/config.xml:/etc/clickhouse-server/config.xml'
       - './.docker/clickhouse/single_node/users.xml:/etc/clickhouse-server/users.xml'
 
+  # For testing pre-23.8 string functions switch between UTF8 and non-UTF8 versions (see clickhouse_qp.clj)
+  clickhouse_older_version:
+    image: 'clickhouse/clickhouse-server:23.3-alpine'
+    container_name: 'metabase-driver-clickhouse-server-older-version'
+    ports:
+      - '8124:8123'
+      - '9001:9000'
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    volumes:
+      - './.docker/clickhouse/single_node/config.xml:/etc/clickhouse-server/config.xml'
+      - './.docker/clickhouse/single_node/users.xml:/etc/clickhouse-server/users.xml'
+
   clickhouse_tls:
     build:
       context: ./

src/metabase/driver/clickhouse.clj

@@ -20,7 +20,7 @@
 (driver/register! :clickhouse :parent :sql-jdbc)
 
 (defmethod driver/display-name :clickhouse [_] "ClickHouse")
-(def ^:private product-name "metabase/1.3.3")
+(def ^:private product-name "metabase/1.3.4")
 
 (defmethod driver/prettify-native-form :clickhouse [_ native-form]
   (sql.u/format-sql-and-fix-params :mysql native-form))
@@ -94,6 +94,20 @@
 (defmethod ddl.i/format-name :clickhouse [_ table-or-field-name]
   (str/replace table-or-field-name #"-" "_"))
 
+(def ^:private version-query
+  "WITH s AS (SELECT version() AS ver, splitByChar('.', ver) AS verSplit) SELECT s.ver, toInt32(verSplit[1]), toInt32(verSplit[2]) FROM s")
+(defmethod driver/dbms-version :clickhouse
+  [driver database]
+  (sql-jdbc.execute/do-with-connection-with-options
+    driver database nil
+    (fn [^java.sql.Connection conn]
+      (with-open [stmt (.prepareStatement conn version-query)
+                  rset (.executeQuery stmt)]
+        (when (.next rset)
+          {:version          (.getString rset 1)
+           :semantic-version {:major (.getInt rset 2)
+                              :minor (.getInt rset 3)}})))))
+
 ;;; ------------------------------------------ User Impersonation ------------------------------------------
 
 (defmethod driver.sql/set-role-statement :clickhouse

src/metabase/driver/clickhouse_qp.clj

@@ -5,13 +5,16 @@
             [honey.sql :as sql]
             [java-time.api :as t]
             [metabase [util :as u]]
+            [metabase.driver :as driver]
             [metabase.driver.clickhouse-nippy]
             [metabase.driver.common.parameters.dates :as params.dates]
             [metabase.driver.sql-jdbc [execute :as sql-jdbc.execute]]
             [metabase.driver.sql.parameters.substitution :as sql.params.substitution]
             [metabase.driver.sql.query-processor :as sql.qp :refer [add-interval-honeysql-form]]
             [metabase.driver.sql.util.unprepare :as unprepare]
+            [metabase.lib.metadata :as lib.metadata]
             [metabase.mbql.util :as mbql.u]
+            [metabase.query-processor.store :as qp.store]
             [metabase.util.date-2 :as u.date]
             [metabase.util.honey-sql-2 :as h2x]
             [metabase.util.log :as log])
@@ -31,6 +34,18 @@
 (defmethod sql.qp/quote-style       :clickhouse [_] :mysql)
 (defmethod sql.qp/honey-sql-version :clickhouse [_] 2)
 
+(defn- clickhouse-version []
+  (let [db (lib.metadata/database (qp.store/metadata-provider))]
+    (qp.store/cached ::clickhouse-version (driver/dbms-version :clickhouse db))))
+
+(defn- with-min-version [major minor default-fn fallback-fn]
+  (let [version (clickhouse-version)]
+    (if (or (> (get-in version [:semantic-version :major]) major)
+            (and (= (get-in version [:semantic-version :major]) major)
+                 (>= (get-in version [:semantic-version :minor]) minor)))
+      default-fn
+      fallback-fn)))
+
 (defmethod sql.qp/date [:clickhouse :day-of-week]
   [_ _ expr]
   ;; a tick in the function name prevents HSQL2 to make the function call UPPERCASE
@@ -289,19 +304,22 @@
 
 (defmethod sql.qp/->honeysql [:clickhouse :starts-with]
   [_ [_ field value options]]
-  (clickhouse-string-fn :'startsWithUTF8 field value options))
+  (let [starts-with (with-min-version 23 8 :'startsWithUTF8 :'startsWith)]
+    (clickhouse-string-fn starts-with field value options)))
 
 (defmethod sql.qp/->honeysql [:clickhouse :ends-with]
   [_ [_ field value options]]
-  (clickhouse-string-fn :'endsWithUTF8 field value options))
+  (let [ends-with (with-min-version 23 8 :'endsWithUTF8 :'endsWith)]
+    (clickhouse-string-fn ends-with field value options)))
 
 (defmethod sql.qp/->honeysql [:clickhouse :contains]
   [_ [_ field value options]]
   (let [hsql-field (sql.qp/->honeysql :clickhouse field)
-        hsql-value (sql.qp/->honeysql :clickhouse value)]
-    (if (get options :case-sensitive true)
-      [:> [:'positionUTF8                hsql-field hsql-value] 0]
-      [:> [:'positionCaseInsensitiveUTF8 hsql-field hsql-value] 0])))
+        hsql-value (sql.qp/->honeysql :clickhouse value)
+        position-fn (if (get options :case-sensitive true)
+                      :'positionUTF8
+                      :'positionCaseInsensitiveUTF8)]
+    [:> [position-fn hsql-field hsql-value] 0]))
 
 (defmethod sql.qp/->honeysql [:clickhouse :datetime-diff]
   [driver [_ x y unit]]