diff --git a/.docker/clickhouse/single_node_tls/Dockerfile b/.docker/clickhouse/single_node_tls/Dockerfile index 12641cd..5303ff1 100644 --- a/.docker/clickhouse/single_node_tls/Dockerfile +++ b/.docker/clickhouse/single_node_tls/Dockerfile @@ -1,4 +1,4 @@ -FROM clickhouse/clickhouse-server:23.8-alpine +FROM clickhouse/clickhouse-server:24.1-alpine COPY .docker/clickhouse/single_node_tls/certificates /etc/clickhouse-server/certs RUN chown clickhouse:clickhouse -R /etc/clickhouse-server/certs \ && chmod 600 /etc/clickhouse-server/certs/* \ diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 0f6c566..20785dc 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -41,6 +41,7 @@ jobs: down-flags: "--volumes" services: | clickhouse + clickhouse_older_version clickhouse_tls - name: Install Clojure CLI @@ -66,11 +67,23 @@ jobs: run: yarn build-static-viz # Use custom deps.edn containing "user/clickhouse" alias to include driver sources - - name: Run tests + - name: Prepare deps.edn run: | mkdir -p /home/runner/.config/clojure cat modules/drivers/clickhouse/.github/deps.edn | sed -e "s|PWD|$PWD|g" > /home/runner/.config/clojure/deps.edn - DRIVERS=clickhouse clojure -X:dev:drivers:drivers-dev:test:user/clickhouse + + - name: Run ClickHouse driver tests with 23.3 + env: + DRIVERS: clickhouse + MB_CLICKHOUSE_TEST_PORT: 8124 + run: | + clojure -X:dev:drivers:drivers-dev:test:user/clickhouse :only metabase.driver.clickhouse-test + + - name: Run all tests with the latest ClickHouse version + env: + DRIVERS: clickhouse + run: | + clojure -X:dev:drivers:drivers-dev:test:user/clickhouse - name: Build ClickHouse driver run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index f5ed959..6afc953 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +# 1.3.4 + +### New features + +* If introspected ClickHouse version is lower than 23.8, the driver will not use [startsWithUTF8](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswithutf8) and fall back to its [non-UTF8 counterpart](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswith) instead. There is a drawback in this compatibility mode: potentially incorrect filtering results when working with non-latin strings. If your use case includes filtering by columns with such strings and you experience these issues, consider upgrading your ClickHouse server to 23.8+. ([#224](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/224)) + # 1.3.3 ### Bug fixes diff --git a/README.md b/README.md index 19f68e7..884b778 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ docker run -d -p 3000:3000 \ | 0.46.x | 1.1.7 | | 0.47.x | 1.2.3 | | 0.47.7+ | 1.2.5 | -| 0.48.x | 1.3.2 | +| 0.48.x | 1.3.4 | ## Creating a Metabase Docker image with ClickHouse driver @@ -142,6 +142,7 @@ The driver should work fine for many use cases. Please consider the following it * As the underlying JDBC driver version does not support columns with `AggregateFunction` type, these columns are excluded from the table metadata and data browser result sets to prevent sync or data browsing errors. * If the past month/week/quarter/year filter over a DateTime64 column is not working as intended, this is likely due to a [type conversion issue](https://github.com/ClickHouse/ClickHouse/pull/50280). See [this report](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/164) for more details. This issue was resolved as of ClickHouse 23.5. +* If introspected ClickHouse version is lower than 23.8, the driver will not use [startsWithUTF8](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswithutf8) and fall back to its [non-UTF8 counterpart](https://clickhouse.com/docs/en/sql-reference/functions/string-functions#startswith) instead. There is a drawback in this compatibility mode: potentially incorrect filtering results when working with non-latin strings. If your use case includes filtering by columns with such strings and you experience these issues, consider upgrading your ClickHouse server to 23.8+. ## Contributing diff --git a/docker-compose.yml b/docker-compose.yml index 3581563..863d7ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.8' services: clickhouse: - image: 'clickhouse/clickhouse-server:23.8-alpine' + image: 'clickhouse/clickhouse-server:24.1-alpine' container_name: 'metabase-driver-clickhouse-server' ports: - '8123:8123' @@ -14,6 +14,21 @@ services: - './.docker/clickhouse/single_node/config.xml:/etc/clickhouse-server/config.xml' - './.docker/clickhouse/single_node/users.xml:/etc/clickhouse-server/users.xml' + # For testing pre-23.8 string functions switch between UTF8 and non-UTF8 versions (see clickhouse_qp.clj) + clickhouse_older_version: + image: 'clickhouse/clickhouse-server:23.3-alpine' + container_name: 'metabase-driver-clickhouse-server-older-version' + ports: + - '8124:8123' + - '9001:9000' + ulimits: + nofile: + soft: 262144 + hard: 262144 + volumes: + - './.docker/clickhouse/single_node/config.xml:/etc/clickhouse-server/config.xml' + - './.docker/clickhouse/single_node/users.xml:/etc/clickhouse-server/users.xml' + clickhouse_tls: build: context: ./ diff --git a/src/metabase/driver/clickhouse.clj b/src/metabase/driver/clickhouse.clj index 690daca..791d8a5 100644 --- a/src/metabase/driver/clickhouse.clj +++ b/src/metabase/driver/clickhouse.clj @@ -20,7 +20,7 @@ (driver/register! :clickhouse :parent :sql-jdbc) (defmethod driver/display-name :clickhouse [_] "ClickHouse") -(def ^:private product-name "metabase/1.3.3") +(def ^:private product-name "metabase/1.3.4") (defmethod driver/prettify-native-form :clickhouse [_ native-form] (sql.u/format-sql-and-fix-params :mysql native-form)) @@ -94,6 +94,20 @@ (defmethod ddl.i/format-name :clickhouse [_ table-or-field-name] (str/replace table-or-field-name #"-" "_")) +(def ^:private version-query + "WITH s AS (SELECT version() AS ver, splitByChar('.', ver) AS verSplit) SELECT s.ver, toInt32(verSplit[1]), toInt32(verSplit[2]) FROM s") +(defmethod driver/dbms-version :clickhouse + [driver database] + (sql-jdbc.execute/do-with-connection-with-options + driver database nil + (fn [^java.sql.Connection conn] + (with-open [stmt (.prepareStatement conn version-query) + rset (.executeQuery stmt)] + (when (.next rset) + {:version (.getString rset 1) + :semantic-version {:major (.getInt rset 2) + :minor (.getInt rset 3)}}))))) + ;;; ------------------------------------------ User Impersonation ------------------------------------------ (defmethod driver.sql/set-role-statement :clickhouse diff --git a/src/metabase/driver/clickhouse_qp.clj b/src/metabase/driver/clickhouse_qp.clj index 8fa4533..13cd049 100644 --- a/src/metabase/driver/clickhouse_qp.clj +++ b/src/metabase/driver/clickhouse_qp.clj @@ -5,13 +5,16 @@ [honey.sql :as sql] [java-time.api :as t] [metabase [util :as u]] + [metabase.driver :as driver] [metabase.driver.clickhouse-nippy] [metabase.driver.common.parameters.dates :as params.dates] [metabase.driver.sql-jdbc [execute :as sql-jdbc.execute]] [metabase.driver.sql.parameters.substitution :as sql.params.substitution] [metabase.driver.sql.query-processor :as sql.qp :refer [add-interval-honeysql-form]] [metabase.driver.sql.util.unprepare :as unprepare] + [metabase.lib.metadata :as lib.metadata] [metabase.mbql.util :as mbql.u] + [metabase.query-processor.store :as qp.store] [metabase.util.date-2 :as u.date] [metabase.util.honey-sql-2 :as h2x] [metabase.util.log :as log]) @@ -31,6 +34,18 @@ (defmethod sql.qp/quote-style :clickhouse [_] :mysql) (defmethod sql.qp/honey-sql-version :clickhouse [_] 2) +(defn- clickhouse-version [] + (let [db (lib.metadata/database (qp.store/metadata-provider))] + (qp.store/cached ::clickhouse-version (driver/dbms-version :clickhouse db)))) + +(defn- with-min-version [major minor default-fn fallback-fn] + (let [version (clickhouse-version)] + (if (or (> (get-in version [:semantic-version :major]) major) + (and (= (get-in version [:semantic-version :major]) major) + (>= (get-in version [:semantic-version :minor]) minor))) + default-fn + fallback-fn))) + (defmethod sql.qp/date [:clickhouse :day-of-week] [_ _ expr] ;; a tick in the function name prevents HSQL2 to make the function call UPPERCASE @@ -289,19 +304,22 @@ (defmethod sql.qp/->honeysql [:clickhouse :starts-with] [_ [_ field value options]] - (clickhouse-string-fn :'startsWithUTF8 field value options)) + (let [starts-with (with-min-version 23 8 :'startsWithUTF8 :'startsWith)] + (clickhouse-string-fn starts-with field value options))) (defmethod sql.qp/->honeysql [:clickhouse :ends-with] [_ [_ field value options]] - (clickhouse-string-fn :'endsWithUTF8 field value options)) + (let [ends-with (with-min-version 23 8 :'endsWithUTF8 :'endsWith)] + (clickhouse-string-fn ends-with field value options))) (defmethod sql.qp/->honeysql [:clickhouse :contains] [_ [_ field value options]] (let [hsql-field (sql.qp/->honeysql :clickhouse field) - hsql-value (sql.qp/->honeysql :clickhouse value)] - (if (get options :case-sensitive true) - [:> [:'positionUTF8 hsql-field hsql-value] 0] - [:> [:'positionCaseInsensitiveUTF8 hsql-field hsql-value] 0]))) + hsql-value (sql.qp/->honeysql :clickhouse value) + position-fn (if (get options :case-sensitive true) + :'positionUTF8 + :'positionCaseInsensitiveUTF8)] + [:> [position-fn hsql-field hsql-value] 0])) (defmethod sql.qp/->honeysql [:clickhouse :datetime-diff] [driver [_ x y unit]] diff --git a/test/metabase/driver/clickhouse_data_types_test.clj b/test/metabase/driver/clickhouse_data_types_test.clj new file mode 100644 index 0000000..83239dd --- /dev/null +++ b/test/metabase/driver/clickhouse_data_types_test.clj @@ -0,0 +1,521 @@ +(ns metabase.driver.clickhouse-test + #_{:clj-kondo/ignore [:unsorted-required-namespaces]} + (:require [cljc.java-time.local-date :as local-date] + [cljc.java-time.offset-date-time :as offset-date-time] + [clojure.test :refer :all] + [metabase.query-processor.test-util :as qp.test] + [metabase.test :as mt] + [metabase.test.data :as data] + [metabase.test.data [interface :as tx]] + [metabase.test.data.clickhouse :as ctd])) + +(deftest ^:parallel clickhouse-decimals + (mt/test-driver + :clickhouse + (data/dataset + (tx/dataset-definition "metabase_tests_decimal" + ["test-data-decimal" + [{:field-name "my_money" + :base-type {:native "Decimal(12,4)"}}] + [[1.0] [23.1337] [42.0] [42.0]]]) + (testing "simple division" + (is + (= 21.0 + (-> (data/run-mbql-query test-data-decimal + {:expressions {:divided [:/ $my_money 2]} + :filter [:> [:expression :divided] 1.0] + :breakout [[:expression :divided]] + :order-by [[:desc [:expression :divided]]] + :limit 1}) + qp.test/first-row last float)))) + (testing "divided decimal precision" + (is + (= 1.8155331831916208 + (-> (data/run-mbql-query test-data-decimal + {:expressions {:divided [:/ 42 $my_money]} + :filter [:= $id 2] + :limit 1}) + qp.test/first-row last double))))))) + +(deftest ^:parallel clickhouse-array-string + (mt/test-driver + :clickhouse + (is + (= "[foo, bar]" + (-> (data/dataset + (tx/dataset-definition "metabase_tests_array_string" + ["test-data-array-string" + [{:field-name "my_array" + :base-type {:native "Array(String)"}}] + [[(into-array (list "foo" "bar"))]]]) + (data/run-mbql-query test-data-array-string {:limit 1})) + qp.test/first-row + last))))) + +(deftest ^:parallel clickhouse-array-uint64 + (mt/test-driver + :clickhouse + (is + (= "[23, 42]" + (-> (data/dataset + (tx/dataset-definition "metabase_tests_array_uint" + ["test-data-array-uint64" + [{:field-name "my_array" + :base-type {:native "Array(UInt64)"}}] + [[(into-array (list 23 42))]]]) + (data/run-mbql-query test-data-array-uint64 {:limit 1})) + qp.test/first-row + last))))) + +(deftest ^:parallel clickhouse-array-of-arrays + (mt/test-driver + :clickhouse + (let [row1 (into-array (list + (into-array (list "foo" "bar")) + (into-array (list "qaz" "qux")))) + row2 (into-array nil) + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_arrays" + ["test-data-array-of-arrays" + [{:field-name "my_array_of_arrays" + :base-type {:native "Array(Array(String))"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-arrays {})) + result (ctd/rows-without-index query-result)] + (is (= [["[[foo, bar], [qaz, qux]]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-low-cardinality-array + (mt/test-driver + :clickhouse + (let [row1 (into-array (list "foo" "bar")) + row2 (into-array nil) + query-result (data/dataset + (tx/dataset-definition "metabase_tests_low_cardinality_array" + ["test-data-low-cardinality-array" + [{:field-name "my_low_card_array" + :base-type {:native "Array(LowCardinality(String))"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-low-cardinality-array {})) + result (ctd/rows-without-index query-result)] + (is (= [["[foo, bar]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-array-of-nullables + (mt/test-driver + :clickhouse + (let [row1 (into-array (list "foo" nil "bar")) + row2 (into-array nil) + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_nullables" + ["test-data-array-of-nullables" + [{:field-name "my_array_of_nullables" + :base-type {:native "Array(Nullable(String))"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-nullables {})) + result (ctd/rows-without-index query-result)] + (is (= [["[foo, null, bar]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-array-of-booleans + (mt/test-driver + :clickhouse + (let [row1 (into-array (list true false true)) + row2 (into-array nil) + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_booleans" + ["test-data-array-of-booleans" + [{:field-name "my_array_of_booleans" + :base-type {:native "Array(Boolean)"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-booleans {})) + result (ctd/rows-without-index query-result)] + (is (= [["[true, false, true]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-array-of-uint8 + (mt/test-driver + :clickhouse + (let [row1 (into-array (list 42 100 2)) + row2 (into-array nil) + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_uint8" + ["test-data-array-of-uint8" + [{:field-name "my_array_of_uint8" + :base-type {:native "Array(UInt8)"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-uint8 {})) + result (ctd/rows-without-index query-result)] + (is (= [["[42, 100, 2]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-array-of-floats + (mt/test-driver + :clickhouse + (let [row1 (into-array (list 1.2 3.4)) + row2 (into-array nil) + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_floats" + ["test-data-array-of-floats" + [{:field-name "my_array_of_floats" + :base-type {:native "Array(Float64)"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-floats {})) + result (ctd/rows-without-index query-result)] + (is (= [["[1.2, 3.4]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-array-of-dates + (mt/test-driver + :clickhouse + (let [row1 (into-array + (list + (local-date/parse "2022-12-06") + (local-date/parse "2021-10-19"))) + row2 (into-array nil) + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_dates" + ["test-data-array-of-dates" + [{:field-name "my_array_of_dates" + :base-type {:native "Array(Date)"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-dates {})) + result (ctd/rows-without-index query-result)] + (is (= [["[2022-12-06, 2021-10-19]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-array-of-date32 + (mt/test-driver + :clickhouse + (let [row1 (into-array + (list + (local-date/parse "2122-12-06") + (local-date/parse "2099-10-19"))) + row2 (into-array nil) + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_date32" + ["test-data-array-of-date32" + [{:field-name "my_array_of_date32" + :base-type {:native "Array(Date32)"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-date32 {})) + result (ctd/rows-without-index query-result)] + (is (= [["[2122-12-06, 2099-10-19]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-array-of-datetime + (mt/test-driver + :clickhouse + (let [row1 (into-array + (list + (offset-date-time/parse "2022-12-06T18:28:31Z") + (offset-date-time/parse "2021-10-19T13:12:44Z"))) + row2 (into-array nil) + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_datetime" + ["test-data-array-of-datetime" + [{:field-name "my_array_of_datetime" + :base-type {:native "Array(DateTime)"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-datetime {})) + result (ctd/rows-without-index query-result)] + (is (= [["[2022-12-06T18:28:31, 2021-10-19T13:12:44]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-array-of-datetime64 + (mt/test-driver + :clickhouse + (let [row1 (into-array + (list + (offset-date-time/parse "2022-12-06T18:28:31.123Z") + (offset-date-time/parse "2021-10-19T13:12:44.456Z"))) + row2 (into-array nil) + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_datetime64" + ["test-data-array-of-datetime64" + [{:field-name "my_array_of_datetime64" + :base-type {:native "Array(DateTime64(3))"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-datetime64 {})) + result (ctd/rows-without-index query-result)] + (is (= [["[2022-12-06T18:28:31.123, 2021-10-19T13:12:44.456]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-array-of-decimals + (mt/test-driver + :clickhouse + (let [row1 (into-array (list "12345123.123456789" "78.245")) + row2 nil + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_decimals" + ["test-data-array-of-decimals" + [{:field-name "my_array_of_decimals" + :base-type {:native "Array(Decimal(18, 9))"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-decimals {})) + result (ctd/rows-without-index query-result)] + (is (= [["[12345123.123456789, 78.245000000]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-array-of-tuples + (mt/test-driver + :clickhouse + (let [row1 (into-array (list (list "foobar" 1234) (list "qaz" 0))) + row2 nil + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_tuples" + ["test-data-array-of-tuples" + [{:field-name "my_array_of_tuples" + :base-type {:native "Array(Tuple(String, UInt32))"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-tuples {})) + result (ctd/rows-without-index query-result)] + (is (= [["[[foobar, 1234], [qaz, 0]]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-array-of-uuids + (mt/test-driver + :clickhouse + (let [row1 (into-array (list "2eac427e-7596-11ed-a1eb-0242ac120002" + "2eac44f4-7596-11ed-a1eb-0242ac120002")) + row2 nil + query-result (data/dataset + (tx/dataset-definition "metabase_tests_array_of_uuids" + ["test-data-array-of-uuids" + [{:field-name "my_array_of_uuids" + :base-type {:native "Array(UUID)"}}] + [[row1] [row2]]]) + (data/run-mbql-query test-data-array-of-uuids {})) + result (ctd/rows-without-index query-result)] + (is (= [["[2eac427e-7596-11ed-a1eb-0242ac120002, 2eac44f4-7596-11ed-a1eb-0242ac120002]"], ["[]"]] result))))) + +(deftest ^:parallel clickhouse-nullable-strings + (mt/test-driver + :clickhouse + (data/dataset + (tx/dataset-definition + "metabase_tests_nullable_strings" + ["test-data-nullable-strings" + [{:field-name "mystring" :base-type :type/Text}] + [["foo"] ["bar"] [" "] [""] [nil]]]) + (testing "null strings count" + (is (= 2 + (-> (data/run-mbql-query test-data-nullable-strings + {:filter [:is-null $mystring] + :aggregation [:count]}) + qp.test/first-row last)))) + (testing "nullable strings not null filter" + (is (= 3 + (-> (data/run-mbql-query test-data-nullable-strings + {:filter [:not-null $mystring] + :aggregation [:count]}) + qp.test/first-row last)))) + (testing "filter nullable string by value" + (is (= 1 + (-> (data/run-mbql-query test-data-nullable-strings + {:filter [:= $mystring "foo"] + :aggregation [:count]}) + qp.test/first-row last))))))) + +(deftest ^:parallel clickhouse-non-latin-strings + (mt/test-driver + :clickhouse + (testing "basic filtering" + (is (= [[1 "Я_1"] [3 "Я_2"] [4 "Я"]] + (qp.test/formatted-rows + [int str] + :format-nil-values + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (data/run-mbql-query + metabase_test_lowercases + {:filter [:contains $mystring "Я"]})))))))) + (testing "case-insensitive non-latin filtering" + (is (= [[1 "Я_1"] [3 "Я_2"] [4 "Я"] [5 "я"]] + (qp.test/formatted-rows + [int str] + :format-nil-values + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (data/run-mbql-query + metabase_test_lowercases + {:filter [:contains $mystring "Я" + {:case-sensitive false}]})))))))))) + +(deftest ^:parallel clickhouse-datetime64-filter + (mt/test-driver + :clickhouse + (let [row1 "2022-03-03 03:03:03.333" + row2 "2022-03-03 03:03:03.444" + row3 "2022-03-03 03:03:03" + query-result (data/dataset + (tx/dataset-definition "metabase_tests_datetime64" + ["test-data-datetime64" + [{:field-name "milli_sec" + :base-type {:native "DateTime64(3)"}}] + [[row1] [row2] [row3]]]) + (data/run-mbql-query test-data-datetime64 {:filter [:= $milli_sec "2022-03-03T03:03:03.333Z"]})) + result (ctd/rows-without-index query-result)] + (is (= [["2022-03-03T03:03:03.333Z"]] result))))) + +(deftest ^:parallel clickhouse-datetime-filter + (mt/test-driver + :clickhouse + (let [row1 "2022-03-03 03:03:03" + row2 "2022-03-03 03:03:04" + row3 "2022-03-03 03:03:05" + query-result (data/dataset + (tx/dataset-definition "metabase_tests_datetime" + ["test-data-datetime" + [{:field-name "second" + :base-type {:native "DateTime"}}] + [[row1] [row2] [row3]]]) + (data/run-mbql-query test-data-datetime {:filter [:= $second "2022-03-03T03:03:04Z"]})) + result (ctd/rows-without-index query-result)] + (is (= [["2022-03-03T03:03:04Z"]] result))))) + +(deftest ^:parallel clickhouse-booleans + (mt/test-driver + :clickhouse + (let [[row1 row2 row3 row4] [["#1" true] ["#2" false] ["#3" false] ["#4" true]] + query-result (data/dataset + (tx/dataset-definition "metabase_tests_booleans" + ["test-data-booleans" + [{:field-name "name" + :base-type :type/Text} + {:field-name "is_active" + :base-type :type/Boolean}] + [row1 row2 row3 row4]]) + (data/run-mbql-query test-data-booleans {:filter [:= $is_active false]})) + rows (qp.test/rows query-result) + result (map #(drop 1 %) rows)] ; remove db "index" which is the first column in the result set + (is (= [row2 row3] result))))) + +(deftest ^:parallel clickhouse-enums-values-test + (mt/test-driver + :clickhouse + (testing "select enums values as strings" + (is (= [["foo" "house" "qaz"] + ["foo bar" "click" "qux"] + ["bar" "house" "qaz"]] + (qp.test/formatted-rows + [str str str] + :format-nil-values + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (data/run-mbql-query + enums_test + {})))))))) + (testing "filtering enum values" + (is (= [["useqa"]] + (qp.test/formatted-rows + [str] + :format-nil-values + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (data/run-mbql-query + enums_test + {:expressions {"test" [:concat + [:substring $enum2 3 3] + [:substring $enum3 1 2]]} + :fields [[:expression "test"]] + :filter [:= $enum1 "foo"]})))))))))) + +(deftest ^:parallel clickhouse-ipv4query-test + (mt/test-driver + :clickhouse + (is (= [[1]] + (qp.test/formatted-rows + [int] + :format-nil-values + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (data/run-mbql-query + ipaddress_test + {:filter [:= $ipvfour "127.0.0.1"] + :aggregation [[:count]]}))))))))) + +(deftest ^:parallel clickhouse-ip-serialization-test + (mt/test-driver + :clickhouse + (is (= [["127.0.0.1" "0:0:0:0:0:ffff:7f00:1"] + ["0.0.0.0" "0:0:0:0:0:ffff:0:0"] + [nil nil]] + (qp.test/formatted-rows + [str str] + (ctd/do-with-test-db + (fn [db] (data/with-db db (data/run-mbql-query ipaddress_test {}))))))))) + +(defn- map-as-string [^java.util.LinkedHashMap m] (.toString m)) +(deftest ^:parallel clickhouse-simple-map-test + (mt/test-driver + :clickhouse + (is (= [["{key1=1, key2=10}"] ["{key1=2, key2=20}"] ["{key1=3, key2=30}"]] + (qp.test/formatted-rows + [map-as-string] + :format-nil-values + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (data/run-mbql-query + maps_test + {}))))))))) + +(deftest ^:parallel clickhouse-datetime-diff-nullable + (mt/test-driver + :clickhouse + (is (= [[170 202] [nil nil] [nil nil] [nil nil]] + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (->> (data/run-mbql-query + datetime_diff_nullable + {:fields [[:expression "dt64,dt"] + [:expression "dt64,d"]] + :expressions + {"dt64,dt" [:datetime-diff $dt64 $dt :day] + "dt64,d" [:datetime-diff $dt64 $d :day]}}) + (mt/formatted-rows [int int]))))))))) + +;; Metabase has pretty extensive testing for sum-where and count-where +;; However, this ClickHouse-specific corner case is not covered +(deftest ^:parallel clickhouse-sum-where-numeric-types + (mt/test-driver + :clickhouse + (testing "int values (with matching rows)" + (is (= [[8]] + (qp.test/formatted-rows + [int] + :format-nil-values + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (data/run-mbql-query + sum_if_test_int + {:aggregation [[:sum-where $int_value [:= $discriminator "bar"]]]})))))))) + (testing "int values (no matching rows)" + (is (= [[0]] + (qp.test/formatted-rows + [int] + :format-nil-values + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (data/run-mbql-query + sum_if_test_int + {:aggregation [[:sum-where $int_value [:= $discriminator "qaz"]]]})))))))) + (testing "double values (with matching rows)" + (is (= [[9.27]] + (qp.test/formatted-rows + [double] + :format-nil-values + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (data/run-mbql-query + sum_if_test_float + {:aggregation [[:sum-where $float_value [:= $discriminator "bar"]]]})))))))) + (testing "double values (no matching rows)" + (is (= [[0.0]] + (qp.test/formatted-rows + [double] + :format-nil-values + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (data/run-mbql-query + sum_if_test_float + {:aggregation [[:sum-where $float_value [:= $discriminator "qaz"]]]})))))))))) diff --git a/test/metabase/driver/clickhouse_base_types_test.clj b/test/metabase/driver/clickhouse_introspection_test.clj similarity index 64% rename from test/metabase/driver/clickhouse_base_types_test.clj rename to test/metabase/driver/clickhouse_introspection_test.clj index 1031ef8..3db586d 100644 --- a/test/metabase/driver/clickhouse_base_types_test.clj +++ b/test/metabase/driver/clickhouse_introspection_test.clj @@ -3,8 +3,15 @@ (:require [clojure.test :refer :all] [metabase.driver :as driver] + [metabase.driver.common :as driver.common] + [metabase.models.database :refer [Database]] + [metabase.query-processor :as qp] + [metabase.query-processor.test-util :as qp.test] [metabase.test :as mt] - [metabase.test.data.clickhouse :as ctd])) + [metabase.test.data :as data] + [metabase.test.data.clickhouse :as ctd] + [metabase.test.data.interface :as tx] + [toucan2.tools.with-temp :as t2.with-temp])) (defn- desc-table [table-name] @@ -12,7 +19,7 @@ (:fields (ctd/do-with-test-db #(driver/describe-table :clickhouse % {:name table-name})))))) -(deftest clickhouse-base-types-test +(deftest ^:parallel clickhouse-base-types-test-enums (mt/test-driver :clickhouse (testing "enums" @@ -41,7 +48,11 @@ :database-required false, :database-type "Nullable(Enum16('SHOW DATABASES' = 0, 'SHOW TABLES' = 1, 'SHOW COLUMNS' = 2))", :name "c6"}} - (desc-table table-name))))) + (desc-table table-name))))))) + +(deftest ^:parallel clickhouse-base-types-test-dates + (mt/test-driver + :clickhouse (testing "dates" (let [table-name "date_base_types"] (is (= #{{:base-type :type/Date, @@ -60,7 +71,11 @@ :database-required false, :database-type "Nullable(Date32)", :name "c4"}} - (desc-table table-name))))) + (desc-table table-name))))))) + +(deftest ^:parallel clickhouse-base-types-test-datetimes + (mt/test-driver + :clickhouse (testing "datetimes" (let [table-name "datetime_base_types"] (is (= #{{:base-type :type/DateTime, @@ -95,7 +110,11 @@ :database-required false, :database-type "Nullable(DateTime)", :name "c8"}} - (desc-table table-name))))) + (desc-table table-name))))))) + +(deftest ^:parallel clickhouse-base-types-test-integers + (mt/test-driver + :clickhouse (testing "integers" (let [table-name "integer_base_types"] (is (= #{{:base-type :type/Integer, @@ -150,7 +169,11 @@ :database-required false, :database-type "Nullable(Int32)", :name "c13"}} - (desc-table table-name))))) + (desc-table table-name))))))) + +(deftest ^:parallel clickhouse-base-types-test-numerics + (mt/test-driver + :clickhouse (testing "numerics" (let [table-name "numeric_base_types"] (is (= #{{:base-type :type/Float, @@ -193,7 +216,11 @@ :database-required false, :database-type "Nullable(Decimal(76, 42))", :name "c10"}} - (desc-table table-name))))) + (desc-table table-name))))))) + +(deftest ^:parallel clickhouse-base-types-test-strings + (mt/test-driver + :clickhouse (testing "strings" (let [table-name "string_base_types"] (is (= #{{:base-type :type/Text, @@ -216,7 +243,11 @@ :database-required true, :database-type "LowCardinality(FixedString(4))", :name "c5"}} - (desc-table table-name))))) + (desc-table table-name))))))) + +(deftest ^:parallel clickhouse-base-types-test-arrays + (mt/test-driver + :clickhouse (testing "arrays" (let [table-name "array_base_types"] (is (= #{{:base-type :type/Array, @@ -235,7 +266,11 @@ :database-required true, :database-type "Array(Array(Array(String)))", :name "c4"}} - (desc-table table-name))))) + (desc-table table-name))))))) + +(deftest ^:parallel clickhouse-base-types-test-low-cardinality-nullable + (mt/test-driver + :clickhouse (testing "low cardinality nullable" (let [table-name "low_cardinality_nullable_base_types"] (is (= #{{:base-type :type/Text, @@ -246,7 +281,11 @@ :database-required true, :database-type "LowCardinality(Nullable(FixedString(16)))", :name "c2"}} - (desc-table table-name))))) + (desc-table table-name))))))) + +(deftest ^:parallel clickhouse-base-types-test-misc + (mt/test-driver + :clickhouse (testing "everything else" (let [table-name "misc_base_types"] (is (= #{{:base-type :type/Boolean, @@ -290,3 +329,120 @@ :database-type "Tuple(String, Int32)", :name "c10"}} (desc-table table-name))))))) + +(deftest ^:parallel clickhouse-boolean-type-metadata + (mt/test-driver + :clickhouse + (let [result (-> {:query "SELECT false, 123, true"} mt/native-query qp/process-query) + [[c1 _ c3]] (-> result qp.test/rows)] + (testing "column should be of type :type/Boolean" + (is (= :type/Boolean (-> result :data :results_metadata :columns first :base_type))) + (is (= :type/Boolean (transduce identity (driver.common/values->base-type) [c1, c3]))) + (is (= :type/Boolean (driver.common/class->base-type (class c1)))))))) + +(def ^:private base-field + {:database-is-auto-increment false + :json-unfolding false + :database-required true}) + +(deftest ^:parallel clickhouse-filtered-aggregate-functions-test-table-metadata + (mt/test-driver + :clickhouse + (is (= {:name "aggregate_functions_filter_test" + :fields #{(merge base-field + {:name "idx" + :database-type "UInt8" + :base-type :type/Integer + :database-position 0}) + (merge base-field + {:name "lowest_value" + :database-type "SimpleAggregateFunction(min, UInt8)" + :base-type :type/Integer + :database-position 2}) + (merge base-field + {:name "count" + :database-type "SimpleAggregateFunction(sum, Int64)" + :base-type :type/BigInteger + :database-position 3})}} + (ctd/do-with-test-db + (fn [db] + (driver/describe-table :clickhouse db {:name "aggregate_functions_filter_test"}))))))) + +(deftest ^:parallel clickhouse-filtered-aggregate-functions-test-result-set + (mt/test-driver + :clickhouse + (is (= [[42 144 255255]] + (qp.test/formatted-rows + [int int int] + :format-nil-values + (ctd/do-with-test-db + (fn [db] + (data/with-db db + (data/run-mbql-query + aggregate_functions_filter_test + {}))))))))) +(def ^:private test-tables + #{{:description nil, + :name "table1", + :schema "metabase_db_scan_test"} + {:description nil, + :name "table2", + :schema "metabase_db_scan_test"}}) + +(deftest ^:parallel clickhouse-describe-database-single + (mt/test-driver + :clickhouse + (t2.with-temp/with-temp + [Database db {:engine :clickhouse + :details (merge {:scan-all-databases nil} + (tx/dbdef->connection-details + :clickhouse :db + {:database-name "metabase_db_scan_test"}))}] + (let [describe-result (driver/describe-database :clickhouse db)] + (is (= {:tables test-tables} describe-result)))))) + +(deftest ^:parallel clickhouse-describe-database-all + (mt/test-driver + :clickhouse + (t2.with-temp/with-temp + [Database db {:engine :clickhouse + :details (merge {:scan-all-databases true} + (tx/dbdef->connection-details + :clickhouse :db + {:database-name "default"}))}] + (let [describe-result (driver/describe-database :clickhouse db)] + ;; check the existence of at least some test tables here + (doseq [table test-tables] + (is (contains? (:tables describe-result) table))) + ;; should not contain any ClickHouse system tables + (is (not (some #(= (:schema %) "system") + (:tables describe-result)))) + (is (not (some #(= (:schema %) "information_schema") + (:tables describe-result)))) + (is (not (some #(= (:schema %) "INFORMATION_SCHEMA") + (:tables describe-result)))))))) + +(deftest ^:parallel clickhouse-describe-database-multiple + (mt/test-driver + :clickhouse + (t2.with-temp/with-temp + [Database db {:engine :clickhouse + :details (tx/dbdef->connection-details + :clickhouse :db + {:database-name "metabase_db_scan_test information_schema"})}] + (let [{:keys [tables] :as _describe-result} + (driver/describe-database :clickhouse db) + tables-table {:name "tables" + :description nil + :schema "information_schema"} + columns-table {:name "columns" + :description nil + :schema "information_schema"}] + + ;; tables from `metabase_db_scan_test` + (doseq [table test-tables] + (is (contains? tables table))) + + ;; tables from `information_schema` + (is (contains? tables tables-table)) + (is (contains? tables columns-table)))))) diff --git a/test/metabase/driver/clickhouse_substitution_test.clj b/test/metabase/driver/clickhouse_substitution_test.clj index 291955b..22b520a 100644 --- a/test/metabase/driver/clickhouse_substitution_test.clj +++ b/test/metabase/driver/clickhouse_substitution_test.clj @@ -2,8 +2,6 @@ #_{:clj-kondo/ignore [:unsorted-required-namespaces]} (:require [clojure.test :refer :all] [java-time.api :as t] - [metabase.driver.clickhouse-base-types-test] - [metabase.driver.clickhouse-temporal-bucketing-test] [metabase.query-processor :as qp] [metabase.test :as mt] [metabase.test.data :as data] @@ -38,7 +36,7 @@ (s/defn ^:private local-date-now :- LocalDate [] (LocalDate/now clock)) (s/defn ^:private local-date-time-now :- LocalDateTime [] (LocalDateTime/now clock)) -(deftest clickhouse-variables-field-filters-datetime-and-datetime64 +(deftest ^:parallel clickhouse-variables-field-filters-datetime-and-datetime64 (mt/test-driver :clickhouse (mt/with-clock clock @@ -149,7 +147,7 @@ (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next12years" db))))))))))))))) -(deftest clickhouse-variables-field-filters-date-and-date32 +(deftest ^:parallel clickhouse-variables-field-filters-date-and-date32 (mt/test-driver :clickhouse (mt/with-clock clock diff --git a/test/metabase/driver/clickhouse_temporal_bucketing_test.clj b/test/metabase/driver/clickhouse_temporal_bucketing_test.clj index 011692b..b0625de 100644 --- a/test/metabase/driver/clickhouse_temporal_bucketing_test.clj +++ b/test/metabase/driver/clickhouse_temporal_bucketing_test.clj @@ -12,7 +12,7 @@ ;; start_of_year == '2022-01-01 00:00:00' ;; mid_of_year == '2022-06-20 06:32:54' ;; end_of_year == '2022-12-31 23:59:59' -(deftest clickhouse-temporal-bucketing-server-tz +(deftest ^:parallel clickhouse-temporal-bucketing-server-tz (mt/test-driver :clickhouse (defn- start-of-year [unit] @@ -102,7 +102,7 @@ (is (= [[4]] (end-of-year :quarter-of-year))))))) -(deftest clickhouse-temporal-bucketing-column-tz +(deftest ^:parallel clickhouse-temporal-bucketing-column-tz (mt/test-driver :clickhouse (defn- start-of-year [unit] diff --git a/test/metabase/driver/clickhouse_test.clj b/test/metabase/driver/clickhouse_test.clj index fe7f1e3..663a561 100644 --- a/test/metabase/driver/clickhouse_test.clj +++ b/test/metabase/driver/clickhouse_test.clj @@ -2,38 +2,45 @@ "Tests for specific behavior of the ClickHouse driver." #_{:clj-kondo/ignore [:unsorted-required-namespaces]} (:require [cljc.java-time.format.date-time-formatter :as date-time-formatter] - [cljc.java-time.local-date :as local-date] [cljc.java-time.offset-date-time :as offset-date-time] [cljc.java-time.temporal.chrono-unit :as chrono-unit] [clojure.test :refer :all] [metabase.driver :as driver] - [metabase.driver.clickhouse-base-types-test] + [metabase.driver.clickhouse-data-types-test] + [metabase.driver.clickhouse-introspection-test] [metabase.driver.clickhouse-substitution-test] [metabase.driver.clickhouse-temporal-bucketing-test] - [metabase.driver.common :as driver.common] [metabase.driver.sql :as driver.sql] [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] - [metabase.models.database :refer [Database]] [metabase.query-processor :as qp] [metabase.query-processor.test-util :as qp.test] [metabase.test :as mt] [metabase.test.data :as data] [metabase.test.data [interface :as tx]] [metabase.test.data.clickhouse :as ctd] - [taoensso.nippy :as nippy] - [toucan2.tools.with-temp :as t2.with-temp])) + [taoensso.nippy :as nippy])) (set! *warn-on-reflection* true) +(use-fixtures :once ctd/create-test-db!) -(deftest clickhouse-server-timezone +(deftest ^:parallel clickhouse-version + (mt/test-driver + :clickhouse + (let [version (driver/dbms-version :clickhouse (mt/db))] + (is (number? (get-in version [:semantic-version :major]))) + (is (number? (get-in version [:semantic-version :minor]))) + (is (string? (get version :version)))))) + +(deftest ^:parallel clickhouse-server-timezone (mt/test-driver :clickhouse (is (= "UTC" - (let [spec (sql-jdbc.conn/connection-details->spec :clickhouse {})] + (let [details (tx/dbdef->connection-details :clickhouse :db {:database-name "default"}) + spec (sql-jdbc.conn/connection-details->spec :clickhouse details)] (driver/db-default-timezone :clickhouse spec)))))) -(deftest clickhouse-now-converted-to-timezone +(deftest ^:parallel clickhouse-now-converted-to-timezone (mt/test-driver :clickhouse (let [[[utc-now shanghai-now]] @@ -48,457 +55,7 @@ (offset-date-time/parse utc-now date-time-formatter/iso-offset-date-time) (offset-date-time/parse shanghai-now date-time-formatter/iso-offset-date-time)))))))) -(deftest clickhouse-decimals - (mt/test-driver - :clickhouse - (data/dataset - (tx/dataset-definition "metabase_tests_decimal" - ["test-data-decimal" - [{:field-name "my_money" - :base-type {:native "Decimal(12,4)"}}] - [[1.0] [23.1337] [42.0] [42.0]]]) - (testing "simple division" - (is - (= 21.0 - (-> (data/run-mbql-query test-data-decimal - {:expressions {:divided [:/ $my_money 2]} - :filter [:> [:expression :divided] 1.0] - :breakout [[:expression :divided]] - :order-by [[:desc [:expression :divided]]] - :limit 1}) - qp.test/first-row last float)))) - (testing "divided decimal precision" - (is - (= 1.8155331831916208 - (-> (data/run-mbql-query test-data-decimal - {:expressions {:divided [:/ 42 $my_money]} - :filter [:= $id 2] - :limit 1}) - qp.test/first-row last double))))))) - -(deftest clickhouse-array-string - (mt/test-driver - :clickhouse - (is - (= "[foo, bar]" - (-> (data/dataset - (tx/dataset-definition "metabase_tests_array_string" - ["test-data-array-string" - [{:field-name "my_array" - :base-type {:native "Array(String)"}}] - [[(into-array (list "foo" "bar"))]]]) - (data/run-mbql-query test-data-array-string {:limit 1})) - qp.test/first-row - last))))) - -(deftest clickhouse-array-uint64 - (mt/test-driver - :clickhouse - (is - (= "[23, 42]" - (-> (data/dataset - (tx/dataset-definition "metabase_tests_array_uint" - ["test-data-array-uint64" - [{:field-name "my_array" - :base-type {:native "Array(UInt64)"}}] - [[(into-array (list 23 42))]]]) - (data/run-mbql-query test-data-array-uint64 {:limit 1})) - qp.test/first-row - last))))) - -(deftest clickhouse-array-of-arrays - (mt/test-driver - :clickhouse - (let [row1 (into-array (list - (into-array (list "foo" "bar")) - (into-array (list "qaz" "qux")))) - row2 (into-array nil) - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_arrays" - ["test-data-array-of-arrays" - [{:field-name "my_array_of_arrays" - :base-type {:native "Array(Array(String))"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-arrays {})) - result (ctd/rows-without-index query-result)] - (is (= [["[[foo, bar], [qaz, qux]]"], ["[]"]] result))))) - -(deftest clickhouse-low-cardinality-array - (mt/test-driver - :clickhouse - (let [row1 (into-array (list "foo" "bar")) - row2 (into-array nil) - query-result (data/dataset - (tx/dataset-definition "metabase_tests_low_cardinality_array" - ["test-data-low-cardinality-array" - [{:field-name "my_low_card_array" - :base-type {:native "Array(LowCardinality(String))"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-low-cardinality-array {})) - result (ctd/rows-without-index query-result)] - (is (= [["[foo, bar]"], ["[]"]] result))))) - -(deftest clickhouse-array-of-nullables - (mt/test-driver - :clickhouse - (let [row1 (into-array (list "foo" nil "bar")) - row2 (into-array nil) - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_nullables" - ["test-data-array-of-nullables" - [{:field-name "my_array_of_nullables" - :base-type {:native "Array(Nullable(String))"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-nullables {})) - result (ctd/rows-without-index query-result)] - (is (= [["[foo, null, bar]"], ["[]"]] result))))) - -(deftest clickhouse-array-of-booleans - (mt/test-driver - :clickhouse - (let [row1 (into-array (list true false true)) - row2 (into-array nil) - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_booleans" - ["test-data-array-of-booleans" - [{:field-name "my_array_of_booleans" - :base-type {:native "Array(Boolean)"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-booleans {})) - result (ctd/rows-without-index query-result)] - (is (= [["[true, false, true]"], ["[]"]] result))))) - -(deftest clickhouse-array-of-uint8 - (mt/test-driver - :clickhouse - (let [row1 (into-array (list 42 100 2)) - row2 (into-array nil) - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_uint8" - ["test-data-array-of-uint8" - [{:field-name "my_array_of_uint8" - :base-type {:native "Array(UInt8)"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-uint8 {})) - result (ctd/rows-without-index query-result)] - (is (= [["[42, 100, 2]"], ["[]"]] result))))) - -(deftest clickhouse-array-of-floats - (mt/test-driver - :clickhouse - (let [row1 (into-array (list 1.2 3.4)) - row2 (into-array nil) - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_floats" - ["test-data-array-of-floats" - [{:field-name "my_array_of_floats" - :base-type {:native "Array(Float64)"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-floats {})) - result (ctd/rows-without-index query-result)] - (is (= [["[1.2, 3.4]"], ["[]"]] result))))) - -(deftest clickhouse-array-of-dates - (mt/test-driver - :clickhouse - (let [row1 (into-array - (list - (local-date/parse "2022-12-06") - (local-date/parse "2021-10-19"))) - row2 (into-array nil) - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_dates" - ["test-data-array-of-dates" - [{:field-name "my_array_of_dates" - :base-type {:native "Array(Date)"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-dates {})) - result (ctd/rows-without-index query-result)] - (is (= [["[2022-12-06, 2021-10-19]"], ["[]"]] result))))) - -(deftest clickhouse-array-of-date32 - (mt/test-driver - :clickhouse - (let [row1 (into-array - (list - (local-date/parse "2122-12-06") - (local-date/parse "2099-10-19"))) - row2 (into-array nil) - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_date32" - ["test-data-array-of-date32" - [{:field-name "my_array_of_date32" - :base-type {:native "Array(Date32)"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-date32 {})) - result (ctd/rows-without-index query-result)] - (is (= [["[2122-12-06, 2099-10-19]"], ["[]"]] result))))) - -(deftest clickhouse-array-of-datetime - (mt/test-driver - :clickhouse - (let [row1 (into-array - (list - (offset-date-time/parse "2022-12-06T18:28:31Z") - (offset-date-time/parse "2021-10-19T13:12:44Z"))) - row2 (into-array nil) - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_datetime" - ["test-data-array-of-datetime" - [{:field-name "my_array_of_datetime" - :base-type {:native "Array(DateTime)"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-datetime {})) - result (ctd/rows-without-index query-result)] - (is (= [["[2022-12-06T18:28:31, 2021-10-19T13:12:44]"], ["[]"]] result))))) - -(deftest clickhouse-array-of-datetime64 - (mt/test-driver - :clickhouse - (let [row1 (into-array - (list - (offset-date-time/parse "2022-12-06T18:28:31.123Z") - (offset-date-time/parse "2021-10-19T13:12:44.456Z"))) - row2 (into-array nil) - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_datetime64" - ["test-data-array-of-datetime64" - [{:field-name "my_array_of_datetime64" - :base-type {:native "Array(DateTime64(3))"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-datetime64 {})) - result (ctd/rows-without-index query-result)] - (is (= [["[2022-12-06T18:28:31.123, 2021-10-19T13:12:44.456]"], ["[]"]] result))))) - -(deftest clickhouse-array-of-decimals - (mt/test-driver - :clickhouse - (let [row1 (into-array (list "12345123.123456789" "78.245")) - row2 nil - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_decimals" - ["test-data-array-of-decimals" - [{:field-name "my_array_of_decimals" - :base-type {:native "Array(Decimal(18, 9))"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-decimals {})) - result (ctd/rows-without-index query-result)] - (is (= [["[12345123.123456789, 78.245000000]"], ["[]"]] result))))) - -(deftest clickhouse-array-of-tuples - (mt/test-driver - :clickhouse - (let [row1 (into-array (list (list "foobar" 1234) (list "qaz" 0))) - row2 nil - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_tuples" - ["test-data-array-of-tuples" - [{:field-name "my_array_of_tuples" - :base-type {:native "Array(Tuple(String, UInt32))"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-tuples {})) - result (ctd/rows-without-index query-result)] - (is (= [["[[foobar, 1234], [qaz, 0]]"], ["[]"]] result))))) - -(deftest clickhouse-array-of-uuids - (mt/test-driver - :clickhouse - (let [row1 (into-array (list "2eac427e-7596-11ed-a1eb-0242ac120002" - "2eac44f4-7596-11ed-a1eb-0242ac120002")) - row2 nil - query-result (data/dataset - (tx/dataset-definition "metabase_tests_array_of_uuids" - ["test-data-array-of-uuids" - [{:field-name "my_array_of_uuids" - :base-type {:native "Array(UUID)"}}] - [[row1] [row2]]]) - (data/run-mbql-query test-data-array-of-uuids {})) - result (ctd/rows-without-index query-result)] - (is (= [["[2eac427e-7596-11ed-a1eb-0242ac120002, 2eac44f4-7596-11ed-a1eb-0242ac120002]"], ["[]"]] result))))) - -(deftest clickhouse-nullable-strings - (mt/test-driver - :clickhouse - (data/dataset - (tx/dataset-definition - "metabase_tests_nullable_strings" - ["test-data-nullable-strings" - [{:field-name "mystring" :base-type :type/Text}] - [["foo"] ["bar"] [" "] [""] [nil]]]) - (testing "null strings count" - (is (= 2 - (-> (data/run-mbql-query test-data-nullable-strings - {:filter [:is-null $mystring] - :aggregation [:count]}) - qp.test/first-row last)))) - (testing "nullable strings not null filter" - (is (= 3 - (-> (data/run-mbql-query test-data-nullable-strings - {:filter [:not-null $mystring] - :aggregation [:count]}) - qp.test/first-row last)))) - (testing "filter nullable string by value" - (is (= 1 - (-> (data/run-mbql-query test-data-nullable-strings - {:filter [:= $mystring "foo"] - :aggregation [:count]}) - qp.test/first-row last))))))) - -(deftest clickhouse-non-latin-strings - (mt/test-driver - :clickhouse - (testing "basic filtering" - (is (= [[1 "Я_1"] [3 "Я_2"] [4 "Я"]] - (qp.test/formatted-rows - [int str] - :format-nil-values - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (data/run-mbql-query - metabase_test_lowercases - {:filter [:contains $mystring "Я"]})))))))) - (testing "case-insensitive non-latin filtering" - (is (= [[1 "Я_1"] [3 "Я_2"] [4 "Я"] [5 "я"]] - (qp.test/formatted-rows - [int str] - :format-nil-values - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (data/run-mbql-query - metabase_test_lowercases - {:filter [:contains $mystring "Я" - {:case-sensitive false}]})))))))))) - -(deftest clickhouse-datetime64-filter - (mt/test-driver - :clickhouse - (let [row1 "2022-03-03 03:03:03.333" - row2 "2022-03-03 03:03:03.444" - row3 "2022-03-03 03:03:03" - query-result (data/dataset - (tx/dataset-definition "metabase_tests_datetime64" - ["test-data-datetime64" - [{:field-name "milli_sec" - :base-type {:native "DateTime64(3)"}}] - [[row1] [row2] [row3]]]) - (data/run-mbql-query test-data-datetime64 {:filter [:= $milli_sec "2022-03-03T03:03:03.333Z"]})) - result (ctd/rows-without-index query-result)] - (is (= [["2022-03-03T03:03:03.333Z"]] result))))) - -(deftest clickhouse-datetime-filter - (mt/test-driver - :clickhouse - (let [row1 "2022-03-03 03:03:03" - row2 "2022-03-03 03:03:04" - row3 "2022-03-03 03:03:05" - query-result (data/dataset - (tx/dataset-definition "metabase_tests_datetime" - ["test-data-datetime" - [{:field-name "second" - :base-type {:native "DateTime"}}] - [[row1] [row2] [row3]]]) - (data/run-mbql-query test-data-datetime {:filter [:= $second "2022-03-03T03:03:04Z"]})) - result (ctd/rows-without-index query-result)] - (is (= [["2022-03-03T03:03:04Z"]] result))))) - -(deftest clickhouse-booleans - (mt/test-driver - :clickhouse - (let [[row1 row2 row3 row4] [["#1" true] ["#2" false] ["#3" false] ["#4" true]] - query-result (data/dataset - (tx/dataset-definition "metabase_tests_booleans" - ["test-data-booleans" - [{:field-name "name" - :base-type :type/Text} - {:field-name "is_active" - :base-type :type/Boolean}] - [row1 row2 row3 row4]]) - (data/run-mbql-query test-data-booleans {:filter [:= $is_active false]})) - rows (qp.test/rows query-result) - result (map #(drop 1 %) rows)] ; remove db "index" which is the first column in the result set - (is (= [row2 row3] result))))) - -(def ^:private base-field - {:database-is-auto-increment false - :json-unfolding false - :database-required true}) - -(deftest clickhouse-enums-values-test - (mt/test-driver - :clickhouse - (testing "select enums values as strings" - (is (= [["foo" "house" "qaz"] - ["foo bar" "click" "qux"] - ["bar" "house" "qaz"]] - (qp.test/formatted-rows - [str str str] - :format-nil-values - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (data/run-mbql-query - enums_test - {})))))))) - (testing "filtering enum values" - (is (= [["useqa"]] - (qp.test/formatted-rows - [str] - :format-nil-values - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (data/run-mbql-query - enums_test - {:expressions {"test" [:concat - [:substring $enum2 3 3] - [:substring $enum3 1 2]]} - :fields [[:expression "test"]] - :filter [:= $enum1 "foo"]})))))))))) - -(deftest clickhouse-ipv4query-test - (mt/test-driver - :clickhouse - (is (= [[1]] - (qp.test/formatted-rows - [int] - :format-nil-values - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (data/run-mbql-query - ipaddress_test - {:filter [:= $ipvfour "127.0.0.1"] - :aggregation [[:count]]}))))))))) - -(deftest clickhouse-ip-serialization-test - (mt/test-driver - :clickhouse - (is (= [["127.0.0.1" "0:0:0:0:0:ffff:7f00:1"] - ["0.0.0.0" "0:0:0:0:0:ffff:0:0"] - [nil nil]] - (qp.test/formatted-rows - [str str] - (ctd/do-with-test-db - (fn [db] (data/with-db db (data/run-mbql-query ipaddress_test {}))))))))) - -(defn- map-as-string [^java.util.LinkedHashMap m] (.toString m)) -(deftest clickhouse-simple-map-test - (mt/test-driver - :clickhouse - (is (= [["{key1=1, key2=10}"] ["{key1=2, key2=20}"] ["{key1=3, key2=30}"]] - (qp.test/formatted-rows - [map-as-string] - :format-nil-values - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (data/run-mbql-query - maps_test - {}))))))))) - -(deftest clickhouse-connection-string +(deftest ^:parallel clickhouse-connection-string (testing "connection with no additional options" (is (= ctd/default-connection-params (sql-jdbc.conn/connection-details->spec @@ -528,17 +85,7 @@ :clickhouse {:dbname nil}))))) -(deftest clickhouse-boolean-result-metadata - (mt/test-driver - :clickhouse - (let [result (-> {:query "SELECT false, 123, true"} mt/native-query qp/process-query) - [[c1 _ c3]] (-> result qp.test/rows)] - (testing "column should be of type :type/Boolean" - (is (= :type/Boolean (-> result :data :results_metadata :columns first :base_type))) - (is (= :type/Boolean (transduce identity (driver.common/values->base-type) [c1, c3]))) - (is (= :type/Boolean (driver.common/class->base-type (class c1)))))))) - -(deftest clickhouse-tls +(deftest ^:parallel clickhouse-tls (mt/test-driver :clickhouse (let [working-dir (System/getProperty "user.dir") @@ -566,148 +113,7 @@ :dbname "default system" :additional-options additional-options})))))))) -(deftest clickhouse-filtered-aggregate-functions-test - (mt/test-driver - :clickhouse - (testing "AggregateFunction columns are filtered" - (testing "from the table metadata" - (is (= {:name "aggregate_functions_filter_test" - :fields #{(merge base-field - {:name "idx" - :database-type "UInt8" - :base-type :type/Integer - :database-position 0}) - (merge base-field - {:name "lowest_value" - :database-type "SimpleAggregateFunction(min, UInt8)" - :base-type :type/Integer - :database-position 2}) - (merge base-field - {:name "count" - :database-type "SimpleAggregateFunction(sum, Int64)" - :base-type :type/BigInteger - :database-position 3})}} - (ctd/do-with-test-db - (fn [db] - (driver/describe-table :clickhouse db {:name "aggregate_functions_filter_test"})))))) - (testing "from the result set" - (is (= [[42 144 255255]] - (qp.test/formatted-rows - [int int int] - :format-nil-values - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (data/run-mbql-query - aggregate_functions_filter_test - {}))))))))))) - -(deftest clickhouse-describe-database - (let [test-tables - #{{:description nil, - :name "table1", - :schema "metabase_db_scan_test"} - {:description nil, - :name "table2", - :schema "metabase_db_scan_test"}}] - (testing "scanning a single database" - (t2.with-temp/with-temp - [Database db {:engine :clickhouse - :details {:dbname "metabase_db_scan_test" - :scan-all-databases nil}}] - (let [describe-result (driver/describe-database :clickhouse db)] - (is (= - {:tables test-tables} - describe-result))))) - (testing "scanning all databases" - (t2.with-temp/with-temp - [Database db {:engine :clickhouse - :details {:dbname "default" - :scan-all-databases true}}] - (let [describe-result (driver/describe-database :clickhouse db)] - ;; check the existence of at least some test tables here - (doseq [table test-tables] - (is (contains? (:tables describe-result) - table))) - ;; should not contain any ClickHouse system tables - (is (not (some #(= (:schema %) "system") - (:tables describe-result)))) - (is (not (some #(= (:schema %) "information_schema") - (:tables describe-result)))) - (is (not (some #(= (:schema %) "INFORMATION_SCHEMA") - (:tables describe-result))))))) - (testing "scanning multiple databases" - (t2.with-temp/with-temp - [Database db {:engine :clickhouse - :details {:dbname "metabase_db_scan_test information_schema"}}] - (let [{:keys [tables] :as _describe-result} - (driver/describe-database :clickhouse db) - tables-table {:name "tables" - :description nil - :schema "information_schema"} - columns-table {:name "columns" - :description nil - :schema "information_schema"}] - - ;; tables from `metabase_db_scan_test` - (doseq [table test-tables] - (is (contains? tables table))) - - ;; tables from `information_schema` - (is (contains? tables tables-table)) - (is (contains? tables columns-table))))))) - -;; Metabase has pretty extensive testing for sum-where and count-where -;; However, this ClickHouse-specific corner case is not covered -(deftest clickhouse-sum-where - (mt/test-driver - :clickhouse - (testing "int values (with matching rows)" - (is (= [[8]] - (qp.test/formatted-rows - [int] - :format-nil-values - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (data/run-mbql-query - sum_if_test_int - {:aggregation [[:sum-where $int_value [:= $discriminator "bar"]]]})))))))) - (testing "int values (no matching rows)" - (is (= [[0]] - (qp.test/formatted-rows - [int] - :format-nil-values - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (data/run-mbql-query - sum_if_test_int - {:aggregation [[:sum-where $int_value [:= $discriminator "qaz"]]]})))))))) - (testing "double values (with matching rows)" - (is (= [[9.27]] - (qp.test/formatted-rows - [double] - :format-nil-values - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (data/run-mbql-query - sum_if_test_float - {:aggregation [[:sum-where $float_value [:= $discriminator "bar"]]]})))))))) - (testing "double values (no matching rows)" - (is (= [[0.0]] - (qp.test/formatted-rows - [double] - :format-nil-values - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (data/run-mbql-query - sum_if_test_float - {:aggregation [[:sum-where $float_value [:= $discriminator "qaz"]]]})))))))))) - -(deftest clickhouse-nippy +(deftest ^:parallel clickhouse-nippy (mt/test-driver :clickhouse (testing "UnsignedByte" @@ -727,7 +133,9 @@ (mt/test-driver :clickhouse (let [default-role (driver.sql/default-database-role :clickhouse nil) - spec (sql-jdbc.conn/connection-details->spec :clickhouse {:user "metabase_test_user"})] + details (merge {:user "metabase_test_user"} + (tx/dbdef->connection-details :clickhouse :db {:database-name "default"})) + spec (sql-jdbc.conn/connection-details->spec :clickhouse details)] (testing "default role is NONE" (is (= default-role "NONE"))) (testing "does not throw with an existing role" @@ -749,7 +157,7 @@ (fn [^java.sql.Connection conn] (driver/set-role! :clickhouse conn "asdf"))))))))) -(deftest clickhouse-query-formatting +(deftest ^:parallel clickhouse-query-formatting (mt/test-driver :clickhouse (let [query (data/mbql-query venues {:fields [$id] :order-by [[:asc $id]] :limit 5}) @@ -760,28 +168,13 @@ (testing "pretty" (is (= "SELECT\n `test_data`.`venues`.`id` AS `id`\nFROM\n `test_data`.`venues`\nORDER BY\n `test_data`.`venues`.`id` ASC\nLIMIT\n 5" pretty)))))) -(deftest clickhouse-datetime-diff-nullable - (mt/test-driver - :clickhouse - (is (= [[170 202] [nil nil] [nil nil] [nil nil]] - (ctd/do-with-test-db - (fn [db] - (data/with-db db - (->> (data/run-mbql-query - datetime_diff_nullable - {:fields [[:expression "dt64,dt"] - [:expression "dt64,d"]] - :expressions - {"dt64,dt" [:datetime-diff $dt64 $dt :day] - "dt64,d" [:datetime-diff $dt64 $d :day]}}) - (mt/formatted-rows [int int]))))))))) - -(deftest clickhouse-can-connect +(deftest ^:parallel clickhouse-can-connect (mt/test-driver :clickhouse - (ctd/create-test-db!) (doall (for [[username password] [["default" ""] ["user_with_password" "foo@bar!"]] database ["default" "Special@Characters~" "'Escaping'"]] (testing (format "User `%s` can connect to `%s`" username database) - (is (true? (driver/can-connect? :clickhouse {:user username :password password :dbname database})))))))) + (let [details (merge {:user username :password password} + (tx/dbdef->connection-details :clickhouse :db {:database-name database}))] + (is (true? (driver/can-connect? :clickhouse details))))))))) diff --git a/test/metabase/test/data/clickhouse.clj b/test/metabase/test/data/clickhouse.clj index b70b5dd..d4fdeb0 100644 --- a/test/metabase/test/data/clickhouse.clj +++ b/test/metabase/test/data/clickhouse.clj @@ -21,6 +21,17 @@ (sql-jdbc.tx/add-test-extensions! :clickhouse) +(def default-connection-params + {:classname "com.clickhouse.jdbc.ClickHouseDriver" + :subprotocol "clickhouse" + :subname "//localhost:8123/default" + :user "default" + :password "" + :ssl false + :use_no_proxy false + :use_server_time_zone_for_dates true + :product_name "metabase/1.3.4"}) + (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Boolean] [_ _] "Boolean") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/BigInteger] [_ _] "Int64") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Char] [_ _] "String") @@ -81,48 +92,34 @@ (defmethod tx/supports-time-type? :clickhouse [_driver] false) -(def default-connection-params - {:classname "com.clickhouse.jdbc.ClickHouseDriver" - :subprotocol "clickhouse" - :subname "//localhost:8123/default" - :user "default" - :password "" - :ssl false - :use_no_proxy false - :use_server_time_zone_for_dates true - :product_name "metabase/1.3.3"}) - (defn rows-without-index "Remove the Metabase index which is the first column in the result set" [query-result] (map #(drop 1 %) (qp.test/rows query-result))) -(defn- test-db-details - [] - {:engine :clickhouse - :details (tx/dbdef->connection-details - :clickhouse :db {:database-name "metabase_test"})}) - (def ^:private test-db-initialized? (atom false)) (defn create-test-db! "Create a ClickHouse database called `metabase_test` and initialize some test data" - [] + [f] (when (not @test-db-initialized?) - (jdbc/with-db-connection - [conn (sql-jdbc.conn/connection-details->spec :clickhouse (test-db-details))] - (let [statements (as-> (slurp "modules/drivers/clickhouse/test/metabase/test/data/datasets.sql") s - (str/split s #";") - (map str/trim s) - (filter seq s))] - (jdbc/db-do-commands conn statements) - (reset! test-db-initialized? true))))) + (let [details (tx/dbdef->connection-details :clickhouse :db {:database-name "metabase_test"})] + (jdbc/with-db-connection + [conn (sql-jdbc.conn/connection-details->spec :clickhouse (merge {:engine :clickhouse} details))] + (let [statements (as-> (slurp "modules/drivers/clickhouse/test/metabase/test/data/datasets.sql") s + (str/split s #";") + (map str/trim s) + (filter seq s))] + (jdbc/db-do-commands conn statements) + (reset! test-db-initialized? true))))) + (f)) (defn do-with-test-db "Execute a test function using the test dataset" {:style/indent 0} [f] - (create-test-db!) (t2.with-temp/with-temp - [Database database (test-db-details)] + [Database database + {:engine :clickhouse + :details (tx/dbdef->connection-details :clickhouse :db {:database-name "metabase_test"})}] (sync-metadata/sync-db-metadata! database) (f database)))