diff --git a/.docker/clickhouse/single_node_tls/Dockerfile b/.docker/clickhouse/single_node_tls/Dockerfile index c6be222..a3fe3fc 100644 --- a/.docker/clickhouse/single_node_tls/Dockerfile +++ b/.docker/clickhouse/single_node_tls/Dockerfile @@ -1,4 +1,4 @@ -FROM clickhouse/clickhouse-server:24.4-alpine +FROM clickhouse/clickhouse-server:24.5-alpine COPY .docker/clickhouse/single_node_tls/certificates /etc/clickhouse-server/certs RUN chown clickhouse:clickhouse -R /etc/clickhouse-server/certs \ && chmod 600 /etc/clickhouse-server/certs/* \ diff --git a/.docker/setup/Dockerfile b/.docker/setup/Dockerfile new file mode 100644 index 0000000..8e60b33 --- /dev/null +++ b/.docker/setup/Dockerfile @@ -0,0 +1,3 @@ +FROM python:3.11.9-alpine +COPY . /app/ +RUN pip install -r /app/requirements.txt diff --git a/.docker/setup/requirements.txt b/.docker/setup/requirements.txt new file mode 100644 index 0000000..6e42168 --- /dev/null +++ b/.docker/setup/requirements.txt @@ -0,0 +1 @@ +requests==2.32.2 diff --git a/.docker/setup/setup.py b/.docker/setup/setup.py new file mode 100644 index 0000000..097d419 --- /dev/null +++ b/.docker/setup/setup.py @@ -0,0 +1,147 @@ +import copy +import logging +import os +import pprint + +import requests + +host = os.environ.get('host') if os.environ.get('host') else 'http://localhost' +port = os.environ.get('port') if os.environ.get('port') else '3000' +admin_email = os.environ.get('admin_email') if os.environ.get('admin_email') else 'admin@example.com' +user_email = os.environ.get('user_email') if os.environ.get('user_email') else 'user@example.com' +password = os.environ.get('password') if os.environ.get('password') else 'metabot1' +site_name = 'ClickHouse test' + +endpoints = { + 'health_check': '/api/health', + 'properties': '/api/session/properties', + 'setup': '/api/setup', + 'database': '/api/database', + 'login': '/api/session', + 'user': '/api/user', +} +for k, v in endpoints.items(): + endpoints[k] = f"{host}:{port}{v}" + +db_base_payload = { + "is_on_demand": False, + "is_full_sync": True, + "is_sample": False, + "cache_ttl": None, + "refingerprint": False, + "auto_run_queries": True, + "schedules": {}, + "details": { + "host": "clickhouse", + "port": 8123, + "user": "default", + "password": None, + "dbname": "default", + "scan-all-databases": False, + "ssl": False, + "tunnel-enabled": False, + "advanced-options": False + }, + "name": "Our ClickHouse", + "engine": "clickhouse" +} + + +def health(): + response = requests.get(endpoints['health_check'], verify=False) + if response.json()['status'] == 'ok': + return 'healthy' + else: + health() + + +def check_response(response, op): + if response.status_code >= 300: + print(f'Unexpected status {response.status_code} for {op}', response.text) + exit(1) + + +if __name__ == '__main__': + print("Checking health") + + if health() == 'healthy' and os.environ.get('retry') is None: + print("Healthy, setting up Metabase") + + session = requests.Session() + session_token = None + try: + token = session.get(endpoints['properties'], verify=False).json()['setup-token'] + setup_payload = { + 'token': f'{token}', + 'user': { + 'first_name': 'Admin', + 'last_name': 'Admin', + 'email': admin_email, + 'site_name': site_name, + 'password': password, + 'password_confirm': password + }, + 'database': None, + 'invite': None, + 'prefs': { + 'site_name': site_name, + 'site_locale': 'en', + 'allow_tracking': False + } + } + print("Getting the setup token") + session_token = session.post(endpoints['setup'], verify=False, json=setup_payload).json()['id'] + except Exception as e: + print("The admin user was already created") + + try: + if session_token is None: + session_token = session.post(endpoints['login'], verify=False, + json={"username": admin_email, "password": password}) + + dbs = session.get(endpoints['database'], verify=False).json() + print("Current databases:") + pprint.pprint(dbs['data']) + + sample_db = next((x for x in dbs['data'] if x['id'] == 1), None) + if sample_db is not None: + print("Deleting the sample database") + res = session.delete(f"{endpoints['database']}/{sample_db['id']}") + check_response(res, 'delete sample db') + else: + print("The sample database was already deleted") + + single_node_db = next((x for x in dbs['data'] + if x['engine'] == 'clickhouse' + and x['details']['host'] == 'clickhouse'), None) + if single_node_db is None: + print("Creating ClickHouse single node db") + single_node_payload = copy.deepcopy(db_base_payload) + single_node_payload['name'] = 'ClickHouse (single node)' + res = session.post(endpoints['database'], verify=False, json=single_node_payload) + check_response(res, 'create single node db') + else: + print("The single node database was already created") + + # cluster_db = next((x for x in dbs['data'] + # if x['engine'] == 'clickhouse' + # and x['details']['host'] == 'nginx'), None) + # if cluster_db is None: + # print("Creating ClickHouse cluster db") + # cluster_db_payload = copy.deepcopy(db_base_payload) + # cluster_db_payload['details']['host'] = 'nginx' + # cluster_db_payload['name'] = 'ClickHouse (cluster)' + # res = session.post(endpoints['database'], verify=False, json=cluster_db_payload) + # check_response(res) + # else: + # print("The cluster database was already created") + + print("Creating a regular user") + user_payload = {"first_name": "User", "last_name": "User", "email": user_email, "password": password} + res = session.post(endpoints['user'], verify=False, json=user_payload) + check_response(res, 'create user') + + print("Done!") + except Exception as e: + logging.exception("Failed to setup Metabase", e) + exit() diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2391e94..3f92c3f 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -10,7 +10,7 @@ on: pull_request: env: - METABASE_VERSION: v0.49.14 + METABASE_VERSION: v0.50.0 jobs: check-local-current-version: @@ -27,9 +27,15 @@ jobs: # and is currently failing for an unknown reason. # metabase.query-processor.middleware.permissions-test does not look like it is related to the driver at all, # but it is failing on the CI only. + # + # FIXME: metabase.models.card-test is failing as of 0.50.0; + # it is unrelated to the driver, likely will be fixed in the future Metabase versions. + # FIXME: metabase.models.dashboard-card-test disabled because it imports from metabase.models.card-test run: | echo "(ns metabase.test.data.dataset-definition-test)" > test/metabase/test/data/dataset_definition_test.clj echo "(ns metabase.query-processor.middleware.permissions-test)" > test/metabase/query_processor/middleware/permissions_test.clj + echo "(ns metabase.models.card-test)" > test/metabase/models/card_test.clj + echo "(ns metabase.models.dashboard-card-test)" > test/metabase/models/dashboard_card_test.clj - name: Checkout Driver Repo uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index 71938b1..85943c4 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ .cpcache .joyride .nrepl-port +.idea diff --git a/CHANGELOG.md b/CHANGELOG.md index b7f4d35..ba44b6e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,24 @@ +# 1.50.0 + +After Metabase 0.50.0, a new naming convention exists for the driver's releases. The new one is intended to reflect the Metabase version the driver is supposed to run on. For example, the driver version 1.50.0 means that it should be used with Metabase v0.50.x or Metabase EE 1.50.x _only_, and it is _not guaranteed_ that this particular version of the driver can work with the previous or the following versions of Metabase. + +### New features + +* Added Metabase 0.50.x support. + +### Improvements + +* Bumped the JDBC driver to [0.6.1](https://github.com/ClickHouse/clickhouse-java/releases/tag/v0.6.1). + +### Bug fixes + +* Fixed the issue where the connection impersonation feature support could be incorrectly reported as disabled. + +### Other + +* The new ClickHouse analyzer, [which is enabled by default in 24.3+](https://clickhouse.com/blog/clickhouse-release-24-03#analyzer-enabled-by-default), is disabled for the queries executed by the driver, as it shows some compatibilities with the queries generated by Metabase (see [this issue](https://github.com/ClickHouse/ClickHouse/issues/64487) for more details). +* The `:window-functions/offset` Metabase feature is currently disabled, as the default implementation generates queries incompatible with ClickHouse. See [this issue](https://github.com/ClickHouse/metabase-clickhouse-driver/issues/245) for tracking. + # 1.5.1 Metabase 0.49.14+ only. diff --git a/README.md b/README.md index 4c34967..51501c1 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,9 @@ docker run -d -p 3000:3000 \ | 0.48.x | 1.3.4 | | 0.49.x | 1.4.0 | | 0.49.14+ | 1.5.1 | +| 0.50.x | 1.50.0 | + +After Metabase 0.50.0, a new naming convention exists for the driver's releases. The new one is intended to reflect the Metabase version the driver is supposed to run on. For example, the driver version 1.50.0 means that it should be used with Metabase v0.50.x or Metabase EE 1.50.x _only_, and it is _not guaranteed_ that this particular version of the driver can work with the previous or the following versions of Metabase. ## Creating a Metabase Docker image with ClickHouse driver diff --git a/deps.edn b/deps.edn index 2a7843c..b7006f6 100644 --- a/deps.edn +++ b/deps.edn @@ -3,7 +3,7 @@ :deps {com.clickhouse/clickhouse-jdbc$http - {:mvn/version "0.6.0-patch5" + {:mvn/version "0.6.1" :exclusions [com.clickhouse/clickhouse-cli-client$shaded com.clickhouse/clickhouse-grpc-client$shaded]} com.widdindustries/cljc.java-time {:mvn/version "0.1.21"}}} diff --git a/docker-compose.yml b/docker-compose.yml index 25b2df5..6d4db69 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3.8' services: ########################################################################################################## @@ -6,7 +5,7 @@ services: ########################################################################################################## clickhouse: - image: 'clickhouse/clickhouse-server:24.4-alpine' + image: 'clickhouse/clickhouse-server:24.5-alpine' container_name: 'metabase-driver-clickhouse-server' ports: - '8123:8123' @@ -65,7 +64,7 @@ services: ########################################################################################################## clickhouse_cluster_node1: - image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.4-alpine}' + image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.5-alpine}' ulimits: nofile: soft: 262144 @@ -82,7 +81,7 @@ services: - './.docker/clickhouse/users.xml:/etc/clickhouse-server/users.xml' clickhouse_cluster_node2: - image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.4-alpine}' + image: 'clickhouse/clickhouse-server:${CLICKHOUSE_VERSION-24.5-alpine}' ulimits: nofile: soft: 262144 @@ -114,8 +113,9 @@ services: ########################################################################################################## metabase: - image: metabase/metabase-enterprise:v1.49.14 + image: metabase/metabase-enterprise:v1.50.0 container_name: metabase-with-clickhouse-driver + hostname: metabase environment: 'MB_HTTP_TIMEOUT': '5000' 'JAVA_TIMEZONE': 'UTC' @@ -124,3 +124,24 @@ services: volumes: - '../../../resources/modules/clickhouse.metabase-driver.jar:/plugins/clickhouse.jar' - './.docker/clickhouse/single_node_tls/certificates/ca.crt:/certs/ca.crt' + healthcheck: + test: curl --fail -X GET -I http://localhost:3000/api/health || exit 1 + interval: 15s + timeout: 5s + retries: 10 + + setup: + build: .docker/setup/. + container_name: metabase-clickhouse-setup + volumes: + - .docker/setup/setup.py:/app/setup.py + depends_on: + metabase: + condition: service_healthy + command: python /app/setup.py + environment: + host: http://metabase + port: 3000 + admin_email: 'admin@example.com' + user_email: 'user@example.com' + password: 'metabot1' diff --git a/resources/metabase-plugin.yaml b/resources/metabase-plugin.yaml index 26dc32b..b50f939 100644 --- a/resources/metabase-plugin.yaml +++ b/resources/metabase-plugin.yaml @@ -1,6 +1,6 @@ info: name: Metabase ClickHouse Driver - version: 1.5.1 + version: 1.50.0 description: Allows Metabase to connect to ClickHouse databases. contact-info: name: ClickHouse diff --git a/src/metabase/driver/clickhouse.clj b/src/metabase/driver/clickhouse.clj index 8b2e64e..40eb308 100644 --- a/src/metabase/driver/clickhouse.clj +++ b/src/metabase/driver/clickhouse.clj @@ -4,7 +4,7 @@ (:require [clojure.core.memoize :as memoize] [clojure.string :as str] [honey.sql :as sql] - [metabase [config :as config]] + [metabase.config :as config] [metabase.driver :as driver] [metabase.driver.clickhouse-introspection] [metabase.driver.clickhouse-nippy] @@ -12,8 +12,8 @@ [metabase.driver.clickhouse-version :as clickhouse-version] [metabase.driver.ddl.interface :as ddl.i] [metabase.driver.sql :as driver.sql] - [metabase.driver.sql-jdbc [common :as sql-jdbc.common] - [connection :as sql-jdbc.conn]] + [metabase.driver.sql-jdbc.common :as sql-jdbc.common] + [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] [metabase.driver.sql.query-processor :as sql.qp] [metabase.driver.sql.util :as sql.u] @@ -23,7 +23,7 @@ (set! *warn-on-reflection* true) -(driver/register! :clickhouse :parent :sql-jdbc) +(driver/register! :clickhouse :parent #{:sql-jdbc}) (defmethod driver/display-name :clickhouse [_] "ClickHouse") (def ^:private product-name "metabase/1.5.0") @@ -39,7 +39,9 @@ :test/jvm-timezone-setting false :schemas true :datetime-diff true - :upload-with-auto-pk false}] + :upload-with-auto-pk false + :window-functions/offset false}] + (defmethod driver/database-supports? [:clickhouse feature] [_driver _feature _db] supported?)) (def ^:private default-connection-details @@ -69,7 +71,9 @@ ;; and https://github.com/ClickHouse/clickhouse-java/issues/1634#issuecomment-2110392634 :databaseTerm "schema" :remember_last_set_roles true - :http_connection_provider "HTTP_URL_CONNECTION"} + :http_connection_provider "HTTP_URL_CONNECTION" + ;; See https://github.com/ClickHouse/ClickHouse/issues/64487 + :custom_http_params "allow_experimental_analyzer=0"} (sql-jdbc.common/handle-additional-options details :separator-style :url)))) (def ^:private ^{:arglists '([db-details])} cloud? diff --git a/src/metabase/driver/clickhouse_nippy.clj b/src/metabase/driver/clickhouse_nippy.clj index a4853b5..7980295 100644 --- a/src/metabase/driver/clickhouse_nippy.clj +++ b/src/metabase/driver/clickhouse_nippy.clj @@ -2,11 +2,14 @@ (:require [taoensso.nippy :as nippy]) (:import [java.io DataInput DataOutput])) +(set! *warn-on-reflection* false) + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; com.clickhouse.data.value.UnsignedByte ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (nippy/extend-freeze com.clickhouse.data.value.UnsignedByte :clickhouse/UnsignedByte [^com.clickhouse.data.value.UnsignedByte x ^DataOutput data-output] + ;; can't enable *warn-on-reflection* because of this call (nippy/freeze-to-out! data-output (.toString x))) (nippy/extend-thaw :clickhouse/UnsignedByte diff --git a/src/metabase/driver/clickhouse_qp.clj b/src/metabase/driver/clickhouse_qp.clj index 2edd02c..85f3412 100644 --- a/src/metabase/driver/clickhouse_qp.clj +++ b/src/metabase/driver/clickhouse_qp.clj @@ -4,13 +4,13 @@ (:require [clojure.string :as str] [honey.sql :as sql] [java-time.api :as t] - [metabase [util :as u]] [metabase.driver.clickhouse-nippy] [metabase.driver.clickhouse-version :as clickhouse-version] - [metabase.driver.sql-jdbc [execute :as sql-jdbc.execute]] + [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] [metabase.driver.sql.query-processor :as sql.qp :refer [add-interval-honeysql-form]] [metabase.driver.sql.util.unprepare :as unprepare] - [metabase.mbql.util :as mbql.u] + [metabase.legacy-mbql.util :as mbql.u] + [metabase.util :as u] [metabase.util.date-2 :as u.date] [metabase.util.honey-sql-2 :as h2x] [metabase.util.log :as log]) diff --git a/src/metabase/driver/clickhouse_version.clj b/src/metabase/driver/clickhouse_version.clj index 564bb1c..620466e 100644 --- a/src/metabase/driver/clickhouse_version.clj +++ b/src/metabase/driver/clickhouse_version.clj @@ -48,7 +48,7 @@ ;; used from the Driver overrides; we have access to the DB object (let [version (driver/dbms-version :clickhouse db) semantic (:semantic-version version)] - (driver.u/semantic-version-gte [major minor] [(:major semantic) (:minor semantic)])))) + (driver.u/semantic-version-gte [(:major semantic) (:minor semantic)] [major minor])))) (defn with-min "Execute `f` if the ClickHouse version is greater or equal to `major.minor` (e.g., 24.4); diff --git a/test/metabase/driver/clickhouse_data_types_test.clj b/test/metabase/driver/clickhouse_data_types_test.clj index 83239dd..b414658 100644 --- a/test/metabase/driver/clickhouse_data_types_test.clj +++ b/test/metabase/driver/clickhouse_data_types_test.clj @@ -6,22 +6,22 @@ [metabase.query-processor.test-util :as qp.test] [metabase.test :as mt] [metabase.test.data :as data] - [metabase.test.data [interface :as tx]] + [metabase.test.data.interface :as tx] [metabase.test.data.clickhouse :as ctd])) (deftest ^:parallel clickhouse-decimals (mt/test-driver :clickhouse (data/dataset - (tx/dataset-definition "metabase_tests_decimal" - ["test-data-decimal" + (tx/dataset-definition "mbt" + ["decimals" [{:field-name "my_money" :base-type {:native "Decimal(12,4)"}}] [[1.0] [23.1337] [42.0] [42.0]]]) (testing "simple division" (is (= 21.0 - (-> (data/run-mbql-query test-data-decimal + (-> (data/run-mbql-query decimals {:expressions {:divided [:/ $my_money 2]} :filter [:> [:expression :divided] 1.0] :breakout [[:expression :divided]] @@ -31,7 +31,7 @@ (testing "divided decimal precision" (is (= 1.8155331831916208 - (-> (data/run-mbql-query test-data-decimal + (-> (data/run-mbql-query decimals {:expressions {:divided [:/ 42 $my_money]} :filter [:= $id 2] :limit 1}) diff --git a/test/metabase/driver/clickhouse_impersonation_test.clj b/test/metabase/driver/clickhouse_impersonation_test.clj index f9973cb..74ad54f 100644 --- a/test/metabase/driver/clickhouse_impersonation_test.clj +++ b/test/metabase/driver/clickhouse_impersonation_test.clj @@ -7,7 +7,7 @@ [metabase.driver.sql :as driver.sql] [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] - [metabase.models [database :refer [Database]]] + [metabase.models.database :refer [Database]] [metabase.query-processor.store :as qp.store] [metabase.sync :as sync] [metabase.test :as mt] @@ -74,7 +74,7 @@ (testing "should support the impersonation feature" (t2.with-temp/with-temp [Database db {:engine :clickhouse :details {:user "default" :port 8123}}] - (is (driver/database-supports? :clickhouse :connection-impersonation db) true))) + (is (true? (driver/database-supports? :clickhouse :connection-impersonation db))))) (let [statements ["CREATE DATABASE IF NOT EXISTS `metabase_test_role_db`;" "CREATE OR REPLACE TABLE `metabase_test_role_db`.`some_table` (i Int32) ENGINE = MergeTree ORDER BY (i);" "INSERT INTO `metabase_test_role_db`.`some_table` VALUES (42), (144);" @@ -92,7 +92,7 @@ (testing "should support the impersonation feature" (t2.with-temp/with-temp [Database db {:engine :clickhouse :details {:user "default" :port 8127}}] - (is (driver/database-supports? :clickhouse :connection-impersonation db) true))) + (is (true? (driver/database-supports? :clickhouse :connection-impersonation db))))) (let [statements ["CREATE DATABASE IF NOT EXISTS `metabase_test_role_db` ON CLUSTER '{cluster}';" "CREATE OR REPLACE TABLE `metabase_test_role_db`.`some_table` ON CLUSTER '{cluster}' (i Int32) ENGINE ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}/{shard}', '{replica}') @@ -112,7 +112,7 @@ (testing "should NOT support the impersonation feature" (t2.with-temp/with-temp [Database db {:engine :clickhouse :details {:user "default" :port 8124}}] - (is (driver/database-supports? :clickhouse :connection-impersonation db) true))))))) + (is (false? (driver/database-supports? :clickhouse :connection-impersonation db))))))))) (deftest conn-impersonation-test-clickhouse (mt/test-driver @@ -149,9 +149,9 @@ (t2.with-temp/with-temp [Database db cluster-details] (mt/with-db db (sync/sync-database! db) - (defn- check-impersonation + (defn- check-impersonation! [roles expected] - (advanced-perms.api.tu/with-impersonations + (advanced-perms.api.tu/with-impersonations! {:impersonations [{:db-id (mt/id) :attribute "impersonation_attr"}] :attributes {"impersonation_attr" roles}} (is (= expected @@ -166,9 +166,9 @@ mt/process-query mt/rows))) - (check-impersonation "row_a" [["a"]]) - (check-impersonation "row_b" [["b"]]) - (check-impersonation "row_c" [["c"]]) - (check-impersonation "row_a,row_c" [["a"] ["c"]]) - (check-impersonation "row_b,row_c" [["b"] ["c"]]) - (check-impersonation "row_a,row_b,row_c" [["a"] ["b"] ["c"]]))))))) + (check-impersonation! "row_a" [["a"]]) + (check-impersonation! "row_b" [["b"]]) + (check-impersonation! "row_c" [["c"]]) + (check-impersonation! "row_a,row_c" [["a"] ["c"]]) + (check-impersonation! "row_b,row_c" [["b"] ["c"]]) + (check-impersonation! "row_a,row_b,row_c" [["a"] ["b"] ["c"]]))))))) diff --git a/test/metabase/driver/clickhouse_substitution_test.clj b/test/metabase/driver/clickhouse_substitution_test.clj index 8e38c5e..21a2086 100644 --- a/test/metabase/driver/clickhouse_substitution_test.clj +++ b/test/metabase/driver/clickhouse_substitution_test.clj @@ -5,7 +5,7 @@ [metabase.query-processor :as qp] [metabase.test :as mt] [metabase.test.data :as data] - [metabase.test.data [interface :as tx]] + [metabase.test.data.interface :as tx] [metabase.test.data.clickhouse :as ctd] [metabase.util :as u] [schema.core :as s]) @@ -56,7 +56,7 @@ (doseq [base-type ["DateTime" "DateTime64"]] (testing base-type (testing "on specific" - (let [db (format "metabase_tests_variables_replacement_on_specific_%s" + (let [db (format "mb_vars_on_x_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusHours now 14) @@ -73,7 +73,7 @@ (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "2019-11-30T22:40:00" db))))))))) (testing "past/next minutes" - (let [db (format "metabase_tests_variables_replacement_past_next_minutes_%s" + (let [db (format "mb_vars_m_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusHours now 14) @@ -90,7 +90,7 @@ (is (= [[(->iso-str row3)]] (ctd/rows-without-index (qp/process-query (get-mbql "next30minutes" db))))))))) (testing "past/next hours" - (let [db (format "metabase_tests_variables_replacement_past_next_hours_%s" + (let [db (format "mb_vars__past_next_hours_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusHours now 14) @@ -107,7 +107,7 @@ (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next12hours" db))))))))) (testing "past/next days" - (let [db (format "metabase_tests_variables_replacement_past_next_days_%s" + (let [db (format "mb_vars_d_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusDays now 14) @@ -124,7 +124,7 @@ (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next12days" db))))))))) (testing "past/next months/quarters" - (let [db (format "metabase_tests_variables_replacement_past_next_months_quarters_%s" + (let [db (format "mb_vars_m_q_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusMonths now 14) @@ -147,7 +147,7 @@ (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next3quarters" db))))))))) (testing "past/next years" - (let [db (format "metabase_tests_variables_replacement_past_next_years_%s" + (let [db (format "mb_vars_y_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusYears now 14) @@ -184,7 +184,7 @@ (doseq [base-type ["Date" "Date32"]] (testing base-type (testing "on specific date" - (let [db (format "metabase_tests_variables_replacement_on_specific_%s" + (let [db (format "mb_vars_on_x_%s" (u/lower-case-en base-type)) now (local-date-time-now) row1 (.minusDays now 14) @@ -197,7 +197,7 @@ (is (= [[(->iso-str row2)]] (ctd/rows-without-index (qp/process-query (get-mbql "2019-11-30" db)))))))) (testing "past/next days" - (let [db (format "metabase_tests_variables_replacement_past_next_days_%s" + (let [db (format "mb_vars_d_%s" (u/lower-case-en base-type)) now (local-date-now) row1 (.minusDays now 14) @@ -214,7 +214,7 @@ (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next12days" db))))))))) (testing "past/next months/quarters" - (let [db (format "metabase_tests_variables_replacement_past_next_months_quarters_%s" + (let [db (format "mb_vars_m_q_%s" (u/lower-case-en base-type)) now (local-date-now) row1 (.minusMonths now 14) @@ -237,7 +237,7 @@ (is (= [[(->iso-str row4)]] (ctd/rows-without-index (qp/process-query (get-mbql "next3quarters" db))))))))) (testing "past/next years" - (let [db (format "metabase_tests_variables_replacement_past_next_years_%s" + (let [db (format "mb_vars_y_%s" (u/lower-case-en base-type)) now (local-date-now) row1 (.minusYears now 14) @@ -271,7 +271,7 @@ (->iso-str-ldt [^LocalDateTime ldt] (t/format "yyyy-MM-dd'T'HH:mm:ss'Z'" ldt))] - (let [db "metabase_tests_field_filters_null_dates" + (let [db "mb_vars_null_dates" now-ld (local-date-now) now-ldt (local-date-time-now) table ["test_table" diff --git a/test/metabase/driver/clickhouse_test.clj b/test/metabase/driver/clickhouse_test.clj index b59724e..2425429 100644 --- a/test/metabase/driver/clickhouse_test.clj +++ b/test/metabase/driver/clickhouse_test.clj @@ -13,12 +13,13 @@ [metabase.driver.clickhouse-substitution-test] [metabase.driver.clickhouse-temporal-bucketing-test] [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] - [metabase.models [database :refer [Database]]] + [metabase.models.database :refer [Database]] [metabase.query-processor :as qp] + [metabase.query-processor.compile :as qp.compile] [metabase.query-processor.test-util :as qp.test] [metabase.test :as mt] [metabase.test.data :as data] - [metabase.test.data [interface :as tx]] + [metabase.test.data.interface :as tx] [metabase.test.data.clickhouse :as ctd] [taoensso.nippy :as nippy] [toucan2.tools.with-temp :as t2.with-temp])) @@ -171,7 +172,7 @@ (mt/test-driver :clickhouse (let [query (data/mbql-query venues {:fields [$id] :order-by [[:asc $id]] :limit 5}) - {compiled :query} (qp/compile-and-splice-parameters query) + {compiled :query} (qp.compile/compile-and-splice-parameters query) pretty (driver/prettify-native-form :clickhouse compiled)] (testing "compiled" (is (= "SELECT `test_data`.`venues`.`id` AS `id` FROM `test_data`.`venues` ORDER BY `test_data`.`venues`.`id` ASC LIMIT 5" compiled))) diff --git a/test/metabase/test/data/clickhouse.clj b/test/metabase/test/data/clickhouse.clj index 7b3ab1f..239e749 100644 --- a/test/metabase/test/data/clickhouse.clj +++ b/test/metabase/test/data/clickhouse.clj @@ -8,16 +8,15 @@ [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] [metabase.driver.sql-jdbc.execute :as sql-jdbc.execute] [metabase.driver.sql.util :as sql.u] - [metabase.models [database :refer [Database]]] + [metabase.models.database :refer [Database]] [metabase.query-processor.test-util :as qp.test] [metabase.sync.sync-metadata :as sync-metadata] - [metabase.test.data - [interface :as tx] - [sql-jdbc :as sql-jdbc.tx]] + [metabase.test.data.interface :as tx] [metabase.test.data.sql :as sql.tx] - [metabase.test.data.sql-jdbc - [execute :as execute] - [load-data :as load-data]] + [metabase.test.data.sql-jdbc :as sql-jdbc.tx] + [metabase.test.data.sql-jdbc.execute :as execute] + [metabase.test.data.sql-jdbc.load-data :as load-data] + [metabase.util.log :as log] [toucan2.tools.with-temp :as t2.with-temp]) (:import [com.clickhouse.jdbc.internal ClickHouseStatementImpl])) @@ -35,7 +34,8 @@ :product_name "metabase/1.5.0" :databaseTerm "schema" :remember_last_set_roles true - :http_connection_provider "HTTP_URL_CONNECTION"}) + :http_connection_provider "HTTP_URL_CONNECTION" + :custom_http_params "allow_experimental_analyzer=0"}) (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/Boolean] [_ _] "Boolean") (defmethod sql.tx/field-base-type->sql-type [:clickhouse :type/BigInteger] [_ _] "Int64") @@ -68,6 +68,13 @@ ([_ db-name table-name] [db-name table-name]) ([_ db-name table-name field-name] [db-name table-name field-name])) +(defmethod tx/create-db! :clickhouse + [driver {:keys [database-name], :as db-def} & options] + (let [database-name (ddl.i/format-name driver database-name)] + (log/infof "Creating ClickHouse database %s" (pr-str database-name)) + ;; call the default impl for SQL JDBC drivers + (apply (get-method tx/create-db! :sql-jdbc/test-extensions) driver db-def options))) + (defn- quote-name [name] (sql.u/quote-name :clickhouse :field (ddl.i/format-name :clickhouse name))) @@ -149,21 +156,21 @@ ([statements details-map] (exec-statements statements details-map nil)) ([statements details-map clickhouse-settings] - (sql-jdbc.execute/do-with-connection-with-options - :clickhouse - (sql-jdbc.conn/connection-details->spec :clickhouse (merge {:engine :clickhouse} details-map)) - {:write? true} - (fn [^java.sql.Connection conn] - (doseq [statement statements] - (println "Executing:" statement) - (with-open [jdbcStmt (.createStatement conn)] - (let [^ClickHouseStatementImpl clickhouseStmt (.unwrap jdbcStmt ClickHouseStatementImpl) - request (.getRequest clickhouseStmt)] - (when clickhouse-settings - (doseq [[k v] clickhouse-settings] (.set request k v))) - (with-open [_response (-> request - (.query ^String statement) - (.executeAndWait))])))))))) + (sql-jdbc.execute/do-with-connection-with-options + :clickhouse + (sql-jdbc.conn/connection-details->spec :clickhouse (merge {:engine :clickhouse} details-map)) + {:write? true} + (fn [^java.sql.Connection conn] + (doseq [statement statements] + (println "Executing:" statement) + (with-open [jdbcStmt (.createStatement conn)] + (let [^ClickHouseStatementImpl clickhouseStmt (.unwrap jdbcStmt ClickHouseStatementImpl) + request (.getRequest clickhouseStmt)] + (when clickhouse-settings + (doseq [[k v] clickhouse-settings] (.set request k v))) + (with-open [_response (-> request + (.query ^String statement) + (.executeAndWait))])))))))) (defn do-with-test-db "Execute a test function using the test dataset"