From 3a00680dcdfb79024db50c02c428e87e1fce0213 Mon Sep 17 00:00:00 2001 From: EugeneTorap Date: Fri, 4 Nov 2022 19:35:34 +0300 Subject: [PATCH 1/6] Use a new official CH driver: clickhouse-connect --- docs/docs/databases/clickhouse.mdx | 12 ++++++------ docs/docs/databases/installing-database-drivers.mdx | 2 +- setup.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/docs/databases/clickhouse.mdx b/docs/docs/databases/clickhouse.mdx index 7afcc50c15cd9..2d81f9c48a654 100644 --- a/docs/docs/databases/clickhouse.mdx +++ b/docs/docs/databases/clickhouse.mdx @@ -10,33 +10,33 @@ version: 1 To use ClickHouse with Superset, you will need to add the following Python library: ``` -clickhouse-sqlalchemy>=0.2.2 +clickhouse-connect>=0.3.8 ``` If running Superset using Docker Compose, add the following to your `./docker/requirements-local.txt` file: ``` -clickhouse-sqlalchemy>=0.2.2 +clickhouse-connect>=0.3.8 ``` The recommended connector library for ClickHouse is -[sqlalchemy-clickhouse](https://github.com/cloudflare/sqlalchemy-clickhouse). +[clickhouse-connect](https://github.com/ClickHouse/clickhouse-connect). The expected connection string is formatted as follows: ``` -clickhouse+native://:@:/[?options…]clickhouse://{username}:{password}@{hostname}:{port}/{database} +clickhousedb+connect://:@:/[?options…]clickhouse://{username}:{password}@{hostname}:{port}/{database} ``` Here's a concrete example of a real connection string: ``` -clickhouse+native://demo:demo@github.demo.trial.altinity.cloud/default?secure=true +clickhousedb+connect://demo:demo@github.demo.trial.altinity.cloud/default?secure=true ``` If you're using Clickhouse locally on your computer, you can get away with using a native protocol URL that uses the default user without a password (and doesn't encrypt the connection): ``` -clickhouse+native://localhost/default +clickhousedb+connect://localhost/default ``` diff --git a/docs/docs/databases/installing-database-drivers.mdx b/docs/docs/databases/installing-database-drivers.mdx index c2dc1159e6566..29ababb92c7f1 100644 --- a/docs/docs/databases/installing-database-drivers.mdx +++ b/docs/docs/databases/installing-database-drivers.mdx @@ -35,7 +35,7 @@ A list of some of the recommended packages. | [Ascend.io](/docs/databases/ascend) | `pip install impyla` | `ascend://{username}:{password}@{hostname}:{port}/{database}?auth_mechanism=PLAIN;use_ssl=true` | | [Azure MS SQL](/docs/databases/sql-server) | `pip install pymssql` | `mssql+pymssql://UserName@presetSQL:TestPassword@presetSQL.database.windows.net:1433/TestSchema` | | [Big Query](/docs/databases/bigquery) | `pip install pybigquery` | `bigquery://{project_id}` | -| [ClickHouse](/docs/databases/clickhouse) | `pip install clickhouse-sqlalchemy` | `clickhouse+native://{username}:{password}@{hostname}:{port}/{database}` | +| [ClickHouse](/docs/databases/clickhouse) | `pip install clickhouse-connect` | `clickhousedb+connect://{username}:{password}@{hostname}:{port}/{database}` | | [CockroachDB](/docs/databases/cockroachdb) | `pip install cockroachdb` | `cockroachdb://root@{hostname}:{port}/{database}?sslmode=disable` | | [Dremio](/docs/databases/dremio) | `pip install sqlalchemy_dremio` | `dremio://user:pwd@host:31010/` | | [Elasticsearch](/docs/databases/elasticsearch) | `pip install elasticsearch-dbapi` | `elasticsearch+http://{user}:{password}@{host}:9200/` | diff --git a/setup.py b/setup.py index cad76a9572997..2ed427ad33dc9 100644 --- a/setup.py +++ b/setup.py @@ -130,7 +130,7 @@ def get_git_sha() -> str: "pybigquery>=0.4.10", "google-cloud-bigquery>=2.4.0", ], - "clickhouse": ["clickhouse-sqlalchemy>=0.2.2, <0.3"], + "clickhouse": ["clickhouse-connect>=0.3.8, <0.4"], "cockroachdb": ["cockroachdb>=0.3.5, <0.4"], "cors": ["flask-cors>=2.0.0"], "crate": ["crate[sqlalchemy]>=0.26.0, <0.27"], From a0b69a94985e0c5178de2732e1672b93b04cab52 Mon Sep 17 00:00:00 2001 From: EugeneTorap Date: Fri, 4 Nov 2022 19:37:06 +0300 Subject: [PATCH 2/6] Fix doc --- docs/docs/databases/clickhouse.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/databases/clickhouse.mdx b/docs/docs/databases/clickhouse.mdx index 2d81f9c48a654..b2164a135cc5b 100644 --- a/docs/docs/databases/clickhouse.mdx +++ b/docs/docs/databases/clickhouse.mdx @@ -34,7 +34,7 @@ Here's a concrete example of a real connection string: clickhousedb+connect://demo:demo@github.demo.trial.altinity.cloud/default?secure=true ``` -If you're using Clickhouse locally on your computer, you can get away with using a native protocol URL that +If you're using Clickhouse locally on your computer, you can get away with using a http protocol URL that uses the default user without a password (and doesn't encrypt the connection): ``` From e9dd2b54bb7bcf5d51aeb498a73274d3cd4aa5dd Mon Sep 17 00:00:00 2001 From: EugeneTorap Date: Thu, 10 Nov 2022 23:02:24 +0300 Subject: [PATCH 3/6] Remove '+connect' in URL conn and use 0.4.1 version of the lib --- docs/docs/databases/clickhouse.mdx | 10 +++++----- docs/docs/databases/installing-database-drivers.mdx | 2 +- setup.py | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/docs/databases/clickhouse.mdx b/docs/docs/databases/clickhouse.mdx index b2164a135cc5b..424e50da9352c 100644 --- a/docs/docs/databases/clickhouse.mdx +++ b/docs/docs/databases/clickhouse.mdx @@ -10,13 +10,13 @@ version: 1 To use ClickHouse with Superset, you will need to add the following Python library: ``` -clickhouse-connect>=0.3.8 +clickhouse-connect>=0.4.1 ``` If running Superset using Docker Compose, add the following to your `./docker/requirements-local.txt` file: ``` -clickhouse-connect>=0.3.8 +clickhouse-connect>=0.4.1 ``` The recommended connector library for ClickHouse is @@ -25,18 +25,18 @@ The recommended connector library for ClickHouse is The expected connection string is formatted as follows: ``` -clickhousedb+connect://:@:/[?options…]clickhouse://{username}:{password}@{hostname}:{port}/{database} +clickhousedb://:@:/[?options…]clickhouse://{username}:{password}@{hostname}:{port}/{database} ``` Here's a concrete example of a real connection string: ``` -clickhousedb+connect://demo:demo@github.demo.trial.altinity.cloud/default?secure=true +clickhousedb://demo:demo@github.demo.trial.altinity.cloud/default?secure=true ``` If you're using Clickhouse locally on your computer, you can get away with using a http protocol URL that uses the default user without a password (and doesn't encrypt the connection): ``` -clickhousedb+connect://localhost/default +clickhousedb://localhost/default ``` diff --git a/docs/docs/databases/installing-database-drivers.mdx b/docs/docs/databases/installing-database-drivers.mdx index 29ababb92c7f1..01c85bc0e12fc 100644 --- a/docs/docs/databases/installing-database-drivers.mdx +++ b/docs/docs/databases/installing-database-drivers.mdx @@ -35,7 +35,7 @@ A list of some of the recommended packages. | [Ascend.io](/docs/databases/ascend) | `pip install impyla` | `ascend://{username}:{password}@{hostname}:{port}/{database}?auth_mechanism=PLAIN;use_ssl=true` | | [Azure MS SQL](/docs/databases/sql-server) | `pip install pymssql` | `mssql+pymssql://UserName@presetSQL:TestPassword@presetSQL.database.windows.net:1433/TestSchema` | | [Big Query](/docs/databases/bigquery) | `pip install pybigquery` | `bigquery://{project_id}` | -| [ClickHouse](/docs/databases/clickhouse) | `pip install clickhouse-connect` | `clickhousedb+connect://{username}:{password}@{hostname}:{port}/{database}` | +| [ClickHouse](/docs/databases/clickhouse) | `pip install clickhouse-connect` | `clickhousedb://{username}:{password}@{hostname}:{port}/{database}` | | [CockroachDB](/docs/databases/cockroachdb) | `pip install cockroachdb` | `cockroachdb://root@{hostname}:{port}/{database}?sslmode=disable` | | [Dremio](/docs/databases/dremio) | `pip install sqlalchemy_dremio` | `dremio://user:pwd@host:31010/` | | [Elasticsearch](/docs/databases/elasticsearch) | `pip install elasticsearch-dbapi` | `elasticsearch+http://{user}:{password}@{host}:9200/` | diff --git a/setup.py b/setup.py index 2ed427ad33dc9..eabea273d8e64 100644 --- a/setup.py +++ b/setup.py @@ -130,7 +130,7 @@ def get_git_sha() -> str: "pybigquery>=0.4.10", "google-cloud-bigquery>=2.4.0", ], - "clickhouse": ["clickhouse-connect>=0.3.8, <0.4"], + "clickhouse": ["clickhouse-connect>=0.4.1, <0.5"], "cockroachdb": ["cockroachdb>=0.3.5, <0.4"], "cors": ["flask-cors>=2.0.0"], "crate": ["crate[sqlalchemy]>=0.26.0, <0.27"], From 61626ecab4b74677e63d089c81aadafdf7ea79ef Mon Sep 17 00:00:00 2001 From: EugeneTorap Date: Thu, 10 Nov 2022 23:26:27 +0300 Subject: [PATCH 4/6] Refactor test_clickhouse.py --- .../db_engine_specs/clickhouse_tests.py | 47 ------------------- .../db_engine_specs/test_clickhouse.py | 46 ++++++++++++++++++ 2 files changed, 46 insertions(+), 47 deletions(-) delete mode 100644 tests/integration_tests/db_engine_specs/clickhouse_tests.py create mode 100644 tests/unit_tests/db_engine_specs/test_clickhouse.py diff --git a/tests/integration_tests/db_engine_specs/clickhouse_tests.py b/tests/integration_tests/db_engine_specs/clickhouse_tests.py deleted file mode 100644 index c8019a8758db7..0000000000000 --- a/tests/integration_tests/db_engine_specs/clickhouse_tests.py +++ /dev/null @@ -1,47 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -from unittest import mock - -import pytest - -from superset.db_engine_specs.clickhouse import ClickHouseEngineSpec -from superset.db_engine_specs.exceptions import SupersetDBAPIDatabaseError -from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec - - -class TestClickHouseDbEngineSpec(TestDbEngineSpec): - def test_convert_dttm(self): - dttm = self.get_dttm() - - self.assertEqual( - ClickHouseEngineSpec.convert_dttm("DATE", dttm), "toDate('2019-01-02')" - ) - - self.assertEqual( - ClickHouseEngineSpec.convert_dttm("DATETIME", dttm), - "toDateTime('2019-01-02 03:04:05')", - ) - - def test_execute_connection_error(self): - from urllib3.exceptions import NewConnectionError - - cursor = mock.Mock() - cursor.execute.side_effect = NewConnectionError( - "Dummypool", message="Exception with sensitive data" - ) - with pytest.raises(SupersetDBAPIDatabaseError) as ex: - ClickHouseEngineSpec.execute(cursor, "SELECT col1 from table1") diff --git a/tests/unit_tests/db_engine_specs/test_clickhouse.py b/tests/unit_tests/db_engine_specs/test_clickhouse.py new file mode 100644 index 0000000000000..66de221eca24c --- /dev/null +++ b/tests/unit_tests/db_engine_specs/test_clickhouse.py @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from datetime import datetime +from unittest import mock + +import pytest + +from tests.unit_tests.fixtures.common import dttm + + +def test_convert_dttm(dttm: datetime) -> None: + from superset.db_engine_specs.clickhouse import ClickHouseEngineSpec + + assert ClickHouseEngineSpec.convert_dttm("DATE", dttm) == "toDate('2019-01-02')" + assert ( + ClickHouseEngineSpec.convert_dttm("DATETIME", dttm) + == "toDateTime('2019-01-02 03:04:05')" + ) + + +def test_execute_connection_error(): + from urllib3.exceptions import NewConnectionError + + from superset.db_engine_specs.clickhouse import ClickHouseEngineSpec + from superset.db_engine_specs.exceptions import SupersetDBAPIDatabaseError + + cursor = mock.Mock() + cursor.execute.side_effect = NewConnectionError( + "Dummypool", message="Exception with sensitive data" + ) + with pytest.raises(SupersetDBAPIDatabaseError) as ex: + ClickHouseEngineSpec.execute(cursor, "SELECT col1 from table1") From b2ba0f0e8690ec2464228ff860b77fb32033929f Mon Sep 17 00:00:00 2001 From: EugeneTorap Date: Thu, 10 Nov 2022 23:34:12 +0300 Subject: [PATCH 5/6] Fix mypy --- tests/unit_tests/db_engine_specs/test_clickhouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/db_engine_specs/test_clickhouse.py b/tests/unit_tests/db_engine_specs/test_clickhouse.py index 66de221eca24c..6be69b25dd5a0 100644 --- a/tests/unit_tests/db_engine_specs/test_clickhouse.py +++ b/tests/unit_tests/db_engine_specs/test_clickhouse.py @@ -32,7 +32,7 @@ def test_convert_dttm(dttm: datetime) -> None: ) -def test_execute_connection_error(): +def test_execute_connection_error() -> None: from urllib3.exceptions import NewConnectionError from superset.db_engine_specs.clickhouse import ClickHouseEngineSpec From e3ba2392ab463a17acf739d5f3bf30e508f78e59 Mon Sep 17 00:00:00 2001 From: EugeneTorap Date: Fri, 11 Nov 2022 11:22:30 +0300 Subject: [PATCH 6/6] Fix mypy --- tests/unit_tests/db_engine_specs/test_clickhouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/db_engine_specs/test_clickhouse.py b/tests/unit_tests/db_engine_specs/test_clickhouse.py index 6be69b25dd5a0..ca01c304f6327 100644 --- a/tests/unit_tests/db_engine_specs/test_clickhouse.py +++ b/tests/unit_tests/db_engine_specs/test_clickhouse.py @@ -40,7 +40,7 @@ def test_execute_connection_error() -> None: cursor = mock.Mock() cursor.execute.side_effect = NewConnectionError( - "Dummypool", message="Exception with sensitive data" + "Dummypool", "Exception with sensitive data" ) with pytest.raises(SupersetDBAPIDatabaseError) as ex: ClickHouseEngineSpec.execute(cursor, "SELECT col1 from table1")