ClickHouse · Enmk · Apr 13, 2020 · Mar 9, 2020 · Mar 9, 2020 · Mar 11, 2020
diff --git a/.gitmodules b/.gitmodules
@@ -1,20 +1,20 @@
 [submodule "contrib/poco"]
-	path = contrib/poco
-	url = https://github.com/pocoproject/poco.git
-	branch = master
+    path = contrib/poco
+    url = https://github.com/pocoproject/poco.git
+    branch = master
 [submodule "contrib/ssl"]
-	path = contrib/ssl
-	url = https://github.com/ClickHouse-Extras/ssl.git
-	branch = master
+    path = contrib/ssl
+    url = https://github.com/ClickHouse-Extras/ssl.git
+    branch = master
 [submodule "contrib/nanodbc"]
-	path = contrib/nanodbc
-	url = https://github.com/nanodbc/nanodbc.git
-	branch = master
+    path = contrib/nanodbc
+    url = https://github.com/nanodbc/nanodbc.git
+    branch = master
 [submodule "contrib/googletest"]
-	path = contrib/googletest
-	url = https://github.com/google/googletest.git
-	branch = master
+    path = contrib/googletest
+    url = https://github.com/google/googletest.git
+    branch = master
 [submodule "contrib/folly"]
-	path = contrib/folly
-	url = https://github.com/facebook/folly.git
-	branch = master
+    path = contrib/folly
+    url = https://github.com/facebook/folly.git
+    branch = master
diff --git a/.travis.yml b/.travis.yml
@@ -42,6 +42,7 @@ jobs:
           packages:
             - gcc
             - g++
+            - libicu-dev
             - unixodbc
             - unixodbc-bin
             - unixodbc-dev
@@ -70,6 +71,7 @@ jobs:
           packages:
             - gcc
             - g++
+            - libicu-dev
             - unixodbc
             - unixodbc-bin
             - unixodbc-dev
@@ -98,6 +100,7 @@ jobs:
           packages:
             - gcc
             - g++
+            - libicu-dev
             - unixodbc
             - unixodbc-bin
             - unixodbc-dev
@@ -129,6 +132,7 @@ jobs:
             - openssl
             - libssl-dev
             - libpoco-dev
+            - libicu-dev
             - googletest
             - unixodbc
             - unixodbc-bin
@@ -157,6 +161,7 @@ jobs:
               key_url: 'https://apt.kitware.com/keys/kitware-archive-latest.asc'
           packages:
             - clang
+            - libicu-dev
             - unixodbc
             - unixodbc-bin
             - unixodbc-dev
@@ -185,6 +190,7 @@ jobs:
           packages:
             - gcc
             - g++
+            - libicu-dev
             - iodbc
             - libiodbc2
             - libiodbc2-dev
@@ -212,6 +218,7 @@ jobs:
               key_url: 'https://apt.kitware.com/keys/kitware-archive-latest.asc'
           packages:
             - clang
+            - libicu-dev
             - iodbc
             - libiodbc2
             - libiodbc2-dev
@@ -256,6 +263,7 @@ jobs:
       addons:
         homebrew:
           packages:
+            - icu4c
             - libiodbc
           update: true
 
@@ -275,6 +283,7 @@ jobs:
           packages:
             - openssl
             - poco
+            - icu4c
             - libiodbc
           update: true
 
@@ -292,6 +301,7 @@ jobs:
       addons:
         homebrew:
           packages:
+            - icu4c
             - unixodbc
           update: true
 
@@ -430,6 +440,9 @@ install: |-
 
 .configure: &configure |-
   CMAKE_CONFIGURE_ARGS="-DTEST_DSN_LIST=\"ClickHouse DSN (ANSI);ClickHouse DSN (Unicode);ClickHouse DSN (ANSI, RBWNAT)\" $CMAKE_CONFIGURE_EXTRA_ARGS"
+  if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then
+    CMAKE_CONFIGURE_ARGS="-DICU_ROOT=$(brew --prefix)/opt/icu4c $CMAKE_CONFIGURE_ARGS"
+  fi
   if [[ "$EXTERNAL_THIRD_PARTY" == "yes" ]]; then
     CMAKE_CONFIGURE_ARGS="-DCH_ODBC_PREFER_BUNDLED_THIRD_PARTIES=OFF $CMAKE_CONFIGURE_ARGS"
     if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -42,10 +42,12 @@ option (CH_ODBC_ALLOW_UNSAFE_DISPATCH "Allow unchecked handle dispatching (may s
 option (CH_ODBC_ENABLE_SSL "Enable SSL (required for utilizing https:// interface, etc.)" ON)
 option (CH_ODBC_ENABLE_INSTALL "Enable install targets (required for packaging)" ON)
 cmake_dependent_option (CH_ODBC_ENABLE_TESTING "Enable test targets" ON "BUILD_TESTING" OFF)
+cmake_dependent_option (CH_ODBC_USE_ICU "Use ICU library, instead of C++ STD, for Unicode conversions" ON "NOT MSVC" OFF)
 option (CH_ODBC_PREFER_BUNDLED_THIRD_PARTIES "Prefer bundled over system variants of third party libraries" ON)
 cmake_dependent_option (CH_ODBC_PREFER_BUNDLED_POCO "Prefer bundled over system variants of Poco library" ON "CH_ODBC_PREFER_BUNDLED_THIRD_PARTIES" OFF)
 cmake_dependent_option (CH_ODBC_PREFER_BUNDLED_SSL "Prefer bundled over system variants of SSL library" ON "CH_ODBC_PREFER_BUNDLED_POCO" OFF)
 cmake_dependent_option (CH_ODBC_PREFER_BUNDLED_FOLLY "Prefer bundled over system variants of Folly library" ON "CH_ODBC_PREFER_BUNDLED_THIRD_PARTIES" OFF)
+cmake_dependent_option (CH_ODBC_PREFER_BUNDLED_ICU "Prefer bundled over system variants of ICU library" OFF "CH_ODBC_PREFER_BUNDLED_THIRD_PARTIES" OFF)
 cmake_dependent_option (CH_ODBC_PREFER_BUNDLED_GOOGLETEST "Prefer bundled over system variants of Google Test library" ON "CH_ODBC_PREFER_BUNDLED_THIRD_PARTIES" OFF)
 cmake_dependent_option (CH_ODBC_PREFER_BUNDLED_NANODBC "Prefer bundled over system variants of nanodbc library" ON "CH_ODBC_PREFER_BUNDLED_THIRD_PARTIES" OFF)
 option (CH_ODBC_RUNTIME_LINK_STATIC "Link with compiler and language runtime statically" OFF)
@@ -137,6 +139,13 @@ if (NOT CH_ODBC_PREFER_BUNDLED_FOLLY)
 #   find_package (Folly)
 endif ()
 
+if (CH_ODBC_USE_ICU)
+    if (CH_ODBC_PREFER_BUNDLED_ICU)
+        message (WARNING "ICU: using bundled variant of the library currently not supported")
+    endif ()
+    find_package (ICU COMPONENTS uc REQUIRED)
+endif ()
+
 if (CH_ODBC_ENABLE_TESTING)
     if (NOT CH_ODBC_PREFER_BUNDLED_GOOGLETEST)
         find_package (GTest)

diff --git a/README.md b/README.md
@@ -177,15 +177,15 @@ Another run-time dependecies are `C++ Redistributable for Visual Studio 2017` or
 Homebrew: execute the following in the terminal (assuming you have [Homebrew](https://brew.sh/) installed):
 
 ```sh
-brew install poco openssl libiodbc
+brew install poco openssl icu4c libiodbc
 ```
 
 #### UnixODBC <!-- omit in toc -->
 
 Homebrew: execute the following in the terminal (assuming you have [Homebrew](https://brew.sh/) installed):
 
 ```sh
-brew install poco openssl unixodbc
+brew install poco openssl icu4c unixodbc
 ```
 
 ### Run-time dependencies: Red Hat/CentOS
@@ -195,15 +195,15 @@ brew install poco openssl unixodbc
 Execute the following in the terminal:
 
 ```sh
-sudo yum install openssl unixODBC
+sudo yum install openssl libicu unixODBC
 ```
 
 #### iODBC <!-- omit in toc -->
 
 Execute the following in the terminal:
 
 ```sh
-sudo yum install openssl libiodbc
+sudo yum install openssl libicu libiodbc
 ```
 
 ### Run-time dependencies: Debian/Ubuntu
@@ -213,15 +213,15 @@ sudo yum install openssl libiodbc
 Execute the following in the terminal:
 
 ```sh
-sudo apt install openssl unixodbc
+sudo apt install openssl libicu unixodbc
 ```
 
 #### iODBC <!-- omit in toc -->
 
 Execute the following in the terminal:
 
 ```sh
-sudo apt install openssl libiodbc2
+sudo apt install openssl libicu libiodbc2
 ```
 
 ### Configuration: MDAC/WDAC (Microsoft/Windows Data Access Components)
@@ -368,15 +368,15 @@ You will need Xcode 10 or later and Command Line Tools to be installed, as well
 Homebrew: execute the following in the terminal (assuming you have [Homebrew](https://brew.sh/) installed):
 
 ```sh
-brew install git cmake make poco openssl libiodbc
+brew install git cmake make poco openssl icu4c libiodbc
 ```
 
 #### Build-time dependencies: UnixODBC <!-- omit in toc -->
 
 Homebrew: execute the following in the terminal (assuming you have [Homebrew](https://brew.sh/) installed):
 
 ```sh
-brew install git cmake make poco openssl unixodbc
+brew install git cmake make poco openssl icu4c unixodbc
 ```
 
 #### Build steps <!-- omit in toc -->
@@ -397,7 +397,7 @@ cd build
 # Configuration options for the project can be specified in the next command in a form of '-Dopt=val'.
 
 # You may also add '-G Xcode' to the next command, in order to use Xcode as a build system or IDE, and generate the solution and project files instead of Makefile.
-cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ..
+cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DOPENSSL_ROOT_DIR=$(brew --prefix)/opt/openssl -DICU_ROOT=$(brew --prefix)/opt/icu4c ..
 ```
 
 Build the generated solution in-place:
@@ -430,7 +430,7 @@ sudo yum install epel-release
 sudo yum groupinstall "Development Tools"
 sudo yum install centos-release-scl
 sudo yum install devtoolset-8
-sudo yum install git cmake3 openssl-devel unixODBC-devel
+sudo yum install git cmake3 openssl-devel libicu-devel unixODBC-devel
 ```
 
 #### Build-time dependencies: iODBC <!-- omit in toc -->
@@ -442,7 +442,7 @@ sudo yum install epel-release
 sudo yum groupinstall "Development Tools"
 sudo yum install centos-release-scl
 sudo yum install devtoolset-8
-sudo yum install git cmake3 openssl-devel libiodbc-devel
+sudo yum install git cmake3 openssl-devel libicu-devel libiodbc-devel
 ```
 
 #### Build steps <!-- omit in toc -->
@@ -491,15 +491,15 @@ cmake3 --build . --config RelWithDebInfo --target test
 Execute the following in the terminal:
 
 ```sh
-sudo apt install build-essential git cmake libpoco-dev libssl-dev unixodbc-dev
+sudo apt install build-essential git cmake libpoco-dev libssl-dev libicu-dev unixodbc-dev
 ```
 
 #### Build-time dependencies: iODBC <!-- omit in toc -->
 
 Execute the following in the terminal:
 
 ```sh
-sudo apt install build-essential git cmake libpoco-dev libssl-dev libiodbc2-dev
+sudo apt install build-essential git cmake libpoco-dev libssl-dev libicu-dev libiodbc2-dev
 ```
 
 #### Build steps <!-- omit in toc -->

diff --git a/contrib/folly b/contrib/folly
diff --git a/driver/CMakeLists.txt b/driver/CMakeLists.txt
@@ -3,6 +3,7 @@ function (declare_odbc_lib_targets libname UNICODE)
 # See the usage of each workaround in the code, for comments.
 unset (WORKAROUND_ALLOW_UNSAFE_DISPATCH)
 unset (WORKAROUND_DISABLE_SSL)
+unset (WORKAROUND_USE_ICU)
 unset (WORKAROUND_ENABLE_TRIM_TRAILING_NULL)
 unset (WORKAROUND_ENABLE_DEFINE_SQLBindParam)
 unset (WORKAROUND_ENABLE_NO_UNICODE_CHARS_IN_COLUMN_NAMES_IN_TESTS)
@@ -31,6 +32,10 @@ if (NOT CH_ODBC_ENABLE_SSL)
     set (WORKAROUND_DISABLE_SSL 1)
 endif()
 
+if (CH_ODBC_USE_ICU)
+    set (WORKAROUND_USE_ICU 1)
+endif ()
+
 message (STATUS "${libname}: ${DRIVER_PREFIX}")
 
 configure_file (
@@ -40,6 +45,7 @@ configure_file (
 
 unset (WORKAROUND_ALLOW_UNSAFE_DISPATCH)
 unset (WORKAROUND_DISABLE_SSL)
+unset (WORKAROUND_USE_ICU)
 unset (WORKAROUND_ENABLE_TRIM_TRAILING_NULL)
 unset (WORKAROUND_ENABLE_DEFINE_SQLBindParam)
 unset (WORKAROUND_ENABLE_NO_UNICODE_CHARS_IN_COLUMN_NAMES_IN_TESTS)
@@ -48,6 +54,8 @@ unset (WORKAROUND_ENABLE_NO_UNICODE_CHARS_IN_COLUMN_NAMES_IN_TESTS)
 add_library (${libname}-impl STATIC
     utils/type_parser.cpp
     utils/type_info.cpp
+    utils/unicode_converter.cpp
+    utils/conversion_context.cpp
 
     config/config.cpp
 
@@ -75,8 +83,16 @@ add_library (${libname}-impl STATIC
     platform/platform.h
 
     utils/utils.h
-    utils/unicode_conv.h
     utils/iostream_debug_helpers.h
+    utils/amortized_istream_reader.h
+    utils/resize_without_initialization.h
+    utils/object_pool.h
+    utils/string_pool.h
+    utils/unicode_converter.h
+    utils/conversion_context.h
+    utils/conversion.h
+    utils/conversion_std.h
+    utils/conversion_icu.h
     utils/type_parser.h
     utils/type_info.h
 
@@ -154,6 +170,12 @@ target_link_libraries (${libname}-impl
     PUBLIC Threads::Threads
 )
 
+if (CH_ODBC_USE_ICU)
+    target_link_libraries (${libname}-impl
+        PUBLIC ICU::uc
+    )
+endif ()
+
 add_library (${libname} SHARED
     api/odbc.cpp
     ${WIN_SOURCES}

diff --git a/driver/api/odbc.cpp b/driver/api/odbc.cpp
@@ -478,7 +478,7 @@ SQLRETURN SQL_API EXPORTED_FUNCTION_MAYBE_W(SQLDriverConnect)(
             if (StringLength1 > 0)
                 out_buffer_length = StringLength1;
             else if (StringLength1 == SQL_NTS)
-                out_buffer_length = NTSStringLength(InConnectionString) + 1; // +1 for null terminating character
+                out_buffer_length = stringBufferLength(InConnectionString);
             else
                 out_buffer_length = 1024; // ...as per SQLDriverConnect() doc: "Applications should allocate at least 1,024 characters for this buffer."
         }

diff --git a/driver/config/config.h b/driver/config/config.h
@@ -1,8 +1,10 @@
 #pragma once
 
+#include "driver/platform/platform.h"
 #include "driver/utils/utils.h"
 
 #include <string>
+#include <map>
 
 using key_value_map_t = std::map<std::string, std::string, UTF8CaseInsensitiveCompare>;
 

diff --git a/driver/format/ODBCDriver2.cpp b/driver/format/ODBCDriver2.cpp
@@ -1,4 +1,5 @@
 #include "driver/format/ODBCDriver2.h"
+#include "driver/utils/resize_without_initialization.h"
 
 ODBCDriver2ResultSet::ODBCDriver2ResultSet(AmortizedIStreamReader & stream, std::unique_ptr<ResultMutator> && mutator)
     : ResultSet(stream, std::move(mutator))

diff --git a/driver/format/RowBinaryWithNamesAndTypes.cpp b/driver/format/RowBinaryWithNamesAndTypes.cpp
@@ -1,4 +1,5 @@
 #include "driver/format/RowBinaryWithNamesAndTypes.h"
+#include "driver/utils/resize_without_initialization.h"
 
 #include <ctime>
 

diff --git a/driver/platform/config_cmake.h.in b/driver/platform/config_cmake.h.in
@@ -16,6 +16,8 @@
 // See the usage of each workaround in the code, for comments.
 #cmakedefine WORKAROUND_ALLOW_UNSAFE_DISPATCH
 #cmakedefine WORKAROUND_DISABLE_SSL
+#cmakedefine WORKAROUND_USE_ICU
+#cmakedefine WORKAROUND_ICU_USE_EXPLICIT_PIVOTING
 #cmakedefine WORKAROUND_ENABLE_TRIM_TRAILING_NULL
 #cmakedefine WORKAROUND_ENABLE_DEFINE_SQLBindParam
 #cmakedefine WORKAROUND_ENABLE_NO_UNICODE_CHARS_IN_COLUMN_NAMES_IN_TESTS
diff --git a/driver/platform/win/setup.cpp b/driver/platform/win/setup.cpp
@@ -3,6 +3,7 @@
 #include "driver/platform/platform.h"
 #include "driver/platform/win/resource.h"
 #include "driver/utils/utils.h"
+#include "driver/utils/conversion.h"
 #include "driver/config/config.h"
 #include "driver/config/ini_defines.h"
 

diff --git a/driver/result_set.cpp b/driver/result_set.cpp
@@ -304,7 +304,7 @@ std::unique_ptr<ResultReader> make_result_reader(const std::string & format, std
         return std::make_unique<ODBCDriver2ResultReader>(raw_stream, std::move(mutator));
     }
     else if (format == "RowBinaryWithNamesAndTypes") {
-        if (!is_little_endian())
+        if (!isLittleEndian())
             throw std::runtime_error("'" + format + "' format is supported only on little-endian platforms");
 
         return std::make_unique<RowBinaryWithNamesAndTypesResultReader>(raw_stream, std::move(mutator));