From f4e96fad52ea5a6c7e821b9a322eed4a0cb78f9a Mon Sep 17 00:00:00 2001 From: HuangWei Date: Mon, 19 Jun 2023 17:28:30 +0800 Subject: [PATCH 1/5] fix: dlopen error, install udf dir and doc --- CMakeLists.txt | 1 + docs/zh/openmldb_sql/ddl/CREATE_FUNCTION.md | 6 +++++- hybridse/src/udf/dynamic_lib_manager.cc | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4fd6a04090d..0d784ec5dd9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -321,3 +321,4 @@ install( DESTINATION tools PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ WORLD_EXECUTE WORLD_READ ) +install(DIRECTORY DESTINATION udf) diff --git a/docs/zh/openmldb_sql/ddl/CREATE_FUNCTION.md b/docs/zh/openmldb_sql/ddl/CREATE_FUNCTION.md index 72befdaa1b4..37951623922 100644 --- a/docs/zh/openmldb_sql/ddl/CREATE_FUNCTION.md +++ b/docs/zh/openmldb_sql/ddl/CREATE_FUNCTION.md @@ -53,4 +53,8 @@ CREATE AGGREGATE FUNCTION special_sum(x BIGINT) RETURNS BIGINT OPTIONS (FILE = ' 创建一个输入参数是bigint并且支持null, 返回类型是bigint并且支持返回null聚合函数，动态库文件为libtest_udf.so ```sql CREATE AGGREGATE FUNCTION count_null(x BIGINT) RETURNS BIGINT OPTIONS (FILE = 'libtest_udf.so', ARG_NULLABLE=true, RETURN_NULLABLE=true); -``` \ No newline at end of file +``` + +```{warning} +如果集群版本<=0.8.0，TabletServer部署目录中默认没有`udf`目录，启动TabletServer后再创建目录并拷贝udf动态库，将无法生效（环境变量问题）。需保证TabletServer启动前存在udf目录，如果已启动，需要将所有TabletServer重启使环境变量生效。 +``` diff --git a/hybridse/src/udf/dynamic_lib_manager.cc b/hybridse/src/udf/dynamic_lib_manager.cc index 7fca58c0ae8..060804ee5b7 100644 --- a/hybridse/src/udf/dynamic_lib_manager.cc +++ b/hybridse/src/udf/dynamic_lib_manager.cc @@ -45,7 +45,7 @@ base::Status DynamicLibManager::ExtractFunction(const std::string& name, bool is } if (!so_handle) { void* handle = dlopen(file.c_str(), RTLD_LAZY); - CHECK_TRUE(handle != nullptr, common::kExternalUDFError, "can not open the dynamic library: " + file) + CHECK_TRUE(handle != nullptr, common::kExternalUDFError, "can not open the dynamic library: " + file + "error: " + dlerror()) so_handle = std::make_shared(handle); std::lock_guard lock(mu_); handle_map_.emplace(file, so_handle); From e26893e4fba25407fd4f43bb0b39d292e54a141a Mon Sep 17 00:00:00 2001 From: HuangWei Date: Mon, 19 Jun 2023 17:59:18 +0800 Subject: [PATCH 2/5] doc --- docs/zh/deploy/install_deploy.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/zh/deploy/install_deploy.md b/docs/zh/deploy/install_deploy.md index 73d83892e9b..d7c288ef965 100644 --- a/docs/zh/deploy/install_deploy.md +++ b/docs/zh/deploy/install_deploy.md @@ -4,7 +4,17 @@ ### 操作系统 -发行的预编译包支持：CentOS 7.x, Ubuntu 20.04, SUSE 12 SP3, macOS 12。对于其他操作系统发行版本，预编译包未做充分测试，无法保证其完全兼容。你可以尝试 [从源码编译](compile.md)，来支持其他的操作系统。 +发行的预编译包支持：CentOS 7.x, Ubuntu 20.04, SUSE 12 SP3, macOS 12。因此Linux要求glibc version >= 2.17。对于其他操作系统发行版本，预编译包未做充分测试，无法保证其完全兼容。你可以尝试 [从源码编译](compile.md)，来支持其他的操作系统。 + +```{note} +Linux可通过以下命令来检查系统的支持情况。 +````shell +cat /etc/redhat-release # redhat only +ldd --version +strings /lib64/libc.so.6 | grep ^GLIBC_ +```` +通常ldd版本2.17，libc.so.6中也会有`GLIBC_2.17`，也就是该系统支持glibc 2.17的程序/动态库运行。如果系统的glibc版本低于2.17，则需要尝试从源码编译。 +``` ### 第三方组件依赖 From 771ebd6c7df6cacfbb88e018d9b63d881b557407 Mon Sep 17 00:00:00 2001 From: HuangWei Date: Mon, 19 Jun 2023 18:07:16 +0800 Subject: [PATCH 3/5] fix --- docs/zh/deploy/install_deploy.md | 2 +- docs/zh/openmldb_sql/udf_develop_guide.md | 2 +- hybridse/src/udf/dynamic_lib_manager.cc | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/zh/deploy/install_deploy.md b/docs/zh/deploy/install_deploy.md index d7c288ef965..1e6aa5922f9 100644 --- a/docs/zh/deploy/install_deploy.md +++ b/docs/zh/deploy/install_deploy.md @@ -13,7 +13,7 @@ cat /etc/redhat-release # redhat only ldd --version strings /lib64/libc.so.6 | grep ^GLIBC_ ```` -通常ldd版本2.17，libc.so.6中也会有`GLIBC_2.17`，也就是该系统支持glibc 2.17的程序/动态库运行。如果系统的glibc版本低于2.17，则需要尝试从源码编译。 +通常ldd版本>=2.17，libc.so.6中也会有`GLIBC_2.17`，也就是该系统支持glibc 2.17的程序/动态库运行。如果系统的glibc版本低于2.17，则需要尝试从源码编译。 ``` ### 第三方组件依赖 diff --git a/docs/zh/openmldb_sql/udf_develop_guide.md b/docs/zh/openmldb_sql/udf_develop_guide.md index 8b0f867d58e..bfe581068ab 100644 --- a/docs/zh/openmldb_sql/udf_develop_guide.md +++ b/docs/zh/openmldb_sql/udf_develop_guide.md @@ -153,7 +153,7 @@ g++ -shared -o libtest_udf.so examples/test_udf.cc -I /work/OpenMLDB/include -st ``` ### 2.3 拷贝动态库 -编译过的动态库需要被拷贝到 TaskManager 和 tablets中。如果 TaskManager 和 tablets中不存在`udf`目录，请先创建。 +编译过的动态库需要被拷贝到 TaskManager 和 tablets中。如果 TaskManager 和 tablets中不存在`udf`目录，请先创建并重启这些进程（保证环境变量生效）。 - tablet的UDF目录是 `path_to_tablet/udf`。 - TaskManager的UDF目录是 `path_to_taskmanager/taskmanager/bin/udf`。 diff --git a/hybridse/src/udf/dynamic_lib_manager.cc b/hybridse/src/udf/dynamic_lib_manager.cc index 060804ee5b7..d76aa65f9a2 100644 --- a/hybridse/src/udf/dynamic_lib_manager.cc +++ b/hybridse/src/udf/dynamic_lib_manager.cc @@ -45,7 +45,7 @@ base::Status DynamicLibManager::ExtractFunction(const std::string& name, bool is } if (!so_handle) { void* handle = dlopen(file.c_str(), RTLD_LAZY); - CHECK_TRUE(handle != nullptr, common::kExternalUDFError, "can not open the dynamic library: " + file + "error: " + dlerror()) + CHECK_TRUE(handle != nullptr, common::kExternalUDFError, "can not open the dynamic library: " + file + ", error: " + dlerror()) so_handle = std::make_shared(handle); std::lock_guard lock(mu_); handle_map_.emplace(file, so_handle); From 8b894125ba6278bed842351c00a94f34fa8892c3 Mon Sep 17 00:00:00 2001 From: HuangWei Date: Sun, 25 Jun 2023 15:02:32 +0800 Subject: [PATCH 4/5] fix --- hybridse/src/udf/dynamic_lib_manager.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/hybridse/src/udf/dynamic_lib_manager.cc b/hybridse/src/udf/dynamic_lib_manager.cc index d76aa65f9a2..8a4daef3e82 100644 --- a/hybridse/src/udf/dynamic_lib_manager.cc +++ b/hybridse/src/udf/dynamic_lib_manager.cc @@ -31,8 +31,8 @@ DynamicLibManager::~DynamicLibManager() { handle_map_.clear(); } -base::Status DynamicLibManager::ExtractFunction(const std::string& name, bool is_aggregate, - const std::string& file, std::vector* funs) { +base::Status DynamicLibManager::ExtractFunction(const std::string& name, bool is_aggregate, const std::string& file, + std::vector* funs) { CHECK_TRUE(funs != nullptr, common::kExternalUDFError, "funs is nullptr") std::shared_ptr so_handle; { @@ -45,7 +45,8 @@ base::Status DynamicLibManager::ExtractFunction(const std::string& name, bool is } if (!so_handle) { void* handle = dlopen(file.c_str(), RTLD_LAZY); - CHECK_TRUE(handle != nullptr, common::kExternalUDFError, "can not open the dynamic library: " + file + ", error: " + dlerror()) + CHECK_TRUE(handle != nullptr, common::kExternalUDFError, + "can not open the dynamic library: " + file + ", error: " + dlerror()) so_handle = std::make_shared(handle); std::lock_guard lock(mu_); handle_map_.emplace(file, so_handle); @@ -96,8 +97,7 @@ base::Status DynamicLibManager::RemoveHandler(const std::string& file) { } } if (so_handle) { - CHECK_TRUE(dlclose(so_handle->handle) == 0, common::kExternalUDFError, - "dlclose run error. file is " + file) + CHECK_TRUE(dlclose(so_handle->handle) == 0, common::kExternalUDFError, "dlclose run error. file is " + file) } return {}; } From c3409c6e357adef0105a6f3f78e0b67394fc17a3 Mon Sep 17 00:00:00 2001 From: HuangWei Date: Mon, 26 Jun 2023 10:13:52 +0800 Subject: [PATCH 5/5] add --- docs/zh/deploy/install_deploy.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/zh/deploy/install_deploy.md b/docs/zh/deploy/install_deploy.md index 1e6aa5922f9..d9a23be6db0 100644 --- a/docs/zh/deploy/install_deploy.md +++ b/docs/zh/deploy/install_deploy.md @@ -9,6 +9,7 @@ ```{note} Linux可通过以下命令来检查系统的支持情况。 ````shell +cat /etc/os-release # most linux cat /etc/redhat-release # redhat only ldd --version strings /lib64/libc.so.6 | grep ^GLIBC_