diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 8be488306c4a18..e520b3cbb1c094 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 3.12.0) +cmake_minimum_required(VERSION 3.19.2) # set CMAKE_C_COMPILER, this must set before project command if (DEFINED ENV{DORIS_GCC_HOME}) @@ -130,14 +130,13 @@ endif() set(Boost_DEBUG FALSE) set(Boost_USE_MULTITHREADED ON) set(BOOST_ROOT ${THIRDPARTY_DIR}) +set(Boost_NO_BOOST_CMAKE OFF) if (NOT APPLE) - find_package(Boost 1.55.0 REQUIRED COMPONENTS thread regex filesystem system date_time program_options) + find_package(Boost 1.73.0 REQUIRED COMPONENTS regex system filesystem thread date_time program_options) else() - find_package(Boost 1.55.0 COMPONENTS thread regex filesystem system date_time program_options) + find_package(Boost 1.73.0 COMPONENTS thread regex system filesystem date_time program_options) endif() -include_directories(${Boost_INCLUDE_DIRS}) -message(STATUS ${Boost_LIBRARIES}) set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools") @@ -313,10 +312,13 @@ check_function_exists(sched_getcpu HAVE_SCHED_GETCPU) # -fno-omit-frame-pointers: Keep frame pointer for functions in register set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wall -Wno-sign-compare -Wno-unknown-pragmas -pthread") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -fno-strict-aliasing -fno-omit-frame-pointer") -set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=gnu++11 -D__STDC_FORMAT_MACROS") +set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -std=gnu++17 -D__STDC_FORMAT_MACROS") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -Wno-deprecated -Wno-vla") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_DATE_TIME_POSIX_TIME_STD_CONFIG") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_SYSTEM_NO_DEPRECATED") +# https://github.com/boostorg/uuid/issues/92 +# We need to avoid using SYS_getrandom system calls +set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DBOOST_UUID_RANDOM_PROVIDER_FORCE_POSIX=1") if ("${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "x86" OR "${CMAKE_BUILD_TARGET_ARCH}" STREQUAL "x86_64") set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -msse4.2") endif() @@ -344,7 +346,7 @@ set(CXX_GCC_FLAGS "${CXX_GCC_FLAGS} -g -Wno-unused-local-typedefs") # Debug information is stored as dwarf2 to be as compatible as possible # -Werror: compile warnings should be errors when using the toolchain compiler. # Only enable for debug builds because this is what we test in pre-commit tests. -set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -Werror -ggdb3 -O0 -gdwarf-2") +set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -ggdb3 -O0 -gdwarf-2") # For CMAKE_BUILD_TYPE=Release # -O3: Enable all compiler optimizations @@ -449,7 +451,12 @@ set(DORIS_DEPENDENCIES librdkafka libs2 snappy - ${Boost_LIBRARIES} + Boost::regex + Boost::system + Boost::filesystem + Boost::thread + Boost::date_time + Boost::program_options thrift thriftnb glog diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt index 9f04d41dd7acd7..c6da8ede2e4eba 100644 --- a/be/src/exec/CMakeLists.txt +++ b/be/src/exec/CMakeLists.txt @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -cmake_minimum_required(VERSION 2.6) - # where to put generated libraries set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/exec") diff --git a/be/src/exec/decompressor.cpp b/be/src/exec/decompressor.cpp index 56c74cb9e34f09..fe805a9524f66b 100644 --- a/be/src/exec/decompressor.cpp +++ b/be/src/exec/decompressor.cpp @@ -63,7 +63,7 @@ std::string Decompressor::debug_info() { // Gzip GzipDecompressor::GzipDecompressor(bool is_deflate) - : Decompressor(_is_deflate ? CompressType::DEFLATE : CompressType::GZIP), + : Decompressor(is_deflate ? CompressType::DEFLATE : CompressType::GZIP), _is_deflate(is_deflate) {} GzipDecompressor::~GzipDecompressor() { diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h index ca71dbab4d4829..e5fa57eaabd497 100644 --- a/be/src/exec/olap_common.h +++ b/be/src/exec/olap_common.h @@ -435,16 +435,21 @@ void ColumnValueRange::convert_to_fixed_value() { return; } + // Incrementing boolean is denied in C++17, So we use int as bool type + using type = std::conditional_t::value, int, T>; + type low_value = _low_value; + type high_value = _high_value; + if (_low_op == FILTER_LARGER) { - ++_low_value; + ++low_value; } - for (T v = _low_value; v < _high_value; ++v) { + for (auto v = low_value; v < high_value; ++v) { _fixed_values.insert(v); } if (_high_op == FILTER_LESS_OR_EQUAL) { - _fixed_values.insert(_high_value); + _fixed_values.insert(high_value); } } diff --git a/be/src/exec/scan_node.cpp b/be/src/exec/scan_node.cpp index f355fe58de5739..f292ed31f2c414 100644 --- a/be/src/exec/scan_node.cpp +++ b/be/src/exec/scan_node.cpp @@ -17,8 +17,6 @@ #include "exec/scan_node.h" -#include - namespace doris { const std::string ScanNode::_s_bytes_read_counter = "BytesRead"; diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index 80755e247203cb..5ea00952ba65b4 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -21,7 +21,7 @@ #include #include -#include +#include #include #include "agent/utils.h" @@ -336,7 +336,7 @@ void add_default_path_handlers(WebPageHandler* web_page_handler, web_page_handler->register_page("/logs", "Logs", logs_handler, false /* is_on_nav_bar */); web_page_handler->register_page("/varz", "Configs", config_handler, true /* is_on_nav_bar */); web_page_handler->register_page( - "/memz", "Memory", boost::bind(&mem_usage_handler, process_mem_tracker, _1, _2), + "/memz", "Memory", boost::bind(&mem_usage_handler, process_mem_tracker, boost::placeholders::_1, boost::placeholders::_2), true /* is_on_nav_bar */); web_page_handler->register_page("/mem_tracker", "MemTracker", mem_tracker_handler, true /* is_on_nav_bar */); @@ -345,7 +345,7 @@ void add_default_path_handlers(WebPageHandler* web_page_handler, web_page_handler->register_page("/cpu", "CPU Profile", cpu_handler, true /* is_on_nav_bar */); register_thread_display_page(web_page_handler); web_page_handler->register_template_page("/tablets_page", "Tablets", - boost::bind(&display_tablets_callback, _1, _2), + boost::bind(&display_tablets_callback, boost::placeholders::_1, boost::placeholders::_2), true /* is_on_nav_bar */); } diff --git a/be/src/http/ev_http_server.cpp b/be/src/http/ev_http_server.cpp index d5bd907b51efd2..63d211c7834a90 100644 --- a/be/src/http/ev_http_server.cpp +++ b/be/src/http/ev_http_server.cpp @@ -147,7 +147,7 @@ Status EvHttpServer::_bind() { ss << "convert address failed, host=" << _host << ", port=" << _port; return Status::InternalError(ss.str()); } - _server_fd = butil::tcp_listen(point, true); + _server_fd = butil::tcp_listen(point); if (_server_fd < 0) { char buf[64]; std::stringstream ss; diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index 06ddbeb74e92e7..b600e5bc09f0cf 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -18,6 +18,7 @@ #ifndef DORIS_BE_SRC_OLAP_ROWSET_ROWSET_H #define DORIS_BE_SRC_OLAP_ROWSET_ROWSET_H +#include #include #include #include diff --git a/be/src/olap/rowset/unique_rowset_id_generator.h b/be/src/olap/rowset/unique_rowset_id_generator.h index cb5d56645de2d3..22909bc1133795 100644 --- a/be/src/olap/rowset/unique_rowset_id_generator.h +++ b/be/src/olap/rowset/unique_rowset_id_generator.h @@ -17,6 +17,8 @@ #pragma once +#include + #include "olap/rowset/rowset_id_generator.h" #include "util/spinlock.h" #include "util/uid_util.h" diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp index ed73ce242f1f0a..670cd2103243bd 100644 --- a/be/src/olap/storage_engine.cpp +++ b/be/src/olap/storage_engine.cpp @@ -597,6 +597,7 @@ OLAPStatus StorageEngine::_start_trash_sweep(double* usage) { time_t now = time(nullptr); //获取UTC时间 tm local_tm_now; + local_tm_now.tm_isdst = 0; if (localtime_r(&now, &local_tm_now) == nullptr) { LOG(WARNING) << "fail to localtime_r time. time=" << now; return OLAP_ERR_OS_ERROR; @@ -721,6 +722,7 @@ OLAPStatus StorageEngine::_do_sweep(const string& scan_root, const time_t& local string dir_name = item->path().filename().string(); string str_time = dir_name.substr(0, dir_name.find('.')); tm local_tm_create; + local_tm_create.tm_isdst = 0; if (strptime(str_time.c_str(), "%Y%m%d%H%M%S", &local_tm_create) == nullptr) { LOG(WARNING) << "fail to strptime time. [time=" << str_time << "]"; res = OLAP_ERR_OS_ERROR; diff --git a/be/src/olap/task/engine_batch_load_task.cpp b/be/src/olap/task/engine_batch_load_task.cpp index dc71791c5a9592..a2718f063c7311 100644 --- a/be/src/olap/task/engine_batch_load_task.cpp +++ b/be/src/olap/task/engine_batch_load_task.cpp @@ -161,7 +161,7 @@ AgentStatus EngineBatchLoadTask::_get_tmp_file_dir(const string& root_path, stri boost::system::error_code error_code; boost::filesystem::create_directories(*download_path, error_code); - if (0 != error_code) { + if (error_code.failed()) { status = DORIS_ERROR; LOG(WARNING) << "create download dir failed.path: " << *download_path << ", error code: " << error_code; diff --git a/be/src/runtime/buffered_block_mgr2.cc b/be/src/runtime/buffered_block_mgr2.cc index 8ac9451ee8f0ed..a59d6bf1a5a243 100644 --- a/be/src/runtime/buffered_block_mgr2.cc +++ b/be/src/runtime/buffered_block_mgr2.cc @@ -781,7 +781,7 @@ Status BufferedBlockMgr2::write_unpinned_block(Block* block) { } disk_id %= _io_mgr->num_local_disks(); DiskIoMgr::WriteRange::WriteDoneCallback callback = - bind(mem_fn(&BufferedBlockMgr2::write_complete), this, block, _1); + bind(mem_fn(&BufferedBlockMgr2::write_complete), this, block, boost::placeholders::_1); block->_write_range = _obj_pool.add( new DiskIoMgr::WriteRange(tmp_file->path(), file_offset, disk_id, callback)); block->_tmp_file = tmp_file; diff --git a/be/src/runtime/buffered_tuple_stream2.cc b/be/src/runtime/buffered_tuple_stream2.cc index 43a974b3b4f97a..e0bcd5b141bb76 100644 --- a/be/src/runtime/buffered_tuple_stream2.cc +++ b/be/src/runtime/buffered_tuple_stream2.cc @@ -17,7 +17,7 @@ #include "runtime/buffered_tuple_stream2.h" -#include +#include #include "runtime/descriptors.h" #include "runtime/row_batch.h" diff --git a/be/src/runtime/buffered_tuple_stream3.cc b/be/src/runtime/buffered_tuple_stream3.cc index d021b1f9659947..126798727a0630 100644 --- a/be/src/runtime/buffered_tuple_stream3.cc +++ b/be/src/runtime/buffered_tuple_stream3.cc @@ -17,7 +17,7 @@ #include -#include +#include #include "runtime/buffered_tuple_stream3.inline.h" #include "runtime/bufferpool/reservation_tracker.h" diff --git a/be/src/runtime/bufferpool/buffer_allocator.cc b/be/src/runtime/bufferpool/buffer_allocator.cc index e3350e545726e3..8dbdb453dd3319 100644 --- a/be/src/runtime/bufferpool/buffer_allocator.cc +++ b/be/src/runtime/bufferpool/buffer_allocator.cc @@ -17,7 +17,7 @@ #include "runtime/bufferpool/buffer_allocator.h" -#include +#include #include #include "common/atomic.h" @@ -721,8 +721,7 @@ std::string BufferPool::FreeBufferArena::DebugString() { << " free buffers: " << lists.num_free_buffers.load() << " low water mark: " << lists.low_water_mark << " clean pages: " << lists.num_clean_pages.load() << " "; - lists.clean_pages.iterate(boost::bind(Page::DebugStringCallback, &ss, _1)); - + lists.clean_pages.iterate(boost::bind(Page::DebugStringCallback, &ss, boost::placeholders::_1)); ss << "\n"; } return ss.str(); diff --git a/be/src/runtime/bufferpool/buffer_pool.cc b/be/src/runtime/bufferpool/buffer_pool.cc index 38c324c3de0b3e..f2749e4fa2b3c8 100644 --- a/be/src/runtime/bufferpool/buffer_pool.cc +++ b/be/src/runtime/bufferpool/buffer_pool.cc @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include +#include #include #include @@ -700,11 +700,11 @@ string BufferPool::Client::DebugString() { << " in_flight_write_bytes: " << in_flight_write_pages_.bytes() << " reservation: " << reservation_.DebugString(); ss << "\n " << pinned_pages_.size() << " pinned pages: "; - pinned_pages_.iterate(boost::bind(Page::DebugStringCallback, &ss, _1)); + pinned_pages_.iterate(boost::bind(Page::DebugStringCallback, &ss, boost::placeholders::_1)); ss << "\n " << dirty_unpinned_pages_.size() << " dirty unpinned pages: "; - dirty_unpinned_pages_.iterate(boost::bind(Page::DebugStringCallback, &ss, _1)); + dirty_unpinned_pages_.iterate(boost::bind(Page::DebugStringCallback, &ss, boost::placeholders::_1)); ss << "\n " << in_flight_write_pages_.size() << " in flight write pages: "; - in_flight_write_pages_.iterate(boost::bind(Page::DebugStringCallback, &ss, _1)); + in_flight_write_pages_.iterate(boost::bind(Page::DebugStringCallback, &ss, boost::placeholders::_1)); return ss.str(); } diff --git a/be/src/runtime/client_cache.h b/be/src/runtime/client_cache.h index 8286853a3c8b00..b84db1c3c4412c 100644 --- a/be/src/runtime/client_cache.h +++ b/be/src/runtime/client_cache.h @@ -18,7 +18,7 @@ #ifndef DORIS_BE_RUNTIME_CLIENT_CACHE_H #define DORIS_BE_RUNTIME_CLIENT_CACHE_H -#include +#include #include #include #include @@ -191,12 +191,12 @@ class ClientCache { ClientCache() : _client_cache_helper() { _client_factory = boost::bind(boost::mem_fn(&ClientCache::make_client), - this, _1, _2); + this, boost::placeholders::_1, boost::placeholders::_2); } ClientCache(int max_cache_size) : _client_cache_helper(max_cache_size) { _client_factory = boost::bind(boost::mem_fn(&ClientCache::make_client), - this, _1, _2); + this, boost::placeholders::_1, boost::placeholders::_2); } // Close all clients connected to the supplied address, (e.g., in diff --git a/be/src/runtime/data_stream_mgr.h b/be/src/runtime/data_stream_mgr.h index 7758a46043eb2a..2fec5ef897ce5c 100644 --- a/be/src/runtime/data_stream_mgr.h +++ b/be/src/runtime/data_stream_mgr.h @@ -104,7 +104,7 @@ class DataStreamMgr { // less-than ordering for pair struct ComparisonOp { bool operator()(const std::pair& a, - const std::pair& b) { + const std::pair& b) const { if (a.first.hi < b.first.hi) { return true; } else if (a.first.hi > b.first.hi) { diff --git a/be/src/runtime/data_stream_recvr.cc b/be/src/runtime/data_stream_recvr.cc index 944c7d7dc7eccd..0962b3c34cfe2e 100644 --- a/be/src/runtime/data_stream_recvr.cc +++ b/be/src/runtime/data_stream_recvr.cc @@ -368,7 +368,7 @@ Status DataStreamRecvr::create_merger(const TupleRowComparator& less_than) { for (int i = 0; i < _sender_queues.size(); ++i) { child_input_batch_suppliers.emplace_back( - bind(mem_fn(&SenderQueue::get_batch), _sender_queues[i], _1)); + bind(mem_fn(&SenderQueue::get_batch), _sender_queues[i], boost::placeholders::_1)); } RETURN_IF_ERROR(_merger->prepare(child_input_batch_suppliers)); return Status::OK(); @@ -403,13 +403,13 @@ Status DataStreamRecvr::create_parallel_merger(const TupleRowComparator& less_th less_than, &_row_desc, _profile, mem_tracker, batch_size, false)); vector input_batch_suppliers; for (int j = i; j < std::min((size_t)i + step, _sender_queues.size()); ++j) { - input_batch_suppliers.emplace_back( - bind(mem_fn(&SenderQueue::get_batch), _sender_queues[j], _1)); + input_batch_suppliers.emplace_back(bind(mem_fn(&SenderQueue::get_batch), + _sender_queues[j], boost::placeholders::_1)); } child_merger->prepare(input_batch_suppliers); - child_input_batch_suppliers.emplace_back( - bind(mem_fn(&SortedRunMerger::get_batch), child_merger.get(), _1)); + child_input_batch_suppliers.emplace_back(bind(mem_fn(&SortedRunMerger::get_batch), + child_merger.get(), boost::placeholders::_1)); _child_mergers.emplace_back(std::move(child_merger)); } RETURN_IF_ERROR(_merger->prepare(child_input_batch_suppliers, true)); diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index 089921ac3473d5..bc22aeeed4a987 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -20,7 +20,7 @@ #include #include -#include +#include #include #include @@ -167,9 +167,9 @@ FragmentExecState::FragmentExecState(const TUniqueId& query_id, _fragment_instance_id(fragment_instance_id), _backend_num(backend_num), _exec_env(exec_env), - _executor(exec_env, - boost::bind(boost::mem_fn(&FragmentExecState::coordinator_callback), this, - _1, _2, _3)), + _executor(exec_env, boost::bind(boost::mem_fn(&FragmentExecState::coordinator_callback), + this, boost::placeholders::_1, boost::placeholders::_2, + boost::placeholders::_3)), _timeout_second(-1), _fragments_ctx(fragments_ctx) { _start_time = DateTimeValue::local_time(); @@ -184,9 +184,9 @@ FragmentExecState::FragmentExecState(const TUniqueId& query_id, _backend_num(backend_num), _exec_env(exec_env), _coord_addr(coord_addr), - _executor(exec_env, - boost::bind(boost::mem_fn(&FragmentExecState::coordinator_callback), this, - _1, _2, _3)), + _executor(exec_env, boost::bind(boost::mem_fn(&FragmentExecState::coordinator_callback), + this, boost::placeholders::_1, boost::placeholders::_2, + boost::placeholders::_3)), _timeout_second(-1) { _start_time = DateTimeValue::local_time(); } @@ -385,17 +385,17 @@ FragmentMgr::FragmentMgr(ExecEnv* exec_env) }); auto s = Thread::create( - "FragmentMgr", "cancel_timeout_plan_fragment", - [this]() { this->cancel_worker(); }, &_cancel_thread); + "FragmentMgr", "cancel_timeout_plan_fragment", [this]() { this->cancel_worker(); }, + &_cancel_thread); CHECK(s.ok()) << s.to_string(); // TODO(zc): we need a better thread-pool // now one user can use all the thread pool, others have no resource. s = ThreadPoolBuilder("FragmentMgrThreadPool") - .set_min_threads(config::fragment_pool_thread_num_min) - .set_max_threads(config::fragment_pool_thread_num_max) - .set_max_queue_size(config::fragment_pool_queue_size) - .build(&_thread_pool); + .set_min_threads(config::fragment_pool_thread_num_min) + .set_max_threads(config::fragment_pool_thread_num_max) + .set_max_queue_size(config::fragment_pool_queue_size) + .build(&_thread_pool); CHECK(s.ok()) << s.to_string(); } diff --git a/be/src/runtime/result_buffer_mgr.cpp b/be/src/runtime/result_buffer_mgr.cpp index 3889818feea381..247acc00940a81 100644 --- a/be/src/runtime/result_buffer_mgr.cpp +++ b/be/src/runtime/result_buffer_mgr.cpp @@ -17,7 +17,7 @@ #include "runtime/result_buffer_mgr.h" -#include +#include #include "gen_cpp/PaloInternalService_types.h" #include "gen_cpp/types.pb.h" diff --git a/be/src/runtime/snapshot_loader.cpp b/be/src/runtime/snapshot_loader.cpp index 1e863ed8146a7c..eccd0b04f8cd4b 100644 --- a/be/src/runtime/snapshot_loader.cpp +++ b/be/src/runtime/snapshot_loader.cpp @@ -298,7 +298,7 @@ Status SnapshotLoader::download(const std::map& src_to status = FileUtils::md5sum(full_local_file, &downloaded_md5sum); if (!status.ok()) { std::stringstream ss; - ss << "failed to get md5sum of file: " << full_local_file; + ss << "failed to get md5sum of file: " << full_local_file << ", err: " << status.get_error_msg(); LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } diff --git a/be/src/runtime/spill_sorter.cc b/be/src/runtime/spill_sorter.cc index 163dc3377cfcd4..24985df23ad9c8 100644 --- a/be/src/runtime/spill_sorter.cc +++ b/be/src/runtime/spill_sorter.cc @@ -1316,7 +1316,7 @@ Status SpillSorter::create_merger(int num_runs) { RETURN_IF_ERROR(run->prepare_read()); // Run::get_next_batch() is used by the merger to retrieve a batch of rows to merge // from this run. - merge_runs.push_back(bind(mem_fn(&Run::get_next_batch), run, _1)); + merge_runs.push_back(bind(mem_fn(&Run::get_next_batch), run, boost::placeholders::_1)); _sorted_runs.pop_front(); _merging_runs.push_back(run); } diff --git a/be/src/runtime/user_function_cache.cpp b/be/src/runtime/user_function_cache.cpp index 8661adeb33a1b4..f173feb3493179 100644 --- a/be/src/runtime/user_function_cache.cpp +++ b/be/src/runtime/user_function_cache.cpp @@ -19,6 +19,7 @@ #include // boost::is_any_of #include // boost::algorithm::ends_with +#include #include #include diff --git a/be/src/util/thrift_server.cpp b/be/src/util/thrift_server.cpp index 1f79082106505d..dc5a5edbe856db 100644 --- a/be/src/util/thrift_server.cpp +++ b/be/src/util/thrift_server.cpp @@ -94,12 +94,15 @@ class ThriftServer::ThriftServerEventProcessor bool _signal_fired; // The time, in milliseconds, to wait for a server to come up - static const int TIMEOUT_MS = 2500; + const static int TIMEOUT_MS; // Called in a separate thread; wraps TNonBlockingServer::serve in an exception handler void supervise(); }; +// https://stackoverflow.com/questions/5391973/undefined-reference-to-static-const-int +const int ThriftServer::ThriftServerEventProcessor::TIMEOUT_MS = 2500; + Status ThriftServer::ThriftServerEventProcessor::start_and_wait_for_server() { // Locking here protects against missed notifications if Supervise executes quickly boost::unique_lock lock(_signal_lock); diff --git a/be/test/exprs/topn_function_test.cpp b/be/test/exprs/topn_function_test.cpp index 7648a25c3e1f95..ac2c9e987e78ba 100644 --- a/be/test/exprs/topn_function_test.cpp +++ b/be/test/exprs/topn_function_test.cpp @@ -87,7 +87,7 @@ void test_topn_accuracy(FunctionContext* ctx, int key_space, int space_expand_ra random_strs[i] = gen_random(10); } - zipf_distribution zf(key_space, zipf_distribution_exponent); + zipf_distribution zf(key_space - 1, zipf_distribution_exponent); std::random_device rd; std::mt19937 gen(rd()); @@ -113,7 +113,7 @@ void test_topn_accuracy(FunctionContext* ctx, int key_space, int space_expand_ra std::uniform_int_distribution<> dist(0, PARALLEL-1); for (uint32_t i = 0; i < TOTAL_RECORDS; ++i) { // generate zipf_distribution - uint32_t index = zf(gen) - 1; + uint32_t index = zf(gen); // choose one single topn to update topn_single(ctx, random_strs[index], single_dst_str[dist(random_gen)], accuracy_map); } diff --git a/env.sh b/env.sh index 0d105e04856471..7c179a0d866440 100755 --- a/env.sh +++ b/env.sh @@ -16,6 +16,8 @@ # under the License. # check DORIS_HOME +export LC_ALL=C + if [[ -z ${DORIS_HOME} ]]; then echo "Error: DORIS_HOME is not set" exit 1 @@ -61,6 +63,23 @@ if [[ ! "$(printf '%s\n' "$required_ver" "$gcc_ver" | sort -V | head -n1)" = "$r exit 1 fi +# register keyword is forbidden to use in C++17 +# the C++ code generated by flex that remove register keyword after version 2.6.0 +# so we need check flex version here to avoid compilation failed +flex_ver=`flex --version | awk '{print $2}'` +required_ver="2.6.0" +if [[ ! "$(printf '%s\n' "$required_ver" "$flex_ver" | sort -V | head -n1)" = "$required_ver" ]]; then + echo "Error: flex version (${flex_ver}) must be greater than or equal to ${required_ver}" + exit 1 +fi + +# find binutils +if test -x ${DORIS_GCC_HOME}/bin/ld ;then + export DORIS_BIN_UTILS=${DORIS_GCC_HOME}/bin/ +else + export DORIS_BIN_UTILS=/usr/bin/ +fi + # export CLANG COMPATIBLE FLAGS export CLANG_COMPATIBLE_FLAGS=`echo | ${DORIS_GCC_HOME}/bin/gcc -Wp,-v -xc++ - -fsyntax-only 2>&1 \ | grep -E '^\s+/' | awk '{print "-I" $1}' | tr '\n' ' '` diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java index 0f98a889b53868..27206c664a39cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java @@ -1167,7 +1167,7 @@ public class Config extends ConfigBase { * If set to true, Doris will check if the compiled and running versions of Java are compatible */ @ConfField - public static boolean check_java_version = true; + public static boolean check_java_version = false; /** * control materialized view diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index aef8599dd78241..7cc907d3d9a488 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -162,7 +162,7 @@ build_libevent() { CFLAGS="-std=c99 -fPIC -D_BSD_SOURCE -fno-omit-frame-pointer -g -ggdb -O2 -I${TP_INCLUDE_DIR}" \ LDFLAGS="-L${TP_LIB_DIR}" \ ./configure --prefix=$TP_INSTALL_DIR --enable-shared=no --disable-samples --disable-libevent-regress - make -j$PARALLEL && make install + make -j $PARALLEL && make install } build_openssl() { @@ -181,7 +181,7 @@ build_openssl() { CFLAGS="-fPIC" \ LIBDIR="lib" \ ./Configure --prefix=$TP_INSTALL_DIR -zlib -shared ${OPENSSL_PLATFORM} - make -j$PARALLEL && make install_sw + make -j $PARALLEL && make install_sw # NOTE(zc): remove this dynamic library files to make libcurl static link. # If I don't remove this files, I don't known how to make libcurl link static library if [ -f $TP_INSTALL_DIR/lib64/libcrypto.so ]; then @@ -213,7 +213,7 @@ build_thrift() { mv compiler/cpp/thrifty.hh compiler/cpp/thrifty.h fi - make -j$PARALLEL && make install + make -j $PARALLEL && make install } # llvm @@ -241,7 +241,7 @@ build_llvm() { rm -rf CMakeCache.txt CMakeFiles/ LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \ ${CMAKE_CMD} -G "${GENERATOR}" -DLLVM_REQUIRES_RTTI:Bool=True -DLLVM_TARGETS_TO_BUILD=${LLVM_TARGET} -DLLVM_ENABLE_TERMINFO=OFF LLVM_BUILD_LLVM_DYLIB:BOOL=OFF -DLLVM_ENABLE_PIC=true -DBUILD_SHARED_LIBS=OFF -DCMAKE_BUILD_TYPE="RELEASE" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/llvm ../$LLVM_SOURCE - ${BUILD_SYSTEM} -j$PARALLEL REQUIRES_RTTI=1 && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL REQUIRES_RTTI=1 && ${BUILD_SYSTEM} install } # protobuf @@ -257,7 +257,7 @@ build_protobuf() { cd src sed -i 's/^AM_LDFLAGS\(.*\)$/AM_LDFLAGS\1 -all-static/' Makefile cd - - make -j$PARALLEL && make install + make -j $PARALLEL && make install } # gflags @@ -269,7 +269,7 @@ build_gflags() { rm -rf CMakeCache.txt CMakeFiles/ ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ -DCMAKE_POSITION_INDEPENDENT_CODE=On ../ - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } # glog @@ -285,7 +285,7 @@ build_glog() { LDFLAGS="-L${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ ./configure --prefix=$TP_INSTALL_DIR --enable-frame-pointers --disable-shared --enable-static - make -j$PARALLEL && make install + make -j $PARALLEL && make install } # gtest @@ -297,7 +297,7 @@ build_gtest() { rm -rf CMakeCache.txt CMakeFiles/ ${CMAKE_CMD} -G "${GENERATOR}" -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ -DCMAKE_POSITION_INDEPENDENT_CODE=On ../ - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } # rapidjson @@ -319,7 +319,7 @@ build_snappy() { -DCMAKE_POSITION_INDEPENDENT_CODE=On \ -DCMAKE_INSTALL_INCLUDEDIR=$TP_INCLUDE_DIR/snappy \ -DSNAPPY_BUILD_TESTS=0 ../ - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install #build for libarrow.a cp $TP_INCLUDE_DIR/snappy/snappy-c.h $TP_INCLUDE_DIR/snappy-c.h && \ @@ -344,7 +344,7 @@ build_gperftools() { LD_LIBRARY_PATH="${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ ./configure --prefix=$TP_INSTALL_DIR/gperftools --disable-shared --enable-static --disable-libunwind --with-pic --enable-frame-pointers - make -j$PARALLEL && make install + make -j $PARALLEL && make install } # zlib @@ -356,7 +356,7 @@ build_zlib() { LDFLAGS="-L${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ ./configure --prefix=$TP_INSTALL_DIR --static - make -j$PARALLEL && make install + make -j $PARALLEL && make install } # lz4 @@ -364,7 +364,7 @@ build_lz4() { check_if_source_exist $LZ4_SOURCE cd $TP_SOURCE_DIR/$LZ4_SOURCE - make -j$PARALLEL install PREFIX=$TP_INSTALL_DIR \ + make -j $PARALLEL install PREFIX=$TP_INSTALL_DIR \ INCLUDEDIR=$TP_INCLUDE_DIR/lz4/ } @@ -374,7 +374,7 @@ build_bzip() { cd $TP_SOURCE_DIR/$BZIP_SOURCE CFLAGS="-fPIC" - make -j$PARALLEL install PREFIX=$TP_INSTALL_DIR + make -j $PARALLEL install PREFIX=$TP_INSTALL_DIR } # lzo2 @@ -386,7 +386,7 @@ build_lzo2() { LDFLAGS="-L${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ ./configure --prefix=$TP_INSTALL_DIR --disable-shared --enable-static - make -j$PARALLEL && make install + make -j $PARALLEL && make install } # curl @@ -398,8 +398,9 @@ build_curl() { LDFLAGS="-L${TP_LIB_DIR}" LIBS="-lcrypto -lssl -lcrypto -ldl" \ CFLAGS="-fPIC" \ ./configure --prefix=$TP_INSTALL_DIR --disable-shared --enable-static \ - --without-librtmp --without-libssh2 --without-nghttp2 --with-ssl=${TP_INSTALL_DIR} --without-libidn2 --disable-ldap --enable-ipv6 - make -j$PARALLEL && make install + --without-librtmp --with-ssl=${TP_INSTALL_DIR} --without-libidn2 --disable-ldap --enable-ipv6 \ + --without-libssh2 + make -j $PARALLEL && make install } # re2 @@ -408,7 +409,7 @@ build_re2() { cd $TP_SOURCE_DIR/$RE2_SOURCE ${CMAKE_CMD} -G "${GENERATOR}" -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR - ${BUILD_SYSTEM} -j$PARALLEL install + ${BUILD_SYSTEM} -j $PARALLEL install } # boost @@ -416,29 +417,28 @@ build_boost() { check_if_source_exist $BOOST_SOURCE cd $TP_SOURCE_DIR/$BOOST_SOURCE - echo "using gcc : doris : ${CXX} ; " > tools/build/src/user-config.jam - ./bootstrap.sh --prefix=$TP_INSTALL_DIR - ./b2 --toolset=gcc-doris link=static -d0 -j$PARALLEL --without-mpi --without-graph --without-graph_parallel --without-python cxxflags="-std=c++11 -fPIC -I$TP_INCLUDE_DIR -L$TP_LIB_DIR" install + ./bootstrap.sh --prefix=$TP_INSTALL_DIR + ./b2 link=static runtime-link=static -j $PARALLEL --without-mpi --without-graph --without-graph_parallel --without-python cxxflags="-std=c++11 -g -fPIC -I$TP_INCLUDE_DIR -L$TP_LIB_DIR" install } # mysql build_mysql() { check_if_source_exist $MYSQL_SOURCE - check_if_source_exist $BOOST_FOR_MYSQL_SOURCE + check_if_source_exist $BOOST_SOURCE cd $TP_SOURCE_DIR/$MYSQL_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR rm -rf CMakeCache.txt CMakeFiles/ - if [ ! -d $BOOST_FOR_MYSQL_SOURCE ]; then - cp -rf $TP_SOURCE_DIR/$BOOST_FOR_MYSQL_SOURCE ./ + if [ ! -d $BOOST_SOURCE ]; then + cp -rf $TP_SOURCE_DIR/$BOOST_SOURCE ./ fi - ${CMAKE_CMD} -G "${GENERATOR}" ../ -DWITH_BOOST=`pwd`/$BOOST_FOR_MYSQL_SOURCE -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/mysql/ \ + ${CMAKE_CMD} -G "${GENERATOR}" ../ -DWITH_BOOST=`pwd`/$BOOST_SOURCE -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/mysql/ \ -DCMAKE_INCLUDE_PATH=$TP_INCLUDE_DIR -DWITHOUT_SERVER=1 -DWITH_ZLIB=$TP_INSTALL_DIR \ -DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-O3 -g -fabi-version=2 -fno-omit-frame-pointer -fno-strict-aliasing -std=gnu++11" \ -DDISABLE_SHARED=1 -DBUILD_SHARED_LIBS=0 - ${BUILD_SYSTEM} -j$PARALLEL mysqlclient + ${BUILD_SYSTEM} -j $PARALLEL mysqlclient # copy headers manually rm -rf ../../../installed/include/mysql/ @@ -457,7 +457,7 @@ build_mysql() { build_leveldb() { check_if_source_exist $LEVELDB_SOURCE cd $TP_SOURCE_DIR/$LEVELDB_SOURCE - CXXFLAGS="-fPIC" make -j$PARALLEL + CXXFLAGS="-fPIC" make -j $PARALLEL cp out-static/libleveldb.a ../../installed/lib/libleveldb.a cp -r include/leveldb ../../installed/include/ } @@ -471,9 +471,10 @@ build_brpc() { rm -rf CMakeCache.txt CMakeFiles/ LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" \ ${CMAKE_CMD} -G "${GENERATOR}" -v -DBUILD_SHARED_LIBS=0 -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ - -DBRPC_WITH_GLOG=ON -DWITH_GLOG=ON -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \ + -DBRPC_WITH_GLOG=ON -DWITH_GLOG=ON -DGFLAGS_LIBRARY=$TP_INSTALL_DIR/lib/libgflags.a -DGLOG_LIB=$TP_INSTALL_DIR/lib \ + -DGFLAGS_INCLUDE_DIR=$TP_INSTALL_DIR/include -DGLOG_LIB=$TP_INSTALL_DIR/lib/libglog.a -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \ -DPROTOBUF_PROTOC_EXECUTABLE=$TP_INSTALL_DIR/bin/protoc .. - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } # rocksdb @@ -483,7 +484,7 @@ build_rocksdb() { cd $TP_SOURCE_DIR/$ROCKSDB_SOURCE CFLAGS="-I ${TP_INCLUDE_DIR} -I ${TP_INCLUDE_DIR}/snappy -I ${TP_INCLUDE_DIR}/lz4" CXXFLAGS="-fPIC -Wno-deprecated-copy -Wno-stringop-truncation -Wno-pessimizing-move" LDFLAGS="-static-libstdc++ -static-libgcc" \ - PORTABLE=1 make USE_RTTI=1 -j$PARALLEL static_lib + PORTABLE=1 make USE_RTTI=1 -j $PARALLEL static_lib cp librocksdb.a ../../installed/lib/librocksdb.a cp -r include/rocksdb ../../installed/include/ } @@ -498,7 +499,7 @@ build_librdkafka() { LDFLAGS="-L${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ ./configure --prefix=$TP_INSTALL_DIR --enable-static --disable-sasl - make -j$PARALLEL && make install + make -j $PARALLEL && make install } # libunixodbc @@ -511,7 +512,7 @@ build_libunixodbc() { LDFLAGS="-L${TP_LIB_DIR}" \ CFLAGS="-fPIC" \ ./configure --prefix=$TP_INSTALL_DIR --with-included-ltdl --enable-static=yes - make -j$PARALLEL && make install + make -j $PARALLEL && make install } # flatbuffers @@ -523,7 +524,7 @@ build_flatbuffers() { CXXFLAGS="-fPIC -Wno-class-memaccess" \ LDFLAGS="-static-libstdc++ -static-libgcc" \ ${CMAKE_CMD} -G "${GENERATOR}" .. - ${BUILD_SYSTEM} -j$PARALLEL + ${BUILD_SYSTEM} -j $PARALLEL cp flatc ../../../installed/bin/flatc cp -r ../include/flatbuffers ../../../installed/include/flatbuffers cp libflatbuffers.a ../../../installed/lib/libflatbuffers.a @@ -543,8 +544,10 @@ build_arrow() { export ARROW_Thrift_URL=${TP_SOURCE_DIR}/${THRIFT_NAME} export LDFLAGS="-L${TP_LIB_DIR} -static-libstdc++ -static-libgcc" + #--trace-expand \ ${CMAKE_CMD} -G "${GENERATOR}" -DARROW_PARQUET=ON -DARROW_IPC=ON -DARROW_USE_GLOG=off \ -DARROW_BUILD_SHARED=OFF -DARROW_BUILD_STATIC=ON -DARROW_WITH_ZSTD=ON \ + -DLZ4_INCLUDE_DIR=$TP_INSTALL_DIR \ -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ -DCMAKE_INSTALL_LIBDIR=lib64 \ -DARROW_BOOST_USE_SHARED=OFF -DARROW_GFLAGS_USE_SHARED=OFF \ @@ -553,12 +556,17 @@ build_arrow() { -Dgflags_ROOT=$TP_INSTALL_DIR/ \ -DSnappy_ROOT=$TP_INSTALL_DIR/ \ -DGLOG_ROOT=$TP_INSTALL_DIR/ \ + -DGFLAGS_LIBRARY=$TP_INSTALL_DIR/lib/libgflags.a \ + -DGLOG_LIB=$TP_INSTALL_DIR/lib/libglog.a \ -DLZ4_ROOT=$TP_INSTALL_DIR/ \ + -DLZ4_INCLUDE_DIR=$TP_INSTALL_DIR/include/lz4 \ -DZSTD_SOURCE=BUNDLED \ + -DZLIB_LIBRARY=$TP_INSTALL_DIR/lib/libz.a -DZLIB_INCLUDE_DIR=$TP_INSTALL_DIR/include \ + -DGFLAGS_LIBRARY=$TP_INSTALL_DIR/lib/libgflags.a \ -Ddouble-conversion_SOURCE=BUNDLED \ -DThrift_ROOT=$TP_INSTALL_DIR/ .. - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install #copy dep libs cp -rf ./jemalloc_ep-prefix/src/jemalloc_ep/dist/lib/libjemalloc_pic.a $TP_INSTALL_DIR/lib64/libjemalloc.a @@ -587,8 +595,9 @@ build_s2() { -DGFLAGS_ROOT_DIR="$TP_INSTALL_DIR/include" \ -DWITH_GFLAGS=ON \ -DGLOG_ROOT_DIR="$TP_INSTALL_DIR/include" \ + -DGFLAGS_LIBRARY=$TP_INSTALL_DIR/lib/libgflags.a \ -DWITH_GLOG=ON .. - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } # bitshuffle @@ -616,26 +625,26 @@ build_bitshuffle() { fi tmp_obj=bitshuffle_${arch}_tmp.o dst_obj=bitshuffle_${arch}.o - ${CC:-gcc} $EXTRA_CFLAGS $arch_flag -std=c99 -I$PREFIX/include/lz4/ -O3 -DNDEBUG -fPIC -c \ + ${CC:-$DORIS_GCC_HOME/bin/gcc} $EXTRA_CFLAGS $arch_flag -std=c99 -I$PREFIX/include/lz4/ -O3 -DNDEBUG -fPIC -c \ "src/bitshuffle_core.c" \ "src/bitshuffle.c" \ "src/iochain.c" # Merge the object files together to produce a combined .o file. - ld -r -o $tmp_obj bitshuffle_core.o bitshuffle.o iochain.o + $DORIS_BIN_UTILS/ld -r -o $tmp_obj bitshuffle_core.o bitshuffle.o iochain.o # For the AVX2 symbols, suffix them. if [ "$arch" == "avx2" ]; then # Create a mapping file with ' ' on each line. - nm --defined-only --extern-only $tmp_obj | while read addr type sym ; do + $DORIS_BIN_UTILS/nm --defined-only --extern-only $tmp_obj | while read addr type sym ; do echo ${sym} ${sym}_${arch} done > renames.txt - objcopy --redefine-syms=renames.txt $tmp_obj $dst_obj + $DORIS_BIN_UTILS/objcopy --redefine-syms=renames.txt $tmp_obj $dst_obj else mv $tmp_obj $dst_obj fi to_link="$to_link $dst_obj" done rm -f libbitshuffle.a - ar rs libbitshuffle.a $to_link + $DORIS_BIN_UTILS/ar rs libbitshuffle.a $to_link mkdir -p $PREFIX/include/bitshuffle cp libbitshuffle.a $PREFIX/lib/ cp $TP_SOURCE_DIR/$BITSHUFFLE_SOURCE/src/bitshuffle.h $PREFIX/include/bitshuffle/bitshuffle.h @@ -653,7 +662,7 @@ build_croaringbitmap() { ${CMAKE_CMD} -G "${GENERATOR}" -v -DROARING_BUILD_STATIC=ON -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ -DCMAKE_INCLUDE_PATH="$TP_INSTALL_DIR/include" \ -DENABLE_ROARING_TESTS=OFF .. - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } #orc build_orc() { @@ -673,7 +682,7 @@ build_orc() { -DBUILD_CPP_TESTS=OFF \ -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } #cctz @@ -681,7 +690,7 @@ build_cctz() { check_if_source_exist $CCTZ_SOURCE cd $TP_SOURCE_DIR/$CCTZ_SOURCE export PREFIX=$TP_INSTALL_DIR - make -j$PARALLEL && make install + make -j $PARALLEL && make install } # all js and csss related @@ -716,7 +725,7 @@ build_aws_c_common() { cd $TP_SOURCE_DIR/$AWS_C_COMMON_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } # aws-c-event-stream @@ -726,7 +735,7 @@ build_aws_c_event_stream() { mkdir -p $BUILD_DIR && cd $BUILD_DIR cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF \ -DBUILD_TESTING=OFF - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } # aws-checksums @@ -735,7 +744,7 @@ build_aws_checksums() { cd $TP_SOURCE_DIR/$AWS_CHECKSUMS_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } # aws-c-io @@ -744,7 +753,7 @@ build_aws_c_io() { cd $TP_SOURCE_DIR/$AWS_C_IO_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } # aws-s2n @@ -753,7 +762,7 @@ build_aws_s2n() { cd $TP_SOURCE_DIR/$AWS_S2N_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } # aws-c-cal @@ -762,7 +771,7 @@ build_aws_c_cal() { cd $TP_SOURCE_DIR/$AWS_C_CAL_SOURCE mkdir -p $BUILD_DIR && cd $BUILD_DIR cmake -G "${GENERATOR}" .. -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } # aws_sdk @@ -773,7 +782,7 @@ build_aws_sdk() { $CMAKE_CMD -G "${GENERATOR}" .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR \ -DBUILD_DEPS=OFF -DCMAKE_PREFIX_PATH=$TP_INSTALL_DIR -DBUILD_SHARED_LIBS=OFF -DENABLE_TESTING=OFF \ -DCMAKE_MODULE_PATH=$TP_INSTALL_DIR/lib64/cmake -DBUILD_ONLY="s3" - ${BUILD_SYSTEM} -j$PARALLEL && ${BUILD_SYSTEM} install + ${BUILD_SYSTEM} -j $PARALLEL && ${BUILD_SYSTEM} install } # See https://github.com/apache/incubator-doris/issues/2910 # LLVM related codes have already be removed in master, so there is @@ -799,7 +808,6 @@ build_snappy build_gperftools build_curl build_re2 -build_mysql build_thrift build_leveldb build_brpc @@ -813,6 +821,7 @@ build_croaringbitmap build_orc build_cctz build_tsan_header +build_mysql build_aws_c_common build_aws_s2n build_aws_c_cal diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index f2799659570ce3..d34bb4db1899f4 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -251,6 +251,7 @@ echo "Finished patching $RE2_SOURCE" cd $TP_SOURCE_DIR/$MYSQL_SOURCE if [ ! -f $PATCHED_MARK ]; then patch -p0 < $TP_PATCH_DIR/mysql-5.7.18.patch + patch -Rp0 < $TP_PATCH_DIR/mysql-5.7.18-boost.patch touch $PATCHED_MARK fi cd - @@ -303,15 +304,6 @@ fi cd - echo "Finished patching $LZ4_SOURCE" -# brpc patch to disable shared library -cd $TP_SOURCE_DIR/$BRPC_SOURCE -if [ ! -f $PATCHED_MARK ] && [ $BRPC_SOURCE == "incubator-brpc-0.9.5" ]; then - patch -p0 < $TP_PATCH_DIR/incubator-brpc-0.9.5.patch - touch $PATCHED_MARK -fi -cd - -echo "Finished patching $BRPC_SOURCE" - # s2 patch to disable shared library cd $TP_SOURCE_DIR/$S2_SOURCE if [ ! -f $PATCHED_MARK ]; then diff --git a/thirdparty/patches/incubator-brpc-0.9.5.patch b/thirdparty/patches/incubator-brpc-0.9.5.patch deleted file mode 100644 index a717f5f99a55f6..00000000000000 --- a/thirdparty/patches/incubator-brpc-0.9.5.patch +++ /dev/null @@ -1,237 +0,0 @@ ---- CMakeLists.txt 2018-10-30 12:00:24.000000000 +0800 -+++ CMakeLists.txt 2020-01-19 15:30:38.260000000 +0800 -@@ -104,8 +104,11 @@ - - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - #required by butil/crc32.cc to boost performance for 10x -- if(NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.4)) -+ if((CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)") AND NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.4)) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4 -msse4.2") -+ elseif((CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")) -+ # segmentation fault in libcontext -+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-gcse -w") - endif() - if(NOT (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0)) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-aligned-new") ---- src/bthread/context.cpp 2018-10-30 12:00:24.000000000 +0800 -+++ src/bthread/context.cpp 2020-01-19 15:48:18.070000000 +0800 -@@ -600,3 +600,106 @@ - ); - - #endif -+ -+#if defined(BTHREAD_CONTEXT_PLATFORM_linux_arm64) && defined(BTHREAD_CONTEXT_COMPILER_gcc) -+__asm ( -+".cpu generic+fp+simd\n" -+".text\n" -+".align 2\n" -+".global bthread_jump_fcontext\n" -+".type bthread_jump_fcontext, %function\n" -+"bthread_jump_fcontext:\n" -+" # prepare stack for GP + FPU\n" -+" sub sp, sp, #0xb0\n" -+"# Because gcc may save integer registers in fp registers across a\n" -+"# function call we cannot skip saving the fp registers.\n" -+"#\n" -+"# Do not reinstate this test unless you fully understand what you\n" -+"# are doing.\n" -+"#\n" -+"# # test if fpu env should be preserved\n" -+"# cmp w3, #0\n" -+"# b.eq 1f\n" -+" # save d8 - d15\n" -+" stp d8, d9, [sp, #0x00]\n" -+" stp d10, d11, [sp, #0x10]\n" -+" stp d12, d13, [sp, #0x20]\n" -+" stp d14, d15, [sp, #0x30]\n" -+"1:\n" -+" # save x19-x30\n" -+" stp x19, x20, [sp, #0x40]\n" -+" stp x21, x22, [sp, #0x50]\n" -+" stp x23, x24, [sp, #0x60]\n" -+" stp x25, x26, [sp, #0x70]\n" -+" stp x27, x28, [sp, #0x80]\n" -+" stp x29, x30, [sp, #0x90]\n" -+" # save LR as PC\n" -+" str x30, [sp, #0xa0]\n" -+" # store RSP (pointing to context-data) in first argument (x0).\n" -+" # STR cannot have sp as a target register\n" -+" mov x4, sp\n" -+" str x4, [x0]\n" -+" # restore RSP (pointing to context-data) from A2 (x1)\n" -+" mov sp, x1\n" -+"# # test if fpu env should be preserved\n" -+"# cmp w3, #0\n" -+"# b.eq 2f\n" -+" # load d8 - d15\n" -+" ldp d8, d9, [sp, #0x00]\n" -+" ldp d10, d11, [sp, #0x10]\n" -+" ldp d12, d13, [sp, #0x20]\n" -+" ldp d14, d15, [sp, #0x30]\n" -+"2:\n" -+" # load x19-x30\n" -+" ldp x19, x20, [sp, #0x40]\n" -+" ldp x21, x22, [sp, #0x50]\n" -+" ldp x23, x24, [sp, #0x60]\n" -+" ldp x25, x26, [sp, #0x70]\n" -+" ldp x27, x28, [sp, #0x80]\n" -+" ldp x29, x30, [sp, #0x90]\n" -+" # use third arg as return value after jump\n" -+" # and as first arg in context function\n" -+" mov x0, x2\n" -+" # load pc\n" -+" ldr x4, [sp, #0xa0]\n" -+" # restore stack from GP + FPU\n" -+" add sp, sp, #0xb0\n" -+" ret x4\n" -+".size bthread_jump_fcontext,.-bthread_jump_fcontext\n" -+"# Mark that we don't need executable stack.\n" -+".section .note.GNU-stack,\"\",%progbits\n" -+); -+ -+#endif -+ -+#if defined(BTHREAD_CONTEXT_PLATFORM_linux_arm64) && defined(BTHREAD_CONTEXT_COMPILER_gcc) -+__asm ( -+".cpu generic+fp+simd\n" -+".text\n" -+".align 2\n" -+".global bthread_make_fcontext\n" -+".type bthread_make_fcontext, %function\n" -+"bthread_make_fcontext:\n" -+" # shift address in x0 (allocated stack) to lower 16 byte boundary\n" -+" and x0, x0, ~0xF\n" -+" # reserve space for context-data on context-stack\n" -+" sub x0, x0, #0xb0\n" -+" # third arg of bthread_make_fcontext() == address of context-function\n" -+" # store address as a PC to jump in\n" -+" str x2, [x0, #0xa0]\n" -+" # save address of finish as return-address for context-function\n" -+" # will be entered after context-function returns (LR register)\n" -+" adr x1, finish\n" -+" str x1, [x0, #0x98]\n" -+" ret x30 \n" -+"finish:\n" -+" # exit code is zero\n" -+" mov x0, #0\n" -+" # exit application\n" -+" bl _exit\n" -+".size bthread_make_fcontext,.-bthread_make_fcontext\n" -+"# Mark that we don't need executable stack.\n" -+".section .note.GNU-stack,\"\",%progbits\n" -+); -+ -+#endif ---- src/bthread/context.h 2018-10-30 12:00:24.000000000 +0800 -+++ src/bthread/context.h 2020-01-19 14:15:37.170000000 +0800 -@@ -30,6 +30,9 @@ - #elif __arm__ - #define BTHREAD_CONTEXT_PLATFORM_linux_arm32 - #define BTHREAD_CONTEXT_CALL_CONVENTION -+ #elif __aarch64__ -+ #define BTHREAD_CONTEXT_PLATFORM_linux_arm64 -+ #define BTHREAD_CONTEXT_CALL_CONVENTION - #endif - - #elif defined(__MINGW32__) || defined (__MINGW64__) ---- src/bthread/processor.h 2018-10-30 12:00:24.000000000 +0800 -+++ src/bthread/processor.h 2020-01-19 15:23:50.410000000 +0800 -@@ -20,9 +20,13 @@ - #define BTHREAD_PROCESSOR_H - - // Pause instruction to prevent excess processor bus usage, only works in GCC --# ifndef cpu_relax --# define cpu_relax() asm volatile("pause\n": : :"memory") --# endif -+#ifndef cpu_relax -+#if defined(ARCH_CPU_ARM_FAMILY) -+# define cpu_relax() asm volatile("yield\n": : :"memory") -+#else -+#define cpu_relax() asm volatile("pause\n": : :"memory") -+#endif -+#endif - - // Compile read-write barrier - # ifndef barrier ---- src/butil/config.h 1970-01-01 08:00:00.000000000 +0800 -+++ src/butil/config.h 2020-01-19 12:39:01.070000000 +0800 -@@ -0,0 +1,6 @@ -+#ifndef BUTIL_CONFIG_H -+#define BUTIL_CONFIG_H -+ -+#define BRPC_WITH_GLOG 1 -+ -+#endif // BUTIL_CONFIG_H ---- src/butil/third_party/snappy/snappy-internal.h 2018-10-30 12:00:24.000000000 +0800 -+++ src/butil/third_party/snappy/snappy-internal.h 2020-01-19 14:59:50.610000000 +0800 -@@ -132,7 +132,7 @@ - matched += 4; - } - if (LittleEndian::IsLittleEndian() && s2 <= s2_limit - 4) { -- uint32 x = UNALIGNED_LOAD32(s2) ^ UNALIGNED_LOAD32(s1 + matched); -+ uint32_t x = UNALIGNED_LOAD32(s2) ^ UNALIGNED_LOAD32(s1 + matched); - int matching_bits = Bits::FindLSBSetNonZero(x); - matched += matching_bits >> 3; - } else { ---- src/butil/third_party/snappy/snappy-stubs-internal.h 2018-10-30 12:00:24.000000000 +0800 -+++ src/butil/third_party/snappy/snappy-stubs-internal.h 2020-01-19 14:49:12.990000000 +0800 -@@ -35,6 +35,7 @@ - #include - #include - #include -+#include - #include "butil/compiler_specific.h" - #include "butil/basictypes.h" - #include "butil/sys_byteorder.h" -@@ -114,8 +115,8 @@ - // See if that would be more efficient on platforms supporting it, - // at least for copies. - --inline uint64_tUNALIGNED_LOAD64(const void *p) { -- uint64_tt; -+inline uint64_t UNALIGNED_LOAD64(const void *p) { -+ uint64_t t; - memcpy(&t, p, sizeof t); - return t; - } -@@ -141,8 +142,8 @@ - return t; - } - --inline uint64_tUNALIGNED_LOAD64(const void *p) { -- uint64_tt; -+inline uint64_t UNALIGNED_LOAD64(const void *p) { -+ uint64_t t; - memcpy(&t, p, sizeof t); - return t; - } -@@ -155,7 +156,7 @@ - memcpy(p, &v, sizeof v); - } - --inline void UNALIGNED_STORE64(void *p, uint64_tv) { -+inline void UNALIGNED_STORE64(void *p, uint64_t v) { - memcpy(p, &v, sizeof v); - } - ---- src/CMakeLists.txt 2018-10-30 12:00:24.000000000 +0800 -+++ src/CMakeLists.txt 2020-01-19 15:58:52.000000000 +0800 -@@ -47,11 +47,13 @@ - target_link_libraries(protoc-gen-mcpack brpc-shared) - - #install directory --install(TARGETS brpc-shared -- RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} -- LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} -- ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} -- ) -+if(BUILD_SHARED_LIBS) -+ install(TARGETS brpc-shared -+ RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} -+ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} -+ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} -+ ) -+endif() - - install(TARGETS brpc-static - RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} diff --git a/thirdparty/patches/mysql-5.7.18-boost.patch b/thirdparty/patches/mysql-5.7.18-boost.patch new file mode 100644 index 00000000000000..e3f761332e75da --- /dev/null +++ b/thirdparty/patches/mysql-5.7.18-boost.patch @@ -0,0 +1,20 @@ +--- cmake/boost.cmake ++++ cmake/boost.cmake +@@ -31,7 +31,7 @@ + # we assume that the correct version (see below) + # is installed on the compile host in the standard location. + +-SET(BOOST_PACKAGE_NAME "boost_1_73_0") ++SET(BOOST_PACKAGE_NAME "boost_1_59_0") + SET(BOOST_TARBALL "${BOOST_PACKAGE_NAME}.tar.gz") + SET(BOOST_DOWNLOAD_URL + "http://sourceforge.net/projects/boost/files/boost/1.59.0/${BOOST_TARBALL}" +@@ -262,7 +262,7 @@ IF(NOT BOOST_MAJOR_VERSION EQUAL 10) + COULD_NOT_FIND_BOOST() + ENDIF() + +-IF(NOT BOOST_MINOR_VERSION EQUAL 73) ++IF(NOT BOOST_MINOR_VERSION EQUAL 59) + MESSAGE(WARNING "Boost minor version found is ${BOOST_MINOR_VERSION} " + "we need 59" + ) diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh index 6f8dc123db24b4..6a3b06512a2100 100644 --- a/thirdparty/vars.sh +++ b/thirdparty/vars.sh @@ -170,10 +170,10 @@ RE2_SOURCE=re2-2017-05-01 RE2_MD5SUM="4aa65a0b22edacb7ddcd7e4aec038dcf" # boost -BOOST_DOWNLOAD="https://dl.bintray.com/boostorg/release/1.64.0/source/boost_1_64_0.tar.gz" -BOOST_NAME=boost_1_64_0.tar.gz -BOOST_SOURCE=boost_1_64_0 -BOOST_MD5SUM="319c6ffbbeccc366f14bb68767a6db79" +BOOST_DOWNLOAD="https://dl.bintray.com/boostorg/release/1.73.0/source/boost_1_73_0.tar.gz" +BOOST_NAME=boost_1_73_0.tar.gz +BOOST_SOURCE=boost_1_73_0 +BOOST_MD5SUM="4036cd27ef7548b8d29c30ea10956196" # mysql MYSQL_DOWNLOAD="https://github.com/mysql/mysql-server/archive/mysql-5.7.18.tar.gz" @@ -181,12 +181,6 @@ MYSQL_NAME=mysql-5.7.18.tar.gz MYSQL_SOURCE=mysql-server-mysql-5.7.18 MYSQL_MD5SUM="58598b10dce180e4d1fbdd7cf5fa68d6" -# boost for mysql -BOOST_FOR_MYSQL_DOWNLOAD="http://sourceforge.net/projects/boost/files/boost/1.59.0/boost_1_59_0.tar.gz" -BOOST_FOR_MYSQL_NAME=boost_1_59_0.tar.gz -BOOST_FOR_MYSQL_SOURCE=boost_1_59_0 -BOOST_FOR_MYSQL_MD5SUM="51528a0e3b33d9e10aaa311d9eb451e3" - # unix odbc ODBC_DOWNLOAD="http://www.unixodbc.org/unixODBC-2.3.7.tar.gz" ODBC_NAME=unixODBC-2.3.7.tar.gz @@ -200,10 +194,10 @@ LEVELDB_SOURCE=leveldb-1.20 LEVELDB_MD5SUM="298b5bddf12c675d6345784261302252" # brpc -BRPC_DOWNLOAD="https://github.com/apache/incubator-brpc/archive/0.9.5.tar.gz" -BRPC_NAME=incubator-brpc-0.9.5.tar.gz -BRPC_SOURCE=incubator-brpc-0.9.5 -BRPC_MD5SUM="c9f46e4c97a9cd5f836ba2c6c56978dd" +BRPC_DOWNLOAD="https://github.com/apache/incubator-brpc/archive/0.9.7.tar.gz" +BRPC_NAME=incubator-brpc-0.9.7.tar.gz +BRPC_SOURCE=incubator-brpc-0.9.7 +BRPC_MD5SUM="a5b79339d139d1c55d39689c0a69bcef" # rocksdb ROCKSDB_DOWNLOAD="https://github.com/facebook/rocksdb/archive/v5.14.2.tar.gz" @@ -370,7 +364,6 @@ CURL RE2 BOOST MYSQL -BOOST_FOR_MYSQL ODBC LEVELDB BRPC