Rebase ray-project#3

cpp/CMakeLists.txt

@@ -768,12 +768,14 @@ endif()
 set(ARROW_SHARED_PRIVATE_LINK_LIBS
   ${ARROW_STATIC_LINK_LIBS}
   ${BOOST_SYSTEM_LIBRARY}
+  ${BOOST_THREAD_LIBRARY}
   ${BOOST_FILESYSTEM_LIBRARY}
   ${BOOST_REGEX_LIBRARY})
 
 list(APPEND
   ARROW_STATIC_LINK_LIBS
   ${BOOST_SYSTEM_LIBRARY}
+  ${BOOST_THREAD_LIBRARY}
   ${BOOST_FILESYSTEM_LIBRARY}
   ${BOOST_REGEX_LIBRARY})
 
@@ -792,6 +794,11 @@ if (NOT MSVC)
     ${CMAKE_DL_LIBS})
 endif()
 
+if (UNIX AND NOT APPLE)
+  set(ARROW_LINK_LIBS ${ARROW_LINK_LIBS} -lrt)
+  set(ARROW_STATIC_LINK_LIBS ${ARROW_STATIC_LINK_LIBS} -lrt)
+endif()
+
 set(ARROW_TEST_STATIC_LINK_LIBS
   arrow_testing_static
   arrow_static
@@ -805,6 +812,7 @@ set(ARROW_TEST_SHARED_LINK_LIBS
   ${ARROW_LINK_LIBS}
   double-conversion_static
   ${BOOST_SYSTEM_LIBRARY}
+  ${BOOST_THREAD_LIBRARY}
   ${BOOST_FILESYSTEM_LIBRARY}
   ${BOOST_REGEX_LIBRARY}
   gtest_main_static

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -369,11 +369,14 @@ if (ARROW_BOOST_VENDORED)
   set(BOOST_BUILD_LINK "static")
   set(BOOST_STATIC_SYSTEM_LIBRARY
     "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_system${CMAKE_STATIC_LIBRARY_SUFFIX}")
+  set(BOOST_STATIC_THREAD_LIBRARY
+      "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_thread${CMAKE_STATIC_LIBRARY_SUFFIX}")
   set(BOOST_STATIC_FILESYSTEM_LIBRARY
     "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_filesystem${CMAKE_STATIC_LIBRARY_SUFFIX}")
   set(BOOST_STATIC_REGEX_LIBRARY
     "${BOOST_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}boost_regex${CMAKE_STATIC_LIBRARY_SUFFIX}")
   set(BOOST_SYSTEM_LIBRARY boost_system_static)
+  set(BOOST_THREAD_LIBRARY boost_thread_static)
   set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
   set(BOOST_REGEX_LIBRARY boost_regex_static)
 
@@ -384,12 +387,13 @@ if (ARROW_BOOST_VENDORED)
   else()
     set(BOOST_BUILD_PRODUCTS
       ${BOOST_STATIC_SYSTEM_LIBRARY}
+      ${BOOST_STATIC_THREAD_LIBRARY}
       ${BOOST_STATIC_FILESYSTEM_LIBRARY}
       ${BOOST_STATIC_REGEX_LIBRARY})
     set(BOOST_CONFIGURE_COMMAND
       "./bootstrap.sh"
       "--prefix=${BOOST_PREFIX}"
-      "--with-libraries=filesystem,regex,system")
+      "--with-libraries=filesystem,regex,system,thread")
     if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
       set(BOOST_BUILD_VARIANT "debug")
     else()
@@ -436,17 +440,20 @@ else()
     if (ARROW_BOOST_HEADER_ONLY)
       find_package(Boost REQUIRED)
     else()
-      find_package(Boost COMPONENTS regex system filesystem REQUIRED)
+      find_package(Boost COMPONENTS regex system thread filesystem REQUIRED)
       if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
         set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+        set(BOOST_SHARED_THREAD_LIBRARY ${Boost_THREAD_LIBRARY_DEBUG})
         set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
         set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
       else()
         set(BOOST_SHARED_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+        set(BOOST_SHARED_THREAD_LIBRARY ${Boost_THREAD_LIBRARY_RELEASE})
         set(BOOST_SHARED_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
         set(BOOST_SHARED_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
       endif()
       set(BOOST_SYSTEM_LIBRARY boost_system_shared)
+      set(BOOST_THREAD_LIBRARY boost_thread_shared)
       set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_shared)
       set(BOOST_REGEX_LIBRARY boost_regex_shared)
     endif()
@@ -457,17 +464,20 @@ else()
     if (ARROW_BOOST_HEADER_ONLY)
       find_package(Boost REQUIRED)
     else()
-      find_package(Boost COMPONENTS regex system filesystem REQUIRED)
+      find_package(Boost COMPONENTS regex system thread atomic chrono date_time filesystem REQUIRED)
       if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
         set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_DEBUG})
+        set(BOOST_STATIC_THREAD_LIBRARY ${Boost_THREAD_LIBRARY_DEBUG})
         set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_DEBUG})
         set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_DEBUG})
       else()
         set(BOOST_STATIC_SYSTEM_LIBRARY ${Boost_SYSTEM_LIBRARY_RELEASE})
+        set(BOOST_STATIC_THREAD_LIBRARY ${Boost_THREAD_LIBRARY_RELEASE})
         set(BOOST_STATIC_FILESYSTEM_LIBRARY ${Boost_FILESYSTEM_LIBRARY_RELEASE})
         set(BOOST_STATIC_REGEX_LIBRARY ${Boost_REGEX_LIBRARY_RELEASE})
       endif()
       set(BOOST_SYSTEM_LIBRARY boost_system_static)
+      set(BOOST_THREAD_LIBRARY boost_thread_static)
       set(BOOST_FILESYSTEM_LIBRARY boost_filesystem_static)
       set(BOOST_REGEX_LIBRARY boost_regex_static)
     endif()
@@ -482,6 +492,10 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
       STATIC_LIB "${BOOST_STATIC_SYSTEM_LIBRARY}"
       SHARED_LIB "${BOOST_SHARED_SYSTEM_LIBRARY}")
 
+  ADD_THIRDPARTY_LIB(boost_thread
+          STATIC_LIB "${BOOST_STATIC_THREAD_LIBRARY}"
+          SHARED_LIB "${BOOST_SHARED_THREAD_LIBRARY}")
+
   ADD_THIRDPARTY_LIB(boost_filesystem
       STATIC_LIB "${BOOST_STATIC_FILESYSTEM_LIBRARY}"
       SHARED_LIB "${BOOST_SHARED_FILESYSTEM_LIBRARY}")
@@ -490,7 +504,7 @@ if (NOT ARROW_BOOST_HEADER_ONLY)
       STATIC_LIB "${BOOST_STATIC_REGEX_LIBRARY}"
       SHARED_LIB "${BOOST_SHARED_REGEX_LIBRARY}")
 
-  SET(ARROW_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY})
+  SET(ARROW_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_THREAD_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY})
 endif()
 
 include_directories(SYSTEM ${Boost_INCLUDE_DIR})

cpp/src/arrow/flight/flight-benchmark.cc

@@ -146,7 +146,7 @@ Status RunPerformanceTest(const int port) {
 
   std::shared_ptr<ThreadPool> pool;
   RETURN_NOT_OK(ThreadPool::Make(FLAGS_num_threads, &pool));
-  std::vector<std::future<Status>> tasks;
+  std::vector<Future<Status>> tasks;
   for (const auto& endpoint : plan->endpoints()) {
     tasks.emplace_back(pool->Submit(ConsumeStream, endpoint));
   }

cpp/src/arrow/python/CMakeLists.txt

@@ -55,6 +55,8 @@ endif()
 set(ARROW_PYTHON_SHARED_LINK_LIBS
   arrow_shared
   ${PYTHON_OTHER_LIBS}
+  ${BOOST_THREAD_LIBRARY}
+  ${Boost_SYSTEM_LIBRARY}
 )
 
 if (WIN32)

cpp/src/arrow/util/memory.h

@@ -60,7 +60,7 @@ void parallel_memcopy(uint8_t* dst, const uint8_t* src, int64_t nbytes,
   // Each thread gets a "chunk" of k blocks.
 
   // Start all parallel memcpy tasks and handle leftovers while threads run.
-  std::vector<std::future<void*>> futures;
+  std::vector<Future<void*>> futures;
 
   for (int i = 0; i < num_threads; i++) {
     futures.emplace_back(pool->Submit(wrap_memcpy, dst + prefix + i * chunk_size,

cpp/src/arrow/util/parallel.h

@@ -35,8 +35,7 @@ namespace internal {
 template <class FUNCTION>
 Status ParallelFor(int num_tasks, FUNCTION&& func) {
   auto pool = internal::GetCpuThreadPool();
-  std::vector<std::future<Status>> futures(num_tasks);
-
+  std::vector<Future<Status>> futures(num_tasks);
   for (int i = 0; i < num_tasks; ++i) {
     futures[i] = pool->Submit(func, i);
   }

cpp/src/arrow/util/thread-pool.h

@@ -33,12 +33,19 @@
 #include <type_traits>
 #include <utility>
 
+#define BOOST_THREAD_PROVIDES_FUTURE
+#include <boost/thread.hpp>
+#include <boost/thread/future.hpp>
+
 #include "arrow/status.h"
 #include "arrow/util/macros.h"
 #include "arrow/util/visibility.h"
 
 namespace arrow {
 
+template<class R>
+using Future = boost::future<R>;
+
 /// \brief Get the capacity of the global thread pool
 ///
 /// Return the number of worker threads in the thread pool to which
@@ -64,12 +71,10 @@ namespace detail {
 // to std::function.
 template <typename R, typename... Args>
 struct packaged_task_wrapper {
-  using PackagedTask = std::packaged_task<R(Args...)>;
-
-  explicit packaged_task_wrapper(PackagedTask&& task)
+  using PackagedTask = boost::packaged_task<R>;
+   explicit packaged_task_wrapper(PackagedTask&& task)
       : task_(std::make_shared<PackagedTask>(std::forward<PackagedTask>(task))) {}
-
-  void operator()(Args&&... args) { return (*task_)(std::forward<Args>(args)...); }
+   void operator()(Args&&... args) { return (*task_)(std::forward<Args>(args)...); }
   std::shared_ptr<PackagedTask> task_;
 };
 
@@ -118,10 +123,10 @@ class ARROW_EXPORT ThreadPool {
   // only occurs if the ThreadPool is shutting down).
   template <typename Function, typename... Args,
             typename Result = typename std::result_of<Function && (Args && ...)>::type>
-  std::future<Result> Submit(Function&& func, Args&&... args) {
+  Future<Result> Submit(Function&& func, Args&&... args) {
     // Trying to templatize std::packaged_task with Function doesn't seem
     // to work, so go through std::bind to simplify the packaged signature
-    using PackagedTask = std::packaged_task<Result()>;
+    using PackagedTask = boost::packaged_task<Result>;
     auto task = PackagedTask(std::bind(std::forward<Function>(func), args...));
     auto fut = task.get_future();
 

cpp/src/arrow/util/utf8.cc

@@ -17,6 +17,9 @@
 
 #include <mutex>
 
+#include <boost/thread.hpp>
+#include <boost/thread/future.hpp>
+
 #include "arrow/util/logging.h"
 #include "arrow/util/utf8.h"
 
@@ -73,10 +76,10 @@ ARROW_EXPORT void CheckUTF8Initialized() {
 
 }  // namespace internal
 
-static std::once_flag utf8_initialized;
+static boost::once_flag utf8_initialized = BOOST_ONCE_INIT;
 
 void InitializeUTF8() {
-  std::call_once(utf8_initialized, internal::InitializeLargeTable);
+  boost::call_once(internal::InitializeLargeTable, utf8_initialized);
 }
 
 }  // namespace util

cpp/src/parquet/arrow/reader.cc

@@ -53,6 +53,7 @@ using arrow::BooleanArray;
 using arrow::ChunkedArray;
 using arrow::Column;
 using arrow::Field;
+using arrow::Future;
 using arrow::Int32Array;
 using arrow::ListArray;
 using arrow::MemoryPool;
@@ -504,7 +505,7 @@ Status FileReader::Impl::ReadRowGroup(int row_group_index,
   };
 
   if (use_threads_) {
-    std::vector<std::future<Status>> futures;
+    std::vector<Future<Status>> futures;
     auto pool = ::arrow::internal::GetCpuThreadPool();
     for (int i = 0; i < num_columns; i++) {
       futures.push_back(pool->Submit(ReadColumnFunc, i));
@@ -551,7 +552,7 @@ Status FileReader::Impl::ReadTable(const std::vector<int>& indices,
   };
 
   if (use_threads_) {
-    std::vector<std::future<Status>> futures;
+    std::vector<Future<Status>> futures;
     auto pool = ::arrow::internal::GetCpuThreadPool();
     for (int i = 0; i < num_fields; i++) {
       futures.push_back(pool->Submit(ReadColumnFunc, i));

cpp/src/plasma/CMakeLists.txt

@@ -79,8 +79,8 @@ set(PLASMA_SRCS
   protocol.cc
   thirdparty/ae/ae.c)
 
-set(PLASMA_LINK_LIBS arrow_shared)
-set(PLASMA_STATIC_LINK_LIBS arrow_static)
+set(PLASMA_LINK_LIBS ${ARROW_LINK_LIBS} ${BOOST_THREAD_LIBRARY} arrow_shared)
+set(PLASMA_STATIC_LINK_LIBS ${ARROW_STATIC_LINK_LIBS} ${BOOST_THREAD_LIBRARY} arrow_static)
 
 if (ARROW_CUDA)
   set(PLASMA_LINK_LIBS ${PLASMA_LINK_LIBS} arrow_cuda_shared)

cpp/src/plasma/client.cc

@@ -662,7 +662,7 @@ bool PlasmaClient::Impl::ComputeObjectHashParallel(XXH64_state_t* hash_state,
   // | num_threads * chunk_size | suffix |, where chunk_size = k * block_size.
   // Each thread gets a "chunk" of k blocks, except the suffix thread.
 
-  std::vector<std::future<void>> futures;
+  std::vector<arrow::Future<void>> futures;
   for (int i = 0; i < num_threads; i++) {
     futures.push_back(pool->Submit(
         ComputeBlockHash, reinterpret_cast<uint8_t*>(data_address) + i * chunk_size,

python/CMakeLists.txt

@@ -202,9 +202,12 @@ find_package(PythonLibsNew REQUIRED)
 find_package(NumPy REQUIRED)
 include(UseCython)
 
+find_package(Boost COMPONENTS system thread atomic chrono date_time filesystem regex REQUIRED)
+
 include_directories(SYSTEM
   ${NUMPY_INCLUDE_DIRS}
   ${PYTHON_INCLUDE_DIRS}
+  ${Boost_INCLUDE_DIR}
   src)
 
 ############################################################
@@ -336,10 +339,14 @@ if (PYARROW_BUNDLE_ARROW_CPP)
       # disable autolinking in boost
       add_definitions(-DBOOST_ALL_NO_LIB)
     endif()
-    find_package(Boost COMPONENTS system filesystem regex REQUIRED)
+    find_package(Boost COMPONENTS system thread atomic chrono date_time filesystem regex REQUIRED)
     bundle_boost_lib(Boost_REGEX_LIBRARY)
     bundle_boost_lib(Boost_FILESYSTEM_LIBRARY)
     bundle_boost_lib(Boost_SYSTEM_LIBRARY)
+    bundle_boost_lib(Boost_THREAD_LIBRARY)
+    bundle_boost_lib(Boost_ATOMIC_LIBRARY)
+    bundle_boost_lib(Boost_CHRONO_LIBRARY)
+    bundle_boost_lib(Boost_DATE_TIME_LIBRARY)
   endif()
 
   bundle_zlib()

python/manylinux1/scripts/build_boost.sh

@@ -25,13 +25,13 @@ mkdir /arrow_boost
 pushd /boost_${BOOST_VERSION_UNDERSCORE}
 ./bootstrap.sh
 ./b2 tools/bcp
-./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time system regex build algorithm locale format variant /arrow_boost
+./dist/bin/bcp --namespace=arrow_boost --namespace-alias filesystem date_time system thread atomic chrono regex build algorithm locale format variant /arrow_boost
 popd
 
 pushd /arrow_boost
 ls -l
 ./bootstrap.sh
-./bjam dll-path="'\$ORIGIN/'" cxxflags='-std=c++11 -fPIC' cflags=-fPIC linkflags="-std=c++11" variant=release link=shared --prefix=/arrow_boost_dist --with-filesystem --with-date_time --with-system --with-regex install
+./bjam dll-path="'\$ORIGIN/'" cxxflags='-std=c++11 -fPIC' cflags=-fPIC linkflags="-std=c++11" variant=release link=shared --prefix=/arrow_boost_dist --with-filesystem --with-date_time --with-system --with-thread --with-atomic --with-chrono --with-regex install
 popd
 rm -rf boost_${BOOST_VERSION_UNDERSCORE}.tar.gz boost_${BOOST_VERSION_UNDERSCORE} arrow_boost
 # Boost always install header-only parts but they also take up quite some space.

python/pyarrow/__init__.py

@@ -44,13 +44,6 @@ def parse_git(root, **kwargs):
 
 import pyarrow.compat as compat
 
-# Workaround for https://issues.apache.org/jira/browse/ARROW-2657
-# and https://issues.apache.org/jira/browse/ARROW-2920
-if _sys.platform in ('linux', 'linux2'):
-    compat.import_tensorflow_extension()
-    compat.import_pytorch_extension()
-
-
 from pyarrow.lib import cpu_count, set_cpu_count
 from pyarrow.lib import (null, bool_,
                          int8, int16, int32, int64,