Build script flag for enabling Arrow/Parquet (#345)

- build.sh has an optional flag --arrow to turn on Arrow/Parquet support. - If this flag is given: - Arrow will be downloaded/built/installed automatically by build.sh. - Readers/writers for Arrow/Parquet will be available in daphne. - In the source code, a macro constant USE_ARROW will be defined. - Arrow test cases with run (guarded by #ifdef USE_ARROW). - libboost-dev must be present as an external dependency. Co-authored-by: Eric Mier <[email protected]> Co-authored-by: Patrick Damme <[email protected]>
daphne-eu · Jul 19, 2022 · 52f49d8 · 52f49d8
1 parent d0ca13f
commit 52f49d8
Show file tree

Hide file tree

Showing 3 changed files with 40 additions and 5 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -127,9 +127,12 @@ endif()
 
 option(USE_ARROW "Whether to activate compilation of Arrow/Parquet features" OFF)
 if(USE_ARROW)
-    find_package(Arrow CONFIG REQUIRED)
+    find_package(Arrow CONFIG REQUIRED
+        PATHS thirdparty/installed/lib/cmake/arrow
+        NO_DEFAULT_PATH
+    )
     find_package(Parquet CONFIG REQUIRED
-        PATHS /usr/local/lib/cmake/arrow
+        PATHS thirdparty/installed/lib/cmake/arrow
         NO_DEFAULT_PATH
     )
     link_libraries(arrow_shared parquet_shared)

diff --git a/build.sh b/build.sh
@@ -41,6 +41,7 @@ function printHelp {
     echo "  --cleanAll        Remove all thirdparty library directories for a build from scratch"
     echo "  -nf, --no-fancy   Suppress all colored and animated output"
     echo "  -y, --yes         Accept all prompts"
+    echo "  --arrow           Compile with support for Arrow/Parquet files"
 }
 
 #******************************************************************************
@@ -282,6 +283,7 @@ function cleanBuildDirs() {
       "${thirdpartyPath}/nlohmannjson_v"*".install.success" \
       "${thirdpartyPath}/openBlas_v"*".install.success" \
       "${thirdpartyPath}/llvm_v"*".install.success" \
+      "${thirdpartyPath}/arrow_v"*".install.success" \
       "${llvmCommitFilePath}")
 
     clean dirs files
@@ -304,6 +306,8 @@ function cleanAll() {
       "${thirdpartyPath}/openBlas_v"*".install.success" \
       "${thirdpartyPath}/openBlas_v"*".download.success" \
       "${thirdpartyPath}/llvm_v"*".install.success" \
+      "${thirdpartyPath}/arrow_v"*".install.success" \
+      "${thirdpartyPath}/arrow_v"*".download.success" \
       "${llvmCommitFilePath}")
 
     clean dirs files
@@ -342,6 +346,7 @@ openBlasVersion=0.3.19
 abslVersion=20211102.0
 grpcVersion=1.38.0
 nlohmannjsonVersion=3.10.5
+arrowVersion=d9d78946607f36e25e9d812a5cc956bd00ab2bc9
 
 #******************************************************************************
 # Set some prefixes, paths and dirs
@@ -375,6 +380,7 @@ par_clean="0"
 par_acceptAll="0"
 unknown_options=""
 BUILD_CUDA="-DUSE_CUDA=OFF"
+BUILD_ARROW="-DUSE_ARROW=OFF"
 BUILD_DEBUG="-DCMAKE_BUILD_TYPE=Release"
 
 while [[ $# -gt 0 ]]; do
@@ -404,6 +410,10 @@ while [[ $# -gt 0 ]]; do
             echo using CUDA
             export BUILD_CUDA="-DUSE_CUDA=ON"
             ;;
+        --arrow)
+            echo using ARROW
+            BUILD_ARROW="-DUSE_ARROW=ON"
+            ;;
         --debug)
             echo building DEBUG version
             export BUILD_DEBUG="-DCMAKE_BUILD_TYPE=Debug"
@@ -630,6 +640,28 @@ else
     daphne_msg "No need to build GRPC again."
 fi
 
+#------------------------------------------------------------------------------
+# Arrow / Parquet
+#------------------------------------------------------------------------------
+arrowDirName="arrow"
+if [[ "$BUILD_ARROW" == "-DUSE_ARROW=ON" ]]; then
+    if ! is_dependency_downloaded "arrow_v${arrowVersion}"; then
+        rm -rf ${sourcePrefix}/${arrowDirName}
+        git clone -n https://github.com/apache/arrow.git ${sourcePrefix}/${arrowDirName}
+        cd ${sourcePrefix}/${arrowDirName}
+        git checkout $arrowVersion
+        dependency_download_success "arrow_v${arrowVersion}"
+    fi
+    if ! is_dependency_installed "arrow_v${arrowVersion}"; then
+        cmake -G Ninja -S "${sourcePrefix}/${arrowDirName}/cpp" -B "${buildPrefix}/${arrowDirName}" \
+            -DCMAKE_INSTALL_PREFIX=${installPrefix} \
+            -DARROW_CSV=ON -DARROW_FILESYSTEM=ON -DARROW_PARQUET=ON
+        cmake --build "${buildPrefix}/${arrowDirName}" --target install
+        dependency_install_success "arrow_v${arrowVersion}"
+    else
+        daphne_msg "No need to build Arrow again."
+    fi
+fi
 
 #------------------------------------------------------------------------------
 # Build MLIR
@@ -676,7 +708,7 @@ fi
 
 daphne_msg "Build Daphne"
 
-cmake -S "$projectRoot" -B "$daphneBuildDir" -G Ninja $BUILD_CUDA $BUILD_DEBUG \
+cmake -S "$projectRoot" -B "$daphneBuildDir" -G Ninja $BUILD_CUDA $BUILD_ARROW $BUILD_DEBUG \
   -DCMAKE_PREFIX_PATH="$installPrefix" -DANTLR_VERSION="$antlrVersion"  \
   -DMLIR_DIR="$buildPrefix/$llvmName/lib/cmake/mlir/" \
   -DLLVM_DIR="$buildPrefix/$llvmName/lib/cmake/llvm/"
@@ -686,4 +718,4 @@ cmake --build "$daphneBuildDir" --target "$target"
 build_ts_end=$(date +%s%N)
 daphne_msg "Successfully built Daphne://${target} (took $(printableTimestamp $((build_ts_end - build_ts_begin))))"
 
-set +e
+set +e
diff --git a/doc/GettingStarted.md b/doc/GettingStarted.md
@@ -48,6 +48,7 @@ Newer versions should work as well, older versions might work as well.
 | java (e.g. openjdk) | 11 (1.7 should be fine) |                                                                                                                                              |
 | gfortran            | 9.3.0                   |                                                                                                                                              |
 | uuid-dev            |                         |                                                                                                                                              |
+| libboost-dev        | 1.71.0.0 | Only required when building with support for Arrow (`--arrow`) |
 
 ##### Hardware
 
@@ -96,7 +97,6 @@ If the build fails in between (e.g., due to missing packages), multiple build di
 
 See [this page](/doc/development/BuildingDaphne) for more information.
 
-
 ### Running the Tests
 
 ```bash