[GLUTEN-7139][VL] Manage Velox Arrow through vcpkg

.github/workflows/velox_weekly.yml

@@ -160,7 +160,7 @@ jobs:
           echo "JAVA_HOME: $JAVA_HOME"
           cd $GITHUB_WORKSPACE/
           ./dev/vcpkg/setup-build-depends.sh
-          ./dev/package-vcpkg.sh --build_arrow=ON --spark_version=3.5
+          ./dev/package-vcpkg.sh --spark_version=3.5
           cd $GITHUB_WORKSPACE/tools/gluten-it
           $GITHUB_WORKSPACE/build/mvn -ntp clean install -Pspark-3.5
       - name: Run TPC-H

cpp/CMake/ConfigArrow.cmake

@@ -20,6 +20,14 @@ set(ARROW_STATIC_LIBRARY_SUFFIX ".a")
 set(ARROW_LIB_NAME "arrow")
 set(ARROW_BUNDLED_DEPS "arrow_bundled_dependencies")
 
+if(ENABLE_GLUTEN_VCPKG)
+  find_package(Arrow CONFIG REQUIRED)
+  if(NOT TARGET Arrow::arrow)
+    add_library(Arrow::arrow INTERFACE IMPORTED)
+    target_link_libraries(Arrow::arrow INTERFACE Arrow::arrow_static)
+  endif()
+endif()
+
 set(ARROW_INSTALL_DIR "${ARROW_HOME}/install")
 set(ARROW_LIB_DIR "${ARROW_INSTALL_DIR}/lib")
 set(ARROW_LIB64_DIR "${ARROW_INSTALL_DIR}/lib64")

cpp/CMakeLists.txt

@@ -68,6 +68,7 @@ endif()
 
 set(GLUTEN_PREFIX_INCLUDE_DIRS)
 if(CMAKE_INSTALL_PREFIX
+   AND NOT ENABLE_GLUTEN_VCPKG
    AND NOT CMAKE_INSTALL_PREFIX STREQUAL "/usr/local"
    AND IS_DIRECTORY "${CMAKE_INSTALL_PREFIX}/include")
   list(APPEND GLUTEN_PREFIX_INCLUDE_DIRS "${CMAKE_INSTALL_PREFIX}/include")
@@ -191,6 +192,11 @@ find_package(Threads REQUIRED)
 find_package(JNI REQUIRED)
 
 find_package(glog REQUIRED)
+if(ENABLE_GLUTEN_VCPKG
+   AND TARGET glog::glog
+   AND NOT TARGET google::glog)
+  add_library(google::glog ALIAS glog::glog)
+endif()
 
 if(BUILD_TESTS)
   set(GLUTEN_GTEST_MIN_VERSION "1.13.0")

cpp/core/CMakeLists.txt

@@ -201,8 +201,10 @@ if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
   endif()
 endif()
 
-find_arrow_lib(${ARROW_LIB_NAME})
-find_arrow_lib(${ARROW_BUNDLED_DEPS})
+if(NOT ENABLE_GLUTEN_VCPKG)
+  find_arrow_lib(${ARROW_LIB_NAME})
+  find_arrow_lib(${ARROW_BUNDLED_DEPS})
+endif()
 
 if(ENABLE_QAT)
   include(BuildQATzip)
@@ -251,8 +253,10 @@ if(BUILD_TESTS)
   add_subdirectory(tests)
 endif()
 
-target_link_libraries(gluten PUBLIC Arrow::arrow
-                                    Arrow::arrow_bundled_dependencies)
+target_link_libraries(gluten PUBLIC Arrow::arrow)
+if(NOT ENABLE_GLUTEN_VCPKG)
+  target_link_libraries(gluten PUBLIC Arrow::arrow_bundled_dependencies)
+endif()
 target_link_libraries(gluten PRIVATE google::glog)
 
 install(TARGETS gluten DESTINATION ${CMAKE_INSTALL_LIBDIR})

cpp/velox/CMakeLists.txt

@@ -246,6 +246,11 @@ endif()
 
 add_library(velox SHARED ${VELOX_SRCS})
 
+if(ENABLE_GLUTEN_VCPKG)
+  find_package(xxHash CONFIG REQUIRED)
+  target_link_libraries(velox PUBLIC xxHash::xxhash)
+endif()
+
 # Keep ABI sensitive Velox feature macros in sync
 set(GLUTEN_VELOX_CMAKE_CACHE "${VELOX_BUILD_PATH}/CMakeCache.txt")
 set(GLUTEN_VELOX_ENABLE_PARQUET ${VELOX_ENABLE_PARQUET})
@@ -342,12 +347,28 @@ if(BUILD_TESTS)
   import_library(
     facebook::velox::file_test_utils
     ${VELOX_BUILD_PATH}/velox/common/file/tests/libvelox_file_test_utils.a)
+  import_library(
+    facebook::velox::vector_fuzzer_util
+    ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_vector_fuzzer_util.a)
+  import_library(
+    facebook::velox::constrained_vector_generator
+    ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_constrained_vector_generator.a
+  )
+  target_link_libraries(facebook::velox::constrained_vector_generator
+                        INTERFACE facebook::velox::vector_fuzzer_util)
+  import_library(
+    facebook::velox::vector_fuzzer
+    ${VELOX_BUILD_PATH}/velox/vector/fuzzer/libvelox_vector_fuzzer.a)
+  target_link_libraries(
+    facebook::velox::vector_fuzzer
+    INTERFACE facebook::velox::vector_fuzzer_util
+              facebook::velox::constrained_vector_generator)
   import_library(
     facebook::velox::exec_test_lib
     ${VELOX_BUILD_PATH}/velox/exec/tests/utils/libvelox_exec_test_lib.a)
   target_link_libraries(
     facebook::velox::exec_test_lib
-    INTERFACE facebook::velox::vector_test_lib
+    INTERFACE facebook::velox::vector_test_lib facebook::velox::vector_fuzzer
               facebook::velox::dwio_common_test
               facebook::velox::file_test_utils)
   target_link_libraries(velox PUBLIC facebook::velox::exec_test_lib)
@@ -422,8 +443,10 @@ endif()
 target_link_libraries(velox PUBLIC Folly::folly)
 
 # Link thrift libraries - check vcpkg first, then fall back to system libraries
-# Determine vcpkg triplet directory based on architecture
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
+# Use the active vcpkg triplet when available, then fall back to architecture.
+if(VCPKG_TARGET_TRIPLET)
+  set(VCPKG_TRIPLET_DIR "${VCPKG_TARGET_TRIPLET}")
+elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)|(amd64)|(AMD64)")
   set(VCPKG_TRIPLET_DIR "x64-linux-avx")
 elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)")
   set(VCPKG_TRIPLET_DIR "arm64-linux-neon")
@@ -443,8 +466,8 @@ if(DEFINED VCPKG_INSTALLED_DIR
   )
   target_link_libraries(
     velox
-    PRIVATE ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a
-            ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftprotocol.a)
+    PUBLIC ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a
+           ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftprotocol.a)
 else()
   set(THRIFT_LIBRARY_DIRS)
   if(CMAKE_INSTALL_PREFIX)

cpp/velox/benchmarks/CMakeLists.txt

@@ -20,6 +20,9 @@ target_include_directories(
                                 ${CMAKE_SOURCE_DIR}/core)
 target_link_libraries(velox_benchmark_common PUBLIC velox benchmark::benchmark
                                                     google::glog)
+if(BUILD_TESTS)
+  target_link_libraries(velox_benchmark_common PRIVATE GTest::gtest)
+endif()
 
 function(add_velox_benchmark BM_EXEC BM_FILE)
   add_executable(${BM_EXEC} ${BM_FILE})

cpp/velox/operators/writer/VeloxParquetDataSource.h

@@ -17,7 +17,6 @@
 
 #pragma once
 
-#include <arrow/filesystem/filesystem.h>
 #include <arrow/memory_pool.h>
 #include <arrow/record_batch.h>
 #include <arrow/type.h>

dev/builddep-veloxbe-inc.sh

@@ -139,9 +139,11 @@ else
         export VCPKG_TRIPLET="x64-linux-avx"
     fi
 
+    export VCPKG_MANIFEST_DIR="$GLUTEN_DIR/dev/vcpkg"
     export VCPKG_TRIPLET_INSTALL_DIR="$GLUTEN_DIR/dev/vcpkg/vcpkg_installed/${VCPKG_TRIPLET}"
     export CMAKE_TOOLCHAIN_FILE="$GLUTEN_DIR/dev/vcpkg/toolchain.cmake"
-    export PKG_CONFIG_PATH="${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
+    unset PKG_CONFIG_PATH
+    export PKG_CONFIG_LIBDIR="${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig"
     export GLUTEN_VCPKG_ENABLED="${VCPKG_ROOT}"
 fi
 

dev/builddeps-veloxbe.sh

@@ -45,6 +45,7 @@ VELOX_BRANCH=""
 VELOX_HOME="$GLUTEN_DIR/ep/build-velox/build/velox_ep"
 VELOX_PARAMETER=""
 BUILD_ARROW=ON
+BUILD_ARROW_EXPLICIT=OFF
 SPARK_VERSION=ALL
 
 # set default number of threads as cpu cores minus 2
@@ -141,6 +142,7 @@ do
         ;;
         --build_arrow=*)
         BUILD_ARROW="${arg#*=}"
+        BUILD_ARROW_EXPLICIT=ON
         shift # Remove argument name from processing
         ;;
         --num_threads=*)
@@ -158,6 +160,18 @@ do
     esac
 done
 
+function vcpkg_is_active {
+    [ "$ENABLE_VCPKG" = "ON" ] || [ -n "${GLUTEN_VCPKG_ENABLED:-}" ]
+}
+
+if vcpkg_is_active; then
+    if [ "$BUILD_ARROW_EXPLICIT" = "ON" ] && [ "$BUILD_ARROW" = "ON" ]; then
+        echo "ERROR: --build_arrow=ON is deprecated with --enable_vcpkg=ON; Arrow is managed by Gluten vcpkg." >&2
+        exit 1
+    fi
+    BUILD_ARROW=OFF
+fi
+
 if [[ "$(uname)" == "Darwin" ]]; then
     export INSTALL_PREFIX=${INSTALL_PREFIX:-${VELOX_HOME}/deps-install}
     if [[ "$INSTALL_PREFIX" == "/usr/local" || "$INSTALL_PREFIX" == /usr/local/* ]]; then
@@ -223,6 +237,10 @@ concat_velox_param
 export VELOX_HOME
 
 function build_arrow {
+  if vcpkg_is_active; then
+    echo "ERROR: build_arrow is deprecated with --enable_vcpkg=ON; Arrow is managed by Gluten vcpkg." >&2
+    return 1
+  fi
   local GLUTEN_BUILD_TYPE="$BUILD_TYPE"
   if [ ! -d "$VELOX_HOME" ]; then
     get_velox
@@ -273,8 +291,10 @@ function build_gluten_cpp {
   )
 
   if [ -n "${INSTALL_PREFIX:-}" ]; then
-    GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX")
     GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX")
+    if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ]; then
+      GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX")
+    fi
   fi
   if [ $OS == 'Darwin' ]; then
     if [[ -n "${INSTALL_PREFIX:-}" && "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then

dev/docker/Dockerfile.centos7-gcc13-static-build

@@ -43,10 +43,10 @@ RUN set -ex; \
     yum remove gcc -y && yum clean all; \
     yes | cp -rf /usr/share/aclocal/* /usr/local/share/aclocal/; \
     mkdir -p ${VCPKG_PATH}; \
-    echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \
+    echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \
     cd /opt/gluten; \
-    bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \
-                                       --enable_hdfs=ON --enable_abfs=ON  build_arrow; \
+    bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \
+                                       --enable_hdfs=ON --enable_abfs=ON'; \
     ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \
     dnf clean all; \
     rm -rf /opt/gluten; \

dev/docker/Dockerfile.centos7-static-build

@@ -39,12 +39,12 @@ RUN set -ex; \
     echo "check_certificate = off" >> ~/.wgetrc; \
     cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \
     mkdir -p ${VCPKG_PATH}; \
-    echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \
+    echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \
     cd /opt/gluten; \
     source /opt/rh/devtoolset-11/enable; \
     source /opt/rh/rh-git227/enable; \
-    bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \
-                                       --enable_hdfs=ON --enable_abfs=ON  build_arrow; \
+    bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \
+                                       --enable_hdfs=ON --enable_abfs=ON'; \
     ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \
     dnf clean all; \
     rm -rf /opt/gluten; \

dev/docker/Dockerfile.centos8-gcc13-static-build

@@ -37,14 +37,14 @@ RUN set -ex; \
     dnf clean all; \
     git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \
     mkdir -p ${VCPKG_PATH}; \
-    echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \
+    echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \
     if [ "$(uname -m)" = "aarch64" ]; then \
         export CPU_TARGET="aarch64"; \
         export VCPKG_FORCE_SYSTEM_BINARIES=1; \
     fi; \
     cd /opt/gluten; \
-    bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \
-                                    --enable_hdfs=ON --enable_abfs=ON  build_arrow; \
+    bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \
+                                    --enable_hdfs=ON --enable_abfs=ON'; \
     ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \
     rm -rf /opt/gluten; \
     rm -rf /root/.cache/ccache;

dev/docker/Dockerfile.centos8-static-build

@@ -37,16 +37,16 @@ RUN set -ex; \
     git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \
     cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \
     mkdir -p ${VCPKG_PATH}; \
-    echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \
+    echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \
     if [ "$(uname -m)" = "aarch64" ]; then \
         export CPU_TARGET="aarch64"; \
         export VCPKG_FORCE_SYSTEM_BINARIES=1; \
     fi; \
     source /opt/rh/gcc-toolset-11/enable; \
     cd /opt/gluten; \
     export SSL_VERIFY=false; \
-    bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \
-                                    --enable_hdfs=ON --enable_abfs=ON  build_arrow; \
+    bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \
+                                    --enable_hdfs=ON --enable_abfs=ON'; \
     ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \
     rm -rf /opt/gluten; \
     rm -rf /root/.cache/ccache;

dev/docker/Dockerfile.centos9-static-build

@@ -33,15 +33,15 @@ RUN set -ex; \
     git clone --depth=1 https://github.com/apache/gluten /opt/gluten; \
     cd /opt/gluten && bash ./dev/vcpkg/setup-build-depends.sh; \
     mkdir -p ${VCPKG_PATH}; \
-    echo "Build arrow, then install the native libs to system paths and jar package to .m2/ directory."; \
+    echo "vcpkg installs the native dependencies and Maven resolves Java dependencies."; \
     if [ "$(uname -m)" = "aarch64" ]; then \
         export CPU_TARGET="aarch64"; \
         export VCPKG_FORCE_SYSTEM_BINARIES=1; \
     fi; \
     source /opt/rh/gcc-toolset-12/enable; \
     cd /opt/gluten; \
-    bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \
-                                    --enable_hdfs=ON --enable_abfs=ON  build_arrow; \
+    bash -c 'source ./dev/vcpkg/env.sh --build_tests=ON --enable_s3=ON --enable_gcs=ON \
+                                    --enable_hdfs=ON --enable_abfs=ON'; \
     ./build/mvn dependency:go-offline -Pbackends-velox -Piceberg -Pdelta -Pspark-3.5 -DskipTests; \
     rm -rf /opt/gluten; \
     rm -rf /root/.cache/ccache;

dev/vcpkg/env.sh

@@ -29,6 +29,9 @@ export VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg"
 export VCPKG_TRIPLET=$([ "${CPU_TARGET:-}" = "aarch64" ] && echo "arm64-linux-neon" || echo "x64-linux-avx")
 export VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET}
 
+unset PKG_CONFIG_PATH
+export PKG_CONFIG_LIBDIR=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig
+
 ${SCRIPT_ROOT}/init.sh "$@"
 
 if [ "${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then
@@ -45,9 +48,7 @@ if [ "${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then
     export VCPKG_ROOT=${VCPKG_ROOT}
     export VCPKG_MANIFEST_DIR=${SCRIPT_ROOT}
     export VCPKG_TRIPLET=${VCPKG_TRIPLET}
-
     export CMAKE_TOOLCHAIN_FILE=${SCRIPT_ROOT}/toolchain.cmake
-    export PKG_CONFIG_PATH=${VCPKG_TRIPLET_INSTALL_DIR}/lib/pkgconfig:${VCPKG_TRIPLET_INSTALL_DIR}/share/pkgconfig:${PKG_CONFIG_PATH:-}
     export PATH="${EXPORT_TOOLS_PATH}:$PATH"
 
     export GLUTEN_VCPKG_ENABLED=${VCPKG_ROOT}

dev/vcpkg/ports/arrow/android.patch

@@ -0,0 +1,27 @@
+diff --git a/cpp/src/arrow/CMakeLists.txt b/cpp/src/arrow/CMakeLists.txt
+index 6dc8358..2b91efa 100644
+--- a/cpp/src/arrow/CMakeLists.txt
++++ b/cpp/src/arrow/CMakeLists.txt
+@@ -166,7 +166,7 @@ if(WIN32)
+   list(APPEND ARROW_SYSTEM_LINK_LIBS "ws2_32")
+ endif()
+
+-if(NOT WIN32 AND NOT APPLE)
++if(NOT WIN32 AND NOT APPLE AND NOT ANDROID)
+   # Pass -lrt on Linux only
+   list(APPEND ARROW_SYSTEM_LINK_LIBS rt)
+ endif()
+diff --git a/cpp/src/arrow/vendored/musl/strptime.c b/cpp/src/arrow/vendored/musl/strptime.c
+index 41912fd..0ea36e9 100644
+--- a/cpp/src/arrow/vendored/musl/strptime.c
++++ b/cpp/src/arrow/vendored/musl/strptime.c
+@@ -18,7 +18,9 @@
+ #undef HAVE_LANGINFO
+
+ #ifndef _WIN32
++# if !(defined(__ANDROID__) && __ANDROID_API__ < 26)
+ #define HAVE_LANGINFO 1
++# endif
+ #endif
+
+ #ifdef HAVE_LANGINFO