Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 45 additions & 81 deletions .github/workflows/velox_weekly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -35,40 +35,8 @@ concurrency:
cancel-in-progress: true

jobs:
build-on-centos-7:
strategy:
fail-fast: false
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
container: centos:7
steps:
- name: Build Gluten native libraries and package
run: |
sed -i -e 's|mirrorlist=|#mirrorlist=|g' /etc/yum.repos.d/CentOS-* || true
sed -i -e 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true
yum update -y
yum install -y epel-release sudo dnf
yum install -y centos-release-scl
rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
sed -i \
-e 's/^mirrorlist/#mirrorlist/' \
-e 's/^#baseurl/baseurl/' \
-e 's/mirror\.centos\.org/vault.centos.org/' \
/etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
yum install -y devtoolset-11
source /opt/rh/devtoolset-11/enable
yum install -y java-1.8.0-openjdk-devel patch wget git perl python3-pip
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
export PATH=$JAVA_HOME/bin:$PATH

# action/checkout does not work centos7 anymore, so we clone the branch instead.
git clone -b main --depth=1 https://github.com/apache/gluten.git && cd gluten/
if [ ${{ github.event_name }} = "pull_request" ]; then
git fetch origin ${{ github.ref }}:pr_branch && git checkout pr_branch
fi
./dev/package.sh --spark_version=3.5

build-on-centos:
weekly-build-on-centos:
strategy:
fail-fast: false
matrix:
Expand All @@ -79,42 +47,33 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Update mirror list
run: |
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
- name: build
run: |
yum update -y
yum install -y epel-release sudo dnf
if [ "${{ matrix.os }}" = "centos:7" ]; then
yum install -y centos-release-scl
rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
sed -i \
-e 's/^mirrorlist/#mirrorlist/' \
-e 's/^#baseurl/baseurl/' \
-e 's/mirror\.centos\.org/vault.centos.org/' \
/etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
yum install -y devtoolset-11
source /opt/rh/devtoolset-11/enable
elif [ "${{ matrix.os }}" = "quay.io/centos/centos:stream9" ]; then
dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
source /opt/rh/gcc-toolset-12/enable || exit 1
else
dnf install -y --setopt=install_weak_deps=False gcc-toolset-11
source /opt/rh/gcc-toolset-11/enable || exit 1
if [ "${{ matrix.os }}" = "centos:8" ]; then
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
dnf update -y
dnf install -y centos-release-stream
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-* || true
sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Stream-* || true
dnf swap -y --allowerasing centos-{linux,stream}-repos
sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-Stream-* || true
sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-Stream-* || true
fi
dnf install -y epel-release sudo dnf
dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
source /opt/rh/gcc-toolset-12/enable || exit 1
yum install -y java-1.8.0-openjdk-devel patch wget git perl
export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
export PATH=$JAVA_HOME/bin:$PATH

export VELOX_BUILD_SHARED=ON
cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5

build-on-ubuntu:
weekly-build-on-ubuntu:
strategy:
fail-fast: false
matrix:
os: [ "ubuntu:22.04" ] # skip "ubuntu:20.04" due to not enough disk space
os: [ "ubuntu:22.04" ]
if: ${{ startsWith(github.repository, 'apache/') }}
runs-on: ubuntu-22.04
container: ${{ matrix.os }}
Expand All @@ -125,30 +84,35 @@ jobs:
# sed -i 's|http://archive|http://us.archive|g' /etc/apt/sources.list
# To avoid the prompt for region selection during installing tzdata.
export DEBIAN_FRONTEND=noninteractive
apt-get update && apt-get install -y sudo maven wget git curl
if [ "${{ matrix.os }}" = "ubuntu:20.04" ]; then
sudo apt install -y software-properties-common
sudo add-apt-repository ppa:ubuntu-toolchain-r/test
sudo apt update && sudo apt install -y gcc-11 g++-11 build-essential cmake
export CC=/usr/bin/gcc-11
export CXX=/usr/bin/g++-11
wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz
tar xf v0.8.1.tar.gz
cd xxHash-0.8.1/
make && make install
cd /tmp
wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz
tar xf v1.13.0.tar.gz && cd googletest-1.13.0/
export CXXFLAGS=-fPIC
cmake . && make install -j
apt autoremove -y
rm -rf /tmp/*
fi
sudo apt-get install -y openjdk-8-jdk python3-pip cmake
export INSTALL_PREFIX=/usr/
apt-get update && apt-get install -y sudo maven wget git curl gcc-11 g++-11 build-essential cmake openjdk-8-jdk python3-pip
pip3 install cmake==3.31.4
cd /tmp
wget https://github.com/gflags/gflags/archive/v2.2.2.tar.gz -O gflags.tar.gz
tar -xzf gflags.tar.gz && cd gflags-2.2.2
cmake -B build \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DBUILD_SHARED_LIBS=ON \
-DBUILD_STATIC_LIBS=ON \
-DBUILD_gflags_LIB=ON \
-DCMAKE_INSTALL_PREFIX=/usr/local \
-DCMAKE_BUILD_TYPE=Release
cmake --build build -j4 --target install
wget https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.1.tar.gz
tar xf v0.8.1.tar.gz
cd xxHash-0.8.1/
make && make install
cd /tmp
wget https://github.com/google/googletest/archive/refs/tags/v1.13.0.tar.gz
tar xf v1.13.0.tar.gz && cd googletest-1.13.0/
export CXXFLAGS=-fPIC
cmake . && make install -j
apt autoremove -y
export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
export VELOX_BUILD_SHARED=ON
cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5

build-on-openeuler:
weekly-build-on-openeuler:
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -177,7 +141,7 @@ jobs:
$TPCH_TEST --extra-conf=spark.gluten.loadLibFromJar=true \
--extra-conf=spark.gluten.loadLibOS=openEuler --extra-conf=spark.gluten.loadLibOSVersion=24.03

build-on-openeuler-vcpkg:
weekly-build-on-openeuler-vcpkg:
strategy:
fail-fast: false
matrix:
Expand Down Expand Up @@ -205,7 +169,7 @@ jobs:
cd $GITHUB_WORKSPACE/tools/gluten-it
$TPCH_TEST

build-on-macos:
weekly-build-on-macos:
strategy:
fail-fast: false
if: ${{ startsWith(github.repository, 'apache/') }}
Expand Down
34 changes: 23 additions & 11 deletions cpp/CMake/ConfigArrow.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -44,20 +44,32 @@ function(FIND_ARROW_LIB LIB_NAME)
endif()
message(STATUS "Found Arrow library: ${ARROW_LIB_${LIB_NAME}}")

# Get the parent-parent directory of the lib file. For example:
#
# * ${ARROW_LIB_${LIB_NAME}}: /usr/local/lib/libarrow.a
# * ${ARROW_LIB_INCLUDE_DIR}: /usr/local
#
# Then we can get our include directory: /usr/local/include
get_filename_component(ARROW_LIB_INCLUDE_DIR "${ARROW_LIB_${LIB_NAME}}"
PATH)
get_filename_component(ARROW_LIB_INCLUDE_DIR "${ARROW_LIB_INCLUDE_DIR}"
PATH)
# Walk up from the library directory until we find a parent that contains an
# "include/arrow/api.h" anchor file. This handles both conventional layouts
# (/usr/local/lib/libarrow.a → /usr/local/include) and multi-arch ones
# (/usr/lib/x86_64-linux-gnu/libarrow.a → /usr/include) without hard-coding
# the number of directory levels to strip.
get_filename_component(_arrow_search_dir "${ARROW_LIB_${LIB_NAME}}" PATH)
set(ARROW_LIB_INCLUDE_DIR "")
foreach(_level RANGE 5)
if(EXISTS "${_arrow_search_dir}/include/arrow/api.h")
set(ARROW_LIB_INCLUDE_DIR "${_arrow_search_dir}/include")
break()
endif()
get_filename_component(_arrow_search_dir "${_arrow_search_dir}" PATH)
endforeach()
if(NOT ARROW_LIB_INCLUDE_DIR)
message(
FATAL_ERROR
"Could not locate Arrow headers near ${ARROW_LIB_${LIB_NAME}}. "
"Set ARROW_HOME to the Arrow installation prefix (the directory "
"that contains lib/ and include/).")
endif()
message(STATUS "Found Arrow include: ${ARROW_LIB_INCLUDE_DIR}")

set_target_properties(
Arrow::${LIB_NAME}
PROPERTIES IMPORTED_LOCATION ${ARROW_LIB_${LIB_NAME}}
INTERFACE_INCLUDE_DIRECTORIES ${ARROW_LIB_INCLUDE_DIR}/include)
INTERFACE_INCLUDE_DIRECTORIES ${ARROW_LIB_INCLUDE_DIR})
endif()
endfunction()
21 changes: 21 additions & 0 deletions cpp/core/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,27 @@ add_dependencies(gluten jni_proto)
if(NOT CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_options(
gluten PRIVATE -Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/symbols.map)
# libgluten statically links libstdc++/libgcc (see dev/vcpkg/toolchain.cmake's
# -static-libstdc++ -static-libgcc). The version script above has no "local:
# *" catch-all (it must keep Arrow/substrait/etc. exported for libvelox to
# resolve against), so by default the bundled gcc-13 C++ runtime symbols also
# stay in libgluten's dynamic symbol table -- and the std::locale::id objects
# that index the facet table are emitted as STB_GNU_UNIQUE. When another
# libstdc++ is already loaded in the process (the system gcc-11 libstdc++.so.6
# pulled in by the JVM inside the centos-9 image), the loader collapses those
# unique ids into one process-wide definition; libgluten then indexes its
# gcc-13 facet table with the system id value, reads a bogus num_put/codecvt
# facet pointer, and crashes with SIGSEGV during Velox backend initialization
# (do_unshift, si_addr=0x30). -Bsymbolic does not help because STB_GNU_UNIQUE
# overrides it. --exclude-libs demotes the symbols coming from the named C++
# runtime archives to STB_LOCAL, dropping them from the dynamic symbol table
# entirely (this also clears their STB_GNU_UNIQUE binding) so they can neither
# interpose nor be interposed -- the same isolation libvelox already gets from
# its "local: *" map. Symbols from the other static deps (Arrow, substrait,
# ...) come from different archives and stay exported.
target_link_options(
gluten PRIVATE -Wl,--exclude-libs,libstdc++.a -Wl,--exclude-libs,libgcc.a
-Wl,--exclude-libs,libgcc_eh.a)
endif()

if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0)
Expand Down
16 changes: 13 additions & 3 deletions cpp/velox/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -413,16 +413,26 @@ if(DEFINED VCPKG_INSTALLED_DIR
PRIVATE ${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftcpp2.a
${VCPKG_INSTALLED_DIR}/${VCPKG_TRIPLET_DIR}/lib/libthriftprotocol.a)
else()
message(STATUS "Using system thrift libraries from /usr/local/lib")
set(THRIFT_LIBRARY_DIRS)
if(CMAKE_INSTALL_PREFIX)
list(APPEND THRIFT_LIBRARY_DIRS "${CMAKE_INSTALL_PREFIX}/lib")
endif()
foreach(PREFIX_PATH IN LISTS CMAKE_PREFIX_PATH)
list(APPEND THRIFT_LIBRARY_DIRS "${PREFIX_PATH}/lib" "${PREFIX_PATH}")
endforeach()
list(APPEND THRIFT_LIBRARY_DIRS /usr/local/lib)
list(REMOVE_DUPLICATES THRIFT_LIBRARY_DIRS)

message(STATUS "Searching thrift libraries in ${THRIFT_LIBRARY_DIRS}")
find_library(
THRIFTCPP2_LIB
NAMES thriftcpp2
PATHS /usr/local/lib
PATHS ${THRIFT_LIBRARY_DIRS}
NO_DEFAULT_PATH)
find_library(
THRIFTPROTOCOL_LIB
NAMES thriftprotocol
PATHS /usr/local/lib
PATHS ${THRIFT_LIBRARY_DIRS}
NO_DEFAULT_PATH)

if(THRIFTCPP2_LIB AND THRIFTPROTOCOL_LIB)
Expand Down
1 change: 1 addition & 0 deletions dev/build-helper-functions.sh
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,7 @@ function cmake_install {
if [[ "${INSTALL_PREFIX:-}" == "/usr/local" || "${INSTALL_PREFIX:-}" == /usr/local/* ]]; then
echo "INFO: INSTALL_PREFIX=${INSTALL_PREFIX} is under /usr/local; keeping /usr/local visible to CMake." >&2
else
COMPILER_FLAGS="$COMPILER_FLAGS -isystem /usr/local/include"
MACOS_ISOLATION_FLAGS="-DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON \
-DCMAKE_IGNORE_PREFIX_PATH=/usr/local \
-DCMAKE_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake \
Expand Down
7 changes: 5 additions & 2 deletions dev/build-thirdparty.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,13 +32,16 @@ function process_setup_ubuntu_2004 {

function process_setup_ubuntu_2204 {
cp /usr/lib/${ARCH}-linux-gnu/{libre2.so.9,libdouble-conversion.so.3,libglog.so.0,libgflags.so.2.2,libevent-2.1.so.7,libsnappy.so.1,libunwind.so.8,libcurl.so.4,libxml2.so.2,libicui18n.so.70,libicuuc.so.70,libnghttp2.so.14,libldap-2.5.so.0,liblber-2.5.so.0,librtmp.so.1,libsasl2.so.2,libssh.so.4,libicudata.so.70} $THIRDPARTY_LIB/
cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_regex.so.1.84.0} $THIRDPARTY_LIB/
find /usr/ -name "libboost_context.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
find /usr/ -name "libboost_regex.so.1.84.0" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
}

function process_setup_centos_9 {
cp /lib64/{libre2.so.9,libdouble-conversion.so.3,libevent-2.1.so.7,libdwarf.so.0,libicudata.so.67,libicui18n.so.67,libicuuc.so.67,libsodium.so.23} $THIRDPARTY_LIB/
cp /usr/local/lib/{libboost_context.so.1.84.0,libboost_filesystem.so.1.84.0,libboost_program_options.so.1.84.0,libboost_regex.so.1.84.0,libboost_system.so.1.84.0,libboost_thread.so.1.84.0,libboost_atomic.so.1.84.0} $THIRDPARTY_LIB/
cp /usr/local/lib64/{libgflags.so.2.2,libglog.so.1,libgeos.so.3.10.7} $THIRDPARTY_LIB/
cp /usr/local/lib/libgflags.so.2.2 $THIRDPARTY_LIB/
find /usr/local/lib /usr/local/lib64 -name "libglog.so.1" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
find /usr/local/lib /usr/local/lib64 -name "libgeos.so.3.10.7" -exec cp {} $THIRDPARTY_LIB/ \; 2>/dev/null
}

function process_setup_centos_8 {
Expand Down
53 changes: 30 additions & 23 deletions dev/builddeps-veloxbe.sh
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,8 @@ if [[ "$(uname)" == "Darwin" ]]; then
if [[ "$INSTALL_PREFIX" == "/usr/local" || "$INSTALL_PREFIX" == /usr/local/* ]]; then
echo "INFO: INSTALL_PREFIX=$INSTALL_PREFIX is under /usr/local; keeping /usr/local visible to CMake." >&2
fi
elif [ -n "${INSTALL_PREFIX:-}" ]; then
export INSTALL_PREFIX
fi

function concat_velox_param {
Expand Down Expand Up @@ -244,34 +246,39 @@ function build_gluten_cpp {
mkdir build
cd build

GLUTEN_CMAKE_OPTIONS="-DBUILD_VELOX_BACKEND=ON \
-DCMAKE_BUILD_TYPE=$BUILD_TYPE \
-DVELOX_HOME=$VELOX_HOME \
-DBUILD_TESTS=$BUILD_TESTS \
-DBUILD_EXAMPLES=$BUILD_EXAMPLES \
-DBUILD_BENCHMARKS=$BUILD_BENCHMARKS \
-DENABLE_JEMALLOC_STATS=$ENABLE_JEMALLOC_STATS \
-DENABLE_QAT=$ENABLE_QAT \
-DENABLE_GCS=$ENABLE_GCS \
-DENABLE_S3=$ENABLE_S3 \
-DENABLE_HDFS=$ENABLE_HDFS \
-DENABLE_ABFS=$ENABLE_ABFS \
-DENABLE_GPU=$ENABLE_GPU \
-DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-DENABLE_ENHANCED_FEATURES=$ENABLE_ENHANCED_FEATURES"
GLUTEN_CMAKE_OPTIONS=(
"-DBUILD_VELOX_BACKEND=ON"
"-DCMAKE_BUILD_TYPE=$BUILD_TYPE"
"-DVELOX_HOME=$VELOX_HOME"
"-DBUILD_TESTS=$BUILD_TESTS"
"-DBUILD_EXAMPLES=$BUILD_EXAMPLES"
"-DBUILD_BENCHMARKS=$BUILD_BENCHMARKS"
"-DENABLE_JEMALLOC_STATS=$ENABLE_JEMALLOC_STATS"
"-DENABLE_QAT=$ENABLE_QAT"
"-DENABLE_GCS=$ENABLE_GCS"
"-DENABLE_S3=$ENABLE_S3"
"-DENABLE_HDFS=$ENABLE_HDFS"
"-DENABLE_ABFS=$ENABLE_ABFS"
"-DENABLE_GPU=$ENABLE_GPU"
"-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
"-DENABLE_ENHANCED_FEATURES=$ENABLE_ENHANCED_FEATURES"
)

if [ -n "${INSTALL_PREFIX:-}" ]; then
GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_PREFIX_PATH=$INSTALL_PREFIX")
GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX")
fi
if [ $OS == 'Darwin' ]; then
GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX"
if [[ "$INSTALL_PREFIX" != "/usr/local" && "$INSTALL_PREFIX" != /usr/local/* ]]; then
GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON"
GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_IGNORE_PREFIX_PATH=/usr/local"
GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake"
GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_SYSTEM_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake"
if [[ "${INSTALL_PREFIX:-}" != "/usr/local" && "${INSTALL_PREFIX:-}" != /usr/local/* ]]; then
GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_NO_SYSTEM_FROM_IMPORTED=ON")
GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_IGNORE_PREFIX_PATH=/usr/local")
GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake")
GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_SYSTEM_IGNORE_PATH=/usr/local;/usr/local/include;/usr/local/lib;/usr/local/lib/cmake")
fi
GLUTEN_CMAKE_OPTIONS+=" -DCMAKE_CXX_FLAGS=-Wno-inconsistent-missing-override -Wno-macro-redefined"
GLUTEN_CMAKE_OPTIONS+=("-DCMAKE_CXX_FLAGS=-Wno-inconsistent-missing-override -Wno-macro-redefined")
fi

cmake -G Ninja $GLUTEN_CMAKE_OPTIONS ..
cmake -G Ninja "${GLUTEN_CMAKE_OPTIONS[@]}" ..
ninja -j $NUM_THREADS
}

Expand Down
2 changes: 1 addition & 1 deletion dev/vcpkg/setup-build-depends.sh
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ install_openeuler_24.03() {
gcc g++ cmake ninja-build perl-IPC-Cmd autoconf autoconf-archive automake libtool \
java-1.8.0-openjdk java-1.8.0-openjdk-devel python3-devel python3-pip libstdc++-static

pip install cmake==3.28.3
pip install cmake==3.31.4

install_maven_from_source
}
Expand Down
Loading
Loading