diff --git a/.github/workflows/velox_backend_arm.yml b/.github/workflows/velox_backend_arm.yml index 466ca6447e..f7346b64b2 100644 --- a/.github/workflows/velox_backend_arm.yml +++ b/.github/workflows/velox_backend_arm.yml @@ -137,20 +137,21 @@ jobs: cpp-test-udf-test: runs-on: ubuntu-24.04-arm - container: apache/gluten:centos-8-jdk8 + container: apache/gluten:centos-9-jdk8 steps: - uses: actions/checkout@v4 - name: Get Ccache uses: actions/cache/restore@v4 with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos8-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} restore-keys: | - ccache-centos8-release-shared-${{runner.arch}} + ccache-centos9-release-shared-${{runner.arch}} - name: Build Gluten native libraries run: | df -a - bash dev/ci-velox-buildshared-centos-8.sh + sed -i "s|gflags_static|gflags_shared|g" /usr/local/lib/cmake/folly/folly-targets.cmake # TODO: remove after upgrading folly to 2024.09.30 or later which has fixed the gflags linkage issue + bash dev/ci-velox-buildshared-centos-9.sh ccache -s - name: Run CPP unit test run: | diff --git a/.github/workflows/velox_backend_cache.yml b/.github/workflows/velox_backend_cache.yml index 11d99c21c9..6f54bf9f4c 100644 --- a/.github/workflows/velox_backend_cache.yml +++ b/.github/workflows/velox_backend_cache.yml @@ -83,34 +83,34 @@ jobs: path: '${{ env.CCACHE_DIR }}' key: ccache-centos8-release-default-${{runner.arch}}-${{github.sha}} - cache-shared-lib-centos-8: + cache-shared-lib-centos-9: if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ${{ matrix.os }} strategy: matrix: os: [ ubuntu-22.04, ubuntu-24.04-arm ] - container: apache/gluten:centos-8-jdk8 + container: apache/gluten:centos-9-jdk8 steps: - uses: actions/checkout@v4 - name: Get Ccache uses: actions/cache/restore@v3 with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos8-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} restore-keys: | - ccache-centos8-release-shared-${{runner.arch}} + ccache-centos9-release-shared-${{runner.arch}} - name: Build Gluten shared libraries run: | df -a export CCACHE_MAXSIZE=1G - bash dev/ci-velox-buildshared-centos-8.sh + bash dev/ci-velox-buildshared-centos-9.sh ccache -s - name: Save Ccache uses: actions/cache/save@v3 id: ccache with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos8-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} cache-enhanced-native-lib-centos-7: if: ${{ startsWith(github.repository, 'apache/') }} @@ -139,7 +139,7 @@ jobs: path: '${{ env.CCACHE_DIR }}' key: ccache-enhanced-centos7-release-default-${{github.sha}} - cache-shared-lib-centos-9: + cache-shared-lib-centos-9-cudf: if: ${{ startsWith(github.repository, 'apache/') }} runs-on: ${{ matrix.os }} strategy: @@ -156,7 +156,7 @@ jobs: uses: actions/cache/restore@v3 with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos9-cudf-release-shared-${{runner.arch}}-${{github.sha}} restore-keys: | ccache-centos9-release-shared-${{runner.arch}} - name: Build Gluten shared libraries @@ -187,4 +187,4 @@ jobs: id: ccache with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos9-cudf-release-shared-${{runner.arch}}-${{github.sha}} diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 8672e5ecc4..11d906d53e 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -1168,20 +1168,21 @@ jobs: cpp-test-udf-test: runs-on: ubuntu-22.04 - container: apache/gluten:centos-8-jdk8 + container: apache/gluten:centos-9-jdk8 steps: - uses: actions/checkout@v4 - name: Get Ccache uses: actions/cache/restore@v4 with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos8-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} restore-keys: | - ccache-centos8-release-shared-${{runner.arch}} + ccache-centos9-release-shared-${{runner.arch}} - name: Build Gluten native libraries run: | df -a - bash dev/ci-velox-buildshared-centos-8.sh + sed -i "s|gflags_static|gflags_shared|g" /usr/local/lib/cmake/folly/folly-targets.cmake # TODO remove after image update + bash dev/ci-velox-buildshared-centos-9.sh ccache -s - name: Run CPP unit test run: | @@ -1245,9 +1246,9 @@ jobs: uses: actions/cache/restore@v4 with: path: '${{ env.CCACHE_DIR }}' - key: ccache-centos9-release-shared-${{runner.arch}}-${{github.sha}} + key: ccache-centos9-cudf-release-shared-${{runner.arch}}-${{github.sha}} restore-keys: | - ccache-centos9-release-shared-${{runner.arch}} + ccache-centos9-cudf-release-shared-${{runner.arch}} - name: Build Gluten native libraries run: | docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:centos-9-jdk8-cudf bash -c " diff --git a/dev/ci-velox-buildshared-centos-9.sh b/dev/ci-velox-buildshared-centos-9.sh new file mode 100755 index 0000000000..d4008d125c --- /dev/null +++ b/dev/ci-velox-buildshared-centos-9.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +source /opt/rh/gcc-toolset-12/enable + +if [ "$(uname -m)" = "aarch64" ]; then + export CPU_TARGET="aarch64"; + export VCPKG_FORCE_SYSTEM_BINARIES=1; +fi + +./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --build_tests=ON \ + --build_examples=ON --build_benchmarks=ON diff --git a/ep/build-velox/src/get-velox.sh b/ep/build-velox/src/get-velox.sh index 2d819f974b..2ffcc56092 100755 --- a/ep/build-velox/src/get-velox.sh +++ b/ep/build-velox/src/get-velox.sh @@ -18,8 +18,8 @@ set -exu CURRENT_DIR=$(cd "$(dirname "$BASH_SOURCE")"; pwd) VELOX_REPO=https://github.com/IBM/velox.git -VELOX_BRANCH=dft-2026_05_29 -VELOX_ENHANCED_BRANCH=ibm-2026_05_29 +VELOX_BRANCH=dft-2026_06_02 +VELOX_ENHANCED_BRANCH=ibm-2026_06_02 VELOX_HOME="" RUN_SETUP_SCRIPT=ON ENABLE_ENHANCED_FEATURES=OFF @@ -69,7 +69,7 @@ function process_setup_ubuntu { } function process_setup_centos9 { - sed -i "s|-DFOLLY_HAVE_INT128_T=ON|-DFOLLY_HAVE_INT128_T=ON -DFOLLY_NO_EXCEPTION_TRACER=ON|g" scripts/setup-common.sh + sed -i "s|-DGFLAGS_SHARED=FALSE|-DGFLAGS_SHARED=TRUE|g" scripts/setup-common.sh echo "Using setup script from Velox" }