From 44b62355a9304b12d0fec0e7f6574cc8e034326e Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 19 Jan 2026 13:58:02 +0000 Subject: [PATCH 01/11] [VL] enable dynamic openssl in vcpkg Signed-off-by: Yuan fix Signed-off-by: Yuan install libcrypto/libssl Signed-off-by: Yuan fix Signed-off-by: Yuan --- .github/workflows/velox_backend_x86.yml | 1 - .../x64-linux-avx-dynamic-openssl.cmake | 19 +++ dev/vcpkg/vcpkg-dynamic-openssl.json | 122 ++++++++++++++++++ 3 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake create mode 100644 dev/vcpkg/vcpkg-dynamic-openssl.json diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 3c9efc44bd..7342c62a15 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -68,7 +68,6 @@ jobs: ccache-centos7-release-default - name: Build Gluten native libraries run: | - docker pull apache/gluten:vcpkg-centos-7-gcc13 docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7-gcc13 bash -c " set -e yum install tzdata -y diff --git a/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake b/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake new file mode 100644 index 0000000000..ba069e033e --- /dev/null +++ b/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake @@ -0,0 +1,19 @@ +set(VCPKG_BUILD_TYPE release) +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +set(VCPKG_C_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -mbmi2") +set(VCPKG_CXX_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++20 -mbmi2") +set(VCPKG_LINKER_FLAGS "-static-libstdc++ -static-libgcc") + +# require BMI2 support for snappy on GCC-13 +if("${PORT}" STREQUAL "snappy") + list(APPEND VCPKG_CMAKE_CONFIGURE_OPTIONS -DSNAPPY_HAVE_BMI2=ON) +endif() + +if(PORT STREQUAL "openssl") + set(VCPKG_LIBRARY_LINKAGE dynamic) +endif() diff --git a/dev/vcpkg/vcpkg-dynamic-openssl.json b/dev/vcpkg/vcpkg-dynamic-openssl.json new file mode 100644 index 0000000000..e22da17943 --- /dev/null +++ b/dev/vcpkg/vcpkg-dynamic-openssl.json @@ -0,0 +1,122 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", + "builtin-baseline": "4334d8b4c8916018600212ab4dd4bbdc343065d1", + "default-features": ["velox"], + "features": { + "velox": { + "description": "Velox backend", + "dependencies": [ + "libevent", + { + "name": "openssl", + "features": ["fips"] + } + "zstd", + "lz4", + "double-conversion", + "bzip2", + "curl", + "lzo", + "re2", + "gflags", + "glog", + "snappy", + "libdwarf", + "fmt", + { + "name": "folly", + "features": ["zstd", "lz4", "lzma", "snappy"] + }, + "boost-atomic", + "boost-context", + "boost-circular-buffer", + "boost-date-time", + "boost-format", + "boost-filesystem", + "boost-program-options", + "boost-process", + "boost-regex", + "boost-locale", + "boost-system", + "boost-stacktrace", + "boost-sort", + "boost-thread", + "boost-multiprecision", + "boost-uuid", + "boost-variant", + "gtest", + "xsimd", + "simdjson", + "xxhash", + "protobuf", + "benchmark", + "icu", + "thrift", + "libstemmer", + "geos", + "grpc" + ] + }, + "velox-s3": { + "description": "Velox S3 Support", + "dependencies": [ + { + "name": "aws-sdk-cpp", + "features": [ + "identity-management", + "s3", + "sts", + "cognito-identity" + ] + } + ] + }, + "velox-gcs": { + "description": "Velox GCS Support", + "dependencies": [ + { + "name": "google-cloud-cpp", + "features": [ + "storage" + ], + "version>=": "2.22.0" + } + ] + }, + "velox-abfs": { + "description": "Velox ABFS Support", + "dependencies": [ + "azure-storage-blobs-cpp", + "azure-storage-files-datalake-cpp", + "azure-identity-cpp" + ] + }, + "duckdb": { + "description": "DUCKDB", + "dependencies": [ + "duckdb" + ] + }, + "qat": { + "description": "QAT", + "dependencies": [ + { + "name": "zstd", + "version>=": "1.5.4" + } + ] + } + }, + "overrides": [ + { "name": "fmt", "version": "10.1.1" }, + { "name": "liblzma", "version": "5.6.2" }, + { "name": "xsimd", "version": "10.0.0" }, + { "name": "protobuf", "version": "3.21.8"}, + { "name": "abseil", "version": "20230802.1"}, + { "name": "re2", "version": "2023-07-01"}, + { "name": "glog", "version": "0.6.0"}, + { "name": "gflags", "version": "2.2.2"}, + { "name": "azure-identity-cpp", "version": "1.11.0"}, + { "name": "grpc", "version": "1.51.1"} + ] +} From f7e9cdd9c6c89ab05bbf09da8eee39313bf4f1db Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 25 May 2026 15:06:27 +0100 Subject: [PATCH 02/11] refactor Signed-off-by: Yuan --- dev/vcpkg/env.sh | 5 +++++ dev/vcpkg/triplets/x64-linux-avx-base.cmake | 15 +++++++++++++++ .../x64-linux-avx-dynamic-openssl.cmake | 18 ++---------------- dev/vcpkg/triplets/x64-linux-avx.cmake | 16 +--------------- dev/vcpkg/vcpkg-dynamic-openssl.json | 2 +- dev/vcpkg/vcpkg.json | 9 +++++++++ 6 files changed, 33 insertions(+), 32 deletions(-) create mode 100644 dev/vcpkg/triplets/x64-linux-avx-base.cmake diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh index a86a953e2b..c6f2906eec 100755 --- a/dev/vcpkg/env.sh +++ b/dev/vcpkg/env.sh @@ -26,7 +26,11 @@ SCRIPT_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" export VCPKG_ROOT="$SCRIPT_ROOT/.vcpkg" export VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg" +export VCPKG_DYNAMIC_OPENSSL=${VCPKG_DYNAMIC_OPENSSL:-OFF} export VCPKG_TRIPLET=$([ "${CPU_TARGET:-}" = "aarch64" ] && echo "arm64-linux-neon" || echo "x64-linux-avx") +if [ "${VCPKG_DYNAMIC_OPENSSL}" = "ON" ] && [ "${VCPKG_TRIPLET}" = "x64-linux-avx" ]; then + export VCPKG_TRIPLET="x64-linux-avx-dynamic-openssl" +fi export VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET} ${SCRIPT_ROOT}/init.sh "$@" @@ -44,6 +48,7 @@ if [ "${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then export VCPKG_ROOT=${VCPKG_ROOT} export VCPKG_MANIFEST_DIR=${SCRIPT_ROOT} + export VCPKG_MANIFEST_FEATURES=$([ "${VCPKG_DYNAMIC_OPENSSL}" = "ON" ] && echo "dynamic-openssl" || echo "") export VCPKG_TRIPLET=${VCPKG_TRIPLET} export CMAKE_TOOLCHAIN_FILE=${SCRIPT_ROOT}/toolchain.cmake diff --git a/dev/vcpkg/triplets/x64-linux-avx-base.cmake b/dev/vcpkg/triplets/x64-linux-avx-base.cmake new file mode 100644 index 0000000000..1488a2bc26 --- /dev/null +++ b/dev/vcpkg/triplets/x64-linux-avx-base.cmake @@ -0,0 +1,15 @@ +set(VCPKG_BUILD_TYPE release) +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +set(VCPKG_C_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -mbmi2") +set(VCPKG_CXX_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++20 -mbmi2") +set(VCPKG_LINKER_FLAGS "-static-libstdc++ -static-libgcc") + +# require BMI2 support for snappy on GCC-13 +if("${PORT}" STREQUAL "snappy") + list(APPEND VCPKG_CMAKE_CONFIGURE_OPTIONS -DSNAPPY_HAVE_BMI2=ON) +endif() \ No newline at end of file diff --git a/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake b/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake index ba069e033e..bd1f04ec40 100644 --- a/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake +++ b/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake @@ -1,19 +1,5 @@ -set(VCPKG_BUILD_TYPE release) -set(VCPKG_TARGET_ARCHITECTURE x64) -set(VCPKG_CRT_LINKAGE dynamic) -set(VCPKG_LIBRARY_LINKAGE static) +include("${CMAKE_CURRENT_LIST_DIR}/x64-linux-avx-base.cmake") -set(VCPKG_CMAKE_SYSTEM_NAME Linux) - -set(VCPKG_C_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -mbmi2") -set(VCPKG_CXX_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++20 -mbmi2") -set(VCPKG_LINKER_FLAGS "-static-libstdc++ -static-libgcc") - -# require BMI2 support for snappy on GCC-13 -if("${PORT}" STREQUAL "snappy") - list(APPEND VCPKG_CMAKE_CONFIGURE_OPTIONS -DSNAPPY_HAVE_BMI2=ON) -endif() - -if(PORT STREQUAL "openssl") +if("${PORT}" STREQUAL "openssl") set(VCPKG_LIBRARY_LINKAGE dynamic) endif() diff --git a/dev/vcpkg/triplets/x64-linux-avx.cmake b/dev/vcpkg/triplets/x64-linux-avx.cmake index 50f1bdebc9..617536dafb 100644 --- a/dev/vcpkg/triplets/x64-linux-avx.cmake +++ b/dev/vcpkg/triplets/x64-linux-avx.cmake @@ -1,15 +1 @@ -set(VCPKG_BUILD_TYPE release) -set(VCPKG_TARGET_ARCHITECTURE x64) -set(VCPKG_CRT_LINKAGE dynamic) -set(VCPKG_LIBRARY_LINKAGE static) - -set(VCPKG_CMAKE_SYSTEM_NAME Linux) - -set(VCPKG_C_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -mbmi2") -set(VCPKG_CXX_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++20 -mbmi2") -set(VCPKG_LINKER_FLAGS "-static-libstdc++ -static-libgcc") - -# require BMI2 support for snappy on GCC-13 -if("${PORT}" STREQUAL "snappy") - list(APPEND VCPKG_CMAKE_CONFIGURE_OPTIONS -DSNAPPY_HAVE_BMI2=ON) -endif() +include("${CMAKE_CURRENT_LIST_DIR}/x64-linux-avx-base.cmake") diff --git a/dev/vcpkg/vcpkg-dynamic-openssl.json b/dev/vcpkg/vcpkg-dynamic-openssl.json index e22da17943..9545120b0e 100644 --- a/dev/vcpkg/vcpkg-dynamic-openssl.json +++ b/dev/vcpkg/vcpkg-dynamic-openssl.json @@ -10,7 +10,7 @@ { "name": "openssl", "features": ["fips"] - } + }, "zstd", "lz4", "double-conversion", diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json index aba59968f2..ebe2329ab0 100644 --- a/dev/vcpkg/vcpkg.json +++ b/dev/vcpkg/vcpkg.json @@ -54,6 +54,15 @@ "grpc" ] }, + "dynamic-openssl": { + "description": "Enable dynamic OpenSSL with fips feature", + "dependencies": [ + { + "name": "openssl", + "features": ["fips"] + } + ] + }, "velox-s3": { "description": "Velox S3 Support", "dependencies": [ From 8b8d3f5464d50a7fc50aa2f7c691c622a0acf2a9 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 25 May 2026 15:10:41 +0100 Subject: [PATCH 03/11] adding arm support Signed-off-by: Yuan --- dev/vcpkg/env.sh | 8 ++++++-- dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake | 5 +++++ 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh index c6f2906eec..9b96d82ba6 100755 --- a/dev/vcpkg/env.sh +++ b/dev/vcpkg/env.sh @@ -28,8 +28,12 @@ export VCPKG_ROOT="$SCRIPT_ROOT/.vcpkg" export VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg" export VCPKG_DYNAMIC_OPENSSL=${VCPKG_DYNAMIC_OPENSSL:-OFF} export VCPKG_TRIPLET=$([ "${CPU_TARGET:-}" = "aarch64" ] && echo "arm64-linux-neon" || echo "x64-linux-avx") -if [ "${VCPKG_DYNAMIC_OPENSSL}" = "ON" ] && [ "${VCPKG_TRIPLET}" = "x64-linux-avx" ]; then - export VCPKG_TRIPLET="x64-linux-avx-dynamic-openssl" +if [ "${VCPKG_DYNAMIC_OPENSSL}" = "ON" ]; then + if [ "${VCPKG_TRIPLET}" = "arm64-linux-neon" ]; then + export VCPKG_TRIPLET="arm64-linux-neon-dynamic-openssl" + elif [ "${VCPKG_TRIPLET}" = "x64-linux-avx" ]; then + export VCPKG_TRIPLET="x64-linux-avx-dynamic-openssl" + fi fi export VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET} diff --git a/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake b/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake new file mode 100644 index 0000000000..1535ca6a66 --- /dev/null +++ b/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake @@ -0,0 +1,5 @@ +include("${CMAKE_CURRENT_LIST_DIR}/arm64-linux-neon.cmake") + +if("${PORT}" STREQUAL "openssl") + set(VCPKG_LIBRARY_LINKAGE dynamic) +endif() \ No newline at end of file From 3116f64d30fc603858bb614057733f8377869810 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 25 May 2026 15:22:37 +0100 Subject: [PATCH 04/11] refine Signed-off-by: Yuan --- dev/vcpkg/env.sh | 9 -- dev/vcpkg/init.sh | 5 +- dev/vcpkg/vcpkg-dynamic-openssl.json | 122 --------------------------- 3 files changed, 4 insertions(+), 132 deletions(-) delete mode 100644 dev/vcpkg/vcpkg-dynamic-openssl.json diff --git a/dev/vcpkg/env.sh b/dev/vcpkg/env.sh index 9b96d82ba6..a86a953e2b 100755 --- a/dev/vcpkg/env.sh +++ b/dev/vcpkg/env.sh @@ -26,15 +26,7 @@ SCRIPT_ROOT="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" export VCPKG_ROOT="$SCRIPT_ROOT/.vcpkg" export VCPKG="$SCRIPT_ROOT/.vcpkg/vcpkg" -export VCPKG_DYNAMIC_OPENSSL=${VCPKG_DYNAMIC_OPENSSL:-OFF} export VCPKG_TRIPLET=$([ "${CPU_TARGET:-}" = "aarch64" ] && echo "arm64-linux-neon" || echo "x64-linux-avx") -if [ "${VCPKG_DYNAMIC_OPENSSL}" = "ON" ]; then - if [ "${VCPKG_TRIPLET}" = "arm64-linux-neon" ]; then - export VCPKG_TRIPLET="arm64-linux-neon-dynamic-openssl" - elif [ "${VCPKG_TRIPLET}" = "x64-linux-avx" ]; then - export VCPKG_TRIPLET="x64-linux-avx-dynamic-openssl" - fi -fi export VCPKG_TRIPLET_INSTALL_DIR=${SCRIPT_ROOT}/vcpkg_installed/${VCPKG_TRIPLET} ${SCRIPT_ROOT}/init.sh "$@" @@ -52,7 +44,6 @@ if [ "${GLUTEN_VCPKG_ENABLED:-}" != "${VCPKG_ROOT}" ]; then export VCPKG_ROOT=${VCPKG_ROOT} export VCPKG_MANIFEST_DIR=${SCRIPT_ROOT} - export VCPKG_MANIFEST_FEATURES=$([ "${VCPKG_DYNAMIC_OPENSSL}" = "ON" ] && echo "dynamic-openssl" || echo "") export VCPKG_TRIPLET=${VCPKG_TRIPLET} export CMAKE_TOOLCHAIN_FILE=${SCRIPT_ROOT}/toolchain.cmake diff --git a/dev/vcpkg/init.sh b/dev/vcpkg/init.sh index 1247a14493..8b4f01b03e 100755 --- a/dev/vcpkg/init.sh +++ b/dev/vcpkg/init.sh @@ -85,7 +85,10 @@ if [ "$ENABLE_GCS" = "ON" ]; then EXTRA_FEATURES+="--x-feature=velox-gcs " fi if [ "$ENABLE_ABFS" = "ON" ]; then - EXTRA_FEATURES+="--x-feature=velox-abfs" + EXTRA_FEATURES+="--x-feature=velox-abfs " +fi +if [ "${VCPKG_DYNAMIC_OPENSSL:-OFF}" = "ON" ]; then + EXTRA_FEATURES+="--x-feature=dynamic-openssl " fi diff --git a/dev/vcpkg/vcpkg-dynamic-openssl.json b/dev/vcpkg/vcpkg-dynamic-openssl.json deleted file mode 100644 index 9545120b0e..0000000000 --- a/dev/vcpkg/vcpkg-dynamic-openssl.json +++ /dev/null @@ -1,122 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", - "builtin-baseline": "4334d8b4c8916018600212ab4dd4bbdc343065d1", - "default-features": ["velox"], - "features": { - "velox": { - "description": "Velox backend", - "dependencies": [ - "libevent", - { - "name": "openssl", - "features": ["fips"] - }, - "zstd", - "lz4", - "double-conversion", - "bzip2", - "curl", - "lzo", - "re2", - "gflags", - "glog", - "snappy", - "libdwarf", - "fmt", - { - "name": "folly", - "features": ["zstd", "lz4", "lzma", "snappy"] - }, - "boost-atomic", - "boost-context", - "boost-circular-buffer", - "boost-date-time", - "boost-format", - "boost-filesystem", - "boost-program-options", - "boost-process", - "boost-regex", - "boost-locale", - "boost-system", - "boost-stacktrace", - "boost-sort", - "boost-thread", - "boost-multiprecision", - "boost-uuid", - "boost-variant", - "gtest", - "xsimd", - "simdjson", - "xxhash", - "protobuf", - "benchmark", - "icu", - "thrift", - "libstemmer", - "geos", - "grpc" - ] - }, - "velox-s3": { - "description": "Velox S3 Support", - "dependencies": [ - { - "name": "aws-sdk-cpp", - "features": [ - "identity-management", - "s3", - "sts", - "cognito-identity" - ] - } - ] - }, - "velox-gcs": { - "description": "Velox GCS Support", - "dependencies": [ - { - "name": "google-cloud-cpp", - "features": [ - "storage" - ], - "version>=": "2.22.0" - } - ] - }, - "velox-abfs": { - "description": "Velox ABFS Support", - "dependencies": [ - "azure-storage-blobs-cpp", - "azure-storage-files-datalake-cpp", - "azure-identity-cpp" - ] - }, - "duckdb": { - "description": "DUCKDB", - "dependencies": [ - "duckdb" - ] - }, - "qat": { - "description": "QAT", - "dependencies": [ - { - "name": "zstd", - "version>=": "1.5.4" - } - ] - } - }, - "overrides": [ - { "name": "fmt", "version": "10.1.1" }, - { "name": "liblzma", "version": "5.6.2" }, - { "name": "xsimd", "version": "10.0.0" }, - { "name": "protobuf", "version": "3.21.8"}, - { "name": "abseil", "version": "20230802.1"}, - { "name": "re2", "version": "2023-07-01"}, - { "name": "glog", "version": "0.6.0"}, - { "name": "gflags", "version": "2.2.2"}, - { "name": "azure-identity-cpp", "version": "1.11.0"}, - { "name": "grpc", "version": "1.51.1"} - ] -} From 30980cdc2a62def17e25e7a64d0dc469f81300ab Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 25 May 2026 15:31:02 +0100 Subject: [PATCH 05/11] remove deprecated file Signed-off-by: Yuan --- .github/workflows/velox_backend_x86.yml | 1 + dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake | 5 ----- 2 files changed, 1 insertion(+), 5 deletions(-) delete mode 100644 dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 7342c62a15..3c9efc44bd 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -68,6 +68,7 @@ jobs: ccache-centos7-release-default - name: Build Gluten native libraries run: | + docker pull apache/gluten:vcpkg-centos-7-gcc13 docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7-gcc13 bash -c " set -e yum install tzdata -y diff --git a/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake b/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake deleted file mode 100644 index 1535ca6a66..0000000000 --- a/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake +++ /dev/null @@ -1,5 +0,0 @@ -include("${CMAKE_CURRENT_LIST_DIR}/arm64-linux-neon.cmake") - -if("${PORT}" STREQUAL "openssl") - set(VCPKG_LIBRARY_LINKAGE dynamic) -endif() \ No newline at end of file From d02e9b4997f6c238ed3aa39137bebca824e2be04 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 25 May 2026 15:41:38 +0100 Subject: [PATCH 06/11] Revert "remove deprecated file" This reverts commit 30980cdc2a62def17e25e7a64d0dc469f81300ab. --- .github/workflows/velox_backend_x86.yml | 1 - dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 3c9efc44bd..7342c62a15 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -68,7 +68,6 @@ jobs: ccache-centos7-release-default - name: Build Gluten native libraries run: | - docker pull apache/gluten:vcpkg-centos-7-gcc13 docker run -v $GITHUB_WORKSPACE:/work -w /work apache/gluten:vcpkg-centos-7-gcc13 bash -c " set -e yum install tzdata -y diff --git a/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake b/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake new file mode 100644 index 0000000000..1535ca6a66 --- /dev/null +++ b/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake @@ -0,0 +1,5 @@ +include("${CMAKE_CURRENT_LIST_DIR}/arm64-linux-neon.cmake") + +if("${PORT}" STREQUAL "openssl") + set(VCPKG_LIBRARY_LINKAGE dynamic) +endif() \ No newline at end of file From 3289a34d945ddf4b8fe112549d8409a9154390d4 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 25 May 2026 15:43:31 +0100 Subject: [PATCH 07/11] fix --- dev/vcpkg/triplets/x64-linux-avx-base.cmake | 15 --------------- .../triplets/x64-linux-avx-dynamic-openssl.cmake | 2 +- dev/vcpkg/triplets/x64-linux-avx.cmake | 16 +++++++++++++++- 3 files changed, 16 insertions(+), 17 deletions(-) delete mode 100644 dev/vcpkg/triplets/x64-linux-avx-base.cmake diff --git a/dev/vcpkg/triplets/x64-linux-avx-base.cmake b/dev/vcpkg/triplets/x64-linux-avx-base.cmake deleted file mode 100644 index 1488a2bc26..0000000000 --- a/dev/vcpkg/triplets/x64-linux-avx-base.cmake +++ /dev/null @@ -1,15 +0,0 @@ -set(VCPKG_BUILD_TYPE release) -set(VCPKG_TARGET_ARCHITECTURE x64) -set(VCPKG_CRT_LINKAGE dynamic) -set(VCPKG_LIBRARY_LINKAGE static) - -set(VCPKG_CMAKE_SYSTEM_NAME Linux) - -set(VCPKG_C_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -mbmi2") -set(VCPKG_CXX_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++20 -mbmi2") -set(VCPKG_LINKER_FLAGS "-static-libstdc++ -static-libgcc") - -# require BMI2 support for snappy on GCC-13 -if("${PORT}" STREQUAL "snappy") - list(APPEND VCPKG_CMAKE_CONFIGURE_OPTIONS -DSNAPPY_HAVE_BMI2=ON) -endif() \ No newline at end of file diff --git a/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake b/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake index bd1f04ec40..6205eab565 100644 --- a/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake +++ b/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake @@ -1,4 +1,4 @@ -include("${CMAKE_CURRENT_LIST_DIR}/x64-linux-avx-base.cmake") +include("${CMAKE_CURRENT_LIST_DIR}/x64-linux-avx.cmake") if("${PORT}" STREQUAL "openssl") set(VCPKG_LIBRARY_LINKAGE dynamic) diff --git a/dev/vcpkg/triplets/x64-linux-avx.cmake b/dev/vcpkg/triplets/x64-linux-avx.cmake index 617536dafb..1488a2bc26 100644 --- a/dev/vcpkg/triplets/x64-linux-avx.cmake +++ b/dev/vcpkg/triplets/x64-linux-avx.cmake @@ -1 +1,15 @@ -include("${CMAKE_CURRENT_LIST_DIR}/x64-linux-avx-base.cmake") +set(VCPKG_BUILD_TYPE release) +set(VCPKG_TARGET_ARCHITECTURE x64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE static) + +set(VCPKG_CMAKE_SYSTEM_NAME Linux) + +set(VCPKG_C_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -mbmi2") +set(VCPKG_CXX_FLAGS "-mavx2 -mfma -mavx -mf16c -mlzcnt -std=c++20 -mbmi2") +set(VCPKG_LINKER_FLAGS "-static-libstdc++ -static-libgcc") + +# require BMI2 support for snappy on GCC-13 +if("${PORT}" STREQUAL "snappy") + list(APPEND VCPKG_CMAKE_CONFIGURE_OPTIONS -DSNAPPY_HAVE_BMI2=ON) +endif() \ No newline at end of file From 357869bc6a8abb82d10137dd1928c27d8d90b5e2 Mon Sep 17 00:00:00 2001 From: Yuan Date: Mon, 25 May 2026 15:45:30 +0100 Subject: [PATCH 08/11] merge with original cmake Signed-off-by: Yuan --- dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake | 5 ----- dev/vcpkg/triplets/arm64-linux-neon.cmake | 4 ++++ dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake | 5 ----- dev/vcpkg/triplets/x64-linux-avx.cmake | 4 ++++ 4 files changed, 8 insertions(+), 10 deletions(-) delete mode 100644 dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake delete mode 100644 dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake diff --git a/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake b/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake deleted file mode 100644 index 1535ca6a66..0000000000 --- a/dev/vcpkg/triplets/arm64-linux-neon-dynamic-openssl.cmake +++ /dev/null @@ -1,5 +0,0 @@ -include("${CMAKE_CURRENT_LIST_DIR}/arm64-linux-neon.cmake") - -if("${PORT}" STREQUAL "openssl") - set(VCPKG_LIBRARY_LINKAGE dynamic) -endif() \ No newline at end of file diff --git a/dev/vcpkg/triplets/arm64-linux-neon.cmake b/dev/vcpkg/triplets/arm64-linux-neon.cmake index c02fe5b442..92f03d1c4f 100644 --- a/dev/vcpkg/triplets/arm64-linux-neon.cmake +++ b/dev/vcpkg/triplets/arm64-linux-neon.cmake @@ -8,3 +8,7 @@ set(VCPKG_CMAKE_SYSTEM_NAME Linux) set(VCPKG_C_FLAGS "-march=armv8-a+crc") set(VCPKG_CXX_FLAGS "-march=armv8-a+crc -std=c++20") set(VCPKG_LINKER_FLAGS "-static-libstdc++ -static-libgcc") + +if("${PORT}" STREQUAL "openssl" AND "$ENV{VCPKG_DYNAMIC_OPENSSL}" STREQUAL "ON") + set(VCPKG_LIBRARY_LINKAGE dynamic) +endif() diff --git a/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake b/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake deleted file mode 100644 index 6205eab565..0000000000 --- a/dev/vcpkg/triplets/x64-linux-avx-dynamic-openssl.cmake +++ /dev/null @@ -1,5 +0,0 @@ -include("${CMAKE_CURRENT_LIST_DIR}/x64-linux-avx.cmake") - -if("${PORT}" STREQUAL "openssl") - set(VCPKG_LIBRARY_LINKAGE dynamic) -endif() diff --git a/dev/vcpkg/triplets/x64-linux-avx.cmake b/dev/vcpkg/triplets/x64-linux-avx.cmake index 1488a2bc26..71600d4cb7 100644 --- a/dev/vcpkg/triplets/x64-linux-avx.cmake +++ b/dev/vcpkg/triplets/x64-linux-avx.cmake @@ -12,4 +12,8 @@ set(VCPKG_LINKER_FLAGS "-static-libstdc++ -static-libgcc") # require BMI2 support for snappy on GCC-13 if("${PORT}" STREQUAL "snappy") list(APPEND VCPKG_CMAKE_CONFIGURE_OPTIONS -DSNAPPY_HAVE_BMI2=ON) +endif() + +if("${PORT}" STREQUAL "openssl" AND "$ENV{VCPKG_DYNAMIC_OPENSSL}" STREQUAL "ON") + set(VCPKG_LIBRARY_LINKAGE dynamic) endif() \ No newline at end of file From 422fe94cd168b14267a9cc3048a2a2b8056d0712 Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 26 May 2026 13:39:28 +0100 Subject: [PATCH 09/11] skip copying libs Signed-off-by: Yuan --- dev/vcpkg/init.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dev/vcpkg/init.sh b/dev/vcpkg/init.sh index 8b4f01b03e..6683ba9151 100755 --- a/dev/vcpkg/init.sh +++ b/dev/vcpkg/init.sh @@ -100,7 +100,9 @@ $VCPKG install --no-print-usage \ # needed by 'releases/libvelox.so' mkdir -p $VCPKG_TRIPLET_INSTALL_DIR/debug/lib/ cp $VCPKG_TRIPLET_INSTALL_DIR/lib/libz.a $VCPKG_TRIPLET_INSTALL_DIR/debug/lib -cp $VCPKG_TRIPLET_INSTALL_DIR/lib/libssl.a $VCPKG_TRIPLET_INSTALL_DIR/debug/lib -cp $VCPKG_TRIPLET_INSTALL_DIR/lib/libcrypto.a $VCPKG_TRIPLET_INSTALL_DIR/debug/lib +if [ "${VCPKG_DYNAMIC_OPENSSL:-OFF}" = "OFF" ]; then + cp $VCPKG_TRIPLET_INSTALL_DIR/lib/libssl.a $VCPKG_TRIPLET_INSTALL_DIR/debug/lib + cp $VCPKG_TRIPLET_INSTALL_DIR/lib/libcrypto.a $VCPKG_TRIPLET_INSTALL_DIR/debug/lib +fi cp $VCPKG_TRIPLET_INSTALL_DIR/lib/liblzma.a $VCPKG_TRIPLET_INSTALL_DIR/debug/lib cp $VCPKG_TRIPLET_INSTALL_DIR/lib/libdwarf.a $VCPKG_TRIPLET_INSTALL_DIR/debug/lib From baf52255c94791b9b77122596cbca68f7e0985ce Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 26 May 2026 13:41:46 +0100 Subject: [PATCH 10/11] add test --- .github/workflows/velox_backend_x86.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 7342c62a15..eeff896112 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -77,6 +77,7 @@ jobs: export CCACHE_MAXSIZE=1G mkdir -p /work/.ccache ccache -sz + export VCPKG_DYNAMIC_OPENSSL=ON bash dev/ci-velox-buildstatic-centos-7.sh ccache -s mkdir -p /work/.m2/repository/org/apache/arrow/ @@ -294,11 +295,11 @@ jobs: - name: Setup java run: | if [ "${{ matrix.java }}" = "java-17" ]; then - yum update -y && yum install -y java-17-openjdk-devel wget + yum update -y && yum install -y java-17-openjdk-devel wget openssl-devel elif [ "${{ matrix.java }}" = "java-11" ]; then - yum update -y && yum install -y java-11-openjdk-devel wget + yum update -y && yum install -y java-11-openjdk-devel wget openssl-devel else - yum update -y && yum install -y java-1.8.0-openjdk-devel wget + yum update -y && yum install -y java-1.8.0-openjdk-devel wget openssl-devel fi - name: Set environment variables run: | From dcf9816e9c4e6d8ef94ab9f86fa81f8b69faadce Mon Sep 17 00:00:00 2001 From: Yuan Date: Tue, 26 May 2026 13:51:14 +0100 Subject: [PATCH 11/11] Revert "add test" This reverts commit baf52255c94791b9b77122596cbca68f7e0985ce. --- .github/workflows/velox_backend_x86.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index eeff896112..7342c62a15 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -77,7 +77,6 @@ jobs: export CCACHE_MAXSIZE=1G mkdir -p /work/.ccache ccache -sz - export VCPKG_DYNAMIC_OPENSSL=ON bash dev/ci-velox-buildstatic-centos-7.sh ccache -s mkdir -p /work/.m2/repository/org/apache/arrow/ @@ -295,11 +294,11 @@ jobs: - name: Setup java run: | if [ "${{ matrix.java }}" = "java-17" ]; then - yum update -y && yum install -y java-17-openjdk-devel wget openssl-devel + yum update -y && yum install -y java-17-openjdk-devel wget elif [ "${{ matrix.java }}" = "java-11" ]; then - yum update -y && yum install -y java-11-openjdk-devel wget openssl-devel + yum update -y && yum install -y java-11-openjdk-devel wget else - yum update -y && yum install -y java-1.8.0-openjdk-devel wget openssl-devel + yum update -y && yum install -y java-1.8.0-openjdk-devel wget fi - name: Set environment variables run: |