Merge branch 'ThrottledAsyncTaskSchedulerWaitForCallback' into combined2

Author: Rafał Hibner

.env

@@ -49,12 +49,12 @@ ULIMIT_CORE=-1
 ALMALINUX=8
 ALPINE_LINUX=3.18
 DEBIAN=12
-FEDORA=39
+FEDORA=42
 UBUNTU=22.04
 
 # Default versions for various dependencies
 CLANG_TOOLS=14
-CMAKE=3.25.0
+CMAKE=3.26.0
 CUDA=11.7.1
 DASK=latest
 DOTNET=8.0

.github/workflows/release_candidate.yml

@@ -62,7 +62,7 @@ jobs:
           else
             version=$(grep '^set(ARROW_VERSION ' cpp/CMakeLists.txt | \
                         grep -E -o '[0-9]+\.[0-9]+\.[0-9]+')
-            rc_num=0
+            rc_num=999
             git config user.name "github-actions[bot]"
             git config user.email "github-actions[bot]@users.noreply.github.com"
             git tag \
@@ -89,14 +89,16 @@ jobs:
             "dev/release/utils-create-release-tarball.sh ${VERSION} ${RC_NUM}" \
             apache-arrow-${VERSION}.tar.gz
           dev/release/utils-create-release-tarball.sh ${VERSION} ${RC_NUM}
-          echo "RELEASE_TARBALL=apache-arrow-${VERSION}.tar.gz" >> ${GITHUB_ENV}
-          dev/release/utils-generate-checksum.sh "apache-arrow-${VERSION}.tar.gz"
+          RELEASE_TARBALL=apache-arrow-${VERSION}.tar.gz
+          echo "RELEASE_TARBALL=${RELEASE_TARBALL}" >> ${GITHUB_ENV}
+          dev/release/run-rat.sh "${RELEASE_TARBALL}"
+          dev/release/utils-generate-checksum.sh "${RELEASE_TARBALL}"
           if [ -n "${ARROW_GPG_SECRET_KEY}" ]; then
             echo "${ARROW_GPG_SECRET_KEY}" | gpg --import
             gpg \
               --armor \
               --detach-sign \
-              --local-user "${ARROW_GPG_SECRET_KEY}" \
+              --local-user "${ARROW_GPG_KEY_UID}" \
               --output "${RELEASE_TARBALL}.asc" \
               "${RELEASE_TARBALL}"
           fi

.github/workflows/verify_rc.yml

@@ -41,8 +41,9 @@ env:
 
 jobs:
   target:
+    name: Target
     runs-on: ubuntu-latest
-    timeout: 5
+    timeout-minutes: 5
     outputs:
       version: ${{ steps.detect.outputs.version }}
       rc: ${{ steps.detect.outputs.rc }}
@@ -54,10 +55,9 @@ jobs:
         run: |
           case "${GITHUB_EVENT_NAME}" in
             workflow_dispatch)
-              tag="${{ input.rc_tag }}"
+              tag="${{ inputs.rc_tag }}"
               ;;
             pull_request)
-              env | sort
               tag="$(gh release list \
                        --jq '.[] | select(.isPrerelease) | .tagName' \
                        --json tagName,isPrerelease \
@@ -75,8 +75,8 @@ jobs:
           echo "rc=${rc}" >> "${GITHUB_OUTPUT}"
 
   apt:
-    needs: target
     name: APT
+    needs: target
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 30
     strategy:
@@ -91,19 +91,41 @@ jobs:
       VERSION: ${{ needs.target.outputs.version }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
       - name: Run
         run: |
           dev/release/verify-release-candidate.sh ${VERSION} ${RC}
       - name: Verify the previous version
+        # TODO: We may re-enable this in the future.
+        # There are some problems for now:
+        # * We need to specify the previous versions for all
+        #   dependencies explicitly. For example, "apt install
+        #   libarrow-glib-dev=20.0.0-1" doesn't work. We need "apt
+        #   install libarrow-glib-dev=20.0.0-1
+        #   libarrow-acero-dev=20.0.0-1 libparquet-dev=20.0.0-1
+        #   gir1.2-arrow-1.0=20.0.0-1 libparquet-dev=20.0.0-1
+        #   libarrow-dev=20.0.0-1"
+        continue-on-error: true
         run: |
           major_version=${VERSION%%.*}
-          previous_major_version$((major_version - 1))
+          previous_major_version=$((major_version - 1))
           previous_version=${previous_major_version}.0.0
-          rc=0 # This number isn't used for APT verification
-          dev/release/verify-release-candidate.sh ${previous_version} ${rc}
+          previous_tag=apache-arrow-${previous_version}
+          git checkout ${previous_tag}
+          # This is workaround. dev/release/verify-release-candidate.sh
+          # in < 21.0.0 doesn't accept 20.0.0 as the VERSION argument.
+          # We can remove this workaround after 21.0.0 release.
+          sed \
+            -i \
+            -e 's/^\(ensure_source_directory\)$/# \1/' \
+            -e 's/^\(test_source_distribution\)$/# \1/' \
+            dev/release/verify-release-candidate.sh
+          dev/release/verify-release-candidate.sh ${previous_version}
 
   binary:
     name: Binary
+    needs: target
     runs-on: ubuntu-latest
     timeout-minutes: 30
     env:
@@ -118,6 +140,7 @@ jobs:
 
   wheels-linux:
     name: Wheels Linux
+    needs: target
     runs-on: ubuntu-latest
     timeout-minutes: 30
     strategy:
@@ -165,6 +188,7 @@ jobs:
 
   wheels-macos:
     name: Wheels macOS
+    needs: target
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 30
     strategy:
@@ -187,6 +211,7 @@ jobs:
 
   wheels-windows:
     name: Wheels Windows
+    needs: target
     runs-on: windows-latest
     timeout-minutes: 45
     env:
@@ -215,6 +240,7 @@ jobs:
 
   yum:
     name: Yum
+    needs: target
     runs-on: ${{ matrix.runs-on }}
     timeout-minutes: 30
     strategy:
@@ -229,13 +255,32 @@ jobs:
       VERSION: ${{ needs.target.outputs.version }}
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          fetch-depth: 0
       - name: Run
         run: |
           dev/release/verify-release-candidate.sh ${VERSION} ${RC}
       - name: Verify the previous version
+        # TODO: We may re-enable this in the future.
+        # There are some problems for now:
+        # * x86_64: libLLVM.so.18.1 needed by gandiva2000-libs on AlmaLinux 9
+        # * arm64: libarrow.so.2000.0.0: refers the
+        #   "std::condition_variable::wait(std::unique_lock<std::mutex>&)@GLIBCXX_3.4.30"
+        #   (not "...@@GLIBCXX_3.4.30" nor "...@GLIBCXX_3.4.11") symbol on
+        #   AmazonLinux 2023.
+        continue-on-error: true
         run: |
           major_version=${VERSION%%.*}
-          previous_major_version$((major_version - 1))
+          previous_major_version=$((major_version - 1))
           previous_version=${previous_major_version}.0.0
-          rc=0 # This number isn't used for Yum verification
-          dev/release/verify-release-candidate.sh ${previous_version} ${rc}
+          previous_tag=apache-arrow-${previous_version}
+          git checkout ${previous_tag}
+          # This is workaround. dev/release/verify-release-candidate.sh
+          # in < 21.0.0 doesn't accept 20.0.0 as the VERSION argument.
+          # We can remove this workaround after 21.0.0 release.
+          sed \
+            -i \
+            -e 's/^\(ensure_source_directory\)$/# \1/' \
+            -e 's/^\(test_source_distribution\)$/# \1/' \
+            dev/release/verify-release-candidate.sh
+          dev/release/verify-release-candidate.sh ${previous_version}

.pre-commit-config.yaml

@@ -329,6 +329,7 @@ repos:
           ?^ci/scripts/install_spark\.sh$|
           ?^ci/scripts/install_vcpkg\.sh$|
           ?^ci/scripts/integration_arrow_build\.sh$|
+          ?^ci/scripts/integration_arrow\.sh$|
           ?^ci/scripts/integration_dask\.sh$|
           ?^ci/scripts/integration_spark\.sh$|
           ?^ci/scripts/matlab_build\.sh$|

ci/docker/conda-cpp.dockerfile

@@ -27,6 +27,7 @@ RUN /arrow/ci/scripts/install_minio.sh latest /opt/conda
 ARG python=3.10
 
 # install the required conda packages into the test environment
+# use `mold` to work around issues with GNU `ld` (GH-47015).
 COPY ci/conda_env_cpp.txt \
      ci/conda_env_gandiva.txt \
      /arrow/ci/
@@ -36,6 +37,7 @@ RUN mamba install -q -y \
         compilers \
         doxygen \
         libnuma \
+        mold \
         python=${python} \
         valgrind && \
     mamba clean --all --yes
@@ -73,6 +75,7 @@ ENV ARROW_ACERO=ON \
     ARROW_S3_MODULE=ON \
     ARROW_SUBSTRAIT=ON \
     ARROW_USE_CCACHE=ON \
+    ARROW_USE_MOLD=ON \
     ARROW_WITH_BROTLI=ON \
     ARROW_WITH_BZ2=ON \
     ARROW_WITH_LZ4=ON \

ci/docker/fedora-39-cpp.dockerfile ci/docker/fedora-42-cpp.dockerfileci/docker/fedora-39-cpp.dockerfile renamed to ci/docker/fedora-42-cpp.dockerfile

@@ -16,7 +16,7 @@
 # under the License.
 
 ARG arch
-FROM ${arch}/fedora:39
+FROM ${arch}/fedora:42
 ARG arch
 
 # install dependencies

ci/scripts/cpp_test.sh

@@ -126,6 +126,9 @@ fi
 
 if [ "${ARROW_FUZZING}" == "ON" ]; then
     # Fuzzing regression tests
+    # Some fuzz regression files may trigger huge memory allocations,
+    # let the allocator return null instead of aborting.
+    export ASAN_OPTIONS="$ASAN_OPTIONS allocator_may_return_null=1"
     "${binary_output_dir}/arrow-ipc-stream-fuzz" "${ARROW_TEST_DATA}"/arrow-ipc-stream/crash-*
     "${binary_output_dir}/arrow-ipc-stream-fuzz" "${ARROW_TEST_DATA}"/arrow-ipc-stream/*-testcase-*
     "${binary_output_dir}/arrow-ipc-file-fuzz" "${ARROW_TEST_DATA}"/arrow-ipc-file/*-testcase-*

ci/scripts/install_gcs_testbench.bat

@@ -17,7 +17,7 @@
 
 @echo on
 
-set GCS_TESTBENCH_VERSION="v0.40.0"
+set GCS_TESTBENCH_VERSION="v0.55.0"
 
 set PIPX_FLAGS=--verbose
 if NOT "%PIPX_PYTHON%"=="" (

ci/scripts/install_gcs_testbench.sh

@@ -36,7 +36,7 @@ esac
 
 version=$1
 if [[ "${version}" = "default" ]]; then
-  version="v0.39.0"
+  version="v0.55.0"
 fi
 
 # The Python to install pipx with

ci/scripts/integration_arrow.sh

@@ -24,16 +24,16 @@ build_dir=${2}
 
 gold_dir=$arrow_dir/testing/data/arrow-ipc-stream/integration
 
-: ${ARROW_INTEGRATION_CPP:=ON}
-: ${ARROW_INTEGRATION_CSHARP:=ON}
+: "${ARROW_INTEGRATION_CPP:=ON}"
+: "${ARROW_INTEGRATION_CSHARP:=ON}"
 
-: ${ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS:=cpp,csharp}
+: "${ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS:=cpp,csharp}"
 export ARCHERY_INTEGRATION_TARGET_IMPLEMENTATIONS
 
-. ${arrow_dir}/ci/scripts/util_log.sh
+. "${arrow_dir}/ci/scripts/util_log.sh"
 
 github_actions_group_begin "Integration: Prepare: Archery"
-pip install -e $arrow_dir/dev/archery[integration]
+pip install -e "$arrow_dir/dev/archery[integration]"
 github_actions_group_end
 
 github_actions_group_begin "Integration: Prepare: Dependencies"
@@ -58,16 +58,19 @@ export PYTHONFAULTHANDLER=1
 export GOMEMLIMIT=200MiB
 export GODEBUG=gctrace=1,clobberfree=1
 
+ARCHERY_WITH_CPP=$([ "$ARROW_INTEGRATION_CPP" == "ON" ] && echo "1" || echo "0")
+ARCHERY_WITH_CSHARP=$([ "$ARROW_INTEGRATION_CSHARP" == "ON" ] && echo "1" || echo "0")
+
 # Rust can be enabled by exporting ARCHERY_INTEGRATION_WITH_RUST=1
 time archery integration \
     --run-c-data \
     --run-ipc \
     --run-flight \
-    --with-cpp=$([ "$ARROW_INTEGRATION_CPP" == "ON" ] && echo "1" || echo "0") \
-    --with-csharp=$([ "$ARROW_INTEGRATION_CSHARP" == "ON" ] && echo "1" || echo "0") \
-    --gold-dirs=$gold_dir/0.14.1 \
-    --gold-dirs=$gold_dir/0.17.1 \
-    --gold-dirs=$gold_dir/1.0.0-bigendian \
-    --gold-dirs=$gold_dir/1.0.0-littleendian \
-    --gold-dirs=$gold_dir/2.0.0-compression \
-    --gold-dirs=$gold_dir/4.0.0-shareddict \
+    --with-cpp="${ARCHERY_WITH_CPP}" \
+    --with-csharp="${ARCHERY_WITH_CSHARP}"\
+    --gold-dirs="$gold_dir/0.14.1" \
+    --gold-dirs="$gold_dir/0.17.1" \
+    --gold-dirs="$gold_dir/1.0.0-bigendian" \
+    --gold-dirs="$gold_dir/1.0.0-littleendian" \
+    --gold-dirs="$gold_dir/2.0.0-compression" \
+    --gold-dirs="$gold_dir/4.0.0-shareddict" \