Merge branch 'BackpressureCombiner' into combined2

Author: Rafał Hibner

.pre-commit-config.yaml

@@ -341,6 +341,7 @@ repos:
           ?^ci/scripts/python_build_emscripten\.sh$|
           ?^ci/scripts/python_build\.sh$|
           ?^ci/scripts/python_sdist_build\.sh$|
+          ?^ci/scripts/python_sdist_test\.sh$|
           ?^ci/scripts/python_wheel_unix_test\.sh$|
           ?^ci/scripts/r_build\.sh$|
           ?^ci/scripts/r_revdepcheck\.sh$|

ci/scripts/cpp_build.sh

@@ -221,7 +221,6 @@ else
     -DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \
     -DARROW_S3=${ARROW_S3:-OFF} \
     -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL:-DEFAULT} \
-    -DARROW_SKYHOOK=${ARROW_SKYHOOK:-OFF} \
     -DARROW_SUBSTRAIT=${ARROW_SUBSTRAIT:-OFF} \
     -DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \
     -DARROW_TEST_MEMCHECK=${ARROW_TEST_MEMCHECK:-OFF} \

ci/scripts/generate_dataset.py

@@ -53,13 +53,17 @@ fi
 if [ -n "${PYARROW_VERSION:-}" ]; then
   sdist="${arrow_dir}/python/dist/pyarrow-${PYARROW_VERSION}.tar.gz"
 else
-  sdist=$(ls ${arrow_dir}/python/dist/pyarrow-*.tar.gz | sort -r | head -n1)
+  sdist=$(echo "${arrow_dir}"/python/dist/pyarrow-*.tar.gz | sort -r | head -n1)
 fi
 
 if [ -n "${ARROW_PYTHON_VENV:-}" ]; then
+  # We don't need to follow this external file.
+  # See also: https://www.shellcheck.net/wiki/SC1091
+  #
+  # shellcheck source=/dev/null
   . "${ARROW_PYTHON_VENV}/bin/activate"
 fi
 
-${PYTHON:-python} -m pip install ${sdist}
+${PYTHON:-python} -m pip install "${sdist}"
 
-pytest -r s ${PYTEST_ARGS:-} --pyargs pyarrow
+pytest -r s "${PYTEST_ARGS:-}" --pyargs pyarrow

ci/scripts/integration_skyhook.sh

@@ -706,10 +706,6 @@ if(ARROW_GANDIVA)
   add_subdirectory(src/gandiva)
 endif()
 
-if(ARROW_SKYHOOK)
-  add_subdirectory(src/skyhook)
-endif()
-
 if(ARROW_BUILD_EXAMPLES)
   add_custom_target(runexample ctest -L example)
   add_subdirectory(examples/arrow)

ci/scripts/python_sdist_test.sh

@@ -179,7 +179,6 @@
         "ARROW_BUILD_EXAMPLES": "ON",
         "ARROW_BUILD_UTILITIES": "ON",
         "ARROW_FLIGHT_SQL_ODBC": "ON",
-        "ARROW_SKYHOOK": "ON",
         "ARROW_TENSORFLOW": "ON",
         "PARQUET_BUILD_EXAMPLES": "ON",
         "PARQUET_BUILD_EXECUTABLES": "ON"

cpp/CMakeLists.txt

@@ -34,7 +34,3 @@ FILES=($(find $FORMAT_DIR -name '*.fbs'))
 FILES+=("$SOURCE_DIR/arrow/ipc/feather.fbs")
 
 $FLATC -o "$OUT_DIR" "${FILES[@]}"
-
-# Skyhook flatbuffers
-$FLATC -o "$SOURCE_DIR/skyhook/protocol" \
-  "$SOURCE_DIR/skyhook/protocol/ScanRequest.fbs"

cpp/CMakePresets.json

@@ -407,15 +407,6 @@ takes precedence over ccache if a storage backend is configured" ON)
                 DEPENDS
                 ARROW_S3)
 
-  define_option(ARROW_SKYHOOK
-                "Build the Skyhook libraries"
-                OFF
-                DEPENDS
-                ARROW_DATASET
-                ARROW_PARQUET
-                ARROW_WITH_LZ4
-                ARROW_WITH_SNAPPY)
-
   define_option(ARROW_SUBSTRAIT
                 "Build the Arrow Substrait Consumer Module"
                 OFF

cpp/build-support/update-flatbuffers.sh

@@ -188,19 +188,6 @@ if(ARROW_PARQUET AND ARROW_DATASET)
   endif()
 
   add_arrow_example(udf_example)
-
-  if(ARROW_SKYHOOK)
-    if(ARROW_BUILD_SHARED)
-      list(APPEND DATASET_EXAMPLES_LINK_LIBS arrow_skyhook_shared)
-    else()
-      list(APPEND DATASET_EXAMPLES_LINK_LIBS arrow_skyhook_static)
-    endif()
-
-    add_arrow_example(dataset_skyhook_scan_example EXTRA_LINK_LIBS
-                      ${DATASET_EXAMPLES_LINK_LIBS})
-    add_dependencies(dataset-skyhook-scan-example parquet)
-  endif()
-
 endif()
 
 if(ARROW_GANDIVA)