diff --git a/ci/scripts/python_build.bat b/ci/scripts/python_build.bat index 78392b9eea10..7f9358494cf2 100644 --- a/ci/scripts/python_build.bat +++ b/ci/scripts/python_build.bat @@ -168,7 +168,7 @@ if %ARROW_PARQUET% == ON ( ) if %PARQUET_REQUIRE_ENCRYPTION% == ON ( set PYARROW_WITH_PARQUET_ENCRYPTION=enabled -) else if %ARROW_ACERO% == OFF ( +) else if %PARQUET_REQUIRE_ENCRYPTION% == OFF ( set PYARROW_WITH_PARQUET_ENCRYPTION=disabled ) else ( set PYARROW_WITH_PARQUET_ENCRYPTION=auto diff --git a/ci/scripts/python_sdist_build.sh b/ci/scripts/python_sdist_build.sh index 7bea9c3dc10c..5979cbac2e4b 100755 --- a/ci/scripts/python_sdist_build.sh +++ b/ci/scripts/python_sdist_build.sh @@ -23,9 +23,8 @@ source_dir=${1}/python pushd "${source_dir}" export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-} -# Meson dist must be run from a VCS, so initiate a dummy repo -git init . -git add --all . -git commit -m "dummy commit for meson dist" -${PYTHON:-python} -m build --sdist . +# The mounted source directory is already a git repository, so mark it as safe +git config --global --add safe.directory "${1}" +${PYTHON:-python} -m pip install build +${PYTHON:-python} -m build --sdist . -Csetup-args="-Dsdist=true" popd diff --git a/ci/scripts/python_wheel_macos_build.sh b/ci/scripts/python_wheel_macos_build.sh index bd61154430e0..7df0d2f3aca5 100755 --- a/ci/scripts/python_wheel_macos_build.sh +++ b/ci/scripts/python_wheel_macos_build.sh @@ -156,38 +156,110 @@ popd echo "=== (${PYTHON_VERSION}) Building wheel ===" export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE} export PYARROW_BUNDLE_ARROW_CPP=1 -export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR} -export PYARROW_WITH_ACERO=${ARROW_ACERO} -export PYARROW_WITH_AZURE=${ARROW_AZURE} -export PYARROW_WITH_DATASET=${ARROW_DATASET} -export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT} -export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA} -export PYARROW_WITH_GCS=${ARROW_GCS} -export PYARROW_WITH_HDFS=${ARROW_HDFS} -export PYARROW_WITH_ORC=${ARROW_ORC} -export PYARROW_WITH_PARQUET=${ARROW_PARQUET} -export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION} -export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT} -export PYARROW_WITH_S3=${ARROW_S3} -export PYARROW_CMAKE_OPTIONS="-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL}" + +PYARROW_WITH_ACERO=$(case "$ARROW_ACERO" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_AZURE=$(case "$ARROW_AZURE" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +# CUDA support for wheels should be disabled? +PYARROW_WITH_CUDA=$(case "$ARROW_CUDA" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_DATASET=$(case "$ARROW_DATASET" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "enabled" ;; + esac) +PYARROW_WITH_FLIGHT=$(case "$ARROW_FLIGHT" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_GANDIVA=$(case "$ARROW_GANDIVA" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_GCS=$(case "$ARROW_GCS" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_HDFS=$(case "$ARROW_HDFS" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "enabled" ;; + esac) +PYARROW_WITH_ORC=$(case "$ARROW_ORC" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_PARQUET=$(case "$ARROW_PARQUET" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_PARQUET_ENCRYPTION=$(case "$PARQUET_REQUIRE_ENCRYPTION" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "enabled" ;; + esac) +PYARROW_WITH_S3=$(case "$ARROW_S3" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_SUBSTRAIT=$(case "$ARROW_SUBSTRAIT" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) + export ARROW_HOME=${build_dir}/install -# PyArrow build configuration -export CMAKE_PREFIX_PATH=${build_dir}/install # Set PyArrow version explicitly export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION} +# Meson sdist requires setuptools_scm to be able to get the version from git +git config --global --add safe.directory ${source_dir} + pushd ${source_dir}/python -python setup.py bdist_wheel +python -m build --sdist --wheel . \ + -Csetup-args="-Dbuildtype=${PYARROW_BUILD_TYPE}" \ + -Csetup-args="-Dacero=${PYARROW_WITH_ACERO}" \ + -Csetup-args="-Dazure=${PYARROW_WITH_AZURE}" \ + -Csetup-args="-Dcuda=${PYARROW_WITH_CUDA}" \ + -Csetup-args="-Ddataset=${PYARROW_WITH_DATASET}" \ + -Csetup-args="-Dflight=${PYARROW_WITH_FLIGHT}" \ + -Csetup-args="-Dgandiva=${PYARROW_WITH_GANDIVA}" \ + -Csetup-args="-Dgcs=${PYARROW_WITH_GCS}" \ + -Csetup-args="-Dhdfs=${PYARROW_WITH_HDFS}" \ + -Csetup-args="-Dorc=${PYARROW_WITH_ORC}" \ + -Csetup-args="-Dparquet=${PYARROW_WITH_PARQUET}" \ + -Csetup-args="-Dparquet_require_encryption=${PYARROW_WITH_PARQUET_ENCRYPTION}" \ + -Csetup-args="-Ds3=${PYARROW_WITH_S3}" \ + -Csetup-args="-Dsubstrait=${PYARROW_WITH_SUBSTRAIT}" \ + -Ccompile-args="-v" \ + -Csetup-args="--pkg-config-path=${ARROW_HOME}/lib/pkgconfig" popd echo "=== (${PYTHON_VERSION}) Show dynamic libraries the wheel depend on ===" -deps=$(delocate-listdeps ${source_dir}/python/dist/*.whl) +delocate-listdeps ${source_dir}/python/dist/*.whl -if echo $deps | grep -v "^pyarrow/lib\(arrow\|gandiva\|parquet\)"; then +echo "=== (${PYTHON_VERSION}) Bundle shared libraries into wheel ===" +delocate-wheel -w ${source_dir}/python/repaired_wheels -v ${source_dir}/python/dist/*.whl + +echo "=== (${PYTHON_VERSION}) Validate there are no non-bundled shared dependencies ===" +deps=$(delocate-listdeps ${source_dir}/python/repaired_wheels/*.whl) +if echo $deps | grep -v "^pyarrow/.dylibs/lib\(arrow\|gandiva\|parquet\)"; then echo "There are non-bundled shared library dependencies." exit 1 fi - -# Move the verified wheels -mkdir -p ${source_dir}/python/repaired_wheels -mv ${source_dir}/python/dist/*.whl ${source_dir}/python/repaired_wheels/ diff --git a/ci/scripts/python_wheel_windows_build.bat b/ci/scripts/python_wheel_windows_build.bat index b4b7fed99fd4..486acd883ffb 100644 --- a/ci/scripts/python_wheel_windows_build.bat +++ b/ci/scripts/python_wheel_windows_build.bat @@ -114,26 +114,109 @@ echo "=== (%PYTHON%) Building wheel ===" set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE% set PYARROW_BUILD_VERBOSE=1 set PYARROW_BUNDLE_ARROW_CPP=ON -set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR% -set PYARROW_CMAKE_OPTIONS="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION%" -set PYARROW_WITH_ACERO=%ARROW_ACERO% -set PYARROW_WITH_DATASET=%ARROW_DATASET% -set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT% -set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA% -set PYARROW_WITH_GCS=%ARROW_GCS% -set PYARROW_WITH_HDFS=%ARROW_HDFS% -set PYARROW_WITH_ORC=%ARROW_ORC% -set PYARROW_WITH_PARQUET=%ARROW_PARQUET% -set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION% -set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT% -set PYARROW_WITH_S3=%ARROW_S3% -set ARROW_HOME=C:\arrow-dist -set CMAKE_PREFIX_PATH=C:\arrow-dist + +if %ARROW_ACERO% == ON ( + set PYARROW_WITH_ACERO=enabled +) else if %ARROW_ACERO% == OFF ( + set PYARROW_WITH_ACERO=disabled +) else ( + set PYARROW_WITH_ACERO=auto +) +if %ARROW_DATASET% == ON ( + set PYARROW_WITH_DATASET=enabled +) else if %ARROW_DATASET% == OFF ( + set PYARROW_WITH_DATASET=disabled +) else ( + set PYARROW_WITH_DATASET=auto +) +if %ARROW_FLIGHT% == ON ( + set PYARROW_WITH_FLIGHT=enabled +) else if %ARROW_FLIGHT% == OFF ( + set PYARROW_WITH_FLIGHT=disabled +) else ( + set PYARROW_WITH_FLIGHT=auto +) +if %ARROW_GANDIVA% == ON ( + set PYARROW_WITH_GANDIVA=enabled +) else if %ARROW_GANDIVA% == OFF ( + set PYARROW_WITH_GANDIVA=disabled +) else ( + set PYARROW_WITH_GANDIVA=auto +) +if %ARROW_GCS% == ON ( + set PYARROW_WITH_GCS=enabled +) else if %ARROW_GCS% == OFF ( + set PYARROW_WITH_GCS=disabled +) else ( + set PYARROW_WITH_GCS=auto +) +if %ARROW_HDFS% == ON ( + set PYARROW_WITH_HDFS=enabled +) else if %ARROW_HDFS% == OFF ( + set PYARROW_WITH_HDFS=disabled +) else ( + set PYARROW_WITH_HDFS=auto +) +if %ARROW_ORC% == ON ( + set PYARROW_WITH_ORC=enabled +) else if %ARROW_ORC% == OFF ( + set PYARROW_WITH_ORC=disabled +) else ( + set PYARROW_WITH_ORC=auto +) +if %ARROW_PARQUET% == ON ( + set PYARROW_WITH_PARQUET=enabled +) else if %ARROW_PARQUET% == OFF ( + set PYARROW_WITH_PARQUET=disabled +) else ( + set PYARROW_WITH_PARQUET=auto +) +if %PARQUET_REQUIRE_ENCRYPTION% == ON ( + set PYARROW_WITH_PARQUET_ENCRYPTION=enabled +) else if %PARQUET_REQUIRE_ENCRYPTION% == OFF ( + set PYARROW_WITH_PARQUET_ENCRYPTION=disabled +) else ( + set PYARROW_WITH_PARQUET_ENCRYPTION=auto +) +if %ARROW_SUBSTRAIT% == ON ( + set PYARROW_WITH_SUBSTRAIT=enabled +) else if %ARROW_SUBSTRAIT% == OFF ( + set PYARROW_WITH_SUBSTRAIT=disabled +) else ( + set PYARROW_WITH_SUBSTRAIT=auto +) +if %ARROW_S3% == ON ( + set PYARROW_WITH_S3=enabled +) else if %ARROW_S3% == OFF ( + set PYARROW_WITH_S3=disabled +) else ( + set PYARROW_WITH_S3=auto +) + +@REM Meson sdist requires setuptools_scm to be able to get the version from git +git config --global --add safe.directory C:\arrow pushd C:\arrow\python +@REM TODO: Remove once docker rebuild works correctly +@REM See: https://github.com/apache/arrow/issues/48947 +%PYTHON_CMD% -m pip install -U build || exit /B 1 + @REM Build wheel -%PYTHON_CMD% setup.py bdist_wheel || exit /B 1 +%PYTHON_CMD% -m build --sdist --wheel . ^ + -Csetup-args="-Dbuildtype=%CMAKE_BUILD_TYPE%" ^ + -Csetup-args="-Dacero=%PYARROW_WITH_ACERO%" ^ + -Csetup-args="-Ddataset=%PYARROW_WITH_DATASET%" ^ + -Csetup-args="-Dflight=%PYARROW_WITH_FLIGHT%" ^ + -Csetup-args="-Dgandiva=%PYARROW_WITH_GANDIVA%" ^ + -Csetup-args="-Dgcs=%PYARROW_WITH_GCS%" ^ + -Csetup-args="-Dhdfs=%PYARROW_WITH_HDFS%" ^ + -Csetup-args="-Dorc=%PYARROW_WITH_ORC%" ^ + -Csetup-args="-Dparquet=%PYARROW_WITH_PARQUET%" ^ + -Csetup-args="-Dparquet_require_encryption=%PYARROW_WITH_PARQUET_ENCRYPTION%" ^ + -Csetup-args="-Dsubstrait=%PYARROW_WITH_SUBSTRAIT%" ^ + -Csetup-args="-Ds3=%PYARROW_WITH_S3%" ^ + -Csetup-args="--cmake-prefix-path=C:\arrow-dist" || exit /B 1 @REM Repair the wheel with delvewheel @REM @@ -142,11 +225,16 @@ pushd C:\arrow\python @REM required by multiple Python libraries in the same process. %PYTHON_CMD% -m pip install delvewheel || exit /B 1 +@REM Copy .lib files to bin directory for delvewheel to find them +copy C:\arrow-dist\lib\*.lib C:\arrow-dist\bin\ || exit /B 1 + for /f %%i in ('dir dist\pyarrow-*.whl /B') do (set WHEEL_NAME=%cd%\dist\%%i) || exit /B 1 echo "Wheel name: %WHEEL_NAME%" %PYTHON_CMD% -m delvewheel repair -vv ^ - --ignore-existing --with-mangle ^ + --ignore-existing --with-mangle --include-imports ^ + --no-mangle "arrow.dll;arrow_python.dll;arrow_acero.dll;arrow_dataset.dll;arrow_flight.dll;arrow_substrait.dll;parquet.dll" ^ + --add-path "C:\arrow-dist\bin" ^ -w repaired_wheels %WHEEL_NAME% || exit /B 1 popd diff --git a/ci/scripts/python_wheel_xlinux_build.sh b/ci/scripts/python_wheel_xlinux_build.sh index a3fbeb3c0b3c..5608441ad20e 100755 --- a/ci/scripts/python_wheel_xlinux_build.sh +++ b/ci/scripts/python_wheel_xlinux_build.sh @@ -148,26 +148,98 @@ check_arrow_visibility echo "=== (${PYTHON_VERSION}) Building wheel ===" export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE} export PYARROW_BUNDLE_ARROW_CPP=1 -export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR} -export PYARROW_CMAKE_OPTIONS="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=${CMAKE_INTERPROCEDURAL_OPTIMIZATION}" -export PYARROW_WITH_ACERO=${ARROW_ACERO} -export PYARROW_WITH_AZURE=${ARROW_AZURE} -export PYARROW_WITH_DATASET=${ARROW_DATASET} -export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT} -export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA} -export PYARROW_WITH_GCS=${ARROW_GCS} -export PYARROW_WITH_HDFS=${ARROW_HDFS} -export PYARROW_WITH_ORC=${ARROW_ORC} -export PYARROW_WITH_PARQUET=${ARROW_PARQUET} -export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION} -export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT} -export PYARROW_WITH_S3=${ARROW_S3} + +PYARROW_WITH_ACERO=$(case "$ARROW_ACERO" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_AZURE=$(case "$ARROW_AZURE" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +# CUDA support for wheels should be disabled? +PYARROW_WITH_CUDA=$(case "$ARROW_CUDA" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_DATASET=$(case "$ARROW_DATASET" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "enabled" ;; + esac) +PYARROW_WITH_FLIGHT=$(case "$ARROW_FLIGHT" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_GANDIVA=$(case "$ARROW_GANDIVA" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_GCS=$(case "$ARROW_GCS" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_HDFS=$(case "$ARROW_HDFS" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "enabled" ;; + esac) +PYARROW_WITH_ORC=$(case "$ARROW_ORC" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_PARQUET=$(case "$ARROW_PARQUET" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_PARQUET_ENCRYPTION=$(case "$PARQUET_REQUIRE_ENCRYPTION" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "enabled" ;; + esac) +PYARROW_WITH_S3=$(case "$ARROW_S3" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) +PYARROW_WITH_SUBSTRAIT=$(case "$ARROW_SUBSTRAIT" in + ON) echo "enabled" ;; + OFF) echo "disabled" ;; + *) echo "auto" ;; + esac) export ARROW_HOME=/tmp/arrow-dist # PyArrow build configuration export CMAKE_PREFIX_PATH=/tmp/arrow-dist +# Meson sdist requires setuptools_scm to be able to get the version from git +git config --global --add safe.directory /arrow + pushd /arrow/python -python setup.py bdist_wheel +python -m build --sdist --wheel . \ + -Csetup-args="-Dbuildtype=${PYARROW_BUILD_TYPE}" \ + -Csetup-args="-Dacero=${PYARROW_WITH_ACERO}" \ + -Csetup-args="-Dazure=${PYARROW_WITH_AZURE}" \ + -Csetup-args="-Dcuda=${PYARROW_WITH_CUDA}" \ + -Csetup-args="-Ddataset=${PYARROW_WITH_DATASET}" \ + -Csetup-args="-Dflight=${PYARROW_WITH_FLIGHT}" \ + -Csetup-args="-Dgandiva=${PYARROW_WITH_GANDIVA}" \ + -Csetup-args="-Dgcs=${PYARROW_WITH_GCS}" \ + -Csetup-args="-Dhdfs=${PYARROW_WITH_HDFS}" \ + -Csetup-args="-Dorc=${PYARROW_WITH_ORC}" \ + -Csetup-args="-Dparquet=${PYARROW_WITH_PARQUET}" \ + -Csetup-args="-Dparquet_require_encryption=${PYARROW_WITH_PARQUET_ENCRYPTION}" \ + -Csetup-args="-Ds3=${PYARROW_WITH_S3}" \ + -Csetup-args="-Dsubstrait=${PYARROW_WITH_SUBSTRAIT}" \ + -Ccompile-args="-v" \ + -Csetup-args="--pkg-config-path=${ARROW_HOME}/lib/pkgconfig" echo "=== Strip symbols from wheel ===" mkdir -p dist/temp-fix-wheel @@ -189,6 +261,12 @@ popd rm -rf dist/temp-fix-wheel +echo "=== (${PYTHON_VERSION}) Show wheel details before repairing ===" +auditwheel -v show dist/pyarrow-*.whl + echo "=== (${PYTHON_VERSION}) Tag the wheel with ${LINUX_WHEEL_KIND}${LINUX_WHEEL_VERSION} ===" auditwheel repair dist/pyarrow-*.whl -w repaired_wheels + +echo "=== (${PYTHON_VERSION}) Show wheel details after repairing ===" +auditwheel -v show repaired_wheels/pyarrow-*.whl popd diff --git a/python/meson.build b/python/meson.build index c65a83786839..4c2c41dbc891 100644 --- a/python/meson.build +++ b/python/meson.build @@ -18,7 +18,6 @@ project( 'pyarrow', 'cython', - 'cpp', version: run_command( 'python', '-m', @@ -32,6 +31,11 @@ project( default_options: ['buildtype=release', 'cpp_std=c++20'], ) +if not get_option('sdist') + add_languages('cpp', native: false) + cc = meson.get_compiler('cpp') +endif + py = import('python').find_installation(pure: false) install_data('NOTICE.txt', install_dir: py.get_install_dir() / 'pyarrow') @@ -86,6 +90,4 @@ print(incdir) numpy_dep = declare_dependency(include_directories: incdir_numpy) endif -cc = meson.get_compiler('cpp') - subdir('pyarrow') diff --git a/python/pyarrow/meson.build b/python/pyarrow/meson.build index 9b0a81e1e63f..d51d2c3754e5 100644 --- a/python/pyarrow/meson.build +++ b/python/pyarrow/meson.build @@ -215,136 +215,144 @@ if needs_parquet endif endif -gnu_symbol_visibility = host_machine.system() == 'darwin' ? 'default' : 'inlineshidden' +# Avoid building Pyarrow if it is a source distribution. +if not get_option('sdist') -if get_option('default_library') == 'static' - pyarrow_private_args = ['-DARROW_PYTHON_STATIC'] - pyarrow_public_args = ['-DARROW_PYTHON_STATIC'] -else - pyarrow_private_args = ['-DARROW_PYTHON_EXPORTING'] - pyarrow_public_args = [] -endif + gnu_symbol_visibility = host_machine.system() == 'darwin' ? 'default' : 'inlineshidden' -pyarrow_lib = library( - 'arrow_python', - sources: pyarrow_srcs, - include_directories: ['src'], - dependencies: libpyarrow_deps + [ - numpy_dep, - cython_generated_dep, - py.dependency(), - ], - cpp_args: pyarrow_private_args, - install: true, - install_dir: py.get_install_dir() / 'pyarrow', - gnu_symbol_visibility: gnu_symbol_visibility, - override_options: ['b_lundef=false'], -) -pyarrow_lib_dep = declare_dependency( - link_with: [pyarrow_lib], - compile_args: pyarrow_public_args, -) - -if needs_flight if get_option('default_library') == 'static' - pyarrow_flight_private_args = ['-DARROW_PYFLIGHT_STATIC'] - pyarrow_flight_public_args = ['-DARROW_PYFLIGHT_STATIC'] + pyarrow_private_args = ['-DARROW_PYTHON_STATIC'] + pyarrow_public_args = ['-DARROW_PYTHON_STATIC'] else - pyarrow_flight_private_args = ['-DARROW_PYFLIGHT_EXPORTING'] - pyarrow_flight_public_args = [] + pyarrow_private_args = ['-DARROW_PYTHON_EXPORTING'] + pyarrow_public_args = [] endif - pyarrow_flight_lib = library( - 'arrow_flight_lib', - sources: ['src/arrow/python/flight.cc'], - dependencies: [arrow_flight_dep, pyarrow_lib_dep, py.dependency()], + pyarrow_lib = library( + 'arrow_python', + sources: pyarrow_srcs, include_directories: ['src'], - cpp_args: pyarrow_flight_private_args, + dependencies: libpyarrow_deps + [ + numpy_dep, + cython_generated_dep, + py.dependency(), + ], + cpp_args: pyarrow_private_args, install: true, install_dir: py.get_install_dir() / 'pyarrow', gnu_symbol_visibility: gnu_symbol_visibility, override_options: ['b_lundef=false'], ) - - pyarrow_flight_dep = declare_dependency( - link_with: [pyarrow_flight_lib], - dependencies: [arrow_flight_dep], - compile_args: pyarrow_flight_public_args, + pyarrow_lib_dep = declare_dependency( + link_with: [pyarrow_lib], + compile_args: pyarrow_public_args, ) - cython_modules += {'_flight': {'dependencies': [pyarrow_flight_dep]}} -endif -if needs_parquet_encryption - if get_option('default_library') == 'static' - pyarrow_pq_enc_private_args = [ - '-DARROW_PYTHON_PARQUET_ENCRYPTION_STATIC', - ] - pyarrow_pq_enc_public_args = [ - '-DARROW_PYTHON_PARQUET_ENCRYPTION_STATIC', - ] - else - pyarrow_pq_enc_private_args = [ - '-DARROW_PYTHON_PARQUET_ENCRYPTION_EXPORTING', - ] - pyarrow_pq_enc_public_args = [] - endif + if needs_flight + if get_option('default_library') == 'static' + pyarrow_flight_private_args = ['-DARROW_PYFLIGHT_STATIC'] + pyarrow_flight_public_args = ['-DARROW_PYFLIGHT_STATIC'] + else + pyarrow_flight_private_args = ['-DARROW_PYFLIGHT_EXPORTING'] + pyarrow_flight_public_args = [] + endif + + pyarrow_flight_lib = library( + 'arrow_flight_lib', + sources: ['src/arrow/python/flight.cc'], + dependencies: [arrow_flight_dep, pyarrow_lib_dep, py.dependency()], + include_directories: ['src'], + cpp_args: pyarrow_flight_private_args, + install: true, + install_dir: py.get_install_dir() / 'pyarrow', + gnu_symbol_visibility: gnu_symbol_visibility, + override_options: ['b_lundef=false'], + ) - pyarrow_encryption_lib = library( - 'arrow_python_parquet_encryption', - sources: ['src/arrow/python/parquet_encryption.cc'], - include_directories: ['src'], - dependencies: [parquet_dep, pyarrow_lib_dep, py.dependency()], - cpp_args: pyarrow_pq_enc_private_args, - install: true, - install_dir: py.get_install_dir() / 'pyarrow', - gnu_symbol_visibility: gnu_symbol_visibility, - override_options: ['b_lundef=false'], - ) + pyarrow_flight_dep = declare_dependency( + link_with: [pyarrow_flight_lib], + dependencies: [arrow_flight_dep], + compile_args: pyarrow_flight_public_args, + ) + cython_modules += {'_flight': {'dependencies': [pyarrow_flight_dep]}} + endif - pyarrow_parquet_encryption_dep = declare_dependency( - link_with: [pyarrow_encryption_lib], - dependencies: [parquet_dep], - compile_args: pyarrow_pq_enc_public_args, - ) + if needs_parquet_encryption + if get_option('default_library') == 'static' + pyarrow_pq_enc_private_args = [ + '-DARROW_PYTHON_PARQUET_ENCRYPTION_STATIC', + ] + pyarrow_pq_enc_public_args = [ + '-DARROW_PYTHON_PARQUET_ENCRYPTION_STATIC', + ] + else + pyarrow_pq_enc_private_args = [ + '-DARROW_PYTHON_PARQUET_ENCRYPTION_EXPORTING', + ] + pyarrow_pq_enc_public_args = [] + endif + + pyarrow_encryption_lib = library( + 'arrow_python_parquet_encryption', + sources: ['src/arrow/python/parquet_encryption.cc'], + include_directories: ['src'], + dependencies: [parquet_dep, pyarrow_lib_dep, py.dependency()], + cpp_args: pyarrow_pq_enc_private_args, + install: true, + install_dir: py.get_install_dir() / 'pyarrow', + gnu_symbol_visibility: gnu_symbol_visibility, + override_options: ['b_lundef=false'], + ) - cython_modules += { - '_parquet_encryption': { - 'dependencies': [pyarrow_parquet_encryption_dep], - }, - } + pyarrow_parquet_encryption_dep = declare_dependency( + link_with: [pyarrow_encryption_lib], + dependencies: [parquet_dep], + compile_args: pyarrow_pq_enc_public_args, + ) - if needs_dataset cython_modules += { - '_dataset_parquet_encryption': { - 'dependencies': [ - arrow_dataset_dep, - pyarrow_parquet_encryption_dep, - ], + '_parquet_encryption': { + 'dependencies': [pyarrow_parquet_encryption_dep], }, } + + if needs_dataset + cython_modules += { + '_dataset_parquet_encryption': { + 'dependencies': [ + arrow_dataset_dep, + pyarrow_parquet_encryption_dep, + ], + }, + } + endif endif -endif -foreach key, val : cython_modules - cython_mod_name = '@0@.pyx'.format(key) - py.extension_module( - key, - sources: [cython_mod_name], - include_directories: ['src'], - cython_args: cython_args, - dependencies: [arrow_dep, pyarrow_lib_dep, numpy_dep] + val.get( - 'dependencies', - [], - ), - override_options: ['cython_language=cpp'], - install: true, - subdir: 'pyarrow', - install_rpath: '$ORIGIN', - gnu_symbol_visibility: gnu_symbol_visibility, - ) + foreach key, val : cython_modules + cython_mod_name = '@0@.pyx'.format(key) + py.extension_module( + key, + sources: [cython_mod_name], + include_directories: ['src'], + cython_args: cython_args, + dependencies: [arrow_dep, pyarrow_lib_dep, numpy_dep] + val.get( + 'dependencies', + [], + ), + override_options: ['cython_language=cpp'], + install: true, + subdir: 'pyarrow', + install_rpath: '$ORIGIN', + gnu_symbol_visibility: gnu_symbol_visibility, + ) - install_data(cython_mod_name, install_dir: py.get_install_dir() / 'pyarrow') -endforeach + install_data( + cython_mod_name, + install_dir: py.get_install_dir() / 'pyarrow', + ) + endforeach + +endif cython_headers = [ '_acero.pxd', diff --git a/python/pyarrow/tests/test_cython.py b/python/pyarrow/tests/test_cython.py index a142e66db567..91a35bfeb361 100644 --- a/python/pyarrow/tests/test_cython.py +++ b/python/pyarrow/tests/test_cython.py @@ -116,6 +116,9 @@ def test_cython_api(tmpdir): orig_path = sys.path[:] sys.path.insert(0, str(tmpdir)) try: + if sys.platform == 'win32': + for dir in pa.get_library_dirs(): + os.add_dll_directory(dir) mod = __import__('pyarrow_cython_example') check_cython_example_module(mod) finally: @@ -187,6 +190,9 @@ def test_visit_strings(tmpdir): env=subprocess_env) sys.path.insert(0, str(tmpdir)) + if sys.platform == 'win32': + for dir in pa.get_library_dirs(): + os.add_dll_directory(dir) mod = __import__('bound_function_visit_strings') strings = ['a', 'b', 'c'] diff --git a/python/requirements-wheel-build.txt b/python/requirements-wheel-build.txt index ac6388762b4c..769435f4dd85 100644 --- a/python/requirements-wheel-build.txt +++ b/python/requirements-wheel-build.txt @@ -1,3 +1,4 @@ +build cython>=3.1 numpy>=2.0.0 setuptools_scm diff --git a/python/scripts/generate_dist.py b/python/scripts/generate_dist.py index c610564a8571..97e306ccf787 100644 --- a/python/scripts/generate_dist.py +++ b/python/scripts/generate_dist.py @@ -26,9 +26,14 @@ def main(): dest_dir = pathlib.Path(os.environ["MESON_DIST_ROOT"]).resolve() license_file = parent_dir / 'LICENSE.txt' - shutil.copy(license_file, dest_dir) + dest_license = dest_dir / 'LICENSE.txt' + dest_license.unlink(missing_ok=True) + shutil.copy(license_file, dest_license) + notice_file = parent_dir / 'NOTICE.txt' - shutil.copy(notice_file, dest_dir) + dest_notice = dest_dir / 'NOTICE.txt' + dest_notice.unlink(missing_ok=True) + shutil.copy(notice_file, dest_notice) if __name__ == "__main__":