Skip to content

GH-36411: [C++][Python] Some fixes for sdist build and wheel building #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
raulcd wants to merge 1 commit into from
Closed

GH-36411: [C++][Python] Some fixes for sdist build and wheel building #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ci/scripts/python_build.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ if %ARROW_PARQUET% == ON (
)
if %PARQUET_REQUIRE_ENCRYPTION% == ON (
set PYARROW_WITH_PARQUET_ENCRYPTION=enabled
) else if %ARROW_ACERO% == OFF (
) else if %PARQUET_REQUIRE_ENCRYPTION% == OFF (
set PYARROW_WITH_PARQUET_ENCRYPTION=disabled
) else (
set PYARROW_WITH_PARQUET_ENCRYPTION=auto
Expand Down
9 changes: 4 additions & 5 deletions ci/scripts/python_sdist_build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,8 @@ source_dir=${1}/python

pushd "${source_dir}"
export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION:-}
# Meson dist must be run from a VCS, so initiate a dummy repo
git init .
git add --all .
git commit -m "dummy commit for meson dist"
${PYTHON:-python} -m build --sdist .
# The mounted source directory is already a git repository, so mark it as safe
git config --global --add safe.directory "${1}"
${PYTHON:-python} -m pip install build
${PYTHON:-python} -m build --sdist . -Csetup-args="-Dsdist=true"
popd
118 changes: 95 additions & 23 deletions ci/scripts/python_wheel_macos_build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,38 +156,110 @@ popd
echo "=== (${PYTHON_VERSION}) Building wheel ==="
export PYARROW_BUILD_TYPE=${CMAKE_BUILD_TYPE}
export PYARROW_BUNDLE_ARROW_CPP=1
export PYARROW_CMAKE_GENERATOR=${CMAKE_GENERATOR}
export PYARROW_WITH_ACERO=${ARROW_ACERO}
export PYARROW_WITH_AZURE=${ARROW_AZURE}
export PYARROW_WITH_DATASET=${ARROW_DATASET}
export PYARROW_WITH_FLIGHT=${ARROW_FLIGHT}
export PYARROW_WITH_GANDIVA=${ARROW_GANDIVA}
export PYARROW_WITH_GCS=${ARROW_GCS}
export PYARROW_WITH_HDFS=${ARROW_HDFS}
export PYARROW_WITH_ORC=${ARROW_ORC}
export PYARROW_WITH_PARQUET=${ARROW_PARQUET}
export PYARROW_WITH_PARQUET_ENCRYPTION=${PARQUET_REQUIRE_ENCRYPTION}
export PYARROW_WITH_SUBSTRAIT=${ARROW_SUBSTRAIT}
export PYARROW_WITH_S3=${ARROW_S3}
export PYARROW_CMAKE_OPTIONS="-DCMAKE_OSX_ARCHITECTURES=${CMAKE_OSX_ARCHITECTURES} -DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL}"

PYARROW_WITH_ACERO=$(case "$ARROW_ACERO" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_AZURE=$(case "$ARROW_AZURE" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
# CUDA support for wheels should be disabled?
PYARROW_WITH_CUDA=$(case "$ARROW_CUDA" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_DATASET=$(case "$ARROW_DATASET" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "enabled" ;;
esac)
PYARROW_WITH_FLIGHT=$(case "$ARROW_FLIGHT" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_GANDIVA=$(case "$ARROW_GANDIVA" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_GCS=$(case "$ARROW_GCS" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_HDFS=$(case "$ARROW_HDFS" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "enabled" ;;
esac)
PYARROW_WITH_ORC=$(case "$ARROW_ORC" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_PARQUET=$(case "$ARROW_PARQUET" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_PARQUET_ENCRYPTION=$(case "$PARQUET_REQUIRE_ENCRYPTION" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "enabled" ;;
esac)
PYARROW_WITH_S3=$(case "$ARROW_S3" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)
PYARROW_WITH_SUBSTRAIT=$(case "$ARROW_SUBSTRAIT" in
ON) echo "enabled" ;;
OFF) echo "disabled" ;;
*) echo "auto" ;;
esac)

export ARROW_HOME=${build_dir}/install
# PyArrow build configuration
export CMAKE_PREFIX_PATH=${build_dir}/install
# Set PyArrow version explicitly
export SETUPTOOLS_SCM_PRETEND_VERSION=${PYARROW_VERSION}

# Meson sdist requires setuptools_scm to be able to get the version from git
git config --global --add safe.directory ${source_dir}

pushd ${source_dir}/python
python setup.py bdist_wheel
python -m build --sdist --wheel . \
-Csetup-args="-Dbuildtype=${PYARROW_BUILD_TYPE}" \
-Csetup-args="-Dacero=${PYARROW_WITH_ACERO}" \
-Csetup-args="-Dazure=${PYARROW_WITH_AZURE}" \
-Csetup-args="-Dcuda=${PYARROW_WITH_CUDA}" \
-Csetup-args="-Ddataset=${PYARROW_WITH_DATASET}" \
-Csetup-args="-Dflight=${PYARROW_WITH_FLIGHT}" \
-Csetup-args="-Dgandiva=${PYARROW_WITH_GANDIVA}" \
-Csetup-args="-Dgcs=${PYARROW_WITH_GCS}" \
-Csetup-args="-Dhdfs=${PYARROW_WITH_HDFS}" \
-Csetup-args="-Dorc=${PYARROW_WITH_ORC}" \
-Csetup-args="-Dparquet=${PYARROW_WITH_PARQUET}" \
-Csetup-args="-Dparquet_require_encryption=${PYARROW_WITH_PARQUET_ENCRYPTION}" \
-Csetup-args="-Ds3=${PYARROW_WITH_S3}" \
-Csetup-args="-Dsubstrait=${PYARROW_WITH_SUBSTRAIT}" \
-Ccompile-args="-v" \
-Csetup-args="--pkg-config-path=${ARROW_HOME}/lib/pkgconfig"
popd

echo "=== (${PYTHON_VERSION}) Show dynamic libraries the wheel depend on ==="
deps=$(delocate-listdeps ${source_dir}/python/dist/*.whl)
delocate-listdeps ${source_dir}/python/dist/*.whl

if echo $deps | grep -v "^pyarrow/lib\(arrow\|gandiva\|parquet\)"; then
echo "=== (${PYTHON_VERSION}) Bundle shared libraries into wheel ==="
delocate-wheel -w ${source_dir}/python/repaired_wheels -v ${source_dir}/python/dist/*.whl

echo "=== (${PYTHON_VERSION}) Validate there are no non-bundled shared dependencies ==="
deps=$(delocate-listdeps ${source_dir}/python/repaired_wheels/*.whl)
if echo $deps | grep -v "^pyarrow/.dylibs/lib\(arrow\|gandiva\|parquet\)"; then
echo "There are non-bundled shared library dependencies."
exit 1
fi

# Move the verified wheels
mkdir -p ${source_dir}/python/repaired_wheels
mv ${source_dir}/python/dist/*.whl ${source_dir}/python/repaired_wheels/
122 changes: 105 additions & 17 deletions ci/scripts/python_wheel_windows_build.bat
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,26 +114,109 @@ echo "=== (%PYTHON%) Building wheel ==="
set PYARROW_BUILD_TYPE=%CMAKE_BUILD_TYPE%
set PYARROW_BUILD_VERBOSE=1
set PYARROW_BUNDLE_ARROW_CPP=ON
set PYARROW_CMAKE_GENERATOR=%CMAKE_GENERATOR%
set PYARROW_CMAKE_OPTIONS="-DCMAKE_INTERPROCEDURAL_OPTIMIZATION=%CMAKE_INTERPROCEDURAL_OPTIMIZATION%"
set PYARROW_WITH_ACERO=%ARROW_ACERO%
set PYARROW_WITH_DATASET=%ARROW_DATASET%
set PYARROW_WITH_FLIGHT=%ARROW_FLIGHT%
set PYARROW_WITH_GANDIVA=%ARROW_GANDIVA%
set PYARROW_WITH_GCS=%ARROW_GCS%
set PYARROW_WITH_HDFS=%ARROW_HDFS%
set PYARROW_WITH_ORC=%ARROW_ORC%
set PYARROW_WITH_PARQUET=%ARROW_PARQUET%
set PYARROW_WITH_PARQUET_ENCRYPTION=%PARQUET_REQUIRE_ENCRYPTION%
set PYARROW_WITH_SUBSTRAIT=%ARROW_SUBSTRAIT%
set PYARROW_WITH_S3=%ARROW_S3%
set ARROW_HOME=C:\arrow-dist
set CMAKE_PREFIX_PATH=C:\arrow-dist

if %ARROW_ACERO% == ON (
set PYARROW_WITH_ACERO=enabled
) else if %ARROW_ACERO% == OFF (
set PYARROW_WITH_ACERO=disabled
) else (
set PYARROW_WITH_ACERO=auto
)
if %ARROW_DATASET% == ON (
set PYARROW_WITH_DATASET=enabled
) else if %ARROW_DATASET% == OFF (
set PYARROW_WITH_DATASET=disabled
) else (
set PYARROW_WITH_DATASET=auto
)
if %ARROW_FLIGHT% == ON (
set PYARROW_WITH_FLIGHT=enabled
) else if %ARROW_FLIGHT% == OFF (
set PYARROW_WITH_FLIGHT=disabled
) else (
set PYARROW_WITH_FLIGHT=auto
)
if %ARROW_GANDIVA% == ON (
set PYARROW_WITH_GANDIVA=enabled
) else if %ARROW_GANDIVA% == OFF (
set PYARROW_WITH_GANDIVA=disabled
) else (
set PYARROW_WITH_GANDIVA=auto
)
if %ARROW_GCS% == ON (
set PYARROW_WITH_GCS=enabled
) else if %ARROW_GCS% == OFF (
set PYARROW_WITH_GCS=disabled
) else (
set PYARROW_WITH_GCS=auto
)
if %ARROW_HDFS% == ON (
set PYARROW_WITH_HDFS=enabled
) else if %ARROW_HDFS% == OFF (
set PYARROW_WITH_HDFS=disabled
) else (
set PYARROW_WITH_HDFS=auto
)
if %ARROW_ORC% == ON (
set PYARROW_WITH_ORC=enabled
) else if %ARROW_ORC% == OFF (
set PYARROW_WITH_ORC=disabled
) else (
set PYARROW_WITH_ORC=auto
)
if %ARROW_PARQUET% == ON (
set PYARROW_WITH_PARQUET=enabled
) else if %ARROW_PARQUET% == OFF (
set PYARROW_WITH_PARQUET=disabled
) else (
set PYARROW_WITH_PARQUET=auto
)
if %PARQUET_REQUIRE_ENCRYPTION% == ON (
set PYARROW_WITH_PARQUET_ENCRYPTION=enabled
) else if %PARQUET_REQUIRE_ENCRYPTION% == OFF (
set PYARROW_WITH_PARQUET_ENCRYPTION=disabled
) else (
set PYARROW_WITH_PARQUET_ENCRYPTION=auto
)
if %ARROW_SUBSTRAIT% == ON (
set PYARROW_WITH_SUBSTRAIT=enabled
) else if %ARROW_SUBSTRAIT% == OFF (
set PYARROW_WITH_SUBSTRAIT=disabled
) else (
set PYARROW_WITH_SUBSTRAIT=auto
)
if %ARROW_S3% == ON (
set PYARROW_WITH_S3=enabled
) else if %ARROW_S3% == OFF (
set PYARROW_WITH_S3=disabled
) else (
set PYARROW_WITH_S3=auto
)

@REM Meson sdist requires setuptools_scm to be able to get the version from git
git config --global --add safe.directory C:\arrow

pushd C:\arrow\python

@REM TODO: Remove once docker rebuild works correctly
@REM See: https://github.com/apache/arrow/issues/48947
%PYTHON_CMD% -m pip install -U build || exit /B 1

@REM Build wheel
%PYTHON_CMD% setup.py bdist_wheel || exit /B 1
%PYTHON_CMD% -m build --sdist --wheel . ^
-Csetup-args="-Dbuildtype=%CMAKE_BUILD_TYPE%" ^
-Csetup-args="-Dacero=%PYARROW_WITH_ACERO%" ^
-Csetup-args="-Ddataset=%PYARROW_WITH_DATASET%" ^
-Csetup-args="-Dflight=%PYARROW_WITH_FLIGHT%" ^
-Csetup-args="-Dgandiva=%PYARROW_WITH_GANDIVA%" ^
-Csetup-args="-Dgcs=%PYARROW_WITH_GCS%" ^
-Csetup-args="-Dhdfs=%PYARROW_WITH_HDFS%" ^
-Csetup-args="-Dorc=%PYARROW_WITH_ORC%" ^
-Csetup-args="-Dparquet=%PYARROW_WITH_PARQUET%" ^
-Csetup-args="-Dparquet_require_encryption=%PYARROW_WITH_PARQUET_ENCRYPTION%" ^
-Csetup-args="-Dsubstrait=%PYARROW_WITH_SUBSTRAIT%" ^
-Csetup-args="-Ds3=%PYARROW_WITH_S3%" ^
-Csetup-args="--cmake-prefix-path=C:\arrow-dist" || exit /B 1

@REM Repair the wheel with delvewheel
@REM
Expand All @@ -142,11 +225,16 @@ pushd C:\arrow\python
@REM required by multiple Python libraries in the same process.
%PYTHON_CMD% -m pip install delvewheel || exit /B 1

@REM Copy .lib files to bin directory for delvewheel to find them
copy C:\arrow-dist\lib\*.lib C:\arrow-dist\bin\ || exit /B 1

for /f %%i in ('dir dist\pyarrow-*.whl /B') do (set WHEEL_NAME=%cd%\dist\%%i) || exit /B 1
echo "Wheel name: %WHEEL_NAME%"

%PYTHON_CMD% -m delvewheel repair -vv ^
--ignore-existing --with-mangle ^
--ignore-existing --with-mangle --include-imports ^
--no-mangle "arrow.dll;arrow_python.dll;arrow_acero.dll;arrow_dataset.dll;arrow_flight.dll;arrow_substrait.dll;parquet.dll" ^
--add-path "C:\arrow-dist\bin" ^
-w repaired_wheels %WHEEL_NAME% || exit /B 1

popd
Loading
Loading