Skip to content

Commit fe0c99b

Browse files
kszucskszucs
committed
ARROW-13447: [Release] Verification script for arm64 and universal2 macOS wheels
- verification script for arm64 macos wheels - verification tasks for arm64 macos wheels - verification tasks for source release on arm64 macos Closes apache#10799 from kszucs/apple-silicon-verification Authored-by: Krisztián Szűcs <szucs.krisztian@gmail.com> Signed-off-by: Krisztián Szűcs <szucs.krisztian@gmail.com>
1 parent ec91a50 commit fe0c99b

12 files changed

Lines changed: 219 additions & 149 deletions

‎ci/scripts/python_wheel_macos_build.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,3 +159,7 @@ if echo $deps | grep -v "^pyarrow/lib\(arrow\|gandiva\|parquet\|plasma\)"; then
159159
echo "There are non-bundled shared library dependencies."
160160
exit 1
161161
fi
162+
163+
# Move the verified wheels
164+
mkdir -p ${source_dir}/python/repaired_wheels
165+
mv ${source_dir}/python/dist/*.whl ${source_dir}/python/repaired_wheels/

‎ci/scripts/python_wheel_manylinux_test.sh‎

Lines changed: 0 additions & 73 deletions
This file was deleted.

ci/scripts/python_wheel_macos_test.sh renamed to ci/scripts/python_wheel_unix_test.sh

Lines changed: 32 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,22 @@
1717
# specific language governing permissions and limitations
1818
# under the License.
1919

20-
set -ex
20+
set -e
21+
set -x
22+
set -o pipefail
23+
24+
if [ "$#" -ne 1 ]; then
25+
echo "Usage: $0 <arrow-src-dir>"
26+
exit 1
27+
fi
2128

2229
source_dir=${1}
2330

24-
: ${ARROW_S3:=ON}
2531
: ${ARROW_FLIGHT:=ON}
32+
: ${ARROW_S3:=ON}
33+
: ${CHECK_IMPORTS:=ON}
34+
: ${CHECK_UNITTESTS:=ON}
35+
: ${INSTALL_PYARROW:=ON}
2636

2737
export PYARROW_TEST_CYTHON=OFF
2838
export PYARROW_TEST_DATASET=ON
@@ -39,11 +49,14 @@ export PYARROW_TEST_TENSORFLOW=ON
3949
export ARROW_TEST_DATA=${source_dir}/testing/data
4050
export PARQUET_TEST_DATA=${source_dir}/submodules/parquet-testing/data
4151

42-
# Install the built wheels
43-
pip install ${source_dir}/python/dist/*.whl
52+
if [ "${INSTALL_PYARROW}" == "ON" ]; then
53+
# Install the built wheels
54+
pip install ${source_dir}/python/repaired_wheels/*.whl
55+
fi
4456

45-
# Test that the modules are importable
46-
python -c "
57+
if [ "${CHECK_IMPORTS}" == "ON" ]; then
58+
# Test that the modules are importable
59+
python -c "
4760
import pyarrow
4861
import pyarrow._hdfs
4962
import pyarrow.csv
@@ -54,16 +67,18 @@ import pyarrow.orc
5467
import pyarrow.parquet
5568
import pyarrow.plasma
5669
"
57-
58-
if [ "${PYARROW_TEST_S3}" == "ON" ]; then
59-
python -c "import pyarrow._s3fs"
60-
fi
61-
if [ "${PYARROW_TEST_FLIGHT}" == "ON" ]; then
62-
python -c "import pyarrow.flight"
70+
if [ "${PYARROW_TEST_S3}" == "ON" ]; then
71+
python -c "import pyarrow._s3fs"
72+
fi
73+
if [ "${PYARROW_TEST_FLIGHT}" == "ON" ]; then
74+
python -c "import pyarrow.flight"
75+
fi
6376
fi
6477

65-
# Install testing dependencies
66-
pip install -r ${source_dir}/python/requirements-wheel-test.txt
67-
68-
# Execute unittest
69-
pytest -r s --pyargs pyarrow
78+
if [ "${CHECK_UNITTESTS}" == "ON" ]; then
79+
# Install testing dependencies
80+
pip install -U -r ${source_dir}/python/requirements-wheel-test.txt
81+
# Execute unittest, test dependencies must be installed
82+
python -c 'import pyarrow; pyarrow.create_library_symlinks()'
83+
pytest -r s --pyargs pyarrow
84+
fi

‎ci/scripts/python_wheel_windows_test.bat‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
set PYARROW_TEST_CYTHON=OFF
2121
set PYARROW_TEST_DATASET=ON
22+
set PYARROW_TEST_FLIGHT=ON
2223
set PYARROW_TEST_GANDIVA=OFF
2324
set PYARROW_TEST_HDFS=ON
2425
set PYARROW_TEST_ORC=OFF
@@ -27,7 +28,6 @@ set PYARROW_TEST_PARQUET=ON
2728
set PYARROW_TEST_PLASMA=OFF
2829
set PYARROW_TEST_S3=OFF
2930
set PYARROW_TEST_TENSORFLOW=ON
30-
set PYARROW_TEST_FLIGHT=ON
3131

3232
set ARROW_TEST_DATA=C:\arrow\testing\data
3333
set PARQUET_TEST_DATA=C:\arrow\submodules\parquet-testing\data

‎dev/release/verify-release-candidate.sh‎

Lines changed: 50 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -572,29 +572,6 @@ test_binary_distribution() {
572572
fi
573573
}
574574

575-
check_python_imports() {
576-
python << IMPORT_TESTS
577-
import platform
578-
579-
import pyarrow
580-
import pyarrow.parquet
581-
import pyarrow.plasma
582-
import pyarrow.fs
583-
import pyarrow._hdfs
584-
import pyarrow.dataset
585-
import pyarrow.flight
586-
587-
if platform.system() == "Darwin":
588-
macos_version = tuple(map(int, platform.mac_ver()[0].split('.')))
589-
check_s3fs = macos_version >= (10, 13)
590-
else:
591-
check_s3fs = True
592-
593-
if check_s3fs:
594-
import pyarrow._s3fs
595-
IMPORT_TESTS
596-
}
597-
598575
test_linux_wheels() {
599576
local py_arches="3.6m 3.7m 3.8 3.9"
600577
local manylinuxes="2010 2014"
@@ -608,12 +585,7 @@ test_linux_wheels() {
608585
for ml_spec in ${manylinuxes}; do
609586
# check the mandatory and optional imports
610587
pip install python-rc/${VERSION}-rc${RC_NUMBER}/pyarrow-${VERSION}-cp${py_arch//[mu.]/}-cp${py_arch//./}-manylinux${ml_spec}_x86_64.whl
611-
check_python_imports
612-
613-
# install test requirements and execute the tests
614-
pip install -r ${ARROW_DIR}/python/requirements-test.txt
615-
python -c 'import pyarrow; pyarrow.create_library_symlinks()'
616-
pytest --pyargs pyarrow
588+
INSTALL_PYARROW=OFF ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_DIR}
617589
done
618590

619591
conda deactivate
@@ -622,7 +594,23 @@ test_linux_wheels() {
622594

623595
test_macos_wheels() {
624596
local py_arches="3.6m 3.7m 3.8 3.9"
597+
local macos_version=$(sw_vers -productVersion)
598+
local macos_short_version=${macos_version:0:5}
625599

600+
local check_s3=ON
601+
local check_flight=ON
602+
603+
# macOS version <= 10.13
604+
if [ $(echo "${macos_short_version}\n10.14" | sort -V | head -n1) == "${macos_short_version}" ]; then
605+
local check_s3=OFF
606+
fi
607+
# apple silicon processor
608+
if [ "$(uname -m)" = "arm64" ]; then
609+
local py_arches="3.9"
610+
local check_flight=OFF
611+
fi
612+
613+
# verify arch-native wheels inside an arch-native conda environment
626614
for py_arch in ${py_arches}; do
627615
local env=_verify_wheel-${py_arch}
628616
conda create -yq -n ${env} python=${py_arch//m/}
@@ -631,15 +619,42 @@ test_macos_wheels() {
631619

632620
# check the mandatory and optional imports
633621
pip install --find-links python-rc/${VERSION}-rc${RC_NUMBER} pyarrow==${VERSION}
634-
check_python_imports
635-
636-
# install test requirements and execute the tests
637-
pip install -r ${ARROW_DIR}/python/requirements-test.txt
638-
python -c 'import pyarrow; pyarrow.create_library_symlinks()'
639-
pytest --pyargs pyarrow
622+
INSTALL_PYARROW=OFF ARROW_FLIGHT=${check_flight} ARROW_S3=${check_s3} \
623+
${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_DIR}
640624

641625
conda deactivate
642626
done
627+
628+
# verify arm64 and universal2 wheels using an universal2 python binary
629+
# the interpreter should be installed from python.org:
630+
# https://www.python.org/ftp/python/3.9.6/python-3.9.6-macosx10.9.pkg
631+
if [ "$(uname -m)" = "arm64" ]; then
632+
for py_arch in ${py_arches}; do
633+
local pyver=${py_arch//m/}
634+
local python="/Library/Frameworks/Python.framework/Versions/${pyver}/bin/python${pyver}"
635+
636+
# create and activate a virtualenv for testing as arm64
637+
for arch in "arm64" "x86_64"; do
638+
local venv="${ARROW_TMPDIR}/test-${arch}-virtualenv"
639+
$python -m virtualenv $venv
640+
source $venv/bin/activate
641+
pip install -U pip
642+
643+
# install pyarrow's universal2 wheel
644+
pip install \
645+
--find-links python-rc/${VERSION}-rc${RC_NUMBER} \
646+
--target $(python -c 'import site; print(site.getsitepackages()[0])') \
647+
--platform macosx_11_0_universal2 \
648+
--only-binary=:all: \
649+
pyarrow==${VERSION}
650+
# check the imports and execute the unittests
651+
INSTALL_PYARROW=OFF ARROW_FLIGHT=${check_flight} ARROW_S3=${check_s3} \
652+
arch -${arch} ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_DIR}
653+
654+
deactivate
655+
done
656+
done
657+
fi
643658
}
644659

645660
test_wheels() {

‎dev/tasks/python-wheels/github.osx.amd64.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,7 @@ jobs:
104104
$PYTHON -m virtualenv test-env
105105
source test-env/bin/activate
106106
pip install --upgrade pip wheel
107-
arrow/ci/scripts/python_wheel_macos_test.sh $(pwd)/arrow
107+
arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
108108
109-
{{ macros.github_upload_releases("arrow/python/dist/*.whl")|indent }}
110-
{{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }}
109+
{{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
110+
{{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}

‎dev/tasks/python-wheels/github.osx.arm64.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ jobs:
122122
$PYTHON -m virtualenv test-arm64-env
123123
source test-arm64-env/bin/activate
124124
pip install --upgrade pip wheel
125-
arch -arm64 arrow/ci/scripts/python_wheel_macos_test.sh $(pwd)/arrow
125+
arch -arm64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
126126
127127
{% if arch == "universal2" %}
128128
- name: Test Wheel on AMD64
@@ -131,7 +131,7 @@ jobs:
131131
$PYTHON -m virtualenv test-amd64-env
132132
source test-amd64-env/bin/activate
133133
pip install --upgrade pip wheel
134-
arch -x86_64 arrow/ci/scripts/python_wheel_macos_test.sh $(pwd)/arrow
134+
arch -x86_64 arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
135135
{% endif %}
136136

137137
- name: Upload artifacts
@@ -150,4 +150,4 @@ jobs:
150150
env:
151151
CROSSBOW_GITHUB_TOKEN: {{ "${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" }}
152152

153-
{{ macros.github_upload_gemfury("arrow/python/dist/*.whl")|indent }}
153+
{{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}

0 commit comments

Comments
 (0)