Skip to content

Commit cd1c01c

Browse files
koukou
authored
ARROW-17240: [CI][Release] Verify wheels in nightly CI (apache#14319)
.deb and .rpm are already verified by the same verification scripts (dev/release/verify-apt.sh and dev/release/verify-yum.sh) but .whl aren't yet. Authored-by: Sutou Kouhei <kou@clear-code.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
1 parent 9b1dddf commit cd1c01c

8 files changed

Lines changed: 143 additions & 79 deletions

File tree

‎ci/docker/almalinux-8-verify-rc.dockerfile‎

Lines changed: 3 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -18,40 +18,7 @@
1818
ARG arch=amd64
1919
FROM ${arch}/almalinux:8
2020

21-
# A script to install dependencies required for release
22-
# verification Red Hat Enterprise Linux 8 clones in particular
23-
# on AlmaLinux 8 and Rocky Linux 8
24-
25-
RUN dnf -y install 'dnf-command(config-manager)' && \
26-
dnf config-manager --set-enabled powertools && \
27-
dnf -y update && \
28-
dnf -y module disable nodejs && \
29-
dnf -y module enable nodejs:16 && \
30-
dnf -y module disable ruby && \
31-
dnf -y module enable ruby:2.7 && \
32-
dnf -y groupinstall "Development Tools" && \
33-
dnf -y install \
34-
cmake \
35-
git \
36-
gobject-introspection-devel \
37-
java-1.8.0-openjdk-devel \
38-
libcurl-devel \
39-
llvm-devel \
40-
llvm-toolset \
41-
maven \
42-
ncurses-devel \
43-
ninja-build \
44-
nodejs \
45-
openssl-devel \
46-
python38-devel \
47-
python38-pip \
48-
ruby-devel \
49-
sqlite-devel \
50-
wget \
51-
which && \
21+
COPY dev/release/setup-rhel-rebuilds.sh /
22+
RUN /setup-rhel-rebuilds.sh && \
23+
rm /setup-rhel-rebuilds.sh && \
5224
dnf -y clean all
53-
54-
RUN python3 -m pip install -U pip && \
55-
alternatives --set python /usr/bin/python3
56-
57-
RUN npm install -g yarn

‎dev/release/setup-rhel-rebuilds.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121
# verification Red Hat Enterprise Linux 8 clones in particular
2222
# on AlmaLinux 8 and Rocky Linux 8
2323

24+
set -exu
25+
2426
dnf -y install 'dnf-command(config-manager)'
2527
dnf config-manager --set-enabled powertools
2628
dnf -y update
@@ -49,5 +51,8 @@ dnf -y install \
4951
vala-devel \
5052
wget \
5153
which
54+
5255
npm install -g yarn
56+
57+
python3 -m pip install -U pip
5358
alternatives --set python /usr/bin/python3

‎dev/release/verify-release-candidate.sh‎

Lines changed: 23 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1014,7 +1014,7 @@ test_linux_wheels() {
10141014
local arch="x86_64"
10151015
fi
10161016

1017-
local python_versions="3.7m 3.8 3.9 3.10"
1017+
local python_versions="${TEST_PYTHON_VERSIONS:-3.7m 3.8 3.9 3.10}"
10181018
local platform_tags="manylinux_2_17_${arch}.manylinux2014_${arch}"
10191019

10201020
for python in ${python_versions}; do
@@ -1023,7 +1023,7 @@ test_linux_wheels() {
10231023
show_header "Testing Python ${pyver} wheel for platform ${platform}"
10241024
CONDA_ENV=wheel-${pyver}-${platform} PYTHON_VERSION=${pyver} maybe_setup_conda || exit 1
10251025
VENV_ENV=wheel-${pyver}-${platform} PYTHON_VERSION=${pyver} maybe_setup_virtualenv || continue
1026-
pip install pyarrow-${VERSION}-cp${pyver/.}-cp${python/.}-${platform}.whl
1026+
pip install pyarrow-${TEST_PYARROW_VERSION:-${VERSION}}-cp${pyver/.}-cp${python/.}-${platform}.whl
10271027
INSTALL_PYARROW=OFF ARROW_GCS=${check_gcs} ${ARROW_DIR}/ci/scripts/python_wheel_unix_test.sh ${ARROW_SOURCE_DIR}
10281028
done
10291029
done
@@ -1087,23 +1087,31 @@ test_wheels() {
10871087
show_header "Downloading Python wheels"
10881088
maybe_setup_conda python || exit 1
10891089

1090-
local download_dir=${ARROW_TMPDIR}/binaries
1091-
mkdir -p ${download_dir}
1092-
1093-
if [ "$(uname)" == "Darwin" ]; then
1094-
local filter_regex=.*macosx.*
1090+
local wheels_dir=
1091+
if [ "${SOURCE_KIND}" = "local" ]; then
1092+
wheels_dir="${ARROW_SOURCE_DIR}/python/repaired_wheels"
10951093
else
1096-
local filter_regex=.*manylinux.*
1097-
fi
1094+
local download_dir=${ARROW_TMPDIR}/binaries
1095+
mkdir -p ${download_dir}
10981096

1099-
${PYTHON:-python3} $SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
1100-
--package_type python \
1101-
--regex=${filter_regex} \
1102-
--dest=${download_dir}
1097+
if [ "$(uname)" == "Darwin" ]; then
1098+
local filter_regex=.*macosx.*
1099+
else
1100+
local filter_regex=.*manylinux.*
1101+
fi
11031102

1104-
verify_dir_artifact_signatures ${download_dir}
1103+
${PYTHON:-python3} \
1104+
$SOURCE_DIR/download_rc_binaries.py $VERSION $RC_NUMBER \
1105+
--package_type python \
1106+
--regex=${filter_regex} \
1107+
--dest=${download_dir}
1108+
1109+
verify_dir_artifact_signatures ${download_dir}
1110+
1111+
wheels_dir=${download_dir}/python-rc/${VERSION}-rc${RC_NUMBER}
1112+
fi
11051113

1106-
pushd ${download_dir}/python-rc/${VERSION}-rc${RC_NUMBER}
1114+
pushd ${wheels_dir}
11071115

11081116
if [ "$(uname)" == "Darwin" ]; then
11091117
test_macos_wheels

‎dev/tasks/python-wheels/github.linux.amd64.yml‎

Lines changed: 76 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
jobs:
2323
build:
24-
name: "Build wheel for Manylinux {{ manylinux_version }}"
24+
name: "Build wheel for manylinux {{ manylinux_version }}"
2525
runs-on: ubuntu-latest
2626
env:
2727
# archery uses these environment variables
@@ -37,13 +37,88 @@ jobs:
3737
shell: bash
3838
run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION={{ arrow.no_rc_version }} python-wheel-manylinux-{{ manylinux_version }}
3939

40+
- uses: actions/upload-artifact@v3
41+
with:
42+
name: wheel
43+
path: arrow/python/repaired_wheels/*.whl
44+
4045
# TODO(kszucs): auditwheel show
4146
- name: Test wheel
4247
shell: bash
4348
run: |
4449
archery docker run python-wheel-manylinux-test-imports
4550
archery docker run python-wheel-manylinux-test-unittests
4651
52+
- name: Test wheel on AlmaLinux 8
53+
shell: bash
54+
if: |
55+
'{{ python_version }}' == '3.8'
56+
env:
57+
ALMALINUX: "8"
58+
run: |
59+
archery docker run \
60+
-e TEST_DEFAULT=0 \
61+
-e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
62+
-e TEST_PYTHON_VERSIONS={{ python_version }} \
63+
-e TEST_WHEELS=1 \
64+
almalinux-verify-rc
65+
66+
- name: Test wheel on Ubuntu 18.04
67+
shell: bash
68+
if: |
69+
'{{ python_version }}' == '3.8'
70+
env:
71+
UBUNTU: "18.04"
72+
run: |
73+
archery docker run \
74+
-e TEST_DEFAULT=0 \
75+
-e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
76+
-e TEST_PYTHON_VERSIONS={{ python_version }} \
77+
-e TEST_WHEELS=1 \
78+
ubuntu-verify-rc
79+
80+
- name: Test wheel on Ubuntu 18.04
81+
shell: bash
82+
if: |
83+
'{{ python_version }}' == '3.8'
84+
env:
85+
UBUNTU: "18.04"
86+
run: |
87+
archery docker run \
88+
-e TEST_DEFAULT=0 \
89+
-e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
90+
-e TEST_PYTHON_VERSIONS={{ python_version }} \
91+
-e TEST_WHEELS=1 \
92+
ubuntu-verify-rc
93+
94+
- name: Test wheel on Ubuntu 20.04
95+
shell: bash
96+
if: |
97+
'{{ python_version }}' == '3.8'
98+
env:
99+
UBUNTU: "20.04"
100+
run: |
101+
archery docker run \
102+
-e TEST_DEFAULT=0 \
103+
-e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
104+
-e TEST_PYTHON_VERSIONS={{ python_version }} \
105+
-e TEST_WHEELS=1 \
106+
ubuntu-verify-rc
107+
108+
- name: Test wheel on Ubuntu 22.04
109+
shell: bash
110+
if: |
111+
'{{ python_version }}' == '3.10'
112+
env:
113+
UBUNTU: "22.04"
114+
run: |
115+
archery docker run \
116+
-e TEST_DEFAULT=0 \
117+
-e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
118+
-e TEST_PYTHON_VERSIONS={{ python_version }} \
119+
-e TEST_WHEELS=1 \
120+
ubuntu-verify-rc
121+
47122
{{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
48123
{{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}
49124

‎dev/tasks/python-wheels/github.osx.amd64.yml‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ env:
3333

3434
jobs:
3535
build:
36-
name: Build wheel for macOS
36+
name: Build wheel for Python {{ python_version }} on macOS
3737
runs-on: macos-latest
3838
env:
3939
VCPKG_BINARY_SOURCES: 'clear;nuget,GitHub,readwrite'
@@ -103,15 +103,28 @@ jobs:
103103
pip install --upgrade pip wheel
104104
PYTHON=python arrow/ci/scripts/python_wheel_macos_build.sh x86_64 $(pwd)/arrow $(pwd)/build
105105
106+
- uses: actions/upload-artifact@v3
107+
with:
108+
name: wheel
109+
path: arrow/python/repaired_wheels/*.whl
110+
106111
- name: Test Wheel
107112
shell: bash
113+
env:
114+
TEST_DEFAULT: "0"
115+
TEST_PYARROW_VERSION: "{{ arrow.no_rc_version }}"
116+
TEST_WHEEL: "1"
108117
run: |
109-
$PYTHON -m venv test-env
110-
source test-env/bin/activate
111-
pip install --upgrade pip wheel
112-
pip install -r arrow/python/requirements-wheel-test.txt
113-
PYTHON=python arrow/ci/scripts/install_gcs_testbench.sh default
114-
arrow/ci/scripts/python_wheel_unix_test.sh $(pwd)/arrow
118+
case "${PYTHON_VERSION}" in
119+
3.7)
120+
python_version=${PYTHON_VERSION}m
121+
;;
122+
*)
123+
python_version=${PYTHON_VERSION}
124+
;;
125+
esac
126+
export TEST_PYTHON_VERSIONS=${python_version}
127+
arrow/dev/release/verify-release-candidate.sh
115128
116129
{{ macros.github_upload_releases("arrow/python/repaired_wheels/*.whl")|indent }}
117130
{{ macros.github_upload_gemfury("arrow/python/repaired_wheels/*.whl")|indent }}

‎dev/tasks/python-wheels/github.osx.arm64.yml‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ env:
3939

4040
jobs:
4141
build:
42-
name: Build wheel for OS X
42+
name: Build wheel for Python {{ python_version }} on macOS
4343
runs-on: ["self-hosted", "macOS", "arm64"]
4444
steps:
4545
- name: Cleanup
@@ -67,7 +67,7 @@ jobs:
6767
env:
6868
VCPKG_DEFAULT_TRIPLET: arm64-osx-static-release
6969
run: |
70-
vcpkg install \
70+
vcpkg install \
7171
--clean-after-build \
7272
--x-install-root=${VCPKG_ROOT}/installed \
7373
--x-manifest-root=arrow/ci/vcpkg \
@@ -85,14 +85,14 @@ jobs:
8585
$PYTHON -m venv build-arm64-env
8686
source build-arm64-env/bin/activate
8787
pip install --upgrade pip wheel
88-
arrow/ci/scripts/python_wheel_macos_build.sh arm64 $(pwd)/arrow $(pwd)/build
88+
arrow/ci/scripts/python_wheel_macos_build.sh arm64 $(pwd)/arrow $(pwd)/build
8989
9090
{% if arch == "universal2" %}
9191
- name: Install AMD64 Packages
9292
env:
9393
VCPKG_DEFAULT_TRIPLET: amd64-osx-static-release
9494
run: |
95-
vcpkg install \
95+
vcpkg install \
9696
--clean-after-build \
9797
--x-install-root=${VCPKG_ROOT}/installed \
9898
--x-manifest-root=arrow/ci/vcpkg \
@@ -127,6 +127,11 @@ jobs:
127127
mv $fused_wheel arrow/python/repaired_wheels/${fused_wheel/x86_64/universal2}
128128
{% endif %}
129129

130+
- uses: actions/upload-artifact@v3
131+
with:
132+
name: wheel
133+
path: arrow/python/repaired_wheels/*.whl
134+
130135
- name: Test Wheel on ARM64
131136
shell: bash
132137
env:
@@ -164,12 +169,12 @@ jobs:
164169
source crossbow-env/bin/activate
165170
arch -x86_64 pip install -e arrow/dev/archery[crossbow-upload]
166171
arch -x86_64 archery crossbow \
167-
--queue-path $(pwd) \
168-
--queue-remote {{ queue_remote_url }} \
169-
upload-artifacts \
170-
--sha {{ task.branch }} \
171-
--tag {{ task.tag }} \
172-
"arrow/python/repaired_wheels/*.whl"
172+
--queue-path $(pwd) \
173+
--queue-remote {{ queue_remote_url }} \
174+
upload-artifacts \
175+
--sha {{ task.branch }} \
176+
--tag {{ task.tag }} \
177+
"arrow/python/repaired_wheels/*.whl"
173178
env:
174179
CROSSBOW_GITHUB_TOKEN: {{ "${{ secrets.CROSSBOW_GITHUB_TOKEN || secrets.GITHUB_TOKEN }}" }}
175180

‎dev/tasks/tasks.yml‎

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1076,15 +1076,6 @@ tasks:
10761076
params:
10771077
github_runner: "macos-{{ macos_version }}"
10781078
target: "wheels"
1079-
1080-
verify-rc-binaries-wheels-macos-{{ macos_version }}-amd64-conda:
1081-
ci: github
1082-
template: verify-rc/github.macos.amd64.yml
1083-
params:
1084-
env:
1085-
USE_CONDA: 1
1086-
github_runner: "macos-{{ macos_version }}"
1087-
target: "wheels"
10881079
{% endfor %}
10891080

10901081
verify-rc-binaries-wheels-macos-11-arm64:

‎dev/tasks/verify-rc/github.linux.amd64.docker.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121

2222
jobs:
2323
test:
24-
name: "Verify release candidate {{ distro }} source"
24+
name: "Verify release candidate {{ distro }} {{ target }}"
2525
runs-on: ubuntu-latest
2626
{% if env is defined %}
2727
env:

0 commit comments

Comments
 (0)