Repository navigation
184 lines (164 loc) · 7.26 KB
/
Copy pathtest.yml
File metadata and controls
184 lines (164 loc) · 7.26 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# Reusable workflow for running tests.
# Single matrix covers both GIL (abi3 wheel) and free-threaded
# (per-interpreter wheels) builds.
name: Test
on:
workflow_call:
env:
UV_LOCKED: true
jobs:
test-matrix:
runs-on: ubuntu-latest
# Backstop: a hung multiprocessing worker (e.g. during a pickle regression)
# should not block CI longer than this.
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
include:
# GIL builds — all share the same abi3 wheel.
- { python-version: "3.10", wheel-tag: "abi3", freethreaded: false }
- { python-version: "3.11", wheel-tag: "abi3", freethreaded: false }
- { python-version: "3.12", wheel-tag: "abi3", freethreaded: false }
- { python-version: "3.13", wheel-tag: "abi3", freethreaded: false }
- { python-version: "3.14", wheel-tag: "abi3", freethreaded: false }
# Free-threaded builds — one wheel per interpreter.
- { python-version: "3.14t", wheel-tag: "3.14t", freethreaded: true }
steps:
- uses: actions/checkout@v7
- name: Setup Python
id: setup-python
uses: actions/setup-python@v7
with:
python-version: ${{ matrix.python-version }}
freethreaded: ${{ matrix.freethreaded }}
- name: Cache Cargo
uses: actions/cache@v6
with:
path: ~/.cargo
key: cargo-cache-stable-${{ hashFiles('Cargo.lock') }}
- name: Install dependencies
uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d
with:
enable-cache: true
- name: Download pre-built Linux wheel
uses: actions/download-artifact@v8
with:
name: dist-manylinux-x86_64-${{ matrix.wheel-tag }}
path: wheels/
# FFI test wheel only built once (under the abi3 matrix entry in build.yml).
- name: Download pre-built FFI test wheel
if: matrix.wheel-tag == 'abi3'
uses: actions/download-artifact@v8
with:
name: test-ffi-manylinux-x86_64
path: wheels/
- name: Install from pre-built wheels
run: |
set -x
# Create the venv with the setup-python interpreter, then point
# every uv command explicitly at the venv's own interpreter.
# uv's interpreter discovery skips free-threaded builds unless
# asked by exact path, so plain `uv sync` (even with an activated
# venv or --active) re-picks the system 3.12 and recreates .venv.
# Targeting .venv/bin/python keeps sync and pip install in the
# same 3.14t environment as the cp314t wheel.
uv venv --python "${{ steps.setup-python.outputs.python-path }}"
VENV_PY="$PWD/.venv/bin/python"
uv sync --python "$VENV_PY" --dev --no-install-package datafusion
# Search recursively: the FFI artifact bundles more than one
# project, so upload-artifact keeps a `<project>/dist/` prefix
# and the wheels are not all at the top of wheels/.
WHEELS=$(find wheels/ -name "*.whl")
if [ -n "$WHEELS" ]; then
echo "Installing wheels:"
echo "$WHEELS"
# shellcheck disable=SC2086 # intentional split on newlines
uv pip install --python "$VENV_PY" $WHEELS
else
echo "ERROR: No wheels found!"
exit 1
fi
- name: Run tests
env:
RUST_BACKTRACE: 1
# On free-threaded interpreters, fail loud if any C extension
# re-enables the GIL implicitly.
PYTHON_GIL: ${{ matrix.freethreaded && '0' || '' }}
run: |
git submodule update --init
# Use the .venv interpreter directly; uv discovery would skip the
# free-threaded build and re-pick the system 3.12 (see install step).
uv run --python "$PWD/.venv/bin/python" --no-project pytest -v --import-mode=importlib
# Run the top-level examples once. They exercise the public Python API
# that the test suite does not: several are plain scripts with no pytest
# coverage, so nothing caught a broken example before this job existed.
- name: Run example scripts
if: matrix.python-version == '3.12'
run: |
set -euo pipefail
# Extra runtime dependencies used by some examples but not part of
# the dev group: pandas and polars for import.py/export.py, and
# matplotlib for sql-to-pandas.py.
uv pip install --python "$PWD/.venv/bin/python" pandas polars matplotlib
# Examples that cannot run here:
# - sql-parquet-s3.py: needs network access and AWS credentials
# - sql-parquet.py, dataframe-parquet.py, sql-to-pandas.py: need the
# NYC taxi parquet file documented in examples/README.md
# - python-udf-comparisons.py: needs the TPC-H dataset generated
# later by the tpchgen-cli step
# - ray_pickle_expr.py: needs the optional (heavy) `ray` dependency
skip="sql-parquet-s3.py sql-parquet.py dataframe-parquet.py sql-to-pandas.py python-udf-comparisons.py ray_pickle_expr.py"
failed=""
for script in examples/*.py; do
name="$(basename "$script")"
if [[ " $skip " == *" $name "* ]]; then
echo "Skipping $script"
continue
fi
echo "::group::Running $script"
if ! uv run --python "$PWD/.venv/bin/python" --no-project python "$script"; then
failed="$failed $script"
fi
echo "::endgroup::"
done
if [[ -n "$failed" ]]; then
echo "Example scripts failed:$failed"
exit 1
fi
# FFI + TPC-H examples only need to run once; gate to abi3 entries.
- name: FFI unit tests
if: matrix.wheel-tag == 'abi3'
run: |
cd examples/datafusion-ffi-example
uv run --no-project pytest python/tests/_test*.py
cd ../datafusion-ffi-query-planner-example
uv run --no-project pytest python/tests/_test*.py
- name: Run tpchgen-cli to create 1 Gb dataset
if: matrix.wheel-tag == 'abi3'
run: |
mkdir examples/tpch/data
cd examples/tpch/data
uv pip install tpchgen-cli
uv run --no-project tpchgen-cli -s 1 --format=parquet
- name: Run TPC-H examples
if: matrix.wheel-tag == 'abi3'
run: |
cd examples/tpch
uv run --no-project pytest _tests.py