Skip to content

Commit 8181c84

Browse files
authored
CI Fixes (apache#162)
* Set RUST_BACKTRACE 1 * Add protoc action to build.yml for manylinux * Add protoc action to test.yaml matrix * Bump to 17.0.0 for a test * include features * test re-enabling protoc action * enable protoc action in test.yaml * blake formatting * python linter fixes * blake fixes * update run tests action * remove duplicate maturin develop * Include Pip install for datafusion dist * Remove some pip install options that are not needed * Install dist/datafusion*.whl * Use virtualenv python version for pip install * Use vitualenv python version for pytest run * examine the contents of the result whl file to make sure _internal.abi3.so is present * Wrong file extension ... change to .whl from .zip * Try another flavor of pip installing * Fix pyarrow version * update pip install process * testing * doh * Remove previous maturin build since now it happens in the test run section
1 parent 8975d4e commit 8181c84

6 files changed

Lines changed: 44 additions & 86 deletions

File tree

‎.github/actions/setup-builder/action.yaml‎

Lines changed: 0 additions & 39 deletions
This file was deleted.

‎.github/workflows/build.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,9 +97,15 @@ jobs:
9797
name: python-wheel-license
9898
path: .
9999
- run: cat LICENSE.txt
100+
- name: Install Protoc
101+
uses: arduino/setup-protoc@v1
102+
with:
103+
version: '3.x'
100104
- name: Build wheels
101105
uses: PyO3/maturin-action@v1
102106
with:
107+
env:
108+
RUST_BACKTRACE: 1
103109
rust-toolchain: nightly
104110
target: x86_64
105111
manylinux: auto

‎.github/workflows/test.yaml‎

Lines changed: 9 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -53,18 +53,10 @@ jobs:
5353
toolchain: ${{ matrix.toolchain }}
5454
override: true
5555

56-
- name: Install protobuf compiler
57-
shell: bash
58-
run: |
59-
mkdir -p $HOME/d/protoc
60-
cd $HOME/d/protoc
61-
export PROTO_ZIP="protoc-21.4-linux-x86_64.zip"
62-
curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v21.4/$PROTO_ZIP
63-
unzip $PROTO_ZIP
64-
export PATH=$PATH:$HOME/d/protoc/bin
65-
export PROTOC=$HOME/d/protoc/bin
66-
sudo chown -R $(whoami) $HOME/d/protoc
67-
protoc --version
56+
- name: Install Protoc
57+
uses: arduino/setup-protoc@v1
58+
with:
59+
version: '3.x'
6860

6961
- name: Setup Python
7062
uses: actions/setup-python@v4
@@ -112,22 +104,11 @@ jobs:
112104
flake8 --exclude venv --ignore=E501,W503
113105
black --line-length 79 --diff --check .
114106
115-
- name: Build wheels
116-
uses: PyO3/maturin-action@v1
117-
with:
118-
command: build
119-
args: --release --out dist
120-
121107
- name: Run tests
108+
env:
109+
RUST_BACKTRACE: 1
122110
run: |
123111
git submodule update --init
124-
export PATH=$PATH:$HOME/d/protoc/bin
125-
export PROTOC=$HOME/d/protoc/bin
126-
sudo chown -R $(whoami) $HOME/d/protoc
127-
ls -l $HOME/d/protoc/
128-
ls -l $HOME/d/protoc/bin
129-
pip install datafusion-python --no-index --find-links dist --force-reinstall
130-
pip install pytest
131-
cargo clean
132-
maturin develop
133-
RUST_BACKTRACE=1 pytest -v .
112+
source venv/bin/activate
113+
pip install -e . -vv
114+
pytest -v .

‎datafusion/tests/test_dataframe.py‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,12 @@ def struct_df():
5252

5353
return ctx.create_dataframe([[batch]])
5454

55+
5556
@pytest.fixture
5657
def aggregate_df():
5758
ctx = SessionContext()
58-
ctx.register_csv('test', 'testing/data/csv/aggregate_test_100.csv')
59-
return ctx.sql('select c1, sum(c2) from test group by c1')
59+
ctx.register_csv("test", "testing/data/csv/aggregate_test_100.csv")
60+
return ctx.sql("select c1, sum(c2) from test group by c1")
6061

6162

6263
def test_select(df):
@@ -271,10 +272,11 @@ def test_logical_plan(aggregate_df):
271272

272273
assert expected == plan.display()
273274

274-
expected = \
275-
"Projection: test.c1, SUM(test.c2)\n" \
276-
" Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" \
275+
expected = (
276+
"Projection: test.c1, SUM(test.c2)\n"
277+
" Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n"
277278
" TableScan: test"
279+
)
278280

279281
assert expected == plan.display_indent()
280282

@@ -286,25 +288,29 @@ def test_optimized_logical_plan(aggregate_df):
286288

287289
assert expected == plan.display()
288290

289-
expected = \
290-
"Projection: test.c1, SUM(test.c2)\n" \
291-
" Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" \
291+
expected = (
292+
"Projection: test.c1, SUM(test.c2)\n"
293+
" Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n"
292294
" TableScan: test projection=[c1, c2]"
295+
)
293296

294297
assert expected == plan.display_indent()
295298

296299

297300
def test_execution_plan(aggregate_df):
298301
plan = aggregate_df.execution_plan()
299302

300-
expected = "ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n"
303+
expected = (
304+
"ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n"
305+
)
301306

302307
assert expected == plan.display()
303308

304-
expected = \
305-
"ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n" \
306-
" Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n" \
309+
expected = (
310+
"ProjectionExec: expr=[c1@0 as c1, SUM(test.c2)@1 as SUM(test.c2)]\n"
311+
" Aggregate: groupBy=[[test.c1]], aggr=[[SUM(test.c2)]]\n"
307312
" TableScan: test projection=[c1, c2]"
313+
)
308314

309315
indent = plan.display_indent()
310316

@@ -317,7 +323,6 @@ def test_execution_plan(aggregate_df):
317323
assert "CsvExec:" in indent
318324

319325

320-
321326
def test_repartition(df):
322327
df.repartition(2)
323328

‎datafusion/tests/test_substrait.py‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,8 @@
1616
# under the License.
1717

1818
import pyarrow as pa
19-
import pyarrow.dataset as ds
2019

21-
from datafusion import column, literal, SessionContext
20+
from datafusion import SessionContext
2221
from datafusion import substrait as ss
2322
import pytest
2423

@@ -39,8 +38,14 @@ def test_substrait_serialization(ctx):
3938
assert ctx.tables() == {"t"}
4039

4140
# For now just make sure the method calls blow up
42-
substrait_plan = ss.substrait.serde.serialize_to_plan("SELECT * FROM t", ctx)
43-
substrait_bytes = ss.substrait.serde.serialize_bytes("SELECT * FROM t", ctx)
41+
substrait_plan = ss.substrait.serde.serialize_to_plan(
42+
"SELECT * FROM t", ctx
43+
)
44+
substrait_bytes = ss.substrait.serde.serialize_bytes(
45+
"SELECT * FROM t", ctx
46+
)
4447
substrait_plan = ss.substrait.serde.deserialize_bytes(substrait_bytes)
45-
df_logical_plan = ss.substrait.consumer.from_substrait_plan(ctx, substrait_plan)
48+
df_logical_plan = ss.substrait.consumer.from_substrait_plan(
49+
ctx, substrait_plan
50+
)
4651
substrait_plan = ss.substrait.producer.to_substrait_plan(df_logical_plan)

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ classifier = [
4343
"Programming Language :: Rust",
4444
]
4545
dependencies = [
46-
"pyarrow>=1",
46+
"pyarrow>=6.0.1",
4747
]
4848

4949
[project.urls]

0 commit comments

Comments
 (0)