-
Notifications
You must be signed in to change notification settings - Fork 266
74 lines (74 loc) · 2.38 KB
/
python-ci.yml
File metadata and controls
74 lines (74 loc) · 2.38 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
name: Python CI
on: [push, pull_request]
jobs:
test:
strategy:
fail-fast: false
matrix:
include:
- spark-version: 3.5.7
scala-version: 2.12
python-version: "3.10"
java-version: 11
- spark-version: 4.0.1
scala-version: 2.13
python-version: "3.12"
java-version: 17
- spark-version: 4.1.0
scala-version: 2.13
python-version: "3.13"
java-version: 17
runs-on: ubuntu-latest
env:
# define Java options for both official sbt and sbt-extras
JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
SPARK_VERSION: ${{ matrix.spark-version }}
SCALA_VERSION: ${{ matrix.scala-version }}
PIP_REQUESTS_TIMEOUT: 100
steps:
- uses: actions/checkout@v6
- uses: actions/setup-java@v5
with:
distribution: "zulu"
java-version: "${{ matrix.java-version }}"
- uses: actions/cache@v5
with:
path: |
~/.ivy2/cache
key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }}
- uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install and configure Poetry
uses: snok/install-poetry@v1
with:
version: 2.1.3
virtualenvs-create: true
virtualenvs-in-project: false
installer-parallel: true
- name: Build Python package and its dependencies
working-directory: ./python
run: |
poetry install --with=dev
poetry run pip install pyspark[connect]==${{ matrix.spark-version }}
- name: Code style
working-directory: ./python
run: |
poetry run python -m black --check graphframes
poetry run python -m flake8 graphframes
poetry run python -m isort --check graphframes
- name: Build jar
working-directory: ./python
run: |
poetry run python dev/build_jar.py ${{ matrix.spark-version }}
- name: Test
working-directory: ./python
run: |
poetry run python -m pytest
- name: Test SparkConnect
env:
SPARK_CONNECT_MODE_ENABLED: 1
working-directory: ./python
run: |
poetry run python -m pytest