Repository navigation
102 lines (101 loc) · 3.34 KB
/
Copy pathpython-ci.yml
File metadata and controls
102 lines (101 loc) · 3.34 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
name: Python CI
on: [push, pull_request]
permissions:
contents: read
jobs:
# Formatting and lint run as their own job: once (not once per Spark matrix
# entry) and without the Spark/JDK setup, so style feedback is not gated on
# the full test matrix.
#
# Keep the checked paths in sync with .pre-commit-config.yaml.
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install and configure Poetry
uses: snok/install-poetry@v1
with:
version: 2.1.3
virtualenvs-create: true
virtualenvs-in-project: false
installer-parallel: true
- name: Install lint dependencies
working-directory: ./python
run: poetry install --with=dev
- name: black
working-directory: ./python
run: poetry run python -m black --check --config pyproject.toml graphframes tests docs dev ../dev
- name: isort
working-directory: ./python
run: poetry run python -m isort --check --settings-path pyproject.toml graphframes tests docs dev ../dev
- name: flake8
working-directory: ./python
run: poetry run python -m flake8 graphframes tests docs dev ../dev
test:
strategy:
fail-fast: false
matrix:
include:
- spark-version: 3.5.8
scala-version: 2.12
python-version: "3.10"
java-version: 11
- spark-version: 4.0.3
scala-version: 2.13
python-version: "3.12"
java-version: 17
- spark-version: 4.1.2
scala-version: 2.13
python-version: "3.13"
java-version: 17
runs-on: ubuntu-latest
env:
# define Java options for both official sbt and sbt-extras
JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
SPARK_VERSION: ${{ matrix.spark-version }}
SCALA_VERSION: ${{ matrix.scala-version }}
PIP_REQUESTS_TIMEOUT: 100
steps:
- uses: actions/checkout@v6
- uses: actions/setup-java@v5
with:
distribution: "zulu"
java-version: "${{ matrix.java-version }}"
- uses: actions/cache@v5
with:
path: |
~/.ivy2/cache
key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }}
- uses: actions/setup-python@v6
with:
python-version: ${{ matrix.python-version }}
- name: Install and configure Poetry
uses: snok/install-poetry@v1
with:
version: 2.1.3
virtualenvs-create: true
virtualenvs-in-project: false
installer-parallel: true
- name: Build Python package and its dependencies
working-directory: ./python
run: |
poetry install --with=dev
poetry run pip install pyspark[connect]==${{ matrix.spark-version }}
- name: Build jar
working-directory: ./python
run: |
poetry run python dev/build_jar.py ${{ matrix.spark-version }}
- name: Test
working-directory: ./python
run: |
poetry run python -m pytest
- name: Test SparkConnect
env:
SPARK_CONNECT_MODE_ENABLED: 1
working-directory: ./python
run: |
poetry run python -m pytest