-
Notifications
You must be signed in to change notification settings - Fork 266
69 lines (69 loc) · 2.22 KB
/
python-ci.yml
File metadata and controls
69 lines (69 loc) · 2.22 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
name: Python CI
on: [push, pull_request]
jobs:
test:
strategy:
fail-fast: false
matrix:
include:
- spark-version: 3.5.5
scala-version: 2.12
python-version: "3.10"
java-version: 11
- spark-version: 4.0.0
scala-version: 2.13
python-version: 3.12
java-version: 17
runs-on: ubuntu-22.04
env:
# define Java options for both official sbt and sbt-extras
JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
SPARK_VERSION: ${{ matrix.spark-version }}
SCALA_VERSION: ${{ matrix.scala-version }}
steps:
- uses: actions/checkout@v4
- uses: actions/setup-java@v4
with:
distribution: "zulu"
java-version: "${{ matrix.java-version }}"
- uses: actions/cache@v4
with:
path: |
~/.ivy2/cache
key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }}
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install and configure Poetry
uses: snok/install-poetry@v1
with:
version: 2.1.3
virtualenvs-create: true
virtualenvs-in-project: false
installer-parallel: true
- name: Build Python package and its dependencies
working-directory: ./python
run: |
poetry install --with=dev
poetry run pip install pyspark[connect]==${{ matrix.spark-version }}
- name: Code style
working-directory: ./python
run: |
poetry run python -m black --check graphframes
poetry run python -m flake8 graphframes
poetry run python -m isort --check graphframes
- name: Build jar
working-directory: ./python
run: |
poetry run python dev/build_jar.py ${{ matrix.spark-version }}
- name: Test
working-directory: ./python
run: |
poetry run python -m pytest
- name: Test SparkConnect
env:
SPARK_CONNECT_MODE_ENABLED: 1
working-directory: ./python
run: |
poetry run python -m pytest