forked from graphframes/graphframes
-
Notifications
You must be signed in to change notification settings - Fork 0
51 lines (51 loc) · 1.85 KB
/
python-ci.yml
File metadata and controls
51 lines (51 loc) · 1.85 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
name: Python CI
on: [push, pull_request]
jobs:
test:
strategy:
fail-fast: false
matrix:
include:
- spark-version: 3.5.0
scala-version: 2.12.18
python-version: 3.9
- spark-version: 3.4.1
scala-version: 2.12.17
python-version: 3.9
- spark-version: 3.3.3
scala-version: 2.12.15
python-version: 3.9
- spark-version: 3.2.4
scala-version: 2.12.15
python-version: 3.9
runs-on: ubuntu-22.04
env:
# define Java options for both official sbt and sbt-extras
JAVA_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
JVM_OPTS: -Xms2048M -Xmx2048M -Xss6M -XX:ReservedCodeCacheSize=256M -Dfile.encoding=UTF-8
SPARK_VERSION: ${{ matrix.spark-version }}
SCALA_VERSION: ${{ matrix.scala-version }}
steps:
- uses: actions/checkout@v3
- uses: olafurpg/setup-scala@v13
with:
java-version: "openjdk@1.11"
- uses: actions/cache@v3
with:
path: |
~/.ivy2/cache
key: sbt-ivy-cache-spark-${{ matrix.spark-version}}-scala-${{ matrix.scala-version }}
- name: Assembly
run: sbt -v ++${{ matrix.scala-version }} -Dspark.version=${{ matrix.spark-version }} "set test in assembly := {}" assembly
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Install python dependencies
run: |
python -m pip install --upgrade pip wheel
pip install -r ./python/requirements.txt
pip install pyspark==${{ matrix.spark-version }}
- name: Test
run: |
export SPARK_HOME=$(python -c "import os; from importlib.util import find_spec; print(os.path.join(os.path.dirname(find_spec('pyspark').origin)))")
./python/run-tests.sh