locationtech · thomas-maschler · Jan 21, 2023 · Jan 21, 2023 · Jan 21, 2023 · Jan 21, 2023
diff --git a/.github/actions/collect_artefacts/action.yml b/.github/actions/collect_artefacts/action.yml
@@ -0,0 +1,10 @@
+name: upload rasterframes artefacts
+description: upload rasterframes artefacts
+runs:
+  using: "composite"
+  steps:
+    - name: upload core dumps
+      uses: actions/upload-artifact@v3
+      with:
+          name: core-dumps
+          path: /tmp/core_dumps
diff --git a/.github/actions/python_build/action.yml b/.github/actions/python_build/action.yml
@@ -0,0 +1,25 @@
+name: build rasterframes python
+description: build rasterframes python
+runs:
+  using: "composite"
+  steps:
+    - name: Configure python interpreter
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python }}
+    - uses: abatilo/actions-poetry@v2
+    - name: Install dependencies
+      shell: bash
+      run: |
+        make init-python
+    - name: Static checks
+      shell: bash
+      run: make lint-python
+    - name: Run tests
+      shell: bash
+      run: |
+        make test-python
+    - name: Build wheel
+      shell: bash
+      run:
+        make build-python
diff --git a/.github/actions/scala_build/action.yml b/.github/actions/scala_build/action.yml
@@ -0,0 +1,39 @@
+name: build rasterframes scala
+description: build rasterframes scala
+inputs: 
+  skip_tests:
+    description: 'Skip Scala Tests? true or false'
+    default: 'false'
+runs:
+  using: "composite"
+  steps:
+        - uses: coursier/cache-action@v6
+        - name: Configure JDK
+          uses: actions/setup-java@v3
+          with:
+            distribution: temurin
+            java-version: 8
+            cache: sbt
+        - name: Compile Scala
+          shell: bash
+          run: make compile-scala
+
+        - name: Test Scale
+          if: inputs.skip_tests == 'false'
+          shell: bash
+          run: make test-scala
+
+        - name: Build FatJar
+          shell: bash
+          run: make build-scala
+
+        # - name: Collect artifacts
+        #   if: ${{ failure() }}
+        #   shell: bash
+        #   run: |
+        #     mkdir -p /tmp/core_dumps
+        #     ls -lh /tmp
+        #     cp core.* *.hs /tmp/core_dumps/ 2> /dev/null || true
+        #     cp ./core/*.log /tmp/core_dumps/ 2> /dev/null || true
+        #     cp -r /tmp/hsperfdata* /tmp/*.hprof /tmp/core_dumps/ 2> /dev/null || true
+        #     cp repo/core/core/* /tmp/core_dumps/ 2> /dev/null || true
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -0,0 +1,37 @@
+name: build main
+on:
+  push:
+   tags: 
+    - 'v*'
+   branches: 
+      - 'master' 
+      - 'develop'
+      - 'release/*'
+      - 'spark-3.2'
+    branches-ignore:
+      - "python/*"
+      - "scala/*"
+  pull_request:
+    branches:
+      - '**'
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix:
+        python: [ 3.8.10 ]
+        spark: [ 3.2.1 ]
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: build scala
+        uses: ./.github/actions/scala_build
+      - name: build python
+        uses: ./.github/actions/python_build
+      - name: upload artefacts
+        uses: ./.github/actions/upload_artefacts
diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml
@@ -0,0 +1,29 @@
+name: build_python
+# build for python, skip scala tests and R build
+on:
+  push:
+    branches:
+      - "python/*"
+  pull_request:
+    branches:
+      - "python/*"
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix:
+        python: [ 3.8.10 ]
+        spark: [ 3.2.1 ]
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: build scala with skipping tests
+        uses: ./.github/actions/scala_build
+        with: 
+          skip_tests: "true"
+      - name: build python
+        uses: ./.github/actions/python_build
+      # - name: upload artefacts
+      #   uses: ./.github/actions/upload_artefacts
diff --git a/.github/workflows/build_scala.yml b/.github/workflows/build_scala.yml
@@ -0,0 +1,24 @@
+name: build_scala
+on:
+  push:
+    branches:
+      - "scala/"
+  pull_request:
+    branches:
+      - "scala/"
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix:
+        python: [ 3.8.10 ]
+        spark: [ 3.2.1 ]
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: build scala with tests
+        uses: ./.github/actions/scala_build
+      # - name: upload artefacts
+      #   uses: ./.github/actions/upload_artefacts
diff --git a/.gitignore b/.gitignore
@@ -48,3 +48,16 @@ rf-notebook/src/main/notebooks/.ipython
 .bloop
 metals.sbt
 *.parquet/
+
+# Python
+
+.coverage
+.venv
+htmlcov
+dist/
+docs/*.md
+docs/*.ipynb
+__pycache__
+*.pipe/
+.coverage*
+*.jar
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,24 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+
+files: ^python/
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: local
+    hooks:
+      - id: black
+        name: black formatting
+        language: system
+        types: [python]
+        entry: poetry run black
+
+      - id: isort
+        name: isort import sorting
+        language: system
+        types: [python]
+        entry: poetry run isort
+        args: ["--profile", "black"]
diff --git a/Makefile b/Makefile
@@ -0,0 +1,96 @@
+SHELL := /usr/bin/env bash
+
+.PHONY: init test lint build docs notebooks help
+
+help:
+	@echo "init - Setup the repository"
+	@echo "clean - clean all compiled python files, build artifacts and virtual envs. Run \`make init\` anew afterwards."
+	@echo "test - run unit tests"
+	@echo "lint - run linter and checks"
+	@echo "build - build wheel"
+	@echo "docs - build documentations"
+	@echo "help - this command"
+
+
+###############
+# SCALA
+###############
+
+compile-scala:
+	sbt -v -batch compile test:compile it:compile
+
+test-scala: test-core-scala test-datasource-scala test-experimental-scala
+
+test-core-scala:
+	sbt -batch core/test
+
+test-datasource-scala:
+	sbt -batch datasource/test
+
+test-experimental-scala:
+	sbt -batch experimental/test
+
+build-scala:
+	sbt "pyrasterframes/assembly"
+	mkdir -p python/pyrasterframes/jars/
+	cp pyrasterframes/target/scala-2.12/pyrasterframes-assembly-*.jar python/pyrasterframes/jars/
+
+
+################
+# PYTHON
+################
+
+init-python:
+	python -m venv ./.venv
+	./.venv/bin/python -m pip install --upgrade pip
+	poetry install
+	poetry run pre-commit install
+
+test-python: build-scala
+	poetry run pytest -vv python/tests --cov=python/pyrasterframes --cov=python/geomesa_pyspark --cov-report=term-missing
+
+lint-python:
+	poetry run pre-commit run --all-file
+
+build-python: clean-build
+	poetry build
+
+docs-python: clean-md
+	poetry run python python/docs/build_docs.py
+
+notebooks-python: clean-ipynb
+	poetry run python python/docs/build_docs.py --format notebook
+
+clean-python: clean-build-python clean-pyc-python clean-test-python clean-venv-python clean-docs-python clean-notebooks-python
+
+clean-build-python:
+	rm -fr build/
+	rm -fr dist/
+	rm -fr .eggs/
+	rm -fr wheelhouse/
+	find . -name '*.egg-info' -exec rm -fr {} +
+	find . -name '*.egg' -exec rm -f {} +
+	find . -name '*.so' -exec rm -f {} +
+	find . -name '*.c' -exec rm -f {} +
+	find . -name '*.html' -exec rm -f {} +
+
+clean-pyc-python:
+	find . -name '*.pyc' -exec rm -f {} +
+	find . -name '*.pyo' -exec rm -f {} +
+	find . -name '*~' -exec rm -f {} +
+	find . -name '__pycache__' -exec rm -fr {} +
+
+clean-test-python:
+	rm -f .coverage
+	rm -fr htmlcov/
+	rm -fr test*.pipe
+
+
+clean-venv-python:
+	rm -fr .venv/
+
+clean-docs-python:
+	find docs -name '*.md' -exec rm -f {} +
+
+clean-notebooks-python:
+	find docs -name '*.ipynb' -exec rm -f {} +