locationtech · metasim · Sep 27, 2021 · Sep 17, 2021 · Sep 17, 2021 · Sep 17, 2021
diff --git a/.circleci/.dockerignore b/.circleci/.dockerignore
@@ -0,0 +1,3 @@
+*
+!requirements-conda.txt
+!fix-permissions
diff --git a/.circleci/Dockerfile b/.circleci/Dockerfile
@@ -1,63 +1,35 @@
-FROM circleci/openjdk:8-jdk
-
-ENV MINICONDA_VERSION=4.8.2 \
-    MINICONDA_MD5=87e77f097f6ebb5127c77662dfc3165e \
-    CONDA_VERSION=4.8.2 \
-    CONDA_DIR=/opt/conda \
-    PYTHON_VERSION=3.7.7
+FROM circleci/openjdk:11-jdk
+#LABEL org.opencontainers.image.source=https://github.com/locationtech/rasterframes
 
 USER root
 
-ENV PATH=$CONDA_DIR/bin:$PATH
-
-# circleci is 3434
-COPY --chown=3434:3434 fix-permissions /tmp
-
+# See: https://docs.conda.io/projects/conda/en/latest/user-guide/install/rpm-debian.html
 RUN \
-    apt-get update && \
-    apt-get install -yq --no-install-recommends \
-        sudo \
-        wget \
-        bzip2 \
-        file \
-        libtinfo5 \
-        ca-certificates \
-        gettext-base \
-        locales && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+  curl -s https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmor > conda.gpg  && \
+  install -o root -g root -m 644 conda.gpg /usr/share/keyrings/conda-archive-keyring.gpg && \
+  gpg --keyring /usr/share/keyrings/conda-archive-keyring.gpg --no-default-keyring --fingerprint 34161F5BF5EB1D4BFBBB8F0A8AEB4F8B29D82806 && \
+  echo "deb [arch=amd64 signed-by=/usr/share/keyrings/conda-archive-keyring.gpg] https://repo.anaconda.com/pkgs/misc/debrepo/conda stable main" > /etc/apt/sources.list.d/conda.list
 
 RUN \
-    cd /tmp && \
-    mkdir -p $CONDA_DIR && \
-    wget --quiet https://repo.continuum.io/miniconda/Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
-    echo "${MINICONDA_MD5} *Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh" | md5sum -c - && \
-    /bin/bash Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh -f -b -p $CONDA_DIR && \
-    rm Miniconda3-py37_${MINICONDA_VERSION}-Linux-x86_64.sh && \
-    conda config --system --set auto_update_conda false && \
-    conda config --system --set show_channel_urls true && \
-    conda config --system --set channel_priority strict && \
-    if [ ! $PYTHON_VERSION = 'default' ]; then conda install --yes python=$PYTHON_VERSION; fi && \
-    conda list python | grep '^python ' | tr -s ' ' | cut -d '.' -f 1,2 | sed 's/$/.*/' >> $CONDA_DIR/conda-meta/pinned && \
-    conda install --quiet --yes conda && \
-    conda install --quiet --yes pip && \
-    pip config set global.progress_bar off && \
-    echo "$CONDA_DIR/lib" > /etc/ld.so.conf.d/conda.conf && \
-    conda clean --all --force-pkgs-dirs --yes --quiet && \
-    sh /tmp/fix-permissions $CONDA_DIR 2> /dev/null
+  apt-get update && \
+  apt-get install -yq --no-install-recommends conda && \
+  apt-get clean && \
+  rm -rf /var/lib/apt/lists/*
 
-COPY requirements-conda.txt /tmp/
+ENV CONDA_DIR=/opt/conda
+ENV PATH=$CONDA_DIR/bin:$PATH
 
+COPY requirements-conda.txt fix-permissions /tmp
 RUN \
-    conda install --channel conda-forge --no-channel-priority --freeze-installed \
-        --file /tmp/requirements-conda.txt && \
+    conda install --quiet --yes --channel=conda-forge --file=/tmp/requirements-conda.txt && \
+    echo "$CONDA_DIR/lib" > /etc/ld.so.conf.d/conda.conf && \
+    ldconfig && \
     conda clean --all --force-pkgs-dirs --yes --quiet && \
-    sh /tmp/fix-permissions $CONDA_DIR 2> /dev/null && \
-    ldconfig 2> /dev/null
+    sh /tmp/fix-permissions $CONDA_DIR
+
 
 # Work-around for pyproj issue https://github.com/pyproj4/pyproj/issues/415
 ENV PROJ_LIB=/opt/conda/share/proj
 
 USER 3434
-
 WORKDIR /home/circleci
diff --git a/.circleci/Makefile b/.circleci/Makefile
@@ -1,19 +1,27 @@
-IMAGE_NAME=miniconda-gdal
-VERSION=latest
-HOST=docker.pkg.github.com
-REPO=${HOST}/locationtech/rasterframes
-FULL_NAME=${REPO}/${IMAGE_NAME}:${VERSION}
+IMAGE_NAME=circleci-openjdk-conda-gdal
+SHA=$(shell git log -n1 --format=format:"%H" | cut -c 1-7)
+VERSION?=$(SHA)
+HOST=docker.io
+REPO=$(HOST)/s22s
+FULL_NAME=$(REPO)/$(IMAGE_NAME):$(VERSION)
 
-all: build login push
+.DEFAULT_GOAL := help
+help:
+# http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
+	@echo "Usage: make [target]"
+	@echo "Targets: "
+	@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\t\033[36m%-20s\033[0m %s\n", $$1, $$2}'
 
-build:
+all: build push ## Build and then push image
+
+build: ## Build the docker image
 	docker build . -t ${FULL_NAME}
 
-login:
-	docker login ${HOST}
+login: ## Login to the docker registry
+	docker login
 
-push:
+push: login ## Push docker image to registry
 	docker push ${FULL_NAME}
 
-shell: build
+run: build ## Build image and launch shell
 	docker run --rm -it ${FULL_NAME} bash
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -6,11 +6,10 @@ orbs:
     executors:
       default:
         docker:
-          - image: s22s/miniconda-gdal:latest
+          - image: s22s/circleci-openjdk-conda-gdal:b8e30ee
         working_directory: ~/repo
         environment:
-          SBT_VERSION: 1.3.8
-          SBT_OPTS: -Xmx768m
+          SBT_OPTS: "-Xms64m -Xmx1536m -Djava.awt.headless=true -Dsun.io.serialization.extendedDebugInfo=true"
     commands:
       setup:
         description: Setup for sbt build
@@ -24,8 +23,7 @@ orbs:
         steps:
           - run:
               name: "Compile Scala via sbt"
-              command: |-
-                sbt -v -batch compile test:compile it:compile
+              command: sbt -v -batch compile test:compile it:compile
 
   python:
     commands:
@@ -60,6 +58,7 @@ orbs:
                 mkdir -p /tmp/core_dumps
                 ls -lh /tmp
                 cp core.* *.hs /tmp/core_dumps 2> /dev/null || true
+                cp core/* /tmp/core_dumps/ 2> /dev/null || true
                 cp -r /tmp/hsperfdata* /tmp/*.hprof /tmp/core_dumps 2> /dev/null || true
               when: on_fail
 
@@ -125,24 +124,23 @@ jobs:
 
       - run:
           name: "Scala Tests: core"
-          command: sbt -batch core/test
+          command: sbt -v -batch core/test
 
       - run:
           name: "Scala Tests: datasource"
-          command: sbt -batch datasource/test
+          command: sbt -v -batch datasource/test
 
       - run:
           name: "Scala Tests: experimental"
-          command: sbt -batch experimental/test
+          command: sbt -v -batch experimental/test
 
       - run:
           name: "Create PyRasterFrames package"
-          command: |-
-            sbt -v -batch pyrasterframes/package
+          command: sbt -v -batch pyrasterframes/package
 
       - run:
           name: "Python Tests"
-          command: sbt -batch pyrasterframes/test
+          command: sbt -v -batch pyrasterframes/test
 
       - rasterframes/save-artifacts
       - rasterframes/save-cache
@@ -249,4 +247,4 @@ workflows:
       - test
       - it
       - it-no-gdal
-      - docs
+      - docs
diff --git a/.circleci/fix-permissions b/.circleci/fix-permissions
@@ -1,19 +1,4 @@
 #!/usr/bin/env bash
-# set permissions on a directory
-# after any installation, if a directory needs to be (human) user-writable,
-# run this script on it.
-# It will make everything in the directory owned by the group $NB_GID
-# and writable by that group.
-# Deployments that want to set a specific user id can preserve permissions
-# by adding the `--group-add users` line to `docker run`.
-
-# uses find to avoid touching files that already have the right permissions,
-# which would cause massive image explosion
-
-# right permissions are:
-# group=$NB_GID
-# AND permissions include group rwX (directory-execute)
-# AND directories have setuid,setgid bits set
 
 set -e
 

diff --git a/.circleci/requirements-conda.txt b/.circleci/requirements-conda.txt
@@ -1,3 +1,5 @@
-gdal==2.4.4
+python==3.8
+gdal==3.1.2
 libspatialindex
+rasterio[s3]
 rtree
diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml
@@ -0,0 +1,131 @@
+name: Build and Test
+
+on:
+  pull_request:
+    branches: ['**']
+  push:
+    branches: ['main']
+    tags: [v*]
+  release:
+    types: [published]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    container:
+      image: s22s/circleci-openjdk-conda-gdal:b8e30ee
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: coursier/cache-action@v6
+      - uses: olafurpg/setup-scala@v13
+        with:
+          java-version: adopt@1.11
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install Conda dependencies
+        run: |
+          # $CONDA is an environment variable pointing to the root of the miniconda directory
+          $CONDA/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt
+
+      - run: ulimit -c unlimited -S
+
+      # Do just the compilation stage to minimize sbt memory footprint
+      - name: Compile
+        run: sbt -v -batch compile test:compile it:compile
+
+      - name: Core tests
+        run: sbt -batch core/test
+
+      - name: Datasource tests
+        run: sbt -batch datasource/test
+
+      - name: Experimental tests
+        run: sbt -batch experimental/test
+
+      - name: Create PyRasterFrames package
+        run: sbt -v -batch pyrasterframes/package
+
+      - name: Python tests
+        run: sbt -batch pyrasterframes/test
+
+      - name: Collect artifacts
+        if: ${{ failure() }}
+        run: |
+          mkdir -p /tmp/core_dumps
+          ls -lh /tmp
+          cp core.* *.hs /tmp/core_dumps/ 2> /dev/null || true
+          cp ./core/*.log /tmp/core_dumps/ 2> /dev/null || true
+          cp -r /tmp/hsperfdata* /tmp/*.hprof /tmp/core_dumps/ 2> /dev/null || true
+          cp repo/core/core/* /tmp/core_dumps/ 2> /dev/null || true
+
+      - name: Upload core dumps
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: core-dumps
+          path: /tmp/core_dumps
+
+  docs:
+    runs-on: ubuntu-latest
+    container:
+      image: s22s/circleci-openjdk-conda-gdal:b8e30ee
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - uses: coursier/cache-action@v6
+      - uses: olafurpg/setup-scala@v13
+        with:
+          java-version: adopt@1.11
+
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      - name: Install Conda dependencies
+        run: |
+          # $CONDA is an environment variable pointing to the root of the miniconda directory
+          $CONDA/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt
+
+      - run: ulimit -c unlimited -S
+
+      - name: Build documentation
+        run: sbt makeSite
+
+      - name: Collect artifacts
+        if: ${{ failure() }}
+        run: |
+          mkdir -p /tmp/core_dumps
+          cp core.* *.hs /tmp/core_dumps 2> /dev/null || true
+          mkdir -p /tmp/markdown
+          cp pyrasterframes/target/python/docs/*.md /tmp/markdown 2> /dev/null || true
+
+      - name: Upload core dumps
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: core-dumps
+          path: /tmp/core_dumps
+
+      - name: Upload markdown
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: markdown
+          path: /tmp/markdown
+
+      - name: Upload rf-site
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: rf-site
+          path: docs/target/site
diff --git a/.sbtopts b/.sbtopts
diff --git a/.scalafmt.conf b/.scalafmt.conf
@@ -1,4 +1,4 @@
-version = 3.0.3
+version = 3.0.4
 runner.dialect = scala212
 indent.main = 2
 indent.significant = 2