locationtech · metasim · Sep 16, 2021 · May 13, 2021 · May 13, 2021 · May 13, 2021
diff --git a/.gitignore b/.gitignore
@@ -1,7 +1,13 @@
+# Operating System Files
+
+*.DS_Store
+Thumbs.db
+
 *.class
 *.log
 
 # sbt specific
+.bsp
 .cache
 .history
 .lib/

diff --git a/.java-version b/.java-version
diff --git a/.scalafmt.conf b/.scalafmt.conf
@@ -1,16 +1,11 @@
-maxColumn = 138
+version = 3.0.3
+runner.dialect = scala212
+indent.main = 2
+indent.significant = 2
+maxColumn = 150
 continuationIndent.defnSite = 2
-binPack.parentConstructors = true
-binPack.literalArgumentLists = false
-newlines.penalizeSingleSelectMultiArgList = false
-newlines.sometimesBeforeColonInMethodReturnType = false
-align.openParenCallSite = false
-align.openParenDefnSite = false
-docstrings = JavaDoc
-rewriteTokens {
-  "⇒" = "=>"
-  "←" = "<-"
-}
-optIn.selfAnnotationNewline = false
-optIn.breakChainOnFirstMethodDot = true
-importSelectors = BinPack
+assumeStandardLibraryStripMargin = true
+danglingParentheses.preset = true
+rewrite.rules = [SortImports, RedundantBraces, RedundantParens, SortModifiers]
+docstrings.style = Asterisk
+# align.preset = more
diff --git a/.sdkmanrc b/.sdkmanrc
@@ -0,0 +1,3 @@
+# Enable auto-env through the sdkman_auto_env config
+# Add key=value pairs of SDKs to use below
+java=11.0.11.hs-adpt
diff --git a/bench/build.sbt b/bench/build.sbt
@@ -11,7 +11,7 @@ libraryDependencies ++= Seq(
 jmhIterations := Some(5)
 jmhWarmupIterations := Some(8)
 jmhTimeUnit := None
-javaOptions in Jmh := Seq("-Xmx4g")
+Jmh / javaOptions := Seq("-Xmx4g")
 
 // To enable profiling:
 // jmhExtraOptions := Some("-prof jmh.extras.JFR")

diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala
diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/CellTypeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/CellTypeBench.scala
@@ -21,10 +21,9 @@
 
 package org.locationtech.rasterframes.bench
 import java.util.concurrent.TimeUnit
-
 import geotrellis.raster.{CellType, DoubleUserDefinedNoDataCellType, IntUserDefinedNoDataCellType}
 import org.apache.spark.sql.catalyst.InternalRow
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes.encoders.StandardEncoders
 import org.openjdk.jmh.annotations._
 
 @BenchmarkMode(Array(Mode.AverageTime))
@@ -37,16 +36,12 @@ class CellTypeBench {
   def setupData(): Unit = {
     ct = IntUserDefinedNoDataCellType(scala.util.Random.nextInt())
     val o: CellType = DoubleUserDefinedNoDataCellType(scala.util.Random.nextDouble())
-    row = o.toInternalRow
+    row = StandardEncoders.cellTypeEncoder.createSerializer()(o)
   }
 
   @Benchmark
-  def fromRow(): CellType = {
-    row.to[CellType]
-  }
+  def fromRow(): CellType = StandardEncoders.cellTypeEncoder.createDeserializer()(row)
 
   @Benchmark
-  def intoRow(): InternalRow = {
-    ct.toInternalRow
-  }
+  def intoRow(): InternalRow = StandardEncoders.cellTypeEncoder.createSerializer()(ct)
 }
diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala
@@ -26,7 +26,6 @@ import java.util.concurrent.TimeUnit
 import geotrellis.raster.Dimensions
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.rf.TileUDT
-import org.locationtech.rasterframes.tiles.InternalRowTile
 import org.openjdk.jmh.annotations._
 
 @BenchmarkMode(Array(Mode.AverageTime))
@@ -62,15 +61,4 @@ class TileCellScanBench extends SparkEnv {
       tile.getDouble(cols/2, rows/2) +
       tile.getDouble(0, 0)
   }
-
-  @Benchmark
-  def internalRowRead(): Double  = {
-    val tile = new InternalRowTile(tileRow)
-    val cols = tile.cols
-    val rows = tile.rows
-    tile.getDouble(cols - 1, rows - 1) +
-      tile.getDouble(cols/2, rows/2) +
-      tile.getDouble(0, 0)
-  }
 }
-
diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala
@@ -24,8 +24,6 @@ package org.locationtech.rasterframes.bench
 import java.net.URI
 import java.util.concurrent.TimeUnit
 
-import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile
-import org.locationtech.rasterframes.ref.RasterRef
 import geotrellis.raster.Tile
 import geotrellis.vector.Extent
 import org.apache.spark.sql.catalyst.InternalRow
@@ -53,24 +51,23 @@ class TileEncodeBench extends SparkEnv {
   @Setup(Level.Trial)
   def setupData(): Unit = {
     cellTypeName match {
-      case "rasterRef" ⇒
+      case "rasterRef" =>
         val baseCOG = "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_B1.TIF"
         val extent = Extent(253785.0, 3235185.0, 485115.0, 3471015.0)
-        tile = RasterRefTile(RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None))
-      case _ ⇒
+        tile = RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None)
+      case _ =>
         tile = randomTile(tileSize, tileSize, cellTypeName)
     }
   }
 
   @Benchmark
   def encode(): InternalRow = {
-    tileEncoder.toRow(tile)
+    tileEncoder.createSerializer.apply(tile)
   }
 
   @Benchmark
   def roundTrip(): Tile = {
-    val row = tileEncoder.toRow(tile)
-    boundEncoder.fromRow(row)
+    val row = tileEncoder.createSerializer().apply(tile)
+    boundEncoder.createDeserializer().apply(row)
   }
 }
-
diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/package.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/package.scala
@@ -37,10 +37,10 @@ package object bench {
     val cellType = CellType.fromName(cellTypeName)
     val tile = ArrayTile.alloc(cellType, cols, rows)
     if(cellType.isFloatingPoint) {
-      tile.mapDouble(_ ⇒ rnd.nextGaussian())
+      tile.mapDouble(_ => rnd.nextGaussian())
     }
     else {
-      tile.map(_ ⇒ {
+      tile.map(_ => {
         var c = NODATA
         do {
           c = rnd.nextInt(255)

diff --git a/build.sbt b/build.sbt
@@ -30,7 +30,7 @@ lazy val IntegrationTest = config("it") extend Test
 lazy val root = project
   .in(file("."))
   .withId("RasterFrames")
-  .aggregate(core, datasource, pyrasterframes, experimental)
+  .aggregate(core, datasource, pyrasterframes)
   .enablePlugins(RFReleasePlugin)
   .settings(
     publish / skip := true,
@@ -52,19 +52,23 @@ lazy val core = project
     libraryDependencies ++= Seq(
       `slf4j-api`,
       shapeless,
+      frameless excludeAll ExclusionRule("com.github.mpilquist", "simulacrum"),
       `jts-core`,
       `spray-json`,
       geomesa("z3").value,
       geomesa("spark-jts").value,
       spark("core").value % Provided,
       spark("mllib").value % Provided,
       spark("sql").value % Provided,
-      geotrellis("spark").value,
-      geotrellis("raster").value,
-      geotrellis("s3").value,
+      // TODO: scala-uri brings an outdated simulacrum dep
+      // Fix it in GT
+      geotrellis("spark").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
+      geotrellis("raster").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
+      geotrellis("s3").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
       geotrellis("spark-testkit").value % Test excludeAll (
         ExclusionRule(organization = "org.scalastic"),
-        ExclusionRule(organization = "org.scalatest")
+        ExclusionRule(organization = "org.scalatest"),
+        ExclusionRule(organization = "com.github.mpilquist")
       ),
       scaffeine,
       scalatest,
@@ -73,8 +77,8 @@ lazy val core = project
     libraryDependencies ++= {
       val gv = rfGeoTrellisVersion.value
       if (gv.startsWith("3")) Seq[ModuleID](
-        geotrellis("gdal").value,
-        geotrellis("s3-spark").value
+        geotrellis("gdal").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
+        geotrellis("s3-spark").value excludeAll ExclusionRule(organization = "com.github.mpilquist")
       )
       else Seq.empty[ModuleID]
     },
@@ -90,11 +94,11 @@ lazy val core = project
   )
 
 lazy val pyrasterframes = project
-  .dependsOn(core, datasource, experimental)
+  .dependsOn(core, datasource)
   .enablePlugins(RFAssemblyPlugin, PythonBuildPlugin)
   .settings(
     libraryDependencies ++= Seq(
-      geotrellis("s3").value,
+      geotrellis("s3").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
       spark("core").value % Provided,
       spark("mllib").value % Provided,
       spark("sql").value % Provided
@@ -108,11 +112,16 @@ lazy val datasource = project
   .settings(
     moduleName := "rasterframes-datasource",
     libraryDependencies ++= Seq(
-      geotrellis("s3").value,
+      compilerPlugin("org.scalamacros" % "paradise" % "2.1.1" cross CrossVersion.full),
+      sttpCatsCe2,
+      stac4s,
+      geotrellis("s3").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
       spark("core").value % Provided,
       spark("mllib").value % Provided,
       spark("sql").value % Provided
     ),
+    Compile / console / scalacOptions ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) },
+    Test / console / scalacOptions ~= { _.filterNot(Set("-Ywarn-unused-import", "-Ywarn-unused:imports")) },
     console / initialCommands := (console / initialCommands).value +
       """
         |import org.locationtech.rasterframes.datasource.geotrellis._
@@ -130,7 +139,7 @@ lazy val experimental = project
   .settings(
     moduleName := "rasterframes-experimental",
     libraryDependencies ++= Seq(
-      geotrellis("s3").value,
+      geotrellis("s3").value excludeAll ExclusionRule(organization = "com.github.mpilquist"),
       spark("core").value % Provided,
       spark("mllib").value % Provided,
       spark("sql").value % Provided
@@ -180,4 +189,3 @@ lazy val docs = project
 lazy val bench = project
   .dependsOn(core % "compile->test")
   .settings(publish / skip := true)
-
diff --git a/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala
@@ -117,7 +117,7 @@ class RasterSourceIT extends TestEnvironment with TestData {
 
   private def expectedTileCountAndBands(x:Int, y:Int, bandCount:Int = 1) = {
     val imageDimensions = Seq(x.toDouble, y.toDouble)
-    val tilesPerBand = imageDimensions.map(x ⇒ ceil(x / NOMINAL_TILE_SIZE)).product
+    val tilesPerBand = imageDimensions.map(x => ceil(x / NOMINAL_TILE_SIZE)).product
     val bands = Range(0, bandCount)
     val expectedTileCount = tilesPerBand * bands.length
     (expectedTileCount, bands)