locationtech · metasim · Feb 20, 2020 · Oct 21, 2019 · Oct 21, 2019 · Oct 22, 2019
diff --git a/.sbtrc b/.sbtrc
@@ -1 +1 @@
-alias openHere=eval "open .".!
+alias openHere=eval scala.sys.process.Process(Seq("open", ".")).!
diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala
@@ -46,7 +46,7 @@ class CatalystSerializerBench extends SparkEnv {
 
   @Setup(Level.Trial)
   def setupData(): Unit = {
-    crsEnc = StandardEncoders.crsEncoder.resolveAndBind()
+    crsEnc = StandardEncoders.crsSparkEncoder.resolveAndBind()
   }
 
   @Benchmark

diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala
@@ -28,8 +28,7 @@ import org.apache.spark.sql._
 import org.locationtech.rasterframes._
 import org.locationtech.rasterframes.expressions.generators.RasterSourceToRasterRefs
 import org.locationtech.rasterframes.expressions.transformers.RasterRefToTile
-import org.locationtech.rasterframes.model.TileDimensions
-import org.locationtech.rasterframes.ref.RasterSource
+import org.locationtech.rasterframes.ref.RFRasterSource
 import org.openjdk.jmh.annotations._
 
 @BenchmarkMode(Array(Mode.AverageTime))
@@ -43,11 +42,11 @@ class RasterRefBench  extends SparkEnv with LazyLogging {
 
   @Setup(Level.Trial)
   def setupData(): Unit = {
-    val r1 = RasterSource(remoteCOGSingleband1)
-    val r2 = RasterSource(remoteCOGSingleband2)
+    val r1 = RFRasterSource(remoteCOGSingleband1)
+    val r2 = RFRasterSource(remoteCOGSingleband2)
 
     singleDF = Seq((r1, r2)).toDF("B1", "B2")
-      .select(RasterRefToTile(RasterSourceToRasterRefs(Some(TileDimensions(r1.dimensions)), Seq(0), $"B1", $"B2")))
+      .select(RasterRefToTile(RasterSourceToRasterRefs(Some(r1.dimensions), Seq(0), $"B1", $"B2")))
 
     expandedDF = Seq((r1, r2)).toDF("B1", "B2")
       .select(RasterRefToTile(RasterSourceToRasterRefs($"B1", $"B2")))

diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala
@@ -23,6 +23,7 @@ package org.locationtech.rasterframes.bench
 
 import java.util.concurrent.TimeUnit
 
+import geotrellis.raster.Dimensions
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.rf.TileUDT
 import org.locationtech.rasterframes.tiles.InternalRowTile
@@ -56,7 +57,7 @@ class TileCellScanBench extends SparkEnv {
   @Benchmark
   def deserializeRead(): Double  = {
     val tile = TileType.deserialize(tileRow)
-    val (cols, rows) = tile.dimensions
+    val Dimensions(cols, rows) = tile.dimensions
     tile.getDouble(cols - 1, rows - 1) +
       tile.getDouble(cols/2, rows/2) +
       tile.getDouble(0, 0)

diff --git a/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala b/bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala
@@ -30,7 +30,7 @@ import geotrellis.raster.Tile
 import geotrellis.vector.Extent
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.locationtech.rasterframes.ref.{RasterRef, RasterSource}
+import org.locationtech.rasterframes.ref.{RasterRef, RFRasterSource}
 import org.openjdk.jmh.annotations._
 
 @BenchmarkMode(Array(Mode.AverageTime))
@@ -56,7 +56,7 @@ class TileEncodeBench extends SparkEnv {
       case "rasterRef" ⇒
         val baseCOG = "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_B1.TIF"
         val extent = Extent(253785.0, 3235185.0, 485115.0, 3471015.0)
-        tile = RasterRefTile(RasterRef(RasterSource(URI.create(baseCOG)), 0, Some(extent), None))
+        tile = RasterRefTile(RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None))
       case _ ⇒
         tile = randomTile(tileSize, tileSize, cellTypeName)
     }

diff --git a/build.sbt b/build.sbt
@@ -50,12 +50,12 @@ lazy val core = project
   .settings(
     moduleName := "rasterframes",
     libraryDependencies ++= Seq(
+      `slf4j-api`,
       shapeless,
       `jts-core`,
+      `spray-json`,
       geomesa("z3").value,
       geomesa("spark-jts").value,
-      `geotrellis-contrib-vlm`,
-      `geotrellis-contrib-gdal`,
       spark("core").value % Provided,
       spark("mllib").value % Provided,
       spark("sql").value % Provided,
@@ -78,7 +78,7 @@ lazy val core = project
       else Seq.empty[ModuleID]
     },
     buildInfoKeys ++= Seq[BuildInfoKey](
-      moduleName, version, scalaVersion, sbtVersion, rfGeoTrellisVersion, rfGeoMesaVersion, rfSparkVersion
+      version, scalaVersion, rfGeoTrellisVersion, rfGeoMesaVersion, rfSparkVersion
     ),
     buildInfoPackage := "org.locationtech.rasterframes",
     buildInfoObject := "RFBuildInfo",
@@ -133,8 +133,7 @@ lazy val experimental = project
       spark("sql").value % Provided
     ),
     fork in IntegrationTest := true,
-    javaOptions in IntegrationTest := Seq("-Xmx2G"),
-    parallelExecution in IntegrationTest := false
+    javaOptions in IntegrationTest := Seq("-Xmx2G")
   )
 
 lazy val docs = project
@@ -175,8 +174,6 @@ lazy val docs = project
     addMappingsToSiteDir(Compile / paradox / mappings, paradox / siteSubdirName)
   )
 
-//ParadoxMaterialThemePlugin.paradoxMaterialThemeSettings(Paradox)
-
 lazy val bench = project
   .dependsOn(core % "compile->test")
   .settings(publish / skip := true)

diff --git a/core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala
@@ -35,13 +35,13 @@ class RasterRefIT extends TestEnvironment {
       def scene(idx: Int) = URI.create(s"https://landsat-pds.s3.us-west-2.amazonaws.com" +
         s"/c1/L8/176/039/LC08_L1TP_176039_20190703_20190718_01_T1/LC08_L1TP_176039_20190703_20190718_01_T1_B$idx.TIF")
 
-      val redScene = RasterSource(scene(4))
+      val redScene = RFRasterSource(scene(4))
       // [west, south, east, north]
       val area = Extent(31.115, 29.963, 31.148, 29.99).reproject(LatLng, redScene.crs)
 
       val red = RasterRef(redScene, 0, Some(area), None)
-      val green = RasterRef(RasterSource(scene(3)), 0, Some(area), None)
-      val blue = RasterRef(RasterSource(scene(2)), 0, Some(area), None)
+      val green = RasterRef(RFRasterSource(scene(3)), 0, Some(area), None)
+      val blue = RasterRef(RFRasterSource(scene(2)), 0, Some(area), None)
 
       val rf = Seq((red, green, blue)).toDF("red", "green", "blue")
       val df = rf.select(

diff --git a/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala
@@ -44,10 +44,10 @@ class RasterSourceIT extends TestEnvironment with TestData {
         val bURI = new URI(
           "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/016/034/LC08_L1TP_016034_20181003_20181003_01_RT/LC08_L1TP_016034_20181003_20181003_01_RT_B2.TIF")
         val red = time("read B4") {
-          RasterSource(rURI).readAll()
+          RFRasterSource(rURI).readAll()
         }
         val blue = time("read B2") {
-          RasterSource(bURI).readAll()
+          RFRasterSource(bURI).readAll()
         }
         time("test empty") {
           red should not be empty
@@ -69,47 +69,47 @@ class RasterSourceIT extends TestEnvironment with TestData {
 
 
       it("should read JPEG2000 scene") {
-        RasterSource(localSentinel).readAll().flatMap(_.tile.statisticsDouble).size should be(64)
+        RFRasterSource(localSentinel).readAll().flatMap(_.tile.statisticsDouble).size should be(64)
       }
 
       it("should read small MRF scene with one band converted from MODIS HDF") {
         val (expectedTileCount, _) = expectedTileCountAndBands(2400, 2400)
-        RasterSource(modisConvertedMrfPath).readAll().flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
+        RFRasterSource(modisConvertedMrfPath).readAll().flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
       }
 
       it("should read remote HTTP MRF scene") {
         val (expectedTileCount, bands) = expectedTileCountAndBands(6257, 7584, 4)
-        RasterSource(remoteHttpMrfPath).readAll(bands = bands).flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
+        RFRasterSource(remoteHttpMrfPath).readAll(bands = bands).flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
       }
 
       it("should read remote S3 MRF scene") {
         val (expectedTileCount, bands) = expectedTileCountAndBands(6257, 7584, 4)
-        RasterSource(remoteS3MrfPath).readAll(bands = bands).flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
+        RFRasterSource(remoteS3MrfPath).readAll(bands = bands).flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
       }
     }
   } else {
     describe("GDAL missing error support") {
       it("should throw exception reading JPEG2000 scene") {
         intercept[IllegalArgumentException] {
-          RasterSource(localSentinel)
+          RFRasterSource(localSentinel)
         }
       }
 
       it("should throw exception reading MRF scene with one band converted from MODIS HDF") {
         intercept[IllegalArgumentException] {
-          RasterSource(modisConvertedMrfPath)
+          RFRasterSource(modisConvertedMrfPath)
         }
       }
 
       it("should throw exception reading remote HTTP MRF scene") {
         intercept[IllegalArgumentException] {
-          RasterSource(remoteHttpMrfPath)
+          RFRasterSource(remoteHttpMrfPath)
         }
       }
 
       it("should throw exception reading remote S3 MRF scene") {
         intercept[IllegalArgumentException] {
-          RasterSource(remoteS3MrfPath)
+          RFRasterSource(remoteS3MrfPath)
         }
       }
     }

diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf
@@ -5,9 +5,9 @@ rasterframes {
   showable-max-cells = 20
   max-truncate-row-element-length = 40
   raster-source-cache-timeout = 120 seconds
+  jp2-gdal-thread-lock = false
 }
-
-vlm.gdal {
+geotrellis.raster.gdal {
     options {
       // See https://trac.osgeo.org/gdal/wiki/ConfigOptions for options
       //CPL_DEBUG = "OFF"

diff --git a/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala b/core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala
@@ -27,7 +27,7 @@ import org.locationtech.rasterframes.encoders.CatalystSerializer._
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.types.{DataType, UDTRegistration, UserDefinedType, _}
 import org.locationtech.rasterframes.encoders.CatalystSerializer
-import org.locationtech.rasterframes.ref.RasterSource
+import org.locationtech.rasterframes.ref.RFRasterSource
 import org.locationtech.rasterframes.util.KryoSupport
 
 /**
@@ -36,25 +36,25 @@ import org.locationtech.rasterframes.util.KryoSupport
  * @since 9/5/18
  */
 @SQLUserDefinedType(udt = classOf[RasterSourceUDT])
-class RasterSourceUDT extends UserDefinedType[RasterSource] {
+class RasterSourceUDT extends UserDefinedType[RFRasterSource] {
   import RasterSourceUDT._
   override def typeName = "rastersource"
 
   override def pyUDT: String = "pyrasterframes.rf_types.RasterSourceUDT"
 
-  def userClass: Class[RasterSource] = classOf[RasterSource]
+  def userClass: Class[RFRasterSource] = classOf[RFRasterSource]
 
-  override def sqlType: DataType = schemaOf[RasterSource]
+  override def sqlType: DataType = schemaOf[RFRasterSource]
 
-  override def serialize(obj: RasterSource): InternalRow =
+  override def serialize(obj: RFRasterSource): InternalRow =
     Option(obj)
       .map(_.toInternalRow)
       .orNull
 
-  override def deserialize(datum: Any): RasterSource =
+  override def deserialize(datum: Any): RFRasterSource =
     Option(datum)
       .collect {
-        case ir: InternalRow ⇒ ir.to[RasterSource]
+        case ir: InternalRow ⇒ ir.to[RFRasterSource]
       }
       .orNull
 
@@ -65,24 +65,24 @@ class RasterSourceUDT extends UserDefinedType[RasterSource] {
 }
 
 object RasterSourceUDT {
-  UDTRegistration.register(classOf[RasterSource].getName, classOf[RasterSourceUDT].getName)
+  UDTRegistration.register(classOf[RFRasterSource].getName, classOf[RasterSourceUDT].getName)
 
   /** Deserialize a byte array, also used inside the Python API */
-  def from(byteArray: Array[Byte]): RasterSource = CatalystSerializer.CatalystIO.rowIO.create(byteArray).to[RasterSource]
+  def from(byteArray: Array[Byte]): RFRasterSource = CatalystSerializer.CatalystIO.rowIO.create(byteArray).to[RFRasterSource]
 
-  implicit val rasterSourceSerializer: CatalystSerializer[RasterSource] = new CatalystSerializer[RasterSource] {
+  implicit val rasterSourceSerializer: CatalystSerializer[RFRasterSource] = new CatalystSerializer[RFRasterSource] {
 
     override val schema: StructType = StructType(Seq(
       StructField("raster_source_kryo", BinaryType, false)
     ))
 
-    override def to[R](t: RasterSource, io: CatalystIO[R]): R = {
+    override def to[R](t: RFRasterSource, io: CatalystIO[R]): R = {
       val buf = KryoSupport.serialize(t)
       io.create(buf.array())
     }
 
-    override def from[R](row: R, io: CatalystIO[R]): RasterSource = {
-      KryoSupport.deserialize[RasterSource](ByteBuffer.wrap(io.getByteArray(row, 0)))
+    override def from[R](row: R, io: CatalystIO[R]): RFRasterSource = {
+      KryoSupport.deserialize[RFRasterSource](ByteBuffer.wrap(io.getByteArray(row, 0)))
     }
   }
 }
diff --git a/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala b/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala
@@ -2,6 +2,7 @@ package org.apache.spark.sql.rf
 
 import java.lang.reflect.Constructor
 
+import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.catalyst.FunctionIdentifier
 import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
 import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
@@ -12,7 +13,6 @@ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.sources.BaseRelation
 import org.apache.spark.sql.types.DataType
-import org.apache.spark.sql.{AnalysisException, DataFrame, Dataset, SQLContext}
 
 import scala.reflect._
 import scala.util.{Failure, Success, Try}
@@ -23,11 +23,6 @@ import scala.util.{Failure, Success, Try}
  * @since 2/13/18
  */
 object VersionShims {
-  def readJson(sqlContext: SQLContext, rows: Dataset[String]): DataFrame = {
-    // NB: Will get a deprecation warning for Spark 2.2.x
-    sqlContext.read.json(rows.rdd) // <-- deprecation warning expected
-  }
-
   def updateRelation(lr: LogicalRelation, base: BaseRelation): LogicalPlan = {
     val lrClazz = classOf[LogicalRelation]
     val ctor = lrClazz.getConstructors.head.asInstanceOf[Constructor[LogicalRelation]]

diff --git a/core/src/main/scala/org/locationtech/rasterframes/PairRDDConverter.scala b/core/src/main/scala/org/locationtech/rasterframes/PairRDDConverter.scala
@@ -23,7 +23,7 @@ package org.locationtech.rasterframes
 
 import org.locationtech.rasterframes.util._
 import geotrellis.raster.{MultibandTile, Tile, TileFeature}
-import geotrellis.spark.{SpaceTimeKey, SpatialKey}
+import geotrellis.layer._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql._
 import org.apache.spark.sql.rf.TileUDT

diff --git a/core/src/main/scala/org/locationtech/rasterframes/StandardColumns.scala b/core/src/main/scala/org/locationtech/rasterframes/StandardColumns.scala
@@ -25,7 +25,7 @@ import java.sql.Timestamp
 
 import geotrellis.proj4.CRS
 import geotrellis.raster.Tile
-import geotrellis.spark.{SpatialKey, TemporalKey}
+import geotrellis.layer._
 import geotrellis.vector.{Extent, ProjectedExtent}
 import org.apache.spark.sql.functions.col
 import org.locationtech.jts.geom.{Point => jtsPoint, Polygon => jtsPolygon}

diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/ProjectedExtentEncoder.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/ProjectedExtentEncoder.scala
@@ -32,6 +32,6 @@ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
  */
 object ProjectedExtentEncoder {
   def apply(): ExpressionEncoder[ProjectedExtent] = {
-    DelegatingSubfieldEncoder("extent" -> extentEncoder, "crs" -> crsEncoder)
+    DelegatingSubfieldEncoder("extent" -> extentEncoder, "crs" -> crsSparkEncoder)
   }
 }
diff --git a/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala b/core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala
@@ -27,9 +27,8 @@ import java.sql.Timestamp
 import org.locationtech.rasterframes.stats.{CellHistogram, CellStatistics, LocalCellStatistics}
 import org.locationtech.jts.geom.Envelope
 import geotrellis.proj4.CRS
-import geotrellis.raster.{CellSize, CellType, Raster, Tile, TileLayout}
-import geotrellis.spark.tiling.LayoutDefinition
-import geotrellis.spark.{KeyBounds, SpaceTimeKey, SpatialKey, TemporalKey, TemporalProjectedExtent, TileLayerMetadata}
+import geotrellis.raster.{CellSize, CellType, Dimensions, Raster, Tile, TileLayout}
+import geotrellis.layer._
 import geotrellis.vector.{Extent, ProjectedExtent}
 import org.apache.spark.sql.{Encoder, Encoders}
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
@@ -53,7 +52,7 @@ trait StandardEncoders extends SpatialEncoders {
   implicit def singlebandTileEncoder: ExpressionEncoder[Tile] = ExpressionEncoder()
   implicit def rasterEncoder: ExpressionEncoder[Raster[Tile]] = ExpressionEncoder()
   implicit def tileLayerMetadataEncoder[K: TypeTag]: ExpressionEncoder[TileLayerMetadata[K]] = TileLayerMetadataEncoder()
-  implicit def crsEncoder: ExpressionEncoder[CRS] = CRSEncoder()
+  implicit def crsSparkEncoder: ExpressionEncoder[CRS] = CRSEncoder()
   implicit def projectedExtentEncoder: ExpressionEncoder[ProjectedExtent] = ProjectedExtentEncoder()
   implicit def temporalProjectedExtentEncoder: ExpressionEncoder[TemporalProjectedExtent] = TemporalProjectedExtentEncoder()
   implicit def cellTypeEncoder: ExpressionEncoder[CellType] = CellTypeEncoder()
@@ -71,8 +70,7 @@ trait StandardEncoders extends SpatialEncoders {
   implicit def tileContextEncoder: ExpressionEncoder[TileContext] = TileContext.encoder
   implicit def tileDataContextEncoder: ExpressionEncoder[TileDataContext] = TileDataContext.encoder
   implicit def extentTilePairEncoder: Encoder[(ProjectedExtent, Tile)] = Encoders.tuple(projectedExtentEncoder, singlebandTileEncoder)
-
-
+  implicit def tileDimensionsEncoder: Encoder[Dimensions[Int]] = CatalystSerializerEncoder[Dimensions[Int]](true)
 }
 
 object StandardEncoders extends StandardEncoders
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		alias openHere=eval "open .".!
		alias openHere=eval scala.sys.process.Process(Seq("open", ".")).!