Skip to content

Commit b2aa335

Browse files
metasimmetasim
authored
Merge pull request #417 from s22s/feature/raster-spatial-index
Option to spatially partition tiles before tiles are read.
2 parents a5ed5ed + d7b3a67 commit b2aa335

28 files changed

Lines changed: 624 additions & 223 deletions

File tree

‎build.sbt‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ lazy val root = project
3434
.enablePlugins(RFReleasePlugin)
3535
.settings(
3636
publish / skip := true,
37-
clean := clean.dependsOn(`rf-notebook`/clean).value
37+
clean := clean.dependsOn(`rf-notebook`/clean, docs/clean).value
3838
)
3939

4040
lazy val `rf-notebook` = project
@@ -76,7 +76,6 @@ lazy val core = project
7676
buildInfoObject := "RFBuildInfo",
7777
buildInfoOptions := Seq(
7878
BuildInfoOption.ToMap,
79-
BuildInfoOption.BuildTime,
8079
BuildInfoOption.ToJson
8180
)
8281
)

‎core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -53,30 +53,50 @@ trait RasterFunctions {
5353
/** Query the number of (cols, rows) in a Tile. */
5454
def rf_dimensions(col: Column): TypedColumn[Any, TileDimensions] = GetDimensions(col)
5555

56+
/** Extracts the CRS from a RasterSource or ProjectedRasterTile */
57+
def rf_crs(col: Column): TypedColumn[Any, CRS] = GetCRS(col)
58+
5659
/** Extracts the bounding box of a geometry as an Extent */
5760
def st_extent(col: Column): TypedColumn[Any, Extent] = GeometryToExtent(col)
5861

5962
/** Extracts the bounding box from a RasterSource or ProjectedRasterTile */
6063
def rf_extent(col: Column): TypedColumn[Any, Extent] = GetExtent(col)
6164

62-
/** Constructs a XZ2 index in WGS84 from either a Geometry, Extent, ProjectedRasterTile, or RasterSource and its CRS
65+
/** Constructs a XZ2 index in WGS84 from either a Geometry, Extent, ProjectedRasterTile, or RasterSource and its CRS.
6366
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
64-
def rf_spatial_index(targetExtent: Column, targetCRS: Column, indexResolution: Short) = XZ2Indexer(targetExtent, targetCRS, indexResolution)
67+
def rf_xz2_index(targetExtent: Column, targetCRS: Column, indexResolution: Short) = XZ2Indexer(targetExtent, targetCRS, indexResolution)
6568

6669
/** Constructs a XZ2 index in WGS84 from either a Geometry, Extent, ProjectedRasterTile, or RasterSource and its CRS
6770
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
68-
def rf_spatial_index(targetExtent: Column, targetCRS: Column) = XZ2Indexer(targetExtent, targetCRS, 18: Short)
71+
def rf_xz2_index(targetExtent: Column, targetCRS: Column) = XZ2Indexer(targetExtent, targetCRS, 18: Short)
6972

70-
/** Constructs a XZ2 index with level 18 resolution in WGS84 from either a ProjectedRasterTile or RasterSource
73+
/** Constructs a XZ2 index with provided resolution level in WGS84 from either a ProjectedRasterTile or RasterSource.
7174
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
72-
def rf_spatial_index(targetExtent: Column, indexResolution: Short) = XZ2Indexer(targetExtent, indexResolution)
75+
def rf_xz2_index(targetExtent: Column, indexResolution: Short) = XZ2Indexer(targetExtent, indexResolution)
7376

74-
/** Constructs a XZ2 index with level 18 resolution in WGS84 from either a ProjectedRasterTile or RasterSource
77+
/** Constructs a XZ2 index with level 18 resolution in WGS84 from either a ProjectedRasterTile or RasterSource.
7578
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
76-
def rf_spatial_index(targetExtent: Column) = XZ2Indexer(targetExtent, 18: Short)
79+
def rf_xz2_index(targetExtent: Column) = XZ2Indexer(targetExtent, 18: Short)
7780

78-
/** Extracts the CRS from a RasterSource or ProjectedRasterTile */
79-
def rf_crs(col: Column): TypedColumn[Any, CRS] = GetCRS(col)
81+
/** Constructs a Z2 index in WGS84 from either a Geometry, Extent, ProjectedRasterTile, or RasterSource and its CRS.
82+
* First the native extent is extracted or computed, and then center is used as the indexing location.
83+
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
84+
def rf_z2_index(targetExtent: Column, targetCRS: Column, indexResolution: Short) = Z2Indexer(targetExtent, targetCRS, indexResolution)
85+
86+
/** Constructs a Z2 index with index resolution of 31 in WGS84 from either a Geometry, Extent, ProjectedRasterTile, or RasterSource and its CRS.
87+
* First the native extent is extracted or computed, and then center is used as the indexing location.
88+
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
89+
def rf_z2_index(targetExtent: Column, targetCRS: Column) = Z2Indexer(targetExtent, targetCRS, 31: Short)
90+
91+
/** Constructs a Z2 index with the given index resolution in WGS84 from either a ProjectedRasterTile or RasterSource
92+
* First the native extent is extracted or computed, and then center is used as the indexing location.
93+
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
94+
def rf_z2_index(targetExtent: Column, indexResolution: Short) = Z2Indexer(targetExtent, indexResolution)
95+
96+
/** Constructs a Z2 index with index resolution of 31 in WGS84 from either a ProjectedRasterTile or RasterSource
97+
* First the native extent is extracted or computed, and then center is used as the indexing location.
98+
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
99+
def rf_z2_index(targetExtent: Column) = Z2Indexer(targetExtent, 31: Short)
80100

81101
/** Extracts the tile from a ProjectedRasterTile, or passes through a Tile. */
82102
def rf_tile(col: Column): TypedColumn[Any, Tile] = RealizeTile(col)

‎core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala‎

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import org.apache.spark.sql.jts.JTSTypes
3030
import org.apache.spark.sql.rf.{RasterSourceUDT, TileUDT}
3131
import org.apache.spark.sql.types._
3232
import org.apache.spark.unsafe.types.UTF8String
33-
import org.locationtech.jts.geom.Envelope
33+
import org.locationtech.jts.geom.{Envelope, Point}
3434
import org.locationtech.rasterframes.encoders.CatalystSerializer._
3535
import org.locationtech.rasterframes.model.{LazyCRS, TileContext}
3636
import org.locationtech.rasterframes.ref.{ProjectedRasterLike, RasterRef, RasterSource}
@@ -69,13 +69,13 @@ object DynamicExtractors {
6969
}
7070

7171
/** Partial function for pulling a ProjectedRasterLike an input row. */
72-
lazy val projectedRasterLikeExtractor: PartialFunction[DataType, InternalRow ⇒ ProjectedRasterLike] = {
72+
lazy val projectedRasterLikeExtractor: PartialFunction[DataType, Any ⇒ ProjectedRasterLike] = {
7373
case _: RasterSourceUDT ⇒
74-
(row: InternalRow) => row.to[RasterSource](RasterSourceUDT.rasterSourceSerializer)
74+
(input: Any) => input.asInstanceOf[InternalRow].to[RasterSource](RasterSourceUDT.rasterSourceSerializer)
7575
case t if t.conformsTo[ProjectedRasterTile] =>
76-
(row: InternalRow) => row.to[ProjectedRasterTile]
76+
(input: Any) => input.asInstanceOf[InternalRow].to[ProjectedRasterTile]
7777
case t if t.conformsTo[RasterRef] =>
78-
(row: InternalRow) => row.to[RasterRef]
78+
(input: Any) => input.asInstanceOf[InternalRow].to[RasterRef]
7979
}
8080

8181
/** Partial function for pulling a CellGrid from an input row. */
@@ -97,13 +97,36 @@ object DynamicExtractors {
9797
(v: Any) => v.asInstanceOf[InternalRow].to[CRS]
9898
}
9999

100-
lazy val extentLikeExtractor: PartialFunction[DataType, Any ⇒ Extent] = {
101-
case t if org.apache.spark.sql.rf.WithTypeConformity(t).conformsTo(JTSTypes.GeometryTypeInstance) =>
102-
(input: Any) => JTSTypes.GeometryTypeInstance.deserialize(input).getEnvelopeInternal
103-
case t if t.conformsTo[Extent] =>
104-
(input: Any) => input.asInstanceOf[InternalRow].to[Extent]
105-
case t if t.conformsTo[Envelope] =>
106-
(input: Any) => Extent(input.asInstanceOf[InternalRow].to[Envelope])
100+
lazy val extentExtractor: PartialFunction[DataType, Any ⇒ Extent] = {
101+
val base: PartialFunction[DataType, Any ⇒ Extent]= {
102+
case t if org.apache.spark.sql.rf.WithTypeConformity(t).conformsTo(JTSTypes.GeometryTypeInstance) =>
103+
(input: Any) => Extent(JTSTypes.GeometryTypeInstance.deserialize(input).getEnvelopeInternal)
104+
case t if t.conformsTo[Extent] =>
105+
(input: Any) => input.asInstanceOf[InternalRow].to[Extent]
106+
case t if t.conformsTo[Envelope] =>
107+
(input: Any) => Extent(input.asInstanceOf[InternalRow].to[Envelope])
108+
}
109+
110+
val fromPRL = projectedRasterLikeExtractor.andThen(_.andThen(_.extent))
111+
fromPRL orElse base
112+
}
113+
114+
lazy val envelopeExtractor: PartialFunction[DataType, Any => Envelope] = {
115+
val base = PartialFunction[DataType, Any => Envelope] {
116+
case t if org.apache.spark.sql.rf.WithTypeConformity(t).conformsTo(JTSTypes.GeometryTypeInstance) =>
117+
(input: Any) => JTSTypes.GeometryTypeInstance.deserialize(input).getEnvelopeInternal
118+
case t if t.conformsTo[Extent] =>
119+
(input: Any) => input.asInstanceOf[InternalRow].to[Extent].jtsEnvelope
120+
case t if t.conformsTo[Envelope] =>
121+
(input: Any) => input.asInstanceOf[InternalRow].to[Envelope]
122+
}
123+
124+
val fromPRL = projectedRasterLikeExtractor.andThen(_.andThen(_.extent.jtsEnvelope))
125+
fromPRL orElse base
126+
}
127+
128+
lazy val centroidExtractor: PartialFunction[DataType, Any ⇒ Point] = {
129+
extentExtractor.andThen(_.andThen(_.center.jtsGeom))
107130
}
108131

109132
sealed trait TileOrNumberArg

‎core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,8 @@ package object expressions {
135135
registry.registerExpression[RenderPNG.RenderCompositePNG]("rf_render_png")
136136
registry.registerExpression[RGBComposite]("rf_rgb_composite")
137137

138-
registry.registerExpression[XZ2Indexer]("rf_spatial_index")
138+
registry.registerExpression[XZ2Indexer]("rf_xz2_index")
139+
registry.registerExpression[Z2Indexer]("rf_z2_index")
139140

140141
registry.registerExpression[transformers.ReprojectGeometry]("st_reproject")
141142

‎core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/XZ2Indexer.scala‎

Lines changed: 8 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -22,24 +22,16 @@
2222
package org.locationtech.rasterframes.expressions.transformers
2323

2424
import geotrellis.proj4.LatLng
25-
import geotrellis.vector.Extent
2625
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
2726
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
2827
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
2928
import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription}
30-
import org.apache.spark.sql.jts.JTSTypes
31-
import org.apache.spark.sql.rf.RasterSourceUDT
3229
import org.apache.spark.sql.types.{DataType, LongType}
33-
import org.apache.spark.sql.{Column, TypedColumn, rf}
30+
import org.apache.spark.sql.{Column, TypedColumn}
3431
import org.locationtech.geomesa.curve.XZ2SFC
35-
import org.locationtech.jts.geom.{Envelope, Geometry}
36-
import org.locationtech.rasterframes.encoders.CatalystSerializer._
3732
import org.locationtech.rasterframes.expressions.DynamicExtractors._
3833
import org.locationtech.rasterframes.expressions.accessors.GetCRS
39-
import org.locationtech.rasterframes.expressions.row
4034
import org.locationtech.rasterframes.jts.ReprojectionTransformer
41-
import org.locationtech.rasterframes.ref.{RasterRef, RasterSource}
42-
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
4335

4436
/**
4537
* Constructs a XZ2 index in WGS84 from either a Geometry, Extent, ProjectedRasterTile, or RasterSource
@@ -63,52 +55,30 @@ import org.locationtech.rasterframes.tiles.ProjectedRasterTile
6355
case class XZ2Indexer(left: Expression, right: Expression, indexResolution: Short)
6456
extends BinaryExpression with CodegenFallback {
6557

66-
override def nodeName: String = "rf_spatial_index"
58+
override def nodeName: String = "rf_xz2_index"
6759

6860
override def dataType: DataType = LongType
6961

7062
override def checkInputDataTypes(): TypeCheckResult = {
71-
if (!extentLikeExtractor.orElse(projectedRasterLikeExtractor).isDefinedAt(left.dataType))
72-
TypeCheckFailure(s"Input type '${left.dataType}' does not look like something with an Extent or something with one.")
63+
if (!envelopeExtractor.isDefinedAt(left.dataType))
64+
TypeCheckFailure(s"Input type '${left.dataType}' does not look like a geometry, extent, or something with one.")
7365
else if(!crsExtractor.isDefinedAt(right.dataType))
74-
TypeCheckFailure(s"Input type '${right.dataType}' does not look like something with a CRS.")
66+
TypeCheckFailure(s"Input type '${right.dataType}' does not look like a CRS or something with one.")
7567
else TypeCheckSuccess
7668
}
7769

7870
private lazy val indexer = XZ2SFC(indexResolution)
7971

8072
override protected def nullSafeEval(leftInput: Any, rightInput: Any): Any = {
8173
val crs = crsExtractor(right.dataType)(rightInput)
82-
83-
val coords = left.dataType match {
84-
case t if rf.WithTypeConformity(t).conformsTo(JTSTypes.GeometryTypeInstance) =>
85-
JTSTypes.GeometryTypeInstance.deserialize(leftInput)
86-
case t if t.conformsTo[Extent] =>
87-
row(leftInput).to[Extent]
88-
case t if t.conformsTo[Envelope] =>
89-
row(leftInput).to[Envelope]
90-
case _: RasterSourceUDT ⇒
91-
row(leftInput).to[RasterSource](RasterSourceUDT.rasterSourceSerializer).extent
92-
case t if t.conformsTo[ProjectedRasterTile] =>
93-
row(leftInput).to[ProjectedRasterTile].extent
94-
case t if t.conformsTo[RasterRef] =>
95-
row(leftInput).to[RasterRef].extent
96-
}
74+
val coords = envelopeExtractor(left.dataType)(leftInput)
9775

9876
// If no transformation is needed then just normalize to an Envelope
99-
val env = if(crs == LatLng) coords match {
100-
case e: Extent => e.jtsEnvelope
101-
case g: Geometry => g.getEnvelopeInternal
102-
case e: Envelope => e
103-
}
77+
val env = if(crs == LatLng) coords
10478
// Otherwise convert to geometry, transform, and get envelope
10579
else {
10680
val trans = new ReprojectionTransformer(crs, LatLng)
107-
coords match {
108-
case e: Extent => trans(e).getEnvelopeInternal
109-
case g: Geometry => trans(g).getEnvelopeInternal
110-
case e: Envelope => trans(e).getEnvelopeInternal
111-
}
81+
trans(coords).getEnvelopeInternal
11282
}
11383

11484
val index = indexer.index(

‎core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/Z2Indexer.scala‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
/*
2+
* This software is licensed under the Apache 2 license, quoted below.
3+
*
4+
* Copyright 2019 Astraea, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
7+
* use this file except in compliance with the License. You may obtain a copy of
8+
* the License at
9+
*
10+
* [http://www.apache.org/licenses/LICENSE-2.0]
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15+
* License for the specific language governing permissions and limitations under
16+
* the License.
17+
*
18+
* SPDX-License-Identifier: Apache-2.0
19+
*
20+
*/
21+
22+
package org.locationtech.rasterframes.expressions.transformers
23+
24+
import geotrellis.proj4.LatLng
25+
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
26+
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
27+
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
28+
import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription}
29+
import org.apache.spark.sql.types.{DataType, LongType}
30+
import org.apache.spark.sql.{Column, TypedColumn}
31+
import org.locationtech.geomesa.curve.Z2SFC
32+
import org.locationtech.rasterframes.expressions.DynamicExtractors._
33+
import org.locationtech.rasterframes.expressions.accessors.GetCRS
34+
import org.locationtech.rasterframes.jts.ReprojectionTransformer
35+
36+
/**
37+
* Constructs a Z2 index in WGS84 from either a Geometry, Extent, ProjectedRasterTile, or RasterSource. First the
38+
* native extent is extracted or computed, and then center is used as the indexing location.
39+
* This function is useful for [range partitioning](http://spark.apache.org/docs/latest/api/python/pyspark.sql.html?highlight=registerjava#pyspark.sql.DataFrame.repartitionByRange).
40+
* Also see: https://www.geomesa.org/documentation/user/datastores/index_overview.html
41+
*
42+
* @param left geometry-like column
43+
* @param right CRS column
44+
* @param indexResolution resolution level of the space filling curve -
45+
* i.e. how many times the space will be recursively quartered
46+
* 1-31 is typical.
47+
*/
48+
@ExpressionDescription(
49+
usage = "_FUNC_(geom, crs) - Constructs a Z2 index in WGS84/EPSG:4326",
50+
arguments = """
51+
Arguments:
52+
* geom - Geometry or item with Geometry: Extent, ProjectedRasterTile, or RasterSource
53+
* crs - the native CRS of the `geom` column
54+
"""
55+
)
56+
case class Z2Indexer(left: Expression, right: Expression, indexResolution: Short)
57+
extends BinaryExpression with CodegenFallback {
58+
59+
override def nodeName: String = "rf_z2_index"
60+
61+
override def dataType: DataType = LongType
62+
63+
override def checkInputDataTypes(): TypeCheckResult = {
64+
if (!centroidExtractor.isDefinedAt(left.dataType))
65+
TypeCheckFailure(s"Input type '${left.dataType}' does not look like something with a centroid.")
66+
else if(!crsExtractor.isDefinedAt(right.dataType))
67+
TypeCheckFailure(s"Input type '${right.dataType}' does not look like a CRS or something with one.")
68+
else TypeCheckSuccess
69+
}
70+
71+
private lazy val indexer = new Z2SFC(indexResolution)
72+
73+
override protected def nullSafeEval(leftInput: Any, rightInput: Any): Any = {
74+
val crs = crsExtractor(right.dataType)(rightInput)
75+
val coord = centroidExtractor(left.dataType)(leftInput)
76+
77+
val pt = if(crs == LatLng) coord
78+
else {
79+
val trans = new ReprojectionTransformer(crs, LatLng)
80+
trans(coord)
81+
}
82+
83+
indexer.index(pt.getX, pt.getY, lenient = false).z
84+
}
85+
}
86+
87+
object Z2Indexer {
88+
import org.locationtech.rasterframes.encoders.SparkBasicEncoders.longEnc
89+
def apply(targetExtent: Column, targetCRS: Column, indexResolution: Short): TypedColumn[Any, Long] =
90+
new Column(new Z2Indexer(targetExtent.expr, targetCRS.expr, indexResolution)).as[Long]
91+
def apply(targetExtent: Column, targetCRS: Column): TypedColumn[Any, Long] =
92+
new Column(new Z2Indexer(targetExtent.expr, targetCRS.expr, 31)).as[Long]
93+
def apply(targetExtent: Column, indexResolution: Short = 31): TypedColumn[Any, Long] =
94+
new Column(new Z2Indexer(targetExtent.expr, GetCRS(targetExtent.expr), indexResolution)).as[Long]
95+
}

‎core/src/main/scala/org/locationtech/rasterframes/jts/ReprojectionTransformer.scala‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,11 @@
2121

2222
package org.locationtech.rasterframes.jts
2323

24-
import org.locationtech.jts.geom.{CoordinateSequence, Envelope, Geometry, GeometryFactory}
24+
import org.locationtech.jts.geom.{CoordinateSequence, Envelope, Geometry, GeometryFactory, Point}
2525
import org.locationtech.jts.geom.util.GeometryTransformer
2626
import geotrellis.proj4.CRS
2727
import geotrellis.vector.Extent
28+
import org.locationtech.jts.algorithm.Centroid
2829

2930
/**
3031
* JTS Geometry reprojection transformation routine.
@@ -38,6 +39,10 @@ class ReprojectionTransformer(src: CRS, dst: CRS) extends GeometryTransformer {
3839
def apply(geometry: Geometry): Geometry = transform(geometry)
3940
def apply(extent: Extent): Geometry = transform(extent.jtsGeom)
4041
def apply(env: Envelope): Geometry = transform(gf.toGeometry(env))
42+
def apply(pt: Point): Point = {
43+
val t = transform(pt)
44+
gf.createPoint(Centroid.getCentroid(t))
45+
}
4146

4247
override def transformCoordinates(coords: CoordinateSequence, parent: Geometry): CoordinateSequence = {
4348
val fact = parent.getFactory

‎core/src/test/scala/org/locationtech/rasterframes/RasterFunctionsSpec.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import geotrellis.raster._
2828
import geotrellis.raster.render.ColorRamps
2929
import geotrellis.raster.testkit.RasterMatchers
3030
import javax.imageio.ImageIO
31-
import org.apache.spark.sql.{Column, Encoders, TypedColumn}
31+
import org.apache.spark.sql.Encoders
3232
import org.apache.spark.sql.functions._
3333
import org.locationtech.rasterframes.expressions.accessors.ExtractTile
3434
import org.locationtech.rasterframes.model.TileDimensions
@@ -741,7 +741,7 @@ class RasterFunctionsSpec extends TestEnvironment with RasterMatchers {
741741
val withMasked = withMask.withColumn("masked",
742742
rf_inverse_mask_by_value($"tile", $"mask", mask_value))
743743
.withColumn("masked2", rf_mask_by_value($"tile", $"mask", lit(mask_value), true))
744-
withMasked.explain(true)
744+
745745
val result = withMasked.agg(rf_agg_no_data_cells($"tile") < rf_agg_no_data_cells($"masked")).as[Boolean]
746746

747747
result.first() should be(true)

0 commit comments

Comments
 (0)