Skip to content

Commit 14c8dc5

Browse files
metasimmetasim
committed
Merge branch 'release/0.8.2'
2 parents 7710e10 + 315b50d commit 14c8dc5

76 files changed

Lines changed: 1489 additions & 673 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎bench/archive/jmh-results-20190912131522.json‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
[
2+
{
3+
"jmhVersion" : "1.21",
4+
"benchmark" : "org.locationtech.rasterframes.bench.CellTypeBench.fromRow",
5+
"mode" : "avgt",
6+
"threads" : 1,
7+
"forks" : 1,
8+
"jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java",
9+
"jvmArgs" : [
10+
"-Xmx4g"
11+
],
12+
"jdkVersion" : "1.8.0_171",
13+
"vmName" : "Java HotSpot(TM) 64-Bit Server VM",
14+
"vmVersion" : "25.171-b11",
15+
"warmupIterations" : 8,
16+
"warmupTime" : "10 s",
17+
"warmupBatchSize" : 1,
18+
"measurementIterations" : 5,
19+
"measurementTime" : "10 s",
20+
"measurementBatchSize" : 1,
21+
"primaryMetric" : {
22+
"score" : 2.835836726694286,
23+
"scoreError" : 0.19270422040266105,
24+
"scoreConfidence" : [
25+
2.643132506291625,
26+
3.028540947096947
27+
],
28+
"scorePercentiles" : {
29+
"0.0" : 2.795428349144724,
30+
"50.0" : 2.833201108807787,
31+
"90.0" : 2.918007291796378,
32+
"95.0" : 2.918007291796378,
33+
"99.0" : 2.918007291796378,
34+
"99.9" : 2.918007291796378,
35+
"99.99" : 2.918007291796378,
36+
"99.999" : 2.918007291796378,
37+
"99.9999" : 2.918007291796378,
38+
"100.0" : 2.918007291796378
39+
},
40+
"scoreUnit" : "us/op",
41+
"rawData" : [
42+
[
43+
2.795428349144724,
44+
2.79552918770274,
45+
2.837017696019802,
46+
2.918007291796378,
47+
2.833201108807787
48+
]
49+
]
50+
},
51+
"secondaryMetrics" : {
52+
}
53+
},
54+
{
55+
"jmhVersion" : "1.21",
56+
"benchmark" : "org.locationtech.rasterframes.bench.CellTypeBench.intoRow",
57+
"mode" : "avgt",
58+
"threads" : 1,
59+
"forks" : 1,
60+
"jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java",
61+
"jvmArgs" : [
62+
"-Xmx4g"
63+
],
64+
"jdkVersion" : "1.8.0_171",
65+
"vmName" : "Java HotSpot(TM) 64-Bit Server VM",
66+
"vmVersion" : "25.171-b11",
67+
"warmupIterations" : 8,
68+
"warmupTime" : "10 s",
69+
"warmupBatchSize" : 1,
70+
"measurementIterations" : 5,
71+
"measurementTime" : "10 s",
72+
"measurementBatchSize" : 1,
73+
"primaryMetric" : {
74+
"score" : 0.35239272354175755,
75+
"scoreError" : 0.007379508933042929,
76+
"scoreConfidence" : [
77+
0.3450132146087146,
78+
0.3597722324748005
79+
],
80+
"scorePercentiles" : {
81+
"0.0" : 0.34961161284617265,
82+
"50.0" : 0.3525031701325926,
83+
"90.0" : 0.3550145438728285,
84+
"95.0" : 0.3550145438728285,
85+
"99.0" : 0.3550145438728285,
86+
"99.9" : 0.3550145438728285,
87+
"99.99" : 0.3550145438728285,
88+
"99.999" : 0.3550145438728285,
89+
"99.9999" : 0.3550145438728285,
90+
"100.0" : 0.3550145438728285
91+
},
92+
"scoreUnit" : "us/op",
93+
"rawData" : [
94+
[
95+
0.3550145438728285,
96+
0.3522314263912252,
97+
0.3525031701325926,
98+
0.34961161284617265,
99+
0.3526028644659687
100+
]
101+
]
102+
},
103+
"secondaryMetrics" : {
104+
}
105+
}
106+
]
107+
108+

‎bench/archive/jmh-results-20190912133231.json‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
[
2+
{
3+
"jmhVersion" : "1.21",
4+
"benchmark" : "org.locationtech.rasterframes.bench.CellTypeBench.fromRow",
5+
"mode" : "avgt",
6+
"threads" : 1,
7+
"forks" : 1,
8+
"jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java",
9+
"jvmArgs" : [
10+
"-Xmx4g"
11+
],
12+
"jdkVersion" : "1.8.0_171",
13+
"vmName" : "Java HotSpot(TM) 64-Bit Server VM",
14+
"vmVersion" : "25.171-b11",
15+
"warmupIterations" : 8,
16+
"warmupTime" : "10 s",
17+
"warmupBatchSize" : 1,
18+
"measurementIterations" : 5,
19+
"measurementTime" : "10 s",
20+
"measurementBatchSize" : 1,
21+
"primaryMetric" : {
22+
"score" : 0.09014145135858256,
23+
"scoreError" : 0.002394012003996672,
24+
"scoreConfidence" : [
25+
0.0877474393545859,
26+
0.09253546336257923
27+
],
28+
"scorePercentiles" : {
29+
"0.0" : 0.08936698388038906,
30+
"50.0" : 0.08989743516282808,
31+
"90.0" : 0.09083590280173164,
32+
"95.0" : 0.09083590280173164,
33+
"99.0" : 0.09083590280173164,
34+
"99.9" : 0.09083590280173164,
35+
"99.99" : 0.09083590280173164,
36+
"99.999" : 0.09083590280173164,
37+
"99.9999" : 0.09083590280173164,
38+
"100.0" : 0.09083590280173164
39+
},
40+
"scoreUnit" : "us/op",
41+
"rawData" : [
42+
[
43+
0.08989743516282808,
44+
0.09072300570225225,
45+
0.08988392924571173,
46+
0.09083590280173164,
47+
0.08936698388038906
48+
]
49+
]
50+
},
51+
"secondaryMetrics" : {
52+
}
53+
},
54+
{
55+
"jmhVersion" : "1.21",
56+
"benchmark" : "org.locationtech.rasterframes.bench.CellTypeBench.intoRow",
57+
"mode" : "avgt",
58+
"threads" : 1,
59+
"forks" : 1,
60+
"jvm" : "/Library/Java/JavaVirtualMachines/jdk1.8.0_171.jdk/Contents/Home/jre/bin/java",
61+
"jvmArgs" : [
62+
"-Xmx4g"
63+
],
64+
"jdkVersion" : "1.8.0_171",
65+
"vmName" : "Java HotSpot(TM) 64-Bit Server VM",
66+
"vmVersion" : "25.171-b11",
67+
"warmupIterations" : 8,
68+
"warmupTime" : "10 s",
69+
"warmupBatchSize" : 1,
70+
"measurementIterations" : 5,
71+
"measurementTime" : "10 s",
72+
"measurementBatchSize" : 1,
73+
"primaryMetric" : {
74+
"score" : 0.0804223243178353,
75+
"scoreError" : 0.003232629425106684,
76+
"scoreConfidence" : [
77+
0.07718969489272862,
78+
0.08365495374294199
79+
],
80+
"scorePercentiles" : {
81+
"0.0" : 0.07979611499160112,
82+
"50.0" : 0.08015109899040546,
83+
"90.0" : 0.08189719727590557,
84+
"95.0" : 0.08189719727590557,
85+
"99.0" : 0.08189719727590557,
86+
"99.9" : 0.08189719727590557,
87+
"99.99" : 0.08189719727590557,
88+
"99.999" : 0.08189719727590557,
89+
"99.9999" : 0.08189719727590557,
90+
"100.0" : 0.08189719727590557
91+
},
92+
"scoreUnit" : "us/op",
93+
"rawData" : [
94+
[
95+
0.08189719727590557,
96+
0.08020909063591224,
97+
0.07979611499160112,
98+
0.08005811969535213,
99+
0.08015109899040546
100+
]
101+
]
102+
},
103+
"secondaryMetrics" : {
104+
}
105+
}
106+
]
107+
108+

‎bench/src/main/scala/org/locationtech/rasterframes/bench/CellTypeBench.scala‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
* This software is licensed under the Apache 2 license, quoted below.
3+
*
4+
* Copyright 2019 Astraea, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
7+
* use this file except in compliance with the License. You may obtain a copy of
8+
* the License at
9+
*
10+
* [http://www.apache.org/licenses/LICENSE-2.0]
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15+
* License for the specific language governing permissions and limitations under
16+
* the License.
17+
*
18+
* SPDX-License-Identifier: Apache-2.0
19+
*
20+
*/
21+
22+
package org.locationtech.rasterframes.bench
23+
import java.util.concurrent.TimeUnit
24+
25+
import geotrellis.raster.{CellType, DoubleUserDefinedNoDataCellType, IntUserDefinedNoDataCellType}
26+
import org.apache.spark.sql.catalyst.InternalRow
27+
import org.locationtech.rasterframes.encoders.CatalystSerializer._
28+
import org.openjdk.jmh.annotations._
29+
30+
@BenchmarkMode(Array(Mode.AverageTime))
31+
@State(Scope.Benchmark)
32+
@OutputTimeUnit(TimeUnit.MICROSECONDS)
33+
class CellTypeBench {
34+
var row: InternalRow = _
35+
var ct: CellType = _
36+
@Setup(Level.Trial)
37+
def setupData(): Unit = {
38+
ct = IntUserDefinedNoDataCellType(scala.util.Random.nextInt())
39+
val o: CellType = DoubleUserDefinedNoDataCellType(scala.util.Random.nextDouble())
40+
row = o.toInternalRow
41+
}
42+
43+
@Benchmark
44+
def fromRow(): CellType = {
45+
row.to[CellType]
46+
}
47+
48+
@Benchmark
49+
def intoRow(): InternalRow = {
50+
ct.toInternalRow
51+
}
52+
}

‎bench/src/main/scala/org/locationtech/rasterframes/bench/TileExplodeBench.scala‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,12 @@ package org.locationtech.rasterframes.bench
2222

2323
import java.util.concurrent.TimeUnit
2424

25+
import org.apache.spark.sql.catalyst.InternalRow
26+
import org.apache.spark.sql.catalyst.expressions.BoundReference
27+
import org.apache.spark.sql.rf.TileUDT
2528
import org.locationtech.rasterframes._
26-
import org.apache.spark.sql._
27-
import org.apache.spark.sql.functions._
29+
import org.locationtech.rasterframes.expressions.generators.ExplodeTiles
2830
import org.openjdk.jmh.annotations._
29-
3031
/**
3132
*
3233
* @author sfitch
@@ -36,33 +37,32 @@ import org.openjdk.jmh.annotations._
3637
@State(Scope.Benchmark)
3738
@OutputTimeUnit(TimeUnit.MILLISECONDS)
3839
class TileExplodeBench extends SparkEnv {
39-
import spark.implicits._
4040

41-
@Param(Array("uint8", "uint16ud255", "float32", "float64"))
41+
//@Param(Array("uint8", "uint16ud255", "float32", "float64"))
42+
@Param(Array("uint16ud255"))
4243
var cellTypeName: String = _
4344

4445
@Param(Array("256"))
4546
var tileSize: Int = _
4647

47-
@Param(Array("100"))
48+
@Param(Array("2000"))
4849
var numTiles: Int = _
4950

5051
@transient
51-
var tiles: DataFrame = _
52+
var tiles: Array[InternalRow] = _
53+
54+
var exploder: ExplodeTiles = _
5255

5356
@Setup(Level.Trial)
5457
def setupData(): Unit = {
55-
tiles = Seq.fill(numTiles)(randomTile(tileSize, tileSize, cellTypeName))
56-
.toDF("tile").repartition(10)
57-
}
58-
59-
@Benchmark
60-
def arrayExplode() = {
61-
tiles.select(posexplode(rf_tile_to_array_double($"tile"))).count()
58+
tiles = Array.fill(numTiles)(randomTile(tileSize, tileSize, cellTypeName))
59+
.map(t => InternalRow(TileUDT.tileSerializer.toInternalRow(t)))
60+
val expr = BoundReference(0, TileType, true)
61+
exploder = new ExplodeTiles(1.0, None, Seq(expr))
6262
}
63-
6463
@Benchmark
6564
def tileExplode() = {
66-
tiles.select(rf_explode_tiles($"tile")).count()
65+
for(t <- tiles)
66+
exploder.eval(t)
6767
}
6868
}

‎core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* This software is licensed under the Apache 2 license, quoted below.
3+
*
4+
* Copyright 2019 Astraea, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
7+
* use this file except in compliance with the License. You may obtain a copy of
8+
* the License at
9+
*
10+
* [http://www.apache.org/licenses/LICENSE-2.0]
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15+
* License for the specific language governing permissions and limitations under
16+
* the License.
17+
*
18+
* SPDX-License-Identifier: Apache-2.0
19+
*
20+
*/
21+
22+
package org.locationtech.rasterframes.ref
23+
24+
import java.net.URI
25+
26+
import geotrellis.proj4.LatLng
27+
import geotrellis.vector.Extent
28+
import org.locationtech.rasterframes._
29+
import org.locationtech.rasterframes.expressions.aggregates.TileRasterizerAggregate
30+
31+
class RasterRefIT extends TestEnvironment {
32+
describe("practical subregion reads") {
33+
ignore("should construct a natural color composite") {
34+
import spark.implicits._
35+
def scene(idx: Int) = URI.create(s"https://landsat-pds.s3.us-west-2.amazonaws.com" +
36+
s"/c1/L8/176/039/LC08_L1TP_176039_20190703_20190718_01_T1/LC08_L1TP_176039_20190703_20190718_01_T1_B$idx.TIF")
37+
38+
val redScene = RasterSource(scene(4))
39+
// [west, south, east, north]
40+
val area = Extent(31.115, 29.963, 31.148, 29.99).reproject(LatLng, redScene.crs)
41+
42+
val red = RasterRef(redScene, 0, Some(area), None)
43+
val green = RasterRef(RasterSource(scene(3)), 0, Some(area), None)
44+
val blue = RasterRef(RasterSource(scene(2)), 0, Some(area), None)
45+
46+
val rf = Seq((red, green, blue)).toDF("red", "green", "blue")
47+
val df = rf.select(
48+
rf_crs($"red"), rf_extent($"red"), rf_tile($"red"), rf_tile($"green"), rf_tile($"blue"))
49+
.toDF
50+
51+
val raster = TileRasterizerAggregate.collect(df, redScene.crs, None, None)
52+
53+
forEvery(raster.tile.statisticsDouble) { stats =>
54+
stats should be ('defined)
55+
stats.get.dataCells shouldBe > (1000L)
56+
}
57+
58+
//import geotrellis.raster.io.geotiff.{GeoTiffOptions, MultibandGeoTiff, Tiled}
59+
//import geotrellis.raster.io.geotiff.compression.{DeflateCompression, NoCompression}
60+
//import geotrellis.raster.io.geotiff.tags.codes.ColorSpace
61+
//val tiffOptions = GeoTiffOptions(Tiled, DeflateCompression, ColorSpace.RGB)
62+
//MultibandGeoTiff(raster, raster.crs, tiffOptions).write("target/composite.tif")
63+
}
64+
}
65+
}

0 commit comments

Comments
 (0)