|
| 1 | +/* |
| 2 | + * This software is licensed under the Apache 2 license, quoted below. |
| 3 | + * |
| 4 | + * Copyright 2018 Astraea, Inc. |
| 5 | + * |
| 6 | + * Licensed under the Apache License, Version 2.0 (the "License"); you may not |
| 7 | + * use this file except in compliance with the License. You may obtain a copy of |
| 8 | + * the License at |
| 9 | + * |
| 10 | + * [http://www.apache.org/licenses/LICENSE-2.0] |
| 11 | + * |
| 12 | + * Unless required by applicable law or agreed to in writing, software |
| 13 | + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
| 14 | + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
| 15 | + * License for the specific language governing permissions and limitations under |
| 16 | + * the License. |
| 17 | + * |
| 18 | + * SPDX-License-Identifier: Apache-2.0 |
| 19 | + * |
| 20 | + */ |
| 21 | + |
| 22 | +package org.locationtech.rasterframes |
| 23 | + |
| 24 | +import org.apache.spark.sql.functions.{col, explode} |
| 25 | + |
| 26 | +class RasterFramesStatsSpec extends TestEnvironment with TestData { |
| 27 | + |
| 28 | + import spark.implicits._ |
| 29 | + |
| 30 | + val df = TestData.sampleGeoTiff |
| 31 | + .toDF() |
| 32 | + .withColumn("tilePlus2", rf_local_add(col("tile"), 2)) |
| 33 | + |
| 34 | + |
| 35 | + describe("Tile quantiles through built-in functions") { |
| 36 | + |
| 37 | + it("should compute approx percentiles for a single tile col") { |
| 38 | + // Use "explode" |
| 39 | + val result = df |
| 40 | + .select(rf_explode_tiles($"tile")) |
| 41 | + .stat |
| 42 | + .approxQuantile("tile", Array(0.10, 0.50, 0.90), 0.00001) |
| 43 | + |
| 44 | + result.length should be(3) |
| 45 | + |
| 46 | + // computing externally with numpy we arrive at 7963, 10068, 12160 for these quantiles |
| 47 | + result should contain inOrderOnly(7963.0, 10068.0, 12160.0) |
| 48 | + |
| 49 | + // Use "to_array" and built-in explode |
| 50 | + val result2 = df |
| 51 | + .select(explode(rf_tile_to_array_double($"tile")) as "tile") |
| 52 | + .stat |
| 53 | + .approxQuantile("tile", Array(0.10, 0.50, 0.90), 0.00001) |
| 54 | + |
| 55 | + result2.length should be(3) |
| 56 | + |
| 57 | + // computing externally with numpy we arrive at 7963, 10068, 12160 for these quantiles |
| 58 | + result2 should contain inOrderOnly(7963.0, 10068.0, 12160.0) |
| 59 | + |
| 60 | + } |
| 61 | + } |
| 62 | + |
| 63 | + describe("Tile quantiles through custom aggregate") { |
| 64 | + it("should compute approx percentiles for a single tile col") { |
| 65 | + val result = df |
| 66 | + .select(rf_agg_approx_quantiles($"tile", Seq(0.1, 0.5, 0.9))) |
| 67 | + .first() |
| 68 | + |
| 69 | + result.length should be(3) |
| 70 | + |
| 71 | + // computing externally with numpy we arrive at 7963, 10068, 12160 for these quantiles |
| 72 | + result should contain inOrderOnly(7963.0, 10068.0, 12160.0) |
| 73 | + } |
| 74 | + } |
| 75 | +} |
| 76 | + |
0 commit comments