GEOMESA-3575 Fix truncated z-column alignment with z partitions (#3554)

* Fix display of bounds in manage metadata command

Author: elahrvivaz

geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-core/src/main/scala/org/locationtech/geomesa/fs/storage/core/StorageMetadata.scala

@@ -8,7 +8,6 @@
 
 package org.locationtech.geomesa.fs.storage.core
 
-import org.calrissian.mango.types.encoders.lexi.LongEncoder
 import org.calrissian.mango.types.{TypeEncoder, TypeRegistry}
 import org.geotools.api.feature.simple.SimpleFeatureType
 import org.geotools.api.filter.Filter
@@ -18,6 +17,7 @@ import org.locationtech.geomesa.index.index.attribute.AttributeIndexKey
 import org.locationtech.jts.geom._
 
 import java.io.Closeable
+import java.util.HexFormat
 
 /**
   * Metadata interface for managing storage partitions
@@ -172,8 +172,8 @@ object StorageMetadata {
   object Z2Encoder extends TypeEncoder[Point, String] {
 
     private val sfc = Z2SFC
-    private val longEncoder = new LongEncoder()
     private val factory = new GeometryFactory()
+    private val hexFormat = HexFormat.of()
 
     override val getAlias: String = "z2"
 
@@ -183,11 +183,11 @@ object StorageMetadata {
       if (value == null) {
         throw new NullPointerException("Null values are not allowed")
       }
-      longEncoder.encode(sfc.index(value.getX, value.getY))
+      toHex(sfc.index(value.getX, value.getY))
     }
 
     override def decode(value: String): Point = {
-      val (x, y) = sfc.invert(longEncoder.decode(value))
+      val (x, y) = sfc.invert(fromHex(value))
       factory.createPoint(new Coordinate(x, y))
     }
 
@@ -198,7 +198,11 @@ object StorageMetadata {
      * @param maxRanges rough upper bound on the number of ranges to return
      */
     def ranges(queries: Seq[(Double, Double, Double, Double)], maxRanges: Option[Int] = None): Seq[(String, String)] =
-      sfc.ranges(queries, maxRanges = maxRanges).map(r => longEncoder.encode(r.lower) -> longEncoder.encode(r.upper))
+      sfc.ranges(queries, maxRanges = maxRanges).map(r => toHex(r.lower) -> toHex(r.upper))
+
+    // since the first two bits are not used in our z values, drop them so that truncating the hex value aligns with our curve
+    private def toHex(z: Long): String = hexFormat.toHexDigits(z << 2)
+    private def fromHex(hex: String): Long = HexFormat.fromHexDigitsToLong(hex) >>> 2
   }
 
   /**
@@ -207,8 +211,8 @@ object StorageMetadata {
   object XZ2Encoder extends TypeEncoder[Geometry, String] {
 
     private val sfc = XZ2SFC
-    private val longEncoder = new LongEncoder()
     private val factory = new GeometryFactory()
+    private val hexFormat = HexFormat.of()
 
     override val getAlias: String = "xz2"
 
@@ -222,11 +226,11 @@ object StorageMetadata {
       if (env.isNull) {
         throw new NullPointerException("Geometry has a null envelope")
       }
-      longEncoder.encode(sfc.index(env.getMinX, env.getMinY, env.getMaxX, env.getMaxY))
+      hexFormat.toHexDigits(sfc.index(env.getMinX, env.getMinY, env.getMaxX, env.getMaxY))
     }
 
     override def decode(value: String): Geometry = {
-      val (xmin, ymin, xmax, ymax) = sfc.invert(longEncoder.decode(value))
+      val (xmin, ymin, xmax, ymax) = sfc.invert(HexFormat.fromHexDigitsToLong(value))
       val ring = Array(
         new Coordinate(xmin, ymin),
         new Coordinate(xmin, ymax),
@@ -244,6 +248,6 @@ object StorageMetadata {
      * @param maxRanges a rough upper limit on the number of ranges to generate
      */
     def ranges(queries: Seq[(Double, Double, Double, Double)], maxRanges: Option[Int] = None): Seq[(String, String)] =
-      sfc.ranges(queries, maxRanges).map(r => longEncoder.encode(r.lower) -> longEncoder.encode(r.upper))
+      sfc.ranges(queries, maxRanges).map(r => hexFormat.toHexDigits(r.lower) -> hexFormat.toHexDigits(r.upper))
   }
 }

geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-core/src/test/scala/org/locationtech/geomesa/fs/storage/core/StorageMetadataTest.scala

@@ -0,0 +1,37 @@
+/***********************************************************************
+ * Copyright (c) 2013-2025 General Atomics Integrated Intelligence, Inc.
+ * All rights reserved. This program and the accompanying materials
+ * are made available under the terms of the Apache License, Version 2.0
+ * which accompanies this distribution and is available at
+ * https://www.apache.org/licenses/LICENSE-2.0
+ ***********************************************************************/
+
+package org.locationtech.geomesa.fs.storage.core
+
+import org.locationtech.geomesa.features.ScalaSimpleFeature
+import org.locationtech.geomesa.fs.storage.core.StorageMetadata.Z2Encoder
+import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes
+import org.locationtech.geomesa.utils.text.WKTUtils
+import org.locationtech.jts.geom.Point
+import org.specs2.mutable.SpecificationWithJUnit
+
+import java.util.HexFormat
+
+class StorageMetadataTest extends SpecificationWithJUnit {
+
+  private val sft = SimpleFeatureTypes.createType("test", "*geom:Point:srid=4326")
+
+  "Z2Encoder" should {
+    "create truncate-able z values that align with our partition scheme" in {
+      val ps = PartitionSchemeFactory.load(sft, "z2:bits=4")
+      foreach(Seq(-67.5, -22.5, 22.5, 67.5)) { lat =>
+        foreach(Seq(-135, -45, 45, 135)) { lon =>
+          val pt = WKTUtils.read(s"POINT($lon $lat)").asInstanceOf[Point]
+          val partition = ps.getPartition(ScalaSimpleFeature.create(sft, "", pt)).value.toInt
+          val z2 = Z2Encoder.encode(pt)
+          partition mustEqual HexFormat.fromHexDigit(z2.head)
+        }
+      }
+    }
+  }
+}

geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-core/src/test/scala/org/locationtech/geomesa/fs/storage/core/metadata/SchemeFilterExtractionTest.scala

@@ -41,7 +41,7 @@ class SchemeFilterExtractionTest extends SpecificationWithJUnit {
       filters.head.partitions must
         containTheSameElementsAs(Seq(PartitionRange("hours:attribute=dtg", "80066ba0", "80066bb8"), PartitionRange("z2:attribute=geom:bits=2", "3", "4")))
       filters.head.columnBounds mustEqual
-        Seq(ColumnOr(0, Seq(ColumnBound("80000160af049000", "80000160b42aec00"))), ColumnOr(1, Seq(ColumnBound("b000000000000000", "bfffffffffffffff"))))
+        Seq(ColumnOr(0, Seq(ColumnBound("80000160af049000", "80000160b42aec00"))), ColumnOr(1, Seq(ColumnBound("c000000000000000", "fffffffffffffffc"))))
     }
   }
 

geomesa-fs/geomesa-fs-storage/geomesa-fs-storage-core/src/test/scala/org/locationtech/geomesa/fs/storage/core/metadata/TestAbstractMetadata.scala

@@ -13,9 +13,11 @@ import com.typesafe.scalalogging.LazyLogging
 import org.apache.commons.io.FileUtils
 import org.geotools.filter.text.ecql.ECQL
 import org.locationtech.geomesa.features.ScalaSimpleFeature
-import org.locationtech.geomesa.fs.storage.core.StorageMetadata.{ColumnBounds, StorageFile}
+import org.locationtech.geomesa.fs.storage.core.StorageMetadata.{ColumnBounds, StorageFile, Z2Encoder}
 import org.locationtech.geomesa.utils.geotools.SimpleFeatureTypes
 import org.locationtech.geomesa.utils.io.WithClose
+import org.locationtech.geomesa.utils.text.{DateParsing, WKTUtils}
+import org.locationtech.jts.geom.Point
 import org.specs2.mutable.Specification
 
 import java.net.URI
@@ -33,20 +35,35 @@ abstract class TestAbstractMetadata extends Specification with LazyLogging {
 
   // note: ensure that partitions and bounds line up
   val partition1 = partitions("2026-03-05T00:00:00.000Z", "POINT (10 10)") // dtg 8000019cbb4b4c00/8000019cbb823a80
-  val columnBounds1 = Seq(ColumnBounds(0, "name5", "name6"), ColumnBounds(1, "8000019cbb4b4c00", "8000019cbb823a80"),
-    ColumnBounds(2, "b00239e50239e502"/*POINT(1 2)*/, "b085d4c185d4c185"/*POINT(11 12)*/))
+  val columnBounds1 = Seq(
+    ColumnBounds(0, "name5", "name6"),
+    ColumnBounds(1, dateBounds("2026-03-05T00:00"), dateBounds("2026-03-05T01:00")),
+    ColumnBounds(2, z2Bounds("POINT(1 2)"), z2Bounds("POINT(11 12)"))
+  )
 
   val partition2 = partitions("2026-03-04T00:00:00.000Z", "POINT (-10 10)") // dtg 8000019cb624f000/8000019cb65bde80
-  val columnBounds2a = Seq(ColumnBounds(0, "name0", "name4"), ColumnBounds(1, "8000019cb624f000", "8000019cb6406740")/* first 30 minutes */,
-    ColumnBounds(2, "a55229b45229b452"/*POINT(-11 2)*/, "a5d5c490d5c490d5"/*POINT(-1 12)*/))
-  val columnBounds2b = Seq(ColumnBounds(0, "name0", "name4"), ColumnBounds(1, "8000019cb624f000", "8000019cb6406740")/* first 30 minutes */,
-    ColumnBounds(2, "a5c0d485c0d485c0"/*POINT(-20 12)*/, "a5ed4fc5ed4fc5ed"/*POINT(-12 20)*/))
+  val columnBounds2a = Seq(
+    ColumnBounds(0, "name0", "name4"),
+    ColumnBounds(1, dateBounds("2026-03-04T00:00"), dateBounds("2026-03-04T00:30"))/* first 30 minutes */,
+    ColumnBounds(2, z2Bounds("POINT(-11 2)"), z2Bounds("POINT(-1 12)"))
+  )
+  val columnBounds2b = Seq(
+    ColumnBounds(0, "name0", "name4"),
+    ColumnBounds(1, dateBounds("2026-03-04T00:00"), dateBounds("2026-03-04T00:30"))/* first 30 minutes */,
+    ColumnBounds(2, z2Bounds("POINT(-20 12)"), z2Bounds("POINT(-12 20)"))
+  )
 
   val partition3 = partitions("2026-03-03T00:00:00.000Z", "POINT (10 -10)") // dtg 8000019cb0fe9400/8000019cb1358280
-  val columnBounds3a = Seq(ColumnBounds(0, "name7", "name9"), ColumnBounds(1, "8000019cb0fe9400", "8000019cb11a0b40")/* first 30 minutes */,
-    ColumnBounds(2, "9a2a3b6f2a3b6f2a"/*POINT(1 -12)*/, "9aadd64badd64bad"/*POINT(11 -2)*/))
-  val columnBounds3b = Seq(ColumnBounds(0, "name7", "name9"), ColumnBounds(1, "8000019cb11a0b40", "8000019cb1358280")/* last 30 minutes */,
-    ColumnBounds(2, "9a2a3b6f2a3b6f2a"/*POINT(1 -12)*/, "9aadd64badd64bad"/*POINT(11 -2)*/))
+  val columnBounds3a = Seq(
+    ColumnBounds(0, "name7", "name9"),
+    ColumnBounds(1, dateBounds("2026-03-03T00:00"), dateBounds("2026-03-03T00:30"))/* first 30 minutes */,
+    ColumnBounds(2, z2Bounds("POINT(1 -12)"), z2Bounds("POINT(11 -2)"))
+  )
+  val columnBounds3b = Seq(
+    ColumnBounds(0, "name7", "name9"),
+    ColumnBounds(1, dateBounds("2026-03-03T00:30"), dateBounds("2026-03-03T01:00"))/* last 30 minutes */,
+    ColumnBounds(2, z2Bounds("POINT(1 -12)"), z2Bounds("POINT(11 -2)"))
+  )
 
   val f1 = StorageFile("file1", partition1, 0L, bounds = columnBounds1, timestamp = System.currentTimeMillis() - 100)
   val f2a = StorageFile("file2a", partition2, 1L, bounds = columnBounds2a, timestamp = f1.timestamp + 10)
@@ -56,6 +73,9 @@ abstract class TestAbstractMetadata extends Specification with LazyLogging {
 
   val files = Seq(f1, f2a, f2b, f3a, f3b)
 
+  private def dateBounds(dtg: String): String = StorageMetadata.TypeRegistry.encode(DateParsing.parseDate(dtg + ":00.000Z"))
+  private def z2Bounds(pt: String): String = Z2Encoder.encode(WKTUtils.read(pt).asInstanceOf[Point])
+
   private def partitions(dtg: String, geom: String): Partition = {
     val sf = ScalaSimpleFeature.create(sft, "", "", dtg, geom)
     Partition(schemes.map(_.getPartition(sf)))

geomesa-fs/geomesa-fs-tools/src/main/scala/org/locationtech/geomesa/fs/tools/ingest/FsManageMetadataCommand.scala

@@ -94,7 +94,7 @@ object FsManageMetadataCommand extends LazyLogging {
 
             case other => other
           }
-          s"${metadata.sft.getDescriptor(b.attribute).getLocalName.padTo(padding, " ")} [ $lower,$upper ]"
+          s"${metadata.sft.getDescriptor(b.attribute).getLocalName.padTo(padding, ' ')} [ $lower,$upper ]"
         }
         Command.user.info(s"File bounds:\n  ${bounds.mkString("\n  ")}")
       }