[GH-2760] Extend OSM PBF reader to support additional metadata fields (#2776)

jiayuasu · web-flow · commit 1e9ced927f8d · 2026-03-24T16:03:53.000-07:00
diff --git a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/DenseNodeExtractor.java b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/extractors/DenseNodeExtractor.java
@@ -26,23 +26,42 @@ public class DenseNodeExtractor implements Extractor {
   long latOffset;
   long lonOffset;
   long granularity;
+  int dateGranularity;
   long firstId;
   long firstLat;
   long firstLon;
   Integer keyIndex;
 
+  // DenseInfo delta accumulators
+  boolean hasDenseInfo;
+  long firstTimestamp;
+  long firstChangeset;
+  int firstUid;
+  int firstUserSid;
+
   Osmformat.DenseNodes nodes;
 
   public DenseNodeExtractor(
-      Osmformat.DenseNodes nodes, long latOffset, long lonOffset, long granularity) {
+      Osmformat.DenseNodes nodes,
+      long latOffset,
+      long lonOffset,
+      long granularity,
+      int dateGranularity) {
     this.firstId = 0;
     this.firstLat = 0;
     this.firstLon = 0;
     this.latOffset = latOffset;
     this.lonOffset = lonOffset;
     this.granularity = granularity;
+    this.dateGranularity = dateGranularity;
     this.nodes = nodes;
     this.keyIndex = 0;
+
+    this.hasDenseInfo = nodes.hasDenseinfo() && nodes.getDenseinfo().getVersionCount() > 0;
+    this.firstTimestamp = 0;
+    this.firstChangeset = 0;
+    this.firstUid = 0;
+    this.firstUserSid = 0;
   }
 
   public OsmNode extract(int idx, Osmformat.StringTable stringTable) {
@@ -63,7 +82,53 @@ private OsmNode parse(int idx, Osmformat.StringTable stringTable) {
 
     HashMap<String, String> tags = parseTags(stringTable);
 
-    return new OsmNode(id, lat, lon, tags);
+    OsmNode node = new OsmNode(id, lat, lon, tags);
+
+    if (hasDenseInfo) {
+      Osmformat.DenseInfo denseInfo = nodes.getDenseinfo();
+
+      // version is NOT delta-encoded
+      if (denseInfo.getVersionCount() > idx) {
+        node.setVersion(denseInfo.getVersion(idx));
+      }
+
+      // timestamp, changeset, uid, user_sid are delta-encoded
+      if (denseInfo.getTimestampCount() > idx) {
+        long timestamp = denseInfo.getTimestamp(idx) + firstTimestamp;
+        firstTimestamp = timestamp;
+        node.setTimestamp(timestamp * dateGranularity);
+      }
+
+      if (denseInfo.getChangesetCount() > idx) {
+        long changeset = denseInfo.getChangeset(idx) + firstChangeset;
+        firstChangeset = changeset;
+        node.setChangeset(changeset);
+      }
+
+      if (denseInfo.getUidCount() > idx) {
+        int uid = denseInfo.getUid(idx) + firstUid;
+        firstUid = uid;
+        node.setUid(uid);
+      }
+
+      if (denseInfo.getUserSidCount() > idx) {
+        int userSid = denseInfo.getUserSid(idx) + firstUserSid;
+        firstUserSid = userSid;
+        if (userSid > 0) {
+          node.setUser(stringTable.getS(userSid).toStringUtf8());
+        }
+      }
+
+      // visible is NOT delta-encoded, and may not be present
+      if (denseInfo.getVisibleCount() > idx) {
+        node.setVisible(denseInfo.getVisible(idx));
+      } else {
+        // Per OSM PBF spec, missing 'visible' must be treated as true (especially in history files)
+        node.setVisible(true);
+      }
+    }
+
+    return node;
   }
 
   HashMap<String, String> parseTags(Osmformat.StringTable stringTable) {
diff --git a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/features/InfoResolver.java b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/features/InfoResolver.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.sedona.sql.datasources.osmpbf.features;
+
+import org.apache.sedona.sql.datasources.osmpbf.build.Osmformat;
+import org.apache.sedona.sql.datasources.osmpbf.model.OSMEntity;
+
+public class InfoResolver {
+
+  public static void populateInfo(
+      OSMEntity entity,
+      Osmformat.Info info,
+      Osmformat.StringTable stringTable,
+      int dateGranularity) {
+    if (info == null) {
+      return;
+    }
+    entity.setVersion(info.getVersion());
+    entity.setTimestamp((long) info.getTimestamp() * dateGranularity);
+    entity.setChangeset(info.getChangeset());
+    entity.setUid(info.getUid());
+    if (info.getUserSid() > 0) {
+      entity.setUser(stringTable.getS(info.getUserSid()).toStringUtf8());
+    }
+    if (info.hasVisible()) {
+      entity.setVisible(info.getVisible());
+    } else {
+      // Per OSM PBF spec, when HistoricalInformation is a required_feature,
+      // missing "visible" must be treated as true. Default to true when absent.
+      entity.setVisible(true);
+    }
+  }
+}
diff --git a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/BlobIterator.java b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/BlobIterator.java
@@ -72,9 +72,8 @@ public OSMEntity next() {
 
   Iterator<OSMEntity> resolveIterator() {
     return IteratorUtils.chainedIterator(
-        new WayIterator(currentPrimitiveGroup.getWaysList(), primitiveBlock.getStringtable()),
-        new RelationIterator(
-            currentPrimitiveGroup.getRelationsList(), primitiveBlock.getStringtable()),
+        new WayIterator(currentPrimitiveGroup.getWaysList(), primitiveBlock),
+        new RelationIterator(currentPrimitiveGroup.getRelationsList(), primitiveBlock),
         new NodeIterator(currentPrimitiveGroup.getNodesList(), primitiveBlock),
         currentPrimitiveGroup.getDense() != null
             ? new DenseNodeIterator(
@@ -84,7 +83,8 @@ Iterator<OSMEntity> resolveIterator() {
                     currentPrimitiveGroup.getDense(),
                     primitiveBlock.getLatOffset(),
                     primitiveBlock.getLonOffset(),
-                    primitiveBlock.getGranularity()))
+                    primitiveBlock.getGranularity(),
+                    primitiveBlock.getDateGranularity()))
             : Collections.emptyIterator());
   }
 }
diff --git a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/NodeIterator.java b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/NodeIterator.java
@@ -22,6 +22,7 @@
 import java.util.Iterator;
 import java.util.List;
 import org.apache.sedona.sql.datasources.osmpbf.build.Osmformat;
+import org.apache.sedona.sql.datasources.osmpbf.features.InfoResolver;
 import org.apache.sedona.sql.datasources.osmpbf.features.TagsResolver;
 import org.apache.sedona.sql.datasources.osmpbf.model.OSMEntity;
 import org.apache.sedona.sql.datasources.osmpbf.model.OsmNode;
@@ -85,6 +86,13 @@ private OsmNode parse(int idx) {
     HashMap<String, String> tags =
         TagsResolver.resolveTags(node.getKeysCount(), node::getKeys, node::getVals, stringTable);
 
-    return new OsmNode(id, lat, lon, tags);
+    OsmNode osmNode = new OsmNode(id, lat, lon, tags);
+
+    if (node.hasInfo()) {
+      InfoResolver.populateInfo(
+          osmNode, node.getInfo(), stringTable, primitiveBlock.getDateGranularity());
+    }
+
+    return osmNode;
   }
 }
diff --git a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/RelationIterator.java b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/RelationIterator.java
@@ -22,6 +22,7 @@
 import java.util.Iterator;
 import java.util.List;
 import org.apache.sedona.sql.datasources.osmpbf.build.Osmformat;
+import org.apache.sedona.sql.datasources.osmpbf.features.InfoResolver;
 import org.apache.sedona.sql.datasources.osmpbf.features.TagsResolver;
 import org.apache.sedona.sql.datasources.osmpbf.model.OSMEntity;
 import org.apache.sedona.sql.datasources.osmpbf.model.Relation;
@@ -32,12 +33,15 @@ public class RelationIterator implements Iterator<OSMEntity> {
   long relationCount;
   List<Osmformat.Relation> relations;
   Osmformat.StringTable stringTable;
+  Osmformat.PrimitiveBlock primitiveBlock;
 
-  public RelationIterator(List<Osmformat.Relation> relations, Osmformat.StringTable stringTable) {
+  public RelationIterator(
+      List<Osmformat.Relation> relations, Osmformat.PrimitiveBlock primitiveBlock) {
     this.idx = 0;
     this.relationCount = 0;
     this.relations = relations;
-    this.stringTable = stringTable;
+    this.stringTable = primitiveBlock.getStringtable();
+    this.primitiveBlock = primitiveBlock;
 
     if (relations != null) {
       this.relationCount = relations.size();
@@ -79,7 +83,14 @@ private Relation parse(Osmformat.Relation relation) {
         TagsResolver.resolveTags(
             relation.getKeysCount(), relation::getKeys, relation::getVals, stringTable);
 
-    return new Relation(relation.getId(), tags, refs, refTypes, roles);
+    Relation relationEntity = new Relation(relation.getId(), tags, refs, refTypes, roles);
+
+    if (relation.hasInfo()) {
+      InfoResolver.populateInfo(
+          relationEntity, relation.getInfo(), stringTable, primitiveBlock.getDateGranularity());
+    }
+
+    return relationEntity;
   }
 
   private String[] resolveRefRoles(Osmformat.Relation relation) {
diff --git a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/WayIterator.java b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/iterators/WayIterator.java
@@ -22,6 +22,7 @@
 import java.util.Iterator;
 import java.util.List;
 import org.apache.sedona.sql.datasources.osmpbf.build.Osmformat;
+import org.apache.sedona.sql.datasources.osmpbf.features.InfoResolver;
 import org.apache.sedona.sql.datasources.osmpbf.features.TagsResolver;
 import org.apache.sedona.sql.datasources.osmpbf.model.OSMEntity;
 import org.apache.sedona.sql.datasources.osmpbf.model.Way;
@@ -31,12 +32,14 @@ public class WayIterator implements Iterator<OSMEntity> {
   long waysCount;
   List<Osmformat.Way> ways;
   Osmformat.StringTable stringTable;
+  Osmformat.PrimitiveBlock primitiveBlock;
 
-  public WayIterator(List<Osmformat.Way> ways, Osmformat.StringTable stringTable) {
+  public WayIterator(List<Osmformat.Way> ways, Osmformat.PrimitiveBlock primitiveBlock) {
     this.idx = 0;
     this.waysCount = 0;
     this.ways = ways;
-    this.stringTable = stringTable;
+    this.stringTable = primitiveBlock.getStringtable();
+    this.primitiveBlock = primitiveBlock;
 
     if (ways != null) {
       this.waysCount = ways.size();
@@ -79,6 +82,13 @@ private Way parse(Osmformat.Way way) {
     HashMap<String, String> tags =
         TagsResolver.resolveTags(way.getKeysCount(), way::getKeys, way::getVals, stringTable);
 
-    return new Way(way.getId(), tags, refs);
+    Way wayEntity = new Way(way.getId(), tags, refs);
+
+    if (way.hasInfo()) {
+      InfoResolver.populateInfo(
+          wayEntity, way.getInfo(), stringTable, primitiveBlock.getDateGranularity());
+    }
+
+    return wayEntity;
   }
 }
diff --git a/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/model/OSMEntity.java b/spark/common/src/main/java/org/apache/sedona/sql/datasources/osmpbf/model/OSMEntity.java
@@ -31,6 +31,14 @@ public class OSMEntity {
   private String[] refRoles;
   private String[] refTypes;
 
+  // Metadata fields from Info/DenseInfo
+  private Integer version;
+  private Long timestamp; // milliseconds since epoch
+  private Long changeset;
+  private Integer uid;
+  private String user;
+  private Boolean visible;
+
   public OSMEntity(
       long id, double latitude, double longitude, HashMap<String, String> tags, String kind) {
     this.id = id;
@@ -93,4 +101,52 @@ public String[] getRefTypes() {
   public long getId() {
     return id;
   }
+
+  public Integer getVersion() {
+    return version;
+  }
+
+  public void setVersion(Integer version) {
+    this.version = version;
+  }
+
+  public Long getTimestamp() {
+    return timestamp;
+  }
+
+  public void setTimestamp(Long timestamp) {
+    this.timestamp = timestamp;
+  }
+
+  public Long getChangeset() {
+    return changeset;
+  }
+
+  public void setChangeset(Long changeset) {
+    this.changeset = changeset;
+  }
+
+  public Integer getUid() {
+    return uid;
+  }
+
+  public void setUid(Integer uid) {
+    this.uid = uid;
+  }
+
+  public String getUser() {
+    return user;
+  }
+
+  public void setUser(String user) {
+    this.user = user;
+  }
+
+  public Boolean getVisible() {
+    return visible;
+  }
+
+  public void setVisible(Boolean visible) {
+    this.visible = visible;
+  }
 }
diff --git a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/osm/OsmPartitionReader.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/osm/OsmPartitionReader.scala
@@ -89,6 +89,16 @@ case class OsmPartitionReader(
           if (entity.getRefTypes != null)
             ArrayData.toArrayData(entity.getRefTypes.map(x => UTF8String.fromString(x)))
           else null
+        case "changeset" => entity.getChangeset
+        case "timestamp" =>
+          if (entity.getTimestamp != null)
+            entity.getTimestamp * 1000L // ms to microseconds for Spark TimestampType
+          else null
+        case "uid" => entity.getUid
+        case "user" =>
+          if (entity.getUser != null) UTF8String.fromString(entity.getUser) else null
+        case "version" => entity.getVersion
+        case "visible" => entity.getVisible
       }
     }))
   }
diff --git a/spark/common/src/main/scala/org/apache/sedona/sql/datasources/osm/SchemaProvider.scala b/spark/common/src/main/scala/org/apache/sedona/sql/datasources/osm/SchemaProvider.scala
@@ -18,7 +18,7 @@
  */
 package org.apache.sedona.sql.datasources.osm
 
-import org.apache.spark.sql.types.{ArrayType, DoubleType, LongType, MapType, StringType, StructField, StructType}
+import org.apache.spark.sql.types.{ArrayType, BooleanType, DoubleType, IntegerType, LongType, MapType, StringType, StructField, StructType, TimestampType}
 
 trait SchemaProvider {
   def schema: StructType =
@@ -39,5 +39,11 @@ trait SchemaProvider {
           nullable = true),
         StructField("refs", ArrayType(LongType), nullable = true),
         StructField("ref_roles", ArrayType(StringType), nullable = true),
-        StructField("ref_types", ArrayType(StringType), nullable = true)))
+        StructField("ref_types", ArrayType(StringType), nullable = true),
+        StructField("changeset", LongType, nullable = true),
+        StructField("timestamp", TimestampType, nullable = true),
+        StructField("uid", IntegerType, nullable = true),
+        StructField("user", StringType, nullable = true),
+        StructField("version", IntegerType, nullable = true),
+        StructField("visible", BooleanType, nullable = true)))
 }
diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/OsmReaderTest.scala b/spark/common/src/test/scala/org/apache/sedona/sql/OsmReaderTest.scala
