refactor: improve PBF file loop syntax and cleanup JVM args in import script (#42)

- Fixed syntax error in `import.sh` PBF file verification loop.
- Removed unused JVM arguments for cleaner configuration.
feat: optimize building data processing in ImportService

- Added shard ID calculation and WKB serialization for building geometry.
- Removed redundant geometry checks and streamlined shard buffer updates.
- Synchronized shard list modifications for thread safety.

Author: dgraf-gh

scripts/import.sh

@@ -81,7 +81,7 @@ if [ ${#PBF_FILES} -eq 0 ]; then
 fi
 
 # Verify each PBF file exists
-for f in ${PBF_FILES[@}; do
+for f in ${PBF_FILES[@]}; do
     if [ ! -f "$f" ]; then
         echo "Error: PBF file '$f' does not exist"
         exit 1
@@ -128,10 +128,6 @@ JVM_ARGS="$JVM_ARGS -XX:+UseG1GC"
 JVM_ARGS="$JVM_ARGS -XX:MaxGCPauseMillis=200"
 JVM_ARGS="$JVM_ARGS -XX:+UnlockExperimentalVMOptions"
 JVM_ARGS="$JVM_ARGS -XX:+UseTransparentHugePages"
-JVM_ARGS="$JVM_ARGS -XX:+ExitOnOutOfMemoryError"
-JVM_ARGS="$JVM_ARGS -XX:MaxDirectMemorySize=4G"
-JVM_ARGS="$JVM_ARGS -XX:MaxMetaspaceSize=512M"
-JVM_ARGS="$JVM_ARGS -Xss512k"
 JVM_ARGS="$JVM_ARGS --add-exports=java.base/jdk.internal.ref=ALL-UNNAMED"
 JVM_ARGS="$JVM_ARGS --add-exports=java.base/sun.nio.ch=ALL-UNNAMED"
 JVM_ARGS="$JVM_ARGS --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED"

src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java

@@ -916,6 +916,7 @@ private void processBuildingBoundariesFromIndex(RocksDB poiIndexDb, RocksDB node
                                         PoiIndexRec recInner = recsToProcess.get(i);
                                         try {
                                             org.locationtech.jts.geom.Geometry geometry = buildGeometryFromWay(wayIdInner, nodeCache, wayIndexDb, stats);
+
                                             if (geometry == null) continue;
                                             if (!geometry.isValid()) {
                                                 try {
@@ -925,7 +926,10 @@ private void processBuildingBoundariesFromIndex(RocksDB poiIndexDb, RocksDB node
                                                 }
                                             }
                                             if (geometry == null || geometry.isEmpty() || !geometry.isValid()) continue;
-                                            results.add(new BuildingData(wayIdInner, recInner.subtype, null, geometry));
+                                            byte[] wkb = new WKBWriter().write(geometry);
+                                            Coordinate center = geometry.getEnvelopeInternal().centre();
+                                            long shardId = s2Helper.getShardId(center.getY(), center.getX());
+                                            results.add(new BuildingData(wayIdInner, recInner.subtype, null, wkb, shardId));
                                         } catch (Exception e) {
                                             stats.recordError(ImportStatistics.Stage.PROCESSING_BUILDINGS, Kind.GEOMETRY, wayIdInner, "build-building-geometry-batch", e);
                                         }
@@ -947,9 +951,10 @@ private void processBuildingBoundariesFromIndex(RocksDB poiIndexDb, RocksDB node
                                     List<BuildingData> batchResults = f.get();
                                     Map<Long, List<BuildingData>> currentActiveBuffer = shardBufferRef.get();
                                     for (BuildingData r : batchResults) {
-                                        Coordinate center = r.geometry().getEnvelopeInternal().centre();
-                                        long shardId = s2Helper.getShardId(center.getY(), center.getX());
-                                        currentActiveBuffer.computeIfAbsent(shardId, k -> new CopyOnWriteArrayList<>()).add(r);
+                                        List<BuildingData> shardList = currentActiveBuffer.computeIfAbsent(r.shardId, k -> new ArrayList<>());
+                                        synchronized (shardList) {
+                                            shardList.add(r);
+                                        }
                                         stats.incrementBuildingsProcessed();
                                     }
                                 } catch (Exception e) {
@@ -984,7 +989,10 @@ private void processBuildingBoundariesFromIndex(RocksDB poiIndexDb, RocksDB node
                                     }
                                 }
                                 if (geometry == null || geometry.isEmpty() || !geometry.isValid()) continue;
-                                results.add(new BuildingData(wayIdInner, recInner.subtype, null, geometry));
+                                byte[] wkb = new WKBWriter().write(geometry);
+                                Coordinate center = geometry.getEnvelopeInternal().centre();
+                                long shardId = s2Helper.getShardId(center.getY(), center.getX());
+                                results.add(new BuildingData(wayIdInner, recInner.subtype, null, wkb, shardId));
                             } catch (Exception e) {
                                 stats.recordError(ImportStatistics.Stage.PROCESSING_BUILDINGS, Kind.GEOMETRY, wayIdInner, "build-building-geometry-batch", e);
                             }
@@ -1005,9 +1013,10 @@ private void processBuildingBoundariesFromIndex(RocksDB poiIndexDb, RocksDB node
                     List<BuildingData> batchResults = f.get();
                     Map<Long, List<BuildingData>> currentActiveBuffer = shardBufferRef.get();
                     for (BuildingData r : batchResults) {
-                        Coordinate center = r.geometry().getEnvelopeInternal().centre();
-                        long shardId = s2Helper.getShardId(center.getY(), center.getX());
-                        currentActiveBuffer.computeIfAbsent(shardId, k -> new CopyOnWriteArrayList<>()).add(r);
+                        List<BuildingData> shardList = currentActiveBuffer.computeIfAbsent(r.shardId, k -> new ArrayList<>());
+                        synchronized (shardList) {
+                            shardList.add(r);
+                        }
                         stats.incrementBuildingsProcessed();
                     }
                 } catch (Exception e) {
@@ -1053,21 +1062,6 @@ private boolean hasAddressTags(OsmEntity entity) {
         return false;
     }
 
-    private boolean isBuildingWay(OsmWay way) {
-        for (int i = 0; i < way.getNumberOfTags(); i++) {
-            OsmTag tag = way.getTag(i);
-            if ("building".equals(tag.getKey())) {
-                String val = tag.getValue();
-                return switch (val) {
-                    case "yes", "commercial", "retail", "industrial",
-                         "office", "apartments", "residential" -> true;
-                    default -> false;
-                };
-            }
-        }
-        return false;
-    }
-
     private void writeShardBatchAppendOnly(Map<Long, List<PoiData>> shardBuffer, RocksDB appendDb, ImportStatistics stats) throws Exception {
         FlatBufferBuilder builder = new FlatBufferBuilder(1024 * 32);
 
@@ -1308,7 +1302,7 @@ private int serializeBuildingData(FlatBufferBuilder builder, BuildingData buildi
 
         int geometryOff = 0;
         if (building.geometry() != null) {
-            byte[] wkb = new WKBWriter().write(building.geometry());
+            byte[] wkb = building.geometry();
             int geometryDataOff = Geometry.createDataVector(builder, wkb);
             geometryOff = Geometry.createGeometry(builder, geometryDataOff);
         }
@@ -2022,7 +2016,7 @@ private record AddressData(String street, String houseNumber, String postcode, S
     private record BoundaryResultLite(long osmId, int level, String name, String code, org.locationtech.jts.geom.Geometry geometry) {
     }
 
-    private record BuildingData(long id, String name, String code, org.locationtech.jts.geom.Geometry geometry) {
+    private record BuildingData(long id, String name, String code, byte[] geometry, long shardId) {
     }
 
     private Long safeEntityId(EntityContainer container) {