Skip to content

Commit f63d6cf

Browse files
committed
chore: add boundary way metrics and indexing improvements to ImportStatistics and ImportService
- Introduced `boundaryWaysProcessed` and `boundaryPhaseEntitiesRead` metrics in ImportStatistics. - Updated phase 1.1.2 to show detailed boundary processing stats. - Enhanced ImportService with boundary-specific metrics tracking and reset logic.
1 parent 0e49556 commit f63d6cf

2 files changed

Lines changed: 31 additions & 26 deletions

File tree

src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java

Lines changed: 4 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,7 @@ private void indexBoundaryMemberWays(Path pbfFile,
275275
RocksDB wayIndexDb,
276276
RocksDB neededNodesDb,
277277
ImportStatistics stats) throws Exception {
278+
stats.resetBoundaryPhaseEntitiesRead();
278279
final byte[] ONE = new byte[]{1};
279280
try (RocksBatchWriter wayWriter = new RocksBatchWriter(wayIndexDb, 10_000, stats);
280281
RocksBatchWriter neededWriter = new RocksBatchWriter(neededNodesDb, 500_000, stats)) {
@@ -283,6 +284,7 @@ private void indexBoundaryMemberWays(Path pbfFile,
283284

284285
while (iterator.hasNext()) {
285286
EntityContainer container = iterator.next();
287+
stats.incrementBoundaryPhaseEntitiesRead();
286288
if (container.getType() != EntityType.Way) continue;
287289

288290
OsmWay way = (OsmWay) container.getEntity();
@@ -299,7 +301,7 @@ private void indexBoundaryMemberWays(Path pbfFile,
299301
neededWriter.put(s2Helper.longToByteArray(nid), ONE);
300302
}
301303
wayWriter.put(wayKey, s2Helper.longArrayToByteArray(nodeIds));
302-
stats.incrementWaysProcessed();
304+
stats.incrementBoundaryWaysProcessed();
303305
}
304306
});
305307

@@ -308,23 +310,6 @@ private void indexBoundaryMemberWays(Path pbfFile,
308310
}
309311
}
310312

311-
private void updateGridIndexEntry(RocksDB gridIndexDb, long cellId, long osmId) throws Exception {
312-
byte[] key = s2Helper.longToByteArray(cellId);
313-
synchronized (this) {
314-
byte[] existingData = gridIndexDb.get(key);
315-
long[] newArray;
316-
if (existingData == null) {
317-
newArray = new long[]{osmId};
318-
} else {
319-
long[] oldArray = s2Helper.byteArrayToLongArray(existingData);
320-
if (Arrays.stream(oldArray).anyMatch(id -> id == osmId)) return;
321-
newArray = Arrays.copyOf(oldArray, oldArray.length + 1);
322-
newArray[oldArray.length] = osmId;
323-
}
324-
gridIndexDb.put(key, s2Helper.longArrayToByteArray(newArray));
325-
}
326-
}
327-
328313
private void pass1DiscoveryAndIndexing(Path pbfFile, RocksDB wayIndexDb, RocksDB boundaryWayIndexDb, RocksDB neededNodesDb, RocksDB relIndexDb, RocksDB poiIndexDb, ImportStatistics stats) throws Exception {
329314

330315
final byte[] ONE = new byte[]{1};
@@ -1394,7 +1379,7 @@ private void storeBoundary(long osmId, int level, String name, String code,
13941379
hasMir = true;
13951380
}
13961381
} catch (Exception e) {
1397-
// MIR computation failed — boundary still works, just no fast-path optimisation
1382+
// MIR computation failed — boundary still works, just no fast-path optimization
13981383
}
13991384

14001385
int nameOffset = fbb.createString(name != null ? name : "Unknown");

src/main/java/com/dedicatedcode/paikka/service/importer/ImportStatistics.java

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,9 @@ public String toString() {
8989
private final AtomicLong rocksDbWrites = new AtomicLong(0);
9090
private final AtomicLong queueSize = new AtomicLong(0);
9191
private final AtomicLong activeThreads = new AtomicLong(0);
92+
private final AtomicLong boundaryWaysProcessed = new AtomicLong(0);
93+
private final AtomicLong boundaryPhaseEntitiesRead = new AtomicLong(0);
94+
9295
private volatile String currentPhase = "Initializing";
9396
private volatile boolean running = true;
9497
private final long startTime = System.currentTimeMillis();
@@ -97,8 +100,6 @@ public String toString() {
97100
private final AtomicLong compactionEntriesTotal = new AtomicLong(0);
98101
private final AtomicLong compactionEntriesProcessed = new AtomicLong(0);
99102

100-
101-
102103
private volatile long compactionStartTime = 0;
103104

104105
private volatile long datasetBytes;
@@ -217,6 +218,25 @@ public void decrementActiveThreads() {
217218
activeThreads.decrementAndGet();
218219
}
219220

221+
public long getBoundaryWaysProcessed() {
222+
return boundaryWaysProcessed.get();
223+
}
224+
225+
public void incrementBoundaryWaysProcessed() {
226+
boundaryWaysProcessed.incrementAndGet();
227+
}
228+
229+
public long getBoundaryPhaseEntitiesRead() {
230+
return boundaryPhaseEntitiesRead.get();
231+
}
232+
233+
public void incrementBoundaryPhaseEntitiesRead() {
234+
boundaryPhaseEntitiesRead.incrementAndGet();
235+
}
236+
237+
public void resetBoundaryPhaseEntitiesRead() {
238+
boundaryPhaseEntitiesRead.set(0);
239+
}
220240
public String getCurrentPhase() {
221241
return currentPhase;
222242
}
@@ -437,12 +457,12 @@ public void startProgressReporter() {
437457
sb.append(String.format(" │ \033[35mRelations:\033[0m %s", formatCompactNumber(getRelationsFound())));
438458

439459
} else if (phase.contains("1.1.2")) {
440-
long pbfPerSec = phaseSeconds > 0 ? (long)(getEntitiesRead() / phaseSeconds) : 0;
441-
sb.append(String.format("\033[1;36m[%s]\033[0m \033[1mScanning PBF Structure\033[0m", formatTime(elapsed)));
460+
long boundaryEntities = getBoundaryPhaseEntitiesRead();
461+
long pbfPerSec = phaseSeconds > 0 ? (long)(boundaryEntities / phaseSeconds) : 0;
462+
sb.append(String.format("\033[1;36m[%s]\033[0m \033[1mIndexing Boundary Member Ways\033[0m", formatTime(elapsed)));
442463
sb.append(String.format(" │ \033[32mPBF Entities:\033[0m %s \033[33m(%s/s)\033[0m",
443-
formatCompactNumber(getEntitiesRead()), formatCompactRate(pbfPerSec)));
444-
sb.append(String.format(" │ \033[37mNodes Found:\033[0m %s", formatCompactNumber(getNodesFound())));
445-
sb.append(String.format(" │ \033[34mWays Found:\033[0m %s", formatCompactNumber(getWaysProcessed())));
464+
formatCompactNumber(boundaryEntities), formatCompactRate(pbfPerSec)));
465+
sb.append(String.format(" │ \033[34mBoundary Ways:\033[0m %s", formatCompactNumber(getBoundaryWaysProcessed())));
446466
sb.append(String.format(" │ \033[35mRelations:\033[0m %s", formatCompactNumber(getRelationsFound())));
447467

448468
} else if (phase.contains("1.1.3")) {

0 commit comments

Comments
 (0)