Skip to content

Commit 433feea

Browse files
dgraf-ghdgraf-gh
authored
26 bug missing elements in the hiearchy (#28)
* chore: add boundary way indexing and enhance stats handling in ImportService and update.sh - Introduced `boundaryWayIndexDb` for boundary member ways. - Updated ImportStatistics with boundary way metrics. - Refactored `update.sh` to improve test case structure and add retry logic. * chore: remove unused `QueryInfo` class and related properties from `POIResponse` and `ReverseGeocodingService`
1 parent 0d50d9b commit 433feea

7 files changed

Lines changed: 127 additions & 85 deletions

File tree

‎scripts/update.sh‎

Lines changed: 30 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,6 @@ REMOTE_BASE_DIR="/opt/paikka/data"
2727
GEOCODER_ADMIN_URL="http://localhost:8080/admin/refresh-db"
2828
GEOCODER_TEST_URL_BASE="http://localhost:8080/v1/reverse"
2929

30-
# --- Verification Test Cases ---
31-
declare -A TEST_CASES=(
32-
["lat=52.516280&lon=13.377635"]="518071791" # Brandenburger Tor
33-
["lat=48.85826&lon=2.2945008"]="5013364" # Eiffel Tower
34-
["lat=40.68924&lon=-74.044502"]="32965412" # Statue of Liberty
35-
)
36-
3730
# Global variables that will be set by parse_args_and_configure or environment
3831
REMOTE_USER=""
3932
REMOTE_HOST=""
@@ -166,64 +159,73 @@ remote_sync_bundle() {
166159
remote_deploy_and_verify() {
167160
log "REMOTE: Executing remote deployment (Atomic Swap)"
168161

169-
# Convert TEST_CASES to a format that can be passed to remote shell
170-
local test_cases_str=""
171-
for key in "${!TEST_CASES}"; do
172-
test_cases_str+="[\"$key\"]=\"${TEST_CASES[$key]}\" "
173-
done
174-
162+
# shellcheck disable=SC2087
175163
ssh "${REMOTE_USER}@${REMOTE_HOST}" /bin/bash << EOF
176164
set -e
177-
BASE_DIR="${REMOTE_BASE_DIR}"
165+
BASE_DIR="/opt/paikka/data"
178166
API_TOKEN="${GEOCODER_API_TOKEN}"
179-
ADMIN_URL="${GEOCODER_ADMIN_URL}"
180-
TEST_URL_BASE="${GEOCODER_TEST_URL_BASE}"
167+
ADMIN_URL="http://localhost:8080/admin/refresh-db"
168+
TEST_URL_BASE="http://localhost:8080/api/v1/reverse"
181169
NEW_RELEASE_DIR="releases/${LATEST_RELEASE_DIR_NAME}"
182170
LIVE_DATA_SYMLINK="live_data"
183171
184-
# Define TESTS array on remote side
185-
declare -A TESTS=($test_cases_str)
186-
187172
echo_remote() {
188173
echo "[REMOTE] \$1"
189174
}
190175
191176
cd "\$BASE_DIR"
192177
193178
OLD_RELEASE_DIR=""
194-
[ -L "\$LIVE_DATA_SYMLINK" ] && OLD_RELEASE_DIR=\$(readlink \$LIVE_DATA_SYMLINK)
179+
[ -L "\$LIVE_DATA_SYMLINK" ] && OLD_RELEASE_DIR=\$(readlink "\$LIVE_DATA_SYMLINK")
195180
196181
echo_remote "Switching symlink: \$LIVE_DATA_SYMLINK -> \$NEW_RELEASE_DIR"
197182
ln -sfn "\$NEW_RELEASE_DIR" "\$LIVE_DATA_SYMLINK"
198183
199184
echo_remote "Refreshing Geocoder DB..."
200-
HTTP_STATUS=\$(curl -s -o /dev/null -w "%{http_code}" -X POST -H "X-Admin-Token: \$API_TOKEN" "\$ADMIN_URL")
185+
HTTP_STATUS=\$(curl -s -o /dev/null -w "%{http_code}" --max-time 300 -X POST -H "X-Admin-Token: \$API_TOKEN" "\$ADMIN_URL")
201186
202187
if [ "\$HTTP_STATUS" -ne 200 ]; then
203188
echo_remote "ERROR: Refresh failed (\$HTTP_STATUS). Rolling back."
204189
[ -n "\$OLD_RELEASE_DIR" ] && ln -sfn "\$OLD_RELEASE_DIR" "\$LIVE_DATA_SYMLINK"
205190
exit 1
206191
fi
207192
193+
echo_remote "Refresh completed successfully"
194+
208195
# --- 2. Verify ---
209196
echo_remote "Verifying new data..."
210197
VERIFICATION_FAILED=0
211-
for query in "\${!TESTS}"; do
212-
ACTUAL_ID=\$(curl -s "\$TEST_URL_BASE?\$query" | jq -r '.[0].id // "not_found"')
213-
if [ "\$ACTUAL_ID" != "\${TESTS[\$query]}" ]; then
214-
echo_remote " --> FAILED: For \$query, expected '\${TESTS[\$query]}', got '\$ACTUAL_ID'"
198+
199+
QUERIES[0]="lat=52.516280&lon=13.377635"
200+
QUERIES[1]="lat=48.85826&lon=2.2945008"
201+
QUERIES[2]="lat=40.68924&lon=-74.044502"
202+
203+
EXPECTED_IDS[0]="518071791"
204+
EXPECTED_IDS[1]="5013364"
205+
EXPECTED_IDS[2]="32965412"
206+
207+
# Get the number of elements
208+
NUM_TESTS=3
209+
210+
for ((i=0; i<NUM_TESTS; i++)); do
211+
query="\${QUERIES[\$i]}"
212+
expected_id="\${EXPECTED_IDS[\$i]}"
213+
echo_remote "Testing URL: \$TEST_URL_BASE?\$query"
214+
ACTUAL_ID=\$(curl -s --max-time 30 "\$TEST_URL_BASE?\$query" | jq -r '.results[0].id // "not_found"')
215+
echo_remote "Got ID: \$ACTUAL_ID, Expected: \$expected_id"
216+
if [ "\$ACTUAL_ID" != "\$expected_id" ]; then
217+
echo_remote " --> FAILED: For \$query, expected '\$expected_id', got '\$ACTUAL_ID'"
215218
VERIFICATION_FAILED=1
216219
else
217220
echo_remote " --> SUCCESS: Verified query for \$query"
218221
fi
219222
done
220-
221223
# --- 3. Finalize or Rollback ---
222224
if [ \$VERIFICATION_FAILED -eq 1 ]; then
223225
echo_remote "VERIFICATION FAILED. Rolling back and re-refreshing."
224226
if [ -n "\$OLD_RELEASE_DIR" ] && [ -d "\$OLD_RELEASE_DIR" ]; then
225227
ln -sfn "\$OLD_RELEASE_DIR" "\$LIVE_DATA_SYMLINK"
226-
curl -s -o /dev/null -X POST -H "X-Admin-Token: \$API_TOKEN" "\$ADMIN_URL"
228+
curl -s -o /dev/null --max-time 300 -X POST -H "X-Admin-Token: \$API_TOKEN" "\$ADMIN_URL"
227229
echo_remote "Rollback to \$OLD_RELEASE_DIR complete. Faulty data in \$NEW_RELEASE_DIR is kept for inspection."
228230
exit 1
229231
else
@@ -310,4 +312,4 @@ main() {
310312
###
311313
if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
312314
main "$@"
313-
fi
315+
fi

‎src/main/java/com/dedicatedcode/paikka/dto/POIResponse.java‎

Lines changed: 1 addition & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -56,9 +56,6 @@ public class POIResponse {
5656
@JsonProperty("boundary")
5757
private GeoJsonGeometry boundary;
5858

59-
@JsonProperty("query")
60-
private QueryInfo query;
61-
6259
// Constructors
6360
public POIResponse() {}
6461

@@ -96,9 +93,6 @@ public POIResponse() {}
9693
public GeoJsonGeometry getBoundary() { return boundary; }
9794
public void setBoundary(GeoJsonGeometry boundary) { this.boundary = boundary; }
9895

99-
public QueryInfo getQuery() { return query; }
100-
public void setQuery(QueryInfo query) { this.query = query; }
101-
10296
public static class HierarchyItem {
10397
@JsonProperty("level")
10498
private int level;
@@ -155,32 +149,5 @@ public HierarchyItem(int level, String type, String name, String code, long osmI
155149
public String getCode() { return code; }
156150
public void setCode(String code) { this.code = code; }
157151
}
158-
159-
public static class QueryInfo {
160-
@JsonProperty("lat")
161-
private double lat;
162-
163-
@JsonProperty("lon")
164-
private double lon;
165-
166-
@JsonProperty("lang")
167-
private String lang;
168-
169-
public QueryInfo() {}
170-
171-
public QueryInfo(double lat, double lon, String lang) {
172-
this.lat = lat;
173-
this.lon = lon;
174-
this.lang = lang;
175-
}
176-
177-
public double getLat() { return lat; }
178-
public void setLat(double lat) { this.lat = lat; }
179-
180-
public double getLon() { return lon; }
181-
public void setLon(double lon) { this.lon = lon; }
182-
183-
public String getLang() { return lang; }
184-
public void setLang(String lang) { this.lang = lang; }
185-
}
152+
186153
}

‎src/main/java/com/dedicatedcode/paikka/service/ReverseGeocodingService.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -415,9 +415,6 @@ private POIResponse convertPOIToResponse(POIData poi, double queryLat, double qu
415415
}
416416
}
417417

418-
// Query information
419-
response.setQuery(new POIResponse.QueryInfo(queryLat, queryLon, lang));
420-
421418
return response;
422419
}
423420

‎src/main/java/com/dedicatedcode/paikka/service/importer/HierarchyCache.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,9 @@ public record CachedBoundary(int level, String name, String code, long osmId, En
123123
public boolean contains(double lon, double lat) {
124124
if (mir != null && mir.contains(lon, lat)) return true;
125125
if (!mbr.contains(lon, lat)) return false;
126-
return locator.locate(new Coordinate(lon, lat)) != Location.EXTERIOR;
126+
synchronized (this) {
127+
return locator.locate(new Coordinate(lon, lat)) != Location.EXTERIOR;
128+
}
127129
}
128130
}
129131

‎src/main/java/com/dedicatedcode/paikka/service/importer/ImportService.java‎

Lines changed: 71 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,7 @@ public void importData(String pbfFilePath, String dataDir) throws Exception {
105105
Path appendDbPath = dataDirectory.resolve("tmp/append_poi");
106106
Path nodeCacheDbPath = dataDirectory.resolve("tmp/node_cache");
107107
Path wayIndexDbPath = dataDirectory.resolve("tmp/way_index");
108+
Path boundaryWayIndexDbPath = dataDirectory.resolve("tmp/boundary_way_index");
108109
Path neededNodesDbPath = dataDirectory.resolve("tmp/needed_nodes");
109110
Path relIndexDbPath = dataDirectory.resolve("tmp/rel_index");
110111
Path poiIndexDbPath = dataDirectory.resolve("tmp/poi_index");
@@ -114,6 +115,7 @@ public void importData(String pbfFilePath, String dataDir) throws Exception {
114115
cleanupDatabase(gridIndexDbPath);
115116
cleanupDatabase(nodeCacheDbPath);
116117
cleanupDatabase(wayIndexDbPath);
118+
cleanupDatabase(boundaryWayIndexDbPath);
117119
cleanupDatabase(neededNodesDbPath);
118120
cleanupDatabase(relIndexDbPath);
119121
cleanupDatabase(poiIndexDbPath);
@@ -185,6 +187,7 @@ public void importData(String pbfFilePath, String dataDir) throws Exception {
185187
RocksDB gridIndexDb = RocksDB.open(gridOpts, gridIndexDbPath.toString());
186188
RocksDB nodeCache = RocksDB.open(nodeOpts, nodeCacheDbPath.toString());
187189
RocksDB wayIndexDb = RocksDB.open(wayIndexOpts, wayIndexDbPath.toString());
190+
RocksDB neededBoundaryWaysDb = RocksDB.open(wayIndexOpts, boundaryWayIndexDbPath.toString());
188191
RocksDB neededNodesDb = RocksDB.open(neededNodesOpts, neededNodesDbPath.toString());
189192
RocksDB relIndexDb = RocksDB.open(wayIndexOpts, relIndexDbPath.toString());
190193
RocksDB poiIndexDb = RocksDB.open(poiIndexOpts, poiIndexDbPath.toString());
@@ -194,21 +197,23 @@ public void importData(String pbfFilePath, String dataDir) throws Exception {
194197
stats.printPhaseHeader("PASS 1: Discovery & Indexing");
195198
long pass1Start = System.currentTimeMillis();
196199
stats.setCurrentPhase(1, "1.1.1: Discovery & Indexing");
197-
pass1DiscoveryAndIndexing(pbfFile, wayIndexDb, neededNodesDb, relIndexDb, poiIndexDb, stats);
200+
pass1DiscoveryAndIndexing(pbfFile, wayIndexDb, neededBoundaryWaysDb, neededNodesDb, relIndexDb, poiIndexDb, stats);
201+
stats.setCurrentPhase(2, "1.1.2: Indexing boundary member ways");
202+
indexBoundaryMemberWays(pbfFile, neededBoundaryWaysDb, wayIndexDb, neededNodesDb, stats);
198203
stats.printPhaseSummary("PASS 1", pass1Start);
199204

200205
// PASS 2: Nodes Cache, Boundaries, POIs
201206
stats.printPhaseHeader("PASS 2: Nodes Cache, Boundaries, POIs");
202207
long pass2Start = System.currentTimeMillis();
203-
stats.setCurrentPhase(2, "1.1.2: Caching node coordinates");
208+
stats.setCurrentPhase(3, "1.1.3: Caching node coordinates");
204209
cacheNeededNodeCoordinates(pbfFile, neededNodesDb, nodeCache, stats);
205210

206-
stats.setCurrentPhase(3, "1.2: Processing administrative boundaries");
211+
stats.setCurrentPhase(4, "1.2: Processing administrative boundaries");
207212
processAdministrativeBoundariesFromIndex(relIndexDb, nodeCache, wayIndexDb, gridIndexDb, boundariesDb, stats);
208-
stats.setCurrentPhase(4, "2.1: Processing POIs & Sharding");
213+
stats.setCurrentPhase(5, "2.1: Processing POIs & Sharding");
209214
pass2PoiShardingFromIndex(nodeCache, wayIndexDb, appendDb, boundariesDb, poiIndexDb, gridIndexDb, stats);
210215

211-
stats.setCurrentPhase(5, "2.2: Compacting POIs");
216+
stats.setCurrentPhase(6, "2.2: Compacting POIs");
212217
compactShards(appendDb, shardsDb, stats);
213218
stats.stop();
214219
stats.printPhaseSummary("PASS 2", pass2Start);
@@ -226,6 +231,7 @@ public void importData(String pbfFilePath, String dataDir) throws Exception {
226231
gridIndexDbPath,
227232
nodeCacheDbPath,
228233
wayIndexDbPath,
234+
boundaryWayIndexDbPath,
229235
neededNodesDbPath,
230236
relIndexDbPath,
231237
poiIndexDbPath,
@@ -264,6 +270,43 @@ private void writeMetadataFile(Path pbfFile, Path dataDirectory) throws IOExcept
264270
System.out.println("\n\033[1;32mMetadata file written to: " + metadataPath + "\033[0m");
265271
}
266272

273+
private void indexBoundaryMemberWays(Path pbfFile,
274+
RocksDB neededBoundaryWaysDb,
275+
RocksDB wayIndexDb,
276+
RocksDB neededNodesDb,
277+
ImportStatistics stats) throws Exception {
278+
final byte[] ONE = new byte[]{1};
279+
try (RocksBatchWriter wayWriter = new RocksBatchWriter(wayIndexDb, 10_000, stats);
280+
RocksBatchWriter neededWriter = new RocksBatchWriter(neededNodesDb, 500_000, stats)) {
281+
282+
withPbfIterator(pbfFile, iterator -> {
283+
284+
while (iterator.hasNext()) {
285+
EntityContainer container = iterator.next();
286+
if (container.getType() != EntityType.Way) continue;
287+
288+
OsmWay way = (OsmWay) container.getEntity();
289+
byte[] wayKey = s2Helper.longToByteArray(way.getId());
290+
291+
// Only process ways that are needed AND not already indexed
292+
if (neededBoundaryWaysDb.get(wayKey) == null) continue;
293+
if (wayIndexDb.get(wayKey) != null) { continue; }
294+
int n = way.getNumberOfNodes();
295+
long[] nodeIds = new long[n];
296+
for (int j = 0; j < n; j++) {
297+
long nid = way.getNodeId(j);
298+
nodeIds[j] = nid;
299+
neededWriter.put(s2Helper.longToByteArray(nid), ONE);
300+
}
301+
wayWriter.put(wayKey, s2Helper.longArrayToByteArray(nodeIds));
302+
stats.incrementWaysProcessed();
303+
}
304+
});
305+
306+
wayWriter.flush();
307+
neededWriter.flush();
308+
}
309+
}
267310

268311
private void updateGridIndexEntry(RocksDB gridIndexDb, long cellId, long osmId) throws Exception {
269312
byte[] key = s2Helper.longToByteArray(cellId);
@@ -282,10 +325,11 @@ private void updateGridIndexEntry(RocksDB gridIndexDb, long cellId, long osmId)
282325
}
283326
}
284327

285-
private void pass1DiscoveryAndIndexing(Path pbfFile, RocksDB wayIndexDb, RocksDB neededNodesDb, RocksDB relIndexDb, RocksDB poiIndexDb, ImportStatistics stats) throws Exception {
328+
private void pass1DiscoveryAndIndexing(Path pbfFile, RocksDB wayIndexDb, RocksDB boundaryWayIndexDb, RocksDB neededNodesDb, RocksDB relIndexDb, RocksDB poiIndexDb, ImportStatistics stats) throws Exception {
286329

287330
final byte[] ONE = new byte[]{1};
288331
try (RocksBatchWriter wayWriter = new RocksBatchWriter(wayIndexDb, 10_000, stats);
332+
RocksBatchWriter boundaryWayWriter = new RocksBatchWriter(boundaryWayIndexDb, 10_000, stats);
289333
RocksBatchWriter neededWriter = new RocksBatchWriter(neededNodesDb, 500_000, stats);
290334
RocksBatchWriter relWriter = new RocksBatchWriter(relIndexDb, 2_000, stats);
291335
RocksBatchWriter poiWriter = new RocksBatchWriter(poiIndexDb, 20_000, stats)) {
@@ -336,6 +380,13 @@ private void pass1DiscoveryAndIndexing(Path pbfFile, RocksDB wayIndexDb, RocksDB
336380
stats.incrementRelationsFound();
337381
RelRec rec = buildRelRec(relation);
338382
relWriter.put(s2Helper.longToByteArray(relation.getId()), encodeRelRec(rec));
383+
for (long wid : rec.outer) {
384+
boundaryWayWriter.put(s2Helper.longToByteArray(wid), ONE);
385+
}
386+
for (long wid : rec.inner) {
387+
boundaryWayWriter.put(s2Helper.longToByteArray(wid), ONE);
388+
}
389+
339390
}
340391
}
341392
} catch (Exception e) {
@@ -1447,19 +1498,30 @@ private long computeDirectorySize(Path root) {
14471498
}
14481499
}
14491500

1450-
private void recordSizeMetrics(ImportStatistics stats, Path shardsDbPath, Path boundariesDbPath, Path gridIndexDbPath, Path nodeCacheDbPath, Path wayIndexDbPath, Path neededNodesDbPath, Path relIndexDbPath, Path poiIndexDbPath, Path appendDbPath) {
1501+
private void recordSizeMetrics(ImportStatistics stats,
1502+
Path shardsDbPath,
1503+
Path boundariesDbPath,
1504+
Path gridIndexDbPath,
1505+
Path nodeCacheDbPath,
1506+
Path wayIndexDbPath,
1507+
Path boundaryWayIndexDbPath,
1508+
Path neededNodesDbPath,
1509+
Path relIndexDbPath,
1510+
Path poiIndexDbPath,
1511+
Path appendDbPath) {
14511512
long shards = computeDirectorySize(shardsDbPath);
14521513
long boundaries = computeDirectorySize(boundariesDbPath);
14531514
long dataset = shards + boundaries;
14541515

14551516
long grid = computeDirectorySize(gridIndexDbPath);
14561517
long node = computeDirectorySize(nodeCacheDbPath);
14571518
long way = computeDirectorySize(wayIndexDbPath);
1519+
long boundaryWay = computeDirectorySize(boundaryWayIndexDbPath);
14581520
long needed = computeDirectorySize(neededNodesDbPath);
14591521
long rel = computeDirectorySize(relIndexDbPath);
14601522
long poi = computeDirectorySize(poiIndexDbPath);
14611523
long append = computeDirectorySize(appendDbPath);
1462-
long tmpTotal = grid + node + way + needed + rel + poi + append;
1524+
long tmpTotal = grid + node + way + boundaryWay + needed + rel + poi + append;
14631525

14641526
stats.setShardsBytes(shards);
14651527
stats.setBoundariesBytes(boundaries);
@@ -1468,6 +1530,7 @@ private void recordSizeMetrics(ImportStatistics stats, Path shardsDbPath, Path b
14681530
stats.setTmpGridBytes(grid);
14691531
stats.setTmpNodeBytes(node);
14701532
stats.setTmpWayBytes(way);
1533+
stats.setTmpBoundaryWayBytes(boundaryWay);
14711534
stats.setTmpNeededBytes(needed);
14721535
stats.setTmpRelBytes(rel);
14731536
stats.setTmpPoiBytes(poi);

0 commit comments

Comments
 (0)