Skip to content

Commit dfa440f

Browse files
michaelmwumichaelmwu
authored
Fix saved list CID parsing
Fix saved-list place CID extraction to prefer the structured fingerprint over S2 cell IDs, and narrow fallback CID detection.
1 parent 86bc7c8 commit dfa440f

2 files changed

Lines changed: 73 additions & 16 deletions

File tree

‎src/gmaps_scraper/parser.py‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -587,11 +587,11 @@ def _looks_like_address(value: str | None) -> bool:
587587
def _find_cid(node: list[JSONValue] | None) -> str | None:
588588
if node is None:
589589
return None
590-
structured = _find_cid_in_value(_safe_index(node, 6))
590+
structured = _find_cid_in_structured_slot(_safe_index(node, 6))
591591
if structured is not None:
592592
return structured
593593
for value in node:
594-
candidate = _find_cid_in_value(value)
594+
candidate = _find_cid_in_fallback_value(value)
595595
if candidate is not None:
596596
return candidate
597597
return None
@@ -637,7 +637,7 @@ def _metadata_matches_coordinate(
637637
)
638638

639639

640-
def _find_cid_in_value(value: JSONValue | None) -> str | None:
640+
def _find_cid_in_structured_slot(value: JSONValue | None) -> str | None:
641641
if isinstance(value, int):
642642
return _normalize_cid_token(str(value))
643643
if isinstance(value, str):
@@ -655,14 +655,38 @@ def _find_cid_in_value(value: JSONValue | None) -> str | None:
655655
]
656656
if not numeric_texts:
657657
return None
658-
for text in numeric_texts:
659-
if not text.startswith("-"):
660-
return _normalize_cid_token(text)
661658
if len(numeric_texts) >= 2:
662659
return _normalize_cid_token(numeric_texts[1])
663660
return _normalize_cid_token(numeric_texts[0])
664661

665662

663+
def _find_cid_in_fallback_value(value: JSONValue | None) -> str | None:
664+
if isinstance(value, int | str):
665+
return _normalize_fallback_cid_token(value)
666+
if not isinstance(value, list):
667+
return None
668+
owner = _parse_list_owner(value)
669+
if owner is not None and (owner.photo_url is not None or owner.profile_id is not None):
670+
return None
671+
numeric_texts = [
672+
text
673+
for text in (_clean_text(item) for item in value)
674+
if text is not None and _LONG_INTEGER_PATTERN.fullmatch(text) is not None
675+
]
676+
if len(numeric_texts) == 1:
677+
return _normalize_fallback_cid_token(numeric_texts[0])
678+
return None
679+
680+
681+
def _normalize_fallback_cid_token(value: JSONValue | None) -> str | None:
682+
text = str(value) if isinstance(value, int) else _clean_text(value)
683+
if text is None or _LONG_INTEGER_PATTERN.fullmatch(text) is None:
684+
return None
685+
if len(text.removeprefix("-")) < 15:
686+
return None
687+
return _normalize_cid_token(text)
688+
689+
666690
def _normalize_cid_token(value: JSONValue | None) -> str | None:
667691
text = _clean_text(value)
668692
if text is None or _LONG_INTEGER_PATTERN.fullmatch(text) is None:

‎tests/test_parser.py‎

Lines changed: 43 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@
1515
"https://www.google.com/maps/@30.5370705,125.4120472,6z/"
1616
"data=!4m3!11m2!2sTESTLISTABC123456789!3e3?entry=ttu"
1717
)
18+
_NORTHWIND_CELL_ID = "7451636382641713350"
19+
_NORTHWIND_CID = "9055794338847426964"
20+
_HARBOR_CELL_ID = "1234567890123456789"
21+
_HARBOR_CID = "5678901234567890123"
1822
_LIST_NODE = [
1923
["TESTLISTABC123456789", 1, None, 1, 1],
2024
4,
@@ -38,7 +42,7 @@
3842
None,
3943
"Example District",
4044
[None, None, 35.6501307, 139.6868459],
41-
["7451636382641713350", "aux"],
45+
[_NORTHWIND_CELL_ID, _NORTHWIND_CID],
4246
"/g/11northwind",
4347
],
4448
"Northwind Cafe",
@@ -47,7 +51,7 @@
4751
None,
4852
None,
4953
[[[[3, None, "104356373423434804635", "❤️", [1776133481, 81561000]]]]],
50-
[[1], ["7451636382641713350", "aux"]],
54+
[[1], [_NORTHWIND_CELL_ID, _NORTHWIND_CID]],
5155
[1776063335, 302383000],
5256
[1776132745, 850748000],
5357
None,
@@ -66,7 +70,7 @@
6670
None,
6771
"Market Square",
6872
[None, None, 35.6915776, 139.7836109],
69-
["1234567890123456789"],
73+
[_HARBOR_CELL_ID, _HARBOR_CID],
7074
"/g/11harborbakery",
7175
],
7276
"Harbor Bakery",
@@ -75,7 +79,7 @@
7579
None,
7680
None,
7781
[],
78-
[[1], ["1234567890123456789", "aux-2"]],
82+
[[1], [_HARBOR_CELL_ID, _HARBOR_CID]],
7983
[1776063335, 302383000],
8084
[1776132745, 850748000],
8185
None,
@@ -180,7 +184,7 @@ def test_keeps_header_owner_first_when_collecting_collaborators(self) -> None:
180184
},
181185
],
182186
)
183-
self.assertEqual(parsed.places[0].cid, "7451636382641713350")
187+
self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID)
184188
self.assertEqual(
185189
parsed.places[0].maps_url,
186190
"https://www.google.com/maps/search/?api=1&query=Northwind+Cafe%2C+Example+District",
@@ -245,6 +249,35 @@ def test_builds_search_query_url_when_cid_is_missing(self) -> None:
245249
"https://www.google.com/maps/search/?api=1&query=Northwind+Cafe%2C+Example+District",
246250
)
247251

252+
def test_uses_fingerprint_not_positive_s2_cell_from_slot_6(self) -> None:
253+
runtime_state = copy.deepcopy(["noise", _LIST_NODE])
254+
255+
parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state)
256+
257+
self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID)
258+
self.assertNotEqual(parsed.places[0].cid, _NORTHWIND_CELL_ID)
259+
260+
def test_normalizes_negative_slot_6_fingerprint(self) -> None:
261+
runtime_state = copy.deepcopy(["noise", _LIST_NODE])
262+
place_metadata = runtime_state[1][8][0][1]
263+
assert isinstance(place_metadata, list)
264+
place_metadata[6] = ["3765761194353288769", "-782808945063765017"]
265+
266+
parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state)
267+
268+
self.assertEqual(parsed.places[0].cid, "17663935128645786599")
269+
270+
def test_ignores_timestamp_like_fallback_values_when_cid_is_missing(self) -> None:
271+
runtime_state = copy.deepcopy(["noise", _LIST_NODE])
272+
place_metadata = runtime_state[1][8][0][1]
273+
assert isinstance(place_metadata, list)
274+
place_metadata[6] = [None]
275+
place_metadata.append(1776063335)
276+
277+
parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state)
278+
279+
self.assertEqual(parsed.places[0].cid, None)
280+
248281
def test_builds_coordinate_query_url_only_when_no_name_or_address_exist(self) -> None:
249282
runtime_state = [
250283
"noise",
@@ -296,8 +329,8 @@ def test_dedupes_places_with_same_cid(self) -> None:
296329
parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state)
297330

298331
self.assertEqual(len(parsed.places), 2)
299-
self.assertEqual(parsed.places[0].cid, "7451636382641713350")
300-
self.assertEqual(parsed.places[1].cid, "1234567890123456789")
332+
self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID)
333+
self.assertEqual(parsed.places[1].cid, _HARBOR_CID)
301334

302335
def test_keeps_distinct_places_that_share_a_cid(self) -> None:
303336
runtime_state = copy.deepcopy(["noise", _LIST_NODE])
@@ -307,14 +340,14 @@ def test_keeps_distinct_places_that_share_a_cid(self) -> None:
307340
assert isinstance(second_metadata, list)
308341

309342
second_metadata[5] = [None, None, 35.7000000, 139.7800000]
310-
second_metadata[6] = ["7451636382641713350", "-2234567890123456789"]
343+
second_metadata[6] = ["7451636382641713350", _NORTHWIND_CID]
311344
second_metadata[7] = None
312345

313346
parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state)
314347

315348
self.assertEqual(len(parsed.places), 2)
316-
self.assertEqual(parsed.places[0].cid, "7451636382641713350")
317-
self.assertEqual(parsed.places[1].cid, "7451636382641713350")
349+
self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID)
350+
self.assertEqual(parsed.places[1].cid, _NORTHWIND_CID)
318351
self.assertEqual(parsed.places[1].lat, 35.7)
319352

320353
def test_does_not_use_owner_profile_id_as_place_cid(self) -> None:

0 commit comments

Comments
 (0)