Skip to content

Fix saved list CID parsing #49

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
michaelmwu merged 1 commit into from
Jul 7, 2026
Merged

Fix saved list CID parsing #49

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 30 additions & 6 deletions src/gmaps_scraper/parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -587,11 +587,11 @@ def _looks_like_address(value: str | None) -> bool:
def _find_cid(node: list[JSONValue] | None) -> str | None:
if node is None:
return None
structured = _find_cid_in_value(_safe_index(node, 6))
structured = _find_cid_in_structured_slot(_safe_index(node, 6))
if structured is not None:
return structured
for value in node:
candidate = _find_cid_in_value(value)
candidate = _find_cid_in_fallback_value(value)
if candidate is not None:
return candidate
return None
Expand Down Expand Up @@ -637,7 +637,7 @@ def _metadata_matches_coordinate(
)


def _find_cid_in_value(value: JSONValue | None) -> str | None:
def _find_cid_in_structured_slot(value: JSONValue | None) -> str | None:
if isinstance(value, int):
return _normalize_cid_token(str(value))
if isinstance(value, str):
Expand All @@ -655,14 +655,38 @@ def _find_cid_in_value(value: JSONValue | None) -> str | None:
]
if not numeric_texts:
return None
for text in numeric_texts:
if not text.startswith("-"):
return _normalize_cid_token(text)
if len(numeric_texts) >= 2:
return _normalize_cid_token(numeric_texts[1])
return _normalize_cid_token(numeric_texts[0])


def _find_cid_in_fallback_value(value: JSONValue | None) -> str | None:
if isinstance(value, int | str):
return _normalize_fallback_cid_token(value)
if not isinstance(value, list):
return None
owner = _parse_list_owner(value)
if owner is not None and (owner.photo_url is not None or owner.profile_id is not None):
return None
numeric_texts = [
text
for text in (_clean_text(item) for item in value)
if text is not None and _LONG_INTEGER_PATTERN.fullmatch(text) is not None
]
if len(numeric_texts) == 1:
return _normalize_fallback_cid_token(numeric_texts[0])
return None


def _normalize_fallback_cid_token(value: JSONValue | None) -> str | None:
text = str(value) if isinstance(value, int) else _clean_text(value)
if text is None or _LONG_INTEGER_PATTERN.fullmatch(text) is None:
return None
if len(text.removeprefix("-")) < 15:
return None
return _normalize_cid_token(text)


def _normalize_cid_token(value: JSONValue | None) -> str | None:
text = _clean_text(value)
if text is None or _LONG_INTEGER_PATTERN.fullmatch(text) is None:
Expand Down
53 changes: 43 additions & 10 deletions tests/test_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@
"https://www.google.com/maps/@30.5370705,125.4120472,6z/"
"data=!4m3!11m2!2sTESTLISTABC123456789!3e3?entry=ttu"
)
_NORTHWIND_CELL_ID = "7451636382641713350"
_NORTHWIND_CID = "9055794338847426964"
_HARBOR_CELL_ID = "1234567890123456789"
_HARBOR_CID = "5678901234567890123"
_LIST_NODE = [
["TESTLISTABC123456789", 1, None, 1, 1],
4,
Expand All @@ -38,7 +42,7 @@
None,
"Example District",
[None, None, 35.6501307, 139.6868459],
["7451636382641713350", "aux"],
[_NORTHWIND_CELL_ID, _NORTHWIND_CID],
"/g/11northwind",
],
"Northwind Cafe",
Expand All @@ -47,7 +51,7 @@
None,
None,
[[[[3, None, "104356373423434804635", "❤️", [1776133481, 81561000]]]]],
[[1], ["7451636382641713350", "aux"]],
[[1], [_NORTHWIND_CELL_ID, _NORTHWIND_CID]],
[1776063335, 302383000],
[1776132745, 850748000],
None,
Expand All @@ -66,7 +70,7 @@
None,
"Market Square",
[None, None, 35.6915776, 139.7836109],
["1234567890123456789"],
[_HARBOR_CELL_ID, _HARBOR_CID],
"/g/11harborbakery",
],
"Harbor Bakery",
Expand All @@ -75,7 +79,7 @@
None,
None,
[],
[[1], ["1234567890123456789", "aux-2"]],
[[1], [_HARBOR_CELL_ID, _HARBOR_CID]],
[1776063335, 302383000],
[1776132745, 850748000],
None,
Expand Down Expand Up @@ -180,7 +184,7 @@ def test_keeps_header_owner_first_when_collecting_collaborators(self) -> None:
},
],
)
self.assertEqual(parsed.places[0].cid, "7451636382641713350")
self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID)
self.assertEqual(
parsed.places[0].maps_url,
"https://www.google.com/maps/search/?api=1&query=Northwind+Cafe%2C+Example+District",
Expand Down Expand Up @@ -245,6 +249,35 @@ def test_builds_search_query_url_when_cid_is_missing(self) -> None:
"https://www.google.com/maps/search/?api=1&query=Northwind+Cafe%2C+Example+District",
)

def test_uses_fingerprint_not_positive_s2_cell_from_slot_6(self) -> None:
runtime_state = copy.deepcopy(["noise", _LIST_NODE])

parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state)

self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID)
self.assertNotEqual(parsed.places[0].cid, _NORTHWIND_CELL_ID)

def test_normalizes_negative_slot_6_fingerprint(self) -> None:
runtime_state = copy.deepcopy(["noise", _LIST_NODE])
place_metadata = runtime_state[1][8][0][1]
assert isinstance(place_metadata, list)
place_metadata[6] = ["3765761194353288769", "-782808945063765017"]

parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state)

self.assertEqual(parsed.places[0].cid, "17663935128645786599")

def test_ignores_timestamp_like_fallback_values_when_cid_is_missing(self) -> None:
runtime_state = copy.deepcopy(["noise", _LIST_NODE])
place_metadata = runtime_state[1][8][0][1]
assert isinstance(place_metadata, list)
place_metadata[6] = [None]
place_metadata.append(1776063335)

parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state)

self.assertEqual(parsed.places[0].cid, None)

Comment on lines +270 to +280

@coderabbitai coderabbitai Bot Jul 7, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
# Inspect the _find_cid dispatch function to see what value is passed to
# _find_cid_in_fallback_value, and confirm place_metadata's appended item
# is part of that scanned value.
rg -n -B5 -A30 'def _find_cid\(' src/gmaps_scraper/parser.py
rg -n '_LONG_INTEGER_PATTERN\s*=' src/gmaps_scraper/parser.py

Repository: 508-dev/gmaps-scraper

Length of output: 1425

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Inspect the helper chain around fallback CID detection and the test context.
sed -n '520,700p' src/gmaps_scraper/parser.py
printf '\n--- TEST CONTEXT ---\n'
sed -n '230,310p' tests/test_parser.py

Repository: 508-dev/gmaps-scraper

Length of output: 8821

🏁 Script executed:

#!/bin/bash
set -euo pipefail

# Show the exact fallback token normalization logic in isolation.
rg -n -A20 -B10 'def _find_cid_in_fallback_value|def _normalize_fallback_cid_token|def _find_cid\(' src/gmaps_scraper/parser.py

Repository: 508-dev/gmaps-scraper

Length of output: 3115

Assert the fallback scan, not just the final None. This still passes if _find_cid stops iterating fallback values entirely; add an assertion that the appended timestamp-like value is actually consumed by _find_cid_in_fallback_value so the regression test breaks when fallback scanning regresses.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@tests/test_parser.py` around lines 270 - 280, The test in
test_ignores_timestamp_like_fallback_values_when_cid_is_missing only checks the
final parsed cid, so it can still pass if _find_cid stops scanning fallback
values. Update this test to assert the fallback scan itself by verifying that
the appended timestamp-like value is actually handled through
_find_cid_in_fallback_value when parse_saved_list_artifacts runs. Use the
existing helpers and symbols (_find_cid, _find_cid_in_fallback_value,
parse_saved_list_artifacts, and place_metadata) so the test fails if fallback
scanning regresses.

def test_builds_coordinate_query_url_only_when_no_name_or_address_exist(self) -> None:
runtime_state = [
"noise",
Expand Down Expand Up @@ -296,8 +329,8 @@ def test_dedupes_places_with_same_cid(self) -> None:
parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state)

self.assertEqual(len(parsed.places), 2)
self.assertEqual(parsed.places[0].cid, "7451636382641713350")
self.assertEqual(parsed.places[1].cid, "1234567890123456789")
self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID)
self.assertEqual(parsed.places[1].cid, _HARBOR_CID)

def test_keeps_distinct_places_that_share_a_cid(self) -> None:
runtime_state = copy.deepcopy(["noise", _LIST_NODE])
Expand All @@ -307,14 +340,14 @@ def test_keeps_distinct_places_that_share_a_cid(self) -> None:
assert isinstance(second_metadata, list)

second_metadata[5] = [None, None, 35.7000000, 139.7800000]
second_metadata[6] = ["7451636382641713350", "-2234567890123456789"]
second_metadata[6] = ["7451636382641713350", _NORTHWIND_CID]
second_metadata[7] = None

parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state)

self.assertEqual(len(parsed.places), 2)
self.assertEqual(parsed.places[0].cid, "7451636382641713350")
self.assertEqual(parsed.places[1].cid, "7451636382641713350")
self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID)
self.assertEqual(parsed.places[1].cid, _NORTHWIND_CID)
self.assertEqual(parsed.places[1].lat, 35.7)

def test_does_not_use_owner_profile_id_as_place_cid(self) -> None:
Expand Down