From 818c50db8e7c1bd3442a09d2ca58cfb16473f83f Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Wed, 8 Jul 2026 17:02:21 -0700 Subject: [PATCH 1/4] Emit CID aliases from parser --- src/gmaps_scraper/models.py | 3 +++ src/gmaps_scraper/parser.py | 36 ++++++++++++++++++++++++++++++++++++ tests/test_parser.py | 1 + 3 files changed, 40 insertions(+) diff --git a/src/gmaps_scraper/models.py b/src/gmaps_scraper/models.py index 5688eb8..25702e9 100644 --- a/src/gmaps_scraper/models.py +++ b/src/gmaps_scraper/models.py @@ -81,6 +81,7 @@ class Place: lng: float maps_url: str cid: str | None = None + cid_aliases: list[str] = field(default_factory=list) google_id: str | None = None is_favorite: bool = False added_by: ListOwner | None = None @@ -100,6 +101,8 @@ def to_dict(self) -> dict[str, object]: del result["note"] if self.cid is not None: result["cid"] = self.cid + if self.cid_aliases: + result["cid_aliases"] = self.cid_aliases if self.google_id is not None: result["google_id"] = self.google_id if self.added_by is not None: diff --git a/src/gmaps_scraper/parser.py b/src/gmaps_scraper/parser.py index 96bf13c..2d308c7 100644 --- a/src/gmaps_scraper/parser.py +++ b/src/gmaps_scraper/parser.py @@ -405,6 +405,7 @@ def _extract_places(node: JSONValue) -> list[Place]: metadata_node = _find_place_metadata(ancestors) address = _extract_address(metadata_node) cid = _find_cid(metadata_node) + cid_aliases = _find_cid_aliases(metadata_node) google_id = _find_google_id(metadata_node) name = _find_place_name(ancestors, address=address, place_record=place_record) note = _find_place_note(place_record, name=name, address=address) @@ -422,6 +423,7 @@ def _extract_places(node: JSONValue) -> list[Place]: lng=lng, ), cid=cid, + cid_aliases=cid_aliases, google_id=google_id, is_favorite=is_favorite, added_by=_find_place_added_by(place_record), @@ -597,6 +599,19 @@ def _find_cid(node: list[JSONValue] | None) -> str | None: return None +def _find_cid_aliases(node: list[JSONValue] | None) -> list[str]: + if node is None: + return [] + structured_value = _safe_index(node, 6) + structured_cid = _find_cid_in_structured_slot(structured_value) + if structured_cid is None: + return [] + return _find_cid_aliases_in_structured_slot( + structured_value, + selected_cid=structured_cid, + ) + + def _find_google_id(node: list[JSONValue] | None) -> str | None: if node is None: return None @@ -660,6 +675,27 @@ def _find_cid_in_structured_slot(value: JSONValue | None) -> str | None: return _normalize_cid_token(numeric_texts[0]) +def _find_cid_aliases_in_structured_slot( + value: JSONValue | None, + *, + selected_cid: str, +) -> list[str]: + if not isinstance(value, list): + return [] + owner = _parse_list_owner(value) + if owner is not None and (owner.photo_url is not None or owner.profile_id is not None): + return [] + aliases: list[str] = [] + for item in value: + text = _clean_text(item) + if text is None or _LONG_INTEGER_PATTERN.fullmatch(text) is None: + continue + alias = _normalize_cid_token(text) + if alias != selected_cid and alias not in aliases: + aliases.append(alias) + return aliases + + def _find_cid_in_fallback_value(value: JSONValue | None) -> str | None: if isinstance(value, int | str): return _normalize_fallback_cid_token(value) diff --git a/tests/test_parser.py b/tests/test_parser.py index 60c6fac..06a27fe 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -255,6 +255,7 @@ def test_uses_fingerprint_not_positive_s2_cell_from_slot_6(self) -> None: parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state) self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID) + self.assertEqual(parsed.places[0].cid_aliases, [_NORTHWIND_CELL_ID]) self.assertNotEqual(parsed.places[0].cid, _NORTHWIND_CELL_ID) def test_normalizes_negative_slot_6_fingerprint(self) -> None: From df51b9eab75841d60176b6a8b8f148fa5a4e3a64 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Wed, 8 Jul 2026 17:16:41 -0700 Subject: [PATCH 2/4] Harden CID alias parsing --- src/gmaps_scraper/models.py | 2 +- src/gmaps_scraper/parser.py | 19 ++++++++++++------- tests/test_parser.py | 14 ++++++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/gmaps_scraper/models.py b/src/gmaps_scraper/models.py index 25702e9..b8b4f0b 100644 --- a/src/gmaps_scraper/models.py +++ b/src/gmaps_scraper/models.py @@ -81,8 +81,8 @@ class Place: lng: float maps_url: str cid: str | None = None - cid_aliases: list[str] = field(default_factory=list) google_id: str | None = None + cid_aliases: list[str] = field(default_factory=list) is_favorite: bool = False added_by: ListOwner | None = None diff --git a/src/gmaps_scraper/parser.py b/src/gmaps_scraper/parser.py index 2d308c7..4348036 100644 --- a/src/gmaps_scraper/parser.py +++ b/src/gmaps_scraper/parser.py @@ -428,14 +428,19 @@ def _extract_places(node: JSONValue) -> list[Place]: is_favorite=is_favorite, added_by=_find_place_added_by(place_record), ) - dedupe_key = ( - google_id - or (f"{cid}:{lat:.6f}:{lng:.6f}" if cid is not None else None) - or f"{place.name}:{lat:.6f}:{lng:.6f}" - ) - if dedupe_key in seen: + dedupe_keys: set[str] + if google_id: + dedupe_keys = {f"gid:{google_id}"} + elif cid is not None: + dedupe_keys = { + f"cid:{cid_value}:{lat:.6f}:{lng:.6f}" + for cid_value in [cid, *cid_aliases] + } + else: + dedupe_keys = {f"name:{place.name}:{lat:.6f}:{lng:.6f}"} + if seen.intersection(dedupe_keys): continue - seen.add(dedupe_key) + seen.update(dedupe_keys) places.append(place) return places diff --git a/tests/test_parser.py b/tests/test_parser.py index 06a27fe..39a258e 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -333,6 +333,20 @@ def test_dedupes_places_with_same_cid(self) -> None: self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID) self.assertEqual(parsed.places[1].cid, _HARBOR_CID) + def test_dedupes_places_with_cid_alias(self) -> None: + runtime_state = copy.deepcopy(["noise", _LIST_NODE]) + duplicate_place = copy.deepcopy(runtime_state[1][8][0]) + duplicate_metadata = duplicate_place[1] + assert isinstance(duplicate_metadata, list) + duplicate_metadata[6] = [_NORTHWIND_CELL_ID] + runtime_state[1][8].append(duplicate_place) + + parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state) + + self.assertEqual(len(parsed.places), 2) + self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID) + self.assertEqual(parsed.places[1].cid, _HARBOR_CID) + def test_keeps_distinct_places_that_share_a_cid(self) -> None: runtime_state = copy.deepcopy(["noise", _LIST_NODE]) second_place = runtime_state[1][8][1] From fc08068fda0e01a2efbafd0433c27d62941b2cbf Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Wed, 8 Jul 2026 17:22:24 -0700 Subject: [PATCH 3/4] Fix CID alias dedupe --- src/gmaps_scraper/parser.py | 14 +++++++------- tests/test_parser.py | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/gmaps_scraper/parser.py b/src/gmaps_scraper/parser.py index 4348036..de3d305 100644 --- a/src/gmaps_scraper/parser.py +++ b/src/gmaps_scraper/parser.py @@ -428,15 +428,15 @@ def _extract_places(node: JSONValue) -> list[Place]: is_favorite=is_favorite, added_by=_find_place_added_by(place_record), ) - dedupe_keys: set[str] + dedupe_keys: set[str] = set() if google_id: - dedupe_keys = {f"gid:{google_id}"} - elif cid is not None: - dedupe_keys = { + dedupe_keys.add(f"gid:{google_id}") + if cid is not None: + dedupe_keys.update({ f"cid:{cid_value}:{lat:.6f}:{lng:.6f}" for cid_value in [cid, *cid_aliases] - } - else: + }) + if not dedupe_keys: dedupe_keys = {f"name:{place.name}:{lat:.6f}:{lng:.6f}"} if seen.intersection(dedupe_keys): continue @@ -696,7 +696,7 @@ def _find_cid_aliases_in_structured_slot( if text is None or _LONG_INTEGER_PATTERN.fullmatch(text) is None: continue alias = _normalize_cid_token(text) - if alias != selected_cid and alias not in aliases: + if alias is not None and alias != selected_cid and alias not in aliases: aliases.append(alias) return aliases diff --git a/tests/test_parser.py b/tests/test_parser.py index 39a258e..5f7e06d 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -339,6 +339,7 @@ def test_dedupes_places_with_cid_alias(self) -> None: duplicate_metadata = duplicate_place[1] assert isinstance(duplicate_metadata, list) duplicate_metadata[6] = [_NORTHWIND_CELL_ID] + duplicate_metadata[7] = None runtime_state[1][8].append(duplicate_place) parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state) From eb929fdc410ac3e541d909107e65c4ce18780f2f Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Wed, 8 Jul 2026 21:43:49 -0700 Subject: [PATCH 4/4] Prefer canonical CID alias dedupe rows --- src/gmaps_scraper/parser.py | 99 ++++++++++++++++++++++++++++++++----- tests/test_parser.py | 43 ++++++++++++++++ 2 files changed, 129 insertions(+), 13 deletions(-) diff --git a/src/gmaps_scraper/parser.py b/src/gmaps_scraper/parser.py index de3d305..df09b5f 100644 --- a/src/gmaps_scraper/parser.py +++ b/src/gmaps_scraper/parser.py @@ -71,6 +71,12 @@ class _Candidate: signal_score: int +@dataclass(frozen=True, slots=True) +class _PlaceDedupeKeys: + primary: set[str] + aliases: set[str] + + class ParseError(RuntimeError): """Raised when a saved list cannot be parsed from the supplied artifacts.""" @@ -388,7 +394,8 @@ def _parse_list_owner(node: JSONValue | None) -> ListOwner | None: def _extract_places(node: JSONValue) -> list[Place]: places: list[Place] = [] - seen: set[str] = set() + primary_key_indexes: dict[str, int] = {} + alias_key_indexes: dict[str, int] = {} for current, ancestors in _walk_json(node): if not _is_coordinate_tuple(current): @@ -428,24 +435,90 @@ def _extract_places(node: JSONValue) -> list[Place]: is_favorite=is_favorite, added_by=_find_place_added_by(place_record), ) - dedupe_keys: set[str] = set() - if google_id: - dedupe_keys.add(f"gid:{google_id}") - if cid is not None: - dedupe_keys.update({ - f"cid:{cid_value}:{lat:.6f}:{lng:.6f}" - for cid_value in [cid, *cid_aliases] - }) - if not dedupe_keys: - dedupe_keys = {f"name:{place.name}:{lat:.6f}:{lng:.6f}"} - if seen.intersection(dedupe_keys): + dedupe_keys = _place_dedupe_keys(place) + duplicate_index = _duplicate_place_index( + dedupe_keys, + primary_key_indexes=primary_key_indexes, + alias_key_indexes=alias_key_indexes, + ) + if duplicate_index is not None: + if _place_dedupe_quality(place) > _place_dedupe_quality(places[duplicate_index]): + places[duplicate_index] = place + _record_place_dedupe_keys( + duplicate_index, + dedupe_keys, + primary_key_indexes=primary_key_indexes, + alias_key_indexes=alias_key_indexes, + ) continue - seen.update(dedupe_keys) places.append(place) + _record_place_dedupe_keys( + len(places) - 1, + dedupe_keys, + primary_key_indexes=primary_key_indexes, + alias_key_indexes=alias_key_indexes, + ) return places +def _place_dedupe_keys(place: Place) -> _PlaceDedupeKeys: + primary: set[str] = set() + aliases: set[str] = set() + if place.google_id: + primary.add(f"gid:{place.google_id}") + if place.cid is not None: + cid_suffix = f"{place.lat:.6f}:{place.lng:.6f}" + primary.add(f"cid:{place.cid}:{cid_suffix}") + aliases.update( + f"cid:{cid_alias}:{cid_suffix}" + for cid_alias in place.cid_aliases + ) + if not primary: + primary.add(f"name:{place.name}:{place.lat:.6f}:{place.lng:.6f}") + return _PlaceDedupeKeys(primary=primary, aliases=aliases) + + +def _duplicate_place_index( + keys: _PlaceDedupeKeys, + *, + primary_key_indexes: dict[str, int], + alias_key_indexes: dict[str, int], +) -> int | None: + for key in keys.primary: + if key in primary_key_indexes: + return primary_key_indexes[key] + for key in keys.aliases: + if key in primary_key_indexes: + return primary_key_indexes[key] + for key in keys.primary: + if key in alias_key_indexes: + return alias_key_indexes[key] + return None + + +def _record_place_dedupe_keys( + index: int, + keys: _PlaceDedupeKeys, + *, + primary_key_indexes: dict[str, int], + alias_key_indexes: dict[str, int], +) -> None: + for key in keys.primary: + primary_key_indexes[key] = index + for key in keys.aliases: + alias_key_indexes[key] = index + + +def _place_dedupe_quality(place: Place) -> tuple[bool, bool, bool, bool]: + return ( + place.google_id is not None, + bool(place.cid_aliases), + place.cid is not None, + place.address is not None, + ) + + def _find_place_metadata(ancestors: Sequence[JSONValue]) -> list[JSONValue] | None: place_record = _find_place_record(ancestors) metadata_node = _place_metadata_from_record(place_record) diff --git a/tests/test_parser.py b/tests/test_parser.py index 5f7e06d..da4885b 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -335,6 +335,9 @@ def test_dedupes_places_with_same_cid(self) -> None: def test_dedupes_places_with_cid_alias(self) -> None: runtime_state = copy.deepcopy(["noise", _LIST_NODE]) + original_metadata = runtime_state[1][8][0][1] + assert isinstance(original_metadata, list) + original_metadata[7] = None duplicate_place = copy.deepcopy(runtime_state[1][8][0]) duplicate_metadata = duplicate_place[1] assert isinstance(duplicate_metadata, list) @@ -348,6 +351,46 @@ def test_dedupes_places_with_cid_alias(self) -> None: self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID) self.assertEqual(parsed.places[1].cid, _HARBOR_CID) + def test_prefers_canonical_place_when_cid_alias_duplicate_appears_first(self) -> None: + runtime_state = copy.deepcopy(["noise", _LIST_NODE]) + duplicate_place = copy.deepcopy(runtime_state[1][8][0]) + duplicate_metadata = duplicate_place[1] + assert isinstance(duplicate_metadata, list) + duplicate_metadata[6] = [_NORTHWIND_CELL_ID] + duplicate_metadata[7] = None + runtime_state[1][8].insert(0, duplicate_place) + + parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state) + + self.assertEqual(len(parsed.places), 2) + self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID) + self.assertEqual(parsed.places[0].cid_aliases, [_NORTHWIND_CELL_ID]) + self.assertEqual(parsed.places[0].google_id, "/g/11northwind") + self.assertEqual(parsed.places[1].cid, _HARBOR_CID) + + def test_keeps_distinct_places_with_same_cid_alias_and_coordinates(self) -> None: + runtime_state = copy.deepcopy(["noise", _LIST_NODE]) + first_place = runtime_state[1][8][0] + second_place = runtime_state[1][8][1] + assert isinstance(first_place, list) + assert isinstance(second_place, list) + first_metadata = first_place[1] + second_metadata = second_place[1] + assert isinstance(first_metadata, list) + assert isinstance(second_metadata, list) + first_metadata[7] = None + second_metadata[5] = [None, None, 35.6501307, 139.6868459] + second_metadata[6] = [_NORTHWIND_CELL_ID, _HARBOR_CID] + second_metadata[7] = None + + parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state) + + self.assertEqual(len(parsed.places), 2) + self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID) + self.assertEqual(parsed.places[1].cid, _HARBOR_CID) + self.assertEqual(parsed.places[0].cid_aliases, [_NORTHWIND_CELL_ID]) + self.assertEqual(parsed.places[1].cid_aliases, [_NORTHWIND_CELL_ID]) + def test_keeps_distinct_places_that_share_a_cid(self) -> None: runtime_state = copy.deepcopy(["noise", _LIST_NODE]) second_place = runtime_state[1][8][1]