diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 642b706..45eb2c4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,7 +32,8 @@ Place entries inside saved lists are detected structurally. The coordinate tuple pattern `[null, null, lat, lng]` is the strongest signal for a saved place record. The parser then uses surrounding parent structures to recover name, address, note, favorite status, Google identifiers, ownership metadata, and a -Maps search URL. +direct CID URL for addressless records when available, or a Maps search URL +otherwise. Saved-list output preserves: @@ -86,6 +87,11 @@ These payloads can contain useful values when the rendered DOM is thin, blocked, or only partially available. Because their shape is undocumented, extraction from preview payloads should favor strongly typed signals: coordinates, plus codes, phone-looking fields, postal addresses, and compact address arrays. +When the final canonical `/maps/place/` URL encodes explicit `!3d`/`!4d` pin +coordinates, those coordinates take precedence: preview payloads can include +unrelated nearby entities or booking offers alongside the selected place. An +`@lat,lng` viewport center remains a fallback only when structured coordinates +are unavailable. ## Legacy Payloads diff --git a/src/gmaps_scraper/cli.py b/src/gmaps_scraper/cli.py index 7ede33c..9484a60 100644 --- a/src/gmaps_scraper/cli.py +++ b/src/gmaps_scraper/cli.py @@ -32,6 +32,7 @@ _merge_llm_place_fields, _merge_place_sources, _place_detail_values, + _prefer_resolved_place_url_coordinates, _repair_source_used_llm, _should_use_llm_repair, collect_place_snapshot, @@ -654,6 +655,10 @@ def _scrape_place_for_debug( snapshot["preview"] if isinstance(snapshot.get("preview"), dict) else {}, ) merged_snapshot = _merge_place_sources(dom_snapshot, preview_snapshot) + merged_snapshot = _prefer_resolved_place_url_coordinates( + merged_snapshot, + resolved_url=resolved_url, + ) details = _build_place_details(place_url, resolved_url=resolved_url, snapshot=merged_snapshot) evidence = _build_place_llm_evidence(merged_snapshot) evidence_hash = _hash_evidence(evidence) diff --git a/src/gmaps_scraper/parser.py b/src/gmaps_scraper/parser.py index df09b5f..7eb59cc 100644 --- a/src/gmaps_scraper/parser.py +++ b/src/gmaps_scraper/parser.py @@ -428,6 +428,7 @@ def _extract_places(node: JSONValue) -> list[Place]: address=address, lat=lat, lng=lng, + cid=cid, ), cid=cid, cid_aliases=cid_aliases, @@ -833,7 +834,10 @@ def _build_maps_url( address: str | None, lat: float, lng: float, + cid: str | None, ) -> str: + if address is None and cid is not None: + return f"https://www.google.com/maps?cid={cid}" query = _build_maps_query(name=name, address=address, lat=lat, lng=lng) return f"https://www.google.com/maps/search/?{urlencode({'api': '1', 'query': query})}" diff --git a/src/gmaps_scraper/place_scraper.py b/src/gmaps_scraper/place_scraper.py index 6b34789..393b474 100644 --- a/src/gmaps_scraper/place_scraper.py +++ b/src/gmaps_scraper/place_scraper.py @@ -49,6 +49,9 @@ _TITLE_SELECTORS = ("h1.DUwDvf", "h1.lfPIob", "div[role='main'] h1") _TITLE_SELECTOR = ", ".join(_TITLE_SELECTORS) +_CANONICAL_COORDINATE_PATTERN = re.compile( + r"!3d(-?\d+(?:\.\d+)?)!4d(-?\d+(?:\.\d+)?)" +) _PLACE_LLM_PROMPT_VERSION = "gmaps-place-repair-v1" _TRANSLATION_MEMORY = TranslationMemory.default() type PlaceLLMRepairer = Callable[[PlaceLLMRepairRequest], Mapping[str, object] | None] @@ -1292,6 +1295,10 @@ def _build_place_details_from_snapshot( (preview_snapshot, "preview"), (search_result_snapshot, "search_result"), ) + merged_snapshot = _prefer_resolved_place_url_coordinates( + merged_snapshot, + resolved_url=resolved_url, + ) details = _build_place_details( place_url, resolved_url=resolved_url, @@ -1932,6 +1939,27 @@ def _looks_like_google_maps_place_url(value: str) -> bool: return parsed.path.startswith("/maps/place/") +def _prefer_resolved_place_url_coordinates( + snapshot: Mapping[str, object], + *, + resolved_url: str | None, +) -> dict[str, object]: + coordinates = _extract_canonical_place_coordinates(resolved_url) + if coordinates is None: + return dict(snapshot) + + merged = dict(snapshot) + raw_field_sources = snapshot.get("field_sources") + field_sources = ( + dict(raw_field_sources) if isinstance(raw_field_sources, Mapping) else {} + ) + merged["lat"], merged["lng"] = coordinates + field_sources["lat"] = "resolved_url" + field_sources["lng"] = "resolved_url" + merged["field_sources"] = field_sources + return merged + + def _collect_review_panel_snapshot(page: Any, *, timeout_ms: int) -> dict[str, object]: try: clicked = page.evaluate(_PLACE_REVIEW_TAB_CLICK_JS) @@ -2021,12 +2049,16 @@ def _build_place_details( category_display_en, category_display_en_source, category_display_en_confidence = ( _derive_category_display_en(category, snapshot) ) - lat = _parse_float(snapshot.get("lat")) - if lat is None: - lat = _extract_coordinate_from_url(resolved_url or source_url, index=0) - lng = _parse_float(snapshot.get("lng")) - if lng is None: - lng = _extract_coordinate_from_url(resolved_url or source_url, index=1) + canonical_coordinates = _extract_canonical_place_coordinates(resolved_url) + if canonical_coordinates is not None: + lat, lng = canonical_coordinates + else: + lat = _parse_float(snapshot.get("lat")) + if lat is None: + lat = _extract_coordinate_from_url(resolved_url or source_url, index=0) + lng = _parse_float(snapshot.get("lng")) + if lng is None: + lng = _extract_coordinate_from_url(resolved_url or source_url, index=1) address = _clean_address_text(snapshot.get("address")) or _extract_address_from_lines( combined_lines ) @@ -4255,14 +4287,42 @@ def _reservation_provider_label_from_url(url: str) -> str: return base.replace("-", " ").replace("_", " ").title() -def _extract_coordinate_from_url(url: str, *, index: int) -> float | None: - match = re.search(r"@(-?\d+(?:\.\d+)?),(-?\d+(?:\.\d+)?)", url) +def _validated_coordinate_match( + match: re.Match[str] | None, +) -> tuple[float, float] | None: if match is None: return None try: - return float(match.group(index + 1)) + lat = float(match.group(1)) + lng = float(match.group(2)) except ValueError: return None + if not _valid_coordinates(lat, lng): + return None + return lat, lng + + +def _extract_coordinate_from_url(url: str, *, index: int) -> float | None: + canonical_match = _CANONICAL_COORDINATE_PATTERN.search(url) + if canonical_match is not None: + coordinates = _validated_coordinate_match(canonical_match) + else: + coordinates = _validated_coordinate_match( + re.search(r"@(-?\d+(?:\.\d+)?),(-?\d+(?:\.\d+)?)", url) + ) + if coordinates is None: + return None + return coordinates[index] + + +def _extract_canonical_place_coordinates( + resolved_url: str | None, +) -> tuple[float, float] | None: + if resolved_url is None or not _looks_like_google_maps_place_url(resolved_url): + return None + return _validated_coordinate_match( + _CANONICAL_COORDINATE_PATTERN.search(resolved_url) + ) def _to_bool(value: object) -> bool: diff --git a/tests/test_parser.py b/tests/test_parser.py index da4885b..a9ac521 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -249,6 +249,37 @@ def test_builds_search_query_url_when_cid_is_missing(self) -> None: "https://www.google.com/maps/search/?api=1&query=Northwind+Cafe%2C+Example+District", ) + def test_builds_direct_cid_url_when_address_is_missing(self) -> None: + runtime_state = copy.deepcopy(["noise", _LIST_NODE]) + place_metadata = runtime_state[1][8][0][1] + assert isinstance(place_metadata, list) + place_metadata[4] = None + + parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state) + + self.assertIsNone(parsed.places[0].address) + self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID) + self.assertEqual( + parsed.places[0].maps_url, + f"https://www.google.com/maps?cid={_NORTHWIND_CID}", + ) + + def test_builds_name_query_url_when_address_and_cid_are_missing(self) -> None: + runtime_state = copy.deepcopy(["noise", _LIST_NODE]) + place_metadata = runtime_state[1][8][0][1] + assert isinstance(place_metadata, list) + place_metadata[4] = None + place_metadata[6] = [None] + + parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state) + + self.assertIsNone(parsed.places[0].address) + self.assertIsNone(parsed.places[0].cid) + self.assertEqual( + parsed.places[0].maps_url, + "https://www.google.com/maps/search/?api=1&query=Northwind+Cafe", + ) + def test_uses_fingerprint_not_positive_s2_cell_from_slot_6(self) -> None: runtime_state = copy.deepcopy(["noise", _LIST_NODE]) diff --git a/tests/test_place_scraper.py b/tests/test_place_scraper.py index 76a7b82..c34274a 100644 --- a/tests/test_place_scraper.py +++ b/tests/test_place_scraper.py @@ -1184,6 +1184,129 @@ def test_build_place_details_prefers_preview_over_search_card_fallback(self) -> assert details.diagnostics is not None self.assertEqual(details.diagnostics.field_sources.get("category"), "preview") + def test_build_place_details_prefers_canonical_resolved_url_coordinates(self) -> None: + details = _build_place_details_from_snapshot( + "https://www.google.com/maps?cid=4015507283657238879", + snapshot={ + "resolved_url": ( + "https://www.google.com/maps/place/La+Paix/" + "@50.8500000,4.3600000,17z/" + "data=!4m6!3m5!1s0x0:0x0!8m2!3d50.8511538!4d4.3650878" + ), + "dom": { + "name": "La Paix", + "address": "Rue Royale 103, 1000 Bruxelles, Belgium", + }, + "preview": { + "lat": 50.901778, + "lng": 4.31745, + }, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertEqual(details.lat, 50.8511538) + self.assertEqual(details.lng, 4.3650878) + assert details.diagnostics is not None + self.assertEqual(details.diagnostics.field_sources.get("lat"), "resolved_url") + self.assertEqual(details.diagnostics.field_sources.get("lng"), "resolved_url") + + def test_build_place_details_keeps_preview_coordinates_without_url_coordinates( + self, + ) -> None: + details = _build_place_details_from_snapshot( + "https://www.google.com/maps?cid=4015507283657238879", + snapshot={ + "resolved_url": "https://www.google.com/maps/place/La+Paix", + "dom": {"name": "La Paix"}, + "preview": { + "lat": 50.8511538, + "lng": 4.3650878, + }, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertEqual(details.lat, 50.8511538) + self.assertEqual(details.lng, 4.3650878) + assert details.diagnostics is not None + self.assertEqual(details.diagnostics.field_sources.get("lat"), "preview") + self.assertEqual(details.diagnostics.field_sources.get("lng"), "preview") + + def test_build_place_details_keeps_preview_coordinates_with_viewport_only_url( + self, + ) -> None: + details = _build_place_details_from_snapshot( + "https://www.google.com/maps?cid=4015507283657238879", + snapshot={ + "resolved_url": ( + "https://www.google.com/maps/place/La+Paix/" + "@50.901778,4.31745,17z" + ), + "dom": {"name": "La Paix"}, + "preview": { + "lat": 50.8511538, + "lng": 4.3650878, + }, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertEqual(details.lat, 50.8511538) + self.assertEqual(details.lng, 4.3650878) + assert details.diagnostics is not None + self.assertEqual(details.diagnostics.field_sources.get("lat"), "preview") + self.assertEqual(details.diagnostics.field_sources.get("lng"), "preview") + + def test_build_place_details_rejects_out_of_range_url_coordinates( + self, + ) -> None: + for resolved_url in ( + ( + "https://www.google.com/maps/place/La+Paix/" + "@50.8500000,4.3600000,17z/" + "data=!4m6!3m5!1s0x0:0x0!8m2!3d950.0!4d4.3650878" + ), + "https://www.google.com/maps/place/La+Paix/@950.0,4.3600000,17z", + ): + with self.subTest(resolved_url=resolved_url): + details = _build_place_details_from_snapshot( + "https://www.google.com/maps?cid=4015507283657238879", + snapshot={ + "resolved_url": resolved_url, + "dom": {"name": "La Paix"}, + "preview": {}, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertIsNone(details.lat) + self.assertIsNone(details.lng) + + def test_build_place_details_keeps_valid_viewport_coordinate_fallback( + self, + ) -> None: + details = _build_place_details_from_snapshot( + "https://www.google.com/maps?cid=4015507283657238879", + snapshot={ + "resolved_url": ( + "https://www.google.com/maps/place/La+Paix/" + "@50.8500000,4.3600000,17z" + ), + "dom": {"name": "La Paix"}, + "preview": {}, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertEqual(details.lat, 50.85) + self.assertEqual(details.lng, 4.36) + def test_build_place_details_backfills_preview_rating_summary_sources(self) -> None: preview = _extract_preview_place_enrichment( (_PREVIEW_FIXTURE_DIR / "rating_review_count.txt").read_text(encoding="utf-8")