Skip to content

Commit 86bc7c8

Browse files
michaelmwumichaelmwu
authored
Harden place extractor fixtures and preview enrichment
Adds browser-backed extractor fixtures, moves the inline place extractor JS into package data, and backfills rating/review count from conservative preview payload signals.
1 parent 71beb48 commit 86bc7c8

10 files changed

Lines changed: 1432 additions & 786 deletions

File tree

‎src/gmaps_scraper/data/place_extractor.js‎

Lines changed: 896 additions & 0 deletions
Large diffs are not rendered by default.

‎src/gmaps_scraper/place_scraper.py‎

Lines changed: 80 additions & 756 deletions
Large diffs are not rendered by default.

‎tests/fixtures/place_pages/about.html‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
<!doctype html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="utf-8">
5+
<title>Fixture About</title>
6+
<style>
7+
body { margin: 0; font-family: Arial, sans-serif; }
8+
[role="main"] { box-sizing: border-box; width: 720px; min-height: 980px; padding: 24px; }
9+
h1 { margin: 0 0 12px; font-size: 32px; }
10+
.F7nice, [data-item-id], section { margin-top: 12px; }
11+
[data-item-id] { display: block; min-height: 32px; }
12+
</style>
13+
</head>
14+
<body>
15+
<main role="main" aria-label="Fixture About Bakery">
16+
<h1 class="DUwDvf">Fixture About Bakery</h1>
17+
<div class="F7nice">
18+
<span><span aria-hidden="true">4.3</span></span>
19+
<span aria-label="678 reviews">(678)</span>
20+
</div>
21+
<button jsaction="category.click">Bakery</button>
22+
<button data-item-id="address"><span class="Io6YTe">9 About Avenue, Fixture City</span></button>
23+
<a data-item-id="authority" href="https://about.fixture.example/">Website</a>
24+
<button data-item-id="phone:tel:+15550102"><span class="Io6YTe">+1 555-0102</span></button>
25+
<div role="tablist">
26+
<button role="tab">Overview</button>
27+
<button role="tab">Reviews</button>
28+
<button role="tab">About</button>
29+
</div>
30+
<div aria-label="About Fixture About Bakery">
31+
<section class="iP2t7d">
32+
<h2>Service options</h2>
33+
<ul>
34+
<li><span aria-label="Offers takeaway">Takeaway</span></li>
35+
<li><span aria-label="Offers delivery">Delivery</span></li>
36+
</ul>
37+
</section>
38+
</div>
39+
</main>
40+
</body>
41+
</html>

‎tests/fixtures/place_pages/limited_view.html‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
<!doctype html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="utf-8">
5+
<title>Limited View Fixture</title>
6+
<style>
7+
body { margin: 0; font-family: Arial, sans-serif; }
8+
[role="main"] { box-sizing: border-box; width: 720px; min-height: 520px; padding: 24px; }
9+
h1 { margin: 0 0 12px; font-size: 32px; }
10+
</style>
11+
</head>
12+
<body>
13+
<main role="main" aria-label="Limited Fixture Place">
14+
<h1 class="DUwDvf">Limited Fixture Place</h1>
15+
<p>This is a limited view of Google Maps.</p>
16+
</main>
17+
</body>
18+
</html>

‎tests/fixtures/place_pages/overview.html‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
<!doctype html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="utf-8">
5+
<meta property="og:image" content="https://example.test/fallback-og.jpg">
6+
<title>Fixture Place</title>
7+
<style>
8+
body { margin: 0; font-family: Arial, sans-serif; }
9+
[role="main"] { box-sizing: border-box; width: 720px; min-height: 900px; padding: 24px; }
10+
h1 { margin: 0 0 12px; font-size: 32px; }
11+
.F7nice, .WeS02d, [data-item-id], .tabs { margin-top: 12px; }
12+
[data-item-id] { display: block; min-height: 32px; }
13+
button, a { font: inherit; }
14+
</style>
15+
</head>
16+
<body>
17+
<main role="main" aria-label="Fixture Cafe">
18+
<div class="RZ66Rb">
19+
<button jsaction="heroHeaderImage.open">
20+
<img src="https://lh3.googleusercontent.com/place-overview.jpg" alt="Fixture Cafe">
21+
</button>
22+
</div>
23+
<h1 class="DUwDvf">Fixture Cafe</h1>
24+
<div class="F7nice" aria-label="4.6 stars, 1,234 reviews">
25+
<span><span aria-hidden="true">4.6</span></span>
26+
<span aria-label="1,234 reviews">(1,234)</span>
27+
</div>
28+
<button jsaction="category.click">Coffee shop</button>
29+
<div class="WeS02d">Compact fixture description for the place overview.</div>
30+
<button data-item-id="address"><span class="Io6YTe">123 Test Street, Fixture City</span></button>
31+
<a data-item-id="authority" href="https://fixture.example/">Website</a>
32+
<button data-item-id="phone:tel:+15550100"><span class="Io6YTe">+1 555-0100</span></button>
33+
<button data-item-id="oloc"><span class="Io6YTe">7FG8+22 Fixture City</span></button>
34+
<div class="tabs" role="tablist">
35+
<button role="tab">Overview</button>
36+
<button role="tab">Reviews</button>
37+
<button role="tab">About</button>
38+
</div>
39+
</main>
40+
</body>
41+
</html>

‎tests/fixtures/place_pages/reviews.html‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
<!doctype html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="utf-8">
5+
<title>Fixture Reviews</title>
6+
<style>
7+
body { margin: 0; font-family: Arial, sans-serif; }
8+
[role="main"] { box-sizing: border-box; width: 720px; min-height: 980px; padding: 24px; }
9+
h1 { margin: 0 0 12px; font-size: 32px; }
10+
.F7nice, [data-item-id], [data-review-id], .topic-row { margin-top: 12px; }
11+
[data-item-id] { display: block; min-height: 32px; }
12+
[data-review-id] { border-top: 1px solid #ddd; padding-top: 12px; }
13+
</style>
14+
</head>
15+
<body>
16+
<main role="main" aria-label="Fixture Reviews Cafe">
17+
<div class="RZ66Rb">
18+
<button jsaction="heroHeaderImage.open">
19+
<img src="https://lh3.googleusercontent.com/place-review-hero.jpg" alt="">
20+
</button>
21+
</div>
22+
<h1 class="DUwDvf">Fixture Reviews Cafe</h1>
23+
<div class="F7nice">
24+
<span><span aria-hidden="true">4.8</span></span>
25+
<span aria-label="2,345 reviews">(2,345)</span>
26+
</div>
27+
<button jsaction="category.click">Restaurant</button>
28+
<button data-item-id="address"><span class="Io6YTe">55 Review Road, Fixture City</span></button>
29+
<a data-item-id="authority" href="https://reviews.fixture.example/">Website</a>
30+
<button data-item-id="phone:tel:+15550101"><span class="Io6YTe">+1 555-0101</span></button>
31+
<div role="tablist">
32+
<button role="tab">Overview</button>
33+
<button role="tab">Reviews</button>
34+
<button role="tab">About</button>
35+
</div>
36+
<div class="topic-row">
37+
<button role="radio" aria-label="coffee, mentioned in 19 reviews">coffee 19</button>
38+
<button role="radio" aria-label="service, mentioned in 12 reviews">service 12</button>
39+
</div>
40+
<article data-review-id="review-placeholder-1">
41+
<button jsaction="image.open">
42+
<img src="https://lh3.googleusercontent.com/reviewer-placeholder.jpg" alt="Reviewer avatar">
43+
</button>
44+
<div class="d4r55">Reviewer Placeholder</div>
45+
<span role="img" aria-label="5 stars"></span>
46+
<span class="rsqaWe">2 months ago</span>
47+
<div class="MyEned"><span class="wiI7pd">Useful visible review snippet.</span></div>
48+
</article>
49+
</main>
50+
</body>
51+
</html>

‎tests/fixtures/place_pages/search_result.html‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
<!doctype html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="utf-8">
5+
<title>Search Result Fixture</title>
6+
<style>
7+
body { margin: 0; font-family: Arial, sans-serif; }
8+
[role="main"] { box-sizing: border-box; width: 840px; min-height: 700px; padding: 24px; }
9+
[role="article"] { width: 420px; min-height: 160px; margin-top: 16px; padding: 12px; }
10+
a { display: block; font-size: 20px; }
11+
</style>
12+
</head>
13+
<body>
14+
<main role="main" aria-label="Results for Fixture Search">
15+
<h1 class="DUwDvf">Results</h1>
16+
<div role="feed">
17+
<div role="article">
18+
<a class="hfpxzc" aria-label="Fixture Search Cafe"
19+
href="https://www.google.com/maps/place/Fixture+Search+Cafe/data=!4m2!3m1!1sfixture-place-id">
20+
Fixture Search Cafe
21+
</a>
22+
<div class="W4Efsd">
23+
<span class="AJB7ye" role="img" aria-label="4.7 stars 321 reviews"></span>
24+
4.7 (321)
25+
</div>
26+
<div class="W4Efsd">Cafe · 10 Search Street</div>
27+
<div class="W4Efsd">Small local search-result description.</div>
28+
</div>
29+
</div>
30+
</main>
31+
</body>
32+
</html>

‎tests/fixtures/preview_payloads/ambiguous_rating_review_count.txt‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
)]}'
2+
[
3+
[
4+
"price-summary",
5+
4.8,
6+
101,
7+
"NT$101"
8+
],
9+
[
10+
"history-summary",
11+
4.6,
12+
2024,
13+
"Founded 2024"
14+
],
15+
[
16+
"non-adjacent-summary",
17+
4.7,
18+
null,
19+
null,
20+
912
21+
],
22+
[
23+
"conflicting-summary-a",
24+
4.5,
25+
120
26+
],
27+
[
28+
"conflicting-summary-b",
29+
4.6,
30+
130
31+
]
32+
]

‎tests/fixtures/preview_payloads/rating_review_count.txt‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
)]}'
2+
[
3+
[
4+
"preview-place",
5+
[
6+
"rating-summary",
7+
4.7,
8+
832
9+
],
10+
[
11+
null,
12+
null,
13+
25.2048,
14+
55.2708
15+
]
16+
]
17+
]

0 commit comments

Comments
 (0)