Skip to content

Commit e6222a9

Browse files
author
tim
committed
Address PR feedback: simplify Hessen coordinate extraction
- Remove redundant URL validation (already in XPath selector) - Add type annotation to _extract_coords_from_osm_url() - Remove mlat/mlon and bbox fallbacks (never used) - Remove link fallback (iframe always present with coordinates) - Add example URL for placeholder coordinates (-1.0, -1.0) - Replace broad Exception with specific ValueError/IndexError - Update README with geolocation statistics (1,863/2,054 schools) Analysis confirmed Hessen iframes always use 'marker' parameter format. The mlat/mlon format only appears in links, which are not needed.
1 parent 57a7553 commit e6222a9

2 files changed

Lines changed: 7 additions & 32 deletions

File tree

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ When available, we try to use the geolocations provided by the data publishers.
4949
| BB | ✅ Yes | WFS |
5050
| HB | ❌ No | - |
5151
| HH | ✅ Yes | WFS |
52-
| HE | ⚠️ Partial (90.7%) | Extracted from OSM iframes on detail pages |
52+
| HE | ⚠️ Partial (90.7%) | Extracted from OSM on detail pages (1,863/2,054 schools). The 191 schools without coordinates include both schools with placeholder coordinates (-1.0, -1.0) that are filtered to null and schools with no map data at all. |
5353
| MV | ❌ No | - |
5454
| NI | ❌ No | - |
5555
| NW | ✅ Yes | Converted from EPSG:25832 in source CSV data |

jedeschule/spiders/hessen.py

Lines changed: 6 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -53,34 +53,16 @@ def parse_list(self, response):
5353
for school in schools:
5454
yield scrapy.Request(school, callback=self.parse_details)
5555

56-
def _extract_coords_from_osm_url(self, url):
57-
"""Extract coordinates from OpenStreetMap URL query parameters"""
58-
if not url or "openstreetmap.org" not in url:
59-
return None, None
60-
56+
def _extract_coords_from_osm_url(self, url: str) -> tuple[float | None, float | None]:
57+
"""Extract coordinates from OpenStreetMap iframe URL marker parameter"""
6158
qs = parse_qs(urlparse(url).query)
6259

63-
# Try marker parameter first (most precise)
60+
# Extract marker parameter (format: "latitude,longitude")
6461
if "marker" in qs and qs["marker"]:
6562
try:
6663
lat_str, lon_str = qs["marker"][0].split(",", 1)
6764
return float(lat_str), float(lon_str)
68-
except Exception:
69-
pass
70-
71-
# Try mlat/mlon parameters
72-
if "mlat" in qs and "mlon" in qs:
73-
try:
74-
return float(qs["mlat"][0]), float(qs["mlon"][0])
75-
except Exception:
76-
pass
77-
78-
# Fallback: bbox center
79-
if "bbox" in qs and qs["bbox"]:
80-
try:
81-
west, south, east, north = map(float, qs["bbox"][0].split(",", 3))
82-
return (south + north) / 2.0, (west + east) / 2.0
83-
except Exception:
65+
except (ValueError, IndexError):
8466
pass
8567

8668
return None, None
@@ -125,21 +107,14 @@ def parse_details(self, response):
125107
# Extract school ID from URL query parameter
126108
school["id"] = response.request.url.split("=")[-1]
127109

128-
# Extract coordinates from OpenStreetMap iframe or link
110+
# Extract coordinates from OpenStreetMap iframe
129111
latitude, longitude = None, None
130-
131-
# Try iframe first
132112
iframe_src = response.xpath('//iframe[contains(@src, "openstreetmap.org")]/@src').get()
133113
if iframe_src:
134114
latitude, longitude = self._extract_coords_from_osm_url(iframe_src)
135115

136-
# Fallback: try "Größere Karte" link
137-
if latitude is None:
138-
osm_link = response.xpath('//a[contains(@href, "openstreetmap.org")]/@href').get()
139-
if osm_link:
140-
latitude, longitude = self._extract_coords_from_osm_url(osm_link)
141-
142116
# Filter out placeholder coordinates (-1.0, -1.0) used by Hessen DB for missing data
117+
# Example: https://schul-db.bildung.hessen.de/schul_db.html/details/?school_no=9642
143118
if latitude == -1.0 and longitude == -1.0:
144119
latitude = None
145120
longitude = None

0 commit comments

Comments
 (0)