Skip to content

Commit fc1aec5

Browse files
njhenrynjhenry
committed
Add exploratory script for downloading elements that existed at any time over a date range.
1 parent e5dd3ff commit fc1aec5

3 files changed

Lines changed: 157 additions & 0 deletions

File tree

‎exploratory/osm_api_query_examples.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Node that was deleted
2+
https://api.openstreetmap.org/api/0.6/node/7807736297/history
3+
4+
# Node with changed metadata
5+
https://api.openstreetmap.org/api/0.6/node/441510760/history

‎exploratory/osm_download.py‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
import overpass
2+
import xml.etree.ElementTree as ET
3+
import requests
4+
import datetime
5+
import time
6+
7+
TIMEOUT = 1000
8+
BBOX = {'ymin': 47.41, 'xmin': -122.48, 'ymax': 47.79, 'xmax': -122.16}
9+
START_DATE = datetime.datetime(2016, 1, 1) # Earliest option is September 13, 2012
10+
END_DATE = datetime.datetime(2026, 1, 1) # Latest
11+
DATE_INTERVAL = datetime.timedelta(days=7)
12+
OSM_KEYS = ['amenity', 'shop', 'healthcare', 'leisure']
13+
14+
# Create date range
15+
date_range = [
16+
START_DATE + i * DATE_INTERVAL
17+
for i in range(((END_DATE - START_DATE) // DATE_INTERVAL) + 1)
18+
]
19+
if date_range[-1] != END_DATE:
20+
date_range.append(END_DATE)
21+
22+
23+
def build_query_string(
24+
date: datetime.datetime,
25+
bbox: dict,
26+
keys: list,
27+
timeout: int
28+
) -> str:
29+
"""
30+
Builds a query string for the given date, bbox, kvs, and timeout.
31+
32+
Args:
33+
date: The date to query for.
34+
bbox: The bounding box to query for.
35+
amenities: The amenities to query for.
36+
timeout: The timeout for the query.
37+
Returns:
38+
A query string.
39+
"""
40+
query_string = f"""
41+
[out:xml][timeout:{timeout}]
42+
[date:"{date.strftime("%Y-%m-%dT00:00:00Z")}"];
43+
(
44+
"""
45+
def add_group(key: str) -> str:
46+
prefix = f"nwr({bbox['ymin']}, {bbox['xmin']}, {bbox['ymax']}, {bbox['xmax']})"
47+
return f"{prefix}[{key}];\n"
48+
for key in keys:
49+
query_string += add_group(key)
50+
query_string += """
51+
);
52+
out ids;
53+
"""
54+
return query_string
55+
56+
consider_ids = {
57+
'node': set(),
58+
'way': set(),
59+
'relation': set(),
60+
}
61+
62+
# Build query string
63+
64+
api = overpass.API(timeout = TIMEOUT)
65+
failed_dates = []
66+
67+
for this_date in date_range:
68+
try:
69+
start_time = time.time()
70+
for key in OSM_KEYS:
71+
# Query all matching elements from this date
72+
query_string = build_query_string(
73+
date = this_date,
74+
bbox = BBOX,
75+
keys = [key],
76+
timeout = TIMEOUT
77+
)
78+
result_xml = api.get(query = query_string, build = False)
79+
# Get all IDs for matching elements; add them to the consider_ids sets
80+
result_etree = ET.fromstring(result_xml)
81+
for e_type in consider_ids:
82+
elements = result_etree.findall(f'.//{e_type}')
83+
for element in elements:
84+
consider_ids[e_type].add(element.get('id'))
85+
print(
86+
f"Successfully queried date {this_date} in {time.time() - start_time} seconds"
87+
)
88+
except Exception as e:
89+
failed_dates.append(this_date)
90+
print(f"Failed to query date {this_date}; adding to failed_dates")
91+
time.sleep(10)
92+
93+
94+
# # Get the changeset for each element
95+
# element_id = consider_ids.pop()
96+
# history_url = f"https://api.openstreetmap.org/api/0.6/node/{element_id}/history"
97+
# history_response = requests.get(history_url)
98+
# history_etree = ET.fromstring(history_response.text)

‎exploratory/overpass_query_examples.txt‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/* History of a node */
2+
timeline(node,7807736297);
3+
for (t["created"]) {
4+
retro (_.val) {
5+
node(7807736297);
6+
out meta;
7+
}
8+
}
9+
10+
/* Timeline of a node */
11+
timeline(node,7807736297);
12+
out meta;
13+
14+
/* Metadata for a node */
15+
node(441510760);
16+
out meta;
17+
18+
/* List of nodes added or deleted during a time range */
19+
[out:xml][timeout:200]
20+
[adiff:"2025-01-07T00:00:00Z","2025-01-10T00:00:00Z"];
21+
nwr(47.41, -122.48, 47.79, -122.16);
22+
compare(delta:1);
23+
out ids;
24+
25+
[out:xml][timeout:200]
26+
[diff:"2025-01-01T00:00:00Z","2025-01-31T00:00:00Z"];
27+
nwr(47.41, -122.48, 47.79, -122.16)["amenity"="restaurant"];
28+
compare(delta:1);
29+
out meta;
30+
31+
/* Full query for a variety of amenities */
32+
[out:xml][timeout:1000]
33+
[date:"2025-01-01T00:00:00Z"];
34+
(
35+
nwr(47.41, -122.48, 47.79, -122.16)["shop"="bicycle"];
36+
nwr(47.41, -122.48, 47.79, -122.16)["shop"="books"];
37+
nwr(47.41, -122.48, 47.79, -122.16)["amenity"="cafe"];
38+
nwr(47.41, -122.48, 47.79, -122.16)["amenity"="community_centre"];
39+
nwr(47.41, -122.48, 47.79, -122.16)["shop"="convenience"];
40+
nwr(47.41, -122.48, 47.79, -122.16)["shop"="general"];
41+
nwr(47.41, -122.48, 47.79, -122.16)["amenity"="dentist"];
42+
nwr(47.41, -122.48, 47.79, -122.16)["healthcare"="dentist"];
43+
nwr(47.41, -122.48, 47.79, -122.16)["shop"="supermarket"];
44+
nwr(47.41, -122.48, 47.79, -122.16)["shop"="charity"];
45+
nwr(47.41, -122.48, 47.79, -122.16)["shop"="second_hand"];
46+
nwr(47.41, -122.48, 47.79, -122.16)["leisure"="playground"];
47+
nwr(47.41, -122.48, 47.79, -122.16)["leisure"="fitness_centre"];
48+
nwr(47.41, -122.48, 47.79, -122.16)["shop"="bakery"];
49+
nwr(47.41, -122.48, 47.79, -122.16)["shop"="hardware"];
50+
nwr(47.41, -122.48, 47.79, -122.16)["shop"="doityourself"];
51+
);
52+
out ids;
53+
54+

0 commit comments

Comments
 (0)