Fix WorldBank urls and outdated data usage#1292
Open
spjuhel wants to merge 19 commits into
Open
GitHub Actions / Petals / Unit Test Results (3.11)
failed
May 28, 2026 in 0s
2 fail, 3 skipped, 290 pass in 9m 36s
Annotations
Check warning on line 0 in climada_petals.entity.exposures.test.test_black_marble.TestEconIndices
github-actions / Petals / Unit Test Results (3.11)
test_fill_econ_indicators_na_pass (climada_petals.entity.exposures.test.test_black_marble.TestEconIndices) failed
climada_petals/tests_xml/tests.xml [took 2m 0s]
Raw output
ValueError: Error loading page https://ddh-openapi.worldbank.org/resources/DR0095334/download
Status: 502
Content: b'The specified CGI application encountered an error and the server terminated the process.'
self = <climada_petals.entity.exposures.test.test_black_marble.TestEconIndices testMethod=test_fill_econ_indicators_na_pass>
def test_fill_econ_indicators_na_pass(self):
"""Test fill_econ_indicators with '' inputs."""
ref_year = 2019
country_isos = {'CHE': [1, 'Switzerland', 'che_geom'],
'ZMB': [2, 'Zambia', 'zmb_geom']
}
gdp = {'CHE': 1.2 * 1e20, 'ZMB': ''}
inc_grp = {'CHE': '', 'ZMB': 4}
kwargs = {'gdp': gdp, 'inc_grp': inc_grp}
> fill_econ_indicators(ref_year, country_isos, SHP_FILE, **kwargs)
climada_petals/entity/exposures/test/test_black_marble.py:301:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
climada_petals/entity/exposures/black_marble.py:276: in fill_econ_indicators
_, inc_grp = income_group(cntry_iso, ref_year, shp_file)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/climada/util/finance.py:117: in income_group
close_year, close_val = world_bank(cntry_iso, ref_year, "INC_GRP")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/climada/util/finance.py:280: in world_bank
file_down = download_file(WORLD_BANK_INC_GRP)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://ddh-openapi.worldbank.org/resources/DR0095334/download'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
req_file = requests.get(url, stream=True)
except IOError as ioe:
raise type(ioe)("Check URL and internet connection: " + str(ioe)) from ioe
if req_file.status_code < 200 or req_file.status_code > 299:
> raise ValueError(
f"Error loading page {url}\n"
f" Status: {req_file.status_code}\n"
f" Content: {req_file.content}"
)
E ValueError: Error loading page https://ddh-openapi.worldbank.org/resources/DR0095334/download
E Status: 502
E Content: b'The specified CGI application encountered an error and the server terminated the process.'
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/climada/util/files_handler.py:97: ValueError
Check warning on line 0 in climada_petals.entity.exposures.test.test_osm_dataloader.TestOSMApiQuery
github-actions / Petals / Unit Test Results (3.11)
test_get_data_overpass (climada_petals.entity.exposures.test.test_osm_dataloader.TestOSMApiQuery) failed
climada_petals/tests_xml/tests.xml [took 2m 8s]
Raw output
Exception: The Overpass API is consistently unavailable
self = <climada_petals.entity.exposures.osm_dataloader.OSMApiQuery object at 0x7fe6b555c390>
query_clause = '[out:json][timeout:180];(nwr["building"](47.36826, 8.5327506, 47.376877, 8.5486078);(._;>;););out;'
read_chunk_size = 100000, end_of_patience = 127
def _insistent_osm_api_query(self, query_clause, read_chunk_size=100000,
end_of_patience=127):
"""Runs a single Overpass API query through overpy.Overpass.query.
In case of failure it tries again after an ever increasing waiting period.
If the waiting period surpasses a given limit an exception is raised.
Parameters:
query_clause (str): the query
read_chunk_size (int): paramter passed over to overpy.Overpass.query
end_of_patience (int): upper limit for the next waiting period to proceed.
Returns:
result as returned by overpy.Overpass.query
"""
api = overpy.Overpass(read_chunk_size=read_chunk_size)
waiting_period = 1
while True:
try:
> return api.query(query_clause)
^^^^^^^^^^^^^^^^^^^^^^^
climada_petals/entity/exposures/osm_dataloader.py:102:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <overpy.Overpass object at 0x7fe6b5a7c4d0>
query = b'[out:json][timeout:180];(nwr["building"](47.36826, 8.5327506, 47.376877, 8.5486078);(._;>;););out;'
def query(self, query: Union[bytes, str]) -> "Result":
"""
Query the Overpass API
:param query: The query string in Overpass QL
:return: The parsed result
"""
if not isinstance(query, bytes):
query = query.encode("utf-8")
retry_num: int = 0
retry_exceptions: List[exception.OverPyException] = []
do_retry: bool = True if self.max_retry_count > 0 else False
while retry_num <= self.max_retry_count:
if retry_num > 0:
time.sleep(self.retry_timeout)
retry_num += 1
try:
f = urlopen(self.url, query)
except HTTPError as e:
f = e
response = f.read(self.read_chunk_size)
while True:
data = f.read(self.read_chunk_size)
if len(data) == 0:
break
response = response + data
f.close()
current_exception: exception.OverPyException
if f.code == 200:
content_type = f.getheader("Content-Type")
if content_type == "application/json":
return self.parse_json(response)
if content_type == "application/osm3s+xml":
return self.parse_xml(response)
current_exception = exception.OverpassUnknownContentType(content_type)
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
if f.code == 400:
msgs: List[str] = []
for msg_raw in self._regex_extract_error_msg.finditer(response):
msg_clean_bytes = self._regex_remove_tag.sub(b"", msg_raw.group("msg"))
try:
msg = msg_clean_bytes.decode("utf-8")
except UnicodeDecodeError:
msg = repr(msg_clean_bytes)
msgs.append(msg)
current_exception = exception.OverpassBadRequest(
query,
msgs=msgs
)
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
if f.code == 429:
current_exception = exception.OverpassTooManyRequests()
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
if f.code == 504:
current_exception = exception.OverpassGatewayTimeout()
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
current_exception = exception.OverpassUnknownHTTPStatusCode(f.code)
if not do_retry:
> raise current_exception
E overpy.exception.OverpassUnknownHTTPStatusCode: Unknown/Unhandled status code: 406
../../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/overpy/__init__.py:195: OverpassUnknownHTTPStatusCode
During handling of the above exception, another exception occurred:
self = <climada_petals.entity.exposures.test.test_osm_dataloader.TestOSMApiQuery testMethod=test_get_data_overpass>
def test_get_data_overpass(self):
"""test methods of OSMApiQuery"""
area_bbox = (8.5327506, 47.368260, 8.5486078, 47.376877)
area_poly = shapely.geometry.Polygon(
[(8.5327506, 47.368260),
(8.5486078, 47.376877),
(8.5486078, 47.39)])
condition_building = '["building"]'
condition_church = '["amenity"="place_of_worship"]'
gdf1 = osm_dl.OSMApiQuery.from_bounding_box(
> area_bbox, condition_building).get_data_overpass()
^^^^^^^^^^^^^^^^^^^
climada_petals/entity/exposures/test/test_osm_dataloader.py:123:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
climada_petals/entity/exposures/osm_dataloader.py:368: in get_data_overpass
result = self._insistent_osm_api_query(query_clause)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <climada_petals.entity.exposures.osm_dataloader.OSMApiQuery object at 0x7fe6b555c390>
query_clause = '[out:json][timeout:180];(nwr["building"](47.36826, 8.5327506, 47.376877, 8.5486078);(._;>;););out;'
read_chunk_size = 100000, end_of_patience = 127
def _insistent_osm_api_query(self, query_clause, read_chunk_size=100000,
end_of_patience=127):
"""Runs a single Overpass API query through overpy.Overpass.query.
In case of failure it tries again after an ever increasing waiting period.
If the waiting period surpasses a given limit an exception is raised.
Parameters:
query_clause (str): the query
read_chunk_size (int): paramter passed over to overpy.Overpass.query
end_of_patience (int): upper limit for the next waiting period to proceed.
Returns:
result as returned by overpy.Overpass.query
"""
api = overpy.Overpass(read_chunk_size=read_chunk_size)
waiting_period = 1
while True:
try:
return api.query(query_clause)
except overpy.exception.OverpassTooManyRequests:
if waiting_period < end_of_patience:
LOGGER.warning("""Too many Overpass API requests -
trying again in {waiting_period} seconds """)
else:
raise Exception("Overpass API is consistently unavailable")
except Exception as exc:
if waiting_period < end_of_patience:
LOGGER.warning(f"""{exc}
Trying again in {waiting_period} seconds""")
else:
> raise Exception(
"The Overpass API is consistently unavailable")
E Exception: The Overpass API is consistently unavailable
climada_petals/entity/exposures/osm_dataloader.py:114: Exception
Loading