Fix WorldBank urls and outdated data usage #1292
Open
GitHub Actions / Petals / Unit Test Results (3.12)
failed
May 15, 2026 in 0s
3 fail, 3 skipped, 289 pass in 5m 49s
Annotations
Check warning on line 0 in climada_petals.entity.exposures.test.test_black_marble.TestEconIndices
github-actions / Petals / Unit Test Results (3.12)
test_fill_econ_indicators_na_pass (climada_petals.entity.exposures.test.test_black_marble.TestEconIndices) failed
climada_petals/tests_xml/tests.xml [took 1s]
Raw output
ImportError: Pandas requires version '3.1.0' or newer of 'openpyxl' (version '3.0.9' currently installed).
self = <climada_petals.entity.exposures.test.test_black_marble.TestEconIndices testMethod=test_fill_econ_indicators_na_pass>
def test_fill_econ_indicators_na_pass(self):
"""Test fill_econ_indicators with '' inputs."""
ref_year = 2019
country_isos = {'CHE': [1, 'Switzerland', 'che_geom'],
'ZMB': [2, 'Zambia', 'zmb_geom']
}
gdp = {'CHE': 1.2 * 1e20, 'ZMB': ''}
inc_grp = {'CHE': '', 'ZMB': 4}
kwargs = {'gdp': gdp, 'inc_grp': inc_grp}
> fill_econ_indicators(ref_year, country_isos, SHP_FILE, **kwargs)
climada_petals/entity/exposures/test/test_black_marble.py:301:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
climada_petals/entity/exposures/black_marble.py:276: in fill_econ_indicators
_, inc_grp = income_group(cntry_iso, ref_year, shp_file)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/climada/util/finance.py:117: in income_group
close_year, close_val = world_bank(cntry_iso, ref_year, "INC_GRP")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/climada/util/finance.py:282: in world_bank
dfr_wb = pd.read_excel(fn_ig, "Country Analytical History", skiprows=5)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/pandas/io/excel/_base.py:495: in read_excel
io = ExcelFile(
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/pandas/io/excel/_base.py:1567: in __init__
self._reader = self._engines[engine](
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/pandas/io/excel/_openpyxl.py:552: in __init__
import_optional_dependency("openpyxl")
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
name = 'openpyxl', extra = '', errors = 'raise', min_version = None
def import_optional_dependency(
name: str,
extra: str = "",
errors: str = "raise",
min_version: str | None = None,
):
"""
Import an optional dependency.
By default, if a dependency is missing an ImportError with a nice
message will be raised. If a dependency is present, but too old,
we raise.
Parameters
----------
name : str
The module name.
extra : str
Additional text to include in the ImportError message.
errors : str {'raise', 'warn', 'ignore'}
What to do when a dependency is not found or its version is too old.
* raise : Raise an ImportError
* warn : Only applicable when a module's version is to old.
Warns that the version is too old and returns None
* ignore: If the module is not installed, return None, otherwise,
return the module, even if the version is too old.
It's expected that users validate the version locally when
using ``errors="ignore"`` (see. ``io/html.py``)
min_version : str, default None
Specify a minimum version that is different from the global pandas
minimum version required.
Returns
-------
maybe_module : Optional[ModuleType]
The imported module, when found and the version is correct.
None is returned when the package is not found and `errors`
is False, or when the package's version is too old and `errors`
is ``'warn'`` or ``'ignore'``.
"""
assert errors in {"warn", "raise", "ignore"}
package_name = INSTALL_MAPPING.get(name)
install_name = package_name if package_name is not None else name
msg = (
f"Missing optional dependency '{install_name}'. {extra} "
f"Use pip or conda to install {install_name}."
)
try:
module = importlib.import_module(name)
except ImportError:
if errors == "raise":
raise ImportError(msg)
return None
# Handle submodules: if we have submodule, grab parent module from sys.modules
parent = name.split(".")[0]
if parent != name:
install_name = parent
module_to_get = sys.modules[install_name]
else:
module_to_get = module
minimum_version = min_version if min_version is not None else VERSIONS.get(parent)
if minimum_version:
version = get_version(module_to_get)
if version and Version(version) < Version(minimum_version):
msg = (
f"Pandas requires version '{minimum_version}' or newer of '{parent}' "
f"(version '{version}' currently installed)."
)
if errors == "warn":
warnings.warn(
msg,
UserWarning,
stacklevel=find_stack_level(),
)
return None
elif errors == "raise":
> raise ImportError(msg)
E ImportError: Pandas requires version '3.1.0' or newer of 'openpyxl' (version '3.0.9' currently installed).
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/pandas/compat/_optional.py:164: ImportError
Check warning on line 0 in climada_petals.entity.exposures.test.test_black_marble.TestEconIndices
github-actions / Petals / Unit Test Results (3.12)
test_fill_econ_indicators_pass (climada_petals.entity.exposures.test.test_black_marble.TestEconIndices) failed
climada_petals/tests_xml/tests.xml [took 0s]
Raw output
ImportError: Pandas requires version '3.1.0' or newer of 'openpyxl' (version '3.0.9' currently installed).
self = <climada_petals.entity.exposures.test.test_black_marble.TestEconIndices testMethod=test_fill_econ_indicators_pass>
def test_fill_econ_indicators_pass(self):
"""Test fill_econ_indicators CHE, ZMB."""
ref_year = 2015
country_isos = {'CHE': [1, 'Switzerland', 'che_geom'],
'ZMB': [2, 'Zambia', 'zmb_geom']
}
> fill_econ_indicators(ref_year, country_isos, SHP_FILE)
climada_petals/entity/exposures/test/test_black_marble.py:264:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
climada_petals/entity/exposures/black_marble.py:276: in fill_econ_indicators
_, inc_grp = income_group(cntry_iso, ref_year, shp_file)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/climada/util/finance.py:117: in income_group
close_year, close_val = world_bank(cntry_iso, ref_year, "INC_GRP")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/climada/util/finance.py:282: in world_bank
dfr_wb = pd.read_excel(fn_ig, "Country Analytical History", skiprows=5)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/pandas/io/excel/_base.py:495: in read_excel
io = ExcelFile(
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/pandas/io/excel/_base.py:1567: in __init__
self._reader = self._engines[engine](
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/pandas/io/excel/_openpyxl.py:552: in __init__
import_optional_dependency("openpyxl")
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
name = 'openpyxl', extra = '', errors = 'raise', min_version = None
def import_optional_dependency(
name: str,
extra: str = "",
errors: str = "raise",
min_version: str | None = None,
):
"""
Import an optional dependency.
By default, if a dependency is missing an ImportError with a nice
message will be raised. If a dependency is present, but too old,
we raise.
Parameters
----------
name : str
The module name.
extra : str
Additional text to include in the ImportError message.
errors : str {'raise', 'warn', 'ignore'}
What to do when a dependency is not found or its version is too old.
* raise : Raise an ImportError
* warn : Only applicable when a module's version is to old.
Warns that the version is too old and returns None
* ignore: If the module is not installed, return None, otherwise,
return the module, even if the version is too old.
It's expected that users validate the version locally when
using ``errors="ignore"`` (see. ``io/html.py``)
min_version : str, default None
Specify a minimum version that is different from the global pandas
minimum version required.
Returns
-------
maybe_module : Optional[ModuleType]
The imported module, when found and the version is correct.
None is returned when the package is not found and `errors`
is False, or when the package's version is too old and `errors`
is ``'warn'`` or ``'ignore'``.
"""
assert errors in {"warn", "raise", "ignore"}
package_name = INSTALL_MAPPING.get(name)
install_name = package_name if package_name is not None else name
msg = (
f"Missing optional dependency '{install_name}'. {extra} "
f"Use pip or conda to install {install_name}."
)
try:
module = importlib.import_module(name)
except ImportError:
if errors == "raise":
raise ImportError(msg)
return None
# Handle submodules: if we have submodule, grab parent module from sys.modules
parent = name.split(".")[0]
if parent != name:
install_name = parent
module_to_get = sys.modules[install_name]
else:
module_to_get = module
minimum_version = min_version if min_version is not None else VERSIONS.get(parent)
if minimum_version:
version = get_version(module_to_get)
if version and Version(version) < Version(minimum_version):
msg = (
f"Pandas requires version '{minimum_version}' or newer of '{parent}' "
f"(version '{version}' currently installed)."
)
if errors == "warn":
warnings.warn(
msg,
UserWarning,
stacklevel=find_stack_level(),
)
return None
elif errors == "raise":
> raise ImportError(msg)
E ImportError: Pandas requires version '3.1.0' or newer of 'openpyxl' (version '3.0.9' currently installed).
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/pandas/compat/_optional.py:164: ImportError
Check warning on line 0 in climada_petals.entity.exposures.test.test_osm_dataloader.TestOSMApiQuery
github-actions / Petals / Unit Test Results (3.12)
test_get_data_overpass (climada_petals.entity.exposures.test.test_osm_dataloader.TestOSMApiQuery) failed
climada_petals/tests_xml/tests.xml [took 2m 9s]
Raw output
Exception: The Overpass API is consistently unavailable
self = <climada_petals.entity.exposures.osm_dataloader.OSMApiQuery object at 0x7f223c6e4470>
query_clause = '[out:json][timeout:180];(nwr["building"](47.36826, 8.5327506, 47.376877, 8.5486078);(._;>;););out;'
read_chunk_size = 100000, end_of_patience = 127
def _insistent_osm_api_query(self, query_clause, read_chunk_size=100000,
end_of_patience=127):
"""Runs a single Overpass API query through overpy.Overpass.query.
In case of failure it tries again after an ever increasing waiting period.
If the waiting period surpasses a given limit an exception is raised.
Parameters:
query_clause (str): the query
read_chunk_size (int): paramter passed over to overpy.Overpass.query
end_of_patience (int): upper limit for the next waiting period to proceed.
Returns:
result as returned by overpy.Overpass.query
"""
api = overpy.Overpass(read_chunk_size=read_chunk_size)
waiting_period = 1
while True:
try:
> return api.query(query_clause)
^^^^^^^^^^^^^^^^^^^^^^^
climada_petals/entity/exposures/osm_dataloader.py:102:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <overpy.Overpass object at 0x7f223c6e42c0>
query = b'[out:json][timeout:180];(nwr["building"](47.36826, 8.5327506, 47.376877, 8.5486078);(._;>;););out;'
def query(self, query: Union[bytes, str]) -> "Result":
"""
Query the Overpass API
:param query: The query string in Overpass QL
:return: The parsed result
"""
if not isinstance(query, bytes):
query = query.encode("utf-8")
retry_num: int = 0
retry_exceptions: List[exception.OverPyException] = []
do_retry: bool = True if self.max_retry_count > 0 else False
while retry_num <= self.max_retry_count:
if retry_num > 0:
time.sleep(self.retry_timeout)
retry_num += 1
try:
f = urlopen(self.url, query)
except HTTPError as e:
f = e
response = f.read(self.read_chunk_size)
while True:
data = f.read(self.read_chunk_size)
if len(data) == 0:
break
response = response + data
f.close()
current_exception: exception.OverPyException
if f.code == 200:
content_type = f.getheader("Content-Type")
if content_type == "application/json":
return self.parse_json(response)
if content_type == "application/osm3s+xml":
return self.parse_xml(response)
current_exception = exception.OverpassUnknownContentType(content_type)
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
if f.code == 400:
msgs: List[str] = []
for msg_raw in self._regex_extract_error_msg.finditer(response):
msg_clean_bytes = self._regex_remove_tag.sub(b"", msg_raw.group("msg"))
try:
msg = msg_clean_bytes.decode("utf-8")
except UnicodeDecodeError:
msg = repr(msg_clean_bytes)
msgs.append(msg)
current_exception = exception.OverpassBadRequest(
query,
msgs=msgs
)
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
if f.code == 429:
current_exception = exception.OverpassTooManyRequests()
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
if f.code == 504:
current_exception = exception.OverpassGatewayTimeout()
if not do_retry:
raise current_exception
retry_exceptions.append(current_exception)
continue
current_exception = exception.OverpassUnknownHTTPStatusCode(f.code)
if not do_retry:
> raise current_exception
E overpy.exception.OverpassUnknownHTTPStatusCode: Unknown/Unhandled status code: 406
../../../../micromamba/envs/climada_env_3.12/lib/python3.12/site-packages/overpy/__init__.py:195: OverpassUnknownHTTPStatusCode
During handling of the above exception, another exception occurred:
self = <climada_petals.entity.exposures.test.test_osm_dataloader.TestOSMApiQuery testMethod=test_get_data_overpass>
def test_get_data_overpass(self):
"""test methods of OSMApiQuery"""
area_bbox = (8.5327506, 47.368260, 8.5486078, 47.376877)
area_poly = shapely.geometry.Polygon(
[(8.5327506, 47.368260),
(8.5486078, 47.376877),
(8.5486078, 47.39)])
condition_building = '["building"]'
condition_church = '["amenity"="place_of_worship"]'
gdf1 = osm_dl.OSMApiQuery.from_bounding_box(
> area_bbox, condition_building).get_data_overpass()
^^^^^^^^^^^^^^^^^^^
climada_petals/entity/exposures/test/test_osm_dataloader.py:123:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
climada_petals/entity/exposures/osm_dataloader.py:368: in get_data_overpass
result = self._insistent_osm_api_query(query_clause)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <climada_petals.entity.exposures.osm_dataloader.OSMApiQuery object at 0x7f223c6e4470>
query_clause = '[out:json][timeout:180];(nwr["building"](47.36826, 8.5327506, 47.376877, 8.5486078);(._;>;););out;'
read_chunk_size = 100000, end_of_patience = 127
def _insistent_osm_api_query(self, query_clause, read_chunk_size=100000,
end_of_patience=127):
"""Runs a single Overpass API query through overpy.Overpass.query.
In case of failure it tries again after an ever increasing waiting period.
If the waiting period surpasses a given limit an exception is raised.
Parameters:
query_clause (str): the query
read_chunk_size (int): paramter passed over to overpy.Overpass.query
end_of_patience (int): upper limit for the next waiting period to proceed.
Returns:
result as returned by overpy.Overpass.query
"""
api = overpy.Overpass(read_chunk_size=read_chunk_size)
waiting_period = 1
while True:
try:
return api.query(query_clause)
except overpy.exception.OverpassTooManyRequests:
if waiting_period < end_of_patience:
LOGGER.warning("""Too many Overpass API requests -
trying again in {waiting_period} seconds """)
else:
raise Exception("Overpass API is consistently unavailable")
except Exception as exc:
if waiting_period < end_of_patience:
LOGGER.warning(f"""{exc}
Trying again in {waiting_period} seconds""")
else:
> raise Exception(
"The Overpass API is consistently unavailable")
E Exception: The Overpass API is consistently unavailable
climada_petals/entity/exposures/osm_dataloader.py:114: Exception
Loading