[Option Appraisal Module] [Package 1: Measures] Split 2: Adds CostIncome class for measure financial aspects management
#1277
GitHub Actions / Core / Unit Test Results (3.11)
failed
May 13, 2026 in 0s
8 fail, 1 skipped, 921 pass in 6m 14s
Annotations
Check warning on line 0 in climada.entity.exposures.test.test_litpop.TestLitPop
github-actions / Core / Unit Test Results (3.11)
test_get_total_value_per_country_gdp (climada.entity.exposures.test.test_litpop.TestLitPop) failed
tests_xml/tests.xml [took 0s]
Raw output
requests.exceptions.SSLError: Internet connection failed while downloading historical income groups: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9aae8f850>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9b28a5250>
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...NB4GolBGpHgigdToQpSRMgEJ9wJC76CA; _cfuvid=a0vhnWvMEmbII4rc38IgNHw1QROlYxp5r1AeHJKOVLk-1778676719900-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9b28a5250>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
> self._validate_conn(conn)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:464:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:1106: in _validate_conn
conn.connect()
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:796: in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:975: in _ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:433: in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:477: in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:517: in wrap_socket
return self.sslsocket_class._create(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1104: in _create
self.do_handshake()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False
@_sslcopydoc
def do_handshake(self, block=False):
self._check_connected()
timeout = self.gettimeout()
try:
if timeout == 0.0 and block:
self.settimeout(None)
> self._sslobj.do_handshake()
E ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1382: SSLCertVerificationError
During handling of the above exception, another exception occurred:
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9aae8f850>
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...NB4GolBGpHgigdToQpSRMgEJ9wJC76CA; _cfuvid=a0vhnWvMEmbII4rc38IgNHw1QROlYxp5r1AeHJKOVLk-1778676719900-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
redirect = False, assert_same_host = False
timeout = Timeout(connect=None, read=None, total=None), pool_timeout = None
release_conn = False, chunked = False, body_pos = None, preload_content = False
decode_content = False, response_kw = {}, destination_scheme = None, conn = None
release_this_conn = True, http_tunnel_required = False, err = None
clean_exit = False
def urlopen( # type: ignore[override]
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | bool | int | None = None,
redirect: bool = True,
assert_same_host: bool = True,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
pool_timeout: int | None = None,
release_conn: bool | None = None,
chunked: bool = False,
body_pos: _TYPE_BODY_POSITION | None = None,
preload_content: bool = True,
decode_content: bool = True,
**response_kw: typing.Any,
) -> BaseHTTPResponse:
"""
Get a connection from the pool and perform an HTTP request. This is the
lowest level call for making a request, so you'll need to specify all
the raw details.
.. note::
More commonly, it's appropriate to use a convenience method
such as :meth:`request`.
.. note::
`release_conn` will only behave as expected if
`preload_content=False` because we want to make
`preload_content=False` the default behaviour someday soon without
breaking backwards compatibility.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param redirect:
If True, automatically handle redirects (status codes 301, 302,
303, 307, 308). Each redirect counts as a retry. Disabling retries
will disable redirect, too.
:param assert_same_host:
If ``True``, will make sure that the host of the pool requests is
consistent else will raise HostChangedError. When ``False``, you can
use the pool on an HTTP proxy and request foreign hosts.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param pool_timeout:
If set and the pool is set to block=True, then this method will
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
connection is available within the time period.
:param bool preload_content:
If True, the response's body will be preloaded into memory.
:param bool decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param release_conn:
If False, then the urlopen call will not release the connection
back into the pool once a response is received (but will release if
you read the entire contents of the response such as when
`preload_content=True`). This is useful if you're not preloading
the response's content immediately. You will need to call
``r.release_conn()`` on the response ``r`` to return the connection
back into the pool. If None, it takes the value of ``preload_content``
which defaults to ``True``.
:param bool chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param int body_pos:
Position to seek to in file-like body in the event of a retry or
redirect. Typically this won't need to be set because urllib3 will
auto-populate the value when needed.
"""
# Ensure that the URL we're connecting to is properly encoded
if url.startswith("/"):
# URLs starting with / are inherently schemeless.
url = to_str(_encode_target(url))
destination_scheme = None
else:
parsed_url = parse_url(url)
destination_scheme = parsed_url.scheme
url = to_str(parsed_url.url)
if headers is None:
headers = self.headers
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
if release_conn is None:
release_conn = preload_content
# Check host
if assert_same_host and not self.is_same_host(url):
raise HostChangedError(self, url, retries)
conn = None
# Track whether `conn` needs to be released before
# returning/raising/recursing. Update this variable if necessary, and
# leave `release_conn` constant throughout the function. That way, if
# the function recurses, the original value of `release_conn` will be
# passed down into the recursive call, and its value will be respected.
#
# See issue #651 [1] for details.
#
# [1] <https://github.com/urllib3/urllib3/issues/651>
release_this_conn = release_conn
http_tunnel_required = connection_requires_http_tunnel(
self.proxy, self.proxy_config, destination_scheme
)
# Merge the proxy headers. Only done when not using HTTP CONNECT. We
# have to copy the headers dict so we can safely change it without those
# changes being reflected in anyone else's copy.
if not http_tunnel_required:
headers = headers.copy() # type: ignore[attr-defined]
headers.update(self.proxy_headers) # type: ignore[union-attr]
# Must keep the exception bound to a separate variable or else Python 3
# complains about UnboundLocalError.
err = None
# Keep track of whether we cleanly exited the except block. This
# ensures we do proper cleanup in finally.
clean_exit = False
# Rewind body position, if needed. Record current position
# for future rewinds in the event of a redirect/retry.
body_pos = set_file_position(body, body_pos)
try:
# Request a connection from the queue.
timeout_obj = self._get_timeout(timeout)
conn = self._get_conn(timeout=pool_timeout)
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
# Is this a closed/new connection that requires CONNECT tunnelling?
if self.proxy is not None and http_tunnel_required and conn.is_closed:
try:
self._prepare_proxy(conn)
except (BaseSSLError, OSError, SocketTimeout) as e:
self._raise_timeout(
err=e, url=self.proxy.url, timeout_value=conn.timeout
)
raise
# If we're going to release the connection in ``finally:``, then
# the response doesn't need to know about the connection. Otherwise
# it will also try to release it and we'll have a double-release
# mess.
response_conn = conn if not release_conn else None
# Make the request on the HTTPConnection object
> response = self._make_request(
conn,
method,
url,
timeout=timeout_obj,
body=body,
headers=headers,
chunked=chunked,
retries=retries,
response_conn=response_conn,
preload_content=preload_content,
decode_content=decode_content,
**response_kw,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:788:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9aae8f850>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9b28a5250>
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...NB4GolBGpHgigdToQpSRMgEJ9wJC76CA; _cfuvid=a0vhnWvMEmbII4rc38IgNHw1QROlYxp5r1AeHJKOVLk-1778676719900-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9b28a5250>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
self._validate_conn(conn)
except (SocketTimeout, BaseSSLError) as e:
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
raise
# _validate_conn() starts the connection to an HTTPS proxy
# so we need to wrap errors with 'ProxyError' here too.
except (
OSError,
NewConnectionError,
TimeoutError,
BaseSSLError,
CertificateError,
SSLError,
) as e:
new_e: Exception = e
if isinstance(e, (BaseSSLError, CertificateError)):
new_e = SSLError(e)
# If the connection didn't successfully connect to it's proxy
# then there
if isinstance(
new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
> raise new_e
E urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:488: SSLError
The above exception was the direct cause of the following exception:
self = <requests.adapters.HTTPAdapter object at 0x7fb9aa3de450>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
> resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:696:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:842: in urlopen
retries = retries.increment(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = Retry(total=0, connect=None, read=False, redirect=None, status=None)
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
response = None
error = SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)'))
_pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9aae8f850>
_stacktrace = <traceback object at 0x7fb9a98de340>
def increment(
self,
method: str | None = None,
url: str | None = None,
response: BaseHTTPResponse | None = None,
error: Exception | None = None,
_pool: ConnectionPool | None = None,
_stacktrace: TracebackType | None = None,
) -> Self:
"""Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
return a response.
:type response: :class:`~urllib3.response.BaseHTTPResponse`
:param Exception error: An error encountered during the request, or
None if the response was received successfully.
:return: A new ``Retry`` object.
"""
if self.total is False and error:
# Disabled, indicate to re-raise the error.
raise reraise(type(error), error, _stacktrace)
total = self.total
if total is not None:
total -= 1
connect = self.connect
read = self.read
redirect = self.redirect
status_count = self.status
other = self.other
cause = "unknown"
status = None
redirect_location = None
if error and self._is_connection_error(error):
# Connect retry?
if connect is False:
raise reraise(type(error), error, _stacktrace)
elif connect is not None:
connect -= 1
elif error and self._is_read_error(error):
# Read retry?
if read is False or method is None or not self._is_method_retryable(method):
raise reraise(type(error), error, _stacktrace)
elif read is not None:
read -= 1
elif error:
# Other retry?
if other is not None:
other -= 1
elif response and response.get_redirect_location():
# Redirect retry?
if redirect is not None:
redirect -= 1
cause = "too many redirects"
response_redirect_location = response.get_redirect_location()
if response_redirect_location:
redirect_location = response_redirect_location
status = response.status
else:
# Incrementing because of a server error like a 500 in
# status_forcelist and the given method is in the allowed_methods
cause = ResponseError.GENERIC_ERROR
if response and response.status:
if status_count is not None:
status_count -= 1
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
status = response.status
history = self.history + (
RequestHistory(method, url, error, status, redirect_location),
)
new_retry = self.new(
total=total,
connect=connect,
read=read,
redirect=redirect,
status=status_count,
other=other,
history=history,
)
if new_retry.is_exhausted():
reason = error or ResponseError(cause)
> raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/retry.py:543: MaxRetryError
During handling of the above exception, another exception occurred:
url = 'http://databank.worldbank.org/data/download/site-content/OGHIST.xls'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
> req_file = requests.get(url, stream=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/files_handler.py:93:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:87: in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:71: in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:651: in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in send
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in <listcomp>
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:292: in resolve_redirects
resp = self.send(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:784: in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <requests.adapters.HTTPAdapter object at 0x7fb9aa3de450>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
except (ProtocolError, OSError) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
> raise SSLError(e, request=request)
E requests.exceptions.SSLError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:727: SSLError
The above exception was the direct cause of the following exception:
cntry_iso = 'CHE', ref_year = 2222, info_ind = 'INC_GRP'
def world_bank(cntry_iso, ref_year, info_ind):
"""Get country's GDP from World Bank's data at a given year, or
closest year value. If no data, get the natural earth's approximation.
Parameters
----------
cntry_iso : str
key = ISO alpha_3 country
ref_year : int
reference year
info_ind : str
indicator of World Bank, e.g. 'NY.GDP.MKTP.CD'. If
'INC_GRP', historical income groups from excel file used.
Returns
-------
int, float
Raises
------
IOError, KeyError, IndexError
"""
if info_ind != "INC_GRP":
cntry_gdp = download_world_bank_indicator(
indicator=info_ind, country_code=cntry_iso, parse_dates=False
)
years = cntry_gdp.index
sort_years = np.abs(years - ref_year).argsort()
close_val = cntry_gdp.iloc[sort_years].dropna()
close_year = close_val.index[0]
close_val = float(close_val.iloc[0])
else: # income group level
fn_ig = SYSTEM_DIR.joinpath("OGHIST.xls")
dfr_wb = pd.DataFrame()
try:
if not fn_ig.is_file():
> file_down = download_file(WORLD_BANK_INC_GRP)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/finance.py:290:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'http://databank.worldbank.org/data/download/site-content/OGHIST.xls'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
req_file = requests.get(url, stream=True)
except IOError as ioe:
> raise type(ioe)("Check URL and internet connection: " + str(ioe)) from ioe
E requests.exceptions.SSLError: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/files_handler.py:95: SSLError
The above exception was the direct cause of the following exception:
self = <climada.entity.exposures.test.test_litpop.TestLitPop testMethod=test_get_total_value_per_country_gdp>
def test_get_total_value_per_country_gdp(self):
"test _get_total_value_per_country get number for gdp"
gdp_togo = lp._get_total_value_per_country("TGO", "gdp", 2010)
gdp_switzerland = lp._get_total_value_per_country("CHE", "gdp", 2222)
> value_switzerland = lp._get_total_value_per_country("CHE", "income_group", 2222)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/entity/exposures/test/test_litpop.py:406:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
climada/entity/exposures/litpop/litpop.py:1243: in _get_total_value_per_country
return gdp_value * (u_fin.income_group(cntry_iso3a, reference_year)[1] + 1)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/finance.py:128: in income_group
close_year, close_val = world_bank(cntry_iso, ref_year, "INC_GRP")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cntry_iso = 'CHE', ref_year = 2222, info_ind = 'INC_GRP'
def world_bank(cntry_iso, ref_year, info_ind):
"""Get country's GDP from World Bank's data at a given year, or
closest year value. If no data, get the natural earth's approximation.
Parameters
----------
cntry_iso : str
key = ISO alpha_3 country
ref_year : int
reference year
info_ind : str
indicator of World Bank, e.g. 'NY.GDP.MKTP.CD'. If
'INC_GRP', historical income groups from excel file used.
Returns
-------
int, float
Raises
------
IOError, KeyError, IndexError
"""
if info_ind != "INC_GRP":
cntry_gdp = download_world_bank_indicator(
indicator=info_ind, country_code=cntry_iso, parse_dates=False
)
years = cntry_gdp.index
sort_years = np.abs(years - ref_year).argsort()
close_val = cntry_gdp.iloc[sort_years].dropna()
close_year = close_val.index[0]
close_val = float(close_val.iloc[0])
else: # income group level
fn_ig = SYSTEM_DIR.joinpath("OGHIST.xls")
dfr_wb = pd.DataFrame()
try:
if not fn_ig.is_file():
file_down = download_file(WORLD_BANK_INC_GRP)
shutil.move(file_down, fn_ig)
dfr_wb = pd.read_excel(fn_ig, "Country Analytical History", skiprows=5)
dfr_wb = dfr_wb.drop(dfr_wb.index[0:5]).set_index("Unnamed: 0")
dfr_wb = dfr_wb.replace(
INCOME_GRP_WB_TABLE.keys(), INCOME_GRP_WB_TABLE.values()
)
except (IOError, requests.exceptions.ConnectionError) as err:
> raise type(err)(
"Internet connection failed while downloading "
"historical income groups: " + str(err)
) from err
E requests.exceptions.SSLError: Internet connection failed while downloading historical income groups: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/finance.py:298: SSLError
Check warning on line 0 in climada.entity.exposures.test.test_litpop.TestLitPop
github-actions / Core / Unit Test Results (3.11)
test_get_total_value_per_country_pc (climada.entity.exposures.test.test_litpop.TestLitPop) failed
tests_xml/tests.xml [took 0s]
Raw output
requests.exceptions.SSLError: Downloading World Bank Wealth Accounting Data failed: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9b27ae750>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9b27ac1d0>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...tgxSJeWfq67TgbmWifPWAXrzlavO7E7A; _cfuvid=RLVVHQqW_l4HR7QjdDQmfZ6.UCRqB8sMexsdNMXYJGs-1778676720740-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9b27ac1d0>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
> self._validate_conn(conn)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:464:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:1106: in _validate_conn
conn.connect()
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:796: in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:975: in _ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:433: in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:477: in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:517: in wrap_socket
return self.sslsocket_class._create(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1104: in _create
self.do_handshake()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False
@_sslcopydoc
def do_handshake(self, block=False):
self._check_connected()
timeout = self.gettimeout()
try:
if timeout == 0.0 and block:
self.settimeout(None)
> self._sslobj.do_handshake()
E ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1382: SSLCertVerificationError
During handling of the above exception, another exception occurred:
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9b27ae750>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...tgxSJeWfq67TgbmWifPWAXrzlavO7E7A; _cfuvid=RLVVHQqW_l4HR7QjdDQmfZ6.UCRqB8sMexsdNMXYJGs-1778676720740-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
redirect = False, assert_same_host = False
timeout = Timeout(connect=None, read=None, total=None), pool_timeout = None
release_conn = False, chunked = False, body_pos = None, preload_content = False
decode_content = False, response_kw = {}, destination_scheme = None, conn = None
release_this_conn = True, http_tunnel_required = False, err = None
clean_exit = False
def urlopen( # type: ignore[override]
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | bool | int | None = None,
redirect: bool = True,
assert_same_host: bool = True,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
pool_timeout: int | None = None,
release_conn: bool | None = None,
chunked: bool = False,
body_pos: _TYPE_BODY_POSITION | None = None,
preload_content: bool = True,
decode_content: bool = True,
**response_kw: typing.Any,
) -> BaseHTTPResponse:
"""
Get a connection from the pool and perform an HTTP request. This is the
lowest level call for making a request, so you'll need to specify all
the raw details.
.. note::
More commonly, it's appropriate to use a convenience method
such as :meth:`request`.
.. note::
`release_conn` will only behave as expected if
`preload_content=False` because we want to make
`preload_content=False` the default behaviour someday soon without
breaking backwards compatibility.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param redirect:
If True, automatically handle redirects (status codes 301, 302,
303, 307, 308). Each redirect counts as a retry. Disabling retries
will disable redirect, too.
:param assert_same_host:
If ``True``, will make sure that the host of the pool requests is
consistent else will raise HostChangedError. When ``False``, you can
use the pool on an HTTP proxy and request foreign hosts.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param pool_timeout:
If set and the pool is set to block=True, then this method will
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
connection is available within the time period.
:param bool preload_content:
If True, the response's body will be preloaded into memory.
:param bool decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param release_conn:
If False, then the urlopen call will not release the connection
back into the pool once a response is received (but will release if
you read the entire contents of the response such as when
`preload_content=True`). This is useful if you're not preloading
the response's content immediately. You will need to call
``r.release_conn()`` on the response ``r`` to return the connection
back into the pool. If None, it takes the value of ``preload_content``
which defaults to ``True``.
:param bool chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param int body_pos:
Position to seek to in file-like body in the event of a retry or
redirect. Typically this won't need to be set because urllib3 will
auto-populate the value when needed.
"""
# Ensure that the URL we're connecting to is properly encoded
if url.startswith("/"):
# URLs starting with / are inherently schemeless.
url = to_str(_encode_target(url))
destination_scheme = None
else:
parsed_url = parse_url(url)
destination_scheme = parsed_url.scheme
url = to_str(parsed_url.url)
if headers is None:
headers = self.headers
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
if release_conn is None:
release_conn = preload_content
# Check host
if assert_same_host and not self.is_same_host(url):
raise HostChangedError(self, url, retries)
conn = None
# Track whether `conn` needs to be released before
# returning/raising/recursing. Update this variable if necessary, and
# leave `release_conn` constant throughout the function. That way, if
# the function recurses, the original value of `release_conn` will be
# passed down into the recursive call, and its value will be respected.
#
# See issue #651 [1] for details.
#
# [1] <https://github.com/urllib3/urllib3/issues/651>
release_this_conn = release_conn
http_tunnel_required = connection_requires_http_tunnel(
self.proxy, self.proxy_config, destination_scheme
)
# Merge the proxy headers. Only done when not using HTTP CONNECT. We
# have to copy the headers dict so we can safely change it without those
# changes being reflected in anyone else's copy.
if not http_tunnel_required:
headers = headers.copy() # type: ignore[attr-defined]
headers.update(self.proxy_headers) # type: ignore[union-attr]
# Must keep the exception bound to a separate variable or else Python 3
# complains about UnboundLocalError.
err = None
# Keep track of whether we cleanly exited the except block. This
# ensures we do proper cleanup in finally.
clean_exit = False
# Rewind body position, if needed. Record current position
# for future rewinds in the event of a redirect/retry.
body_pos = set_file_position(body, body_pos)
try:
# Request a connection from the queue.
timeout_obj = self._get_timeout(timeout)
conn = self._get_conn(timeout=pool_timeout)
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
# Is this a closed/new connection that requires CONNECT tunnelling?
if self.proxy is not None and http_tunnel_required and conn.is_closed:
try:
self._prepare_proxy(conn)
except (BaseSSLError, OSError, SocketTimeout) as e:
self._raise_timeout(
err=e, url=self.proxy.url, timeout_value=conn.timeout
)
raise
# If we're going to release the connection in ``finally:``, then
# the response doesn't need to know about the connection. Otherwise
# it will also try to release it and we'll have a double-release
# mess.
response_conn = conn if not release_conn else None
# Make the request on the HTTPConnection object
> response = self._make_request(
conn,
method,
url,
timeout=timeout_obj,
body=body,
headers=headers,
chunked=chunked,
retries=retries,
response_conn=response_conn,
preload_content=preload_content,
decode_content=decode_content,
**response_kw,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:788:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9b27ae750>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9b27ac1d0>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...tgxSJeWfq67TgbmWifPWAXrzlavO7E7A; _cfuvid=RLVVHQqW_l4HR7QjdDQmfZ6.UCRqB8sMexsdNMXYJGs-1778676720740-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9b27ac1d0>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
self._validate_conn(conn)
except (SocketTimeout, BaseSSLError) as e:
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
raise
# _validate_conn() starts the connection to an HTTPS proxy
# so we need to wrap errors with 'ProxyError' here too.
except (
OSError,
NewConnectionError,
TimeoutError,
BaseSSLError,
CertificateError,
SSLError,
) as e:
new_e: Exception = e
if isinstance(e, (BaseSSLError, CertificateError)):
new_e = SSLError(e)
# If the connection didn't successfully connect to it's proxy
# then there
if isinstance(
new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
> raise new_e
E urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:488: SSLError
The above exception was the direct cause of the following exception:
self = <requests.adapters.HTTPAdapter object at 0x7fb9a9813dd0>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
> resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:696:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:842: in urlopen
retries = retries.increment(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = Retry(total=0, connect=None, read=False, redirect=None, status=None)
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
response = None
error = SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)'))
_pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9b27ae750>
_stacktrace = <traceback object at 0x7fb9b27aeec0>
def increment(
self,
method: str | None = None,
url: str | None = None,
response: BaseHTTPResponse | None = None,
error: Exception | None = None,
_pool: ConnectionPool | None = None,
_stacktrace: TracebackType | None = None,
) -> Self:
"""Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
return a response.
:type response: :class:`~urllib3.response.BaseHTTPResponse`
:param Exception error: An error encountered during the request, or
None if the response was received successfully.
:return: A new ``Retry`` object.
"""
if self.total is False and error:
# Disabled, indicate to re-raise the error.
raise reraise(type(error), error, _stacktrace)
total = self.total
if total is not None:
total -= 1
connect = self.connect
read = self.read
redirect = self.redirect
status_count = self.status
other = self.other
cause = "unknown"
status = None
redirect_location = None
if error and self._is_connection_error(error):
# Connect retry?
if connect is False:
raise reraise(type(error), error, _stacktrace)
elif connect is not None:
connect -= 1
elif error and self._is_read_error(error):
# Read retry?
if read is False or method is None or not self._is_method_retryable(method):
raise reraise(type(error), error, _stacktrace)
elif read is not None:
read -= 1
elif error:
# Other retry?
if other is not None:
other -= 1
elif response and response.get_redirect_location():
# Redirect retry?
if redirect is not None:
redirect -= 1
cause = "too many redirects"
response_redirect_location = response.get_redirect_location()
if response_redirect_location:
redirect_location = response_redirect_location
status = response.status
else:
# Incrementing because of a server error like a 500 in
# status_forcelist and the given method is in the allowed_methods
cause = ResponseError.GENERIC_ERROR
if response and response.status:
if status_count is not None:
status_count -= 1
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
status = response.status
history = self.history + (
RequestHistory(method, url, error, status, redirect_location),
)
new_retry = self.new(
total=total,
connect=connect,
read=read,
redirect=redirect,
status=status_count,
other=other,
history=history,
)
if new_retry.is_exhausted():
reason = error or ResponseError(cause)
> raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/retry.py:543: MaxRetryError
During handling of the above exception, another exception occurred:
url = 'https://databank.worldbank.org/data/download/Wealth-Accounts_CSV.zip'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
> req_file = requests.get(url, stream=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/files_handler.py:93:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:87: in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:71: in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:651: in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in send
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in <listcomp>
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:292: in resolve_redirects
resp = self.send(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:784: in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <requests.adapters.HTTPAdapter object at 0x7fb9a9813dd0>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
except (ProtocolError, OSError) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
> raise SSLError(e, request=request)
E requests.exceptions.SSLError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:727: SSLError
The above exception was the direct cause of the following exception:
cntry_iso = 'POL', ref_year = 2015, variable_name = 'NW.PCA.TO', no_land = True
def world_bank_wealth_account(
cntry_iso, ref_year, variable_name="NW.PCA.TO", no_land=True
):
"""
Download and unzip wealth accounting historical data (1995, 2000, 2005, 2010, 2014)
from World Bank (https://datacatalog.worldbank.org/dataset/wealth-accounting).
Return requested variable for a country (cntry_iso) and a year (ref_year).
Parameters
----------
cntry_iso : str
ISO3-code of country, i.e. "CHN" for China
ref_year : int
reference year
* available in data: 1995, 2000, 2005, 2010, 2014
* other years between 1995 and 2014 are interpolated
* for years outside range, indicator is scaled
proportionally to GDP
variable_name : str
select one variable, i.e.:
'NW.PCA.TO': Produced capital stock of country
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.PCA.PC': Produced capital stock per capita
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.NCA.TO': Total natural capital of country. Natural capital
includes the valuation of fossil fuel energy (oil, gas,
hard and soft coal) and minerals (bauxite, copper, gold,
iron ore, lead, nickel, phosphate, silver, tin, and zinc),
agricultural land (cropland and pastureland),
forests (timber and some nontimber forest products), and
protected areas.
'NW.TOW.TO': Total wealth of country.
Note: Values are measured at market exchange rates in constant 2014 US dollars,
using a country-specific GDP deflator.
no_land : boolean
If True, return produced capital without built-up land value
(applies to 'NW.PCA.*' only). Default: True.
"""
try:
data_file = SYSTEM_DIR.joinpath(FILE_WORLD_BANK_WEALTH_ACC)
if not data_file.is_file():
data_file = SYSTEM_DIR.joinpath(
"Wealth-Accounts_CSV", FILE_WORLD_BANK_WEALTH_ACC
)
if not data_file.is_file():
if not SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").is_dir():
SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").mkdir()
> file_down = download_file(WORLD_BANK_WEALTH_ACC)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/finance.py:467:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://databank.worldbank.org/data/download/Wealth-Accounts_CSV.zip'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
req_file = requests.get(url, stream=True)
except IOError as ioe:
> raise type(ioe)("Check URL and internet connection: " + str(ioe)) from ioe
E requests.exceptions.SSLError: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/files_handler.py:95: SSLError
The above exception was the direct cause of the following exception:
self = <climada.entity.exposures.test.test_litpop.TestLitPop testMethod=test_get_total_value_per_country_pc>
def test_get_total_value_per_country_pc(self):
"test _get_total_value_per_country get number for pc of Poland"
> value = lp._get_total_value_per_country("POL", "pc", 2015)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/entity/exposures/test/test_litpop.py:413:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
climada/entity/exposures/litpop/litpop.py:1226: in _get_total_value_per_country
return u_fin.world_bank_wealth_account(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cntry_iso = 'POL', ref_year = 2015, variable_name = 'NW.PCA.TO', no_land = True
def world_bank_wealth_account(
cntry_iso, ref_year, variable_name="NW.PCA.TO", no_land=True
):
"""
Download and unzip wealth accounting historical data (1995, 2000, 2005, 2010, 2014)
from World Bank (https://datacatalog.worldbank.org/dataset/wealth-accounting).
Return requested variable for a country (cntry_iso) and a year (ref_year).
Parameters
----------
cntry_iso : str
ISO3-code of country, i.e. "CHN" for China
ref_year : int
reference year
* available in data: 1995, 2000, 2005, 2010, 2014
* other years between 1995 and 2014 are interpolated
* for years outside range, indicator is scaled
proportionally to GDP
variable_name : str
select one variable, i.e.:
'NW.PCA.TO': Produced capital stock of country
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.PCA.PC': Produced capital stock per capita
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.NCA.TO': Total natural capital of country. Natural capital
includes the valuation of fossil fuel energy (oil, gas,
hard and soft coal) and minerals (bauxite, copper, gold,
iron ore, lead, nickel, phosphate, silver, tin, and zinc),
agricultural land (cropland and pastureland),
forests (timber and some nontimber forest products), and
protected areas.
'NW.TOW.TO': Total wealth of country.
Note: Values are measured at market exchange rates in constant 2014 US dollars,
using a country-specific GDP deflator.
no_land : boolean
If True, return produced capital without built-up land value
(applies to 'NW.PCA.*' only). Default: True.
"""
try:
data_file = SYSTEM_DIR.joinpath(FILE_WORLD_BANK_WEALTH_ACC)
if not data_file.is_file():
data_file = SYSTEM_DIR.joinpath(
"Wealth-Accounts_CSV", FILE_WORLD_BANK_WEALTH_ACC
)
if not data_file.is_file():
if not SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").is_dir():
SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").mkdir()
file_down = download_file(WORLD_BANK_WEALTH_ACC)
zip_ref = zipfile.ZipFile(file_down, "r")
zip_ref.extractall(SYSTEM_DIR.joinpath("Wealth-Accounts_CSV"))
zip_ref.close()
Path(file_down).unlink()
LOGGER.debug("Download and unzip complete. Unzipping %s", str(data_file))
data_wealth = pd.read_csv(data_file, sep=",", index_col=None, header=0)
except Exception as err:
> raise type(err)(
"Downloading World Bank Wealth Accounting Data failed: " + str(err)
) from err
E requests.exceptions.SSLError: Downloading World Bank Wealth Accounting Data failed: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/finance.py:476: SSLError
Check warning on line 0 in climada.util.test.test_finance.TestWBData
github-actions / Core / Unit Test Results (3.11)
test_income_grp_sxm_1999_pass (climada.util.test.test_finance.TestWBData) failed
tests_xml/tests.xml [took 0s]
Raw output
requests.exceptions.SSLError: Internet connection failed while downloading historical income groups: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9aa20e490>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9aa20f210>
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...lduUwg7I5ClblMQ7eAdA8zTBiWwlEoN4; _cfuvid=G6V6NFNcMnjh.2q1lkuJ6tDrnQPzPUaS.QgjQ2NqNfw-1778676900178-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9aa20f210>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
> self._validate_conn(conn)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:464:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:1106: in _validate_conn
conn.connect()
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:796: in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:975: in _ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:433: in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:477: in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:517: in wrap_socket
return self.sslsocket_class._create(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1104: in _create
self.do_handshake()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False
@_sslcopydoc
def do_handshake(self, block=False):
self._check_connected()
timeout = self.gettimeout()
try:
if timeout == 0.0 and block:
self.settimeout(None)
> self._sslobj.do_handshake()
E ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1382: SSLCertVerificationError
During handling of the above exception, another exception occurred:
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9aa20e490>
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...lduUwg7I5ClblMQ7eAdA8zTBiWwlEoN4; _cfuvid=G6V6NFNcMnjh.2q1lkuJ6tDrnQPzPUaS.QgjQ2NqNfw-1778676900178-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
redirect = False, assert_same_host = False
timeout = Timeout(connect=None, read=None, total=None), pool_timeout = None
release_conn = False, chunked = False, body_pos = None, preload_content = False
decode_content = False, response_kw = {}, destination_scheme = None, conn = None
release_this_conn = True, http_tunnel_required = False, err = None
clean_exit = False
def urlopen( # type: ignore[override]
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | bool | int | None = None,
redirect: bool = True,
assert_same_host: bool = True,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
pool_timeout: int | None = None,
release_conn: bool | None = None,
chunked: bool = False,
body_pos: _TYPE_BODY_POSITION | None = None,
preload_content: bool = True,
decode_content: bool = True,
**response_kw: typing.Any,
) -> BaseHTTPResponse:
"""
Get a connection from the pool and perform an HTTP request. This is the
lowest level call for making a request, so you'll need to specify all
the raw details.
.. note::
More commonly, it's appropriate to use a convenience method
such as :meth:`request`.
.. note::
`release_conn` will only behave as expected if
`preload_content=False` because we want to make
`preload_content=False` the default behaviour someday soon without
breaking backwards compatibility.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param redirect:
If True, automatically handle redirects (status codes 301, 302,
303, 307, 308). Each redirect counts as a retry. Disabling retries
will disable redirect, too.
:param assert_same_host:
If ``True``, will make sure that the host of the pool requests is
consistent else will raise HostChangedError. When ``False``, you can
use the pool on an HTTP proxy and request foreign hosts.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param pool_timeout:
If set and the pool is set to block=True, then this method will
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
connection is available within the time period.
:param bool preload_content:
If True, the response's body will be preloaded into memory.
:param bool decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param release_conn:
If False, then the urlopen call will not release the connection
back into the pool once a response is received (but will release if
you read the entire contents of the response such as when
`preload_content=True`). This is useful if you're not preloading
the response's content immediately. You will need to call
``r.release_conn()`` on the response ``r`` to return the connection
back into the pool. If None, it takes the value of ``preload_content``
which defaults to ``True``.
:param bool chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param int body_pos:
Position to seek to in file-like body in the event of a retry or
redirect. Typically this won't need to be set because urllib3 will
auto-populate the value when needed.
"""
# Ensure that the URL we're connecting to is properly encoded
if url.startswith("/"):
# URLs starting with / are inherently schemeless.
url = to_str(_encode_target(url))
destination_scheme = None
else:
parsed_url = parse_url(url)
destination_scheme = parsed_url.scheme
url = to_str(parsed_url.url)
if headers is None:
headers = self.headers
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
if release_conn is None:
release_conn = preload_content
# Check host
if assert_same_host and not self.is_same_host(url):
raise HostChangedError(self, url, retries)
conn = None
# Track whether `conn` needs to be released before
# returning/raising/recursing. Update this variable if necessary, and
# leave `release_conn` constant throughout the function. That way, if
# the function recurses, the original value of `release_conn` will be
# passed down into the recursive call, and its value will be respected.
#
# See issue #651 [1] for details.
#
# [1] <https://github.com/urllib3/urllib3/issues/651>
release_this_conn = release_conn
http_tunnel_required = connection_requires_http_tunnel(
self.proxy, self.proxy_config, destination_scheme
)
# Merge the proxy headers. Only done when not using HTTP CONNECT. We
# have to copy the headers dict so we can safely change it without those
# changes being reflected in anyone else's copy.
if not http_tunnel_required:
headers = headers.copy() # type: ignore[attr-defined]
headers.update(self.proxy_headers) # type: ignore[union-attr]
# Must keep the exception bound to a separate variable or else Python 3
# complains about UnboundLocalError.
err = None
# Keep track of whether we cleanly exited the except block. This
# ensures we do proper cleanup in finally.
clean_exit = False
# Rewind body position, if needed. Record current position
# for future rewinds in the event of a redirect/retry.
body_pos = set_file_position(body, body_pos)
try:
# Request a connection from the queue.
timeout_obj = self._get_timeout(timeout)
conn = self._get_conn(timeout=pool_timeout)
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
# Is this a closed/new connection that requires CONNECT tunnelling?
if self.proxy is not None and http_tunnel_required and conn.is_closed:
try:
self._prepare_proxy(conn)
except (BaseSSLError, OSError, SocketTimeout) as e:
self._raise_timeout(
err=e, url=self.proxy.url, timeout_value=conn.timeout
)
raise
# If we're going to release the connection in ``finally:``, then
# the response doesn't need to know about the connection. Otherwise
# it will also try to release it and we'll have a double-release
# mess.
response_conn = conn if not release_conn else None
# Make the request on the HTTPConnection object
> response = self._make_request(
conn,
method,
url,
timeout=timeout_obj,
body=body,
headers=headers,
chunked=chunked,
retries=retries,
response_conn=response_conn,
preload_content=preload_content,
decode_content=decode_content,
**response_kw,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:788:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9aa20e490>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9aa20f210>
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...lduUwg7I5ClblMQ7eAdA8zTBiWwlEoN4; _cfuvid=G6V6NFNcMnjh.2q1lkuJ6tDrnQPzPUaS.QgjQ2NqNfw-1778676900178-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9aa20f210>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
self._validate_conn(conn)
except (SocketTimeout, BaseSSLError) as e:
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
raise
# _validate_conn() starts the connection to an HTTPS proxy
# so we need to wrap errors with 'ProxyError' here too.
except (
OSError,
NewConnectionError,
TimeoutError,
BaseSSLError,
CertificateError,
SSLError,
) as e:
new_e: Exception = e
if isinstance(e, (BaseSSLError, CertificateError)):
new_e = SSLError(e)
# If the connection didn't successfully connect to it's proxy
# then there
if isinstance(
new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
> raise new_e
E urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:488: SSLError
The above exception was the direct cause of the following exception:
self = <requests.adapters.HTTPAdapter object at 0x7fb9b27af750>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
> resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:696:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:842: in urlopen
retries = retries.increment(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = Retry(total=0, connect=None, read=False, redirect=None, status=None)
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
response = None
error = SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)'))
_pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9aa20e490>
_stacktrace = <traceback object at 0x7fba2413dcc0>
def increment(
self,
method: str | None = None,
url: str | None = None,
response: BaseHTTPResponse | None = None,
error: Exception | None = None,
_pool: ConnectionPool | None = None,
_stacktrace: TracebackType | None = None,
) -> Self:
"""Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
return a response.
:type response: :class:`~urllib3.response.BaseHTTPResponse`
:param Exception error: An error encountered during the request, or
None if the response was received successfully.
:return: A new ``Retry`` object.
"""
if self.total is False and error:
# Disabled, indicate to re-raise the error.
raise reraise(type(error), error, _stacktrace)
total = self.total
if total is not None:
total -= 1
connect = self.connect
read = self.read
redirect = self.redirect
status_count = self.status
other = self.other
cause = "unknown"
status = None
redirect_location = None
if error and self._is_connection_error(error):
# Connect retry?
if connect is False:
raise reraise(type(error), error, _stacktrace)
elif connect is not None:
connect -= 1
elif error and self._is_read_error(error):
# Read retry?
if read is False or method is None or not self._is_method_retryable(method):
raise reraise(type(error), error, _stacktrace)
elif read is not None:
read -= 1
elif error:
# Other retry?
if other is not None:
other -= 1
elif response and response.get_redirect_location():
# Redirect retry?
if redirect is not None:
redirect -= 1
cause = "too many redirects"
response_redirect_location = response.get_redirect_location()
if response_redirect_location:
redirect_location = response_redirect_location
status = response.status
else:
# Incrementing because of a server error like a 500 in
# status_forcelist and the given method is in the allowed_methods
cause = ResponseError.GENERIC_ERROR
if response and response.status:
if status_count is not None:
status_count -= 1
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
status = response.status
history = self.history + (
RequestHistory(method, url, error, status, redirect_location),
)
new_retry = self.new(
total=total,
connect=connect,
read=read,
redirect=redirect,
status=status_count,
other=other,
history=history,
)
if new_retry.is_exhausted():
reason = error or ResponseError(cause)
> raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/retry.py:543: MaxRetryError
During handling of the above exception, another exception occurred:
url = 'http://databank.worldbank.org/data/download/site-content/OGHIST.xls'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
> req_file = requests.get(url, stream=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/files_handler.py:93:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:87: in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:71: in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:651: in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in send
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in <listcomp>
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:292: in resolve_redirects
resp = self.send(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:784: in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <requests.adapters.HTTPAdapter object at 0x7fb9b27af750>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
except (ProtocolError, OSError) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
> raise SSLError(e, request=request)
E requests.exceptions.SSLError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:727: SSLError
The above exception was the direct cause of the following exception:
cntry_iso = 'SXM', ref_year = 1999, info_ind = 'INC_GRP'
def world_bank(cntry_iso, ref_year, info_ind):
"""Get country's GDP from World Bank's data at a given year, or
closest year value. If no data, get the natural earth's approximation.
Parameters
----------
cntry_iso : str
key = ISO alpha_3 country
ref_year : int
reference year
info_ind : str
indicator of World Bank, e.g. 'NY.GDP.MKTP.CD'. If
'INC_GRP', historical income groups from excel file used.
Returns
-------
int, float
Raises
------
IOError, KeyError, IndexError
"""
if info_ind != "INC_GRP":
cntry_gdp = download_world_bank_indicator(
indicator=info_ind, country_code=cntry_iso, parse_dates=False
)
years = cntry_gdp.index
sort_years = np.abs(years - ref_year).argsort()
close_val = cntry_gdp.iloc[sort_years].dropna()
close_year = close_val.index[0]
close_val = float(close_val.iloc[0])
else: # income group level
fn_ig = SYSTEM_DIR.joinpath("OGHIST.xls")
dfr_wb = pd.DataFrame()
try:
if not fn_ig.is_file():
> file_down = download_file(WORLD_BANK_INC_GRP)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/finance.py:290:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'http://databank.worldbank.org/data/download/site-content/OGHIST.xls'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
req_file = requests.get(url, stream=True)
except IOError as ioe:
> raise type(ioe)("Check URL and internet connection: " + str(ioe)) from ioe
E requests.exceptions.SSLError: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/files_handler.py:95: SSLError
The above exception was the direct cause of the following exception:
self = <climada.util.test.test_finance.TestWBData testMethod=test_income_grp_sxm_1999_pass>
def test_income_grp_sxm_1999_pass(self):
"""Test income_group function Sint Maarten."""
ref_year = 1999
with self.assertLogs("climada.util.finance", level="INFO") as cm:
> ig_year, ig_val = income_group("SXM", ref_year, SHP_FILE)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/test/test_finance.py:85:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
climada/util/finance.py:128: in income_group
close_year, close_val = world_bank(cntry_iso, ref_year, "INC_GRP")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cntry_iso = 'SXM', ref_year = 1999, info_ind = 'INC_GRP'
def world_bank(cntry_iso, ref_year, info_ind):
"""Get country's GDP from World Bank's data at a given year, or
closest year value. If no data, get the natural earth's approximation.
Parameters
----------
cntry_iso : str
key = ISO alpha_3 country
ref_year : int
reference year
info_ind : str
indicator of World Bank, e.g. 'NY.GDP.MKTP.CD'. If
'INC_GRP', historical income groups from excel file used.
Returns
-------
int, float
Raises
------
IOError, KeyError, IndexError
"""
if info_ind != "INC_GRP":
cntry_gdp = download_world_bank_indicator(
indicator=info_ind, country_code=cntry_iso, parse_dates=False
)
years = cntry_gdp.index
sort_years = np.abs(years - ref_year).argsort()
close_val = cntry_gdp.iloc[sort_years].dropna()
close_year = close_val.index[0]
close_val = float(close_val.iloc[0])
else: # income group level
fn_ig = SYSTEM_DIR.joinpath("OGHIST.xls")
dfr_wb = pd.DataFrame()
try:
if not fn_ig.is_file():
file_down = download_file(WORLD_BANK_INC_GRP)
shutil.move(file_down, fn_ig)
dfr_wb = pd.read_excel(fn_ig, "Country Analytical History", skiprows=5)
dfr_wb = dfr_wb.drop(dfr_wb.index[0:5]).set_index("Unnamed: 0")
dfr_wb = dfr_wb.replace(
INCOME_GRP_WB_TABLE.keys(), INCOME_GRP_WB_TABLE.values()
)
except (IOError, requests.exceptions.ConnectionError) as err:
> raise type(err)(
"Internet connection failed while downloading "
"historical income groups: " + str(err)
) from err
E requests.exceptions.SSLError: Internet connection failed while downloading historical income groups: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/finance.py:298: SSLError
Check warning on line 0 in climada.util.test.test_finance.TestWBData
github-actions / Core / Unit Test Results (3.11)
test_wb_income_grp_sxm_pass (climada.util.test.test_finance.TestWBData) failed
tests_xml/tests.xml [took 0s]
Raw output
requests.exceptions.SSLError: Internet connection failed while downloading historical income groups: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a9713cd0>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a9711b10>
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...2vfJ_joYmet.yztw_rGe2JBIEO7xMagA; _cfuvid=IA5lZ_9qIZVmy_6cU1M3Bc4m3mnY1iY_KA8cvSDLiCk-1778676900874-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a9711b10>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
> self._validate_conn(conn)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:464:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:1106: in _validate_conn
conn.connect()
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:796: in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:975: in _ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:433: in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:477: in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:517: in wrap_socket
return self.sslsocket_class._create(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1104: in _create
self.do_handshake()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False
@_sslcopydoc
def do_handshake(self, block=False):
self._check_connected()
timeout = self.gettimeout()
try:
if timeout == 0.0 and block:
self.settimeout(None)
> self._sslobj.do_handshake()
E ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1382: SSLCertVerificationError
During handling of the above exception, another exception occurred:
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a9713cd0>
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...2vfJ_joYmet.yztw_rGe2JBIEO7xMagA; _cfuvid=IA5lZ_9qIZVmy_6cU1M3Bc4m3mnY1iY_KA8cvSDLiCk-1778676900874-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
redirect = False, assert_same_host = False
timeout = Timeout(connect=None, read=None, total=None), pool_timeout = None
release_conn = False, chunked = False, body_pos = None, preload_content = False
decode_content = False, response_kw = {}, destination_scheme = None, conn = None
release_this_conn = True, http_tunnel_required = False, err = None
clean_exit = False
def urlopen( # type: ignore[override]
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | bool | int | None = None,
redirect: bool = True,
assert_same_host: bool = True,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
pool_timeout: int | None = None,
release_conn: bool | None = None,
chunked: bool = False,
body_pos: _TYPE_BODY_POSITION | None = None,
preload_content: bool = True,
decode_content: bool = True,
**response_kw: typing.Any,
) -> BaseHTTPResponse:
"""
Get a connection from the pool and perform an HTTP request. This is the
lowest level call for making a request, so you'll need to specify all
the raw details.
.. note::
More commonly, it's appropriate to use a convenience method
such as :meth:`request`.
.. note::
`release_conn` will only behave as expected if
`preload_content=False` because we want to make
`preload_content=False` the default behaviour someday soon without
breaking backwards compatibility.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param redirect:
If True, automatically handle redirects (status codes 301, 302,
303, 307, 308). Each redirect counts as a retry. Disabling retries
will disable redirect, too.
:param assert_same_host:
If ``True``, will make sure that the host of the pool requests is
consistent else will raise HostChangedError. When ``False``, you can
use the pool on an HTTP proxy and request foreign hosts.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param pool_timeout:
If set and the pool is set to block=True, then this method will
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
connection is available within the time period.
:param bool preload_content:
If True, the response's body will be preloaded into memory.
:param bool decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param release_conn:
If False, then the urlopen call will not release the connection
back into the pool once a response is received (but will release if
you read the entire contents of the response such as when
`preload_content=True`). This is useful if you're not preloading
the response's content immediately. You will need to call
``r.release_conn()`` on the response ``r`` to return the connection
back into the pool. If None, it takes the value of ``preload_content``
which defaults to ``True``.
:param bool chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param int body_pos:
Position to seek to in file-like body in the event of a retry or
redirect. Typically this won't need to be set because urllib3 will
auto-populate the value when needed.
"""
# Ensure that the URL we're connecting to is properly encoded
if url.startswith("/"):
# URLs starting with / are inherently schemeless.
url = to_str(_encode_target(url))
destination_scheme = None
else:
parsed_url = parse_url(url)
destination_scheme = parsed_url.scheme
url = to_str(parsed_url.url)
if headers is None:
headers = self.headers
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
if release_conn is None:
release_conn = preload_content
# Check host
if assert_same_host and not self.is_same_host(url):
raise HostChangedError(self, url, retries)
conn = None
# Track whether `conn` needs to be released before
# returning/raising/recursing. Update this variable if necessary, and
# leave `release_conn` constant throughout the function. That way, if
# the function recurses, the original value of `release_conn` will be
# passed down into the recursive call, and its value will be respected.
#
# See issue #651 [1] for details.
#
# [1] <https://github.com/urllib3/urllib3/issues/651>
release_this_conn = release_conn
http_tunnel_required = connection_requires_http_tunnel(
self.proxy, self.proxy_config, destination_scheme
)
# Merge the proxy headers. Only done when not using HTTP CONNECT. We
# have to copy the headers dict so we can safely change it without those
# changes being reflected in anyone else's copy.
if not http_tunnel_required:
headers = headers.copy() # type: ignore[attr-defined]
headers.update(self.proxy_headers) # type: ignore[union-attr]
# Must keep the exception bound to a separate variable or else Python 3
# complains about UnboundLocalError.
err = None
# Keep track of whether we cleanly exited the except block. This
# ensures we do proper cleanup in finally.
clean_exit = False
# Rewind body position, if needed. Record current position
# for future rewinds in the event of a redirect/retry.
body_pos = set_file_position(body, body_pos)
try:
# Request a connection from the queue.
timeout_obj = self._get_timeout(timeout)
conn = self._get_conn(timeout=pool_timeout)
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
# Is this a closed/new connection that requires CONNECT tunnelling?
if self.proxy is not None and http_tunnel_required and conn.is_closed:
try:
self._prepare_proxy(conn)
except (BaseSSLError, OSError, SocketTimeout) as e:
self._raise_timeout(
err=e, url=self.proxy.url, timeout_value=conn.timeout
)
raise
# If we're going to release the connection in ``finally:``, then
# the response doesn't need to know about the connection. Otherwise
# it will also try to release it and we'll have a double-release
# mess.
response_conn = conn if not release_conn else None
# Make the request on the HTTPConnection object
> response = self._make_request(
conn,
method,
url,
timeout=timeout_obj,
body=body,
headers=headers,
chunked=chunked,
retries=retries,
response_conn=response_conn,
preload_content=preload_content,
decode_content=decode_content,
**response_kw,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:788:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a9713cd0>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a9711b10>
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...2vfJ_joYmet.yztw_rGe2JBIEO7xMagA; _cfuvid=IA5lZ_9qIZVmy_6cU1M3Bc4m3mnY1iY_KA8cvSDLiCk-1778676900874-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a9711b10>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
self._validate_conn(conn)
except (SocketTimeout, BaseSSLError) as e:
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
raise
# _validate_conn() starts the connection to an HTTPS proxy
# so we need to wrap errors with 'ProxyError' here too.
except (
OSError,
NewConnectionError,
TimeoutError,
BaseSSLError,
CertificateError,
SSLError,
) as e:
new_e: Exception = e
if isinstance(e, (BaseSSLError, CertificateError)):
new_e = SSLError(e)
# If the connection didn't successfully connect to it's proxy
# then there
if isinstance(
new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
> raise new_e
E urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:488: SSLError
The above exception was the direct cause of the following exception:
self = <requests.adapters.HTTPAdapter object at 0x7fba24302c50>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
> resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:696:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:842: in urlopen
retries = retries.increment(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = Retry(total=0, connect=None, read=False, redirect=None, status=None)
method = 'GET', url = '/public/ddpext_download/site-content/OGHIST.xls'
response = None
error = SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)'))
_pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a9713cd0>
_stacktrace = <traceback object at 0x7fb9a9712180>
def increment(
self,
method: str | None = None,
url: str | None = None,
response: BaseHTTPResponse | None = None,
error: Exception | None = None,
_pool: ConnectionPool | None = None,
_stacktrace: TracebackType | None = None,
) -> Self:
"""Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
return a response.
:type response: :class:`~urllib3.response.BaseHTTPResponse`
:param Exception error: An error encountered during the request, or
None if the response was received successfully.
:return: A new ``Retry`` object.
"""
if self.total is False and error:
# Disabled, indicate to re-raise the error.
raise reraise(type(error), error, _stacktrace)
total = self.total
if total is not None:
total -= 1
connect = self.connect
read = self.read
redirect = self.redirect
status_count = self.status
other = self.other
cause = "unknown"
status = None
redirect_location = None
if error and self._is_connection_error(error):
# Connect retry?
if connect is False:
raise reraise(type(error), error, _stacktrace)
elif connect is not None:
connect -= 1
elif error and self._is_read_error(error):
# Read retry?
if read is False or method is None or not self._is_method_retryable(method):
raise reraise(type(error), error, _stacktrace)
elif read is not None:
read -= 1
elif error:
# Other retry?
if other is not None:
other -= 1
elif response and response.get_redirect_location():
# Redirect retry?
if redirect is not None:
redirect -= 1
cause = "too many redirects"
response_redirect_location = response.get_redirect_location()
if response_redirect_location:
redirect_location = response_redirect_location
status = response.status
else:
# Incrementing because of a server error like a 500 in
# status_forcelist and the given method is in the allowed_methods
cause = ResponseError.GENERIC_ERROR
if response and response.status:
if status_count is not None:
status_count -= 1
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
status = response.status
history = self.history + (
RequestHistory(method, url, error, status, redirect_location),
)
new_retry = self.new(
total=total,
connect=connect,
read=read,
redirect=redirect,
status=status_count,
other=other,
history=history,
)
if new_retry.is_exhausted():
reason = error or ResponseError(cause)
> raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/retry.py:543: MaxRetryError
During handling of the above exception, another exception occurred:
url = 'http://databank.worldbank.org/data/download/site-content/OGHIST.xls'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
> req_file = requests.get(url, stream=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/files_handler.py:93:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:87: in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:71: in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:651: in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in send
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in <listcomp>
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:292: in resolve_redirects
resp = self.send(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:784: in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <requests.adapters.HTTPAdapter object at 0x7fba24302c50>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
except (ProtocolError, OSError) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
> raise SSLError(e, request=request)
E requests.exceptions.SSLError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:727: SSLError
The above exception was the direct cause of the following exception:
cntry_iso = 'SXM', ref_year = 2012, info_ind = 'INC_GRP'
def world_bank(cntry_iso, ref_year, info_ind):
"""Get country's GDP from World Bank's data at a given year, or
closest year value. If no data, get the natural earth's approximation.
Parameters
----------
cntry_iso : str
key = ISO alpha_3 country
ref_year : int
reference year
info_ind : str
indicator of World Bank, e.g. 'NY.GDP.MKTP.CD'. If
'INC_GRP', historical income groups from excel file used.
Returns
-------
int, float
Raises
------
IOError, KeyError, IndexError
"""
if info_ind != "INC_GRP":
cntry_gdp = download_world_bank_indicator(
indicator=info_ind, country_code=cntry_iso, parse_dates=False
)
years = cntry_gdp.index
sort_years = np.abs(years - ref_year).argsort()
close_val = cntry_gdp.iloc[sort_years].dropna()
close_year = close_val.index[0]
close_val = float(close_val.iloc[0])
else: # income group level
fn_ig = SYSTEM_DIR.joinpath("OGHIST.xls")
dfr_wb = pd.DataFrame()
try:
if not fn_ig.is_file():
> file_down = download_file(WORLD_BANK_INC_GRP)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/finance.py:290:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'http://databank.worldbank.org/data/download/site-content/OGHIST.xls'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
req_file = requests.get(url, stream=True)
except IOError as ioe:
> raise type(ioe)("Check URL and internet connection: " + str(ioe)) from ioe
E requests.exceptions.SSLError: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/files_handler.py:95: SSLError
The above exception was the direct cause of the following exception:
self = <climada.util.test.test_finance.TestWBData testMethod=test_wb_income_grp_sxm_pass>
def test_wb_income_grp_sxm_pass(self):
"""Test world_bank function Sint Maarten."""
ref_year = 2012
> wb_year, wb_val = world_bank("SXM", ref_year, "INC_GRP")
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/test/test_finance.py:74:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cntry_iso = 'SXM', ref_year = 2012, info_ind = 'INC_GRP'
def world_bank(cntry_iso, ref_year, info_ind):
"""Get country's GDP from World Bank's data at a given year, or
closest year value. If no data, get the natural earth's approximation.
Parameters
----------
cntry_iso : str
key = ISO alpha_3 country
ref_year : int
reference year
info_ind : str
indicator of World Bank, e.g. 'NY.GDP.MKTP.CD'. If
'INC_GRP', historical income groups from excel file used.
Returns
-------
int, float
Raises
------
IOError, KeyError, IndexError
"""
if info_ind != "INC_GRP":
cntry_gdp = download_world_bank_indicator(
indicator=info_ind, country_code=cntry_iso, parse_dates=False
)
years = cntry_gdp.index
sort_years = np.abs(years - ref_year).argsort()
close_val = cntry_gdp.iloc[sort_years].dropna()
close_year = close_val.index[0]
close_val = float(close_val.iloc[0])
else: # income group level
fn_ig = SYSTEM_DIR.joinpath("OGHIST.xls")
dfr_wb = pd.DataFrame()
try:
if not fn_ig.is_file():
file_down = download_file(WORLD_BANK_INC_GRP)
shutil.move(file_down, fn_ig)
dfr_wb = pd.read_excel(fn_ig, "Country Analytical History", skiprows=5)
dfr_wb = dfr_wb.drop(dfr_wb.index[0:5]).set_index("Unnamed: 0")
dfr_wb = dfr_wb.replace(
INCOME_GRP_WB_TABLE.keys(), INCOME_GRP_WB_TABLE.values()
)
except (IOError, requests.exceptions.ConnectionError) as err:
> raise type(err)(
"Internet connection failed while downloading "
"historical income groups: " + str(err)
) from err
E requests.exceptions.SSLError: Internet connection failed while downloading historical income groups: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/site-content/OGHIST.xls (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/finance.py:298: SSLError
Check warning on line 0 in climada.util.test.test_finance.TestWBWealthAccount
github-actions / Core / Unit Test Results (3.11)
test_pca_CHE_2008_pass (climada.util.test.test_finance.TestWBWealthAccount) failed
tests_xml/tests.xml [took 0s]
Raw output
requests.exceptions.SSLError: Downloading World Bank Wealth Accounting Data failed: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a2f26490>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a2f27490>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...GuKlCECmOkL4YyRFS3wIGoZpVex6B41o; _cfuvid=avinASwAfsQZZQud8PrHCKU1869GbdgpL4lKo.3PusU-1778676901672-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a2f27490>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
> self._validate_conn(conn)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:464:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:1106: in _validate_conn
conn.connect()
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:796: in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:975: in _ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:433: in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:477: in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:517: in wrap_socket
return self.sslsocket_class._create(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1104: in _create
self.do_handshake()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False
@_sslcopydoc
def do_handshake(self, block=False):
self._check_connected()
timeout = self.gettimeout()
try:
if timeout == 0.0 and block:
self.settimeout(None)
> self._sslobj.do_handshake()
E ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1382: SSLCertVerificationError
During handling of the above exception, another exception occurred:
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a2f26490>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...GuKlCECmOkL4YyRFS3wIGoZpVex6B41o; _cfuvid=avinASwAfsQZZQud8PrHCKU1869GbdgpL4lKo.3PusU-1778676901672-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
redirect = False, assert_same_host = False
timeout = Timeout(connect=None, read=None, total=None), pool_timeout = None
release_conn = False, chunked = False, body_pos = None, preload_content = False
decode_content = False, response_kw = {}, destination_scheme = None, conn = None
release_this_conn = True, http_tunnel_required = False, err = None
clean_exit = False
def urlopen( # type: ignore[override]
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | bool | int | None = None,
redirect: bool = True,
assert_same_host: bool = True,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
pool_timeout: int | None = None,
release_conn: bool | None = None,
chunked: bool = False,
body_pos: _TYPE_BODY_POSITION | None = None,
preload_content: bool = True,
decode_content: bool = True,
**response_kw: typing.Any,
) -> BaseHTTPResponse:
"""
Get a connection from the pool and perform an HTTP request. This is the
lowest level call for making a request, so you'll need to specify all
the raw details.
.. note::
More commonly, it's appropriate to use a convenience method
such as :meth:`request`.
.. note::
`release_conn` will only behave as expected if
`preload_content=False` because we want to make
`preload_content=False` the default behaviour someday soon without
breaking backwards compatibility.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param redirect:
If True, automatically handle redirects (status codes 301, 302,
303, 307, 308). Each redirect counts as a retry. Disabling retries
will disable redirect, too.
:param assert_same_host:
If ``True``, will make sure that the host of the pool requests is
consistent else will raise HostChangedError. When ``False``, you can
use the pool on an HTTP proxy and request foreign hosts.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param pool_timeout:
If set and the pool is set to block=True, then this method will
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
connection is available within the time period.
:param bool preload_content:
If True, the response's body will be preloaded into memory.
:param bool decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param release_conn:
If False, then the urlopen call will not release the connection
back into the pool once a response is received (but will release if
you read the entire contents of the response such as when
`preload_content=True`). This is useful if you're not preloading
the response's content immediately. You will need to call
``r.release_conn()`` on the response ``r`` to return the connection
back into the pool. If None, it takes the value of ``preload_content``
which defaults to ``True``.
:param bool chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param int body_pos:
Position to seek to in file-like body in the event of a retry or
redirect. Typically this won't need to be set because urllib3 will
auto-populate the value when needed.
"""
# Ensure that the URL we're connecting to is properly encoded
if url.startswith("/"):
# URLs starting with / are inherently schemeless.
url = to_str(_encode_target(url))
destination_scheme = None
else:
parsed_url = parse_url(url)
destination_scheme = parsed_url.scheme
url = to_str(parsed_url.url)
if headers is None:
headers = self.headers
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
if release_conn is None:
release_conn = preload_content
# Check host
if assert_same_host and not self.is_same_host(url):
raise HostChangedError(self, url, retries)
conn = None
# Track whether `conn` needs to be released before
# returning/raising/recursing. Update this variable if necessary, and
# leave `release_conn` constant throughout the function. That way, if
# the function recurses, the original value of `release_conn` will be
# passed down into the recursive call, and its value will be respected.
#
# See issue #651 [1] for details.
#
# [1] <https://github.com/urllib3/urllib3/issues/651>
release_this_conn = release_conn
http_tunnel_required = connection_requires_http_tunnel(
self.proxy, self.proxy_config, destination_scheme
)
# Merge the proxy headers. Only done when not using HTTP CONNECT. We
# have to copy the headers dict so we can safely change it without those
# changes being reflected in anyone else's copy.
if not http_tunnel_required:
headers = headers.copy() # type: ignore[attr-defined]
headers.update(self.proxy_headers) # type: ignore[union-attr]
# Must keep the exception bound to a separate variable or else Python 3
# complains about UnboundLocalError.
err = None
# Keep track of whether we cleanly exited the except block. This
# ensures we do proper cleanup in finally.
clean_exit = False
# Rewind body position, if needed. Record current position
# for future rewinds in the event of a redirect/retry.
body_pos = set_file_position(body, body_pos)
try:
# Request a connection from the queue.
timeout_obj = self._get_timeout(timeout)
conn = self._get_conn(timeout=pool_timeout)
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
# Is this a closed/new connection that requires CONNECT tunnelling?
if self.proxy is not None and http_tunnel_required and conn.is_closed:
try:
self._prepare_proxy(conn)
except (BaseSSLError, OSError, SocketTimeout) as e:
self._raise_timeout(
err=e, url=self.proxy.url, timeout_value=conn.timeout
)
raise
# If we're going to release the connection in ``finally:``, then
# the response doesn't need to know about the connection. Otherwise
# it will also try to release it and we'll have a double-release
# mess.
response_conn = conn if not release_conn else None
# Make the request on the HTTPConnection object
> response = self._make_request(
conn,
method,
url,
timeout=timeout_obj,
body=body,
headers=headers,
chunked=chunked,
retries=retries,
response_conn=response_conn,
preload_content=preload_content,
decode_content=decode_content,
**response_kw,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:788:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a2f26490>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a2f27490>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...GuKlCECmOkL4YyRFS3wIGoZpVex6B41o; _cfuvid=avinASwAfsQZZQud8PrHCKU1869GbdgpL4lKo.3PusU-1778676901672-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a2f27490>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
self._validate_conn(conn)
except (SocketTimeout, BaseSSLError) as e:
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
raise
# _validate_conn() starts the connection to an HTTPS proxy
# so we need to wrap errors with 'ProxyError' here too.
except (
OSError,
NewConnectionError,
TimeoutError,
BaseSSLError,
CertificateError,
SSLError,
) as e:
new_e: Exception = e
if isinstance(e, (BaseSSLError, CertificateError)):
new_e = SSLError(e)
# If the connection didn't successfully connect to it's proxy
# then there
if isinstance(
new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
> raise new_e
E urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:488: SSLError
The above exception was the direct cause of the following exception:
self = <requests.adapters.HTTPAdapter object at 0x7fba242453d0>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
> resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:696:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:842: in urlopen
retries = retries.increment(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = Retry(total=0, connect=None, read=False, redirect=None, status=None)
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
response = None
error = SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)'))
_pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a2f26490>
_stacktrace = <traceback object at 0x7fb9a2f26300>
def increment(
self,
method: str | None = None,
url: str | None = None,
response: BaseHTTPResponse | None = None,
error: Exception | None = None,
_pool: ConnectionPool | None = None,
_stacktrace: TracebackType | None = None,
) -> Self:
"""Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
return a response.
:type response: :class:`~urllib3.response.BaseHTTPResponse`
:param Exception error: An error encountered during the request, or
None if the response was received successfully.
:return: A new ``Retry`` object.
"""
if self.total is False and error:
# Disabled, indicate to re-raise the error.
raise reraise(type(error), error, _stacktrace)
total = self.total
if total is not None:
total -= 1
connect = self.connect
read = self.read
redirect = self.redirect
status_count = self.status
other = self.other
cause = "unknown"
status = None
redirect_location = None
if error and self._is_connection_error(error):
# Connect retry?
if connect is False:
raise reraise(type(error), error, _stacktrace)
elif connect is not None:
connect -= 1
elif error and self._is_read_error(error):
# Read retry?
if read is False or method is None or not self._is_method_retryable(method):
raise reraise(type(error), error, _stacktrace)
elif read is not None:
read -= 1
elif error:
# Other retry?
if other is not None:
other -= 1
elif response and response.get_redirect_location():
# Redirect retry?
if redirect is not None:
redirect -= 1
cause = "too many redirects"
response_redirect_location = response.get_redirect_location()
if response_redirect_location:
redirect_location = response_redirect_location
status = response.status
else:
# Incrementing because of a server error like a 500 in
# status_forcelist and the given method is in the allowed_methods
cause = ResponseError.GENERIC_ERROR
if response and response.status:
if status_count is not None:
status_count -= 1
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
status = response.status
history = self.history + (
RequestHistory(method, url, error, status, redirect_location),
)
new_retry = self.new(
total=total,
connect=connect,
read=read,
redirect=redirect,
status=status_count,
other=other,
history=history,
)
if new_retry.is_exhausted():
reason = error or ResponseError(cause)
> raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/retry.py:543: MaxRetryError
During handling of the above exception, another exception occurred:
url = 'https://databank.worldbank.org/data/download/Wealth-Accounts_CSV.zip'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
> req_file = requests.get(url, stream=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/files_handler.py:93:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:87: in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:71: in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:651: in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in send
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in <listcomp>
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:292: in resolve_redirects
resp = self.send(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:784: in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <requests.adapters.HTTPAdapter object at 0x7fba242453d0>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
except (ProtocolError, OSError) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
> raise SSLError(e, request=request)
E requests.exceptions.SSLError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:727: SSLError
The above exception was the direct cause of the following exception:
cntry_iso = 'CHE', ref_year = 2008, variable_name = 'NW.PCA.PC', no_land = 0
def world_bank_wealth_account(
cntry_iso, ref_year, variable_name="NW.PCA.TO", no_land=True
):
"""
Download and unzip wealth accounting historical data (1995, 2000, 2005, 2010, 2014)
from World Bank (https://datacatalog.worldbank.org/dataset/wealth-accounting).
Return requested variable for a country (cntry_iso) and a year (ref_year).
Parameters
----------
cntry_iso : str
ISO3-code of country, i.e. "CHN" for China
ref_year : int
reference year
* available in data: 1995, 2000, 2005, 2010, 2014
* other years between 1995 and 2014 are interpolated
* for years outside range, indicator is scaled
proportionally to GDP
variable_name : str
select one variable, i.e.:
'NW.PCA.TO': Produced capital stock of country
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.PCA.PC': Produced capital stock per capita
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.NCA.TO': Total natural capital of country. Natural capital
includes the valuation of fossil fuel energy (oil, gas,
hard and soft coal) and minerals (bauxite, copper, gold,
iron ore, lead, nickel, phosphate, silver, tin, and zinc),
agricultural land (cropland and pastureland),
forests (timber and some nontimber forest products), and
protected areas.
'NW.TOW.TO': Total wealth of country.
Note: Values are measured at market exchange rates in constant 2014 US dollars,
using a country-specific GDP deflator.
no_land : boolean
If True, return produced capital without built-up land value
(applies to 'NW.PCA.*' only). Default: True.
"""
try:
data_file = SYSTEM_DIR.joinpath(FILE_WORLD_BANK_WEALTH_ACC)
if not data_file.is_file():
data_file = SYSTEM_DIR.joinpath(
"Wealth-Accounts_CSV", FILE_WORLD_BANK_WEALTH_ACC
)
if not data_file.is_file():
if not SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").is_dir():
SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").mkdir()
> file_down = download_file(WORLD_BANK_WEALTH_ACC)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/finance.py:467:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://databank.worldbank.org/data/download/Wealth-Accounts_CSV.zip'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
req_file = requests.get(url, stream=True)
except IOError as ioe:
> raise type(ioe)("Check URL and internet connection: " + str(ioe)) from ioe
E requests.exceptions.SSLError: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/files_handler.py:95: SSLError
The above exception was the direct cause of the following exception:
self = <climada.util.test.test_finance.TestWBWealthAccount testMethod=test_pca_CHE_2008_pass>
def test_pca_CHE_2008_pass(self):
"""Test Prcoessed Capital per capita Switzerland 2008 (interp.)."""
ref_year = 2008
cntry_iso = "CHE"
var_name = "NW.PCA.PC"
> wb_year, wb_val, _ = world_bank_wealth_account(
cntry_iso, ref_year, variable_name=var_name, no_land=0
)
climada/util/test/test_finance.py:226:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cntry_iso = 'CHE', ref_year = 2008, variable_name = 'NW.PCA.PC', no_land = 0
def world_bank_wealth_account(
cntry_iso, ref_year, variable_name="NW.PCA.TO", no_land=True
):
"""
Download and unzip wealth accounting historical data (1995, 2000, 2005, 2010, 2014)
from World Bank (https://datacatalog.worldbank.org/dataset/wealth-accounting).
Return requested variable for a country (cntry_iso) and a year (ref_year).
Parameters
----------
cntry_iso : str
ISO3-code of country, i.e. "CHN" for China
ref_year : int
reference year
* available in data: 1995, 2000, 2005, 2010, 2014
* other years between 1995 and 2014 are interpolated
* for years outside range, indicator is scaled
proportionally to GDP
variable_name : str
select one variable, i.e.:
'NW.PCA.TO': Produced capital stock of country
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.PCA.PC': Produced capital stock per capita
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.NCA.TO': Total natural capital of country. Natural capital
includes the valuation of fossil fuel energy (oil, gas,
hard and soft coal) and minerals (bauxite, copper, gold,
iron ore, lead, nickel, phosphate, silver, tin, and zinc),
agricultural land (cropland and pastureland),
forests (timber and some nontimber forest products), and
protected areas.
'NW.TOW.TO': Total wealth of country.
Note: Values are measured at market exchange rates in constant 2014 US dollars,
using a country-specific GDP deflator.
no_land : boolean
If True, return produced capital without built-up land value
(applies to 'NW.PCA.*' only). Default: True.
"""
try:
data_file = SYSTEM_DIR.joinpath(FILE_WORLD_BANK_WEALTH_ACC)
if not data_file.is_file():
data_file = SYSTEM_DIR.joinpath(
"Wealth-Accounts_CSV", FILE_WORLD_BANK_WEALTH_ACC
)
if not data_file.is_file():
if not SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").is_dir():
SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").mkdir()
file_down = download_file(WORLD_BANK_WEALTH_ACC)
zip_ref = zipfile.ZipFile(file_down, "r")
zip_ref.extractall(SYSTEM_DIR.joinpath("Wealth-Accounts_CSV"))
zip_ref.close()
Path(file_down).unlink()
LOGGER.debug("Download and unzip complete. Unzipping %s", str(data_file))
data_wealth = pd.read_csv(data_file, sep=",", index_col=None, header=0)
except Exception as err:
> raise type(err)(
"Downloading World Bank Wealth Accounting Data failed: " + str(err)
) from err
E requests.exceptions.SSLError: Downloading World Bank Wealth Accounting Data failed: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/finance.py:476: SSLError
Check warning on line 0 in climada.util.test.test_finance.TestWBWealthAccount
github-actions / Core / Unit Test Results (3.11)
test_pca_CUB_2015_pass (climada.util.test.test_finance.TestWBWealthAccount) failed
tests_xml/tests.xml [took 0s]
Raw output
requests.exceptions.SSLError: Downloading World Bank Wealth Accounting Data failed: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a9e0ab90>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a9e09dd0>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...bSf0F6zUEeDMrsMY2_NZIgmaIojt8IXU; _cfuvid=UldjESYp2v96_SukMiADGVrGSSlwyCWTH53gvw.QJq4-1778676902101-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a9e09dd0>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
> self._validate_conn(conn)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:464:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:1106: in _validate_conn
conn.connect()
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:796: in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:975: in _ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:433: in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:477: in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:517: in wrap_socket
return self.sslsocket_class._create(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1104: in _create
self.do_handshake()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False
@_sslcopydoc
def do_handshake(self, block=False):
self._check_connected()
timeout = self.gettimeout()
try:
if timeout == 0.0 and block:
self.settimeout(None)
> self._sslobj.do_handshake()
E ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1382: SSLCertVerificationError
During handling of the above exception, another exception occurred:
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a9e0ab90>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...bSf0F6zUEeDMrsMY2_NZIgmaIojt8IXU; _cfuvid=UldjESYp2v96_SukMiADGVrGSSlwyCWTH53gvw.QJq4-1778676902101-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
redirect = False, assert_same_host = False
timeout = Timeout(connect=None, read=None, total=None), pool_timeout = None
release_conn = False, chunked = False, body_pos = None, preload_content = False
decode_content = False, response_kw = {}, destination_scheme = None, conn = None
release_this_conn = True, http_tunnel_required = False, err = None
clean_exit = False
def urlopen( # type: ignore[override]
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | bool | int | None = None,
redirect: bool = True,
assert_same_host: bool = True,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
pool_timeout: int | None = None,
release_conn: bool | None = None,
chunked: bool = False,
body_pos: _TYPE_BODY_POSITION | None = None,
preload_content: bool = True,
decode_content: bool = True,
**response_kw: typing.Any,
) -> BaseHTTPResponse:
"""
Get a connection from the pool and perform an HTTP request. This is the
lowest level call for making a request, so you'll need to specify all
the raw details.
.. note::
More commonly, it's appropriate to use a convenience method
such as :meth:`request`.
.. note::
`release_conn` will only behave as expected if
`preload_content=False` because we want to make
`preload_content=False` the default behaviour someday soon without
breaking backwards compatibility.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param redirect:
If True, automatically handle redirects (status codes 301, 302,
303, 307, 308). Each redirect counts as a retry. Disabling retries
will disable redirect, too.
:param assert_same_host:
If ``True``, will make sure that the host of the pool requests is
consistent else will raise HostChangedError. When ``False``, you can
use the pool on an HTTP proxy and request foreign hosts.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param pool_timeout:
If set and the pool is set to block=True, then this method will
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
connection is available within the time period.
:param bool preload_content:
If True, the response's body will be preloaded into memory.
:param bool decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param release_conn:
If False, then the urlopen call will not release the connection
back into the pool once a response is received (but will release if
you read the entire contents of the response such as when
`preload_content=True`). This is useful if you're not preloading
the response's content immediately. You will need to call
``r.release_conn()`` on the response ``r`` to return the connection
back into the pool. If None, it takes the value of ``preload_content``
which defaults to ``True``.
:param bool chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param int body_pos:
Position to seek to in file-like body in the event of a retry or
redirect. Typically this won't need to be set because urllib3 will
auto-populate the value when needed.
"""
# Ensure that the URL we're connecting to is properly encoded
if url.startswith("/"):
# URLs starting with / are inherently schemeless.
url = to_str(_encode_target(url))
destination_scheme = None
else:
parsed_url = parse_url(url)
destination_scheme = parsed_url.scheme
url = to_str(parsed_url.url)
if headers is None:
headers = self.headers
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
if release_conn is None:
release_conn = preload_content
# Check host
if assert_same_host and not self.is_same_host(url):
raise HostChangedError(self, url, retries)
conn = None
# Track whether `conn` needs to be released before
# returning/raising/recursing. Update this variable if necessary, and
# leave `release_conn` constant throughout the function. That way, if
# the function recurses, the original value of `release_conn` will be
# passed down into the recursive call, and its value will be respected.
#
# See issue #651 [1] for details.
#
# [1] <https://github.com/urllib3/urllib3/issues/651>
release_this_conn = release_conn
http_tunnel_required = connection_requires_http_tunnel(
self.proxy, self.proxy_config, destination_scheme
)
# Merge the proxy headers. Only done when not using HTTP CONNECT. We
# have to copy the headers dict so we can safely change it without those
# changes being reflected in anyone else's copy.
if not http_tunnel_required:
headers = headers.copy() # type: ignore[attr-defined]
headers.update(self.proxy_headers) # type: ignore[union-attr]
# Must keep the exception bound to a separate variable or else Python 3
# complains about UnboundLocalError.
err = None
# Keep track of whether we cleanly exited the except block. This
# ensures we do proper cleanup in finally.
clean_exit = False
# Rewind body position, if needed. Record current position
# for future rewinds in the event of a redirect/retry.
body_pos = set_file_position(body, body_pos)
try:
# Request a connection from the queue.
timeout_obj = self._get_timeout(timeout)
conn = self._get_conn(timeout=pool_timeout)
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
# Is this a closed/new connection that requires CONNECT tunnelling?
if self.proxy is not None and http_tunnel_required and conn.is_closed:
try:
self._prepare_proxy(conn)
except (BaseSSLError, OSError, SocketTimeout) as e:
self._raise_timeout(
err=e, url=self.proxy.url, timeout_value=conn.timeout
)
raise
# If we're going to release the connection in ``finally:``, then
# the response doesn't need to know about the connection. Otherwise
# it will also try to release it and we'll have a double-release
# mess.
response_conn = conn if not release_conn else None
# Make the request on the HTTPConnection object
> response = self._make_request(
conn,
method,
url,
timeout=timeout_obj,
body=body,
headers=headers,
chunked=chunked,
retries=retries,
response_conn=response_conn,
preload_content=preload_content,
decode_content=decode_content,
**response_kw,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:788:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a9e0ab90>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a9e09dd0>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...bSf0F6zUEeDMrsMY2_NZIgmaIojt8IXU; _cfuvid=UldjESYp2v96_SukMiADGVrGSSlwyCWTH53gvw.QJq4-1778676902101-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fb9a9e09dd0>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
self._validate_conn(conn)
except (SocketTimeout, BaseSSLError) as e:
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
raise
# _validate_conn() starts the connection to an HTTPS proxy
# so we need to wrap errors with 'ProxyError' here too.
except (
OSError,
NewConnectionError,
TimeoutError,
BaseSSLError,
CertificateError,
SSLError,
) as e:
new_e: Exception = e
if isinstance(e, (BaseSSLError, CertificateError)):
new_e = SSLError(e)
# If the connection didn't successfully connect to it's proxy
# then there
if isinstance(
new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
> raise new_e
E urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:488: SSLError
The above exception was the direct cause of the following exception:
self = <requests.adapters.HTTPAdapter object at 0x7fb9a3211450>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
> resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:696:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:842: in urlopen
retries = retries.increment(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = Retry(total=0, connect=None, read=False, redirect=None, status=None)
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
response = None
error = SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)'))
_pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fb9a9e0ab90>
_stacktrace = <traceback object at 0x7fba241de340>
def increment(
self,
method: str | None = None,
url: str | None = None,
response: BaseHTTPResponse | None = None,
error: Exception | None = None,
_pool: ConnectionPool | None = None,
_stacktrace: TracebackType | None = None,
) -> Self:
"""Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
return a response.
:type response: :class:`~urllib3.response.BaseHTTPResponse`
:param Exception error: An error encountered during the request, or
None if the response was received successfully.
:return: A new ``Retry`` object.
"""
if self.total is False and error:
# Disabled, indicate to re-raise the error.
raise reraise(type(error), error, _stacktrace)
total = self.total
if total is not None:
total -= 1
connect = self.connect
read = self.read
redirect = self.redirect
status_count = self.status
other = self.other
cause = "unknown"
status = None
redirect_location = None
if error and self._is_connection_error(error):
# Connect retry?
if connect is False:
raise reraise(type(error), error, _stacktrace)
elif connect is not None:
connect -= 1
elif error and self._is_read_error(error):
# Read retry?
if read is False or method is None or not self._is_method_retryable(method):
raise reraise(type(error), error, _stacktrace)
elif read is not None:
read -= 1
elif error:
# Other retry?
if other is not None:
other -= 1
elif response and response.get_redirect_location():
# Redirect retry?
if redirect is not None:
redirect -= 1
cause = "too many redirects"
response_redirect_location = response.get_redirect_location()
if response_redirect_location:
redirect_location = response_redirect_location
status = response.status
else:
# Incrementing because of a server error like a 500 in
# status_forcelist and the given method is in the allowed_methods
cause = ResponseError.GENERIC_ERROR
if response and response.status:
if status_count is not None:
status_count -= 1
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
status = response.status
history = self.history + (
RequestHistory(method, url, error, status, redirect_location),
)
new_retry = self.new(
total=total,
connect=connect,
read=read,
redirect=redirect,
status=status_count,
other=other,
history=history,
)
if new_retry.is_exhausted():
reason = error or ResponseError(cause)
> raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/retry.py:543: MaxRetryError
During handling of the above exception, another exception occurred:
url = 'https://databank.worldbank.org/data/download/Wealth-Accounts_CSV.zip'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
> req_file = requests.get(url, stream=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/files_handler.py:93:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:87: in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:71: in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:651: in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in send
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in <listcomp>
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:292: in resolve_redirects
resp = self.send(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:784: in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <requests.adapters.HTTPAdapter object at 0x7fb9a3211450>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
except (ProtocolError, OSError) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
> raise SSLError(e, request=request)
E requests.exceptions.SSLError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:727: SSLError
The above exception was the direct cause of the following exception:
cntry_iso = 'CUB', ref_year = 2015, variable_name = 'NW.PCA.TO', no_land = 1
def world_bank_wealth_account(
cntry_iso, ref_year, variable_name="NW.PCA.TO", no_land=True
):
"""
Download and unzip wealth accounting historical data (1995, 2000, 2005, 2010, 2014)
from World Bank (https://datacatalog.worldbank.org/dataset/wealth-accounting).
Return requested variable for a country (cntry_iso) and a year (ref_year).
Parameters
----------
cntry_iso : str
ISO3-code of country, i.e. "CHN" for China
ref_year : int
reference year
* available in data: 1995, 2000, 2005, 2010, 2014
* other years between 1995 and 2014 are interpolated
* for years outside range, indicator is scaled
proportionally to GDP
variable_name : str
select one variable, i.e.:
'NW.PCA.TO': Produced capital stock of country
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.PCA.PC': Produced capital stock per capita
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.NCA.TO': Total natural capital of country. Natural capital
includes the valuation of fossil fuel energy (oil, gas,
hard and soft coal) and minerals (bauxite, copper, gold,
iron ore, lead, nickel, phosphate, silver, tin, and zinc),
agricultural land (cropland and pastureland),
forests (timber and some nontimber forest products), and
protected areas.
'NW.TOW.TO': Total wealth of country.
Note: Values are measured at market exchange rates in constant 2014 US dollars,
using a country-specific GDP deflator.
no_land : boolean
If True, return produced capital without built-up land value
(applies to 'NW.PCA.*' only). Default: True.
"""
try:
data_file = SYSTEM_DIR.joinpath(FILE_WORLD_BANK_WEALTH_ACC)
if not data_file.is_file():
data_file = SYSTEM_DIR.joinpath(
"Wealth-Accounts_CSV", FILE_WORLD_BANK_WEALTH_ACC
)
if not data_file.is_file():
if not SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").is_dir():
SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").mkdir()
> file_down = download_file(WORLD_BANK_WEALTH_ACC)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/finance.py:467:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://databank.worldbank.org/data/download/Wealth-Accounts_CSV.zip'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
req_file = requests.get(url, stream=True)
except IOError as ioe:
> raise type(ioe)("Check URL and internet connection: " + str(ioe)) from ioe
E requests.exceptions.SSLError: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/files_handler.py:95: SSLError
The above exception was the direct cause of the following exception:
self = <climada.util.test.test_finance.TestWBWealthAccount testMethod=test_pca_CUB_2015_pass>
def test_pca_CUB_2015_pass(self):
"""Test Processed Capital value Cuba 2015 (missing value)."""
ref_year = 2015
cntry_iso = "CUB"
> wb_year, wb_val, q = world_bank_wealth_account(cntry_iso, ref_year, no_land=1)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/test/test_finance.py:257:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cntry_iso = 'CUB', ref_year = 2015, variable_name = 'NW.PCA.TO', no_land = 1
def world_bank_wealth_account(
cntry_iso, ref_year, variable_name="NW.PCA.TO", no_land=True
):
"""
Download and unzip wealth accounting historical data (1995, 2000, 2005, 2010, 2014)
from World Bank (https://datacatalog.worldbank.org/dataset/wealth-accounting).
Return requested variable for a country (cntry_iso) and a year (ref_year).
Parameters
----------
cntry_iso : str
ISO3-code of country, i.e. "CHN" for China
ref_year : int
reference year
* available in data: 1995, 2000, 2005, 2010, 2014
* other years between 1995 and 2014 are interpolated
* for years outside range, indicator is scaled
proportionally to GDP
variable_name : str
select one variable, i.e.:
'NW.PCA.TO': Produced capital stock of country
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.PCA.PC': Produced capital stock per capita
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.NCA.TO': Total natural capital of country. Natural capital
includes the valuation of fossil fuel energy (oil, gas,
hard and soft coal) and minerals (bauxite, copper, gold,
iron ore, lead, nickel, phosphate, silver, tin, and zinc),
agricultural land (cropland and pastureland),
forests (timber and some nontimber forest products), and
protected areas.
'NW.TOW.TO': Total wealth of country.
Note: Values are measured at market exchange rates in constant 2014 US dollars,
using a country-specific GDP deflator.
no_land : boolean
If True, return produced capital without built-up land value
(applies to 'NW.PCA.*' only). Default: True.
"""
try:
data_file = SYSTEM_DIR.joinpath(FILE_WORLD_BANK_WEALTH_ACC)
if not data_file.is_file():
data_file = SYSTEM_DIR.joinpath(
"Wealth-Accounts_CSV", FILE_WORLD_BANK_WEALTH_ACC
)
if not data_file.is_file():
if not SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").is_dir():
SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").mkdir()
file_down = download_file(WORLD_BANK_WEALTH_ACC)
zip_ref = zipfile.ZipFile(file_down, "r")
zip_ref.extractall(SYSTEM_DIR.joinpath("Wealth-Accounts_CSV"))
zip_ref.close()
Path(file_down).unlink()
LOGGER.debug("Download and unzip complete. Unzipping %s", str(data_file))
data_wealth = pd.read_csv(data_file, sep=",", index_col=None, header=0)
except Exception as err:
> raise type(err)(
"Downloading World Bank Wealth Accounting Data failed: " + str(err)
) from err
E requests.exceptions.SSLError: Downloading World Bank Wealth Accounting Data failed: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/finance.py:476: SSLError
Check warning on line 0 in climada.util.test.test_finance.TestWBWealthAccount
github-actions / Core / Unit Test Results (3.11)
test_pca_DEU_2010_pass (climada.util.test.test_finance.TestWBWealthAccount) failed
tests_xml/tests.xml [took 0s]
Raw output
requests.exceptions.SSLError: Downloading World Bank Wealth Accounting Data failed: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fba24156250>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fba24157050>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...uBCMzbqt_rpEIrRe28IxkitKKCuT9mnM; _cfuvid=RslpueJ2DeWIkA4b70Qf9kxyvlHOM2GzDuK6Q32VDZQ-1778676902534-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fba24157050>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
> self._validate_conn(conn)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:464:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:1106: in _validate_conn
conn.connect()
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:796: in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:975: in _ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:433: in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:477: in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:517: in wrap_socket
return self.sslsocket_class._create(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1104: in _create
self.do_handshake()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False
@_sslcopydoc
def do_handshake(self, block=False):
self._check_connected()
timeout = self.gettimeout()
try:
if timeout == 0.0 and block:
self.settimeout(None)
> self._sslobj.do_handshake()
E ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1382: SSLCertVerificationError
During handling of the above exception, another exception occurred:
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fba24156250>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...uBCMzbqt_rpEIrRe28IxkitKKCuT9mnM; _cfuvid=RslpueJ2DeWIkA4b70Qf9kxyvlHOM2GzDuK6Q32VDZQ-1778676902534-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
redirect = False, assert_same_host = False
timeout = Timeout(connect=None, read=None, total=None), pool_timeout = None
release_conn = False, chunked = False, body_pos = None, preload_content = False
decode_content = False, response_kw = {}, destination_scheme = None, conn = None
release_this_conn = True, http_tunnel_required = False, err = None
clean_exit = False
def urlopen( # type: ignore[override]
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | bool | int | None = None,
redirect: bool = True,
assert_same_host: bool = True,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
pool_timeout: int | None = None,
release_conn: bool | None = None,
chunked: bool = False,
body_pos: _TYPE_BODY_POSITION | None = None,
preload_content: bool = True,
decode_content: bool = True,
**response_kw: typing.Any,
) -> BaseHTTPResponse:
"""
Get a connection from the pool and perform an HTTP request. This is the
lowest level call for making a request, so you'll need to specify all
the raw details.
.. note::
More commonly, it's appropriate to use a convenience method
such as :meth:`request`.
.. note::
`release_conn` will only behave as expected if
`preload_content=False` because we want to make
`preload_content=False` the default behaviour someday soon without
breaking backwards compatibility.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param redirect:
If True, automatically handle redirects (status codes 301, 302,
303, 307, 308). Each redirect counts as a retry. Disabling retries
will disable redirect, too.
:param assert_same_host:
If ``True``, will make sure that the host of the pool requests is
consistent else will raise HostChangedError. When ``False``, you can
use the pool on an HTTP proxy and request foreign hosts.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param pool_timeout:
If set and the pool is set to block=True, then this method will
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
connection is available within the time period.
:param bool preload_content:
If True, the response's body will be preloaded into memory.
:param bool decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param release_conn:
If False, then the urlopen call will not release the connection
back into the pool once a response is received (but will release if
you read the entire contents of the response such as when
`preload_content=True`). This is useful if you're not preloading
the response's content immediately. You will need to call
``r.release_conn()`` on the response ``r`` to return the connection
back into the pool. If None, it takes the value of ``preload_content``
which defaults to ``True``.
:param bool chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param int body_pos:
Position to seek to in file-like body in the event of a retry or
redirect. Typically this won't need to be set because urllib3 will
auto-populate the value when needed.
"""
# Ensure that the URL we're connecting to is properly encoded
if url.startswith("/"):
# URLs starting with / are inherently schemeless.
url = to_str(_encode_target(url))
destination_scheme = None
else:
parsed_url = parse_url(url)
destination_scheme = parsed_url.scheme
url = to_str(parsed_url.url)
if headers is None:
headers = self.headers
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
if release_conn is None:
release_conn = preload_content
# Check host
if assert_same_host and not self.is_same_host(url):
raise HostChangedError(self, url, retries)
conn = None
# Track whether `conn` needs to be released before
# returning/raising/recursing. Update this variable if necessary, and
# leave `release_conn` constant throughout the function. That way, if
# the function recurses, the original value of `release_conn` will be
# passed down into the recursive call, and its value will be respected.
#
# See issue #651 [1] for details.
#
# [1] <https://github.com/urllib3/urllib3/issues/651>
release_this_conn = release_conn
http_tunnel_required = connection_requires_http_tunnel(
self.proxy, self.proxy_config, destination_scheme
)
# Merge the proxy headers. Only done when not using HTTP CONNECT. We
# have to copy the headers dict so we can safely change it without those
# changes being reflected in anyone else's copy.
if not http_tunnel_required:
headers = headers.copy() # type: ignore[attr-defined]
headers.update(self.proxy_headers) # type: ignore[union-attr]
# Must keep the exception bound to a separate variable or else Python 3
# complains about UnboundLocalError.
err = None
# Keep track of whether we cleanly exited the except block. This
# ensures we do proper cleanup in finally.
clean_exit = False
# Rewind body position, if needed. Record current position
# for future rewinds in the event of a redirect/retry.
body_pos = set_file_position(body, body_pos)
try:
# Request a connection from the queue.
timeout_obj = self._get_timeout(timeout)
conn = self._get_conn(timeout=pool_timeout)
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
# Is this a closed/new connection that requires CONNECT tunnelling?
if self.proxy is not None and http_tunnel_required and conn.is_closed:
try:
self._prepare_proxy(conn)
except (BaseSSLError, OSError, SocketTimeout) as e:
self._raise_timeout(
err=e, url=self.proxy.url, timeout_value=conn.timeout
)
raise
# If we're going to release the connection in ``finally:``, then
# the response doesn't need to know about the connection. Otherwise
# it will also try to release it and we'll have a double-release
# mess.
response_conn = conn if not release_conn else None
# Make the request on the HTTPConnection object
> response = self._make_request(
conn,
method,
url,
timeout=timeout_obj,
body=body,
headers=headers,
chunked=chunked,
retries=retries,
response_conn=response_conn,
preload_content=preload_content,
decode_content=decode_content,
**response_kw,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:788:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fba24156250>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fba24157050>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...uBCMzbqt_rpEIrRe28IxkitKKCuT9mnM; _cfuvid=RslpueJ2DeWIkA4b70Qf9kxyvlHOM2GzDuK6Q32VDZQ-1778676902534-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fba24157050>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
self._validate_conn(conn)
except (SocketTimeout, BaseSSLError) as e:
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
raise
# _validate_conn() starts the connection to an HTTPS proxy
# so we need to wrap errors with 'ProxyError' here too.
except (
OSError,
NewConnectionError,
TimeoutError,
BaseSSLError,
CertificateError,
SSLError,
) as e:
new_e: Exception = e
if isinstance(e, (BaseSSLError, CertificateError)):
new_e = SSLError(e)
# If the connection didn't successfully connect to it's proxy
# then there
if isinstance(
new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
> raise new_e
E urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:488: SSLError
The above exception was the direct cause of the following exception:
self = <requests.adapters.HTTPAdapter object at 0x7fba24118050>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
> resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:696:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:842: in urlopen
retries = retries.increment(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = Retry(total=0, connect=None, read=False, redirect=None, status=None)
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
response = None
error = SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)'))
_pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fba24156250>
_stacktrace = <traceback object at 0x7fba241579c0>
def increment(
self,
method: str | None = None,
url: str | None = None,
response: BaseHTTPResponse | None = None,
error: Exception | None = None,
_pool: ConnectionPool | None = None,
_stacktrace: TracebackType | None = None,
) -> Self:
"""Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
return a response.
:type response: :class:`~urllib3.response.BaseHTTPResponse`
:param Exception error: An error encountered during the request, or
None if the response was received successfully.
:return: A new ``Retry`` object.
"""
if self.total is False and error:
# Disabled, indicate to re-raise the error.
raise reraise(type(error), error, _stacktrace)
total = self.total
if total is not None:
total -= 1
connect = self.connect
read = self.read
redirect = self.redirect
status_count = self.status
other = self.other
cause = "unknown"
status = None
redirect_location = None
if error and self._is_connection_error(error):
# Connect retry?
if connect is False:
raise reraise(type(error), error, _stacktrace)
elif connect is not None:
connect -= 1
elif error and self._is_read_error(error):
# Read retry?
if read is False or method is None or not self._is_method_retryable(method):
raise reraise(type(error), error, _stacktrace)
elif read is not None:
read -= 1
elif error:
# Other retry?
if other is not None:
other -= 1
elif response and response.get_redirect_location():
# Redirect retry?
if redirect is not None:
redirect -= 1
cause = "too many redirects"
response_redirect_location = response.get_redirect_location()
if response_redirect_location:
redirect_location = response_redirect_location
status = response.status
else:
# Incrementing because of a server error like a 500 in
# status_forcelist and the given method is in the allowed_methods
cause = ResponseError.GENERIC_ERROR
if response and response.status:
if status_count is not None:
status_count -= 1
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
status = response.status
history = self.history + (
RequestHistory(method, url, error, status, redirect_location),
)
new_retry = self.new(
total=total,
connect=connect,
read=read,
redirect=redirect,
status=status_count,
other=other,
history=history,
)
if new_retry.is_exhausted():
reason = error or ResponseError(cause)
> raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/retry.py:543: MaxRetryError
During handling of the above exception, another exception occurred:
url = 'https://databank.worldbank.org/data/download/Wealth-Accounts_CSV.zip'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
> req_file = requests.get(url, stream=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/files_handler.py:93:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:87: in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:71: in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:651: in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in send
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in <listcomp>
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:292: in resolve_redirects
resp = self.send(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:784: in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <requests.adapters.HTTPAdapter object at 0x7fba24118050>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
except (ProtocolError, OSError) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
> raise SSLError(e, request=request)
E requests.exceptions.SSLError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:727: SSLError
The above exception was the direct cause of the following exception:
cntry_iso = 'DEU', ref_year = 2010, variable_name = 'NW.PCA.TO', no_land = 0
def world_bank_wealth_account(
cntry_iso, ref_year, variable_name="NW.PCA.TO", no_land=True
):
"""
Download and unzip wealth accounting historical data (1995, 2000, 2005, 2010, 2014)
from World Bank (https://datacatalog.worldbank.org/dataset/wealth-accounting).
Return requested variable for a country (cntry_iso) and a year (ref_year).
Parameters
----------
cntry_iso : str
ISO3-code of country, i.e. "CHN" for China
ref_year : int
reference year
* available in data: 1995, 2000, 2005, 2010, 2014
* other years between 1995 and 2014 are interpolated
* for years outside range, indicator is scaled
proportionally to GDP
variable_name : str
select one variable, i.e.:
'NW.PCA.TO': Produced capital stock of country
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.PCA.PC': Produced capital stock per capita
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.NCA.TO': Total natural capital of country. Natural capital
includes the valuation of fossil fuel energy (oil, gas,
hard and soft coal) and minerals (bauxite, copper, gold,
iron ore, lead, nickel, phosphate, silver, tin, and zinc),
agricultural land (cropland and pastureland),
forests (timber and some nontimber forest products), and
protected areas.
'NW.TOW.TO': Total wealth of country.
Note: Values are measured at market exchange rates in constant 2014 US dollars,
using a country-specific GDP deflator.
no_land : boolean
If True, return produced capital without built-up land value
(applies to 'NW.PCA.*' only). Default: True.
"""
try:
data_file = SYSTEM_DIR.joinpath(FILE_WORLD_BANK_WEALTH_ACC)
if not data_file.is_file():
data_file = SYSTEM_DIR.joinpath(
"Wealth-Accounts_CSV", FILE_WORLD_BANK_WEALTH_ACC
)
if not data_file.is_file():
if not SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").is_dir():
SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").mkdir()
> file_down = download_file(WORLD_BANK_WEALTH_ACC)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/finance.py:467:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://databank.worldbank.org/data/download/Wealth-Accounts_CSV.zip'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
req_file = requests.get(url, stream=True)
except IOError as ioe:
> raise type(ioe)("Check URL and internet connection: " + str(ioe)) from ioe
E requests.exceptions.SSLError: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/files_handler.py:95: SSLError
The above exception was the direct cause of the following exception:
self = <climada.util.test.test_finance.TestWBWealthAccount testMethod=test_pca_DEU_2010_pass>
def test_pca_DEU_2010_pass(self):
"""Test Processed Capital value Germany 2010."""
ref_year = 2010
cntry_iso = "DEU"
> wb_year, wb_val, q = world_bank_wealth_account(cntry_iso, ref_year, no_land=0)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/test/test_finance.py:205:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cntry_iso = 'DEU', ref_year = 2010, variable_name = 'NW.PCA.TO', no_land = 0
def world_bank_wealth_account(
cntry_iso, ref_year, variable_name="NW.PCA.TO", no_land=True
):
"""
Download and unzip wealth accounting historical data (1995, 2000, 2005, 2010, 2014)
from World Bank (https://datacatalog.worldbank.org/dataset/wealth-accounting).
Return requested variable for a country (cntry_iso) and a year (ref_year).
Parameters
----------
cntry_iso : str
ISO3-code of country, i.e. "CHN" for China
ref_year : int
reference year
* available in data: 1995, 2000, 2005, 2010, 2014
* other years between 1995 and 2014 are interpolated
* for years outside range, indicator is scaled
proportionally to GDP
variable_name : str
select one variable, i.e.:
'NW.PCA.TO': Produced capital stock of country
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.PCA.PC': Produced capital stock per capita
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.NCA.TO': Total natural capital of country. Natural capital
includes the valuation of fossil fuel energy (oil, gas,
hard and soft coal) and minerals (bauxite, copper, gold,
iron ore, lead, nickel, phosphate, silver, tin, and zinc),
agricultural land (cropland and pastureland),
forests (timber and some nontimber forest products), and
protected areas.
'NW.TOW.TO': Total wealth of country.
Note: Values are measured at market exchange rates in constant 2014 US dollars,
using a country-specific GDP deflator.
no_land : boolean
If True, return produced capital without built-up land value
(applies to 'NW.PCA.*' only). Default: True.
"""
try:
data_file = SYSTEM_DIR.joinpath(FILE_WORLD_BANK_WEALTH_ACC)
if not data_file.is_file():
data_file = SYSTEM_DIR.joinpath(
"Wealth-Accounts_CSV", FILE_WORLD_BANK_WEALTH_ACC
)
if not data_file.is_file():
if not SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").is_dir():
SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").mkdir()
file_down = download_file(WORLD_BANK_WEALTH_ACC)
zip_ref = zipfile.ZipFile(file_down, "r")
zip_ref.extractall(SYSTEM_DIR.joinpath("Wealth-Accounts_CSV"))
zip_ref.close()
Path(file_down).unlink()
LOGGER.debug("Download and unzip complete. Unzipping %s", str(data_file))
data_wealth = pd.read_csv(data_file, sep=",", index_col=None, header=0)
except Exception as err:
> raise type(err)(
"Downloading World Bank Wealth Accounting Data failed: " + str(err)
) from err
E requests.exceptions.SSLError: Downloading World Bank Wealth Accounting Data failed: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/finance.py:476: SSLError
Check warning on line 0 in climada.util.test.test_finance.TestWBWealthAccount
github-actions / Core / Unit Test Results (3.11)
test_tow_IND_1985_pass (climada.util.test.test_finance.TestWBWealthAccount) failed
tests_xml/tests.xml [took 0s]
Raw output
requests.exceptions.SSLError: Downloading World Bank Wealth Accounting Data failed: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fba24317f10>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fba24315d10>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...vHcaOfJWSz_gREGYOXiQzFh7.1mYJE3k; _cfuvid=KSfFgp3U6bNclPYT2kw4k1AZR8S84HRxabFcLjmqLkg-1778676902966-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fba24315d10>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
> self._validate_conn(conn)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:464:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:1106: in _validate_conn
conn.connect()
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:796: in connect
sock_and_verified = _ssl_wrap_socket_and_match_hostname(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connection.py:975: in _ssl_wrap_socket_and_match_hostname
ssl_sock = ssl_wrap_socket(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:433: in ssl_wrap_socket
ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls, server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/ssl_.py:477: in _ssl_wrap_socket_impl
return ssl_context.wrap_socket(sock, server_hostname=server_hostname)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:517: in wrap_socket
return self.sslsocket_class._create(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1104: in _create
self.do_handshake()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <ssl.SSLSocket [closed] fd=-1, family=2, type=1, proto=6>, block = False
@_sslcopydoc
def do_handshake(self, block=False):
self._check_connected()
timeout = self.gettimeout()
try:
if timeout == 0.0 and block:
self.settimeout(None)
> self._sslobj.do_handshake()
E ssl.SSLCertVerificationError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/ssl.py:1382: SSLCertVerificationError
During handling of the above exception, another exception occurred:
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fba24317f10>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...vHcaOfJWSz_gREGYOXiQzFh7.1mYJE3k; _cfuvid=KSfFgp3U6bNclPYT2kw4k1AZR8S84HRxabFcLjmqLkg-1778676902966-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
redirect = False, assert_same_host = False
timeout = Timeout(connect=None, read=None, total=None), pool_timeout = None
release_conn = False, chunked = False, body_pos = None, preload_content = False
decode_content = False, response_kw = {}, destination_scheme = None, conn = None
release_this_conn = True, http_tunnel_required = False, err = None
clean_exit = False
def urlopen( # type: ignore[override]
self,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | bool | int | None = None,
redirect: bool = True,
assert_same_host: bool = True,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
pool_timeout: int | None = None,
release_conn: bool | None = None,
chunked: bool = False,
body_pos: _TYPE_BODY_POSITION | None = None,
preload_content: bool = True,
decode_content: bool = True,
**response_kw: typing.Any,
) -> BaseHTTPResponse:
"""
Get a connection from the pool and perform an HTTP request. This is the
lowest level call for making a request, so you'll need to specify all
the raw details.
.. note::
More commonly, it's appropriate to use a convenience method
such as :meth:`request`.
.. note::
`release_conn` will only behave as expected if
`preload_content=False` because we want to make
`preload_content=False` the default behaviour someday soon without
breaking backwards compatibility.
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
If ``None`` (default) will retry 3 times, see ``Retry.DEFAULT``. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param redirect:
If True, automatically handle redirects (status codes 301, 302,
303, 307, 308). Each redirect counts as a retry. Disabling retries
will disable redirect, too.
:param assert_same_host:
If ``True``, will make sure that the host of the pool requests is
consistent else will raise HostChangedError. When ``False``, you can
use the pool on an HTTP proxy and request foreign hosts.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param pool_timeout:
If set and the pool is set to block=True, then this method will
block for ``pool_timeout`` seconds and raise EmptyPoolError if no
connection is available within the time period.
:param bool preload_content:
If True, the response's body will be preloaded into memory.
:param bool decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param release_conn:
If False, then the urlopen call will not release the connection
back into the pool once a response is received (but will release if
you read the entire contents of the response such as when
`preload_content=True`). This is useful if you're not preloading
the response's content immediately. You will need to call
``r.release_conn()`` on the response ``r`` to return the connection
back into the pool. If None, it takes the value of ``preload_content``
which defaults to ``True``.
:param bool chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param int body_pos:
Position to seek to in file-like body in the event of a retry or
redirect. Typically this won't need to be set because urllib3 will
auto-populate the value when needed.
"""
# Ensure that the URL we're connecting to is properly encoded
if url.startswith("/"):
# URLs starting with / are inherently schemeless.
url = to_str(_encode_target(url))
destination_scheme = None
else:
parsed_url = parse_url(url)
destination_scheme = parsed_url.scheme
url = to_str(parsed_url.url)
if headers is None:
headers = self.headers
if not isinstance(retries, Retry):
retries = Retry.from_int(retries, redirect=redirect, default=self.retries)
if release_conn is None:
release_conn = preload_content
# Check host
if assert_same_host and not self.is_same_host(url):
raise HostChangedError(self, url, retries)
conn = None
# Track whether `conn` needs to be released before
# returning/raising/recursing. Update this variable if necessary, and
# leave `release_conn` constant throughout the function. That way, if
# the function recurses, the original value of `release_conn` will be
# passed down into the recursive call, and its value will be respected.
#
# See issue #651 [1] for details.
#
# [1] <https://github.com/urllib3/urllib3/issues/651>
release_this_conn = release_conn
http_tunnel_required = connection_requires_http_tunnel(
self.proxy, self.proxy_config, destination_scheme
)
# Merge the proxy headers. Only done when not using HTTP CONNECT. We
# have to copy the headers dict so we can safely change it without those
# changes being reflected in anyone else's copy.
if not http_tunnel_required:
headers = headers.copy() # type: ignore[attr-defined]
headers.update(self.proxy_headers) # type: ignore[union-attr]
# Must keep the exception bound to a separate variable or else Python 3
# complains about UnboundLocalError.
err = None
# Keep track of whether we cleanly exited the except block. This
# ensures we do proper cleanup in finally.
clean_exit = False
# Rewind body position, if needed. Record current position
# for future rewinds in the event of a redirect/retry.
body_pos = set_file_position(body, body_pos)
try:
# Request a connection from the queue.
timeout_obj = self._get_timeout(timeout)
conn = self._get_conn(timeout=pool_timeout)
conn.timeout = timeout_obj.connect_timeout # type: ignore[assignment]
# Is this a closed/new connection that requires CONNECT tunnelling?
if self.proxy is not None and http_tunnel_required and conn.is_closed:
try:
self._prepare_proxy(conn)
except (BaseSSLError, OSError, SocketTimeout) as e:
self._raise_timeout(
err=e, url=self.proxy.url, timeout_value=conn.timeout
)
raise
# If we're going to release the connection in ``finally:``, then
# the response doesn't need to know about the connection. Otherwise
# it will also try to release it and we'll have a double-release
# mess.
response_conn = conn if not release_conn else None
# Make the request on the HTTPConnection object
> response = self._make_request(
conn,
method,
url,
timeout=timeout_obj,
body=body,
headers=headers,
chunked=chunked,
retries=retries,
response_conn=response_conn,
preload_content=preload_content,
decode_content=decode_content,
**response_kw,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:788:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fba24317f10>
conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fba24315d10>
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
body = None
headers = {'User-Agent': 'python-requests/2.34.0', 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept': '*/*', 'Connection': ...vHcaOfJWSz_gREGYOXiQzFh7.1mYJE3k; _cfuvid=KSfFgp3U6bNclPYT2kw4k1AZR8S84HRxabFcLjmqLkg-1778676902966-0.0.1.1-604800000'}
retries = Retry(total=0, connect=None, read=False, redirect=None, status=None)
timeout = Timeout(connect=None, read=None, total=None), chunked = False
response_conn = <HTTPSConnection(host='databankfiles.worldbank.org', port=443) at 0x7fba24315d10>
preload_content = False, decode_content = False, enforce_content_length = True
def _make_request(
self,
conn: BaseHTTPConnection,
method: str,
url: str,
body: _TYPE_BODY | None = None,
headers: typing.Mapping[str, str] | None = None,
retries: Retry | None = None,
timeout: _TYPE_TIMEOUT = _DEFAULT_TIMEOUT,
chunked: bool = False,
response_conn: BaseHTTPConnection | None = None,
preload_content: bool = True,
decode_content: bool = True,
enforce_content_length: bool = True,
) -> BaseHTTPResponse:
"""
Perform a request on a given urllib connection object taken from our
pool.
:param conn:
a connection from one of our connection pools
:param method:
HTTP request method (such as GET, POST, PUT, etc.)
:param url:
The URL to perform the request on.
:param body:
Data to send in the request body, either :class:`str`, :class:`bytes`,
an iterable of :class:`str`/:class:`bytes`, or a file-like object.
:param headers:
Dictionary of custom headers to send, such as User-Agent,
If-None-Match, etc. If None, pool headers are used. If provided,
these headers completely replace any pool-specific headers.
:param retries:
Configure the number of retries to allow before raising a
:class:`~urllib3.exceptions.MaxRetryError` exception.
Pass ``None`` to retry until you receive a response. Pass a
:class:`~urllib3.util.retry.Retry` object for fine-grained control
over different types of retries.
Pass an integer number to retry connection errors that many times,
but no other types of errors. Pass zero to never retry.
If ``False``, then retries are disabled and any exception is raised
immediately. Also, instead of raising a MaxRetryError on redirects,
the redirect response will be returned.
:type retries: :class:`~urllib3.util.retry.Retry`, False, or an int.
:param timeout:
If specified, overrides the default timeout for this one
request. It may be a float (in seconds) or an instance of
:class:`urllib3.util.Timeout`.
:param chunked:
If True, urllib3 will send the body using chunked transfer
encoding. Otherwise, urllib3 will send the body using the standard
content-length form. Defaults to False.
:param response_conn:
Set this to ``None`` if you will handle releasing the connection or
set the connection to have the response release it.
:param preload_content:
If True, the response's body will be preloaded during construction.
:param decode_content:
If True, will attempt to decode the body based on the
'content-encoding' header.
:param enforce_content_length:
Enforce content length checking. Body returned by server must match
value of Content-Length header, if present. Otherwise, raise error.
"""
self.num_requests += 1
timeout_obj = self._get_timeout(timeout)
timeout_obj.start_connect()
conn.timeout = Timeout.resolve_default_timeout(timeout_obj.connect_timeout)
try:
# Trigger any extra validation we need to do.
try:
self._validate_conn(conn)
except (SocketTimeout, BaseSSLError) as e:
self._raise_timeout(err=e, url=url, timeout_value=conn.timeout)
raise
# _validate_conn() starts the connection to an HTTPS proxy
# so we need to wrap errors with 'ProxyError' here too.
except (
OSError,
NewConnectionError,
TimeoutError,
BaseSSLError,
CertificateError,
SSLError,
) as e:
new_e: Exception = e
if isinstance(e, (BaseSSLError, CertificateError)):
new_e = SSLError(e)
# If the connection didn't successfully connect to it's proxy
# then there
if isinstance(
new_e, (OSError, NewConnectionError, TimeoutError, SSLError)
) and (conn and conn.proxy and not conn.has_connected_to_proxy):
new_e = _wrap_proxy_error(new_e, conn.proxy.scheme)
> raise new_e
E urllib3.exceptions.SSLError: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:488: SSLError
The above exception was the direct cause of the following exception:
self = <requests.adapters.HTTPAdapter object at 0x7fba24225cd0>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
> resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:696:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/connectionpool.py:842: in urlopen
retries = retries.increment(
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = Retry(total=0, connect=None, read=False, redirect=None, status=None)
method = 'GET', url = '/public/ddpext_download/Wealth-Accounts_CSV.zip'
response = None
error = SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)'))
_pool = <urllib3.connectionpool.HTTPSConnectionPool object at 0x7fba24317f10>
_stacktrace = <traceback object at 0x7fba24314700>
def increment(
self,
method: str | None = None,
url: str | None = None,
response: BaseHTTPResponse | None = None,
error: Exception | None = None,
_pool: ConnectionPool | None = None,
_stacktrace: TracebackType | None = None,
) -> Self:
"""Return a new Retry object with incremented retry counters.
:param response: A response object, or None, if the server did not
return a response.
:type response: :class:`~urllib3.response.BaseHTTPResponse`
:param Exception error: An error encountered during the request, or
None if the response was received successfully.
:return: A new ``Retry`` object.
"""
if self.total is False and error:
# Disabled, indicate to re-raise the error.
raise reraise(type(error), error, _stacktrace)
total = self.total
if total is not None:
total -= 1
connect = self.connect
read = self.read
redirect = self.redirect
status_count = self.status
other = self.other
cause = "unknown"
status = None
redirect_location = None
if error and self._is_connection_error(error):
# Connect retry?
if connect is False:
raise reraise(type(error), error, _stacktrace)
elif connect is not None:
connect -= 1
elif error and self._is_read_error(error):
# Read retry?
if read is False or method is None or not self._is_method_retryable(method):
raise reraise(type(error), error, _stacktrace)
elif read is not None:
read -= 1
elif error:
# Other retry?
if other is not None:
other -= 1
elif response and response.get_redirect_location():
# Redirect retry?
if redirect is not None:
redirect -= 1
cause = "too many redirects"
response_redirect_location = response.get_redirect_location()
if response_redirect_location:
redirect_location = response_redirect_location
status = response.status
else:
# Incrementing because of a server error like a 500 in
# status_forcelist and the given method is in the allowed_methods
cause = ResponseError.GENERIC_ERROR
if response and response.status:
if status_count is not None:
status_count -= 1
cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status)
status = response.status
history = self.history + (
RequestHistory(method, url, error, status, redirect_location),
)
new_retry = self.new(
total=total,
connect=connect,
read=read,
redirect=redirect,
status=status_count,
other=other,
history=history,
)
if new_retry.is_exhausted():
reason = error or ResponseError(cause)
> raise MaxRetryError(_pool, url, reason) from reason # type: ignore[arg-type]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
E urllib3.exceptions.MaxRetryError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/urllib3/util/retry.py:543: MaxRetryError
During handling of the above exception, another exception occurred:
url = 'https://databank.worldbank.org/data/download/Wealth-Accounts_CSV.zip'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
> req_file = requests.get(url, stream=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/files_handler.py:93:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:87: in get
return request("get", url, params=params, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/api.py:71: in request
return session.request(method=method, url=url, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:651: in request
resp = self.send(prep, **send_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in send
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:805: in <listcomp>
history = [resp for resp in gen]
^^^^^^^^^^^^^^^^^^^^^^
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:292: in resolve_redirects
resp = self.send(
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/sessions.py:784: in send
r = adapter.send(request, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
self = <requests.adapters.HTTPAdapter object at 0x7fba24225cd0>
request = <PreparedRequest [GET]>, stream = True, timeout = None, verify = True
cert = None, proxies = OrderedDict()
def send(
self,
request: PreparedRequest,
stream: bool = False,
timeout: _t.TimeoutType = None,
verify: _t.VerifyType = True,
cert: _t.CertType = None,
proxies: dict[str, str] | None = None,
) -> Response:
"""Sends PreparedRequest object. Returns Response object.
:param request: The :class:`PreparedRequest <PreparedRequest>` being sent.
:param stream: (optional) Whether to stream the request content.
:param timeout: (optional) How long to wait for the server to send
data before giving up, as a float, or a :ref:`(connect timeout,
read timeout) <timeouts>` tuple.
:type timeout: float or tuple or urllib3 Timeout object
:param verify: (optional) Either a boolean, in which case it controls whether
we verify the server's TLS certificate, or a string, in which case it
must be a path to a CA bundle to use
:param cert: (optional) Any user-provided SSL certificate to be trusted.
:param proxies: (optional) The proxies dictionary to apply to the request.
:rtype: requests.Response
"""
assert _is_prepared(request)
try:
conn = self.get_connection_with_tls_context(
request, verify, proxies=proxies, cert=cert
)
except LocationValueError as e:
raise InvalidURL(e, request=request)
self.cert_verify(conn, request.url, verify, cert)
url = self.request_url(request, proxies)
self.add_headers(
request,
stream=stream,
timeout=timeout,
verify=verify,
cert=cert,
proxies=proxies,
)
chunked = not (request.body is None or "Content-Length" in request.headers)
if isinstance(timeout, tuple):
try:
connect, read = timeout
resolved_timeout = TimeoutSauce(connect=connect, read=read)
except ValueError:
raise ValueError(
f"Invalid timeout {timeout}. Pass a (connect, read) timeout tuple, "
f"or a single float to set both timeouts to the same value."
)
elif isinstance(timeout, TimeoutSauce):
resolved_timeout = timeout
else:
resolved_timeout = TimeoutSauce(connect=timeout, read=timeout)
try:
resp = conn.urlopen(
method=request.method,
url=url,
body=request.body, # type: ignore[arg-type] # urllib3 stubs don't accept Iterable[bytes | str]
headers=request.headers, # type: ignore[arg-type] # urllib3#3072
redirect=False,
assert_same_host=False,
preload_content=False,
decode_content=False,
retries=self.max_retries,
timeout=resolved_timeout,
chunked=chunked,
)
except (ProtocolError, OSError) as err:
raise ConnectionError(err, request=request)
except MaxRetryError as e:
if isinstance(e.reason, ConnectTimeoutError):
# TODO: Remove this in 3.0.0: see #2811
if not isinstance(e.reason, NewConnectionError):
raise ConnectTimeout(e, request=request)
if isinstance(e.reason, ResponseError):
raise RetryError(e, request=request)
if isinstance(e.reason, _ProxyError):
raise ProxyError(e, request=request)
if isinstance(e.reason, _SSLError):
# This branch is for urllib3 v1.22 and later.
> raise SSLError(e, request=request)
E requests.exceptions.SSLError: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
../../../micromamba/envs/climada_env_3.11/lib/python3.11/site-packages/requests/adapters.py:727: SSLError
The above exception was the direct cause of the following exception:
cntry_iso = 'IND', ref_year = 1985, variable_name = 'NW.TOW.TO', no_land = True
def world_bank_wealth_account(
cntry_iso, ref_year, variable_name="NW.PCA.TO", no_land=True
):
"""
Download and unzip wealth accounting historical data (1995, 2000, 2005, 2010, 2014)
from World Bank (https://datacatalog.worldbank.org/dataset/wealth-accounting).
Return requested variable for a country (cntry_iso) and a year (ref_year).
Parameters
----------
cntry_iso : str
ISO3-code of country, i.e. "CHN" for China
ref_year : int
reference year
* available in data: 1995, 2000, 2005, 2010, 2014
* other years between 1995 and 2014 are interpolated
* for years outside range, indicator is scaled
proportionally to GDP
variable_name : str
select one variable, i.e.:
'NW.PCA.TO': Produced capital stock of country
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.PCA.PC': Produced capital stock per capita
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.NCA.TO': Total natural capital of country. Natural capital
includes the valuation of fossil fuel energy (oil, gas,
hard and soft coal) and minerals (bauxite, copper, gold,
iron ore, lead, nickel, phosphate, silver, tin, and zinc),
agricultural land (cropland and pastureland),
forests (timber and some nontimber forest products), and
protected areas.
'NW.TOW.TO': Total wealth of country.
Note: Values are measured at market exchange rates in constant 2014 US dollars,
using a country-specific GDP deflator.
no_land : boolean
If True, return produced capital without built-up land value
(applies to 'NW.PCA.*' only). Default: True.
"""
try:
data_file = SYSTEM_DIR.joinpath(FILE_WORLD_BANK_WEALTH_ACC)
if not data_file.is_file():
data_file = SYSTEM_DIR.joinpath(
"Wealth-Accounts_CSV", FILE_WORLD_BANK_WEALTH_ACC
)
if not data_file.is_file():
if not SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").is_dir():
SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").mkdir()
> file_down = download_file(WORLD_BANK_WEALTH_ACC)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
climada/util/finance.py:467:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
url = 'https://databank.worldbank.org/data/download/Wealth-Accounts_CSV.zip'
download_dir = None, overwrite = True
def download_file(url, download_dir=None, overwrite=True):
"""Download file from url to given target folder and provide full path of the downloaded file.
Parameters
----------
url : str
url containing data to download
download_dir : Path or str, optional
the parent directory of the eventually downloaded file
default: local_data.save_dir as defined in climada.conf
overwrite : bool, optional
whether or not an already existing file at the target location should be overwritten,
by default True
Returns
-------
str
the full path to the eventually downloaded file
"""
file_name = url.split("/")[-1]
if file_name.strip() == "":
raise ValueError(f"cannot download {url} as a file")
download_path = (
CONFIG.local_data.save_dir.dir() if download_dir is None else Path(download_dir)
)
file_path = download_path.absolute().joinpath(file_name)
if file_path.exists():
if not file_path.is_file() or not overwrite:
raise FileExistsError(f"cannot download to {file_path}")
try:
req_file = requests.get(url, stream=True)
except IOError as ioe:
> raise type(ioe)("Check URL and internet connection: " + str(ioe)) from ioe
E requests.exceptions.SSLError: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/files_handler.py:95: SSLError
The above exception was the direct cause of the following exception:
self = <climada.util.test.test_finance.TestWBWealthAccount testMethod=test_tow_IND_1985_pass>
def test_tow_IND_1985_pass(self):
"""Test Total Wealth value India 1985 (outside year range)."""
ref_year = 1985
cntry_iso = "IND"
var_name = "NW.TOW.TO"
> wb_year, wb_val, _ = world_bank_wealth_account(
cntry_iso, ref_year, variable_name=var_name
)
climada/util/test/test_finance.py:241:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cntry_iso = 'IND', ref_year = 1985, variable_name = 'NW.TOW.TO', no_land = True
def world_bank_wealth_account(
cntry_iso, ref_year, variable_name="NW.PCA.TO", no_land=True
):
"""
Download and unzip wealth accounting historical data (1995, 2000, 2005, 2010, 2014)
from World Bank (https://datacatalog.worldbank.org/dataset/wealth-accounting).
Return requested variable for a country (cntry_iso) and a year (ref_year).
Parameters
----------
cntry_iso : str
ISO3-code of country, i.e. "CHN" for China
ref_year : int
reference year
* available in data: 1995, 2000, 2005, 2010, 2014
* other years between 1995 and 2014 are interpolated
* for years outside range, indicator is scaled
proportionally to GDP
variable_name : str
select one variable, i.e.:
'NW.PCA.TO': Produced capital stock of country
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.PCA.PC': Produced capital stock per capita
incl. manufactured or built assets such as machinery,
equipment, and physical structures
and value of built-up urban land (24% mark-up)
'NW.NCA.TO': Total natural capital of country. Natural capital
includes the valuation of fossil fuel energy (oil, gas,
hard and soft coal) and minerals (bauxite, copper, gold,
iron ore, lead, nickel, phosphate, silver, tin, and zinc),
agricultural land (cropland and pastureland),
forests (timber and some nontimber forest products), and
protected areas.
'NW.TOW.TO': Total wealth of country.
Note: Values are measured at market exchange rates in constant 2014 US dollars,
using a country-specific GDP deflator.
no_land : boolean
If True, return produced capital without built-up land value
(applies to 'NW.PCA.*' only). Default: True.
"""
try:
data_file = SYSTEM_DIR.joinpath(FILE_WORLD_BANK_WEALTH_ACC)
if not data_file.is_file():
data_file = SYSTEM_DIR.joinpath(
"Wealth-Accounts_CSV", FILE_WORLD_BANK_WEALTH_ACC
)
if not data_file.is_file():
if not SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").is_dir():
SYSTEM_DIR.joinpath("Wealth-Accounts_CSV").mkdir()
file_down = download_file(WORLD_BANK_WEALTH_ACC)
zip_ref = zipfile.ZipFile(file_down, "r")
zip_ref.extractall(SYSTEM_DIR.joinpath("Wealth-Accounts_CSV"))
zip_ref.close()
Path(file_down).unlink()
LOGGER.debug("Download and unzip complete. Unzipping %s", str(data_file))
data_wealth = pd.read_csv(data_file, sep=",", index_col=None, header=0)
except Exception as err:
> raise type(err)(
"Downloading World Bank Wealth Accounting Data failed: " + str(err)
) from err
E requests.exceptions.SSLError: Downloading World Bank Wealth Accounting Data failed: Check URL and internet connection: HTTPSConnectionPool(host='databankfiles.worldbank.org', port=443): Max retries exceeded with url: /public/ddpext_download/Wealth-Accounts_CSV.zip (Caused by SSLError(SSLCertVerificationError(1, '[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: certificate has expired (_ssl.c:1016)')))
climada/util/finance.py:476: SSLError
Loading