Merge branch 'develop' into feature/interpolate_exceedance

Author: ValentinGebhart

CHANGELOG.md

@@ -27,7 +27,7 @@ Code freeze date: YYYY-MM-DD
 
 ### Removed
 - `climada.util.earth_engine.py` Google Earth Engine methods did not facilitate direct use of GEE data in CLIMADA. Code was relocated to [climada-snippets](https://github.com/CLIMADA-project/climada-snippets). [#1109](https://github.com/CLIMADA-project/climada_python/pull/1109)
-- `doc.climada_util_earth_engine.ipynb` Tutorial about GEE not relevant to CLIMADA Core. Tutorial notebook was relocated to [climada-snippets](https://github.com/CLIMADA-project/climada-snippets).[#1109](https://github.com/CLIMADA-project/climada_python/pull/1109)
+- `doc.climada_util_earth_engine.ipynb` Tutorial about GEE not relevant to CLIMADA Core. Tutorial notebook was relocated to [climada-snippets](https://github.com/CLIMADA-project/climada-snippets). [#1109](https://github.com/CLIMADA-project/climada_python/pull/1109)
 
 ## 6.1.0
 
@@ -160,9 +160,9 @@ Removed:
 - `climada.entity.impact_funcs.trop_cyclone.ImpfSetTropCyclone.get_impf_id_regions_per_countries` function [#1034](https://github.com/CLIMADA-project/climada_python/pull/1034)
 - `climada.hazard.tc_tracks.BasinBoundsStorm` Enum class `climada.hazard.tc_tracks.subset_by_basin` function [#1031](https://github.com/CLIMADA-project/climada_python/pull/1031)
 - `climada.hazard.tc_tracks.TCTracks.subset_years` function [#1023](https://github.com/CLIMADA-project/climada_python/pull/1023)
--`climada.hazard.tc_tracks.compute_track_density` function, `climada.hazard.tc_tracks.compute_genesis_density` function, `climada.hazard.plot.plot_track_density` function
+- `climada.hazard.tc_tracks.compute_track_density` function, `climada.hazard.tc_tracks.compute_genesis_density` function, `climada.hazard.plot.plot_track_density` function
  [#1003](https://github.com/CLIMADA-project/climada_python/pull/1003)
--`climada.hazard.tc_tracks.TCTracks.from_FAST` function, add Australia basin (AU) [#993](https://github.com/CLIMADA-project/climada_python/pull/993)
+- `climada.hazard.tc_tracks.TCTracks.from_FAST` function, add Australia basin (AU) [#993](https://github.com/CLIMADA-project/climada_python/pull/993)
 - Add `osm-flex` package to CLIMADA core [#981](https://github.com/CLIMADA-project/climada_python/pull/981)
 - `doc.tutorial.climada_entity_Exposures_osm.ipynb` tutorial explaining how to use `osm-flex` with CLIMADA
 - `climada.util.coordinates.bounding_box_global` function [#980](https://github.com/CLIMADA-project/climada_python/pull/980)

climada/data/system/entity_template.xlsx

@@ -41,22 +41,18 @@
 from climada.entity import Exposures, ImpactFunc, ImpactFuncSet
 from climada.entity.entity_def import Entity
 from climada.hazard import Hazard
-from climada.util.api_client import Client
+from climada.test import get_test_file
 from climada.util.constants import (
     ENT_DEMO_FUTURE,
     ENT_DEMO_TODAY,
-    EXP_DEMO_H5,
     HAZ_DEMO_H5,
     TEST_UNC_OUTPUT_COSTBEN,
     TEST_UNC_OUTPUT_IMPACT,
 )
 
-test_unc_output_impact = Client().get_dataset_file(
-    name=TEST_UNC_OUTPUT_IMPACT, status="test_dataset"
-)
-test_unc_output_costben = Client().get_dataset_file(
-    name=TEST_UNC_OUTPUT_COSTBEN, status="test_dataset"
-)
+EXP_DEMO_H5 = get_test_file("exp_demo_today", file_format="hdf5")
+test_unc_output_impact = get_test_file(TEST_UNC_OUTPUT_IMPACT)
+test_unc_output_costben = get_test_file(TEST_UNC_OUTPUT_COSTBEN)
 
 
 def impf_dem(x_paa=1, x_mdd=1):
@@ -578,7 +574,7 @@ def test_calc_sensitivity_all_pass(self):
                 "sensitivity_kwargs": {"S": 10, "seed": 12345},
                 "test_param_name": ["x_exp", 0],
                 "test_si_name": ["CV", 16],
-                "test_si_value": [0.25000, 2],
+                "test_si_value": [0.250000, 2],
             },
             "hdmr": {
                 "sampling_method": "saltelli",
@@ -587,7 +583,7 @@ def test_calc_sensitivity_all_pass(self):
                 "sensitivity_kwargs": {},
                 "test_param_name": ["x_exp", 2],
                 "test_si_name": ["Sa", 4],
-                "test_si_value": [0.004658, 3],
+                "test_si_value": [0.004649, 3],
             },
             "ff": {
                 "sampling_method": "ff",
@@ -618,7 +614,7 @@ def test_calc_sensitivity_all_pass(self):
                 },
                 "test_param_name": ["x_exp", 0],
                 "test_si_name": ["dgsm", 8],
-                "test_si_value": [1.697516e-01, 9],
+                "test_si_value": [0.1697516, 9],
             },
             "fast": {
                 "sampling_method": "fast_sampler",
@@ -627,7 +623,7 @@ def test_calc_sensitivity_all_pass(self):
                 "sensitivity_kwargs": {"M": 4, "seed": 12345},
                 "test_param_name": ["x_exp", 0],
                 "test_si_name": ["S1_conf", 8],
-                "test_si_value": [0.671396, 1],
+                "test_si_value": [0.671546, 1],
             },
             "rbd_fast": {
                 "sampling_method": "saltelli",
@@ -636,7 +632,7 @@ def test_calc_sensitivity_all_pass(self):
                 "sensitivity_kwargs": {"M": 4, "seed": 12345},
                 "test_param_name": ["x_exp", 0],
                 "test_si_name": ["S1_conf", 4],
-                "test_si_value": [0.152609, 4],
+                "test_si_value": [0.129919, 4],
             },
             "morris": {
                 "sampling_method": "morris",
@@ -645,7 +641,7 @@ def test_calc_sensitivity_all_pass(self):
                 "sensitivity_kwargs": {},
                 "test_param_name": ["x_exp", 0],
                 "test_si_name": ["mu", 1],
-                "test_si_value": [5066460029.63911, 8],
+                "test_si_value": [7935400297.813827, 8],
             },
         }
 
@@ -700,7 +696,7 @@ def test_sensitivity_method(
                 haz_unc,
                 sensitivity_method,
                 method_params,
-                places=2 if sensitivity_method == "rbd_fast" else 5,
+                places=5,
             )
 
 

climada/engine/unsequa/test/test_unsequa.py

@@ -403,6 +403,9 @@ def __init__(
 
         self.description = self._consolidate(meta, "description", description)
         self.ref_year = self._consolidate(meta, "ref_year", ref_year, DEF_REF_YEAR)
+
+        if geodata.shape[0] > 0:
+            value_unit = self._consolidate(geodata.iloc[0], "value_unit", value_unit)
         self.value_unit = self._consolidate(
             meta, "value_unit", value_unit, DEF_VALUE_UNIT
         )
@@ -644,15 +647,17 @@ def assign_centroids(
 
         Caution: nearest neighbourg matching can introduce serious artefacts
         such as:
-            - exposure and hazard centroids with shifted grids can lead
-            to systematically wrong assignements.
-            - hazard centroids covering larger areas than exposures may lead
-            to sub-optimal matching if the threshold is too large
-            - projected crs often diverge at the anti-meridian and close points
-            on either side will be at a large distance. For proper handling
-            of the anti-meridian please use degree coordinates in EPSG:4326.
-            This might be relevant for countries like the Fidji or the US that
-            cross the anti-meridian.
+
+        - exposure and hazard centroids with shifted grids can lead
+          to systematically wrong assignements.
+        - hazard centroids covering larger areas than exposures may lead
+          to sub-optimal matching if the threshold is too large
+        - projected crs often diverge at the anti-meridian and close points
+          on either side will be at a large distance. For proper handling
+          of the anti-meridian please use degree coordinates in EPSG:4326.
+          This might be relevant for countries like the Fidji or the US that
+          cross the anti-meridian.
+
 
         Users are free to implement their own matching alrogithm and save the
         matching centroid index in the appropriate column ``centr_[hazard.HAZ_TYPE]``.

climada/entity/exposures/base.py

@@ -182,7 +182,7 @@ def test__init__mda_in_kwargs(self):
     def test_read_raster_pass(self):
         """from_raster"""
         exp = Exposures.from_raster(
-            HAZ_DEMO_FL, window=Window(10, 20, 50, 60), attrs={"value_unit": "USD"}
+            HAZ_DEMO_FL, window=Window(10, 20, 50, 60), attrs={"value_unit": "PKR"}
         )
         exp.check()
         self.assertTrue(u_coord.equal_crs(exp.crs, DEF_CRS))
@@ -204,7 +204,7 @@ def test_read_raster_pass(self):
         self.assertAlmostEqual(
             exp.gdf["value"].values.reshape((60, 50))[25, 12], 0.056825936
         )
-        self.assertEqual(exp.value_unit, "USD")
+        self.assertEqual(exp.value_unit, "PKR")
 
     def test_assign_raster_pass(self):
         """Test assign_centroids with raster hazard"""
@@ -429,16 +429,23 @@ def test_read_template_pass(self):
         exp_df = Exposures(df)
         # set metadata
         exp_df.ref_year = 2020
-        exp_df.value_unit = "XSD"
+        exp_df.value_unit = "PAK"
         exp_df.check()
 
+    def test_handling_unit_conflicts_pass(self):
+        """Check that the value_unit is correctly set when there are conflicting value_unit definitions in the data frame and the meta attribute."""
+        df = pd.read_excel(ENT_TEMPLATE_XLS)
+        exp_df = Exposures(df, meta={"value_unit": "XSD"}, value_unit="XSD")
+        exp_df.check()
+        self.assertEqual(exp_df.value_unit, "XSD")
+        with self.assertRaises(ValueError) as cm:
+            exp_df = Exposures(df, meta={"value_unit": "XSD"}, value_unit="PAK")
+
     def test_io_hdf5_pass(self):
         """write and read hdf5"""
-        exp = Exposures(pd.read_excel(ENT_TEMPLATE_XLS), crs="epsg:32632")
-
-        # set metadata
-        exp.ref_year = 2020
-        exp.value_unit = "XSD"
+        exp = Exposures(
+            pd.read_excel(ENT_TEMPLATE_XLS), crs="epsg:32632", ref_year=2020
+        )
 
         # add another geometry column
         exp.data["geocol2"] = exp.data.geometry.copy(deep=True)

climada/entity/exposures/test/test_base.py

@@ -36,10 +36,12 @@
 from climada.entity.measures.measure_set import MeasureSet
 from climada.hazard.base import Hazard
 from climada.test import get_test_file
-from climada.util.constants import EXP_DEMO_H5, HAZ_DEMO_H5
+from climada.util.constants import HAZ_DEMO_H5
 
 DATA_DIR = CONFIG.measures.test_data.dir()
 
+EXP_DEMO_H5 = get_test_file("exp_demo_today", file_format="hdf5")
+
 HAZ_TEST_TC: Path = get_test_file("test_tc_florida", file_format="hdf5")
 """
 Hazard test file from Data API: Hurricanes from 1851 to 2011 over Florida with 100 centroids.

climada/entity/measures/test/test_base.py

@@ -2281,7 +2281,9 @@ def compute_track_density(
         wind_min: float = None,
         wind_max: float = None,
     ) -> tuple[np.ndarray, np.ndarray, np.ndarray]:
-        """Compute tropical cyclone track density. Before using this function,
+        """Compute tropical cyclone track density.
+
+        Before using this function,
         apply the same temporal resolution to all tracks by calling :py:meth:`equal_timestep` on
         the TCTrack object. Due to the computational cost of the this function, it is not
         recommended to use a grid resolution higher tha 0.1°. Also note that the time step (in hours)
@@ -2291,7 +2293,7 @@ def compute_track_density(
         it returns the absolute count per bin. To plot the output of this function,
         use :py:meth:`plot_track_density`.
 
-        Parameters:
+        Parameters
         ----------
         tc_track: TCTracks object
             track object containing a list of all tracks
@@ -2310,7 +2312,8 @@ def compute_track_density(
         wind_max: float (optional), default: None
             Maximal wind speed below which to select tracks (exclusive if wind_min is also provided,
             otherwise inclusive).
-        Returns:
+
+        Returns
         -------
         hist_count: np.ndarray
             2D matrix containing the absolute count per grid cell of track point.
@@ -2319,8 +2322,8 @@ def compute_track_density(
         lon_bins: np.ndarray
             longitude bins in which the point were counted
 
-        Example:
-        --------
+        Example
+        -------
         >>> tc_tracks = TCTrack.from_ibtracs_netcdf("path_to_file")
         >>> tc_tracks.equal_timestep(time_step_h = 1)
         >>> hist_count, _, _ = compute_track_density(tc_track = tc_tracks, res = 2)

climada/hazard/tc_tracks.py

@@ -1049,7 +1049,7 @@ def _apply_decay_coeffs(track, v_rel, p_rel, land_geom, s_rel):
                 # if there is no further landfall, correct until the end of
                 # the track
                 end_cor = track["time"].size
-            rndn = 0.1 * float(np.abs(np.random.normal(size=1) * 5) + 6)
+            rndn = 0.1 * float(np.abs(np.random.default_rng().normal() * 5) + 6)
             r_diff = (
                 track["central_pressure"][land_sea].values
                 - track["central_pressure"][land_sea - 1].values

climada/hazard/tc_tracks_synth.py

@@ -28,7 +28,7 @@
 import numpy as np
 import xarray as xr
 from pyproj import CRS
-from scipy.sparse import csr_matrix
+from scipy.sparse import csr_array, csr_matrix
 
 from climada.hazard.base import Hazard
 from climada.util.constants import DEF_CRS
@@ -104,8 +104,8 @@ def _assert_default_types(self, hazard):
         self.assertIsInstance(hazard.event_id, np.ndarray)
         self.assertIsInstance(hazard.event_name, list)
         self.assertIsInstance(hazard.frequency, np.ndarray)
-        self.assertIsInstance(hazard.intensity, csr_matrix)
-        self.assertIsInstance(hazard.fraction, csr_matrix)
+        self.assertIsInstance(hazard.intensity, csr_matrix | csr_array)
+        self.assertIsInstance(hazard.fraction, csr_matrix | csr_array)
         self.assertIsInstance(hazard.date, np.ndarray)
 
     def test_load_path(self):
@@ -149,8 +149,11 @@ def _load_and_assert(**kwargs):
     def test_type_error(self):
         """Calling 'from_xarray_raster' with wrong data type should throw"""
         # Passing a DataArray
-        with xr.open_dataset(self.netcdf_path) as dset, self.assertRaisesRegex(
-            TypeError, "This method only supports passing xr.Dataset"
+        with (
+            xr.open_dataset(self.netcdf_path) as dset,
+            self.assertRaisesRegex(
+                TypeError, "This method only supports passing xr.Dataset"
+            ),
         ):
             Hazard.from_xarray_raster(dset["intensity"], "", "")
 

climada/hazard/test/test_xarray.py

@@ -58,7 +58,7 @@ def _to_csr_matrix(array: xr.DataArray) -> sparse.csr_matrix:
         output_dtypes=[array.dtype],
     )
     sparse_coo = array.compute().data  # Load into memory
-    return sparse_coo.tocsr()  # Convert sparse.COO to scipy.sparse.csr_matrix
+    return sparse_coo.tocsr()  # Convert sparse.COO to scipy.sparse.csr_array
 
 
 # Define accessors for xarray DataArrays