Square up caching code to make it easier to read. Room for improvement

Author: jmcvey3

.github/workflows/main.yml

@@ -83,7 +83,7 @@ jobs:
         shell: bash -l {0}
         run: |
           conda activate mhkit-env
-          pip install -e  ".[all,dev]"
+          pip install -e ".[all,dev]"
 
       - name: Prepare non-hindcast API data
         shell: bash -l {0}
@@ -134,7 +134,7 @@ jobs:
         shell: bash -l {0}
         run: |
           conda activate mhkit-env
-          pip install -e  ".[all,dev]"
+          pip install -e ".[all,dev]"
 
       - name: Prepare Wave Hindcast data
         shell: bash -l {0}
@@ -183,7 +183,7 @@ jobs:
         shell: bash -l {0}
         run: |
           conda activate mhkit-env
-          pip install -e  ".[all,dev]"
+          pip install -e ".[all,dev]"
 
       - name: Prepare Wind Hindcast data
         shell: bash -l {0}
@@ -290,7 +290,7 @@ jobs:
         shell: bash -l {0}
         run: |
           python -m pip install --upgrade pip wheel
-          pip install -e  ".[all,dev]"
+          pip install -e ".[all,dev]"
 
       - name: Reinstall h5py and netCDF4 with system libraries
         if: runner.os == 'Linux'
@@ -362,7 +362,7 @@ jobs:
         shell: bash -l {0}
         run: |
           conda activate mhkit-env
-          pip install -e  ".[all,dev]" --no-deps
+          pip install -e ".[all,dev]" --no-deps
 
       - name: Download Wave Hindcast data from artifact
         uses: actions/download-artifact@v4
@@ -514,7 +514,7 @@ jobs:
           needs.prepare-nonhindcast-cache.result == 'success' &&
           needs.prepare-wave-hindcast-cache.result == 'skipped' &&
           needs.prepare-wind-hindcast-cache.result == 'skipped' &&
-          needs.check-changes.outputs.should-run-hindcast == 'false'    
+          needs.check-changes.outputs.should-run-hindcast == 'false'
         ) ||
         (
           needs.prepare-nonhindcast-cache.result == 'success' &&
@@ -584,7 +584,7 @@ jobs:
         shell: bash -l {0}
         run: |
           conda activate mhkit-env
-          pip install -e  ".[all,dev]" --no-deps
+          pip install -e ".[all,dev]" --no-deps
 
       - name: Download non-hindcast data
         uses: actions/download-artifact@v4

mhkit/tests/wave/io/hindcast/test_wind_toolkit.py

@@ -40,7 +40,7 @@ def setUpClass(self):
             index_col="time_index",
             names=["time_index", "windspeed_10m_0", "windspeed_10m_1"],
             header=0,
-            dtype={"windspeed_10m_0": "float32", "windspeed_10m_1": "float32"},
+            dtype={"windspeed_10m_0": "float64", "windspeed_10m_1": "float64"},
         )
         self.ml.index = pd.to_datetime(self.ml.index)
 

mhkit/tests/wave/io/test_cdip.py

@@ -128,14 +128,14 @@ def test_request_parse_workflow_multiyear(self):
         expected_index0 = datetime(year1, 1, 1)
         expected_index_final = datetime(year2, 12, 31)
 
-        wave1D = data["data"]["wave"]
+        wave1D = data["wave"]
         self.assertEqual(wave1D.index[0].floor("D").to_pydatetime(), expected_index0)
 
         self.assertEqual(
             wave1D.index[-1].floor("D").to_pydatetime(), expected_index_final
         )
 
-        for key, wave2D in data["data"]["wave2D"].items():
+        for key, wave2D in data["wave2D"].items():
             self.assertEqual(
                 wave2D.index[0].floor("D").to_pydatetime(), expected_index0
             )
@@ -158,7 +158,7 @@ def test_plot_boxplot(self):
         )
 
         plt.figure()
-        wave.graphics.plot_boxplot(data["data"]["wave"]["waveHs"])
+        wave.graphics.plot_boxplot(data["wave"]["waveHs"])
         plt.savefig(filename, format="png")
         plt.close()
 
@@ -181,9 +181,9 @@ def test_plot_compendium(self):
 
         plt.figure()
         wave.graphics.plot_compendium(
-            data["data"]["wave"]["waveHs"],
-            data["data"]["wave"]["waveTp"],
-            data["data"]["wave"]["waveDp"],
+            data["wave"]["waveHs"],
+            data["wave"]["waveTp"],
+            data["wave"]["waveDp"],
         )
         plt.savefig(filename, format="png")
         plt.close()

mhkit/utils/cache.py

@@ -119,9 +119,18 @@ def _load_cache(file_extension, cache_filepath):
             )
         elif file_extension == ".pkl":
             with open(cache_filepath, "rb") as f:
-                data = pickle.load(f)  # dictionary with 'data' and 'metadata' keys
-
-        return data
+                data_dict = pickle.load(f)  # some format of 'data' and 'metadata'
+                # TODO Must be a better way to handle these if we are writing the cache.
+                if isinstance(data_dict, tuple):
+                    # CDIP call returns dictionary in size 1 tuple
+                    if len(data_dict) == 1 and isinstance(data_dict[0], dict):
+                        data_dict = data_dict[0]
+                    # WPTO hindcast call returns size 2 tuple
+                    if len(data_dict) == 2:
+                        return data_dict[0], data_dict[1]
+                data, metadata = data_dict["data"], data_dict["metadata"]
+
+        return data, metadata
 
     def _write_cache(data, metadata, file_extension, cache_filepath):
         """Store data in the cache file based on the extension."""
@@ -154,7 +163,8 @@ def _write_cache(data, metadata, file_extension, cache_filepath):
     if os.path.isfile(cache_filepath) and (
         cache_content is None or cache_content["data"] is None
     ):
-        return _load_cache(file_extension, cache_filepath) + (cache_filepath,)
+        data, metadata = _load_cache(file_extension, cache_filepath)
+        return data, metadata, cache_filepath
 
     # Store data in cache if provided
     if cache_content and cache_content["data"] is not None:

mhkit/wave/io/cdip.py

@@ -378,20 +378,20 @@ def request_parse_workflow(
                         "write_json": None,
                     },
                 )
-            multiyear_data[year] = year_data["data"]
+            multiyear_data[year] = year_data
 
-        for data_key in year_data["data"].keys():
+        for data_key in year_data.keys():
             if data_key.endswith("2D"):
-                data["data"][data_key] = {}
-                for data_key2D in year_data["data"][data_key].keys():
+                data[data_key] = {}
+                for data_key2D in year_data[data_key].keys():
                     data_list = []
                     for year in years:
                         data2D = multiyear_data[year][data_key][data_key2D]
                         data_list.append(data2D)
-                    data["data"][data_key][data_key2D] = pd.concat(data_list)
+                    data[data_key][data_key2D] = pd.concat(data_list)
             else:
                 data_list = [multiyear_data[year][data_key] for year in years]
-                data["data"][data_key] = pd.concat(data_list)
+                data[data_key] = pd.concat(data_list)
 
     if buoy_name:
         try: