Upgrade dependency for testing, fix nexrad S3 archive, cloud option not mandatory

Author: ghiggi

pyproject.toml

@@ -25,7 +25,6 @@ classifiers = [
     "Programming Language :: Python",
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3 :: Only",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
@@ -46,7 +45,7 @@ dependencies = [
     "pandas",
     "tqdm",
 ]
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 dynamic = ["version"]
 
 [tool.setuptools]
@@ -60,6 +59,7 @@ write_to = "radar_api/_version.py"
 
 [project.optional-dependencies]
 dev = ["pre-commit", "loghub",
+       "xarray", "xradar",
        "black[jupyter]", "blackdoc", "codespell", "ruff",
        "pytest", "pytest-cov", "pytest-mock", "pytest-check", "pytest-sugar",
        "pytest-watcher", "deepdiff",

radar_api/etc/network/NEXRAD.yaml

@@ -1,7 +1,7 @@
 network: "NEXRAD"
 description: "NOAA NEXRAD radar network"
 public_data: True
-cloud_directory_pattern: "s3://noaa-nexrad-level2/{time:%Y}/{time:%m}/{time:%d}/{radar:s}"
+cloud_directory_pattern: "s3://unidata-nexrad-level2/{time:%Y}/{time:%m}/{time:%d}/{radar:s}"
 local_directory_pattern: "{base_dir}/NEXRAD/{time:%Y}/{time:%m}/{time:%d}/{time:%H}/{radar:s}"
 filename_patterns:
   - "{radar_acronym:4s}{start_time:%Y%m%d_%H%M%S}_V0{version:1s}"

radar_api/io.py

@@ -228,6 +228,8 @@ def get_directory_pattern(protocol, network):
         directory_pattern = get_network_info(network)["cloud_directory_pattern"]
     else:
         directory_pattern = get_network_info(network)["local_directory_pattern"]
+    if directory_pattern is None:
+        raise NotImplementedError(f"protocol {protocol} is not implemented for {network}.")
     return directory_pattern
 
 

radar_api/search.py

@@ -56,7 +56,7 @@ def get_pattern_shortest_time_component(directory_pattern):
     if any(s in directory_pattern for s in ["{time:%m}", "{time:%b}", "{time:%B}"]):
         return "MS"
     if any(s in directory_pattern for s in ["{time:%Y}", "{time:%y}"]):
-        return "Y"  # Y-DEC
+        return "YE"  # Y-DEC
     raise NotImplementedError
 
 
@@ -67,7 +67,7 @@ def get_list_timesteps(start_time, end_time, freq):
     end = pd.to_datetime(end_time)
 
     # Round start_time and end_time to the frequency resolution
-    # "Y" --> set month and days to 01 and zero out hour, minute, second
+    # "YE" --> set month and days to 01 and zero out hour, minute, second
     # "MS" --> set day to 01 and zero out hour, minute, second
     # "D" -> zero out hour, minute, second
     # "h" -> zero out minute, second
@@ -87,7 +87,7 @@ def get_list_timesteps(start_time, end_time, freq):
             new_start_year = start.year
         start = pd.to_datetime(datetime.datetime(new_start_year, new_start_month, 1))
         end = pd.to_datetime(datetime.datetime(end.year, end.month, 1))
-    elif freq == "Y":  # Y-DEC
+    elif freq == "YE":  # Y-DEC
         start = pd.to_datetime(datetime.datetime(start.year - 1, 1, 1))
         end = pd.to_datetime(datetime.datetime(end.year, 12, 31))
     else:

radar_api/tests/test_io.py

@@ -175,7 +175,10 @@ def test_get_network_filename_patterns(network):
 @pytest.mark.parametrize("network", NETWORKS)
 def test_get_directory_pattern_cloud(network):
     """Test get_directory_pattern for a cloud protocol (e.g. s3)."""
-    pattern = get_directory_pattern(protocol="s3", network=network)
+    try:
+        pattern = get_directory_pattern(protocol="s3", network=network)
+    except NotImplementedError:
+        pytest.skip(f"protocol s3 is not implemented for network {network}")
     assert isinstance(pattern, str)
 
 

radar_api/tests/test_readers.py

@@ -42,7 +42,7 @@
 
 def test_get_simplecache_file():
     """Test file simple caching with fsspec."""
-    filepath = "s3://noaa-nexrad-level2/2023/01/01/KABR/KABR20230101_000142_V06"
+    filepath = "s3://unidata-nexrad-level2/2023/01/01/KABR/KABR20230101_000142_V06"
     file = get_simplecache_file(filepath)
     assert isinstance(file, str)
 

radar_api/tests/test_search.py

@@ -69,7 +69,7 @@ def test_year_component(self):
         """Test get_pattern_shortest_time_component returns 'Y' for pattern with {time:%Y}."""
         pattern = "some_path/{time:%Y}"
         freq = get_pattern_shortest_time_component(pattern)
-        assert freq == "Y"
+        assert freq == "YE"
 
     def test_not_implemented(self):
         """Test get_pattern_shortest_time_component raises NotImplementedError if unknown format."""
@@ -145,7 +145,7 @@ def test_freq_y(self):
         """Test get_list_timesteps with freq='Y' (yearly)."""
         start_time = "2024-06-10"
         end_time = "2026-02-01"
-        times = get_list_timesteps(start_time, end_time, freq="Y")
+        times = get_list_timesteps(start_time, end_time, freq="YE")
         # freq='Y' (YE-DEC) means shift start year back by 1 => 2023,
         assert list(times) == [
             pd.Timestamp("2023-12-31"),
@@ -170,7 +170,7 @@ def test_directories_s3_day(self):
         protocol = "s3"
         base_dir = None
 
-        # directory_pattern for protocol="s3" is 's3://noaa-nexrad-level2/{time:%Y}/{time:%m}/{time:%d}/{radar:s}'
+        # directory_pattern for protocol="s3" is 's3://unidata-nexrad-level2/{time:%Y}/{time:%m}/{time:%d}/{radar:s}'
         # => The last time component is {time:%d} => freq='D'
         # => We'll generate times for 2023-06-30, 2023-07-01, 2023-07-02
 
@@ -184,9 +184,9 @@ def test_directories_s3_day(self):
         )
         expected = [
             # Because the function does start_time - 1 day, then floors to day ...
-            "s3://noaa-nexrad-level2/2023/06/30/KFSD",
-            "s3://noaa-nexrad-level2/2023/07/01/KFSD",
-            "s3://noaa-nexrad-level2/2023/07/02/KFSD",
+            "s3://unidata-nexrad-level2/2023/06/30/KFSD",
+            "s3://unidata-nexrad-level2/2023/07/01/KFSD",
+            "s3://unidata-nexrad-level2/2023/07/02/KFSD",
         ]
         assert paths == expected