add cdippy lib version info to requests and mock requests in tests (#17)

* add cdippy lib version info to requests

add loggers to cdipnc and url_utils

mock all unit tests

* fix uninitialized variable in mop_data

* fix 'start' dates interpreted as local time rather than utc

Author: haileyajohnson

.gitignore

@@ -4,7 +4,9 @@
 .venv
 .vscode
 **/.coverage
+uv.lock
 
 # build files
 **/*.egg-info
-**/__pycache__
+**/__pycache__
+**/build

cdippy/cdipnc.py

@@ -1,6 +1,7 @@
 from datetime import datetime, timedelta, timezone
 import os
 
+import logging
 import netCDF4
 
 import numpy as np
@@ -12,6 +13,9 @@
 import cdippy.url_utils as uu
 
 
+logger = logging.getLogger(__name__)
+
+
 class CDIPnc:
     """A base class used by the class StnData for retrieving data from
     CDIP netCDF (nc) files located either locally or remotely.
@@ -144,7 +148,7 @@ def set_request_info(
             is set to True, only nonpub records will be returned.
         """
         if start is None:
-            start = datetime(1975, 1, 1)
+            start = datetime(1975, 1, 1).replace(tzinfo=timezone.utc)
         if end is None:
             end = datetime.now(timezone.utc)
         self.set_timespan(start, end)
@@ -156,14 +160,19 @@ def set_request_info(
     def set_timespan(self, start, end):
         """Sets request timespan"""
         if isinstance(start, str):
-            self.start_dt = datetime.strptime(start, "%Y-%m-%d %H:%M:%S")
+            self.start_dt = datetime.strptime(start, "%Y-%m-%d %H:%M:%S").replace(
+                tzinfo=timezone.utc
+            )
         else:
             self.start_dt = start
         if isinstance(end, str):
-            self.end_dt = datetime.strptime(end, "%Y-%m-%d %H:%M:%S")
+            self.end_dt = datetime.strptime(end, "%Y-%m-%d %H:%M:%S").replace(
+                tzinfo=timezone.utc
+            )
         else:
             self.end_dt = end
         self.start_stamp = cu.datetime_to_timestamp(self.start_dt)
+
         self.end_stamp = cu.datetime_to_timestamp(self.end_dt)
 
     def get_request(self) -> dict:
@@ -365,23 +374,25 @@ def __get_indices(self, times: list, start_stamp: int, end_stamp: int) -> tuple:
         e_idx = bisect_right(times, end_stamp, s_idx)
         return s_idx, e_idx
 
-    def get_nc(self, url: str = None) -> netCDF4.Dataset:
-        if url is None:
+    def get_nc(self, url: str = None, retry: bool = False) -> netCDF4.Dataset:
+        if not url:
             url = self.url
-        # Check if the html page or file exists
-        if (
-            url[0:4] == "http" and not uu.url_exists(url + ".html")
-        ) and not os.path.isfile(url):
-            return None
         try:
-            nc = netCDF4.Dataset(url)
-        except Exception:
+            return netCDF4.Dataset(url)
+        except Exception as e:
             # Try again if unsuccessful (nc file not ready? THREDDS problem?)
-            try:
-                nc = netCDF4.Dataset(url)
-            except Exception:
-                nc = None
-        return nc
+            if retry:
+                logger.warning(
+                    msg=f"Retrying to open dataset at {url} due to an unexpected exception: {e}"
+                )
+                try:
+                    return netCDF4.Dataset(url)
+                except Exception:
+                    pass
+            logger.exception(
+                msg=f"Failed to open dataset at {url} due to an unexpected exception: {e}"
+            )
+            return None
 
     def byte_arr_to_string(self, b_arr: np.ma.masked_array) -> str:
         if np.ma.is_masked(b_arr):
@@ -1027,7 +1038,3 @@ def __init__(self, stn, data_dir=None, org=None):
         """For parameters see CDIPnc.set_dataset_info."""
         CDIPnc.__init__(self, data_dir)
         self.set_dataset_info(stn, org, "realtimexy")
-
-
-if __name__ == "__main__":
-    pass

cdippy/mopdata.py

@@ -1,4 +1,4 @@
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from bisect import bisect_left
 
 from cdippy.cdipnc import CDIPnc
@@ -235,7 +235,9 @@ def get_series(
 
         if start is not None and end is None:  # Target time
             if isinstance(start, str):
-                start = datetime.strptime(start, "%Y-%m-%d %H:%M:%S")
+                start = datetime.strptime(start, "%Y-%m-%d %H:%M:%S").replace(
+                    tzinfo=timezone.utc
+                )
             ts_I = self.get_target_timespan(
                 cu.datetime_to_timestamp(start), target_records, prefix + "Time"
             )
@@ -284,6 +286,7 @@ def get_target_timespan(
         # i_b will be possibly one more than the last index
         i_b = min(i_b, last_idx)
         # Target timestamp is exactly equal to a data time
+        closest_idx = None
         if i_b == last_idx or stamps[i_b] == target_timestamp:
             closest_idx = i_b
         elif i_b > 0:

cdippy/nchashes.py

@@ -7,10 +7,9 @@ class NcHashes:
 
     hashes_url = "http://cdip.ucsd.edu/data_access/metadata/wavecdf_by_datemod.txt"
     new_hashes = {}
-    hash_pkl = "HASH.pkl"
 
-    def __init__(self):
-        self.load_hash_table()
+    def __init__(self, hash_file_location=""):
+        self.hash_pkl = hash_file_location + "/HASH.pkl"
 
     def load_hash_table(self):
         lines = uu.read_url(self.hashes_url).strip().split("\n")
@@ -36,7 +35,7 @@ def compare_hash_tables(self) -> list:
         changed = []
         if old_hashes:
             if len(self.new_hashes) == 0:
-                self.load_hash_table()
+                return []
             for key in self.new_hashes:
                 if key not in old_hashes.keys() or (
                     key in old_hashes.keys() and old_hashes[key] != self.new_hashes[key]

cdippy/ncstats.py

@@ -54,7 +54,7 @@ def deployment_summary(self) -> dict:
         return result
 
     def load_nc_files(self, types: list = ["realtime", "historic", "archive"]) -> dict:
-        """Returns netcdf4 objects of a station's netcdf files"""
+        """Returns netCDF4 objects of a station's netcdf files"""
         self.nc_files = self.get_nc_files(types)
 
     def load_file(self, nc_filename: str):

cdippy/ndbc.py

@@ -25,17 +25,22 @@ def get_stn_info(wmo_id):
     uu.rfindt(root, results, "description")
 
 
-def get_wmo_id(stn):
+def get_wmo_id(
+    stn,
+    store=True,
+    filepath=".",
+):
     """Queries cdip wmo id table for a given station. Drops pickle file locally."""
-    pkl_fl = "./WMO_IDS.pkl"
+    pkl_fl = filepath + "/WMO_IDS.pkl" if store else None
     now = datetime.now(timezone.utc)
-    if now.minute == 23 or not os.path.isfile(pkl_fl):
+    if not pkl_fl or now.minute == 23 or not os.path.isfile(pkl_fl):
         url = "/".join([cdip_base, "wmo_ids"])
         r = uu.read_url(url)
         ids = {}
         for line in r.splitlines():
             ids[line[0:3]] = line[5:].strip()
-        cu.pkl_dump(ids, pkl_fl)
+        if pkl_fl:
+            cu.pkl_dump(ids, pkl_fl)
     else:
         ids = cu.pkl_load(pkl_fl)
     if stn in ids:

cdippy/stndata.py

@@ -503,7 +503,7 @@ def __merge_request(self):
         return result
 
     def get_nc_files(self, types: list = nc_file_types) -> dict:
-        """Returns dict of netcdf4 objects of a station's netcdf files"""
+        """Returns dict of netCDF4 objects of a station's netcdf files"""
         result = {}
         for ftype in types:
             if ftype == "historic":

cdippy/url_utils.py

@@ -1,7 +1,31 @@
 """Methods for working with urllib scraping web pages"""
 
+import logging
+import tomllib
+from urllib import request, error
 import xml.etree.ElementTree as ET
-import urllib.request
+
+with open("pyproject.toml", "rb") as f:
+    pyproject = tomllib.load(f)
+
+version = pyproject["project"]["version"]
+cdippy_lib = f"CDIPpy/{version}"
+
+_headers = {"User-Agent": cdippy_lib}
+logger = logging.getLogger(__name__)
+
+
+def _make_cdippy_request(url):
+    req = request.Request(url, headers=_headers)
+    try:
+        return request.urlopen(req)
+    except error.URLError as e:
+        logger.exception(f"URL error: {e.reason}")
+    except error.HTTPError as e:
+        logger.exception(f"HTTP error: request to {url} returned {e.code} - {e.reason}")
+    except Exception as e:
+        logger.exception(e)
+    return None
 
 
 def rfindta(el, r, tag, attr):
@@ -24,23 +48,11 @@ def rfindt(el, r, tag):
         r.append(el.text)
 
 
-def url_exists(url):
-    req = urllib.request.Request(url)
-    try:
-        urllib.request.urlopen(req)
-    except Exception:
-        return False
-    else:
-        return True
-
-
 def read_url(url):
-    try:
-        r = urllib.request.urlopen(url).read().decode("UTF-8")
-    except Exception:
-        return None
-    return r
+    response = _make_cdippy_request(url)
+    return response.read().decode("UTF-8") if response else None
 
 
 def load_et_root(url):
-    return ET.fromstring(urllib.request.urlopen(url).read())
+    response = _make_cdippy_request(url)
+    return ET.fromstring(response.read()) if response else None

cdippy/utils.py

@@ -1,7 +1,7 @@
 import os
 import errno
 import pickle as pkl
-import time
+import calendar
 from datetime import datetime
 import pytz
 
@@ -71,7 +71,7 @@ def cdip_datetime(cdip_str: str) -> datetime:
 
 
 def datetime_to_timestamp(dt: datetime) -> int:
-    return time.mktime(dt.timetuple())
+    return calendar.timegm(dt.utctimetuple())
 
 
 def timestamp_to_datetime(ts: int) -> datetime:

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cdippy"
-version = "0.1.0"
+version = "0.1.0-dev"
 description = "CDIP python library"
 readme = { file = "README.md", content-type = "text/markdown" }
 license = { text = "BSD License" }