Fixed read_vtide and got rid of stray matplotlib import for a defunct __main__

Author: Eli

dms_datastore/read_multi.py

@@ -3,7 +3,6 @@
 
 import os
 import pandas as pd
-import matplotlib.pyplot as plt
 from dms_datastore.read_ts import read_ts, read_yaml_header
 from dms_datastore import dstore_config
 from dms_datastore.filename import *
@@ -371,25 +370,4 @@ def ts_multifile_read(
     return full
 
 
-if __name__ == "__main__":
-    # NCRO example
-
-    dirname = "//cnrastore-bdo/Modeling_Data/continuous_station_repo_beta/formatted_1yr"
-    rpats = ["ncro_gle_b9532000_temp*.csv", "cdec_gle*temp*.csv"]
-    pats = [os.path.join(dirname, p) for p in rpats]
-    ts = ts_multifile(pats)
-    print(ts)
-    ts.plot()
-    plt.show()
-
-    # Example for USGS
-    # usgs_list = ['lib','ucs','srv','dsj','dws','sdi','fpt','lps','mld','sjj','sjg']
-    # for nseries in usgs_list:
-    #    print(nseries)
-    #
-    #    dirname = "//cnrastore-bdo/Modeling_Data/continuous_station_repo/raw/"
-    #    pat = os.path.join(dirname,f"usgs_{nseries}_*turbidity_*.rdb")
-    #    ts = ts_multifile_read(pat,column_name=nseries)
-    #    print(ts)
-    #    ts.plot()
-    #    plt.show()
+

dms_datastore/read_ts.py

@@ -1248,22 +1248,76 @@ def vtide_date_parser(*args):
     return dtm.datetime.strptime(x, "%Y%m%dT%H%M")
 
 
-def read_vtide(fpath_pattern, start=None, end=None, selector=None, force_regular=False, freq=None, **kwargs):
-    ts = csv_retrieve_ts(
-        fpath_pattern,
-        start,
-        end,
-        force_regular,
-        selector=selector,
-        format_compatible_fn=lambda x: True,
-        qaqc_selector=None,
-        parsedates=[0, 1],
-        indexcol=0,
-        header=None,
-        sep=r"\\s+",
-        comment="#",
+def read_vtide_file(path, comment="#", sep=r"\s+", header=None, **kwargs):
+    """Read a single no-header vtide text file with fixed date/time columns."""
+    dset = pd.read_csv(
+        path,
+        sep=sep,
+        header=header,
+        comment=comment,
+        dtype={0: str, 1: str},
+        **kwargs,
     )
 
+    if dset.shape[1] < 2:
+        raise ValueError(f"Vtide file {path} must contain at least date/time columns")
+
+    date_part = dset[0].astype(str).str.strip()
+    time_part = dset[1].astype(str).str.strip()
+
+    if not time_part.str.contains(":").all():
+        time_part = time_part.str.zfill(4).str.replace(r"^(\d{2})(\d{2})$", r"\1:\2", regex=True)
+
+    dt_str = date_part + "T" + time_part
+
+    try:
+        idx = pd.to_datetime(dt_str, format="%Y%m%dT%H%M", errors="raise")
+    except ValueError:
+        try:
+            idx = pd.to_datetime(dt_str, format="%Y-%m-%dT%H:%M", errors="raise")
+        except ValueError:
+            idx = pd.to_datetime(dt_str, errors="raise")
+
+    dset.index = idx
+    dset.index.name = "datetime"
+    dset = dset.drop(columns=[0, 1])
+
+    return dset
+
+
+def read_vtide(fpath_pattern, start=None, end=None, selector=None, force_regular=False, freq=None, **kwargs):
+    contains_glob = any(ch in fpath_pattern for ch in "*?[]")
+
+    if contains_glob:
+        ts = csv_retrieve_ts(
+            fpath_pattern,
+            start,
+            end,
+            force_regular,
+            selector=selector,
+            format_compatible_fn=lambda x: True,
+            qaqc_selector=None,
+            parsedates=None,
+            indexcol=None,
+            header=None,
+            sep=r"\s+",
+            comment="#",
+            **kwargs,
+        )
+
+        if ts is not None and isinstance(ts, pd.DataFrame) and 0 in ts.columns and 1 in ts.columns:
+            dt_str = ts[0].astype(str).str.strip() + "T" + ts[1].astype(str).str.zfill(4).str.strip()
+            ts.index = pd.to_datetime(dt_str, format="%Y%m%dT%H%M", errors="raise")
+            ts.index.name = "datetime"
+            ts = ts.drop(columns=[0, 1])
+
+        return ts
+
+    ts = read_vtide_file(fpath_pattern, **kwargs)
+
+    if selector is not None:
+        ts = ts[selector] if isinstance(selector, (list, tuple)) else ts[[selector]]
+
     return ts
 
 

dms_datastore/write_ts.py

@@ -118,6 +118,11 @@ def write_ts_csv(
     **kwargs : other
     Other items that will be passed to write_csv
     """
+    # Series support: convert to single-column DataFrame while preserving the index and column name.
+    if isinstance(ts, pd.Series):
+        col_name = ts.name if ts.name is not None else "value"
+        ts = ts.to_frame(name=col_name)
+
     former_index = ts.index.name
     if former_index != "datetime" and not overwrite_conventions:
         # warnings.warn("Index will be renamed datetime in file according to specification. Copy made")
@@ -139,10 +144,17 @@ def write_ts_csv(
             s = max(pd.Timestamp(bnd[0], 1, 1), ts.first_valid_index())
             e = min(pd.Timestamp(bnd[1], 12, 31, 23, 59, 59), ts.last_valid_index())
             tssub = ts.loc[s:e]
-            if (
-                tssub.count() < 16
-            ).all():  # require 15 values per column. all() is for multiple columns
-                continue
+
+            count = tssub.count()
+            if hasattr(count, "all"):
+                # DataFrame path: all columns should have at least 16 values
+                if not (count >= 16).all():
+                    continue
+            else:
+                # Series path: count is scalar
+                if count < 16:
+                    continue
+
             new_date_range_str = f"{bnd[0]}_{bnd[1]}"
 
             if single_year_label:

tests/test_write_ts.py

@@ -59,3 +59,28 @@ def test_write_ts_csv_stringio_with_metadata(sample_ts):
     assert "station_id: ABC" in contents
     assert "units: ft" in contents
     assert "value" in contents
+
+
+def test_write_ts_csv_stringio_series():
+    s = pd.Series(
+        [1.1, 2.2, 3.3, 4.4, 5.5],
+        index=pd.date_range("2020-01-01", periods=5, freq="h", name="datetime"),
+        name="value",
+    )
+    buf = io.StringIO()
+    write_ts_csv(s, buf)
+
+    contents = buf.getvalue()
+    assert "# format: dwr-dms-1.0" in contents
+    assert "datetime" in contents
+    assert "value" in contents
+
+    buf.seek(0)
+    lines = [line for line in buf if not line.startswith("#")]
+    roundtrip = pd.read_csv(
+        io.StringIO("".join(lines)),
+        index_col="datetime",
+        parse_dates=True,
+    )
+    assert list(roundtrip.index) == list(s.index)
+    assert list(roundtrip["value"]) == list(s.values)