Improve time interval handling and irregular time series identification.

Author: Eli

dms_datastore/read_ts.py

@@ -11,7 +11,7 @@
 import os
 import fnmatch
 from vtools.functions.merge import *
-from vtools.data.vtime import days, minutes
+from vtools.data.vtime import days, minutes, hours, months, seconds, years, to_timedelta
 from dms_datastore.filename import extract_year_fname
 
 __all__ = [
@@ -1341,7 +1341,7 @@ def read_ts(
             if "freq" in kwargs and freq is not None:
                 raise ValueError("freq must be None if passed in kwargs to avoid conflict")
             if force_regular: 
-                if  freq in (None, "None"): freq = "infer"
+                if  freq in (None, "None","none"): freq = "infer"
             else:                
                 if freq not in ["None", None]:
                     raise ValueError("freq must be None or 'None' if force_regular is False")
@@ -1433,7 +1433,6 @@ def infer_freq_robust(
     index, preferred=["h", "15min", "6min", "10min", "h", "d"], **kwargs
 ):
     index = index.round("1min")
-
     if len(index) < 8:
         # not enough to quibble, use the 8 points
         f = pd.infer_freq(index)
@@ -1452,8 +1451,9 @@ def infer_freq_robust(
             f = pd.infer_freq(index[0:7])
         if f is None:
             for p in preferred:
-                freq = pd.tseries.frequencies.to_offset(p)
+                freq = to_timedelta(p)
                 tester = index.round(p)
+
                 diff = (index - tester) < (freq / 5)
                 frac = diff.mean()
                 if frac > 0.98:

examples/dropbox/dropbox_spec_ccf.yaml

@@ -9,7 +9,7 @@ data:
   - name: ccfb
     skip: False
     collect:
-      file_pattern: "CC_Gate_Positions*2024*.csv"
+      file_pattern: "CC_Gate_Positions*.csv"
       wildcard: time_overlap    # time_sharded | time_overlap | data | none. 
             # If time_sharded, indicates non-overlapping timestamps that uses year chunking. 
             # If time_overlap, indicates wildcards are ordered by date 20250531(lexicographic must = chronological)
@@ -27,7 +27,7 @@ data:
          transition: prefer_first
       selector: null  # Usually a column name. When read_last_resort_csv is used, use null.
     transforms:
-      - name: dst_tz      # SCADA is delivered local time. Often this is not needed from better data sources
+      - name: dst_st      # SCADA is delivered local time. Often this is not needed from better data sources
         args:
           src_tz: US/Pacific
           target_tz: ${target_tz}
@@ -55,10 +55,10 @@ data:
     output:
       staging_dir: ./drop_staging/formatted
       write_args:
-        float_format: "%0.3f"   # optional, fallback = 0.6f
+        float_format: "%0.3f"
         chunk_years: False
       repo_dir: ./fake_repo/formatted
-      merge_priority: staged
+      merge_priority: staged      # reconcile
       allow_new_series: true
       inspection:
         recent_years: 3