Renamed some dropbox materials so that they will be together in parent directory. Lower cased "value" in incoming data.

Author: Eli

…datastore/config_data/daily_dropbox.yaml …datastore/config_data/dropbox_daily.yamldms_datastore/config_data/daily_dropbox.yaml renamed to dms_datastore/config_data/dropbox_daily.yaml dms_datastore/config_data/daily_dropbox.yaml renamed to dms_datastore/config_data/dropbox_daily.yaml

@@ -1,10 +1,15 @@
 
+modeling_data: //cnrastore-bdo/Modeling_Data
+dropbox_home: //cnrastore-bdo/Modeling_Data/repo_staging_auto/dropbox
+target_tz: "Etc/GMT+8"    # Odd, but the correct one in POSIX/pandas for PST.
+target_tz_label: PST 
+
 data:
+
   - name: daily
     collect:
       file_pattern: "{source}_{station_id}_{agency_id}_flow_{syear}_{eyear}.csv"
-      #location: "//cnrastore-bdo/Modeling_Data/repo_staging_auto/daily/raw"
-      location: ./raw
+      location: "//cnrastore-bdo/Modeling_Data/repo_staging_auto/daily/raw"
       recursive_search: false
       reader: read_ts
       selector: value
@@ -18,7 +23,8 @@ data:
       subloc: default
       unit: ft^3/s
       freq: D
-      time_zone: Etc/GMT+8
+      time_zone: ${target_tz}
+      time_zone_label: ${target_tz_label}
       latitude: registry_lookup
       longitude: registry_lookup
       projection_x_coordinate: registry_lookup
@@ -40,13 +46,54 @@ data:
           recent_years: 3
           p3: 0.15
           p10: 0.05      
-          
+
+# This recipe operates on the scraped website from EBMUD. One of the daily time series from populate_daily
+# will catch occasional updates from USGS. A third recipe is in 
+  - name: moke ebmud
+    collect:
+      file_pattern: "mokelumne_flow_*.csv"
+      location: "${dropbox_home}/mokelumne"
+      recursive_search: false
+      reader: read_ts
+      wildcard: time_shard
+      selector: null
+    metadata:
+      station_id: wbr
+      source: ebmud
+      agency_id: registry_lookup
+      agency: registry_lookup      
+      freq: D
+      param: flow
+      unit: ft^3/s
+      subloc: default
+      time_zone: ${target_tz}
+      time_zone_label: ${target_tz_label}
+      latitude: registry_lookup
+      longitude: registry_lookup
+      projection_x_coordinate: registry_lookup
+      projection_y_coordinate: registry_lookup
+      agency_lat: registry_lookup
+      agency_lon: registry_lookup
+      # Coordinates are auto-populated from the station registry.
+    output:
+      repo_name: daily_formatted
+      staging:
+        dir: ./staging
+        write_args:
+          float_format: "%.4f"
+      reconcile:
+        prefer: repo
+        allow_new_series: true
+        inspection:
+          recent_years: 3
+          p3: 0.15
+          p10: 0.05
+
 
   - name: daily_cx2
     collect:
       file_pattern: "{source}_{station_id}_{agency_id}_kmdist_{syear}_{eyear}.csv"
-      #location: "//cnrastore-bdo/Modeling_Data/repo_staging_auto/daily/raw"
-      location: ./raw
+      location: "//cnrastore-bdo/Modeling_Data/repo_staging_auto/daily/raw"
       recursive_search: false
       reader: read_ts
       selector: value

dms_datastore/config_data/dstore_config.yaml

@@ -4,7 +4,7 @@ variable_definitions: variables.csv
 sublocations: station_subloc.csv
 ncro_inventory: ncro_inventory_full.csv
 daily_data_manifest: daily_data_manifest.csv
-daily_dropbox_spec: daily_dropbox.yaml
+dropbox_daily: dropbox_daily.yaml
 file_deletion_list: non_15_min_files_checked.txt
 compare_excepts_formatted: compare_excepts_formatted.txt
 des_rationalize_time_spec: des_rationalize_cfg.yaml

dms_datastore/dropbox_data.py

@@ -354,9 +354,10 @@ def _maybe_rename_value_column(ts, splice_args):
       splice_args:
         rename: {old: new}   # dict rename
     """
-    # Auto-normalize 'VALUE' -> 'value' for univariate series
-    if "VALUE" in ts.columns and "value" not in ts.columns:
-        ts = ts.rename(columns={"VALUE": "value"})
+    # Auto-normalize any case variant of 'value' -> 'value' for univariate series
+    value_cols = [c for c in ts.columns if c.lower() == "value" and c != "value"]
+    if value_cols:
+        ts = ts.rename(columns={c: "value" for c in value_cols})
 
     if not splice_args:
         return ts