Allow dropbox to work off a config variable (first case is daily_dropbox_spec)

Author: Eli

dms_datastore/config_data/daily_dropbox.yaml

@@ -0,0 +1,39 @@
+
+data:
+  - name: daily
+    collect:
+      file_pattern: "{source}_{station_id}_{agency_id}_{param}_{syear}_{eyear}.csv"
+      location: "//cnrastore-bdo/Modeling_Data/repo_staging_auto/daily/raw"
+      recursive_search: false
+      reader: read_ts
+      selector: value
+
+    metadata:
+      station_id: infer_from_filename
+      source: infer_from_filename
+      agency_id: infer_from_filename
+      agency: registry_lookup
+      param: infer_from_filename
+      subloc: default
+      unit: ft^3/s
+      freq: D
+      time_zone: Etc/GMT+8
+      latitude: registry_lookup
+      longitude: registry_lookup
+      projection_x_coordinate: registry_lookup
+      projection_y_coordinate: registry_lookup
+      
+    output:
+      repo_name: daily_formatted
+      staging:
+        dir: staging
+        float_format: "%0.3f"
+        chunk_years: False
+      reconcile:
+        #repo_data_dir: ./fake_repo
+        prefer: staged
+        allow_new_series: true
+        inspection:
+          recent_years: 3
+          p3: 0.15
+          p10: 0.05      

dms_datastore/config_data/dstore_config.yaml

@@ -5,7 +5,7 @@ sublocations: station_subloc.csv
 ncro_inventory: ncro_inventory_full.csv
 
 daily_data_manifest: daily_data_manifest.csv
-
+daily_dropbox_spec: daily_dropbox.yaml
 
 file_deletion_list: non_15_min_files_checked.txt
 compare_excepts_formatted: compare_excepts_formatted.txt

dms_datastore/dropbox_data.py

@@ -4,12 +4,13 @@
 import yaml
 import glob
 import click
+import dms_datastore.dstore_config
 from vtools import dst_st, ts_merge, ts_splice,ts_coarsen
 from omegaconf import OmegaConf
 from vtools.data.indexing import infer_freq_robust
 from dms_datastore.read_ts import read_ts, infer_freq_robust
 from dms_datastore.write_ts import write_ts_csv
-from dms_datastore.dstore_config import repo_registry, repo_config
+from dms_datastore.dstore_config import repo_registry, repo_config, config_file
 from dms_datastore.filename import interpret_fname, meta_to_filename, naming_spec
 from dms_datastore.reconcile_data import update_repo
 import logging
@@ -419,7 +420,7 @@ def _require_lower(name):
         )
 
 
-def get_data(spec, selected_names=None):
+def apply_dropbox_workflow(spec, selected_names=None):
     logger.info("dropbox: loaded %d recipe entries", len(spec["data"]))
     always_skip = True
 
@@ -750,16 +751,21 @@ def get_data(spec, selected_names=None):
         )            
 
 def dropbox_data(spec_fname, selected_names=None):
+    spec_fname = (
+        spec_fname
+        if os.path.exists(spec_fname)
+        else config_file(spec_fname)
+    )
     spec = get_spec(spec_fname)
-    get_data(spec, selected_names=selected_names)
+    apply_dropbox_workflow(spec, selected_names=selected_names)
 
 
 @click.command(name="dropbox")
 @click.option(
     "--input",
     "spec_fname",
     required=True,
-    type=click.Path(exists=True, dir_okay=False, readable=True),
+    type=str,
     help="YAML file with dropbox specification.",)
 @click.option(
     "--name",