DD-124 download_dcc.py to improve action and comments parsing

Author: dwr-psandhu

dms_datastore/download_dcc.py

@@ -17,7 +17,6 @@ def main(base_dir="data/raw/dxc_gate"):
     """
     Download the Delta Cross Channel gate log from the US Bureau of Reclamation
     https://www.usbr.gov/mp/cvo/vungvari/Ccgates.pdf
-
     """
     utils.ensure_dir(base_dir)
     today = datetime.datetime.now()
@@ -31,7 +30,7 @@ def main(base_dir="data/raw/dxc_gate"):
         fh.write(response.content)
     pages = tabula.read_pdf(
         pdfname, pages="all", guess=False, encoding="ISO-8859-1"  # for windows maybe?
-    )  # columns=['date','time','remarks'])
+    )  # columns=['date','time', 'action', 'remarks'])
     df = pd.concat(pages)
     df.columns = ["date", "time", "value"]
     df = df.dropna()
@@ -41,15 +40,26 @@ def main(base_dir="data/raw/dxc_gate"):
     df = df[["datetime", "value"]]
     df = df.set_index("datetime")
     df = df.sort_index()
-    df["action"] = df["value"].str.split(expand=True)[0]
-    df["comments"] = df["value"].str.split().map(lambda x: " ".join(x[1:]))
+    df["action"] = df["value"].str.split(n=1, expand=True)[0]
+    df["comments"] = df["value"].str.split(n=1, expand=True)[1]
     df = df.drop(columns=["value"])
-    # df['action'].unique()
-    df["action"] = (
-        df["action"]
-        .map({"open": 2, "closed": 0, "gate": 0, "partially": 1, "-": 0, "close": 0})
-        .astype("int")
+    df.loc[df["comments"].str.strip() == "-", "comments"] = ""
+    df.loc[df["comments"].isna(), "comments"] = ""
+    df["comments"] = df["comments"].str.strip()
+    df["action"] = df["action"].map(
+        {
+            "open": "open",
+            "closed": "closed",
+            "gate": "closed",
+            "partially": "partially open",
+            "-": "closed",
+            "close": "closed",
+        }
     )
     conv_dir = os.path.dirname(pdfname).replace("/raw/", "/converted/")
     utils.ensure_dir(conv_dir)
     df.to_csv(os.path.join(conv_dir, fname.split(".")[0] + ".csv"))
+
+
+if __name__ == "__main__":
+    main()