DD-111 adding dcc and smcg gates downloader

Author: dwr-psandhu

dms_datastore/download_dcc.py

@@ -0,0 +1,55 @@
+# Delta Cross Channel
+# Download the gate operations from https://www.usbr.gov/mp/cvo/vungvari/Ccgates.pdf
+
+import os
+import pandas as pd
+import datetime
+import tabula  # for PDF parsing
+import requests
+from . import store_utils as utils
+import click
+
+click.command()
+
+
+@click.option("--base-dir", default="data/raw/dxc_gate")
+def main(base_dir="data/raw/dxc_gate"):
+    """
+    Download the Delta Cross Channel gate log from the US Bureau of Reclamation
+    https://www.usbr.gov/mp/cvo/vungvari/Ccgates.pdf
+
+    """
+    utils.ensure_dir(base_dir)
+    today = datetime.datetime.now()
+    date_str = today.strftime("%Y-%m-%d")
+    url = "https://www.usbr.gov/mp/cvo/vungvari/Ccgates.pdf"
+    response = requests.get(url)
+    assert response.status_code == 200
+    fname = url.split("/")[-1]
+    pdfname = os.path.join(base_dir, fname.split(".")[0] + ".pdf")
+    with open(pdfname, "wb") as fh:
+        fh.write(response.content)
+    pages = tabula.read_pdf(
+        pdfname, pages="all", guess=False, encoding="ISO-8859-1"  # for windows maybe?
+    )  # columns=['date','time','remarks'])
+    df = pd.concat(pages)
+    df.columns = ["date", "time", "value"]
+    df = df.dropna()
+    df["datetime"] = df["date"] + " " + df["time"]
+    df = df[df["datetime"] != "DATE TIME"]
+    df["datetime"] = pd.to_datetime(df["datetime"])
+    df = df[["datetime", "value"]]
+    df = df.set_index("datetime")
+    df = df.sort_index()
+    df["action"] = df["value"].str.split(expand=True)[0]
+    df["comments"] = df["value"].str.split().map(lambda x: " ".join(x[1:]))
+    df = df.drop(columns=["value"])
+    # df['action'].unique()
+    df["action"] = (
+        df["action"]
+        .map({"open": 2, "closed": 0, "gate": 0, "partially": 1, "-": 0, "close": 0})
+        .astype("int")
+    )
+    conv_dir = os.path.dirname(pdfname).replace("/raw/", "/converted/")
+    utils.ensure_dir(conv_dir)
+    df.to_csv(os.path.join(conv_dir, fname.split(".")[0] + ".csv"))

dms_datastore/download_montezuma_gates.py

@@ -0,0 +1,30 @@
+import requests
+import pandas as pd
+from . import store_utils as utils
+import datetime
+import os
+import click
+
+
+@click.command()
+@click.option("--base-dir", default="data/raw/montezuma_gate_log")
+def main(base_dir="data/raw/montezuma_gate_log"):
+    """
+    Download the Montezuma Slough Salinity Control Gates log from the California Natural Resources Agency
+    # https://data.cnra.ca.gov/dataset/suisun-marsh-salinity-control-gates-log/resource/265729e9-4ac0-469e-828b-2564ac077689
+    """
+    utils.ensure_dir(base_dir)
+    today = datetime.datetime.now()
+    date_str = today.strftime("%Y-%m-%d")
+    url = "https://data.cnra.ca.gov/dataset/e76622ca-b6e9-4e78-a08e-deb9580d49b3/resource/265729e9-4ac0-469e-828b-2564ac077689/download/smscg-log.xlsx"
+    response = requests.get(url)
+    assert response.status_code == 200
+    fname = url.split("/")[-1]
+    xlsfname = os.path.join(base_dir, fname.split(".")[0] + ".xlsx")
+    with open(xlsfname, "wb") as fh:
+        fh.write(response.content)
+    df = pd.read_excel(xlsfname, parse_dates=True, index_col=0)
+    df = df.sort_index()
+    conv_dir = os.path.dirname(xlsfname).replace("/raw/", "/converted/")
+    utils.ensure_dir(conv_dir)
+    df.to_csv(os.path.join(conv_dir, fname.split(".")[0] + ".csv"))

environment.yml

@@ -14,6 +14,7 @@ dependencies:
   - openpyxl
   - lxml # for pandas html parsing
   - paramiko # for sftp cimis requests
+  - tabula-py # for pdf parsing
   - pandas>=2
   - numpy<2
   - cfgrib

setup.py

@@ -69,6 +69,8 @@
             "download_mokelumne=dms_datastore.download_mokelumne:main",
             "download_ucdipm=dms_datastore.download_ucdipm:main",
             "download_cimis=dms_datastore.download_cimis:main",
+            "download_dcc=dms_datastore.download_dcc:main",
+            "download_montezuma_gates=dms_datastore.download_montezuma_gates:main",
             "compare_directories=dms_datastore.compare_directories:main",
             "populate_repo=dms_datastore.populate_repo:main",
             "station_info=dms_datastore.station_info:main",