switch to uv, polars, and seaborn; improve legibility with lots of data

.gitignore

@@ -81,9 +81,6 @@ target/
 profile_default/
 ipython_config.py
 
-# pyenv
-.python-version
-
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 #   However, in case of collaboration, if having platform-specific dependencies or dependencies

.python-version

@@ -0,0 +1 @@
+3.14

README.md

@@ -1,12 +1,13 @@
 # Watertable
 ## Analysis and visualization of an observational well on North Pender Island
 
-The well is powered by solar panels and connected over satellite, and it has a depth sensor at the base that sends measurements every hour on the hour.
+The well is solar-powered with a 4G connection; it has a depth sensor at the base that sends measurements every hour on the hour.
 It's been running since 2003.
-This project superimposes every year with data on each other, with the current year highlighted.
-This allows for users to clearly see how full the aquifer is, compared with earlier this year, natural levels, and all previous years.
+Data is sourced from the government of British Columbia's [Aquarius project](https://www2.gov.bc.ca/gov/content/environment/air-land-water/water/water-science-data/water-data-tools/real-time-water-data-reporting).
+This project graphs the current year's data, with the 95% historical range as a backdrop.
+Key numbers (current water level, historical average for this time of year) is presented in easy-to-read text.
 
-The project is hosted by Cloudflare, and is located at [watertable.nove.dev](https://watertable.nove.dev)
+The project is hosted on Cloudflare, and is located at [watertable.nove.dev](https://watertable.nove.dev)
 
 For more details, open the `main.ipynb` document. 
 The most recent code is in generate.py, and that's what's used to update the website, but the notebook has useful information.

generate.py

@@ -1,12 +1,16 @@
 import requests as rq
-import csv
 import datetime
-import numpy as np
-import matplotlib as mpl
+import polars as pl
+import seaborn as sns
+from zoneinfo import ZoneInfo
 import matplotlib.pyplot as plt
+from matplotlib.patches import Patch
+from matplotlib.lines import Line2D
 import matplotlib.dates as mdates
-from pipeline import *
+import matplotlib.ticker as ticker
 
+now = datetime.datetime.now(ZoneInfo("America/Vancouver"))
+current_year = now.year
 
 params = {
     "DataSet": "SGWL.Working@OW283",
@@ -19,65 +23,96 @@
     "Timezone": "-7",
 }
 
+print("Getting API token...")
 token = rq.post(
     url="https://aqrt.nrs.gov.bc.ca/Export/DataSetToken", params=params
 ).json()["Token"]
 params["Token"] = token
-response = rq.get(url="https://aqrt.nrs.gov.bc.ca/Export/DataSet", params=params)
-reader = csv.reader(response.text.splitlines())
-
-
-# with open("data/dataset.csv") as data:
-parsed = filter(lambda x: x, map(lambda x: clean_and_process(x[0], x[1]), reader))
-split = year_splitter(parsed)
-years = list(map(lambda x: np.transpose(unify_year(x)), split))
 
+print("Downloading raw data (this may take a minute)...")
+response = rq.get(url="https://aqrt.nrs.gov.bc.ca/Export/DataSet", params=params)
 
+print("Data downloaded. Processing...")
+df = pl.read_csv(
+    response.content,
+    try_parse_dates=True,
+    skip_lines=5,
+    new_columns=["timestamp", "level"],
+)
+df = df.drop_nans()
+df = df.group_by_dynamic("timestamp", every="1d").agg(pl.col("level").mean())
+df = df.with_columns(
+    (pl.col("level") * -3.28), (pl.col("timestamp").dt.year()).alias("year")
+)
+df = df.remove(
+    (pl.col("timestamp").dt.month() == 2) & (pl.col("timestamp").dt.day() == 29)
+)
+df = df.with_columns(
+    (pl.col("timestamp").dt.replace(year=current_year)),
+    (pl.col("year") == current_year).alias("current_year"),
+)
+df = df.remove((pl.col("year") < 2004))
+
+previous_years = df.filter(~pl.col("current_year"))
+redline = df.filter(pl.col("current_year"))
+
+ordinal_today = now.date().replace(year=1).toordinal()
+historical_average: float = -(
+    previous_years.filter(
+        (pl.col("timestamp").dt.ordinal_day() - ordinal_today).abs() <= 7
+    )
+    .group_by("year")
+    .agg(pl.col("level").mean())
+    .mean()["level"]
+    .item()
+)
+current_level: float = -redline.row(-1, named=True)["level"]
+
+print("Processing complete. Generating graphs...")
 fig, ax = plt.subplots()
 fig.set_size_inches(14, 10)
-
+sns.lineplot(
+    ax=ax,
+    data=previous_years,
+    x="timestamp",
+    y="level",
+    color="teal",
+    errorbar=("pi", 90),
+    linewidth=0,
+)
+sns.lineplot(data=redline, x="timestamp", y="level", ax=ax, color="blue")
+handles = [
+    Patch(facecolor="teal", alpha=0.3, label="historical range"),
+    Line2D([0], [0], color="blue", label=f"{current_year}"),
+]
+
+ax.legend(handles=handles)
+ax.set_xlabel("")
 ax.set_ylabel("feet below the surface")
 ax.set_title("North Pender Island Water Table Status")
-ax.set_xlim(years[-1][0][0], years[0][0][-1])
+ax.set_xlim(datetime.date(current_year, 1, 1), datetime.date(current_year, 12, 31))  # type: ignore[arg-type]
 
-# fancy automatic date labels along x axis
 locator = mdates.AutoDateLocator(minticks=3, maxticks=20)
 formatter = mdates.ConciseDateFormatter(locator)
 ax.xaxis.set_major_locator(locator)
 ax.xaxis.set_major_formatter(formatter)
-
-# chosen for linear perception and contrast with red
-colourmap = mpl.colormaps['viridis'] # type: ignore
-
-# graphing each year with its own label and colour along the viridis colourmap
-year_plots = []
-current_year = datetime.today().year
-for i, year in enumerate(years[7:-1]):
-    year_as_percentage = i / (current_year - 1 - 2010)
-    (year_plot,) = ax.plot(
-        year[0][:-40],
-        rolling_mean(year[1], 40)[:-40],
-        color=colourmap(1 - year_as_percentage),
-        label=i + 2010,
-    )
-    year_plots.append(year_plot)
-
-# current year
-ax.plot(years[-1][0], years[-1][1], color="red", label=current_year)
-
-ax.legend()
-
+ax.yaxis.set_major_locator(ticker.MaxNLocator(integer=True))
 
 fig.savefig("www/output.svg")
 fig.set_size_inches(7, 5)
 fig.savefig("www/output_small.svg")
 
-current_level = abs(years[-1][-1][-1])
-historical = abs(historical_past_two_weeks(years))
+
+def rough_date() -> str:
+    month = now.strftime("%B")
+    prefix = "early " if now.day < 10 else "mid-" if now.day < 20 else "late "
+    return prefix + month
+
 
 with open("index.html") as index:
-    index = index.read().replace("XX", ("%2.1f" % current_level), 1)
-    index = index.replace("XX", ("%2.1f" % historical), 1)
+    index = index.read().replace("XX", now.strftime("%Y-%m-%d, %H:%M PT"), 1)
+    index = index.replace("XX", ("%2.1f" % current_level), 1)
+    index = index.replace("XX", ("%2.1f" % historical_average), 1)
     index = index.replace("this time of year", rough_date())
 with open("www/index.html", "w") as new_index:
     new_index.write(index)
@@ -96,7 +131,7 @@
   </g>""",
             "",
         )
-        .replace('viewBox="0 0 1008 720"', 'viewBox="80 50 915 670"')
+        .replace('viewBox="0 0 1008 720"', 'viewBox="80 60 840 620"')
         .replace('height="720pt"', "")
         .replace('width="1008pt"', "")
     )
@@ -116,9 +151,11 @@
   </g>""",
             "",
         )
-        .replace('viewBox="0 0 504 360"', 'viewBox="15 20 490 350"')
+        .replace('viewBox="0 0 504 360"', 'viewBox="15 20 445 330"')
         .replace('height="360pt"', "")
         .replace('width="504pt"', "")
     )
 with open("www/output_small.svg", "w") as new_graph:
     new_graph.write(graph)
+
+print("Graphs generated. Resulting static webpage saved to www/. Goodbye!")

index.html

@@ -14,7 +14,7 @@
 
 <body>
     <h1>North Pender Island Water Table</h1>
-    <h2 id="currentDate" onload="updateDate()"></h2>
+    <h2 id="currentDate">DATA UPDATED XX</h2>
     <div class="summary">
         <p class="left-col">The water level is currently</p>
         <p id="water-level" class="dyn-important">XX</p>
@@ -25,15 +25,11 @@ <h2 id="currentDate" onload="updateDate()"></h2>
     </div>
     <div class="graph-wrapper">
         <object id="large" type="image/svg+xml" data="/output.svg"
-            alt="A graph showing the water level for this year, and 10 years in the past"
+            alt="A graph showing the water level for this year, and historical 90% percentile interval"
             onload="largeGraphLoad()"></object>
         <object id="small" type="image/svg+xml" data="/output_small.svg"
-            alt="A graph showing the water level for this year, and 10 years in the past"
+            alt="A graph showing the water level for this year, and historical 90% percentile interval%"
             onload="smallGraphLoad()"></object>
-        <p style="display:none">Time range: 2010 - 2021<br>
-            Smoothing: 48 hours<br>
-            AI prediction: no<br>
-            Last updated: 30 minutes ago</p>
     </div>
     <div class="future" style="display:none;">
         <p class="left-col">In the next two weeks, we're forecast to get about</p>

pyproject.toml

@@ -0,0 +1,14 @@
+[project]
+name = "watertable"
+version = "1.0.0"
+description = "friendly local water level monitoring"
+readme = "README.md"
+requires-python = ">=3.14"
+dependencies = [
+    "matplotlib>=3.10.8",
+    "numpy>=2.4.3",
+    "polars>=1.39.3",
+    "pyarrow>=23.0.1",
+    "requests>=2.32.5",
+    "seaborn>=0.13.2",
+]