AP-660: Implements static_files plugin (#62)

- Adds a static_files plugin that serves files
  from ./storage (configurable).
- Adds a "Files" tab to the DagRun page that shows
  files created for that run. Refactors existing
  URLs/storage dirs to use that new location.
- Extracts Jinja2 templating for the main app into
  mokelumne.templates. Templates need to be
  rendered slightly differently for emails vs.
  on-disk indexes, and this cleans up that code.

Author: danschmidt5189

.gitignore

@@ -5,12 +5,14 @@ tmp
 artifacts/*
 logs/*
 download/*
-public/*
+files/*
+!files/.keep
 __pycache__/
 *.py[cod]
 *$py.class
 config
 airflow.cfg
 *.egg-info
 .coverage
-build/
+build/
+uv.lock

README.md

@@ -107,7 +107,6 @@ Important environment variables for our build/environment:
 |`AWS_MODEL_ID`|The model to use. Make sure it's supported on the ARN.|`AWS_MODEL_ID="us.anthropic.claude-haiku-4-5-20251001-v1:0"`|
 |`AWS_MODEL_LABEL`|A human friendly label for the model. Will eventually be displayed in the Tind record.|`AWS_MODEL_LABEL="Claude Haiku 4.5"`|
 |`AWS_MODEL_PROVIDER`|The provider for the model. |`AWS_MODEL_PROVIDER=anthropic`|
-|`MOKELUMNE_PUBLIC_STORAGE`|Path for public assets|`MOKELUMNE_PUBLIC_STORAGE=/opt/airflow/public`|
 |`MOKELUMNE_PUBLIC_URL`|URL to access public assets - must end in `/`|`MOKELUMNE_PUBLIC_URL=https://mokelumne-assets.ucblib.org/`|
 
 Note: The `AIRFLOW_UID` example in `example.env` maps to the reserved `uid` for the `airflow` user in [lap/workflow](https://git.lib.berkeley.edu/lap/workflow/-/wikis/UIDs).

docker-compose.ci.yml

@@ -1,44 +1,45 @@
+x-airflow-common-overrides: &airflow-common-overrides
+  image: ${DOCKER_APP_IMAGE}
+  volumes: !override
+    - airflow_files:/opt/airflow/files
+
 services:
   airflow-apiserver:
+    <<: *airflow-common-overrides
     build: !reset
     develop: !reset
-    image: ${DOCKER_APP_IMAGE}
-    volumes: !reset
 
   airflow-cli:
+    <<: *airflow-common-overrides
     build: !reset
     develop: !reset
-    image: ${DOCKER_APP_IMAGE}
-    volumes: !reset
 
   airflow-dag-processor:
+    <<: *airflow-common-overrides
     build: !reset
     develop: !reset
-    image: ${DOCKER_APP_IMAGE}
-    volumes: !reset
 
   airflow-init:
+    <<: *airflow-common-overrides
     build: !reset
     develop: !reset
-    image: ${DOCKER_APP_IMAGE}
-    volumes: !reset
 
   airflow-scheduler:
+    <<: *airflow-common-overrides
     build: !reset
     develop: !reset
-    image: ${DOCKER_APP_IMAGE}
-    volumes: !reset
 
   airflow-triggerer:
+    <<: *airflow-common-overrides
     build: !reset
     develop: !reset
-    image: ${DOCKER_APP_IMAGE}
-    volumes: !reset
 
   airflow-worker:
+    <<: *airflow-common-overrides
     build: !reset
     develop: !reset
-    image: ${DOCKER_APP_IMAGE}
-    volumes: !reset
 
   flower: !reset
+
+volumes:
+  airflow_files: {}

docker-compose.yml

@@ -69,8 +69,8 @@ x-airflow-common: &airflow-common
   volumes:
     - ./:/opt/airflow
     - ./artifacts:/home/airflow/artifacts
+    - ./files:/opt/airflow/files
     - ./test:/opt/airflow/test
-    - ./public:/opt/airflow/public
   user: "${AIRFLOW_UID:-40093}:0"
 
 services:
@@ -123,11 +123,16 @@ services:
             --conn-password "${TIND_API_KEY}" \
             --conn-description "TIND API connection"
         fi
+        chown -cR airflow:root /opt/airflow/files
+        chmod 0775 /opt/airflow/files
     depends_on:
       postgres:
         condition: service_healthy
     entrypoint: /bin/bash
     restart: on-failure:3
+    # Runs as root to chown the files directory. Necessary in GitHub Actions
+    # where it's initialized as root:root.
+    user: root
 
   airflow-scheduler:
     <<: *airflow-common

files/.keep

@@ -0,0 +1,51 @@
+"""Template renderers for Batch Image emails, web pages, etc.
+
+Jinja2 templates live alongside this module as ``*.html`` files. Helpers in
+this package load them via a FileSystemLoader anchored to ``__file__``.
+"""
+
+from pathlib import Path
+
+from jinja2 import Environment, FileSystemLoader
+
+from mokelumne.plugins.static_files.helpers import static_path_to_url
+
+
+_env = Environment(
+    loader=FileSystemLoader(Path(__file__).parent),
+    autoescape=True,
+)
+
+
+def render_results_html(
+    query: str,
+    processed_path: Path, processed_count: int, processed_success: int, processed_failures: int,
+    fetched_path: Path, fetched_count: int, fetched_success: int, fetched_failures: int,
+    skipped_path: Path, skipped_count: int,
+    embed: bool,
+) -> str:
+    """Render the per-run results summary.
+
+    When ``embed`` is True, asset links resolve inside Airflow's sandboxed
+    "Files" iframe (text/plain downgrade). When False, links are suitable for
+    standalone browser use, e.g. an emailed summary.
+    """
+    return _env.get_template("results.html").render(
+        query=query,
+        processed={
+            "url": static_path_to_url(processed_path, embed=embed),
+            "count": processed_count,
+            "success": processed_success,
+            "failures": processed_failures,
+        },
+        fetched={
+            "url": static_path_to_url(fetched_path, embed=embed),
+            "count": fetched_count,
+            "success": fetched_success,
+            "failures": fetched_failures,
+        },
+        skipped={
+            "url": static_path_to_url(skipped_path, embed=embed),
+            "count": skipped_count,
+        },
+    )

mokelumne/batch_image/templates/__init__.py

@@ -0,0 +1,19 @@
+<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>Batch image description results</title>
+</head>
+<body>
+  <h1>Batch image description results</h1>
+  <p>Query: {{ query }}</p>
+  <dl>
+    <dt><a href="{{ processed.url }}">{{ processed.count }} images processed</a></dt>
+    <dd>{{ processed.success }} succeeded, {{ processed.failures }} failed</dd>
+    <dt><a href="{{ fetched.url }}">{{ fetched.count }} images fetched</a></dt>
+    <dd>{{ fetched.success }} succeeded, {{ fetched.failures }} failed</dd>
+    <dt><a href="{{ skipped.url }}">{{ skipped.count }} records skipped</a></dt>
+    <dd>These records did not match filter criteria</dd>
+  </dl>
+</body>
+</html>

mokelumne/batch_image/templates/results.html

@@ -7,26 +7,25 @@
 import logging
 from collections import namedtuple
 from datetime import datetime, UTC
-from html import escape
 from itertools import filterfalse
 from os import environ as ENV
 from pathlib import Path
 from shutil import copyfile
 from typing import List
-from uuid import uuid4
-
 from airflow.exceptions import AirflowFailException
 from airflow.providers.smtp.operators.smtp import EmailOperator
 from airflow.sdk import dag, task, task_group, Param, get_current_context
 from pymarc.marcxml import map_xml
 
 from langchain_aws import ChatBedrock
+from mokelumne.batch_image.templates import render_results_html
 from mokelumne.dags.fetch_tind_records import write_query_results_to_xml
 from mokelumne.providers.tind.hooks.tind import TindHook
 from mokelumne.util import langfuse
 from mokelumne.util.image_describer import ImageDescriber
 from mokelumne.util.image_fetcher import ImageFetcher, base64_size
-from mokelumne.util.storage import run_dir, public_dir, public_path_to_url
+from mokelumne.plugins.static_files.helpers import static_files_run_dir
+from mokelumne.util.storage import run_dir
 from mokelumne.util.tind_csv_writer import TindCsvWriter, is_single_image_record
 
 
@@ -357,9 +356,14 @@ def summarise_job():
 
         @task
         def generate_id() -> str:
-            """Generate a URL-safe directory for collated output."""
-            output_path = public_dir() / str(uuid4())
-            output_path.mkdir()  # We don't exist_ok=True because it should be unique.
+            """Return the per-run public output directory, creating it if needed.
+
+            The static_files plugin's DAG-run "Files" tab links to
+            ``<STATIC_FILES_ROOT>/<dag_id>/<run_id>/``, so collated outputs must
+            land there for the tab to find them.
+            """
+            context = get_current_context()
+            output_path = static_files_run_dir(context["dag"].dag_id, context["run_id"])
             return str(output_path)
 
         @task
@@ -413,22 +417,26 @@ def count_success_fail_of_csv(csv_file: Path, success: str) -> tuple[int, int, i
                 count_success_fail_of_csv(proc_path, "success")
             )
 
-            template_html = f"""
-<html><head><title>Batch image description results</title></head><body>
-<h1>Batch image description results</h1>
-<p>Query: {escape(params['tind_query'])}</p>
-<dl>
-    <dt><a href="{public_path_to_url(proc_path)}">{proc_count} images processed</a></dt>
-    <dd>{proc_success} succeeded, {proc_failures} failed</dd>
-    <dt><a href="{public_path_to_url(fetched_path)}">{fetch_count} images fetched</a></dt>
-    <dd>{fetch_success} succeeded, {fetch_failures} failed</dd>
-    <dt><a href="{public_path_to_url(skipped_path)}">{skip_count} records skipped</a></dt>
-    <dd>These records did not match filter criteria</dd>
-</dl>
-</body></html>"""
-
-            with (output_path / 'index.html').open('w') as html:
-                html.write(template_html)
+            def render(embed: bool) -> str:
+                return render_results_html(
+                    query=params['tind_query'],
+                    processed_path=proc_path,
+                    processed_count=proc_count,
+                    processed_success=proc_success,
+                    processed_failures=proc_failures,
+                    fetched_path=fetched_path,
+                    fetched_count=fetch_count,
+                    fetched_success=fetch_success,
+                    fetched_failures=fetch_failures,
+                    skipped_path=skipped_path,
+                    skipped_count=skip_count,
+                    embed=embed,
+                )
+
+            # index.html: served from AirFlow's iFrame, so downloads are blocked (embed=True)
+            (output_path / 'index.html').write_text(render(embed=True), encoding='utf-8')
+            # email.html: opened from email client, so downloads are allowed (embed=False)
+            (output_path / 'email.html').write_text(render(embed=False), encoding='utf-8')
 
             return output_dir_str
 
@@ -442,11 +450,10 @@ def notify_user(pub_directory):
 
         @task
         def render_email_template(asset_directory: str) -> str:
-            """Create the HTML template for the email message that will be sent."""
+            """Read the pre-rendered email body for this run."""
             asset_path = Path(str(asset_directory))
 
-            with (asset_path / "index.html").open(encoding='utf-8') as html:
-                return html.read()
+            return (asset_path / "email.html").read_text(encoding='utf-8')
 
         EmailOperator(
             task_id='send_email',

mokelumne/dags/gen_llm_image_descriptions.py

@@ -0,0 +1,40 @@
+from airflow.plugins_manager import AirflowPlugin
+
+from .config import (
+    STATIC_FILES_DAG_RUN_TAB_LABEL,
+    STATIC_FILES_EMBED_PARAM,
+    STATIC_FILES_PLUGIN_NAME,
+    STATIC_FILES_ROOT,
+    STATIC_FILES_ROUTE,
+    STATIC_FILES_URL_PREFIX,
+)
+from .helpers import static_file_path, static_files_root, static_path_to_url
+from .routes import files_router
+
+
+class StaticFilesPlugin(AirflowPlugin):
+    name = STATIC_FILES_PLUGIN_NAME
+
+    fastapi_apps = [
+        {
+            "app": files_router,
+            "name": STATIC_FILES_PLUGIN_NAME,
+            "url_prefix": STATIC_FILES_URL_PREFIX,
+        }
+    ]
+
+    external_views = [
+        {
+            "name": STATIC_FILES_DAG_RUN_TAB_LABEL,
+            "destination": "dag_run",
+            "url_route": STATIC_FILES_ROUTE,
+            "href": f"{STATIC_FILES_URL_PREFIX}/{{DAG_ID}}/{{RUN_ID}}/?{STATIC_FILES_EMBED_PARAM}=1",
+        }
+    ]
+
+    # Adds macros allowing the rest of AirFlow to address files served by this plugin.
+    macros = [
+        static_file_path,
+        static_files_root,
+        static_path_to_url,
+    ]