From 7db8c20276aac47aa73b718a6cc903c02332ab2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 5 Jun 2026 22:39:54 +0200 Subject: [PATCH 1/2] Add server-side daily cache for usage API data Usage page data changes infrequently and is expensive to fetch from the upstream API. Flask now proxies /api/usage/ requests and caches responses as JSON files in /tmp/packit-dashboard-usage-cache/, keyed by endpoint and date. All WSGI workers share the same file-based cache. Stale files from previous days are evicted on cache miss. Frontend fetches usage data from the Flask proxy (same origin) instead of the external API directly. Tabs render all content eagerly (removed mountOnEnter) so all data loads upfront. Vite dev server proxies /api/usage to localhost Flask for local development. - Flask proxy with file-based daily cache over in-memory dict to share across WSGI workers - Frontend switched from VITE_API_URL to relative /api/usage/ paths - All tabs rendered immediately instead of on-demand Assisted-By: Claude Opus 4.6 --- frontend/src/components/usage/Usage.tsx | 1 - .../src/components/usage/UsageInterval.tsx | 4 +- frontend/src/components/usage/UsageList.tsx | 14 ++-- frontend/vite.config.ts | 3 + packit_dashboard/app.py | 2 + packit_dashboard/usage/__init__.py | 2 + packit_dashboard/usage/routes.py | 78 +++++++++++++++++++ 7 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 packit_dashboard/usage/__init__.py create mode 100644 packit_dashboard/usage/routes.py diff --git a/frontend/src/components/usage/Usage.tsx b/frontend/src/components/usage/Usage.tsx index 28ade0f1..08276a2a 100644 --- a/frontend/src/components/usage/Usage.tsx +++ b/frontend/src/components/usage/Usage.tsx @@ -38,7 +38,6 @@ const Usage = () => { fetch( - `${import.meta.env.VITE_API_URL}/usage/intervals?days=${ - granularity.days - }&hours=${granularity.hours}&count=${granularity.count}`, + `/api/usage/intervals?days=${granularity.days}&hours=${granularity.hours}&count=${granularity.count}`, ).then((response) => { if (!response.ok) { throw Promise.reject(response); diff --git a/frontend/src/components/usage/UsageList.tsx b/frontend/src/components/usage/UsageList.tsx index f29b28a5..e1b4d44e 100644 --- a/frontend/src/components/usage/UsageList.tsx +++ b/frontend/src/components/usage/UsageList.tsx @@ -22,14 +22,12 @@ import { Preloader } from "../shared/Preloader"; import { UsageListData } from "./UsageListData"; const fetchDataByGranularity = (granularity: UsageListProps["what"]) => - fetch(`${import.meta.env.VITE_API_URL}/usage/${granularity}`).then( - (response) => { - if (!response.ok) { - throw Promise.reject(response); - } - return response.json(); - }, - ); + fetch(`/api/usage/${granularity}`).then((response) => { + if (!response.ok) { + throw Promise.reject(response); + } + return response.json(); + }); interface UsageListProps { what: "past-day" | "past-week" | "past-month" | "past-year" | "total"; diff --git a/frontend/vite.config.ts b/frontend/vite.config.ts index 46fe0155..3e72f789 100644 --- a/frontend/vite.config.ts +++ b/frontend/vite.config.ts @@ -47,6 +47,9 @@ export default defineConfig(() => ({ ], server: { open: true, + proxy: { + "/api/usage": "http://localhost:5000", + }, }, test: { environment: "happy-dom", diff --git a/packit_dashboard/app.py b/packit_dashboard/app.py index cbc558b3..75c795ab 100644 --- a/packit_dashboard/app.py +++ b/packit_dashboard/app.py @@ -8,6 +8,7 @@ from packit_dashboard.api.routes import api from packit_dashboard.home.routes import home +from packit_dashboard.usage.routes import usage app = Flask( "Packit Service Dashboard", @@ -16,6 +17,7 @@ # Note: Declare any other flask blueprints or routes above this. # Routes declared below this will be rendered by React +app.register_blueprint(usage) app.register_blueprint(home) app.register_blueprint(api) diff --git a/packit_dashboard/usage/__init__.py b/packit_dashboard/usage/__init__.py new file mode 100644 index 00000000..e01ff12d --- /dev/null +++ b/packit_dashboard/usage/__init__.py @@ -0,0 +1,2 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT diff --git a/packit_dashboard/usage/routes.py b/packit_dashboard/usage/routes.py new file mode 100644 index 00000000..7b0498b9 --- /dev/null +++ b/packit_dashboard/usage/routes.py @@ -0,0 +1,78 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import json +from datetime import date +from logging import getLogger +from pathlib import Path + +import requests +from flask import Blueprint, jsonify, request + +from packit_dashboard.config import API_URL + +logger = getLogger("packit_dashboard") + +usage = Blueprint("usage", __name__) + +CACHE_DIR = Path("/tmp/packit-dashboard-usage-cache") + + +def _cache_path(cache_key: str) -> Path: + today = date.today().isoformat() + return CACHE_DIR / f"{cache_key}_{today}.json" + + +def _evict_stale(): + if not CACHE_DIR.exists(): + return + today = date.today().isoformat() + for f in CACHE_DIR.iterdir(): + if not f.name.endswith(f"_{today}.json"): + f.unlink(missing_ok=True) + + +def _get_cached_or_fetch(cache_key: str, url: str) -> dict: + path = _cache_path(cache_key) + + if path.exists(): + return json.loads(path.read_text()) + + _evict_stale() + + response = requests.get(url, timeout=60) + response.raise_for_status() + data = response.json() + + CACHE_DIR.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data)) + return data + + +@usage.route("/api/usage/") +def usage_by_granularity(granularity): + try: + data = _get_cached_or_fetch( + f"usage-{granularity}", + f"{API_URL}/usage/{granularity}", + ) + return jsonify(data) + except Exception: + logger.exception("Failed to fetch usage data for %s", granularity) + return jsonify({"error": "Failed to fetch usage data"}), 502 + + +@usage.route("/api/usage/intervals") +def usage_intervals(): + days = request.args.get("days", "") + hours = request.args.get("hours", "") + count = request.args.get("count", "") + try: + data = _get_cached_or_fetch( + f"usage-intervals-{days}-{hours}-{count}", + f"{API_URL}/usage/intervals?days={days}&hours={hours}&count={count}", + ) + return jsonify(data) + except Exception: + logger.exception("Failed to fetch usage intervals") + return jsonify({"error": "Failed to fetch usage data"}), 502 From 195550e3dc0d7b65eddda8267c7f9e9aa7069158 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Such=C3=BD?= Date: Fri, 5 Jun 2026 22:49:51 +0200 Subject: [PATCH 2/2] Add daily cache warm-up for usage data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract cache logic into cache.py module shared by Flask routes and the new warm_cache.py script. The startup script pre-warms the cache on boot and schedules a background loop that re-fetches all usage endpoints at 00:01 each day. - No crond needed — background sleep loop in run_httpd.sh calculates seconds until next 00:01 and sleeps precisely - Cache files in /tmp shared across WSGI workers, warm_cache populates them before any user request hits Assisted-By: Claude Opus 4.6 --- files/scripts/run_httpd.sh | 8 ++++ packit_dashboard/usage/cache.py | 41 ++++++++++++++++++ packit_dashboard/usage/routes.py | 42 ++---------------- packit_dashboard/usage/warm_cache.py | 64 ++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 39 deletions(-) create mode 100644 packit_dashboard/usage/cache.py create mode 100755 packit_dashboard/usage/warm_cache.py diff --git a/files/scripts/run_httpd.sh b/files/scripts/run_httpd.sh index 3a0abe59..9edba4b4 100755 --- a/files/scripts/run_httpd.sh +++ b/files/scripts/run_httpd.sh @@ -14,6 +14,14 @@ fi # open HTTP/2 connection gets reused API_SERVER_NAME="${DEPLOYMENT}.packit.dev" +# Pre-warm the usage cache at startup and schedule daily refresh at 00:01 +PYTHONPATH=/src python3 -m packit_dashboard.usage.warm_cache || true +(while true; do + seconds_until_midnight=$(( $(date -d "tomorrow 00:01" +%s) - $(date +%s) )) + sleep "${seconds_until_midnight}" + PYTHONPATH=/src python3 -m packit_dashboard.usage.warm_cache || true +done) & + # See "mod_wsgi-express-3 start-server --help" for details on # these options, and the configuration documentation of mod_wsgi: # https://modwsgi.readthedocs.io/en/master/configuration.html diff --git a/packit_dashboard/usage/cache.py b/packit_dashboard/usage/cache.py new file mode 100644 index 00000000..a7c523d7 --- /dev/null +++ b/packit_dashboard/usage/cache.py @@ -0,0 +1,41 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import json +from datetime import date +from pathlib import Path + +import requests + +CACHE_DIR = Path("/tmp/packit-dashboard-usage-cache") + + +def _cache_path(cache_key: str) -> Path: + today = date.today().isoformat() + return CACHE_DIR / f"{cache_key}_{today}.json" + + +def evict_stale(): + if not CACHE_DIR.exists(): + return + today = date.today().isoformat() + for f in CACHE_DIR.iterdir(): + if not f.name.endswith(f"_{today}.json"): + f.unlink(missing_ok=True) + + +def get_cached_or_fetch(cache_key: str, url: str) -> dict: + path = _cache_path(cache_key) + + if path.exists(): + return json.loads(path.read_text()) + + evict_stale() + + response = requests.get(url, timeout=60) + response.raise_for_status() + data = response.json() + + CACHE_DIR.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data)) + return data diff --git a/packit_dashboard/usage/routes.py b/packit_dashboard/usage/routes.py index 7b0498b9..3ed90aa2 100644 --- a/packit_dashboard/usage/routes.py +++ b/packit_dashboard/usage/routes.py @@ -1,58 +1,22 @@ # Copyright Contributors to the Packit project. # SPDX-License-Identifier: MIT -import json -from datetime import date from logging import getLogger -from pathlib import Path -import requests from flask import Blueprint, jsonify, request from packit_dashboard.config import API_URL +from packit_dashboard.usage.cache import get_cached_or_fetch logger = getLogger("packit_dashboard") usage = Blueprint("usage", __name__) -CACHE_DIR = Path("/tmp/packit-dashboard-usage-cache") - - -def _cache_path(cache_key: str) -> Path: - today = date.today().isoformat() - return CACHE_DIR / f"{cache_key}_{today}.json" - - -def _evict_stale(): - if not CACHE_DIR.exists(): - return - today = date.today().isoformat() - for f in CACHE_DIR.iterdir(): - if not f.name.endswith(f"_{today}.json"): - f.unlink(missing_ok=True) - - -def _get_cached_or_fetch(cache_key: str, url: str) -> dict: - path = _cache_path(cache_key) - - if path.exists(): - return json.loads(path.read_text()) - - _evict_stale() - - response = requests.get(url, timeout=60) - response.raise_for_status() - data = response.json() - - CACHE_DIR.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(data)) - return data - @usage.route("/api/usage/") def usage_by_granularity(granularity): try: - data = _get_cached_or_fetch( + data = get_cached_or_fetch( f"usage-{granularity}", f"{API_URL}/usage/{granularity}", ) @@ -68,7 +32,7 @@ def usage_intervals(): hours = request.args.get("hours", "") count = request.args.get("count", "") try: - data = _get_cached_or_fetch( + data = get_cached_or_fetch( f"usage-intervals-{days}-{hours}-{count}", f"{API_URL}/usage/intervals?days={days}&hours={hours}&count={count}", ) diff --git a/packit_dashboard/usage/warm_cache.py b/packit_dashboard/usage/warm_cache.py new file mode 100755 index 00000000..a937c64e --- /dev/null +++ b/packit_dashboard/usage/warm_cache.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +"""Pre-generate usage cache files. Run via cron at 00:01 daily.""" + +import logging +import sys + +from packit_dashboard.config import API_URL +from packit_dashboard.usage.cache import evict_stale, get_cached_or_fetch + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("packit_dashboard") + +GRANULARITIES = ["past-day", "past-week", "past-month", "past-year", "total"] +INTERVALS = [ + {"days": 0, "hours": 1, "count": 24}, + {"days": 1, "hours": 0, "count": 7}, + {"days": 1, "hours": 0, "count": 30}, + {"days": 7, "hours": 0, "count": 52}, +] + + +def main(): + evict_stale() + errors = 0 + + for granularity in GRANULARITIES: + try: + get_cached_or_fetch( + f"usage-{granularity}", + f"{API_URL}/usage/{granularity}", + ) + logger.info("Cached usage/%s", granularity) + except Exception: + logger.exception("Failed to fetch usage/%s", granularity) + errors += 1 + + for interval in INTERVALS: + days, hours, count = interval["days"], interval["hours"], interval["count"] + try: + get_cached_or_fetch( + f"usage-intervals-{days}-{hours}-{count}", + f"{API_URL}/usage/intervals?days={days}&hours={hours}&count={count}", + ) + logger.info( + "Cached usage/intervals days=%s hours=%s count=%s", days, hours, count + ) + except Exception: + logger.exception( + "Failed to fetch usage/intervals days=%s hours=%s count=%s", + days, + hours, + count, + ) + errors += 1 + + if errors: + sys.exit(1) + + +if __name__ == "__main__": + main()