Fix downtime spam and speed up post-recovery detection (#1)

* Detect pre-release site downtime instead of spamming errors

Wizards takes the Secret Lair site down for hours before a new drop, which
used to produce one Discord error notification every 15 minutes per page.
Now the first 2 consecutive failed store-page checks (or a maintenance
holding page) flip the monitor into maintenance mode, fire a single
"Secret Lair Releasing Soon" notification, and poll every 60s instead of
every 3 minutes. When the site comes back, the next check runs
immediately so the new drop is announced within a poll interval instead
of 15+ minutes later. Also adds Cache-Control/Pragma no-cache headers
plus a cache-busting query param so stale CDN responses don't delay
detection of recovery.

* Burst-poll the top of the hour while in maintenance mode

Historically Secret Lair drops have gone live at :00, so a flat 60s
maintenance poll can still leave us up to a minute late. Within
MAINTENANCE_BURST_WINDOW_BEFORE/AFTER minutes of :00, switch to a 15s
poll with ±2s jitter (worst-case detection ~17s after the drop appears).
Outside that window, stay on the slower 60s maintenance interval so we
don't hammer the site for hours of every outage. Also drop the request
timeout to 10s while in maintenance mode so a hung connection can't eat
a whole burst interval.

* Ignore Python bytecode artifacts

---------

Co-authored-by: Claude <noreply@anthropic.com>

Author: epiphanyplx

.gitignore

@@ -1,2 +1,4 @@
 .env
 data/
+__pycache__/
+*.pyc

README.md

@@ -12,6 +12,7 @@ Monitors the [Secret Lair](https://secretlair.wizards.com/us/) store and [Chaos
 
 - Detects new product additions by tracking product IDs
 - Special "CHAOS VAULT IS OPEN" alert when the vault transitions from closed to active
+- Detects pre-release site downtime and fires a single "Releasing Soon" alert (instead of spamming for every failed poll), then polls faster so the new drop is announced quickly when the site returns
 - Rich Discord embeds with product name, price, and direct link
 - Persists state across restarts (won't re-notify on reboot)
 - First run silently catalogues existing products (no spam on initial deploy)
@@ -55,7 +56,14 @@ All config is via environment variables (set in `.env`):
 | Variable | Default | Description |
 |---|---|---|
 | `DISCORD_WEBHOOK_URL` | *(required)* | Discord webhook URL |
-| `POLL_INTERVAL_SECONDS` | `180` | Seconds between checks |
+| `POLL_INTERVAL_SECONDS` | `180` | Seconds between checks when the site is up |
+| `MAINTENANCE_POLL_INTERVAL_SECONDS` | `60` | Poll interval used while the site is down, *outside* the top-of-hour window |
+| `MAINTENANCE_BURST_INTERVAL_SECONDS` | `15` | Fast poll interval while in maintenance mode and near :00 (drops historically go live on the hour) |
+| `MAINTENANCE_BURST_JITTER_SECONDS` | `2` | ±jitter added to the burst interval to avoid looking like a cron job |
+| `MAINTENANCE_BURST_WINDOW_BEFORE` | `2` | Minutes before :00 to start burst-polling |
+| `MAINTENANCE_BURST_WINDOW_AFTER` | `2` | Minutes after :00 to keep burst-polling |
+| `MAINTENANCE_REQUEST_TIMEOUT_SECONDS` | `10` | HTTP timeout while in maintenance mode (so a hung request can't eat a burst interval) |
+| `MAINTENANCE_STRIKE_THRESHOLD` | `2` | Consecutive failed checks required before declaring maintenance mode (filters transient blips) |
 | `MONITOR_SHOP_ALL` | `true` | Also monitor `/us/shopall` |
 | `NOTIFY_ON_START` | `false` | Notify for existing products on first run |
 | `LOG_LEVEL` | `INFO` | Log verbosity |

monitor.py

@@ -8,6 +8,7 @@
 import json
 import logging
 import os
+import random
 import re
 import sys
 import time
@@ -22,6 +23,35 @@
 # ---------------------------------------------------------------------------
 DISCORD_WEBHOOK_URL = os.environ.get("DISCORD_WEBHOOK_URL", "")
 POLL_INTERVAL_SECONDS = int(os.environ.get("POLL_INTERVAL_SECONDS", "180"))  # 3 min default
+# While the site is down (likely a pre-release maintenance window), poll faster
+# so we catch the new drop quickly once it's back.
+MAINTENANCE_POLL_INTERVAL_SECONDS = int(
+    os.environ.get("MAINTENANCE_POLL_INTERVAL_SECONDS", "60")
+)
+# Historical Secret Lair drops have gone live on the hour, so while in
+# maintenance mode we ramp up to a near-realtime poll for a few minutes on
+# either side of :00 and back off to the slower maintenance interval otherwise.
+MAINTENANCE_BURST_INTERVAL_SECONDS = int(
+    os.environ.get("MAINTENANCE_BURST_INTERVAL_SECONDS", "15")
+)
+MAINTENANCE_BURST_JITTER_SECONDS = int(
+    os.environ.get("MAINTENANCE_BURST_JITTER_SECONDS", "2")
+)
+# How many minutes before / after :00 to burst-poll.
+MAINTENANCE_BURST_WINDOW_BEFORE = int(
+    os.environ.get("MAINTENANCE_BURST_WINDOW_BEFORE", "2")
+)
+MAINTENANCE_BURST_WINDOW_AFTER = int(
+    os.environ.get("MAINTENANCE_BURST_WINDOW_AFTER", "2")
+)
+# Shorter request timeout while the site is down so a hung request can't eat
+# the whole poll interval.
+MAINTENANCE_REQUEST_TIMEOUT_SECONDS = int(
+    os.environ.get("MAINTENANCE_REQUEST_TIMEOUT_SECONDS", "10")
+)
+# How many consecutive failed checks before declaring maintenance mode. Filters
+# out transient network blips so we don't ping Discord for one-off failures.
+MAINTENANCE_STRIKE_THRESHOLD = int(os.environ.get("MAINTENANCE_STRIKE_THRESHOLD", "2"))
 STATE_FILE = os.environ.get("STATE_FILE", "/data/state.json")
 LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO").upper()
 USER_AGENT = os.environ.get(
@@ -84,19 +114,52 @@ def save_state(state: dict) -> None:
 PRODUCT_LINK_RE = re.compile(r"/us/product/(\d+)(?:/([^\"'\s]*))?")
 
 
-def fetch_page(url: str) -> str | None:
-    """Fetch a page's HTML. Returns None on failure."""
-    headers = {"User-Agent": USER_AGENT, "Accept-Language": "en-US,en;q=0.9"}
+def fetch_page(url: str, timeout: int = 30) -> str | None:
+    """Fetch a page's HTML. Returns None on failure.
+
+    Sends cache-busting headers so we don't get a stale 'we'll be right back'
+    page from a CDN edge after Wizards brings the site back online.
+    """
+    headers = {
+        "User-Agent": USER_AGENT,
+        "Accept-Language": "en-US,en;q=0.9",
+        "Cache-Control": "no-cache",
+        "Pragma": "no-cache",
+    }
+    sep = "&" if "?" in url else "?"
+    bust_url = f"{url}{sep}_={int(time.time())}"
     try:
-        resp = requests.get(url, headers=headers, timeout=30)
+        resp = requests.get(bust_url, headers=headers, timeout=timeout)
         resp.raise_for_status()
         return resp.text
     except requests.RequestException as e:
-        log.error("Failed to fetch %s: %s", url, e)
-        send_discord_error_notification(str(e), context=f"Failed to fetch `{url}`")
+        log.warning("Failed to fetch %s: %s", url, e)
         return None
 
 
+# Phrases that indicate Wizards has the site behind a maintenance/holding page
+# (HTTP 200, but the body says it's down). Matched case-insensitively.
+_MAINTENANCE_INDICATORS = (
+    "we'll be right back",
+    "we will be right back",
+    "be right back",
+    "under maintenance",
+    "site is temporarily unavailable",
+    "temporarily unavailable",
+    "503 service unavailable",
+    "service is unavailable",
+    "site is currently down",
+)
+
+
+def is_maintenance_page(html: str | None) -> bool:
+    """Detect a 'site down' holding page that loaded as 200 OK."""
+    if not html:
+        return False
+    lower = html.lower()
+    return any(indicator in lower for indicator in _MAINTENANCE_INDICATORS)
+
+
 def parse_products(html: str) -> dict[str, dict]:
     """
     Extract products from HTML.
@@ -233,6 +296,45 @@ def send_discord_notification(products: list[dict], source: str) -> bool:
     return True
 
 
+def send_release_soon_notification() -> bool:
+    """One-shot notification fired when the site goes into maintenance mode.
+
+    Replaces the per-fetch error spam that used to fire during the multi-hour
+    pre-release window.
+    """
+    if not DISCORD_WEBHOOK_URL:
+        return False
+
+    payload = {
+        "username": "Secret Lair Monitor",
+        "avatar_url": "https://cdn-prod.scalefast.com/public/assets/img/resized/"
+                      "wizardsofthecoast-secret-lair/favicon-32.png",
+        "content": "**🃏 Secret Lair Releasing Soon!**",
+        "embeds": [
+            {
+                "title": "Secret Lair site is down",
+                "url": "https://secretlair.wizards.com/us/",
+                "color": 0xF1C40F,  # Yellow
+                "description": (
+                    "The Secret Lair site is currently unavailable, which usually "
+                    "means a new drop is about to go live. Monitoring more "
+                    "frequently — you'll get a notification as soon as the new "
+                    "products appear."
+                ),
+                "timestamp": datetime.now(timezone.utc).isoformat(),
+            }
+        ],
+    }
+    try:
+        resp = requests.post(DISCORD_WEBHOOK_URL, json=payload, timeout=15)
+        resp.raise_for_status()
+        log.info("'Releasing Soon' notification sent")
+        return True
+    except requests.RequestException as e:
+        log.error("Failed to send 'Releasing Soon' notification: %s", e)
+        return False
+
+
 def send_chaos_vault_opened_notification() -> bool:
     """Special notification when the Chaos Vault transitions from closed to open."""
     if not DISCORD_WEBHOOK_URL:
@@ -312,16 +414,83 @@ def send_discord_error_notification(error_msg: str, context: str = "") -> bool:
 # Main loop
 # ---------------------------------------------------------------------------
 
-def run_check(state: dict) -> dict:
-    """Run one check cycle across all monitored pages."""
+def in_top_of_hour_burst_window() -> bool:
+    """True when the wall clock is in the configured top-of-hour burst window.
+
+    Minute-of-hour is the same across all whole-hour timezones, so we don't
+    need to know which timezone Wizards releases in.
+    """
+    minute = datetime.now().minute
+    return (
+        minute >= 60 - MAINTENANCE_BURST_WINDOW_BEFORE
+        or minute <= MAINTENANCE_BURST_WINDOW_AFTER
+    )
+
+
+def compute_sleep_interval(in_maintenance: bool) -> float:
+    """Pick the next sleep duration based on site state and clock position."""
+    if not in_maintenance:
+        return float(POLL_INTERVAL_SECONDS)
+    if in_top_of_hour_burst_window():
+        jitter = random.uniform(
+            -MAINTENANCE_BURST_JITTER_SECONDS, MAINTENANCE_BURST_JITTER_SECONDS
+        )
+        return max(1.0, MAINTENANCE_BURST_INTERVAL_SECONDS + jitter)
+    return float(MAINTENANCE_POLL_INTERVAL_SECONDS)
+
+
+def run_check(state: dict) -> tuple[dict, bool]:
+    """Run one check cycle across all monitored pages.
+
+    Returns (state, in_maintenance). When the site is in maintenance mode we
+    skip product parsing and let the caller use a faster poll interval.
+    """
     known = state.get("known_products", {})
     chaos_vault_was_active = state.get("chaos_vault_active", False)
+    was_in_maintenance = state.get("maintenance_mode", False)
+    strikes = state.get("maintenance_strikes", 0)
+
+    # While we were already in maintenance mode, use a tight request timeout so
+    # one hung connection can't blow past the burst-poll interval.
+    fetch_timeout = (
+        MAINTENANCE_REQUEST_TIMEOUT_SECONDS if was_in_maintenance else 30
+    )
+
+    # Use the main store page as the canonical signal for site health. The
+    # chaos vault page is legitimately empty most of the time, and shop_all
+    # can lag, so we don't want either to be the trigger.
+    store_html = fetch_page(PAGES["store"], timeout=fetch_timeout)
+    store_down = store_html is None or is_maintenance_page(store_html)
 
+    if store_down:
+        strikes += 1
+        log.info(
+            "Secret Lair store appears down (strike %d/%d)",
+            strikes,
+            MAINTENANCE_STRIKE_THRESHOLD,
+        )
+        state["maintenance_strikes"] = strikes
+        if strikes >= MAINTENANCE_STRIKE_THRESHOLD and not was_in_maintenance:
+            send_release_soon_notification()
+            state["maintenance_mode"] = True
+        state["last_check"] = datetime.now(timezone.utc).isoformat()
+        return state, state.get("maintenance_mode", False)
+
+    # Site is up — reset strike counter and announce recovery if needed.
+    state["maintenance_strikes"] = 0
+    if was_in_maintenance:
+        log.info("Secret Lair site is back online — checking for new products")
+        state["maintenance_mode"] = False
+
+    # Process the store page we already fetched, then the rest.
     for source, url in PAGES.items():
-        log.debug("Checking %s: %s", source, url)
-        html = fetch_page(url)
-        if html is None:
-            continue
+        if source == "store":
+            html = store_html
+        else:
+            log.debug("Checking %s: %s", source, url)
+            html = fetch_page(url, timeout=fetch_timeout)
+            if html is None:
+                continue
 
         # Special Chaos Vault open/close detection
         if source == "chaos_vault":
@@ -351,7 +520,7 @@ def run_check(state: dict) -> dict:
 
     state["known_products"] = known
     state["last_check"] = datetime.now(timezone.utc).isoformat()
-    return state
+    return state, False
 
 
 def main() -> None:
@@ -381,7 +550,7 @@ def main() -> None:
         if not notify_on_start:
             globals()["DISCORD_WEBHOOK_URL"] = ""
 
-        state = run_check(state)
+        state, _ = run_check(state)
         save_state(state)
 
         if not notify_on_start:
@@ -392,19 +561,30 @@ def main() -> None:
             len(state.get("known_products", {})),
         )
 
+    in_maintenance = state.get("maintenance_mode", False)
     while True:
+        was_in_maintenance = in_maintenance
         try:
-            state = run_check(state)
+            state, in_maintenance = run_check(state)
             save_state(state)
         except Exception as e:
             log.exception("Unhandled error during check cycle")
             send_discord_error_notification(
                 f"{type(e).__name__}: {e}",
                 context="Unhandled error during check cycle",
             )
+            in_maintenance = state.get("maintenance_mode", False)
+
+        # If we just transitioned from maintenance → up, skip the sleep and
+        # check again immediately so the new drop is announced as soon as it
+        # appears rather than waiting another poll interval.
+        if was_in_maintenance and not in_maintenance:
+            log.info("Recovery detected — re-checking immediately")
+            continue
 
-        log.debug("Sleeping %ds until next check", POLL_INTERVAL_SECONDS)
-        time.sleep(POLL_INTERVAL_SECONDS)
+        interval = compute_sleep_interval(in_maintenance)
+        log.debug("Sleeping %.1fs until next check", interval)
+        time.sleep(interval)
 
 
 if __name__ == "__main__":