Exclude celery.app.trace from database log storage (v2.6.53)

Per-task INFO rows from celery.app.trace generated heavy WAL volume
during maintenance runs, driving PostgreSQL checkpoint IO storms that
stalled the producer loops. Added to the default log-exclusion list
with an additive startup migration backfilling existing installs.

Author: ttlequals0

CHANGELOG.MD

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0).
 
+## [2.6.53] - 2026-06-11
+
+### Fixed
+
+- **Maintenance runs no longer trigger 5-minute checkpoint IO storms.** During the verified cleanup run, recurring 2-7 minute slowdowns aligned to the second with PostgreSQL checkpoint write phases. Cause: `celery.app.trace` logs one INFO row per task ("succeeded in Xs" with the full result dict), and the database log handler stored all of them - ~150-190MB of WAL per 5 minutes at ~850 tasks/s, forcing heavy checkpoints that stalled the producer's own DB operations. `celery.app.trace` is now in the default log-exclusion list, and a startup migration appends it to the stored config on existing installs (additive - user customizations survive). Container/Loki logging is unaffected.
+
 ## [2.6.52] - 2026-06-11
 
 ### Fixed

docker-compose.yml

@@ -63,7 +63,7 @@ services:
 
   # PixelProbe application
   pixelprobe:
-    image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.6.52}
+    image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.6.53}
     container_name: pixelprobe-app
     environment:
       # Security
@@ -131,7 +131,7 @@ services:
 
   # P1 Celery Worker for distributed task processing
   celery-worker:
-    image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.6.52}
+    image: ttlequals0/pixelprobe:${PIXELPROBE_VERSION:-2.6.53}
     container_name: pixelprobe-celery-worker
     command: python celery_worker.py
     environment:

pixelprobe/constants.py

@@ -98,8 +98,11 @@
 CONFIG_LOG_RETENTION_DAYS = 'log_retention_days'
 CONFIG_LOG_EXCLUDE_LOGGERS = 'log_exclude_loggers'
 
-# Default excluded loggers for database log storage
-DEFAULT_LOG_EXCLUDE_LOGGERS = 'urllib3,werkzeug,celery.worker.strategy,celery.bootsteps,kombu,amqp'
+# Default excluded loggers for database log storage.
+# celery.app.trace emits one INFO row per task ("succeeded in Xs" with the
+# full result): at ~850 tasks/s during maintenance runs that is ~150MB of WAL
+# per 5 minutes, driving checkpoint IO storms that stall the producer loops
+DEFAULT_LOG_EXCLUDE_LOGGERS = 'urllib3,werkzeug,celery.worker.strategy,celery.app.trace,celery.bootsteps,kombu,amqp'
 
 # Sentinel value for system (non-scan) logs
 SYSTEM_LOG_ID = 'system'

pixelprobe/migrations/startup.py

@@ -317,6 +317,28 @@ def run_v2_6_49_migrations(db):
         logger.error(f"Migration v2.6.49 failed: {e}")
 
 
+def run_v2_6_53_migrations(db):
+    """Append celery.app.trace to the stored log-exclusion list if absent.
+
+    The exclude config is seeded once (ON CONFLICT DO NOTHING), so default
+    changes never reach existing installs. Additive so user customizations
+    survive. celery.app.trace logs one row per task; at maintenance-run rates
+    that WAL volume drives 5-minute checkpoint IO storms.
+    """
+    try:
+        with migration_connection(db) as conn:
+            conn.execute(text("""
+                UPDATE app_configs
+                SET value = value || ',celery.app.trace'
+                WHERE key = :key
+                  AND ',' || replace(value, ' ', '') || ',' NOT LIKE '%,celery.app.trace,%'
+            """), {'key': CONFIG_LOG_EXCLUDE_LOGGERS})
+            conn.commit()
+            logger.info("v2.6.53 log-exclusion backfill completed")
+    except Exception as e:
+        logger.error(f"Migration v2.6.53 failed: {e}")
+
+
 def create_performance_indexes(db):
     """Create performance indexes"""
     indexes = [
@@ -402,6 +424,12 @@ def _run_all_migrations(db):
     except Exception as e:
         logger.error(f"v2.6.49 migration failed: {e}")
 
+    logger.info("Running v2.6.53 migration (log-exclusion backfill)...")
+    try:
+        run_v2_6_53_migrations(db)
+    except Exception as e:
+        logger.error(f"v2.6.53 migration failed: {e}")
+
     logger.info("Creating performance indexes...")
     try:
         create_performance_indexes(db)

pixelprobe/version.py

@@ -4,7 +4,7 @@
 # Default version - this is the single source of truth
 
 
-_DEFAULT_VERSION = '2.6.52'
+_DEFAULT_VERSION = '2.6.53'
 
 
 # Allow override via environment variable for CI/CD, but default to the hardcoded version