HAM-113: bound /var/log so log spam can't fill the SD card

files/journald-sensor-bounds.conf

@@ -0,0 +1,10 @@
+# Managed by mjolnir-hamma (HAM-113) -> /etc/systemd/journald.conf.d/00-sensor-bounds.conf
+#
+# Bound systemd-journald disk use so a runaway log source (e.g. brokkr
+# science_ingest spamming when the AGS is down, HAM-112) cannot fill the SD
+# card. Do NOT edit on the sensor -- change files/journald-sensor-bounds.conf
+# in the repo and re-run install.sh (or scripts/apply_log_bounds.sh).
+[Journal]
+SystemMaxUse=500M
+SystemKeepFree=1G
+RuntimeMaxUse=100M

files/logrotate-hourly.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+# Managed by mjolnir-hamma (HAM-113) -> /etc/cron.hourly/mjolnir-logrotate
+#
+# Run logrotate hourly so a log that exceeds `maxsize` (set on the rsyslog
+# stanza) is rotated within the hour instead of waiting for the daily run.
+# This is what bounds /var/log under sustained log spam.
+#
+# Safe/idempotent: `daily`/`weekly` stanzas still rotate on their normal
+# cadence; `maxsize` only forces an extra rotation when a log has grown past
+# the cap since the last run.
+/usr/sbin/logrotate /etc/logrotate.conf
+exit 0

scripts/apply_log_bounds.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+#
+# apply_log_bounds.sh (HAM-113)
+#
+# Apply the /var/log size bounds to an ALREADY-DEPLOYED sensor, without a full
+# reinstall. Idempotent -- safe to run repeatedly. Run ON the sensor (needs
+# sudo). This is the fleet-remediation counterpart to configure_log_bounds() in
+# unified_install/lib/hardware.sh.
+#
+# Usage:
+#   sudo bash scripts/apply_log_bounds.sh            # apply the bounds
+#   sudo bash scripts/apply_log_bounds.sh --check    # report status, no changes
+#
+# Applies:
+#   1. journald SystemMaxUse cap (drop-in)
+#   2. maxsize 100M on the rsyslog logrotate stanzas (backup at .mjolnir-orig)
+#   3. hourly logrotate run so maxsize is enforced within the hour
+# and reclaims any current overage (journal vacuum + one logrotate pass).
+
+set -euo pipefail
+
+CHECK_ONLY=false
+if [[ "${1:-}" == "--check" ]]; then
+    CHECK_ONLY=true
+elif [[ -n "${1:-}" ]]; then
+    echo "Unknown argument: $1" >&2
+    echo "Usage: $0 [--check]" >&2
+    exit 2
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+FILES_DIR="$(cd "$SCRIPT_DIR/../files" && pwd)"
+
+JOURNALD_DST="/etc/systemd/journald.conf.d/00-sensor-bounds.conf"
+RSYSLOG_LR="/etc/logrotate.d/rsyslog"
+# Backup MUST live outside /etc/logrotate.d/ -- logrotate reads every file in
+# that dir, so a backup there causes "duplicate log entry" errors.
+RSYSLOG_BAK="/var/backups/logrotate-rsyslog.mjolnir-orig"
+CRON_DST="/etc/cron.hourly/mjolnir-logrotate"
+
+log() { echo "[apply-log-bounds] $*"; }
+
+log "Current /var/log usage:"
+du -sh /var/log 2>/dev/null || true
+df -h / | awk 'NR==1 || /\/$/ {print}'
+
+if [[ "$CHECK_ONLY" == "true" ]]; then
+    log "Status (--check, no changes made):"
+    if [[ -f "$JOURNALD_DST" ]]; then log "  journald cap:      PRESENT"; else log "  journald cap:      MISSING"; fi
+    if grep -q "maxsize" "$RSYSLOG_LR" 2>/dev/null; then log "  rsyslog maxsize:   PRESENT"; else log "  rsyslog maxsize:   MISSING"; fi
+    if [[ -f "$CRON_DST" ]]; then log "  hourly logrotate:  PRESENT"; else log "  hourly logrotate:  MISSING"; fi
+    exit 0
+fi
+
+# --- 1. journald cap ---
+sudo mkdir -p "$(dirname "$JOURNALD_DST")"
+sudo cp "$FILES_DIR/journald-sensor-bounds.conf" "$JOURNALD_DST"
+sudo chmod 0644 "$JOURNALD_DST"
+sudo systemctl restart systemd-journald
+log "journald cap applied and journald restarted"
+
+# --- 2. rsyslog logrotate maxsize (idempotent, with one-time backup) ---
+if [[ -f "$RSYSLOG_LR" ]]; then
+    if grep -q "maxsize" "$RSYSLOG_LR"; then
+        log "rsyslog logrotate already has a maxsize cap; leaving as-is"
+    else
+        sudo mkdir -p "$(dirname "$RSYSLOG_BAK")"
+        sudo cp -a "$RSYSLOG_LR" "$RSYSLOG_BAK"
+        sudo sed -i '/^{/a\    maxsize 100M' "$RSYSLOG_LR"
+        log "added 'maxsize 100M' to $RSYSLOG_LR (backup at $RSYSLOG_BAK)"
+    fi
+else
+    log "WARN: $RSYSLOG_LR not found; skipping rsyslog cap"
+fi
+
+# --- 3. hourly logrotate ---
+sudo cp "$FILES_DIR/logrotate-hourly.sh" "$CRON_DST"
+sudo chmod 0755 "$CRON_DST"
+log "hourly logrotate installed at $CRON_DST"
+
+# --- 4. reclaim any current overage now ---
+sudo journalctl --vacuum-size=500M >/dev/null 2>&1 || true
+sudo /usr/sbin/logrotate /etc/logrotate.conf || true
+
+log "Done. New /var/log usage:"
+du -sh /var/log 2>/dev/null || true

tests/unified/test_log_bounds.py

@@ -0,0 +1,133 @@
+"""
+Tests for the HAM-113 log-size bounds feature.
+
+Covers the shipped config artifacts, the install-time wiring
+(configure_log_bounds in hardware.sh, called from install.sh Phase 4), the
+dry-run manifest operations, and the fleet-remediation script's --check mode.
+"""
+
+import os
+import json
+import subprocess
+from pathlib import Path
+
+import pytest
+
+REPO_ROOT = Path(__file__).parent.parent.parent
+FILES_DIR = REPO_ROOT / "files"
+UNIFIED_DIR = REPO_ROOT / "unified_install"
+
+
+class TestLogBoundsArtifacts:
+    """The shipped config files exist and contain the expected caps."""
+
+    def test_journald_dropin_present_and_bounded(self):
+        conf = (FILES_DIR / "journald-sensor-bounds.conf").read_text()
+        assert "[Journal]" in conf
+        assert "SystemMaxUse=" in conf, "journald drop-in must cap SystemMaxUse"
+
+    def test_hourly_logrotate_script_runs_logrotate(self):
+        script = (FILES_DIR / "logrotate-hourly.sh").read_text()
+        assert script.startswith("#!"), "cron script needs a shebang"
+        assert "logrotate" in script, "hourly job must invoke logrotate"
+
+
+class TestLogBoundsWiring:
+    """configure_log_bounds is defined and invoked in the install flow."""
+
+    def test_function_defined_in_hardware_lib(self):
+        hardware = (UNIFIED_DIR / "lib" / "hardware.sh").read_text()
+        assert "configure_log_bounds()" in hardware
+
+    def test_function_called_in_phase4(self):
+        install = (UNIFIED_DIR / "install.sh").read_text()
+        assert "configure_log_bounds" in install, \
+            "configure_log_bounds must be called from install.sh"
+
+
+class TestLogBoundsManifest:
+    """Running install.sh --dry-run through Phase 4 emits the log-bounds ops."""
+
+    def _run_install_with_hardware(self, tmp_path):
+        manifest_file = tmp_path / "manifest.json"
+        env = os.environ.copy()
+        env["MANIFEST_FILE"] = str(manifest_file)
+        env["HOME"] = str(tmp_path)
+        env["USB_PATH"] = str(tmp_path / "usb")
+        env["FILES_DIR"] = str(FILES_DIR)
+        (tmp_path / "usb").mkdir(exist_ok=True)
+
+        # Run the hardware phase (do NOT --skip-hardware) so configure_log_bounds runs.
+        result = subprocess.run(
+            ["bash", str(UNIFIED_DIR / "install.sh"),
+             "05", "--wifi", "--dry-run",
+             "--skip-packages", "--skip-brokkr", "--skip-extras"],
+            capture_output=True, text=True, env=env, cwd=str(UNIFIED_DIR),
+        )
+        if not manifest_file.exists():
+            pytest.fail(
+                f"Manifest not created.\nstdout: {result.stdout}\nstderr: {result.stderr}")
+        return json.loads(manifest_file.read_text())["operations"]
+
+    def test_manifest_caps_journald(self, tmp_path):
+        ops = self._run_install_with_hardware(tmp_path)
+        journald = [op for op in ops
+                    if op.get("type") == "copy"
+                    and op.get("dst", "").endswith("00-sensor-bounds.conf")]
+        assert journald, f"No journald cap copy op. Ops: {[o.get('type') for o in ops]}"
+
+    def test_manifest_adds_rsyslog_maxsize(self, tmp_path):
+        ops = self._run_install_with_hardware(tmp_path)
+        modify = [op for op in ops
+                  if op.get("type") == "modify"
+                  and op.get("path") == "/etc/logrotate.d/rsyslog"]
+        assert modify, f"No rsyslog maxsize modify op. Ops: {[o.get('type') for o in ops]}"
+
+    def test_manifest_installs_hourly_logrotate(self, tmp_path):
+        ops = self._run_install_with_hardware(tmp_path)
+        cron = [op for op in ops
+                if op.get("type") == "copy"
+                and op.get("dst", "").endswith("cron.hourly/mjolnir-logrotate")]
+        assert cron, f"No hourly logrotate cron op. Ops: {[o.get('type') for o in ops]}"
+
+
+class TestApplyLogBoundsCheck:
+    """The fleet-remediation script's --check mode is read-only and exits 0."""
+
+    def test_check_mode_runs_clean(self):
+        result = subprocess.run(
+            ["bash", str(REPO_ROOT / "scripts" / "apply_log_bounds.sh"), "--check"],
+            capture_output=True, text=True,
+        )
+        assert result.returncode == 0, f"--check failed: {result.stderr}"
+        assert "apply-log-bounds" in result.stdout
+
+    def test_rejects_unknown_arg(self):
+        result = subprocess.run(
+            ["bash", str(REPO_ROOT / "scripts" / "apply_log_bounds.sh"), "--bogus"],
+            capture_output=True, text=True,
+        )
+        assert result.returncode == 2, "unknown arg should exit 2"
+
+
+class TestRsyslogBackupLocation:
+    """Regression, found during the mj05 E2E: the rsyslog backup must NOT be
+    written inside /etc/logrotate.d/ -- logrotate reads every file there, so a
+    backup copy triggers 'duplicate log entry' errors for every log path."""
+
+    def _sources(self):
+        return [
+            (UNIFIED_DIR / "lib" / "hardware.sh").read_text(),
+            (REPO_ROOT / "scripts" / "apply_log_bounds.sh").read_text(),
+        ]
+
+    def test_backup_not_written_into_logrotate_dir(self):
+        for text in self._sources():
+            # the old buggy forms appended .mjolnir-orig to the in-dir path
+            assert "${rsyslog_lr}.mjolnir-orig" not in text
+            assert "${RSYSLOG_LR}.mjolnir-orig" not in text
+            assert "/etc/logrotate.d/rsyslog.mjolnir-orig" not in text
+
+    def test_backup_uses_var_backups(self):
+        for text in self._sources():
+            assert "/var/backups/logrotate-rsyslog.mjolnir-orig" in text

unified_install/install.sh

@@ -349,6 +349,7 @@ if [[ "$SKIP_HARDWARE" != "true" ]]; then
     source "$SCRIPT_DIR/lib/hardware.sh"
     setup_sensor_connection
     setup_automount
+    configure_log_bounds
 else
     log_info "Skipping hardware setup (--skip-hardware)"
 fi

unified_install/lib/hardware.sh

@@ -130,3 +130,87 @@ setup_automount() {
     echo ""
     log_info "Users in 'pi' group can now mount/unmount drives using udisksctl"
 }
+
+# --- Configure Log-Size Bounds (HAM-113) ---
+# Bounds /var/log so a runaway log source (brokkr science_ingest spamming when
+# the AGS is down, HAM-112) cannot fill the SD card. Three defenses:
+#   1. journald SystemMaxUse cap (drop-in)
+#   2. maxsize on the rsyslog logrotate stanzas
+#   3. hourly logrotate run so maxsize is enforced within the hour
+configure_log_bounds() {
+    log_step "Configuring log-size bounds (HAM-113)..."
+
+    local journald_dir="/etc/systemd/journald.conf.d"
+    local journald_src="journald-sensor-bounds.conf"
+    local journald_dst="$journald_dir/00-sensor-bounds.conf"
+    local rsyslog_lr="/etc/logrotate.d/rsyslog"
+    # Backup MUST live outside /etc/logrotate.d/ -- logrotate reads every file
+    # in that dir, so a backup there causes "duplicate log entry" errors.
+    local rsyslog_bak="/var/backups/logrotate-rsyslog.mjolnir-orig"
+    local cron_src="logrotate-hourly.sh"
+    local cron_dst="/etc/cron.hourly/mjolnir-logrotate"
+
+    # --- Step 1: Cap systemd-journald disk use ---
+    log_step "[Log bounds 1/3] Capping systemd-journald..."
+    if [[ "$DRY_RUN" == "true" ]]; then
+        log_dry_run "mkdir -p $journald_dir"
+        log_dry_run "cp $FILES_DIR/$journald_src $journald_dst"
+        log_dry_run "systemctl restart systemd-journald"
+        manifest_add "mkdir" "path" "$journald_dir" "sudo" "true"
+        manifest_add "copy" "src" "$FILES_DIR/$journald_src" "dst" "$journald_dst" "sudo" "true"
+        manifest_add "service" "action" "restart" "unit" "systemd-journald" "sudo" "true"
+    else
+        if [[ -f "$FILES_DIR/$journald_src" ]]; then
+            sudo mkdir -p "$journald_dir"
+            sudo cp "$FILES_DIR/$journald_src" "$journald_dst"
+            sudo chmod 0644 "$journald_dst"
+            sudo systemctl restart systemd-journald
+            log_success "journald disk use capped ($journald_dst)"
+        else
+            log_warn "journald bounds file not found at $FILES_DIR/$journald_src"
+        fi
+    fi
+
+    # --- Step 2: Add maxsize cap to the rsyslog logrotate stanzas ---
+    log_step "[Log bounds 2/3] Adding maxsize cap to $rsyslog_lr..."
+    if [[ "$DRY_RUN" == "true" ]]; then
+        log_dry_run "inject 'maxsize 100M' into $rsyslog_lr (idempotent; backup at $rsyslog_bak)"
+        manifest_add "modify" "path" "$rsyslog_lr" "change" "add maxsize 100M" "sudo" "true"
+    else
+        if [[ -f "$rsyslog_lr" ]]; then
+            if sudo grep -q "maxsize" "$rsyslog_lr"; then
+                log_info "rsyslog logrotate already has a maxsize cap; leaving as-is"
+            else
+                sudo mkdir -p "$(dirname "$rsyslog_bak")"
+                sudo cp -a "$rsyslog_lr" "$rsyslog_bak"
+                sudo sed -i '/^{/a\    maxsize 100M' "$rsyslog_lr"
+                log_success "Added 'maxsize 100M' to $rsyslog_lr (backup at $rsyslog_bak)"
+            fi
+        else
+            log_warn "rsyslog logrotate config not found at $rsyslog_lr"
+        fi
+    fi
+
+    # --- Step 3: Run logrotate hourly so maxsize is enforced within the hour ---
+    log_step "[Log bounds 3/3] Installing hourly logrotate job..."
+    if [[ "$DRY_RUN" == "true" ]]; then
+        log_dry_run "cp $FILES_DIR/$cron_src $cron_dst"
+        log_dry_run "chmod 0755 $cron_dst"
+        manifest_add "copy" "src" "$FILES_DIR/$cron_src" "dst" "$cron_dst" "sudo" "true"
+        manifest_add "chmod" "path" "$cron_dst" "mode" "0755" "sudo" "true"
+    else
+        if [[ -f "$FILES_DIR/$cron_src" ]]; then
+            sudo cp "$FILES_DIR/$cron_src" "$cron_dst"
+            sudo chmod 0755 "$cron_dst"
+            log_success "Hourly logrotate job installed at $cron_dst"
+        else
+            log_warn "Hourly logrotate file not found at $FILES_DIR/$cron_src"
+        fi
+    fi
+
+    log_success "Log-size bounds configured!"
+    echo ""
+    log_info "  - journald: SystemMaxUse capped via $journald_dst"
+    log_info "  - rsyslog:  maxsize 100M per log, rotated hourly"
+    log_info "  - Keeps /var/log well under 2 GB even under sustained log spam"
+}