Add continuous model integrity monitoring with pre-inference verification (M12)

Every 15 minutes, a systemd timer verifies SHA256 hashes of all promoted models.
Tampered models are auto-quarantined and workers restarted. All chat endpoints
now verify model integrity before allowing inference, blocking requests if the
active model fails hash verification. Includes security hardening roadmap (M12-M24).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: SecAI-Hub

docs/roadmap-security-hardening.md

@@ -60,6 +60,12 @@ session:
   # sensitive mode: aggressive worker recycling after each task
   # offline-only: hard-block all network even if airlock is enabled
 
+monitoring:
+  # Model integrity check interval (minutes). The integrity timer verifies
+  # SHA256 hashes of all promoted models against the registry manifest.
+  # On mismatch: model is quarantined, removed from manifest, workers restarted.
+  integrity_interval: 15  # minutes (5, 15, 30, or 60)
+
 logging:
   level: "info"
   store_raw_prompts: false

files/system/etc/secure-ai/config/appliance.yaml

@@ -0,0 +1,40 @@
+[Unit]
+Description=Secure AI Model Integrity Check (verify all model hashes)
+After=secure-ai-registry.service
+Wants=secure-ai-registry.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/libexec/secure-ai/integrity-check.sh
+Environment=REGISTRY_URL=http://127.0.0.1:8470
+Environment=REGISTRY_DIR=/var/lib/secure-ai/registry
+
+# Filesystem access
+ReadOnlyPaths=/usr/libexec/secure-ai
+ReadWritePaths=/var/lib/secure-ai/logs
+ReadWritePaths=/var/lib/secure-ai/quarantine
+PrivateTmp=yes
+ProtectSystem=strict
+ProtectHome=yes
+
+# Kernel protection
+ProtectKernelTunables=yes
+ProtectKernelModules=yes
+ProtectKernelLogs=yes
+ProtectControlGroups=yes
+ProtectClock=yes
+ProtectHostname=yes
+
+# Privilege restriction
+NoNewPrivileges=yes
+RestrictSUIDSGID=yes
+LockPersonality=yes
+RestrictRealtime=yes
+
+# Needs network to talk to registry on localhost
+RestrictAddressFamilies=AF_INET AF_UNIX
+
+# Resource limits
+MemoryMax=256M
+CPUQuota=25%
+TasksMax=16

files/system/usr/lib/systemd/system/secure-ai-integrity.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Secure AI Model Integrity Check Timer (default: every 15 minutes)
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=15min
+RandomizedDelaySec=30
+Persistent=true
+
+[Install]
+WantedBy=timers.target

files/system/usr/lib/systemd/system/secure-ai-integrity.timer

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+#
+# Secure AI Appliance — Continuous Model Integrity Monitor
+#
+# Verifies SHA256 hashes of all promoted models against the registry manifest.
+# On mismatch: quarantines the tampered model, removes it from the manifest,
+# kills any inference process using it, and logs a CRITICAL alert.
+#
+# Run via secure-ai-integrity.timer (default: every 15 minutes).
+
+set -euo pipefail
+
+SECURE_AI_ROOT="/var/lib/secure-ai"
+REGISTRY_URL="${REGISTRY_URL:-http://127.0.0.1:8470}"
+REGISTRY_DIR="${REGISTRY_DIR:-/var/lib/secure-ai/registry}"
+TAMPERED_DIR="${SECURE_AI_ROOT}/quarantine/tampered"
+INTEGRITY_LOG="${SECURE_AI_ROOT}/logs/integrity.jsonl"
+RESULT_FILE="${SECURE_AI_ROOT}/logs/integrity-last.json"
+
+log() {
+    echo "[integrity-check] $*"
+    logger -t secure-ai-integrity "$*"
+}
+
+log_json() {
+    local status="$1" model="$2" detail="$3"
+    local ts
+    ts=$(date -Iseconds)
+    printf '{"timestamp":"%s","status":"%s","model":"%s","detail":"%s"}\n' \
+        "$ts" "$status" "$model" "$detail" >> "$INTEGRITY_LOG"
+}
+
+mkdir -p "$(dirname "$INTEGRITY_LOG")" "$TAMPERED_DIR"
+
+# Fetch the manifest from the registry
+manifest=$(curl -sf "${REGISTRY_URL}/v1/models" 2>/dev/null) || {
+    log "ERROR: cannot reach registry at ${REGISTRY_URL}"
+    log_json "error" "" "registry unreachable"
+    echo '{"status":"error","detail":"registry unreachable","checked_at":"'"$(date -Iseconds)"'"}' > "$RESULT_FILE"
+    exit 1
+}
+
+model_count=$(echo "$manifest" | jq 'length')
+if [ "$model_count" -eq 0 ]; then
+    log "No models in registry. Nothing to verify."
+    echo '{"status":"ok","models_checked":0,"failures":0,"checked_at":"'"$(date -Iseconds)"'"}' > "$RESULT_FILE"
+    exit 0
+fi
+
+log "Verifying ${model_count} model(s)..."
+
+failures=0
+checked=0
+
+for i in $(seq 0 $((model_count - 1))); do
+    name=$(echo "$manifest" | jq -r ".[$i].name")
+    filename=$(echo "$manifest" | jq -r ".[$i].filename")
+    expected=$(echo "$manifest" | jq -r ".[$i].sha256")
+    filepath="${REGISTRY_DIR}/${filename}"
+
+    if [ ! -f "$filepath" ]; then
+        log "CRITICAL: model file missing: ${filename} (${name})"
+        log_json "missing" "$name" "file not found: ${filename}"
+        failures=$((failures + 1))
+        continue
+    fi
+
+    actual=$(sha256sum "$filepath" | awk '{print $1}')
+    checked=$((checked + 1))
+
+    if [ "$actual" = "$expected" ]; then
+        log "OK: ${name} (${expected:0:16}...)"
+        log_json "ok" "$name" "hash verified"
+    else
+        log "CRITICAL: HASH MISMATCH for ${name}!"
+        log "  Expected: ${expected}"
+        log "  Actual:   ${actual}"
+        log_json "tampered" "$name" "expected=${expected} actual=${actual}"
+        failures=$((failures + 1))
+
+        # Quarantine the tampered model
+        log "Quarantining tampered model: ${filename}"
+        mv "$filepath" "${TAMPERED_DIR}/${filename}.tampered.$(date +%s)" 2>/dev/null || true
+
+        # Remove from registry manifest via API
+        log "Removing ${name} from registry..."
+        curl -sf -X DELETE "${REGISTRY_URL}/v1/model/delete?name=${name}" >/dev/null 2>&1 || {
+            log "WARNING: could not remove ${name} from registry via API"
+        }
+
+        # Kill any inference process that might be using this model
+        # The inference worker loads models by path — killing it forces a clean reload
+        if systemctl is-active --quiet secure-ai-inference.service 2>/dev/null; then
+            log "Restarting inference worker to drop potentially poisoned model..."
+            systemctl restart secure-ai-inference.service 2>/dev/null || true
+        fi
+        if systemctl is-active --quiet secure-ai-diffusion.service 2>/dev/null; then
+            log "Restarting diffusion worker to drop potentially poisoned model..."
+            systemctl restart secure-ai-diffusion.service 2>/dev/null || true
+        fi
+    fi
+done
+
+ts=$(date -Iseconds)
+status="ok"
+if [ "$failures" -gt 0 ]; then
+    status="failed"
+    log "INTEGRITY CHECK FAILED: ${failures} model(s) tampered or missing out of ${checked} checked"
+else
+    log "Integrity check passed: ${checked} model(s) verified OK"
+fi
+
+# Write summary for the status API
+cat > "$RESULT_FILE" <<EOF
+{
+  "status": "${status}",
+  "models_checked": ${checked},
+  "failures": ${failures},
+  "checked_at": "${ts}"
+}
+EOF
+
+exit 0

files/system/usr/libexec/secure-ai/integrity-check.sh

@@ -81,6 +81,7 @@ modules:
         - secure-ai-quarantine-watcher.service
         - secure-ai-inference.service
         - secure-ai-diffusion.service
+        - secure-ai-integrity.timer
         - nftables.service
         - secure-ai-firstboot.service
         - secure-ai-tmpdir.mount

recipes/recipe.yml

@@ -311,7 +311,83 @@ func handleDelete(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(map[string]string{"status": "deleted", "name": name})
 }
 
-func handleVerify(w http.ResponseWriter, r *http.Request) {
+func handleVerifyAll(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	manifestMu.RLock()
+	models := make([]Artifact, len(manifest.Models))
+	copy(models, manifest.Models)
+	manifestMu.RUnlock()
+
+	results := make([]map[string]string, 0, len(models))
+	allOk := true
+
+	for _, m := range models {
+		filePath := filepath.Join(registryDir, m.Filename)
+		actual, err := verifyFileHash(filePath, m.SHA256)
+		if err != nil {
+			allOk = false
+			results = append(results, map[string]string{
+				"name":     m.Name,
+				"status":   "failed",
+				"expected": m.SHA256,
+				"actual":   actual,
+				"error":    err.Error(),
+			})
+		} else {
+			results = append(results, map[string]string{
+				"name":   m.Name,
+				"status": "verified",
+				"sha256": actual,
+			})
+		}
+	}
+
+	status := "ok"
+	if !allOk {
+		status = "failed"
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	if !allOk {
+		w.WriteHeader(http.StatusConflict)
+	}
+	json.NewEncoder(w).Encode(map[string]interface{}{
+		"status":  status,
+		"models":  results,
+		"checked": len(results),
+	})
+}
+
+func handleIntegrityStatus(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	resultPath := os.Getenv("INTEGRITY_RESULT_PATH")
+	if resultPath == "" {
+		resultPath = "/var/lib/secure-ai/logs/integrity-last.json"
+	}
+
+	data, err := os.ReadFile(resultPath)
+	if err != nil {
+		w.Header().Set("Content-Type", "application/json")
+		json.NewEncoder(w).Encode(map[string]interface{}{
+			"status":  "unknown",
+			"detail":  "no integrity check has run yet",
+		})
+		return
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	w.Write(data)
+}
+
+func handleVerifyModel(w http.ResponseWriter, r *http.Request) {
 	if r.Method != http.MethodPost {
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
 		return
@@ -339,14 +415,16 @@ func handleVerify(w http.ResponseWriter, r *http.Request) {
 					"expected": m.SHA256,
 					"actual":   actual,
 					"error":    err.Error(),
+					"safe_to_use": "false",
 				})
 				return
 			}
 			w.Header().Set("Content-Type", "application/json")
 			json.NewEncoder(w).Encode(map[string]string{
-				"status": "verified",
-				"name":   name,
-				"sha256": actual,
+				"status":      "verified",
+				"name":        name,
+				"sha256":      actual,
+				"safe_to_use": "true",
 			})
 			return
 		}
@@ -391,7 +469,9 @@ func main() {
 	mux.HandleFunc("/v1/model/path", handleModelPath)
 	mux.HandleFunc("/v1/model/promote", handlePromote)
 	mux.HandleFunc("/v1/model/delete", handleDelete)
-	mux.HandleFunc("/v1/model/verify", handleVerify)
+	mux.HandleFunc("/v1/model/verify", handleVerifyModel)
+	mux.HandleFunc("/v1/models/verify-all", handleVerifyAll)
+	mux.HandleFunc("/v1/integrity/status", handleIntegrityStatus)
 
 	log.Printf("secure-ai-registry listening on %s", bind)
 	if err := http.ListenAndServe(bind, mux); err != nil {