|
| 1 | +# SPDX-License-Identifier: PLMP-1.0-or-later |
| 2 | +# Bustfile — breakage / rollback contract for echidnabot |
| 3 | +# |
| 4 | +# When a build, test, container, dispatch, webhook, or migration goes |
| 5 | +# wrong, this file declares the recovery sequence. Each scenario has |
| 6 | +# an error code, a probe to detect it, and a recovery recipe. |
| 7 | +# Run with: just bust-check |
| 8 | +# Error-code namespace: B### |
| 9 | + |
| 10 | +version: 1 |
| 11 | + |
| 12 | +metadata: |
| 13 | + name: echidnabot-bust-contract |
| 14 | + spec: v0.1.0 |
| 15 | + description: "Breakage rollback for Rust build, sqlx migrations, container, dispatch, observability, webhooks." |
| 16 | + |
| 17 | +# ── Build-Bust ───────────────────────────────────────────────────────── |
| 18 | + |
| 19 | +build: |
| 20 | + |
| 21 | + - name: B001-cargo-build-broken |
| 22 | + description: "cargo build fails after dep update or upstream API change." |
| 23 | + severity: critical |
| 24 | + probe: "! cargo build --offline 2>&1 | grep -q 'error\\['" |
| 25 | + recovery: "just bust-cargo-cache && cargo build" |
| 26 | + escalation: "git revert <last-dep-bump>; cargo update -p <crate> --precise <known-good>" |
| 27 | + |
| 28 | + - name: B002-shared-context-path-broken |
| 29 | + description: "Cargo build fails on missing ../../shared-context (echidnabot lives in gitbot-fleet)." |
| 30 | + severity: high |
| 31 | + probe: "cargo check --offline 2>&1 | grep -q 'failed to read .*shared-context'" |
| 32 | + recovery: "Build from gitbot-fleet/bots/echidnabot/ where the relative path resolves." |
| 33 | + escalation: "see [[feedback_echidnabot_src_abi_namespace_intentional]] for the dual-tree story." |
| 34 | + |
| 35 | + - name: B003-edition-mismatch |
| 36 | + description: "let-chain or other edition-2024 syntax in edition-2021 file." |
| 37 | + severity: high |
| 38 | + probe: "cargo build 2>&1 | grep -q 'experimental.*let.*chain'" |
| 39 | + recovery: "Bump Cargo.toml `edition = \"2024\"` OR rewrite the syntax to edition-compatible form." |
| 40 | + |
| 41 | +# ── Test-Bust ────────────────────────────────────────────────────────── |
| 42 | + |
| 43 | +tests: |
| 44 | + |
| 45 | + - name: B010-cargo-test-broken |
| 46 | + description: "cargo test --lib regression after change." |
| 47 | + severity: critical |
| 48 | + probe: "! cargo test --lib --offline 2>&1 | grep -q 'test result: ok'" |
| 49 | + recovery: "git bisect run cargo test --lib; revert offending commit." |
| 50 | + |
| 51 | + - name: B011-property-test-shrink-too-slow |
| 52 | + description: "tests/property_tests.rs proptest shrinking blows past timeout." |
| 53 | + severity: low |
| 54 | + probe: "cargo test --tests --offline 2>&1 | grep -q 'shrinking'" |
| 55 | + recovery: "Lower PROPTEST_CASES env (e.g. 64); investigate the offending invariant offline." |
| 56 | + |
| 57 | + - name: B012-webhook-e2e-broken |
| 58 | + description: "tests/webhook_e2e_test.rs fails (signature verification / payload decode)." |
| 59 | + severity: high |
| 60 | + probe: "! cargo test --test webhook_e2e_test --offline 2>&1 | grep -q 'test result: ok'" |
| 61 | + recovery: "Check signature header (X-Hub-Signature-256, X-Gitlab-Token, X-Gitea-Signature) routing in src/api/webhooks.rs." |
| 62 | + |
| 63 | +# ── Container-Bust ───────────────────────────────────────────────────── |
| 64 | + |
| 65 | +container: |
| 66 | + |
| 67 | + - name: B020-containerfile-build-broken |
| 68 | + description: "podman build -f Containerfile fails." |
| 69 | + severity: warning |
| 70 | + probe: "! podman build -f Containerfile -q . 2>&1 | tail -3 | grep -qE 'sha256:[0-9a-f]'" |
| 71 | + recovery: "podman system prune -af; rebuild." |
| 72 | + escalation: "check Chainguard base tag drift; stapeln.toml layer cache invalidation." |
| 73 | + |
| 74 | + - name: B021-compose-stack-broken |
| 75 | + description: "docker compose / podman-compose up fails on compose.yml." |
| 76 | + severity: warning |
| 77 | + probe: "podman-compose config 2>&1 | grep -q 'error'" |
| 78 | + recovery: "podman-compose down -v; clear pgdata volume; up fresh." |
| 79 | + escalation: "check echidnabot#69 compose.yml syntax against current Compose v2 spec." |
| 80 | + |
| 81 | +# ── DB-Bust ──────────────────────────────────────────────────────────── |
| 82 | + |
| 83 | +database: |
| 84 | + |
| 85 | + - name: B030-sqlx-migration-failure |
| 86 | + description: "sqlx migrate run failed mid-transaction." |
| 87 | + severity: critical |
| 88 | + probe: "manual — sqlx logs" |
| 89 | + recovery: "sqlx migrate revert; fix the offending migration locally; sqlx migrate run again." |
| 90 | + escalation: "if DB state corrupt: restore from backup; never force-apply a broken migration." |
| 91 | + |
| 92 | + - name: B031-postgres-conn-drop |
| 93 | + description: "Postgres connection pool exhausted / dropped." |
| 94 | + severity: medium |
| 95 | + probe: "grep -q 'PoolTimedOut\\|pool exhausted' /var/log/echidnabot*.log 2>/dev/null" |
| 96 | + recovery: "Increase pool size in src/store/postgres.rs; check for connection leak in long-running query." |
| 97 | + |
| 98 | + - name: B032-sqlite-locked |
| 99 | + description: "SQLite database is locked (concurrent writer)." |
| 100 | + severity: medium |
| 101 | + probe: "grep -q 'database is locked' /var/log/echidnabot*.log 2>/dev/null" |
| 102 | + recovery: "Ensure single-writer convention; switch to WAL mode if not already." |
| 103 | + |
| 104 | +# ── Dispatch-Bust ────────────────────────────────────────────────────── |
| 105 | + |
| 106 | +dispatch: |
| 107 | + |
| 108 | + - name: B040-echidna-server-unreachable |
| 109 | + description: "Calls to echidna server fail (timeout / connection refused)." |
| 110 | + severity: high |
| 111 | + probe: "curl -sf --max-time 5 ${ECHIDNA_API_URL:-http://localhost:8081}/api/health >/dev/null" |
| 112 | + recovery: "Check echidna server alive; check ECHIDNA_API_URL env; check network policy." |
| 113 | + escalation: "fall back to graphql endpoint if REST down (echidna#188 added 3 new GraphQL ops)." |
| 114 | + |
| 115 | + - name: B041-graphql-contract-mismatch |
| 116 | + description: "GraphQL operations echidnabot client calls are absent from echidna server." |
| 117 | + severity: high |
| 118 | + probe: "echidna#180 surfaced this; current state: verifyProof + suggestTactics + proverStatus added in #188." |
| 119 | + recovery: "Ensure echidna server is on commit >= 08771e64 (PR #188 merge)." |
| 120 | + |
| 121 | + - name: B042-job-queue-stalled |
| 122 | + description: "Job queue has > 100 pending items / scheduler not processing." |
| 123 | + severity: high |
| 124 | + probe: "src/scheduler/job_queue.rs depth metric" |
| 125 | + recovery: "Increase scheduler concurrency; investigate backend slowness; check dispatch failure rate." |
| 126 | + |
| 127 | +# ── Observability-Bust ───────────────────────────────────────────────── |
| 128 | + |
| 129 | +observability: |
| 130 | + |
| 131 | + - name: B050-otel-endpoint-unreachable |
| 132 | + description: "OTLP exporter cannot reach OTEL_EXPORTER_OTLP_ENDPOINT." |
| 133 | + severity: low |
| 134 | + probe: "Check observability config; OTLP exporter errors in stderr." |
| 135 | + recovery: "Unset OTEL_EXPORTER_OTLP_ENDPOINT to disable; spans still log via fmt layer." |
| 136 | + |
| 137 | + - name: B051-graceful-shutdown-timeout |
| 138 | + description: "Shutdown timeout fired with N in-flight jobs." |
| 139 | + severity: low |
| 140 | + probe: "grep -q 'graceful shutdown timeout' /var/log/echidnabot*.log" |
| 141 | + recovery: "Increase ECHIDNABOT_SHUTDOWN_TIMEOUT_SECS env; investigate why jobs hang past 30s." |
| 142 | + |
| 143 | +# ── Repo-Bust ────────────────────────────────────────────────────────── |
| 144 | + |
| 145 | +repo: |
| 146 | + |
| 147 | + - name: B060-bad-commit-on-main |
| 148 | + description: "A known-bad commit is on origin/main." |
| 149 | + severity: high |
| 150 | + recovery: "git revert <SHA> on a fresh branch; PR; admin-merge." |
| 151 | + escalation: "force-push to main is PROHIBITED without owner confirmation." |
| 152 | + |
| 153 | + - name: B061-tag-points-at-wrong-sha |
| 154 | + description: "A release tag was cut at the wrong commit." |
| 155 | + severity: medium |
| 156 | + recovery: "git tag -d <tag>; git push origin :refs/tags/<tag>; re-tag at correct SHA; push." |
| 157 | + escalation: "notify any downstream consumers of the tag re-point." |
| 158 | + |
| 159 | +bust-escalation-ladder: |
| 160 | + - 1. revert a single commit (low blast radius) |
| 161 | + - 2. reset local workspace (no remote impact) |
| 162 | + - 3. invalidate a specific cache (just bust-* recipes) |
| 163 | + - 4. force-push to main — PROHIBITED without owner confirmation |
| 164 | + - 5. registry-level (delete release tag, archive package version) — human-only |
0 commit comments