feat(store): durable webhook deliveries with X-GitHub-Delivery idempotency (#2 phase 1) (#44)

New crates/store crate: embedded SQLite (rusqlite, WAL, user_version
migrations) holding webhook_deliveries, tasks, and task_attempts per
docs/durable-task-store.md. The webhook route now records every delivery —
and its routing outcome — atomically BEFORE GitHub hears success:

- X-GitHub-Delivery is required (400 without it); it is the idempotency key
- redelivered ids answer 200 {duplicate:true} and never dispatch twice
- routed tasks get a durable queued row (with superseded tombstones for
  older queued reviews of the same PR); ping/unroutable deliveries are
  recorded as ignored:<reason>
- a store failure answers 500 so GitHub retries instead of losing work

[storage] config with doctor checks; compose gains a data volume; the demo
gains an Act 1b proving redelivery dedup end to end (21 assertions); the
smoke script covers the missing-delivery-id 400.

Worker dispatch still rides the in-process channel in this phase; durable
claims, restart recovery, and retirement of the in-memory task store land
in phase 2.

Signed-off-by: Val Alexander <bunsthedev@gmail.com>

Author: BunsDev

Cargo.lock

@@ -2,6 +2,7 @@
 members = [
     "crates/config",
     "crates/github",
+    "crates/store",
     "crates/webhook",
     "crates/worker",
     "crates/server",
@@ -25,6 +26,7 @@ hmac = "0.12"
 jsonwebtoken = "9"
 octocrab = "0.41"
 reqwest = { version = "0.12", features = ["json"] }
+rusqlite = { version = "0.32", features = ["bundled"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 sha2 = "0.10"

Cargo.toml

@@ -161,7 +161,7 @@ duplicate comments.
 | `coven-code --headless` execution | Partial | Worker spawns headless sessions with a tokenless session brief and enforces task timeouts; result quality depends on the runtime. |
 | Pull request creation | Partial | Opens draft PRs from session results against the repository's resolved default/base branch. |
 | CovenCave task polling | Partial | In-memory task API exists for local oversight; hosted control-plane auth and persistence are planned. |
-| Durable queue / task store | Planned | Required for hosted reliability and restarts. |
+| Durable queue / task store | Partial | Deliveries are persisted and deduplicated by `X-GitHub-Delivery` before GitHub hears success, and every routed task gets a durable record ([design](docs/durable-task-store.md)); worker claims + restart recovery land next. |
 | Hosted tier | Planned | See [Hosted vs self-hosted](docs/hosted-vs-self-hosted.md). |
 | Familiar trust contract | Planned | See [Familiar Contract](FAMILIAR-CONTRACT.md). |
 

README.md

@@ -23,10 +23,14 @@ services:
       - ./keys:/keys:ro
       # Ephemeral per-task workspaces. Keep worker.workspace_root pointed here.
       - coven-workspaces:/tmp/coven-github-tasks
+      # Durable adapter state (webhook idempotency + task records). Point
+      # storage.path at /data/coven-github.db in local.toml.
+      - coven-data:/data
     environment:
       # Adjust log verbosity, e.g. RUST_LOG=coven_github=debug
       RUST_LOG: "coven_github=info"
     restart: unless-stopped
 
 volumes:
   coven-workspaces:
+  coven-data:

compose.yaml

@@ -19,6 +19,11 @@ workspace_root = "/tmp/coven-github-tasks"    # Ephemeral workspace root
 timeout_secs = 600  # 10 minutes per task
 max_retries = 2     # Retries for infra errors (exit code 2)
 
+[storage]
+# Durable adapter state: webhook deliveries (idempotency) and task records.
+# SQLite file — keep it on a persistent volume; parents are created at start.
+path = "data/coven-github.db"
+
 # ── Familiar configuration ──────────────────────────────────────────────────
 # Add one [[familiars]] block per familiar you want to expose as a GitHub bot.
 

config/example.toml

@@ -13,6 +13,29 @@ pub struct Config {
     /// Automatic review trigger policy. Absent section = all lanes off.
     #[serde(default)]
     pub review: ReviewConfig,
+    /// Durable adapter state (issue #2). Absent section = default path.
+    #[serde(default)]
+    pub storage: StorageConfig,
+}
+
+/// Durable store location. See `docs/durable-task-store.md`.
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct StorageConfig {
+    /// SQLite database path; parent directories are created at startup.
+    #[serde(default = "default_storage_path")]
+    pub path: PathBuf,
+}
+
+impl Default for StorageConfig {
+    fn default() -> Self {
+        Self {
+            path: default_storage_path(),
+        }
+    }
+}
+
+fn default_storage_path() -> PathBuf {
+    PathBuf::from("data/coven-github.db")
 }
 
 /// Automatic review trigger policy (issue #10). Lanes default to off.
@@ -264,6 +287,33 @@ impl Config {
             }
         }
 
+        // ── Storage ─────────────────────────────────────────────────────
+        if self.storage.path.is_dir() {
+            out.push(Diagnostic::error(
+                "storage.path",
+                format!(
+                    "'{}' is a directory — storage.path must be the SQLite database file itself.",
+                    self.storage.path.display()
+                ),
+            ));
+        } else if !self.storage.path.exists() {
+            // Startup creates the file and any missing parents; surface where
+            // it will land so operators mount/persist the right volume.
+            let parent = self
+                .storage
+                .path
+                .parent()
+                .filter(|p| !p.as_os_str().is_empty())
+                .map(|p| p.display().to_string())
+                .unwrap_or_else(|| ".".to_string());
+            out.push(Diagnostic::warning(
+                "storage.path",
+                format!(
+                    "database does not exist yet — it will be created under '{parent}' at startup; make sure that path is on a persistent volume."
+                ),
+            ));
+        }
+
         // ── Review policy ───────────────────────────────────────────────
         let known_ids: std::collections::HashSet<&str> =
             self.familiars.iter().map(|f| f.id.as_str()).collect();
@@ -370,6 +420,9 @@ fn next_step_for(field: &str, _message: &str) -> &'static str {
         "review.familiar" => {
             "Set review.familiar (or the repo override's familiar) to the id of a configured [[familiars]] block."
         }
+        "storage.path" => {
+            "Point storage.path at a writable SQLite file location on a persistent volume."
+        }
         _ => "Update this config field, then rerun coven-github doctor.",
     }
 }
@@ -468,6 +521,7 @@ mod tests {
             worker,
             familiars,
             review: ReviewConfig::default(),
+            storage: StorageConfig::default(),
         }
     }
 

crates/config/src/lib.rs

@@ -18,5 +18,6 @@ tracing.workspace = true
 tracing-subscriber.workspace = true
 coven-github-api = { path = "../github" }
 coven-github-config = { path = "../config" }
+coven-github-store = { path = "../store" }
 coven-github-webhook = { path = "../webhook" }
 coven-github-worker = { path = "../worker" }

crates/server/Cargo.toml

@@ -86,6 +86,14 @@ async fn main() -> Result<()> {
 
             let config = Arc::new(config);
 
+            // Durable deliveries + tasks (issue #2): open before serving so a
+            // broken storage path fails the boot, not the first delivery.
+            let store = coven_github_store::Store::open(&config.storage.path)?;
+            tracing::info!(
+                "durable store ready at {}",
+                config.storage.path.display()
+            );
+
             let (task_tx, task_rx) = mpsc::channel(256);
             let task_store = TaskStore::default();
 
@@ -101,6 +109,7 @@ async fn main() -> Result<()> {
                 config: config.clone(),
                 task_tx,
                 task_store,
+                store,
             };
 
             let app = Router::new()

crates/server/src/main.rs

@@ -0,0 +1,17 @@
+[package]
+name = "coven-github-store"
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+chrono.workspace = true
+rusqlite.workspace = true
+serde_json.workspace = true
+tokio.workspace = true
+tracing.workspace = true
+coven-github-api = { path = "../github" }
+
+[dev-dependencies]
+uuid.workspace = true