feat: hosted deployment stack — production compose, /healthz, image publish, ops runbook (#71)

* feat: hosted deployment stack — production compose, /healthz, image publish, ops runbook

The operational half of the HOSTED.md beta gates (the six code gates are
shipped). Single-VM architecture matching what the code wants: single-writer
SQLite and a container worker backend that drives the host Docker daemon.

- deploy/hosted/: production compose (adapter + Caddy TLS ingress +
  Litestream SQLite replication to Cloudflare R2), Caddyfile, litestream.yml,
  .env.example, and an idempotent provision.sh for Ubuntu 24.04 (Docker,
  firewall 22/80/443, layout, docker-gid + uid-10001 permissions).
- /healthz: unauthenticated liveness/readiness route probing the store —
  compose healthcheck and uptime monitors point at it; leaks no tenant data.
- Dockerfile: add curl for the container healthcheck.
- .github/workflows/publish.yml: build/push ghcr.io/opencoven/coven-github
  (latest + SHA on main, semver on tags).
- docs/hosted-deploy.md: first-deploy runbook, monitoring signals, backup and
  restore drill, upgrade and incident basics. Cross-linked from HOSTED.md.

Validated: docker compose config, bash -n provision.sh, full CI gates.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Signed-off-by: Val Alexander <bunsthedev@gmail.com>

* fix(webhook): minimal healthz failure body per review

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Signed-off-by: Val Alexander <bunsthedev@gmail.com>

---------

Signed-off-by: Val Alexander <bunsthedev@gmail.com>
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: BunsDev

.github/workflows/publish.yml

@@ -0,0 +1,45 @@
+# Build and publish the coven-github container image to GHCR.
+# Pushes to main publish `latest` + the commit SHA; version tags publish the
+# semver tag. deploy/hosted/compose.yaml pins deployments to these images.
+name: publish
+
+on:
+  push:
+    branches: [main]
+    tags: ["v*"]
+
+permissions:
+  contents: read
+  packages: write
+
+jobs:
+  image:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: docker/setup-buildx-action@v3
+
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/opencoven/coven-github
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=sha,format=long
+            type=semver,pattern={{version}}
+
+      - uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

Dockerfile

@@ -23,8 +23,9 @@ RUN cargo build --release --locked -p coven-github
 FROM debian:bookworm-slim AS runtime
 
 # ca-certificates: TLS to api.github.com. git: clone target repos.
+# curl: container healthchecks against /healthz (deploy/hosted/compose.yaml).
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends ca-certificates git \
+    && apt-get install -y --no-install-recommends ca-certificates git curl \
     && rm -rf /var/lib/apt/lists/*
 
 # Run as an unprivileged user. Secrets are mounted read-only at runtime, never

HOSTED.md

@@ -88,6 +88,11 @@ Hosted beta should wait for:
 5. Usage metering by installation, repo, familiar, and task.
 6. Cave oversight dashboard for task history and human intervention.
 
+All six code gates are shipped. The operational side — provisioning, TLS
+ingress, continuous store backup, monitoring, restore drills, upgrades — is
+the [hosted deployment runbook](docs/hosted-deploy.md) with its stack in
+[`deploy/hosted/`](deploy/hosted/).
+
 ## Landing Page Copy
 
 Headline:

crates/server/src/main.rs

@@ -12,7 +12,8 @@ use tracing_subscriber::EnvFilter;
 
 use coven_github_config::Config;
 use coven_github_webhook::routes::{
-    audit, handle_webhook, list_memory, list_tasks, revoke_memory, routing, usage, AppState,
+    audit, handle_webhook, healthz, list_memory, list_tasks, revoke_memory, routing, usage,
+    AppState,
 };
 use coven_github_worker as worker;
 
@@ -172,6 +173,7 @@ async fn main() -> Result<()> {
             };
 
             let app = Router::new()
+                .route("/healthz", get(healthz))
                 .route("/webhook", post(handle_webhook))
                 .route("/api/github/tasks", get(list_tasks))
                 .route("/api/github/memory", get(list_memory))

crates/webhook/src/routes.rs

@@ -43,6 +43,23 @@ fn familiar_names(config: &Config) -> std::collections::HashMap<String, String>
         .collect()
 }
 
+/// GET /healthz — liveness/readiness for ingress and uptime checks.
+/// Unauthenticated by design, so it must never expose tenant data: the body
+/// is only `ok` plus whether the durable store answers a trivial read.
+pub async fn healthz(State(state): State<AppState>) -> impl IntoResponse {
+    match state.store.delivery_routing("__healthz__").await {
+        Ok(_) => (StatusCode::OK, Json(json!({"ok": true}))).into_response(),
+        Err(e) => {
+            error!("healthz store probe failed: {e:#}");
+            (
+                StatusCode::SERVICE_UNAVAILABLE,
+                Json(json!({"ok": false})),
+            )
+                .into_response()
+        }
+    }
+}
+
 /// GET /api/github/tasks — task state for CovenCave polling, behind the
 /// tenant boundary (issue #3). `token` mode fails closed; a tenant token sees
 /// only its own installation (optionally narrowed to repositories); the
@@ -2996,6 +3013,25 @@ mod metering_route_tests {
     }
 }
 
+#[cfg(test)]
+mod healthz_tests {
+    use super::tests::app_state;
+    use super::*;
+    use axum::extract::State;
+
+    #[tokio::test]
+    async fn healthz_answers_ok_and_leaks_nothing() {
+        let state = app_state();
+        let response = healthz(State(state)).await.into_response();
+        assert_eq!(response.status(), StatusCode::OK);
+        let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+            .await
+            .expect("body");
+        let json: serde_json::Value = serde_json::from_slice(&bytes).expect("json");
+        assert_eq!(json, serde_json::json!({"ok": true}));
+    }
+}
+
 #[cfg(test)]
 mod billing_route_tests {
     //! Marketplace plan intake: entitlement recording, plan-derived limits,

deploy/coven-github/__pycache__/coven_github_adapter.cpython-314.pyc

@@ -0,0 +1,20 @@
+# Secrets for deploy/hosted/compose.yaml. Copy to .env (mode 0600) and fill
+# in. NEVER commit the filled-in file.
+
+# Public hostname the GitHub App webhook points at.
+WEBHOOK_DOMAIN=gh.opencoven.ai
+
+# Image tag to run (a released ghcr.io/opencoven/coven-github tag).
+COVEN_GITHUB_TAG=latest
+
+# Cloudflare R2 (S3-compatible) for Litestream SQLite replication.
+# Create an R2 API token scoped to one bucket with object read+write.
+R2_ACCOUNT_ID=
+R2_BUCKET=coven-github-store
+R2_ACCESS_KEY_ID=
+R2_SECRET_ACCESS_KEY=
+
+# Host docker group id, so the unprivileged adapter user can drive the
+# daemon for the container worker backend. provision.sh fills this in
+# (getent group docker | cut -d: -f3).
+DOCKER_GID=

deploy/coven-github/__pycache__/test_coven_github_adapter.cpython-314.pyc

@@ -0,0 +1,13 @@
+# TLS ingress for hosted coven-github. Caddy provisions and renews
+# Let's Encrypt certificates automatically for $WEBHOOK_DOMAIN.
+{$WEBHOOK_DOMAIN} {
+	encode gzip
+
+	# GitHub webhook deliveries and the tenant-scoped API.
+	reverse_proxy coven-github:3000
+
+	log {
+		output stdout
+		format console
+	}
+}

deploy/hosted/.env.example

@@ -0,0 +1,77 @@
+# Production Docker Compose for hosted coven-github: adapter + TLS ingress
+# (Caddy) + continuous SQLite replication (Litestream → Cloudflare R2).
+#
+# Runbook: docs/hosted-deploy.md. Secrets come from ./.env (never committed);
+# copy .env.example and fill it in.
+#
+# Layout on the host (created by provision.sh):
+#   /opt/coven-github/
+#     compose.yaml            this file
+#     .env                    secrets (mode 0600)
+#     Caddyfile               TLS ingress
+#     litestream.yml          replication config
+#     config/production.toml  adapter config
+#     keys/app.pem            GitHub App private key (mode 0600)
+#     data/                   SQLite store (replicated)
+
+services:
+  coven-github:
+    image: ghcr.io/opencoven/coven-github:${COVEN_GITHUB_TAG:-latest}
+    command: ["serve", "--config", "/config/production.toml"]
+    restart: unless-stopped
+    expose:
+      - "3000"
+    volumes:
+      - ./config:/config:ro
+      - ./keys:/keys:ro
+      - ./data:/data
+      - ./workspaces:/tmp/coven-github-tasks
+      # Container worker backend: the adapter launches per-task hardened
+      # containers through the host Docker daemon (docs/container-isolation.md).
+      - /var/run/docker.sock:/var/run/docker.sock
+    # The image runs unprivileged (uid 10001); grant the host docker group so
+    # the worker can drive the daemon. provision.sh fills DOCKER_GID in .env.
+    group_add:
+      - "${DOCKER_GID:?set in .env (getent group docker | cut -d: -f3)}"
+    environment:
+      RUST_LOG: "coven_github=info"
+    healthcheck:
+      test: ["CMD", "curl", "-fsS", "http://localhost:3000/healthz"]
+      interval: 30s
+      timeout: 5s
+      retries: 3
+      start_period: 10s
+
+  caddy:
+    image: caddy:2
+    restart: unless-stopped
+    ports:
+      - "80:80"
+      - "443:443"
+    volumes:
+      - ./Caddyfile:/etc/caddy/Caddyfile:ro
+      - caddy-data:/data
+      - caddy-config:/config
+    environment:
+      WEBHOOK_DOMAIN: ${WEBHOOK_DOMAIN:?set in .env, e.g. gh.opencoven.ai}
+    depends_on:
+      - coven-github
+
+  litestream:
+    image: litestream/litestream:0.3
+    restart: unless-stopped
+    command: ["replicate", "-config", "/etc/litestream.yml"]
+    volumes:
+      - ./litestream.yml:/etc/litestream.yml:ro
+      - ./data:/data
+    environment:
+      LITESTREAM_ACCESS_KEY_ID: ${R2_ACCESS_KEY_ID:?set in .env}
+      LITESTREAM_SECRET_ACCESS_KEY: ${R2_SECRET_ACCESS_KEY:?set in .env}
+      R2_BUCKET: ${R2_BUCKET:?set in .env}
+      R2_ACCOUNT_ID: ${R2_ACCOUNT_ID:?set in .env}
+    depends_on:
+      - coven-github
+
+volumes:
+  caddy-data:
+  caddy-config: