Merge pull request #23 from pilot-protocol/feat/site-search-and-submit

Handover prep: cleanup, signed CI/CD, runbook, god-file split, ENV.md, admin lockdown

Author: TeoSlayer

.github/workflows/ci.yml

@@ -20,6 +20,24 @@ jobs:
           go-version: '1.25'
           cache: true
 
+      - name: gofmt (formatting gate)
+        run: |
+          unformatted=$(gofmt -l .)
+          if [ -n "$unformatted" ]; then
+            echo "These files are not gofmt-clean:"; echo "$unformatted"
+            echo "Run: gofmt -w ."; exit 1
+          fi
+
+      - name: go vet
+        run: go vet ./...
+
+      - name: Cross-compile for prod target (linux/arm64)
+        # Production runs on a GH200 (linux/arm64). Pure-Go + CGO_ENABLED=0
+        # means this is a clean cross-compile on the amd64 runner — no QEMU,
+        # no self-hosted runner. Catches a compile break on the prod arch
+        # that the host-arch test build would miss.
+        run: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build ./cmd/cosift
+
       - name: Run tests with coverage
         # cosift's internal/crawler hits ~25s; allow 5m total headroom.
         run: go test -race -coverprofile=coverage.out -covermode=atomic -timeout 5m ./...

.github/workflows/deploy.yml

@@ -0,0 +1,148 @@
+name: deploy
+
+# Build + sign + publish a release artifact. PULL-BASED deploy:
+# this workflow never touches the production box and never holds an SSH key.
+# The box runs cosift-self-update.timer, which polls the latest GitHub Release,
+# verifies sha256 + minisign signature against a public key baked on the box,
+# snapshots, atomically swaps the binary, restarts, and health-gates with
+# auto-rollback. See deploy/scripts/README.md.
+#
+# Triggers:
+#   - push of a version tag (v*)         → cut a release for that tag
+#   - manual workflow_dispatch           → build from a chosen ref (no release
+#                                          unless a tag is also present)
+
+on:
+  push:
+    tags:
+      - 'v*'
+  workflow_dispatch:
+
+permissions:
+  contents: write   # needed to create the GitHub Release + upload assets
+
+jobs:
+  # (a) verify — gate the release on vet + race tests + a smoke subset.
+  verify:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: '1.25'
+          cache: true
+
+      - name: go vet
+        run: go vet ./...
+
+      - name: Tests (race)
+        run: go test -race -timeout 5m ./...
+
+      - name: Smoke subset (build + serve roundtrip)
+        # make smoke builds the binary, crawls, and asserts /healthz + /search.
+        # Guarded with a timeout so a hung crawl can't wedge the release.
+        run: timeout 300 make smoke
+
+  # (b) build — reproducible static arm64 binary, sha256, minisign signature.
+  build:
+    needs: verify
+    runs-on: ubuntu-latest
+    outputs:
+      version: ${{ steps.ver.outputs.version }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0   # full history so version stamping is meaningful
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: '1.25'
+          cache: true
+
+      - name: Resolve version stamp
+        id: ver
+        # Tag name when triggered by a tag push; otherwise the short SHA.
+        run: |
+          if [ "${GITHUB_REF_TYPE}" = "tag" ]; then
+            echo "version=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT"
+          else
+            echo "version=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Build (linux/arm64, static, trimmed)
+        env:
+          CGO_ENABLED: '0'
+          GOOS: linux
+          GOARCH: arm64
+          VERSION: ${{ steps.ver.outputs.version }}
+        # Stamp both main.version and internal/server.Version so /healthz and
+        # /metrics report the shipped version. Matches the Makefile `build`
+        # target's ldflags exactly (kept in sync deliberately).
+        run: |
+          go build -trimpath \
+            -ldflags "-s -w -X main.version=${VERSION} -X github.com/pilot-protocol/cosift/internal/server.Version=${VERSION}" \
+            -o cosift-linux-arm64 ./cmd/cosift
+
+      - name: sha256
+        run: sha256sum cosift-linux-arm64 | tee cosift-linux-arm64.sha256
+
+      - name: Install minisign
+        run: sudo apt-get update && sudo apt-get install -y minisign
+
+      - name: Sign with minisign
+        env:
+          # Repo secret. The matching PUBLIC key is baked on the box at
+          # /etc/cosift/minisign.pub. Generate with `minisign -G` (see
+          # deploy/scripts/README.md). NEVER commit the secret key.
+          MINISIGN_SECRET_KEY: ${{ secrets.MINISIGN_SECRET_KEY }}
+        run: |
+          if [ -z "${MINISIGN_SECRET_KEY}" ]; then
+            echo "::error::MINISIGN_SECRET_KEY secret is not set" >&2
+            exit 1
+          fi
+          printf '%s' "${MINISIGN_SECRET_KEY}" > minisign.key
+          # -W: secret key is unencrypted (no interactive passphrase prompt).
+          # Trusted comment carries the version so the box can sanity-check it.
+          minisign -S -W -s minisign.key \
+            -m cosift-linux-arm64 \
+            -t "cosift ${{ steps.ver.outputs.version }} linux/arm64"
+          rm -f minisign.key
+          # Verification against the embedded public key is done on the box;
+          # here we just confirm the .minisig was produced.
+          test -f cosift-linux-arm64.minisig
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: cosift-release-artifacts
+          path: |
+            cosift-linux-arm64
+            cosift-linux-arm64.sha256
+            cosift-linux-arm64.minisig
+          retention-days: 30
+
+  # (c) release — publish the signed binary as a GitHub Release asset.
+  # Only on a tag push (workflow_dispatch builds the artifact but does not
+  # cut a release).
+  release:
+    needs: build
+    if: github.ref_type == 'tag'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/download-artifact@v7
+        with:
+          name: cosift-release-artifacts
+
+      - name: Create / update GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: ${{ github.ref_name }}
+          name: cosift ${{ github.ref_name }}
+          generate_release_notes: true
+          fail_on_unmatched_files: true
+          files: |
+            cosift-linux-arm64
+            cosift-linux-arm64.sha256
+            cosift-linux-arm64.minisig

Makefile

@@ -22,7 +22,7 @@ smoke:
 	@./scripts/smoke-test.sh
 
 build:
-	CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=$(VERSION) -X github.com/calinteodor/cosift/internal/server.Version=$(VERSION)" -o $(BINARY) $(PKG)
+	CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=$(VERSION) -X github.com/pilot-protocol/cosift/internal/server.Version=$(VERSION)" -o $(BINARY) $(PKG)
 
 # Light-touch verification: compile + vet + unit tests on packages that
 # don't need OPENAI/COHERE keys or live network. Catches latent compile

cmd/cosift/assets/chat.html

@@ -213,6 +213,22 @@
   font-family: var(--mono); font-size: 11px;
   padding: 6px 8px; cursor: pointer;
 }
+.site-input-wrap {
+  display: flex; align-items: center; gap: 4px;
+  border: 1px solid var(--line); border-radius: 8px;
+  background: var(--bg-2); padding: 0 8px;
+}
+.site-input-label {
+  font-family: var(--mono); font-size: 10px; color: var(--ink-dim);
+  white-space: nowrap; user-select: none; flex-shrink: 0;
+}
+.site-input {
+  border: none; background: transparent; color: var(--ink);
+  font-family: var(--mono); font-size: 11px;
+  padding: 5px 0; width: 140px; min-width: 0;
+  outline: none;
+}
+.site-input::placeholder { color: var(--ink-dim); opacity: 0.6; }
 .send-btn {
   padding: 14px 22px;
   background: var(--accent); color: var(--accent-ink);
@@ -324,6 +340,10 @@ <h1>Chat with the corpus.</h1>
   <div class="composer-row">
     <textarea id="input" class="composer-input" rows="1" placeholder="Ask anything…" autofocus></textarea>
     <div class="composer-controls">
+      <div class="site-input-wrap" title="Restrict results to a specific domain, e.g. pilotprotocol.network or docs.example.com/api">
+        <span class="site-input-label">site:</span>
+        <input id="site-filter" class="site-input" type="text" placeholder="domain.com" autocomplete="off" spellcheck="false">
+      </div>
       <select id="mode" title="query = LLM-planned hybrid (default) · answer = direct RAG · research = multi-step planner">
         <option value="query" selected>query</option>
         <option value="answer">answer</option>
@@ -332,7 +352,7 @@ <h1>Chat with the corpus.</h1>
       <button id="send" class="send-btn">Send →</button>
     </div>
   </div>
-  <div class="composer-hint">Enter to send · Shift+Enter newline · /clear to reset</div>
+  <div class="composer-hint">Enter to send · Shift+Enter newline · /clear to reset · site:domain.com to scope</div>
 </div>
 
 <script>
@@ -349,9 +369,16 @@ <h1>Chat with the corpus.</h1>
   var inputEl = document.getElementById('input');
   var sendBtn = document.getElementById('send');
   var modeEl = document.getElementById('mode');
+  var siteEl = document.getElementById('site-filter');
   var emptyEl = document.getElementById('empty-state');
   var clearBtn = document.getElementById('clear-btn');
 
+  // Persist site filter across page loads
+  try { var _s = localStorage.getItem('cosift-site'); if (_s) siteEl.value = _s; } catch(e) {}
+  siteEl.addEventListener('change', function () {
+    try { localStorage.setItem('cosift-site', siteEl.value.trim()); } catch(e) {}
+  });
+
   function esc(s) { return String(s).replace(/[&<>"]/g, function(c){ return ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;'}[c]); }); }
 
   function autoresize() {
@@ -519,7 +546,19 @@ <h1>Chat with the corpus.</h1>
       autoresize();
       return;
     }
-    appendMsg('user', q);
+
+    // Allow inline site:domain.com syntax — strip it from the query and
+    // populate the site filter input. e.g. "pilot tunnels site:pilotprotocol.network"
+    var siteMatch = q.match(/(?:^|\s)site:(\S+)/);
+    var site = siteEl.value.trim();
+    if (siteMatch) {
+      q = q.replace(siteMatch[0], ' ').trim();
+      site = siteMatch[1];
+      siteEl.value = site;
+      try { localStorage.setItem('cosift-site', site); } catch(e) {}
+    }
+
+    appendMsg('user', q + (site ? '  [site: ' + site + ']' : ''));
     inputEl.value = '';
     autoresize();
 
@@ -540,6 +579,7 @@ <h1>Chat with the corpus.</h1>
     // progress (planner sub-queries, per-expansion hit counts, judge
     // drops, synth tokens).
     var url = endpoint + '?q=' + encodeURIComponent(q) + '&k=5&stream=true';
+    if (site) url += '&site=' + encodeURIComponent(site);
 
     try {
       var resp = await fetch(url, { headers: { 'Accept': 'text/event-stream' } });

cmd/cosift/assets/landing.html

@@ -340,6 +340,9 @@ <h2>Search the <em>live</em> <span class="accent">corpus.</span></h2>
         <button class="btn primary" type="submit">Search <span class="arr">→</span></button>
       </form>
       <div class="tryit-knobs">
+        <label>site
+          <input type="text" id="tryit-site" placeholder="domain.com" style="width:140px; font-family:monospace" title="Restrict results to a domain, e.g. pilotprotocol.network" autocomplete="off" spellcheck="false"/>
+        </label>
         <label>retriever
           <select id="tryit-retriever">
             <option value="bm25" selected>bm25</option>
@@ -561,6 +564,8 @@ <h2>Ask <em>anything.</em></h2>
     var p = new URLSearchParams();
     p.set('q', q);
     p.set('k', document.getElementById('tryit-k').value || '5');
+    var site = document.getElementById('tryit-site').value.trim();
+    if (site) p.set('site', site);
     var retr = document.getElementById('tryit-retriever').value;
     if (retr && retr !== 'bm25') p.set('retriever', retr);
     var mmr = document.getElementById('tryit-mmr').value;

cmd/cosift/backfill_host_postings.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/pilot-protocol/cosift/internal/store"
+)
+
+// runBackfillHostPostings rebuilds the 'P' host-partition family from existing
+// 'h' + 'p' data so site= queries can scan a single host's postings
+// (O(site_docs)) via SearchInHost instead of the global lists (O(corpus)).
+//
+// It re-uses the data already on disk — no re-tokenize, no embeddings — so it's
+// a one-time key reshuffle, typically ~1-2h on a multi-million-doc corpus. The
+// store opens read-write because it writes the new family, but it never touches
+// 'p'/'d'/'v', so it is safe to run alongside a live pebble-serve. After it
+// completes, set COSIFT_HOST_PARTITION_READ=1 on the server to route site=
+// queries to the partition (and COSIFT_HOST_PARTITION=1 to keep it fresh on
+// future crawls).
+//
+//	cosift backfill-host-postings -dir /home/ubuntu/cosift-data/pebble
+func runBackfillHostPostings(ctx context.Context, args []string) error {
+	fs := flag.NewFlagSet("backfill-host-postings", flag.ExitOnError)
+	dir := fs.String("dir", "", "PebbleStore directory (required; same dir as pebble-serve -dir)")
+	host := fs.String("host", "", "backfill only this host (fast 'g'-index path); empty = full corpus single-pass")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if *dir == "" {
+		return fmt.Errorf("-dir required")
+	}
+
+	ps, err := store.OpenPebble(*dir)
+	if err != nil {
+		return fmt.Errorf("open store: %w", err)
+	}
+	defer ps.Close()
+
+	start := time.Now()
+	if *host != "" {
+		fmt.Fprintf(os.Stderr, "backfill-host-postings: targeted backfill for host %q (fast 'g'-index path)…\n", *host)
+	} else {
+		fmt.Fprintf(os.Stderr, "backfill-host-postings: full-corpus single-pass 'p'→'P'…\n")
+	}
+	last := start
+	written, err := ps.BackfillHostPostings(ctx, *host, func(n int64) {
+		now := time.Now()
+		if now.Sub(last) >= 5*time.Second {
+			rate := float64(n) / time.Since(start).Seconds()
+			fmt.Fprintf(os.Stderr, "  %d host postings written (%.0f/s, %s elapsed)\n",
+				n, rate, time.Since(start).Round(time.Second))
+			last = now
+		}
+	})
+	if err != nil {
+		return fmt.Errorf("backfill: %w (wrote %d before failing)", err, written)
+	}
+	fmt.Fprintf(os.Stderr, "backfill-host-postings: done — %d host postings in %s\n",
+		written, time.Since(start).Round(time.Second))
+	fmt.Fprintf(os.Stderr, "Next: set COSIFT_HOST_PARTITION_READ=1 (read) + COSIFT_HOST_PARTITION=1 (keep fresh) and restart pebble-serve.\n")
+	return nil
+}