Skip to content

Commit ef95cf6

Browse files
hyperpolymathclaude
hyperpolymath
and
claude
committed
feat: wire VeriSimDB into gv-clade-index alongside Cloudflare KV
Add worker/src/verisimdb.js: dual-write client seeding clade:repos, clade:clades, clade:index collections. Replace KV-only loadIndex with VeriSimDB-first (KV fallback). Add /v1/query/language and /v1/query/tag endpoints backed by VeriSimDB. Add sync/seed-verisimdb.sh to populate VeriSimDB from static JSON on demand. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent b62923a commit ef95cf6

3 files changed

Lines changed: 492 additions & 6 deletions

File tree

‎sync/seed-verisimdb.sh‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
#!/usr/bin/env bash
2+
# SPDX-License-Identifier: PMPL-1.0-or-later
3+
# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell
4+
#
5+
# Seed VeriSimDB with the static JSON data files.
6+
#
7+
# Usage:
8+
# ./sync/seed-verisimdb.sh [VERISIMDB_URL]
9+
#
10+
# VERISIMDB_URL defaults to http://localhost:8080
11+
#
12+
# This script seeds the following VeriSimDB collections:
13+
# clade:repos — one document per repo (from worker/data/repos.json)
14+
# clade:clades — one document per clade (from worker/data/clades.json)
15+
# clade:index — pre-built combined index (from worker/data/index.json)
16+
#
17+
# Run this script after initial setup and after any changes to the static
18+
# JSON data files so that VeriSimDB stays in sync with Cloudflare KV.
19+
20+
set -euo pipefail
21+
22+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
23+
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
24+
DATA_DIR="${REPO_ROOT}/worker/data"
25+
26+
VERISIMDB_URL="${1:-${VERISIMDB_URL:-http://localhost:8080}}"
27+
API_BASE="${VERISIMDB_URL}/api/v1"
28+
29+
echo "[seed-verisimdb] Base URL: ${VERISIMDB_URL}"
30+
echo "[seed-verisimdb] Data dir: ${DATA_DIR}"
31+
32+
# ── Helper ────────────────────────────────────────────────────────────────
33+
34+
put_doc() {
35+
local collection="$1"
36+
local id="$2"
37+
local body="$3"
38+
39+
local url="${API_BASE}/${collection}/${id}"
40+
local http_code
41+
http_code=$(curl -s -o /dev/null -w "%{http_code}" \
42+
-X PUT \
43+
-H "Content-Type: application/json" \
44+
-d "${body}" \
45+
"${url}")
46+
47+
if [[ "$http_code" =~ ^(200|201|204)$ ]]; then
48+
return 0
49+
else
50+
echo "[seed-verisimdb] WARN: PUT ${url} returned ${http_code}" >&2
51+
return 1
52+
fi
53+
}
54+
55+
# ── Seed clades ───────────────────────────────────────────────────────────
56+
57+
echo "[seed-verisimdb] Seeding clade:clades ..."
58+
CLADES_JSON="${DATA_DIR}/clades.json"
59+
if [[ ! -f "${CLADES_JSON}" ]]; then
60+
echo "[seed-verisimdb] ERROR: ${CLADES_JSON} not found" >&2
61+
exit 1
62+
fi
63+
64+
seeded_clades=0
65+
error_clades=0
66+
67+
# Use jq to iterate over the clades array
68+
while IFS= read -r clade_doc; do
69+
code=$(echo "${clade_doc}" | jq -r '.code')
70+
if put_doc "clade:clades" "${code}" "${clade_doc}"; then
71+
seeded_clades=$((seeded_clades + 1))
72+
else
73+
error_clades=$((error_clades + 1))
74+
fi
75+
done < <(jq -c '.[]' "${CLADES_JSON}")
76+
77+
echo "[seed-verisimdb] Clades: seeded=${seeded_clades} errors=${error_clades}"
78+
79+
# ── Seed repos ────────────────────────────────────────────────────────────
80+
81+
echo "[seed-verisimdb] Seeding clade:repos ..."
82+
REPOS_JSON="${DATA_DIR}/repos.json"
83+
if [[ ! -f "${REPOS_JSON}" ]]; then
84+
echo "[seed-verisimdb] ERROR: ${REPOS_JSON} not found" >&2
85+
exit 1
86+
fi
87+
88+
seeded_repos=0
89+
error_repos=0
90+
91+
while IFS= read -r repo_doc; do
92+
name=$(echo "${repo_doc}" | jq -r '.name')
93+
if put_doc "clade:repos" "${name}" "${repo_doc}"; then
94+
seeded_repos=$((seeded_repos + 1))
95+
else
96+
error_repos=$((error_repos + 1))
97+
fi
98+
done < <(jq -c '.[]' "${REPOS_JSON}")
99+
100+
echo "[seed-verisimdb] Repos: seeded=${seeded_repos} errors=${error_repos}"
101+
102+
# ── Seed index ────────────────────────────────────────────────────────────
103+
104+
echo "[seed-verisimdb] Seeding clade:index/latest ..."
105+
INDEX_JSON="${DATA_DIR}/index.json"
106+
if [[ -f "${INDEX_JSON}" ]]; then
107+
if put_doc "clade:index" "latest" "$(cat "${INDEX_JSON}")"; then
108+
echo "[seed-verisimdb] Index: seeded=1"
109+
else
110+
echo "[seed-verisimdb] Index: errors=1"
111+
fi
112+
else
113+
echo "[seed-verisimdb] WARN: ${INDEX_JSON} not found — skipping index seed" >&2
114+
fi
115+
116+
# ── Summary ───────────────────────────────────────────────────────────────
117+
118+
total_seeded=$((seeded_clades + seeded_repos))
119+
total_errors=$((error_clades + error_repos))
120+
121+
echo ""
122+
echo "[seed-verisimdb] Done. total_seeded=${total_seeded} total_errors=${total_errors}"
123+
124+
if [[ "${total_errors}" -gt 0 ]]; then
125+
echo "[seed-verisimdb] Some documents failed to seed. Check VeriSimDB logs." >&2
126+
exit 1
127+
fi

‎worker/src/index.js‎

Lines changed: 46 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,13 @@
1111
// GET /v1/llm?q=... — Structured output for LLM consumption
1212
// GET /v1/dashboard — Ecosystem overview
1313
// GET /v1/health — Health check
14+
// GET /v1/query/language?q=.. — Query repos by language (VeriSimDB-backed)
15+
// GET /v1/query/tag?q=... — Query repos by tag (VeriSimDB-backed)
16+
//
17+
// Storage: Dual-write pattern — Cloudflare KV (primary cache) + VeriSimDB
18+
// (persistent store). loadIndex() prefers VeriSimDB, falls back to KV.
19+
20+
import { loadIndex as vdbLoadIndex, queryByLanguage, queryByTag } from './verisimdb.js';
1421

1522
const CORS_HEADERS = {
1623
'Access-Control-Allow-Origin': '*',
@@ -34,13 +41,10 @@ function error(message, status = 404) {
3441
return json({ error: message }, status);
3542
}
3643

37-
// Load index data from KV (cached in-memory per request)
44+
// Load index data — prefers VeriSimDB, falls back to Cloudflare KV.
45+
// Uses the unified loadIndex from verisimdb.js (dual-store aware).
3846
async function loadIndex(env) {
39-
const cached = await env.CLADE_KV.get('index', { type: 'json' });
40-
if (!cached) {
41-
return null;
42-
}
43-
return cached;
47+
return vdbLoadIndex(env);
4448
}
4549

4650
// Search repos by query string (simple text matching across name + description)
@@ -263,6 +267,8 @@ export default {
263267
'/v1/repos': 'All repos (params: clade, page, limit)',
264268
'/v1/search?q=...': 'Full-text search across repos',
265269
'/v1/llm?q=...': 'Structured output for LLM consumption',
270+
'/v1/query/language?q=...': 'Query repos by language keyword (VeriSimDB)',
271+
'/v1/query/tag?q=...': 'Query repos by tag (VeriSimDB)',
266272
'/v1/health': 'Health check',
267273
},
268274
total_repos: index ? index.total_repos : 'loading',
@@ -271,6 +277,40 @@ export default {
271277
});
272278
}
273279

280+
// Route: GET /v1/query/language?q=rust
281+
// Query repos by primary language keyword (VeriSimDB-backed, falls back to KV)
282+
if (path === '/v1/query/language') {
283+
const q = url.searchParams.get('q');
284+
if (!q || q.trim().length === 0) {
285+
return error('Query parameter q is required', 400);
286+
}
287+
const results = await queryByLanguage(env, q.trim());
288+
return json({
289+
query: q.trim(),
290+
type: 'language',
291+
results,
292+
total: results.length,
293+
source: 'verisimdb',
294+
});
295+
}
296+
297+
// Route: GET /v1/query/tag?q=security
298+
// Query repos by tag / keyword (VeriSimDB-backed, falls back to KV)
299+
if (path === '/v1/query/tag') {
300+
const q = url.searchParams.get('q');
301+
if (!q || q.trim().length === 0) {
302+
return error('Query parameter q is required', 400);
303+
}
304+
const results = await queryByTag(env, q.trim());
305+
return json({
306+
query: q.trim(),
307+
type: 'tag',
308+
results,
309+
total: results.length,
310+
source: 'verisimdb',
311+
});
312+
}
313+
274314
return error('Not found. Try /v1 for API documentation.');
275315
},
276316
};

0 commit comments

Comments
 (0)