Add hype-rag CLI, wasm bindings and examples

Convert repository to a Cargo workspace and bump hypembed to 0.3.0. Add new crates: "hype-rag" (CLI for local RAG with indexing, chunking, sqlite-backed store, and search utilities) and "hypembed-wasm" (wasm-bindgen wrapper exposing a WasmEmbedder). Add unicode-normalization and platform-gated deps, example helper to write a tiny model, Bunny Edge WebAssembly glue (JS/TS + wasm artifact) and a Cloudflare-style embed handler, tests for RAG and wasm paths, and utility scripts to build/verify wasm glue. Also update .gitignore to ignore /mcps/.

Author: Tuntii

.gitignore

@@ -1,4 +1,5 @@
 /target
+/mcps/
 Cargo.lock
 *.swp
 *.swo

Cargo.toml

@@ -1,6 +1,10 @@
+[workspace]
+members = [".", "hype-rag", "hypembed-wasm"]
+resolver = "2"
+
 [package]
 name = "hypembed"
-version = "0.2.1"
+version = "0.3.0"
 edition = "2021"
 rust-version = "1.70"
 description = "Pure-Rust BERT-compatible text embedding inference for local-first applications."
@@ -16,13 +20,19 @@ categories = ["science", "text-processing"]
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 thiserror = "2"
+unicode-normalization = "0.1"
+
+[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
 memmap2 = "0.9"
 rayon = "1.10"
-unicode-normalization = "0.1"
 
 [dev-dependencies]
 criterion = { version = "0.5", features = ["html_reports"] }
 
 [[bench]]
 name = "inference_bench"
 harness = false
+
+[[example]]
+name = "write_tiny_model"
+path = "examples/write_tiny_model.rs"

examples/bunny-edge/embed-handler.js

@@ -0,0 +1,81 @@
+/**
+ * Bunny Edge Script example: POST JSON { "text": "..." } -> embedding vector.
+ *
+ * Prerequisites:
+ * - Regenerate glue: scripts\generate_bunny_glue.cmd (or verify_wasm.cmd)
+ * - Deploy hypembed_wasm.js + hypembed_wasm_bg.wasm alongside this handler
+ * - Provide model bytes (config.json, vocab.txt, model.safetensors) via edge storage or KV
+ */
+
+import init, { WasmEmbedder } from "./hypembed_wasm.js";
+
+let embedder = null;
+let wasmReady = false;
+
+async function ensureWasm() {
+  if (!wasmReady) {
+    await init();
+    wasmReady = true;
+  }
+}
+
+async function loadEmbedder(env) {
+  if (embedder) {
+    return embedder;
+  }
+
+  await ensureWasm();
+
+  const configJson = await env.MODEL_CONFIG.get("config.json", { type: "text" });
+  const vocabTxt = await env.MODEL_VOCAB.get("vocab.txt", { type: "text" });
+  const weights = await env.MODEL_WEIGHTS.get("model.safetensors", { type: "arrayBuffer" });
+
+  embedder = new WasmEmbedder(configJson, vocabTxt, new Uint8Array(weights));
+  return embedder;
+}
+
+export default {
+  async fetch(request, env) {
+    if (request.method !== "POST") {
+      return new Response(JSON.stringify({ error: "POST required" }), {
+        status: 405,
+        headers: { "content-type": "application/json" },
+      });
+    }
+
+    let body;
+    try {
+      body = await request.json();
+    } catch {
+      return new Response(JSON.stringify({ error: "invalid JSON body" }), {
+        status: 400,
+        headers: { "content-type": "application/json" },
+      });
+    }
+
+    const text = body?.text;
+    if (typeof text !== "string" || text.trim().length === 0) {
+      return new Response(JSON.stringify({ error: "text field required" }), {
+        status: 400,
+        headers: { "content-type": "application/json" },
+      });
+    }
+
+    try {
+      const model = await loadEmbedder(env);
+      const vector = model.embed(text);
+      return new Response(
+        JSON.stringify({
+          dim: vector.length,
+          embedding: Array.from(vector),
+        }),
+        { headers: { "content-type": "application/json" } },
+      );
+    } catch (err) {
+      return new Response(
+        JSON.stringify({ error: String(err?.message ?? err) }),
+        { status: 500, headers: { "content-type": "application/json" } },
+      );
+    }
+  },
+};

examples/bunny-edge/hypembed_wasm.d.ts

@@ -0,0 +1,60 @@
+/* tslint:disable */
+/* eslint-disable */
+
+/**
+ * WASM embedder wrapping the hypembed core pipeline.
+ */
+export class WasmEmbedder {
+    free(): void;
+    [Symbol.dispose](): void;
+    /**
+     * Embed a single text and return the vector as a JS Float32Array-compatible Vec.
+     */
+    embed(text: string): Float32Array;
+    /**
+     * Return the model hidden size (embedding dimension).
+     */
+    hidden_size(): number;
+    /**
+     * Create an embedder from in-memory model bytes (no filesystem access).
+     */
+    constructor(config_json: string, vocab_txt: string, weights: Uint8Array);
+}
+
+export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module;
+
+export interface InitOutput {
+    readonly memory: WebAssembly.Memory;
+    readonly __wbg_wasmembedder_free: (a: number, b: number) => void;
+    readonly wasmembedder_embed: (a: number, b: number, c: number) => [number, number, number, number];
+    readonly wasmembedder_hidden_size: (a: number) => number;
+    readonly wasmembedder_new: (a: number, b: number, c: number, d: number, e: number, f: number) => [number, number, number];
+    readonly __wbindgen_externrefs: WebAssembly.Table;
+    readonly __wbindgen_malloc: (a: number, b: number) => number;
+    readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number;
+    readonly __externref_table_dealloc: (a: number) => void;
+    readonly __wbindgen_free: (a: number, b: number, c: number) => void;
+    readonly __wbindgen_start: () => void;
+}
+
+export type SyncInitInput = BufferSource | WebAssembly.Module;
+
+/**
+ * Instantiates the given `module`, which can either be bytes or
+ * a precompiled `WebAssembly.Module`.
+ *
+ * @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated.
+ *
+ * @returns {InitOutput}
+ */
+export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput;
+
+/**
+ * If `module_or_path` is {RequestInfo} or {URL}, makes a request and
+ * for everything else, calls `WebAssembly.instantiate` directly.
+ *
+ * @param {{ module_or_path: InitInput | Promise<InitInput> }} module_or_path - Passing `InitInput` directly is deprecated.
+ *
+ * @returns {Promise<InitOutput>}
+ */
+export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise<InitInput> } | InitInput | Promise<InitInput>): Promise<InitOutput>;