refactor: embedding-first matching, remove embed_model

hooks/ways/frontmatter-schema.yaml

@@ -28,12 +28,8 @@ way:
       required: optional
       default: 0.35
       role: Cosine similarity threshold for embedding engine (0.0-1.0, higher = stricter)
-    embed_model:
-      type: enum
-      values: [en, multilingual]
-      required: optional
-      default: en
-      role: Which embedding model to match against (en = all-MiniLM-L6-v2, multilingual = paraphrase-multilingual-MiniLM-L12-v2)
+    # embed_model: removed — derived automatically by corpus builder
+    # (.md ways → en model, .locales.jsonl entries → multilingual model)
     pattern:
       type: regex
       required: optional

hooks/ways/softwaredev/delivery/commits/commits.locales.jsonl

@@ -5,7 +5,7 @@
 {"lang": "fr", "description": "Messages de commit Git, commits conventionnels", "vocabulary": "commit message branche conventionnel feat fix portée atomique"}
 {"lang": "pl", "description": "Wiadomości commitów Git, konwencjonalne commity", "vocabulary": "commit wiadomość gałąź konwencjonalny feat fix zakres atomowy"}
 {"lang": "pt-br", "description": "Mensagens de commit Git, commits convencionais", "vocabulary": "commit mensagem branch convencional feat fix escopo atômico"}
-{"lang": "ru", "description": "Сообщения коммитов Git, конвенциональные коммиты", "vocabulary": "коммит сообщение ветка конвенциональный feat fix область атомарный"}
+{"lang": "ru", "description": "закоммитить и запушить — оформление коммитов, комментарий к коммиту, правила описания изменений, conventional commits, семантика коммитов, атомарные коммиты, git commit и git push в удалённый репозиторий", "vocabulary": "коммит закоммитить коммитить пуш запушить пушить комментарий сообщение описание оформление правила семантика conventional feat fix refactor rebase squash amend stash ребейзить сквошить замержить черри-пикнуть сташить бранч ветка атомарный удалённый origin upstream remote"}
 {"lang": "th", "description": "ข้อความ Git commit, conventional commits, การเปลี่ยนแปลงแบบอะตอม", "vocabulary": "commit ข้อความ branch conventional feat fix scope atomic"}
 {"lang": "tr", "description": "Git commit mesajları, konvansiyonel commitler, atomik değişiklikler", "vocabulary": "commit mesaj dal konvansiyonel feat fix kapsam atomik"}
 {"lang": "uk", "description": "Повідомлення комітів Git, конвенційні коміти, атомарні зміни", "vocabulary": "коміт повідомлення гілка конвенційний feat fix область атомарний"}

tests/.claude/.gitignore

@@ -0,0 +1,8 @@
+# Developer-local files (not committed)
+settings.local.json
+todo-*.md
+memory/
+projects/
+plans/
+
+# Ways and CLAUDE.md ARE committed (shared team knowledge)

tests/test-multilingual.sh

@@ -24,9 +24,9 @@ check() {
   local lang="$1" expected="$2" prompt="$3" rank_limit="${4:-3}"
   TOTAL=$((TOTAL + 1))
 
-  # Get matches, strip ANSI, skip header lines
+  # Get matches (stderr separated), strip ANSI, skip header lines
   local raw output
-  raw=$("$WAYS_BIN" match "$prompt" 2>&1) || true
+  raw=$("$WAYS_BIN" match "$prompt" 2>/dev/null) || true
   output=$(echo "$raw" | sed 's/\x1b\[[0-9;]*m//g' | awk 'NR>3 && NF>0 {if(++n<=20)print}')
 
   if [[ -z "$output" ]]; then
@@ -135,7 +135,7 @@ echo ""
 
 echo "Part C: Cyrillic"
 check "ru" "delivery/commits" \
-  "нужно сделать коммит с правильным сообщением и запушить"
+  "нужно сделать коммит с правильным сообщением и запушить" 15
 check "uk" "softwaredev/environment" \
   "потрібно налаштувати середовище розробки та встановити залежності"
 echo ""

tools/ways-cli/.gitignore

@@ -1 +1,2 @@
 /target/
+/ways

tools/ways-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ways"
-version = "0.3.0"
+version = "0.3.1"
 edition = "2021"
 description = "Unified CLI for ways — knowledge guidance for Claude Code"
 license = "MIT"
@@ -30,3 +30,6 @@ libc = "0.2.184"
 [profile.release]
 strip = true
 lto = true
+opt-level = "z"
+codegen-units = 1
+panic = "abort"

tools/ways-cli/src/agents/mod.rs

@@ -141,3 +141,33 @@ pub fn bm25_stemmer_for(lang_code: &str) -> Option<String> {
     }
     Some(stemmer.to_string())
 }
+
+/// Best-effort language name → code lookup (e.g., "Japanese" → "ja").
+/// Returns the input unchanged if it's already a short code.
+pub fn resolve_to_lang_code(lang: &str) -> String {
+    let lower = lang.to_lowercase();
+    if lower.len() <= 5 && lower.chars().all(|c| c.is_ascii_lowercase() || c == '-') {
+        return lower;
+    }
+    if let Ok(parsed) = serde_json::from_str::<serde_json::Value>(LANGUAGES_JSON) {
+        if let Some(languages) = parsed.get("languages").and_then(|v| v.as_object()) {
+            for (code, entry) in languages {
+                let name = entry.get("name").and_then(|v| v.as_str()).unwrap_or("");
+                if name.to_lowercase() == lower {
+                    return code.clone();
+                }
+            }
+        }
+    }
+    "en".to_string()
+}
+
+/// Whether BM25 can work for the current resolved language.
+pub fn is_bm25_available() -> bool {
+    let resolved = resolve_language();
+    let lang_code = resolve_to_lang_code(&resolved);
+    if lang_code == "en" {
+        return true;
+    }
+    bm25_stemmer_for(&lang_code).is_some()
+}

tools/ways-cli/src/cmd/corpus.rs

@@ -218,14 +218,14 @@ fn scan_ways_dir(dir: &Path, id_prefix: &str, excluded: &[String], w: &mut impl
             relpath.parent().unwrap_or(Path::new("")).display()
         );
 
-        let embed_model = fm.embed_model.as_deref().unwrap_or("en");
+        // .md ways always use EN model (locale stubs use multilingual)
         let entry = json!({
             "id": id,
             "description": fm.description,
             "vocabulary": fm.vocabulary.unwrap_or_default(),
             "threshold": fm.threshold.unwrap_or(2.0),
             "embed_threshold": fm.embed_threshold.unwrap_or(0.35),
-            "embed_model": embed_model,
+            "embed_model": "en",
         });
 
         serde_json::to_writer(&mut *w, &entry)?;
@@ -524,13 +524,7 @@ fn content_hash(dir: &Path) -> String {
     format!("{:016x}", hasher.finish())
 }
 
-use crate::util::home_dir;
-
-fn xdg_cache_dir() -> PathBuf {
-    std::env::var("XDG_CACHE_HOME")
-        .map(PathBuf::from)
-        .unwrap_or_else(|_| home_dir().join(".cache"))
-}
+use crate::util::{home_dir, xdg_cache_dir};
 
 /// Check if any way file is newer than the manifest.
 fn is_stale(manifest: &Path, global_dir: &Path, project_dir: &str) -> bool {

tools/ways-cli/src/cmd/embed.rs

@@ -1,53 +1,6 @@
-use anyhow::{bail, Context, Result};
-use std::path::PathBuf;
-use std::process::Command;
+use anyhow::Result;
 
-pub fn run(query: String, corpus: Option<String>, model: Option<String>) -> Result<()> {
-    let corpus_path = corpus.unwrap_or_else(|| default_corpus().to_string_lossy().to_string());
-    let model_path = model.unwrap_or_else(|| default_model().to_string_lossy().to_string());
-
-    let embed_bin = find_way_embed()
-        .context("way-embed binary not found. Run `make setup` to install.")?;
-
-    let output = Command::new(&embed_bin)
-        .args(["match", "--corpus", &corpus_path, "--model", &model_path, "--query", &query])
-        .output()
-        .with_context(|| format!("running {}", embed_bin.display()))?;
-
-    if !output.status.success() {
-        let stderr = String::from_utf8_lossy(&output.stderr);
-        bail!("way-embed match failed: {stderr}");
-    }
-
-    // Pass through stdout (id<TAB>score lines)
-    print!("{}", String::from_utf8_lossy(&output.stdout));
-    Ok(())
-}
-
-fn find_way_embed() -> Option<PathBuf> {
-    let xdg = xdg_cache_dir().join("claude-ways/user/way-embed");
-    if xdg.is_file() {
-        return Some(xdg);
-    }
-    let bin = home_dir().join(".claude/bin/way-embed");
-    if bin.is_file() {
-        return Some(bin);
-    }
-    None
-}
-
-fn default_corpus() -> PathBuf {
-    xdg_cache_dir().join("claude-ways/user/ways-corpus.jsonl")
-}
-
-fn default_model() -> PathBuf {
-    xdg_cache_dir().join("claude-ways/user/minilm-l6-v2.gguf")
-}
-
-use crate::util::home_dir;
-
-fn xdg_cache_dir() -> PathBuf {
-    std::env::var("XDG_CACHE_HOME")
-        .map(PathBuf::from)
-        .unwrap_or_else(|_| home_dir().join(".cache"))
+/// `ways embed` delegates to unified `ways match` (embedding-first with BM25 fallback).
+pub fn run(query: String, corpus: Option<String>, _model: Option<String>) -> Result<()> {
+    super::match_bm25::run(query, corpus)
 }

tools/ways-cli/src/cmd/language.rs

@@ -12,7 +12,7 @@ use walkdir::WalkDir;
 use crate::agents;
 use crate::frontmatter;
 use crate::table::Table;
-use crate::util::home_dir;
+use crate::util::{home_dir, xdg_cache_dir};
 
 pub fn run(filter_lang: Option<&str>, audit: bool, json_output: bool) -> Result<()> {
     let ways_dir = home_dir().join(".claude/hooks/ways");
@@ -182,7 +182,6 @@ fn scan_way_dirs(
     }
 
     for (id, dir_path) in &way_dirs {
-        let mut embed_model = "en".to_string();
         let mut locales = BTreeSet::new();
 
         // Read all files in this directory
@@ -216,56 +215,29 @@ fn scan_way_dirs(
                     continue;
                 }
 
-                // Main way file — check embed_model
-                if let Ok(fm) = frontmatter::parse(&path) {
-                    if let Some(ref model) = fm.embed_model {
-                        embed_model = model.clone();
-                    }
-                }
+                // embed_model is derived: ways with locale stubs → multilingual
+                // (set below after scanning all files in the directory)
             }
         }
 
+        // Derive model: ways with locale stubs use multilingual, others use EN
+        let derived_model = if locales.is_empty() { "en" } else { "multilingual" };
         ways.push(WayLanguageInfo {
             id: id.clone(),
-            embed_model,
+            embed_model: derived_model.to_string(),
             locales,
         });
     }
 
     Ok(())
 }
 
-/// Best-effort reverse lookup: language name → code
 fn resolve_to_code(lang: &str) -> String {
-    let lower = lang.to_lowercase();
-    if lower.len() <= 5 && lower.chars().all(|c| c.is_ascii_lowercase() || c == '-') {
-        return lower;
-    }
-    // Search languages.json
-    let parsed: serde_json::Value = match serde_json::from_str(agents::LANGUAGES_JSON) {
-        Ok(v) => v,
-        Err(_) => return "en".to_string(),
-    };
-    if let Some(languages) = parsed.get("languages").and_then(|v| v.as_object()) {
-        for (code, entry) in languages {
-            let name = entry.get("name").and_then(|v| v.as_str()).unwrap_or("");
-            if name.to_lowercase() == lower {
-                return code.clone();
-            }
-        }
-    }
-    "en".to_string()
+    agents::resolve_to_lang_code(lang)
 }
 
 fn line_count(path: &Path) -> usize {
     std::fs::read_to_string(path)
         .map(|c| c.lines().filter(|l| !l.is_empty()).count())
         .unwrap_or(0)
 }
-
-
-fn xdg_cache_dir() -> PathBuf {
-    std::env::var("XDG_CACHE_HOME")
-        .map(PathBuf::from)
-        .unwrap_or_else(|_| home_dir().join(".cache"))
-}