camelCase-aware search tokenization (#13)

* Extract identtoken package from searchrank

Move the identifier splitter (camelCase / separator / letter-digit boundaries)
out of searchrank into a leaf internal/identtoken package so search indexing can
share it. Pure structural change: searchrank behavior is unchanged.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Index camelCase identifier sub-tokens for search

buildSearchContent emitted node names verbatim, so unicode61 FTS treated a
camelCase name as one token: inner-word queries ('limit', 'user', 'suspects')
and any query whose term was not a name prefix returned nothing, starving the
reranker of candidates.

Emit deduplicated camelCase / separator / letter-digit sub-tokens of the name
and qualified name alongside the verbatim name (which still serves whole-name
and prefix matching). Content is backend-agnostic, so SQLite FTS5 and Postgres
tsvector both benefit; ccg build rebuilds the index, so no migration is needed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Document camelCase search sub-tokens and rebuild-after-upgrade

Note that search content now indexes identifier sub-tokens and that an existing
graph needs one full `ccg build` after upgrading, since a pure incremental
workflow only re-indexes changed nodes.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

* Split camelCase query terms into sub-token OR groups

Query sanitizers expanded each term to a single prefix, so a camelCase query
term only matched a whole indexed token. Expand a camelCase term to
"(whole OR (sub1 AND sub2 ...))" for both FTS5 and Postgres tsquery, mirroring
the sub-tokens indexed at build time, via a shared buildPrefixQuery helper.
Add sanitizeRawTokens to keep original case for splitting; sanitizeTokens now
derives from it, so exact-name promotion is unchanged.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: tae2089

guide/architecture.md

@@ -41,6 +41,13 @@ Per-database full-text search backends:
 - **SQLite**: FTS5
 - **PostgreSQL**: tsvector + GIN index
 
+Search content indexes camelCase identifier sub-tokens (`getUserById` also
+indexes `get`, `user`, `by`, `id`), so inner words are searchable. Because this
+content is generated at index time, an existing graph only gains sub-tokens for
+nodes that are re-indexed. Run a full `ccg build` once after upgrading — a
+pure incremental-update workflow only refreshes changed nodes, leaving
+untouched nodes without sub-tokens until they change.
+
 Full builds and explicit postprocess runs rebuild namespace search state.
 Incremental updates refresh only affected search documents and FTS rows, while
 community postprocessing can still be namespace-wide. Persisted stored-flow

guide/ko/architecture.md

@@ -43,6 +43,12 @@ GORM ORM 기반 저장소입니다. SQLite 및 PostgreSQL과 호환됩니다.
 - **SQLite**: FTS5
 - **PostgreSQL**: tsvector + GIN 인덱스
 
+검색 콘텐츠는 camelCase 식별자의 서브토큰을 색인합니다(`getUserById`는 `get`,
+`user`, `by`, `id`도 색인) — 내부 단어로도 검색됩니다. 이 콘텐츠는 색인 시점에
+생성되므로, 기존 그래프는 재색인된 노드에 대해서만 서브토큰을 갖습니다. 업그레이드
+후 `ccg build`로 한 번 전체 재빌드하십시오 — 순수 증분 업데이트만 반복하면 변경된
+노드만 갱신되어, 건드리지 않은 노드는 변경 전까지 서브토큰이 없습니다.
+
 전체 빌드 및 명시적인 사후 처리 실행은 네임스페이스 검색 상태를 재생성합니다. 증분 업데이트는 영향을 받는 검색 문서와 FTS 행만 갱신하는 반면, 커뮤니티 사후 처리는 여전히 네임스페이스 전체에 걸쳐 이루어질 수 있습니다. 영구 저장된 흐름(stored-flow)의 재생성은 전체 사후 처리 실행 및 명시적인 `run_postprocess(flows=true)` 호출 시 구현되어 있으며, 진입점별 흐름 쿼리에는 `trace_flow`를 사용하십시오.
 
 ### 분석 (Analysis) (`internal/analysis/`)

internal/identtoken/identtoken.go

@@ -0,0 +1,48 @@
+// Package identtoken splits source identifiers into lowercased sub-tokens on
+// separators, camelCase boundaries, and letter/digit transitions. It is a leaf
+// utility shared by search indexing (content generation) and search reranking.
+package identtoken
+
+import (
+	"strings"
+	"unicode"
+)
+
+// Split breaks an identifier into lowercased sub-tokens on separators, camelCase
+// boundaries, and letter/digit transitions ("getUserById" -> get, user, by, id;
+// "HTTPServer" -> http, server; "parseHTML5" -> parse, html, 5).
+func Split(s string) []string {
+	runes := []rune(s)
+	var tokens []string
+	var cur []rune
+	flush := func() {
+		if len(cur) > 0 {
+			tokens = append(tokens, strings.ToLower(string(cur)))
+			cur = cur[:0]
+		}
+	}
+	for i, r := range runes {
+		if !isAlnum(r) {
+			flush()
+			continue
+		}
+		if len(cur) > 0 {
+			prev := cur[len(cur)-1]
+			switch {
+			case unicode.IsUpper(r) && (unicode.IsLower(prev) || unicode.IsDigit(prev)):
+				flush() // lower/digit -> Upper: new word
+			case unicode.IsUpper(r) && unicode.IsUpper(prev) && i+1 < len(runes) && unicode.IsLower(runes[i+1]):
+				flush() // acronym tail begins a new word (HTTPServer -> http, server)
+			case unicode.IsDigit(r) != unicode.IsDigit(prev):
+				flush() // letter/digit transition
+			}
+		}
+		cur = append(cur, r)
+	}
+	flush()
+	return tokens
+}
+
+func isAlnum(r rune) bool {
+	return unicode.IsLetter(r) || unicode.IsDigit(r)
+}

internal/identtoken/identtoken_test.go

@@ -0,0 +1,30 @@
+package identtoken
+
+import "testing"
+
+func TestSplit(t *testing.T) {
+	cases := []struct {
+		in   string
+		want []string
+	}{
+		{"getUserById", []string{"get", "user", "by", "id"}},
+		{"HTTPServer", []string{"http", "server"}},
+		{"user_id", []string{"user", "id"}},
+		{"parseHTML5", []string{"parse", "html", "5"}},
+		{"", nil},
+		{"lower", []string{"lower"}},
+	}
+	for _, c := range cases {
+		got := Split(c.in)
+		if len(got) != len(c.want) {
+			t.Errorf("Split(%q)=%v, want %v", c.in, got, c.want)
+			continue
+		}
+		for i := range c.want {
+			if got[i] != c.want[i] {
+				t.Errorf("Split(%q)=%v, want %v", c.in, got, c.want)
+				break
+			}
+		}
+	}
+}

internal/searchrank/searchrank.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"unicode"
 
+	"github.com/tae2089/code-context-graph/internal/identtoken"
 	"github.com/tae2089/code-context-graph/internal/model"
 )
 
@@ -117,7 +118,7 @@ func nameSim(qTokens []string, node model.Node) float64 {
 			continue
 		}
 		lower := strings.ToLower(raw)
-		subs := splitIdentifier(raw) // original case: camelCase boundaries matter
+		subs := identtoken.Split(raw) // original case: camelCase boundaries matter
 		sum := 0.0
 		for _, tok := range qTokens {
 			b := normLevSim(tok, lower)
@@ -132,45 +133,6 @@ func nameSim(qTokens []string, node model.Node) float64 {
 	return best
 }
 
-// splitIdentifier breaks an identifier into lowercased sub-tokens on separators,
-// camelCase boundaries, and letter/digit transitions ("getUserById" -> get,
-// user, by, id; "HTTPServer" -> http, server; "parseHTML5" -> parse, html, 5).
-func splitIdentifier(s string) []string {
-	runes := []rune(s)
-	var tokens []string
-	var cur []rune
-	flush := func() {
-		if len(cur) > 0 {
-			tokens = append(tokens, strings.ToLower(string(cur)))
-			cur = cur[:0]
-		}
-	}
-	for i, r := range runes {
-		if !isAlnum(r) {
-			flush()
-			continue
-		}
-		if len(cur) > 0 {
-			prev := cur[len(cur)-1]
-			switch {
-			case unicode.IsUpper(r) && (unicode.IsLower(prev) || unicode.IsDigit(prev)):
-				flush() // lower/digit -> Upper: new word
-			case unicode.IsUpper(r) && unicode.IsUpper(prev) && i+1 < len(runes) && unicode.IsLower(runes[i+1]):
-				flush() // acronym tail begins a new word (HTTPServer -> http, server)
-			case unicode.IsDigit(r) != unicode.IsDigit(prev):
-				flush() // letter/digit transition
-			}
-		}
-		cur = append(cur, r)
-	}
-	flush()
-	return tokens
-}
-
-func isAlnum(r rune) bool {
-	return unicode.IsLetter(r) || unicode.IsDigit(r)
-}
-
 // pathScore is the fraction of query tokens that appear as file-path segments.
 func pathScore(qTokens []string, node model.Node) float64 {
 	segs := map[string]struct{}{}

internal/searchrank/searchrank_test.go

@@ -131,31 +131,6 @@ func TestFetchLimit(t *testing.T) {
 	}
 }
 
-func TestSplitIdentifier(t *testing.T) {
-	cases := []struct {
-		in   string
-		want []string
-	}{
-		{"getUserById", []string{"get", "user", "by", "id"}},
-		{"HTTPServer", []string{"http", "server"}},
-		{"user_id", []string{"user", "id"}},
-		{"parseHTML5", []string{"parse", "html", "5"}},
-	}
-	for _, c := range cases {
-		got := splitIdentifier(c.in)
-		if len(got) != len(c.want) {
-			t.Errorf("splitIdentifier(%q)=%v, want %v", c.in, got, c.want)
-			continue
-		}
-		for i := range c.want {
-			if got[i] != c.want[i] {
-				t.Errorf("splitIdentifier(%q)=%v, want %v", c.in, got, c.want)
-				break
-			}
-		}
-	}
-}
-
 func TestLevenshtein(t *testing.T) {
 	cases := []struct {
 		a, b string

internal/service/indexer_test.go

@@ -117,6 +117,47 @@ func TestBuildSearchDocuments_IndexesFileBaseAndLanguageTokens(t *testing.T) {
 	}
 }
 
+// camelCase 식별자를 서브토큰으로도 색인해, 내부 단어(user, id 등)로 검색 가능하게 한다.
+func TestBuildSearchContent_EmitsIdentifierSubtokens(t *testing.T) {
+	tests := []struct {
+		name     string
+		node     model.Node
+		contains []string
+	}{
+		{
+			name:     "camelCase name and qualified name split",
+			node:     model.Node{Name: "getUserById", QualifiedName: "svc.getUserById", Kind: model.NodeKindFunction, FilePath: "svc/user.go", Language: "go"},
+			contains: []string{"getuserbyid", "get", "user", "by", "id"},
+		},
+		{
+			name:     "PascalCase class split",
+			node:     model.Node{Name: "UserService", QualifiedName: "pkg.UserService", Kind: model.NodeKindClass, FilePath: "pkg/svc.go", Language: "go"},
+			contains: []string{"userservice", "user", "service"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			content := strings.ToLower(buildSearchContent(tt.node, nil))
+			for _, want := range tt.contains {
+				if !containsToken(content, want) {
+					t.Fatalf("content %q missing subtoken %q", content, want)
+				}
+			}
+		})
+	}
+}
+
+// containsToken checks for a whitespace-delimited token, not a substring, so
+// "id" must appear as its own token rather than inside "identifier".
+func containsToken(content, tok string) bool {
+	for _, f := range strings.Fields(content) {
+		if f == tok {
+			return true
+		}
+	}
+	return false
+}
+
 type recordingGraphStore struct {
 	t             *testing.T
 	ops           []string

internal/service/search.go

@@ -12,6 +12,7 @@ import (
 	"github.com/tae2089/trace"
 
 	"github.com/tae2089/code-context-graph/internal/ctxns"
+	"github.com/tae2089/code-context-graph/internal/identtoken"
 	"github.com/tae2089/code-context-graph/internal/model"
 )
 
@@ -201,6 +202,13 @@ func buildSearchContent(n model.Node, annByNode map[uint]*model.Annotation) stri
 	sb.WriteString(n.QualifiedName)
 	sb.WriteByte(' ')
 	sb.WriteString(string(n.Kind))
+	// Emit identifier sub-tokens so camelCase names are searchable by their
+	// inner words (getUserById -> get, user, by, id). The verbatim name above
+	// preserves existing whole-name and prefix matching.
+	for _, token := range identSubtokens(n.Name, n.QualifiedName) {
+		sb.WriteByte(' ')
+		sb.WriteString(token)
+	}
 	for _, token := range searchPathTokens(n.FilePath) {
 		sb.WriteByte(' ')
 		sb.WriteString(token)
@@ -222,6 +230,24 @@ func buildSearchContent(n model.Node, annByNode map[uint]*model.Annotation) stri
 	return sb.String()
 }
 
+// identSubtokens returns the deduplicated camelCase/separator sub-tokens of a
+// node's name and qualified name. Dedup keeps a token's term frequency from
+// being inflated just because it appears in both the name and the qualified name.
+func identSubtokens(name, qualifiedName string) []string {
+	seen := map[string]struct{}{}
+	var tokens []string
+	for _, raw := range []string{name, qualifiedName} {
+		for _, tok := range identtoken.Split(raw) {
+			if _, ok := seen[tok]; ok {
+				continue
+			}
+			seen[tok] = struct{}{}
+			tokens = append(tokens, tok)
+		}
+	}
+	return tokens
+}
+
 // searchPathTokens derives lowercase filename tokens and optional language aliases for search indexing.
 // @intent 파일 경로 자체도 검색 힌트가 되도록 basename과 언어 별칭을 토큰화한다.
 func searchPathTokens(filePath string) []string {