Skip to content

Commit 8486b25

Browse files
committed
perf: cache parsing and batch graph updates
Reuse versioned parse results and unresolved candidates during builds and updates. Batch flow traversal and reduce resolver allocation churn for large Kotlin corpora.
1 parent 2210713 commit 8486b25

42 files changed

Lines changed: 1411 additions & 84 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

internal/adapters/inbound/cli/build.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/spf13/cobra"
88
"github.com/tae2089/trace"
99

10+
"github.com/tae2089/code-context-graph/internal/app/ingest"
1011
"github.com/tae2089/code-context-graph/internal/app/ingest/workflow"
1112
requestctx "github.com/tae2089/code-context-graph/internal/ctx"
1213
)
@@ -37,10 +38,12 @@ func newBuildCmd(deps *Deps) *cobra.Command {
3738
paths := resolveIncludePaths(includePaths)
3839
fileLimit := resolveMaxFileBytes(maxFileBytes)
3940
totalLimit := resolveMaxTotalParsedBytes(maxTotalParsedBytes)
41+
parseCache, _ := deps.Store.(ingest.ParseCache)
4042
svc := &workflow.Service{
4143
Store: deps.Store,
4244
UnitOfWork: deps.UnitOfWork,
4345
Search: deps.Search,
46+
ParseCache: parseCache,
4447
Walkers: deps.Walkers,
4548
Logger: deps.Logger,
4649
}

internal/adapters/inbound/cli/update.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"github.com/spf13/cobra"
77
"github.com/tae2089/trace"
88

9+
"github.com/tae2089/code-context-graph/internal/app/ingest"
910
"github.com/tae2089/code-context-graph/internal/app/ingest/workflow"
1011
requestctx "github.com/tae2089/code-context-graph/internal/ctx"
1112
)
@@ -40,10 +41,12 @@ func newUpdateCmd(deps *Deps) *cobra.Command {
4041
ns := resolveNamespace(cmd)
4142
ctx = requestctx.WithNamespace(ctx, ns)
4243

44+
parseCache, _ := deps.Store.(ingest.ParseCache)
4345
svc := &workflow.Service{
4446
Store: deps.Store,
4547
UnitOfWork: deps.UnitOfWork,
4648
Search: deps.Search,
49+
ParseCache: parseCache,
4750
Walkers: deps.Walkers,
4851
Logger: deps.Logger,
4952
}

internal/adapters/inbound/mcp/handler_parse.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,12 @@ func (h *handlers) graphService() *workflow.Service {
7171
if candidate, ok := h.deps.Build.Store.(ingest.GraphStore); ok {
7272
graphStore = candidate
7373
}
74+
parseCache, _ := h.deps.Build.Store.(ingest.ParseCache)
7475
return &workflow.Service{
7576
Store: graphStore,
7677
UnitOfWork: h.deps.Build.UnitOfWork,
7778
Search: h.deps.Build.Search,
79+
ParseCache: parseCache,
7880
Parsers: walkers,
7981
Logger: h.logger(),
8082
}

internal/adapters/outbound/graphgorm/store.go

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ func (s *Store) AutoMigrate() error {
4646
&graph.CommunityMembership{},
4747
&graph.Flow{},
4848
&graph.FlowMembership{},
49+
&graph.ParseCacheEntry{},
50+
&graph.UnresolvedEdgeCandidate{},
51+
&graph.UnresolvedIndexState{},
4952
); err != nil {
5053
return err
5154
}
@@ -57,6 +60,43 @@ func (s *Store) AutoMigrate() error {
5760
return nil
5861
}
5962

63+
// LoadParseResult returns a cached parser payload only when every identity field matches.
64+
// @intent prevent stale parser output from being reused across content, path, parser, context, or namespace changes.
65+
func (s *Store) LoadParseResult(ctx context.Context, key ingest.ParseCacheKey) ([]byte, bool, error) {
66+
var entry graph.ParseCacheEntry
67+
ns := requestctx.FromContext(ctx)
68+
result := s.db.WithContext(ctx).Where(
69+
"namespace = ? AND file_path = ? AND source_hash = ? AND parser_version = ? AND context_hash = ?",
70+
ns, key.FilePath, key.SourceHash, key.ParserVersion, key.ContextHash,
71+
).First(&entry)
72+
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
73+
return nil, false, nil
74+
}
75+
if result.Error != nil {
76+
return nil, false, trace.Wrap(result.Error, "load parse cache result")
77+
}
78+
return entry.Payload, true, nil
79+
}
80+
81+
// StoreParseResult replaces the latest cached parser payload for one namespace/file path.
82+
// @intent retain one bounded current cache entry per source path instead of accumulating every historical hash.
83+
// @sideEffect inserts or updates a parse_cache_entries row.
84+
func (s *Store) StoreParseResult(ctx context.Context, key ingest.ParseCacheKey, payload []byte) error {
85+
entry := graph.ParseCacheEntry{
86+
Namespace: requestctx.FromContext(ctx), FilePath: key.FilePath, SourceHash: key.SourceHash,
87+
ParserVersion: key.ParserVersion, ContextHash: key.ContextHash, Payload: payload,
88+
}
89+
if err := s.db.WithContext(ctx).Clauses(clause.OnConflict{
90+
Columns: []clause.Column{{Name: "namespace"}, {Name: "file_path"}},
91+
DoUpdates: clause.AssignmentColumns([]string{
92+
"source_hash", "parser_version", "context_hash", "payload", "updated_at",
93+
}),
94+
}).Create(&entry).Error; err != nil {
95+
return trace.Wrap(err, "store parse cache result")
96+
}
97+
return nil
98+
}
99+
60100
// UpsertNodes stores a batch of nodes keyed by qualified_name.
61101
// @intent apply parsed result nodes in bulk without creating duplicates.
62102
// @sideEffect performs batch inserts and updates on the nodes table.
@@ -257,6 +297,9 @@ func (s *Store) GetFileNodesByPathSuffix(ctx context.Context, suffix string) ([]
257297
// @domainRule connected edges and annotations must also be removed when deleting a file.
258298
func (s *Store) DeleteNodesByFile(ctx context.Context, filePath string) error {
259299
ns := requestctx.FromContext(ctx)
300+
if err := s.db.WithContext(ctx).Where("namespace = ? AND file_path = ?", ns, filePath).Delete(&graph.UnresolvedEdgeCandidate{}).Error; err != nil {
301+
return trace.Wrap(err, "delete file unresolved edges")
302+
}
260303
var nodeIDs []uint
261304
if err := s.db.WithContext(ctx).
262305
Model(&graph.Node{}).
@@ -342,6 +385,12 @@ func (s *Store) DeleteGraph(ctx context.Context) error {
342385
}
343386

344387
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
388+
if err := tx.Where("namespace = ?", ns).Delete(&graph.UnresolvedEdgeCandidate{}).Error; err != nil {
389+
return trace.Wrap(err, "delete namespace unresolved edges")
390+
}
391+
if err := tx.Where("namespace = ?", ns).Delete(&graph.UnresolvedIndexState{}).Error; err != nil {
392+
return trace.Wrap(err, "delete namespace unresolved index state")
393+
}
345394
if len(filePaths) > 0 {
346395
if err := tx.
347396
Where("namespace = ? AND file_path IN ?", ns, filePaths).

internal/adapters/outbound/graphgorm/store_test.go

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,88 @@ func TestAutoMigrate_SQLite(t *testing.T) {
6969
}
7070
}
7171

72+
func TestParseCache_RequiresCompleteIdentityAndReplacesLatestFileEntry(t *testing.T) {
73+
store := setupTestDB(t)
74+
ctx := requestctx.WithNamespace(context.Background(), "repo-a")
75+
v1 := ingest.ParseCacheKey{FilePath: "src/a.go", SourceHash: "hash-1", ParserVersion: "go-v1", ContextHash: "ctx-1"}
76+
if err := store.StoreParseResult(ctx, v1, []byte("first")); err != nil {
77+
t.Fatalf("store v1: %v", err)
78+
}
79+
if got, ok, err := store.LoadParseResult(ctx, v1); err != nil || !ok || string(got) != "first" {
80+
t.Fatalf("load v1 = %q, %v, %v", got, ok, err)
81+
}
82+
if _, ok, err := store.LoadParseResult(ctx, ingest.ParseCacheKey{FilePath: "src/a.go", SourceHash: "hash-2", ParserVersion: "go-v1", ContextHash: "ctx-1"}); err != nil || ok {
83+
t.Fatalf("changed source identity should miss, ok=%v err=%v", ok, err)
84+
}
85+
86+
v2 := ingest.ParseCacheKey{FilePath: "src/a.go", SourceHash: "hash-2", ParserVersion: "go-v2", ContextHash: "ctx-2"}
87+
if err := store.StoreParseResult(ctx, v2, []byte("second")); err != nil {
88+
t.Fatalf("store v2: %v", err)
89+
}
90+
if _, ok, err := store.LoadParseResult(ctx, v1); err != nil || ok {
91+
t.Fatalf("replaced identity should miss, ok=%v err=%v", ok, err)
92+
}
93+
if got, ok, err := store.LoadParseResult(ctx, v2); err != nil || !ok || string(got) != "second" {
94+
t.Fatalf("load v2 = %q, %v, %v", got, ok, err)
95+
}
96+
otherCtx := requestctx.WithNamespace(context.Background(), "repo-b")
97+
if _, ok, err := store.LoadParseResult(otherCtx, v2); err != nil || ok {
98+
t.Fatalf("other namespace should miss, ok=%v err=%v", ok, err)
99+
}
100+
}
101+
102+
func TestDeleteNodesByFile_RemovesOnlyMatchingNamespaceUnresolvedCandidates(t *testing.T) {
103+
store := setupTestDB(t)
104+
ctxA := requestctx.WithNamespace(context.Background(), "ns-a")
105+
ctxB := requestctx.WithNamespace(context.Background(), "ns-b")
106+
107+
for _, ctx := range []context.Context{ctxA, ctxB} {
108+
if err := store.UpsertNodes(ctx, []graph.Node{{
109+
QualifiedName: "pkg.Source", Kind: graph.NodeKindFunction, Name: "Source", FilePath: "source.go", StartLine: 1,
110+
}}); err != nil {
111+
t.Fatalf("UpsertNodes: %v", err)
112+
}
113+
if err := store.UpsertUnresolvedEdges(ctx, []graph.UnresolvedEdgeCandidate{{
114+
LookupKey: "Target", Fingerprint: "calls:source.go:Target:1", FilePath: "source.go", Kind: graph.EdgeKindCalls, Line: 1,
115+
}}); err != nil {
116+
t.Fatalf("UpsertUnresolvedEdges: %v", err)
117+
}
118+
}
119+
120+
if err := store.DeleteNodesByFile(ctxA, "source.go"); err != nil {
121+
t.Fatalf("DeleteNodesByFile: %v", err)
122+
}
123+
gotA, err := store.FindUnresolvedEdgesByLookupKeys(ctxA, []string{"Target"})
124+
if err != nil || len(gotA) != 0 {
125+
t.Fatalf("ns-a candidates = %+v, err=%v; want none", gotA, err)
126+
}
127+
gotB, err := store.FindUnresolvedEdgesByLookupKeys(ctxB, []string{"Target"})
128+
if err != nil || len(gotB) != 1 {
129+
t.Fatalf("ns-b candidates = %+v, err=%v; want one preserved", gotB, err)
130+
}
131+
}
132+
133+
func TestUnresolvedIndexReady_RequiresMatchingVersion(t *testing.T) {
134+
store := setupTestDB(t)
135+
ctx := requestctx.WithNamespace(context.Background(), "repo-a")
136+
137+
if err := store.MarkUnresolvedIndexReady(ctx, "index-v1"); err != nil {
138+
t.Fatalf("MarkUnresolvedIndexReady: %v", err)
139+
}
140+
ready, err := store.UnresolvedIndexReady(ctx, "index-v1")
141+
if err != nil || !ready {
142+
t.Fatalf("matching version ready = %v, err=%v; want true", ready, err)
143+
}
144+
ready, err = store.UnresolvedIndexReady(ctx, "index-v2")
145+
if err != nil || ready {
146+
t.Fatalf("mismatched version ready = %v, err=%v; want false", ready, err)
147+
}
148+
ready, err = store.UnresolvedIndexReady(requestctx.WithNamespace(context.Background(), "repo-b"), "index-v1")
149+
if err != nil || ready {
150+
t.Fatalf("other namespace ready = %v, err=%v; want false", ready, err)
151+
}
152+
}
153+
72154
func TestUpsertNodes_Insert(t *testing.T) {
73155
s := setupTestDB(t)
74156
ctx := context.Background()
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// @index GORM persistence for unresolved-edge reverse lookup and readiness state.
2+
package graphgorm
3+
4+
import (
5+
"context"
6+
"errors"
7+
8+
"gorm.io/gorm"
9+
"gorm.io/gorm/clause"
10+
11+
"github.com/tae2089/trace"
12+
13+
requestctx "github.com/tae2089/code-context-graph/internal/ctx"
14+
"github.com/tae2089/code-context-graph/internal/domain/graph"
15+
)
16+
17+
const unresolvedQueryChunkSize = 400
18+
19+
// UpsertUnresolvedEdges stores reverse-index rows without exposing them as traversable graph edges.
20+
// @intent retain unresolved syntax candidates until a future symbol addition can resolve them.
21+
// @sideEffect inserts unresolved_edge_candidates rows.
22+
func (s *Store) UpsertUnresolvedEdges(ctx context.Context, candidates []graph.UnresolvedEdgeCandidate) error {
23+
if len(candidates) == 0 {
24+
return nil
25+
}
26+
ns := requestctx.FromContext(ctx)
27+
for i := range candidates {
28+
candidates[i].Namespace = ns
29+
}
30+
if err := s.db.WithContext(ctx).Clauses(clause.OnConflict{
31+
Columns: []clause.Column{{Name: "namespace"}, {Name: "fingerprint"}},
32+
DoNothing: true,
33+
}).CreateInBatches(candidates, 500).Error; err != nil {
34+
return trace.Wrap(err, "upsert unresolved edges")
35+
}
36+
return nil
37+
}
38+
39+
// FindUnresolvedEdgesByLookupKeys finds deduplicated syntax edges matching newly added symbol keys.
40+
// @intent use the reverse index to identify affected unchanged source files.
41+
func (s *Store) FindUnresolvedEdgesByLookupKeys(ctx context.Context, keys []string) ([]graph.Edge, error) {
42+
if len(keys) == 0 {
43+
return nil, nil
44+
}
45+
ns := requestctx.FromContext(ctx)
46+
seen := make(map[string]struct{})
47+
var out []graph.Edge
48+
for start := 0; start < len(keys); start += unresolvedQueryChunkSize {
49+
end := min(start+unresolvedQueryChunkSize, len(keys))
50+
var rows []graph.UnresolvedEdgeCandidate
51+
if err := s.db.WithContext(ctx).
52+
Where("namespace = ? AND lookup_key IN ?", ns, keys[start:end]).
53+
Order("file_path ASC").Order("line ASC").Order("fingerprint ASC").
54+
Find(&rows).Error; err != nil {
55+
return nil, trace.Wrap(err, "find unresolved edges by lookup keys")
56+
}
57+
for _, row := range rows {
58+
if _, ok := seen[row.Fingerprint]; ok {
59+
continue
60+
}
61+
seen[row.Fingerprint] = struct{}{}
62+
out = append(out, row.Edge())
63+
}
64+
}
65+
return out, nil
66+
}
67+
68+
// FindUnresolvedEdgesByFiles loads every candidate edge owned by the selected affected files.
69+
// @intent replay import warmup and related edges together after reverse-index selection narrows source files.
70+
func (s *Store) FindUnresolvedEdgesByFiles(ctx context.Context, filePaths []string) ([]graph.Edge, error) {
71+
if len(filePaths) == 0 {
72+
return nil, nil
73+
}
74+
ns := requestctx.FromContext(ctx)
75+
seen := make(map[string]struct{})
76+
var out []graph.Edge
77+
for start := 0; start < len(filePaths); start += unresolvedQueryChunkSize {
78+
end := min(start+unresolvedQueryChunkSize, len(filePaths))
79+
var rows []graph.UnresolvedEdgeCandidate
80+
if err := s.db.WithContext(ctx).
81+
Where("namespace = ? AND file_path IN ?", ns, filePaths[start:end]).
82+
Order("file_path ASC").Order("line ASC").Order("fingerprint ASC").
83+
Find(&rows).Error; err != nil {
84+
return nil, trace.Wrap(err, "find unresolved edges by files")
85+
}
86+
for _, row := range rows {
87+
if _, ok := seen[row.Fingerprint]; ok {
88+
continue
89+
}
90+
seen[row.Fingerprint] = struct{}{}
91+
out = append(out, row.Edge())
92+
}
93+
}
94+
return out, nil
95+
}
96+
97+
// DeleteUnresolvedEdgesByFingerprints removes every lookup-key row for resolved syntax edges.
98+
// @intent keep the reverse index limited to relationships that still lack endpoints.
99+
func (s *Store) DeleteUnresolvedEdgesByFingerprints(ctx context.Context, fingerprints []string) error {
100+
if len(fingerprints) == 0 {
101+
return nil
102+
}
103+
ns := requestctx.FromContext(ctx)
104+
for start := 0; start < len(fingerprints); start += unresolvedQueryChunkSize {
105+
end := min(start+unresolvedQueryChunkSize, len(fingerprints))
106+
if err := s.db.WithContext(ctx).
107+
Where("namespace = ? AND fingerprint IN ?", ns, fingerprints[start:end]).
108+
Delete(&graph.UnresolvedEdgeCandidate{}).Error; err != nil {
109+
return trace.Wrap(err, "delete resolved unresolved edges")
110+
}
111+
}
112+
return nil
113+
}
114+
115+
// UnresolvedIndexReady reports whether a compatible full build populated the namespace's reverse index.
116+
// @intent gate semi-naive update on complete historical unresolved-edge coverage produced by the expected algorithm and parsers.
117+
func (s *Store) UnresolvedIndexReady(ctx context.Context, version string) (bool, error) {
118+
if version == "" {
119+
return false, nil
120+
}
121+
var state graph.UnresolvedIndexState
122+
ns := requestctx.FromContext(ctx)
123+
result := s.db.WithContext(ctx).Where("namespace = ? AND version = ?", ns, version).First(&state)
124+
if errors.Is(result.Error, gorm.ErrRecordNotFound) {
125+
return false, nil
126+
}
127+
if result.Error != nil {
128+
return false, trace.Wrap(result.Error, "load unresolved index state")
129+
}
130+
return true, nil
131+
}
132+
133+
// MarkUnresolvedIndexReady marks the current namespace and producer version after a successful full candidate pass.
134+
// @intent distinguish a compatible legitimately empty reverse index from stale or uninitialized state.
135+
// @sideEffect inserts or updates unresolved_index_states.
136+
func (s *Store) MarkUnresolvedIndexReady(ctx context.Context, version string) error {
137+
if version == "" {
138+
return trace.New("unresolved index version is empty")
139+
}
140+
state := graph.UnresolvedIndexState{Namespace: requestctx.FromContext(ctx), Version: version}
141+
if err := s.db.WithContext(ctx).Clauses(clause.OnConflict{
142+
Columns: []clause.Column{{Name: "namespace"}},
143+
DoUpdates: clause.AssignmentColumns([]string{"version", "updated_at"}),
144+
}).Create(&state).Error; err != nil {
145+
return trace.Wrap(err, "mark unresolved index ready")
146+
}
147+
return nil
148+
}

0 commit comments

Comments
 (0)