Split indexer.go god file into cohesive service files (#2)

indexer.go was 2,456 lines mixing the build pipeline, update pipeline,
spool, package discovery, search refresh, graph-state loading, and file
I/O. Move each concern into its own file within the same package:
build.go, update.go, spool.go, packages.go, search.go, graphstate.go,
fileio.go. indexer.go keeps the GraphService type, interfaces, options,
and shared helpers (2,456 -> 160 lines).

Pure structural change: no declaration added or removed (114 top-level
decls before and after), imports pruned per file, full suite green.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: tae2089

internal/service/build.go

@@ -0,0 +1,137 @@
+// @index Filesystem walking, per-file parsing, and unreadable-file accounting.
+package service
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+
+	"github.com/tae2089/code-context-graph/internal/model"
+	"github.com/tae2089/code-context-graph/internal/parse"
+	"github.com/tae2089/code-context-graph/internal/parse/treesitter"
+	"github.com/tae2089/code-context-graph/internal/pathutil"
+)
+
+// @intent walk candidate source files once while applying recursion, exclude, and include-path policy before parsing.
+func walkMatchingFiles(ctx context.Context, absDir string, opts BuildOptions, fn func(path, relPath string) error) error {
+	return filepath.Walk(absDir, func(path string, info os.FileInfo, err error) error {
+		if ctxErr := ctx.Err(); ctxErr != nil {
+			return ctxErr
+		}
+		if err != nil {
+			return err
+		}
+
+		relPath, _ := filepath.Rel(absDir, path)
+
+		if info.IsDir() {
+			if path != absDir && opts.NoRecursive {
+				return filepath.SkipDir
+			}
+			if pathutil.ShouldSkipDir(info.Name()) || pathutil.MatchExcludes(opts.ExcludePatterns, relPath) {
+				return filepath.SkipDir
+			}
+			if len(opts.IncludePaths) > 0 && path != absDir && !pathutil.MatchIncludePaths(relPath, opts.IncludePaths) {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		if pathutil.MatchExcludes(opts.ExcludePatterns, relPath) {
+			return nil
+		}
+		if len(opts.IncludePaths) > 0 && !pathutil.MatchIncludePaths(relPath, opts.IncludePaths) {
+			return nil
+		}
+		return fn(path, relPath)
+	})
+}
+
+// parseForBuild parses one source file using the comment-aware parser when available.
+// @intent surface comment blocks and language alongside nodes/edges so the binder can attach annotations.
+func parseForBuild(ctx context.Context, parser Parser, relPath string, content []byte) ([]model.Node, []model.Edge, []treesitter.CommentBlock, treesitter.ParseMetadata, string, error) {
+	if mp, ok := parser.(metadataParserWithLanguage); ok {
+		nodes, edges, comments, meta, err := mp.ParseWithCommentsAndMetadata(ctx, relPath, content)
+		return nodes, edges, comments, meta, mp.Language(), err
+	}
+	if cp, ok := parser.(commentParserWithLanguage); ok {
+		nodes, edges, comments, err := cp.ParseWithComments(ctx, relPath, content)
+		return nodes, edges, comments, treesitter.ParseMetadata{}, cp.Language(), err
+	}
+	nodes, edges, err := parser.ParseWithContext(ctx, relPath, content)
+	return nodes, edges, nil, treesitter.ParseMetadata{}, "", err
+}
+
+// unreadableFileSummary aggregates files that could not be stat-ed or read during a build or update pass.
+// @intent let callers surface a single structured failure or warning instead of one log entry per file.
+type unreadableFileSummary struct {
+	count  int
+	sample string
+	files  []string
+}
+
+// add records one more unreadable file, keeping the first occurrence as the sample.
+// @intent collect every offending path while keeping summary output bounded for logs.
+// @mutates s.count, s.sample, s.files
+func (s *unreadableFileSummary) add(relPath string) {
+	s.count++
+	if s.sample == "" {
+		s.sample = relPath
+	}
+	s.files = append(s.files, relPath)
+}
+
+// log emits a single warning describing how many files were skipped during a phase.
+// @intent prevent log spam by collapsing per-file warnings into one phase-tagged entry.
+// @sideEffect writes a warn-level log entry when the summary is non-empty.
+func (s unreadableFileSummary) log(logger *slog.Logger, phase string) {
+	if s.count == 0 || logger == nil {
+		return
+	}
+	logger.Warn("skipped unreadable files", "phase", phase, "count", s.count, "sample", s.sample)
+}
+
+// asError converts the summary into an UnreadableFilesError when at least one file failed.
+// @intent let callers escalate skipped reads into a structured failure when FailOnUnreadable is set.
+func (s unreadableFileSummary) asError() error {
+	if s.count == 0 {
+		return nil
+	}
+	files := append([]string(nil), s.files...)
+	return &UnreadableFilesError{Files: files}
+}
+
+// @intent reject individual files that exceed the configured per-file parse budget before loading them into memory.
+func CheckParseFileSize(relPath string, size int64, maxFileBytes int64) error {
+	if maxFileBytes > 0 && size > maxFileBytes {
+		return fmt.Errorf("parse file %s exceeds max file bytes: %d > %d", relPath, size, maxFileBytes)
+	}
+	return nil
+}
+
+// @intent stop one build or update pass once cumulative parsed bytes would exceed the configured safety limit.
+func CheckTotalParsedBytes(relPath string, current int64, next int64, maxTotalBytes int64) error {
+	if maxTotalBytes > 0 && current+next > maxTotalBytes {
+		return fmt.Errorf("parse file %s exceeds max total parsed bytes: %d > %d", relPath, current+next, maxTotalBytes)
+	}
+	return nil
+}
+
+// toBinderComments converts walker comment blocks into binder comment blocks,
+// preserving docstring bookkeeping required by the Python docstring binding path.
+// @intent keep IsDocstring and OwnerStartLine in sync between walker and binder types
+func toBinderComments(tsComments []treesitter.CommentBlock) []parse.CommentBlock {
+	out := make([]parse.CommentBlock, len(tsComments))
+	for i, c := range tsComments {
+		out[i] = parse.CommentBlock{
+			StartLine:      c.StartLine,
+			EndLine:        c.EndLine,
+			Text:           c.Text,
+			IsDocstring:    c.IsDocstring,
+			OwnerStartLine: c.OwnerStartLine,
+		}
+	}
+	return out
+}

internal/service/fileio.go

@@ -0,0 +1,182 @@
+// @index Existing-graph file-state loading and change/force detection.
+package service
+
+import (
+	"context"
+
+	"gorm.io/gorm"
+
+	"github.com/tae2089/code-context-graph/internal/analysis/incremental"
+	"github.com/tae2089/code-context-graph/internal/ctxns"
+	"github.com/tae2089/code-context-graph/internal/model"
+	"github.com/tae2089/code-context-graph/internal/pathutil"
+)
+
+// ExistingGraphFiles returns namespace-scoped graph file paths currently stored.
+// @intent share deletion-scope discovery across CLI and MCP incremental updates
+func ExistingGraphFiles(ctx context.Context, db *gorm.DB) ([]string, error) {
+	filePaths, _, err := existingGraphFileState(ctx, db)
+	return filePaths, err
+}
+
+// existingGraphFileState loads the namespace-scoped current graph state grouped by file path.
+// @intent provide both deletion-scope file paths and per-file node projections from a single query.
+func existingGraphFileState(ctx context.Context, db *gorm.DB) ([]string, map[string][]model.Node, error) {
+	if db == nil {
+		return nil, map[string][]model.Node{}, nil
+	}
+
+	ns := ctxns.FromContext(ctx)
+	var nodes []graphFileNodeState
+	if err := db.WithContext(ctx).
+		Model(&model.Node{}).
+		Select("id", "file_path", "hash").
+		Where("namespace = ? AND kind <> ?", ns, model.NodeKindPackage).
+		Find(&nodes).Error; err != nil {
+		return nil, nil, err
+	}
+	nodesByFile := make(map[string][]model.Node)
+	fileSeen := make(map[string]struct{})
+	filePaths := make([]string, 0)
+	for _, node := range nodes {
+		minimalNode := model.Node{ID: node.ID, FilePath: node.FilePath, Hash: node.Hash}
+		nodesByFile[node.FilePath] = append(nodesByFile[node.FilePath], minimalNode)
+		if _, ok := fileSeen[node.FilePath]; !ok {
+			fileSeen[node.FilePath] = struct{}{}
+			filePaths = append(filePaths, node.FilePath)
+		}
+	}
+	return filePaths, nodesByFile, nil
+}
+
+// filterExistingStateByInclude restricts existing graph state to file paths that match the include filter.
+// @intent prevent partial-scope updates from deleting files that live outside the requested include paths.
+func filterExistingStateByInclude(filePaths []string, nodesByFile map[string][]model.Node, includePaths []string) ([]string, map[string][]model.Node) {
+	filteredFiles := make([]string, 0, len(filePaths))
+	filteredNodes := make(map[string][]model.Node)
+	for _, fp := range filePaths {
+		if !pathutil.MatchIncludePaths(fp, includePaths) {
+			continue
+		}
+		filteredFiles = append(filteredFiles, fp)
+		filteredNodes[fp] = nodesByFile[fp]
+	}
+	return filteredFiles, filteredNodes
+}
+
+// forceReparseFiles finds files whose edges reference nodes from changed files and therefore must be reparsed.
+// @intent keep cross-file edges consistent by reparsing edge-source files when their referenced nodes change.
+func forceReparseFiles(ctx context.Context, db *gorm.DB, existingNodesByFile map[string][]model.Node, currentHashes map[string]string) (map[string]struct{}, error) {
+	forceFiles := make(map[string]struct{})
+	if db == nil || len(existingNodesByFile) == 0 || len(currentHashes) == 0 {
+		return forceFiles, nil
+	}
+	if !db.Migrator().HasTable(&model.Edge{}) {
+		return forceFiles, nil
+	}
+
+	var changedNodeIDs []uint
+	for filePath, nodes := range existingNodesByFile {
+		if len(nodes) == 0 {
+			continue
+		}
+		currentHash, stillPresent := currentHashes[filePath]
+		if !stillPresent || nodes[0].Hash != currentHash {
+			for _, node := range nodes {
+				changedNodeIDs = append(changedNodeIDs, node.ID)
+			}
+		}
+	}
+	if len(changedNodeIDs) == 0 {
+		return forceFiles, nil
+	}
+
+	ns := ctxns.FromContext(ctx)
+	edgeFileSeen := make(map[string]struct{})
+	for start := 0; start < len(changedNodeIDs); start += forceReparseEdgeChunkSize {
+		end := min(start+forceReparseEdgeChunkSize, len(changedNodeIDs))
+		chunk := changedNodeIDs[start:end]
+		var chunkFiles []string
+		if err := db.WithContext(ctx).
+			Model(&model.Edge{}).
+			Where("namespace = ? AND file_path <> '' AND (from_node_id IN ? OR to_node_id IN ?)", ns, chunk, chunk).
+			Distinct().
+			Pluck("file_path", &chunkFiles).Error; err != nil {
+			return nil, err
+		}
+		for _, filePath := range chunkFiles {
+			edgeFileSeen[filePath] = struct{}{}
+		}
+
+		var relatedImplements []model.Edge
+		if err := db.WithContext(ctx).
+			Model(&model.Edge{}).
+			Where("namespace = ? AND kind = ? AND file_path <> '' AND (from_node_id IN ? OR to_node_id IN ?)", ns, model.EdgeKindImplements, chunk, chunk).
+			Find(&relatedImplements).Error; err != nil {
+			return nil, err
+		}
+		var relatedTypeIDs []uint
+		seenTypeID := make(map[uint]struct{}, len(relatedImplements)*2)
+		for _, edge := range relatedImplements {
+			for _, id := range []uint{edge.FromNodeID, edge.ToNodeID} {
+				if id == 0 {
+					continue
+				}
+				if _, ok := seenTypeID[id]; ok {
+					continue
+				}
+				seenTypeID[id] = struct{}{}
+				relatedTypeIDs = append(relatedTypeIDs, id)
+			}
+		}
+		for relatedStart := 0; relatedStart < len(relatedTypeIDs); relatedStart += forceReparseEdgeChunkSize {
+			relatedEnd := min(relatedStart+forceReparseEdgeChunkSize, len(relatedTypeIDs))
+			relatedChunk := relatedTypeIDs[relatedStart:relatedEnd]
+			var dispatchFiles []string
+			if err := db.WithContext(ctx).
+				Model(&model.Edge{}).
+				Where("namespace = ? AND kind IN ? AND file_path <> '' AND (from_node_id IN ? OR to_node_id IN ?)", ns, model.CallEdgeKinds(), relatedChunk, relatedChunk).
+				Distinct().
+				Pluck("file_path", &dispatchFiles).Error; err != nil {
+				return nil, err
+			}
+			for _, filePath := range dispatchFiles {
+				edgeFileSeen[filePath] = struct{}{}
+			}
+		}
+	}
+	for filePath := range edgeFileSeen {
+		currentHash, stillPresent := currentHashes[filePath]
+		if !stillPresent {
+			continue
+		}
+		nodes := existingNodesByFile[filePath]
+		if len(nodes) == 0 || nodes[0].Hash != currentHash {
+			continue
+		}
+		forceFiles[filePath] = struct{}{}
+	}
+	return forceFiles, nil
+}
+
+// splitForcedFiles partitions inputs into normal and forced-reparse buckets and marks the forced ones.
+// @intent process unchanged-hash forced files separately so the syncer can bypass its hash short-circuit.
+func splitForcedFiles(files map[string]incremental.FileInfo, forceFiles map[string]struct{}) (map[string]incremental.FileInfo, map[string]incremental.FileInfo) {
+	if len(files) == 0 {
+		return nil, nil
+	}
+	if len(forceFiles) == 0 {
+		return files, nil
+	}
+	normal := make(map[string]incremental.FileInfo, len(files))
+	forced := make(map[string]incremental.FileInfo)
+	for filePath, info := range files {
+		if _, ok := forceFiles[filePath]; ok {
+			info.Force = true
+			forced[filePath] = info
+			continue
+		}
+		normal[filePath] = info
+	}
+	return normal, forced
+}