Merge remote-tracking branch 'origin/refactor/indexing-progress' into refactor/indexing-progress

Author: doITmagic

.gitignore

@@ -54,6 +54,9 @@ Thumbs.db
 .env
 .env.local
 
+# Local project config (not for VCS)
+.trello.json
+
 # Temporary files
 tmp/
 temp/

BUGS.md

@@ -155,7 +155,7 @@ In reality, the index may be completely stale — all 232 files could be unindex
 
 ## BUG-003: Top-level Go functions with no AST relations are missing from the vector index
 
-**Status:** Open  
+**Status:** ✅ Fixed (2026-03-10, PR #40)  
 **Date confirmed:** 2026-03-09  
 **Affected component:** Go parser / indexer (`pkg/indexer`, `internal/parser`)  
 **Severity:** Medium — `rag_list_package_exports` and `rag_search` silently omit exported constructor/loader functions

SUGGESTIONS.md

@@ -1,7 +1,16 @@
 # Suggestions
 
-## Incremental indexing resets status to "starting"
+## Incremental indexing resets status to "starting" — ⚠️ Partially addressed
 
-Când se re-indexează incremental un singur fișier, `StartIndexingAsync` suprascrie statusul la `state: "starting"` cu totul de la zero, ștergând informația că 99% din index e deja acolo și funcțional. AI-ul vede `"starting"` + `"processed": 0` și crede că nu are date.
+When a single file is re-indexed incrementally, `StartIndexingAsync` previously
+overwrote the status file with a brand-new object, discarding `Languages` data.
 
-Fix-ul corect ar fi: la indexare incrementală, nu reseta starea la `"starting"` — folosește ceva gen `"updating"` sau păstrează `"completed"` cu un sub-status. Dar asta e un issue separat, nu din PR review-ul curent.
+**Current state (PR #40):**
+- The `State` field is now hidden from external JSON output (`json:"-"`), so AI
+  consumers no longer see `"starting"` / `"completed"` strings.
+- The `Languages` map is now preserved during incremental re-indexing (not wiped).
+
+**Remaining open item:** during incremental re-indexing, `processed` counters reset
+to whatever the incremental run reports. The overall `Languages` snapshot from the
+last full indexing run is kept, but live progress during the incremental pass may
+temporarily show lower counts.

cmd/rag-code-mcp/main.go

@@ -17,7 +17,7 @@ import (
 )
 
 var (
-	Version = "2.1.65"
+	Version = "2.1.67"
 	Commit  = "none"
 	Date    = "24.10.2025"
 )

internal/daemon/run.go

@@ -25,6 +25,7 @@ import (
 	_ "github.com/doITmagic/rag-code-mcp/pkg/parser/docs"
 	_ "github.com/doITmagic/rag-code-mcp/pkg/parser/go"
 	_ "github.com/doITmagic/rag-code-mcp/pkg/parser/html"
+	_ "github.com/doITmagic/rag-code-mcp/pkg/parser/javascript"
 	_ "github.com/doITmagic/rag-code-mcp/pkg/parser/php"
 	_ "github.com/doITmagic/rag-code-mcp/pkg/parser/python"
 	"github.com/doITmagic/rag-code-mcp/pkg/storage"

internal/service/engine/engine.go

@@ -802,8 +802,8 @@ func (e *Engine) popPendingIndex(workspaceID string) (files []string, overflow b
 func (e *Engine) tryStartPendingIndex(root, workspaceID string) {
 	files, overflow := e.popPendingIndex(workspaceID)
 	if overflow {
-		logger.Instance.Info("[IDX] ♻️ Pending changes exceeded limit for ws=%s — triggering full scan", filepath.Base(root))
-		e.StartIndexingAsync(root, workspaceID, nil, false)
+		logger.Instance.Info("[IDX] ♻️ Pending changes exceeded limit for ws=%s — triggering full re-index", filepath.Base(root))
+		e.StartIndexingAsync(root, workspaceID, nil, true)
 		return
 	}
 	if len(files) == 0 {
@@ -817,6 +817,11 @@ func (e *Engine) tryStartPendingIndex(root, workspaceID string) {
 // If recreate=true and a job is already running, the recreate is queued and
 // will start immediately after the current job finishes.
 func (e *Engine) StartIndexingAsync(root, id string, changedFiles []string, recreate bool) {
+	if watch.IsInvalidRoot(root) {
+		logger.Instance.Error("[IDX] ⛔ Refusing to index invalid/dangerous root: %s", root)
+		return
+	}
+
 	if _, loaded := e.indexingJobs.LoadOrStore(id, time.Now()); loaded {
 		// A job is already running. If recreate=true, queue it so it fires
 		// after the current job finishes (via tryStartPendingIndex/defer).
@@ -838,7 +843,17 @@ func (e *Engine) StartIndexingAsync(root, id string, changedFiles []string, recr
 		logger.Instance.Warn("[IDX] ⚠️ %d workspaces indexing simultaneously — Ollama requests will serialize implicitly (ws=%s)", activeCount, filepath.Base(root))
 	}
 
-	indexer.SaveIndexStatus(root, &indexer.IndexStatus{StartedAt: time.Now().UTC().Format(time.RFC3339)})
+	// Preserve existing Languages data on restart so incremental indexing
+	// doesn't show processed=0 to AI consumers. Reset only lifecycle fields.
+	s := indexer.LoadIndexStatus(root)
+	if s == nil {
+		s = &indexer.IndexStatus{}
+	}
+	s.StartedAt = time.Now().UTC().Format(time.RFC3339)
+	s.EndedAt = ""
+	s.Elapsed = ""
+	s.Error = ""
+	indexer.SaveIndexStatus(root, s)
 
 	go func() {
 		defer func() {
@@ -858,33 +873,33 @@ func (e *Engine) StartIndexingAsync(root, id string, changedFiles []string, recr
 			err = e.IndexWorkspace(ctx, root, recreate)
 		}
 
+		finalizeIndexStatus(root, err)
 		if err != nil {
 			logger.Instance.Error("[IDX] ws=%s Background indexing failed: %v", filepath.Base(root), err)
-			s := indexer.LoadIndexStatus(root)
-			if s == nil {
-				s = &indexer.IndexStatus{}
-			}
-			s.Error = err.Error()
-			s.EndedAt = time.Now().UTC().Format(time.RFC3339)
-			if started, pErr := time.Parse(time.RFC3339, s.StartedAt); pErr == nil {
-				s.Elapsed = time.Since(started).Round(time.Second).String()
-			}
-			indexer.SaveIndexStatus(root, s)
 		} else {
 			logger.Instance.Info("[IDX] ✅ ws=%s Background indexing completed", filepath.Base(root))
-			s := indexer.LoadIndexStatus(root)
-			if s == nil {
-				s = &indexer.IndexStatus{}
-			}
-			s.EndedAt = time.Now().UTC().Format(time.RFC3339)
-			if started, pErr := time.Parse(time.RFC3339, s.StartedAt); pErr == nil {
-				s.Elapsed = time.Since(started).Round(time.Second).String()
-			}
-			indexer.SaveIndexStatus(root, s)
 		}
 	}()
 }
 
+// finalizeIndexStatus loads the current status, stamps EndedAt/Elapsed/Error,
+// and saves it atomically. Centralises the logic that was duplicated in the
+// success and error branches of StartIndexingAsync.
+func finalizeIndexStatus(root string, indexErr error) {
+	s := indexer.LoadIndexStatus(root)
+	if s == nil {
+		s = &indexer.IndexStatus{}
+	}
+	s.EndedAt = time.Now().UTC().Format(time.RFC3339)
+	if started, pErr := time.Parse(time.RFC3339, s.StartedAt); pErr == nil {
+		s.Elapsed = time.Since(started).Round(time.Second).String()
+	}
+	if indexErr != nil {
+		s.Error = indexErr.Error()
+	}
+	indexer.SaveIndexStatus(root, s)
+}
+
 // IndexFiles indexes specific files in a workspace.
 func (e *Engine) IndexFiles(ctx context.Context, root string, files []string) error {
 	wctx, err := e.DetectContext(ctx, root)
@@ -953,27 +968,53 @@ func (e *Engine) IndexWorkspace(ctx context.Context, path string, recreate bool)
 	fileCounts := e.indexer.CountAllFiles(wctx.Root, excludePatterns)
 	logger.Instance.Info("[IDX] ws=%s file counts: %v", wsName, fileCounts)
 
-	// Pre-populate index_status.json with the real disk totals so that
-	// even languages with 0 changed files still show correct on_disk counts.
-	{
-		s := indexer.LoadIndexStatus(wctx.Root)
-		if s == nil {
-			s = &indexer.IndexStatus{StartedAt: time.Now().UTC().Format(time.RFC3339)}
-		}
-		if s.Languages == nil {
-			s.Languages = make(map[string]indexer.LangStatus)
-		}
-		for _, lang := range languages {
-			s.Languages[lang] = indexer.LangStatus{OnDisk: fileCounts[lang]}
-		}
-		indexer.SaveIndexStatus(wctx.Root, s)
+	// Sort languages by file count descending so the dominant language is indexed
+	// first and AI search works immediately for the most relevant code.
+	sort.Slice(languages, func(i, j int) bool {
+		return fileCounts[languages[i]] > fileCounts[languages[j]]
+	})
+	logger.Instance.Info("[IDX] ws=%s indexing order: %v", wsName, languages)
+
+	// Load or create a shared in-memory IndexStatus for the entire indexing run.
+	// This avoids calling LoadIndexStatus (JSON read + parse) on every Progress tick.
+	// The single *IndexStatus is updated in-place; only SaveIndexStatus (atomic write)
+	// hits the disk, and only every 10 files.
+	s := indexer.LoadIndexStatus(wctx.Root)
+	if s == nil {
+		s = &indexer.IndexStatus{StartedAt: time.Now().UTC().Format(time.RFC3339)}
+	}
+	if s.Languages == nil {
+		s.Languages = make(map[string]indexer.LangStatus)
 	}
+	// Pre-populate real on_disk counts so languages with 0 changed files still appear.
+	for _, l := range languages {
+		entry := s.Languages[l]
+		entry.OnDisk = fileCounts[l]
+		s.Languages[l] = entry
+	}
+	indexer.SaveIndexStatus(wctx.Root, s)
 
 	var indexErrors []string
 	for _, lang := range languages {
 		diskTotal := fileCounts[lang]
 		collection := wctx.CollectionName(lang)
 		logger.Instance.Info("[IDX] ws=%s lang=%s ▶ starting (on_disk=%d)", wsName, lang, diskTotal)
+
+		// Capture the already-processed count before this run starts.
+		// For incremental runs (only changed files), we accumulate on top of
+		// whatever was already indexed in Qdrant. For a full re-index
+		// (recreate=true, or all files changed), we reset to 0.
+		// baseProcessed is captured once per language, before the Progress
+		// callback fires, so it's safe to close over it.
+		baseProcessed := s.Languages[lang].Processed
+		if recreate {
+			baseProcessed = 0
+		}
+		// firstTick is used to detect on the first Progress callback whether
+		// this is a full re-index (totalFiles >= diskTotal) so we can reset
+		// baseProcessed to 0 and avoid double-counting.
+		firstTick := true
+
 		err := e.indexer.IndexWorkspace(ctx, wctx.Root, collection, indexer.Options{
 			Language:        lang,
 			WorkspaceName:   wsName,
@@ -984,18 +1025,23 @@ func (e *Engine) IndexWorkspace(ctx context.Context, path string, recreate bool)
 				if doneFiles%10 != 0 && doneFiles != totalFiles {
 					return
 				}
-				if s := indexer.LoadIndexStatus(wctx.Root); s != nil {
-
-					if s.Languages == nil {
-						s.Languages = make(map[string]indexer.LangStatus)
+				// On the first tick, decide if this is a full re-index.
+				// If totalFiles covers all on-disk files, reset base to 0
+				// so we don't double-count the existing Processed value.
+				if firstTick {
+					firstTick = false
+					if diskTotal > 0 && totalFiles >= diskTotal {
+						baseProcessed = 0
 					}
-					ls := s.Languages[lang]
-					ls.OnDisk = diskTotal   // real total files on disk for this language
-					ls.Changed = totalFiles // files that needed re-indexing (changedFiles)
-					ls.Processed = doneFiles
-					s.Languages[lang] = ls
-					indexer.SaveIndexStatus(wctx.Root, s)
 				}
+				ls := s.Languages[lang]
+				ls.OnDisk = diskTotal   // real total files on disk
+				ls.Changed = totalFiles // files that needed re-indexing this run
+				// Cumulative total: for incremental runs add to the existing
+				// DB count; for full re-indexes (base=0) start from scratch.
+				ls.Processed = baseProcessed + doneFiles
+				s.Languages[lang] = ls
+				indexer.SaveIndexStatus(wctx.Root, s)
 			},
 		})
 		if err != nil {

internal/service/engine/engine_fallback_search_test.go

@@ -94,9 +94,7 @@ func ValidateEmail(email string) bool {
 	eng := NewEngine(idxSvc, searchSvc, "", &config.Config{})
 	eng.SetResolver(resolver.New(resolver.Dependencies{Detector: &mockDirDetector{root: root}}))
 
-	t.Cleanup(func() {
 
-	})
 
 	return root, eng
 }

internal/service/tools/evaluate_ragcode.go

@@ -126,17 +126,11 @@ func (t *EvaluateRagCodeTool) Execute(ctx context.Context, args map[string]inter
 		}
 	}
 
-
-
 	response := ToolResponse{
 		Status:  "success",
 		Message: b.String(),
 		Data:    data,
-		Context: ContextMetadata{
-			WorkspaceRoot:    workspaceRoot,
-			DetectionSource:  source,
-			IndexingStatus: nil,
-		},
+		Context: ContextFromWorkspaceWithStatus(wctx, t.engine),
 	}
 
 	return response.JSON()

internal/service/tools/list_package_exports.go

@@ -255,9 +255,10 @@ func (t *ListPackageExportsTool) Execute(ctx context.Context, args map[string]in
 		Message: "Found package exports\n\n" + response.String(),
 		Data:    exports,
 		Context: ContextMetadata{
-			WorkspaceRoot:    wctx.Root,
-			DetectionSource:  wctx.DetectionSource,
-			Telemetry:        telemetry.CalculateSavings(baselineBytes, actualBytes),
+			WorkspaceRoot:   wctx.Root,
+			DetectionSource: wctx.DetectionSource,
+			Telemetry:       telemetry.CalculateSavings(baselineBytes, actualBytes),
+			IndexingStatus:  t.engine.GetIndexStatus(wctx.Root),
 		},
 	}
 

internal/service/tools/read_file_context.go

@@ -311,11 +311,7 @@ func (t *ReadFileContextTool) buildResponse(wctx *engine.WorkspaceContext, res C
 		Message: fmt.Sprintf("Extracted %s context for lines %d-%d from %s", res.ContextType, res.StartLine, res.EndLine, res.FilePath),
 	}
 	if wctx != nil {
-		resp.Context = ContextMetadata{
-			WorkspaceRoot:    wctx.Root,
-			DetectionSource:  wctx.DetectionSource,
-			IndexingStatus: nil,
-		}
+		resp.Context = ContextFromWorkspaceWithStatus(wctx, t.engine)
 	}
 
 	baselineBytes := int64(0)