refactor: replace old progressStore with simple IndexStatus file

- Remove progressStore (preRegister, update, carry-over, flusher)
- Remove IndexingProgressSummary, BuildIndexingProgress, formatAge, buildIndexingMessage
- Remove auto-resume from SearchCode/HybridSearchCode (redundant with DetectContext)
- Remove resumeAttempts field from Engine
- Add IndexStatus/LangStatus/SaveIndexStatus/LoadIndexStatus in pkg/indexer/
- Indexer writes OnDisk/Changed/Processed via Progress callback
- Tools read status directly from .ragcode/index_status.json
- Fix TestDetectNoMarkers with AllowedRoots isolation

Author: razvan

cmd/rag-code-mcp/main.go

@@ -17,7 +17,7 @@ import (
 )
 
 var (
-	Version = "2.1.62"
+	Version = "2.1.63"
 	Commit  = "none"
 	Date    = "24.10.2025"
 )

internal/service/engine/engine.go

@@ -45,16 +45,13 @@ type Engine struct {
 	pendingFiles    map[string]map[string]struct{} // workspaceID -> set(filePath)
 	pendingOverflow map[string]bool                // workspaceID -> too many pending changes, fallback to full scan
 
-	progress *progressStore
+
 
 	// detectionCache stores resolved WorkspaceContext with TTL to avoid
 	// repeated full resolver cascades for the same path.
 	detectionCache sync.Map // map[string]*detectionCacheEntry
 
-	// resumeAttempts throttles auto-resume of interrupted indexing.
-	// Key: workspace ID, Value: time.Time of last resume attempt.
-	// Prevents CPU/log churn when indexing keeps failing (e.g. Ollama down).
-	resumeAttempts sync.Map
+
 
 	// connectTriggered tracks whether background indexing was automatically
 	// triggered for a workspace ID upon initial daemon resolution.
@@ -116,19 +113,16 @@ func NewEngine(idx *indexer.Service, srv *search.Service, registryPath string, c
 		resolver:        res,
 		config:          cfg,
 		watchers:        watcherMgr,
-		progress:        newProgressStore(),
+
 		pendingFiles:    make(map[string]map[string]struct{}),
 		pendingOverflow: make(map[string]bool),
 	}
 }
 
-// GetIndexProgress returns the last known indexing progress for a workspace.
-// workspaceRoot is used as a hint to load persisted status from disc if not in memory.
-func (e *Engine) GetIndexProgress(workspaceID, workspaceRoot string) *IndexProgress {
-	if e.progress == nil {
-		return nil
-	}
-	return e.progress.get(workspaceID, workspaceRoot)
+// GetIndexStatus returns the last known indexing status for a workspace.
+// Reads directly from {workspaceRoot}/.ragcode/index_status.json.
+func (e *Engine) GetIndexStatus(workspaceRoot string) *indexer.IndexStatus {
+	return indexer.LoadIndexStatus(workspaceRoot)
 }
 
 // ActiveIndexingJobs returns the IDs of workspaces currently being indexed.
@@ -413,22 +407,7 @@ func (e *Engine) SearchCode(ctx context.Context, filePath, queryText string, lim
 	primaryColl := wctx.CollectionName(primaryLang)
 	t1 := time.Now()
 
-	// Auto-resume: if GlobalPercent is between 1-99 and state is "running",
-	// the indexer was interrupted — trigger a background re-index.
-	if idxStatus := loadIndexStatus(wctx.Root); idxStatus != nil {
-		if idxStatus.WorkspaceID == wctx.ID && idxStatus.GlobalPercent > 0 && idxStatus.GlobalPercent < 100 && idxStatus.State == "running" {
-			if _, ok := e.indexingJobs.Load(wctx.ID); !ok {
-				const resumeCooldown = 5 * time.Minute
-				now := time.Now()
-				if last, loaded := e.resumeAttempts.Load(wctx.ID); !loaded || now.Sub(last.(time.Time)) > resumeCooldown {
-					e.resumeAttempts.Store(wctx.ID, now)
-					logger.Instance.Info("[IDX] ws=%s Indexing interrupted at %d%% — auto-resuming", filepath.Base(wctx.Root), idxStatus.GlobalPercent)
-					e.StartIndexingAsync(wctx.Root, wctx.ID, nil, false)
-				}
-			}
-			logger.Instance.Info("[IDX] ws=%s Indexing in progress (%d%%) — will search available collections", filepath.Base(wctx.Root), idxStatus.GlobalPercent)
-		}
-	}
+
 
 	// Check if the primary collection exists.
 	// If not, trigger background indexing but do NOT block — the fan-out below
@@ -622,21 +601,7 @@ func (e *Engine) HybridSearchCode(ctx context.Context, filePath, queryText strin
 
 	collection := wctx.CollectionName(lang)
 
-	// Auto-resume from index_status.json: if indexing was interrupted, resume it.
-	if idxStatus := loadIndexStatus(wctx.Root); idxStatus != nil {
-		if idxStatus.WorkspaceID == wctx.ID && idxStatus.GlobalPercent > 0 && idxStatus.GlobalPercent < 100 && idxStatus.State == "running" {
-			if _, ok := e.indexingJobs.Load(wctx.ID); !ok {
-				const resumeCooldown = 5 * time.Minute
-				now := time.Now()
-				if last, loaded := e.resumeAttempts.Load(wctx.ID); !loaded || now.Sub(last.(time.Time)) > resumeCooldown {
-					e.resumeAttempts.Store(wctx.ID, now)
-					logger.Instance.Info("[IDX] ws=%s Indexing interrupted at %d%% — auto-resuming", filepath.Base(wctx.Root), idxStatus.GlobalPercent)
-					e.StartIndexingAsync(wctx.Root, wctx.ID, nil, false)
-				}
-			}
-			logger.Instance.Info("[IDX] ws=%s Indexing in progress (%d%%) — will search available collections", filepath.Base(wctx.Root), idxStatus.GlobalPercent)
-		}
-	}
+
 
 	exists, err := e.search.CollectionExists(ctx, collection)
 	if err != nil {
@@ -822,10 +787,7 @@ func (e *Engine) StartIndexingAsync(root, id string, changedFiles []string, recr
 		logger.Instance.Warn("[IDX] ⚠️ %d workspaces indexing simultaneously — Ollama requests will serialize implicitly (ws=%s)", activeCount, filepath.Base(root))
 	}
 
-	jobID := fmt.Sprintf("%s-%d", id, time.Now().UnixNano())
-	if e.progress != nil {
-		e.progress.start(id, root, jobID, time.Now())
-	}
+	indexer.SaveIndexStatus(root, &indexer.IndexStatus{State: "starting", StartedAt: time.Now().UTC().Format(time.RFC3339)})
 
 	go func() {
 		defer func() {
@@ -847,13 +809,18 @@ func (e *Engine) StartIndexingAsync(root, id string, changedFiles []string, recr
 
 		if err != nil {
 			logger.Instance.Error("[IDX] ws=%s Background indexing failed: %v", filepath.Base(root), err)
-			if e.progress != nil {
-				e.progress.fail(id, root, time.Now(), err.Error())
+			if s := indexer.LoadIndexStatus(root); s != nil {
+				s.State = "failed"
+				s.Error = err.Error()
+				s.EndedAt = time.Now().UTC().Format(time.RFC3339)
+				indexer.SaveIndexStatus(root, s)
 			}
 		} else {
 			logger.Instance.Info("[IDX] ✅ ws=%s Background indexing completed", filepath.Base(root))
-			if e.progress != nil {
-				e.progress.complete(id, root, time.Now())
+			if s := indexer.LoadIndexStatus(root); s != nil {
+				s.State = "completed"
+				s.EndedAt = time.Now().UTC().Format(time.RFC3339)
+				indexer.SaveIndexStatus(root, s)
 			}
 		}
 	}()
@@ -916,39 +883,34 @@ func (e *Engine) IndexWorkspace(ctx context.Context, path string, recreate bool)
 
 	wsName := filepath.Base(wctx.Root)
 
-	// Pre-scan: count files for every language before starting indexing.
-	// This gives progressStore an accurate denominator from the start, so
-	// GlobalPercent increases monotonically instead of resetting per language.
 	var excludePatterns []string
 	if e.config != nil {
 		excludePatterns = e.config.Workspace.ExcludePatterns
 	}
-	// Pre-scan: single WalkDir counting files per language before indexing begins.
-	// This gives progressStore an accurate denominator from the start, so
-	// GlobalPercent increases monotonically. One combined walk avoids
-	// O(languages x files) traversals that per-language scans would incur.
-	if e.progress != nil {
-		fileCounts := e.indexer.CountAllFiles(wctx.Root, excludePatterns)
-		for _, lang := range languages {
-			e.progress.preRegister(wctx.ID, lang, fileCounts[lang], time.Now())
-		}
-	}
 
 	var indexErrors []string
 	for _, lang := range languages {
 		collection := wctx.CollectionName(lang)
-		progressCb := func(doneFiles, totalFiles int) {
-			if e.progress != nil {
-				e.progress.update(wctx.ID, lang, doneFiles, totalFiles, time.Now())
-			}
-		}
 		logger.Instance.Info("[IDX] ws=%s lang=%s ▶ starting", wsName, lang)
 		err := e.indexer.IndexWorkspace(ctx, wctx.Root, collection, indexer.Options{
 			Language:        lang,
 			WorkspaceName:   wsName,
 			ExcludePatterns: excludePatterns,
 			Recreate:        recreate,
-			Progress:        progressCb,
+			Progress: func(doneFiles, totalFiles int) {
+				if s := indexer.LoadIndexStatus(wctx.Root); s != nil {
+					s.State = "running"
+					if s.Languages == nil {
+						s.Languages = make(map[string]indexer.LangStatus)
+					}
+					ls := s.Languages[lang]
+					ls.OnDisk = totalFiles
+					ls.Changed = totalFiles
+					ls.Processed = doneFiles
+					s.Languages[lang] = ls
+					indexer.SaveIndexStatus(wctx.Root, s)
+				}
+			},
 		})
 		if err != nil {
 			logger.Instance.Error("[IDX] ws=%s lang=%s ❌ failed: %v", wsName, lang, err)

internal/service/engine/engine_fallback_search_test.go

@@ -95,9 +95,7 @@ func ValidateEmail(email string) bool {
 	eng.SetResolver(resolver.New(resolver.Dependencies{Detector: &mockDirDetector{root: root}}))
 
 	t.Cleanup(func() {
-		if eng.progress != nil {
-			eng.progress.stop()
-		}
+
 	})
 
 	return root, eng

internal/service/engine/engine_nonblocking_search_test.go

@@ -78,9 +78,7 @@ func TestSearchCodeReturnsResultsFromOtherLangsWhenPrimaryMissing(t *testing.T)
 
 	// Cleanup: stop progress flusher
 	t.Cleanup(func() {
-		if eng2.progress != nil {
-			eng2.progress.stop()
-		}
+
 	})
 }
 
@@ -129,9 +127,7 @@ func TestSearchCodeBlocksWhenZeroCollectionsExist(t *testing.T) {
 
 	// Cleanup
 	t.Cleanup(func() {
-		if eng2.progress != nil {
-			eng2.progress.stop()
-		}
+
 		// Wait for bg indexing to drain
 		for i := 0; i < 100; i++ {
 			if len(eng2.ActiveIndexingJobs()) == 0 {
@@ -183,9 +179,7 @@ func TestSearchCodeIndexingInProgressStillSearches(t *testing.T) {
 	// Cleanup the fake job
 	eng2.indexingJobs.Delete(wctx.ID)
 	t.Cleanup(func() {
-		if eng2.progress != nil {
-			eng2.progress.stop()
-		}
+
 	})
 }
 
@@ -234,9 +228,7 @@ func TestHybridSearchCodeReturnsNilWhenCollectionMissing(t *testing.T) {
 
 	// Cleanup
 	t.Cleanup(func() {
-		if eng2.progress != nil {
-			eng2.progress.stop()
-		}
+
 		// Wait for bg indexing to drain
 		for i := 0; i < 100; i++ {
 			if len(eng2.ActiveIndexingJobs()) == 0 {
@@ -289,9 +281,7 @@ func TestHybridSearchCodeStillWorksWhenCollectionExists(t *testing.T) {
 	}
 
 	t.Cleanup(func() {
-		if eng2.progress != nil {
-			eng2.progress.stop()
-		}
+
 	})
 }
 

internal/service/engine/engine_searchcode_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"sync/atomic"
 	"testing"
-	"time"
 
 	"github.com/doITmagic/rag-code-mcp/internal/config"
 	"github.com/doITmagic/rag-code-mcp/internal/service/search"
@@ -316,97 +315,4 @@ func (m *mockDirDetector) DetectFromFilePath(_ context.Context, path string) (*c
 	}, nil
 }
 
-// TestSearchCodeAutoResumesInterruptedIndexing verifies that SearchCode triggers
-// a background re-index when index_status.json shows an interrupted indexing
-// session (state="running", 1 <= GlobalPercent <= 99).
-// It also verifies the 5-minute cooldown prevents repeated resume triggers.
-func TestSearchCodeAutoResumesInterruptedIndexing(t *testing.T) {
-	llmProvider := &countingLLM{}
-	eng := newEngineCountingLLM(&testStore{existing: map[string]bool{}}, llmProvider)
-
-	wsRoot := t.TempDir()
-	eng.SetResolver(resolver.New(resolver.Dependencies{Detector: &mockDirDetector{root: wsRoot}}))
-
-	// Pre-fill the connect trigger so we test the search-based auto-resume,
-	// not the daemon's on-connect auto-warmup.
-	wsID := contract.DeriveWorkspaceID(wsRoot, "", "")
-	eng.connectTriggered.Store(wsID, true)
-
-	// Detect context to learn the workspace ID (needed for index_status.json)
-	wctx, err := eng.DetectContext(context.Background(), "dummy.go")
-	if err != nil || wctx == nil {
-		t.Fatal("Failed to detect workspace context")
-	}
-
-	// Write index_status.json that simulates an interrupted indexing session at 50%
-	now := time.Now()
-	status := IndexProgress{
-		JobID:         "interrupted-test-job",
-		WorkspaceID:   wctx.ID,
-		WorkspaceRoot: wsRoot,
-		State:         "running",
-		GlobalPercent: 50,
-		StartedAt:     now.Add(-10 * time.Minute),
-		UpdatedAt:     now,
-		Languages:     map[string]IndexLanguageProgress{},
-	}
-	saveIndexStatus(wsRoot, &status)
 
-	// Pre-emptively set connectTriggered so that DetectContext doesn't
-	// automatically kick off indexing directly when it is called entirely
-	// bypassing the auto-resume retry mechanism we're trying to test here.
-	eng.connectTriggered.Store(wctx.ID, true)
-
-	// Make the go collection exist so SearchCode gets past the fast-fail check
-	goColl := CollectionNameFor(wctx.ID, "go")
-	store := &multiLangStore{
-		testStore: testStore{
-			existing: map[string]bool{goColl: true},
-		},
-	}
-	eng.SetSearchService(search.NewService(llmProvider, store))
-
-	// First call — auto-resume should fire
-	_, _ = eng.SearchCode(context.Background(), "dummy.go", "test", 10, false)
-
-	// resumeAttempts is updated synchronously before the goroutine starts,
-	// so it is readable immediately after SearchCode returns.
-	if _, ok := eng.resumeAttempts.Load(wctx.ID); !ok {
-		t.Error("expected auto-resume to be triggered: resumeAttempts has no entry for workspace")
-	}
-
-	// Second call within cooldown — should NOT trigger another resume attempt.
-	// We verify this by checking that resumeAttempts still holds the same timestamp.
-	first, okFirst := eng.resumeAttempts.Load(wctx.ID)
-	_, _ = eng.SearchCode(context.Background(), "dummy.go", "test", 10, false)
-	second, okSecond := eng.resumeAttempts.Load(wctx.ID)
-	
-	if okFirst && okSecond {
-		if first.(time.Time) != second.(time.Time) {
-			t.Error("cooldown violated: auto-resume was triggered again within the 5-minute window")
-		}
-	} else {
-		t.Error("expected resumeAttempts to be populated for both calls")
-	}
-
-	// Stop the progress flusher goroutine before the test returns.
-	// Without this the goroutine can still write to wsRoot/.ragcode/ while
-	// t.TempDir() cleanup removes the directory, triggering a race under -race.
-	t.Cleanup(func() {
-		if eng.progress != nil {
-			eng.progress.stop()
-		}
-	})
-
-	// Wait for the background indexing goroutine to drain before test returns.
-	deadline := time.Now().Add(5 * time.Second)
-	for time.Now().Before(deadline) {
-		if len(eng.ActiveIndexingJobs()) == 0 {
-			break
-		}
-		time.Sleep(10 * time.Millisecond)
-	}
-	if jobs := eng.ActiveIndexingJobs(); len(jobs) != 0 {
-		t.Fatalf("background indexing jobs still active after 5s wait: %v", jobs)
-	}
-}

internal/service/engine/engine_sticky_test.go

@@ -88,9 +88,7 @@ func TestCheckAndReindexOnConnect_TriggersReindex(t *testing.T) {
 
 	// Register cleanup immediately — before any Fatal that could skip it.
 	t.Cleanup(func() {
-		if eng.progress != nil {
-			eng.progress.stop()
-		}
+
 		deadline := time.Now().Add(5 * time.Second)
 		for time.Now().Before(deadline) {
 			if len(eng.ActiveIndexingJobs()) == 0 {