fix(index): paginate per-profile rebuild/clear beyond search-page caps

GetToolsByServer capped a single Bleve search at 10000 hits and DeleteAll
at 100000, so a server with >10k indexed tools or an index with >100k docs
would silently drop the overflow during a per-profile rebuild/clear,
leaving the profile index inconsistent with the shared index.

Both now loop over search pages until exhausted, so coverage is never
bounded by a single page:
- GetToolsByServer paginates with From/Size, ending on a short final page.
- DeleteAll deletes page-by-page from offset 0 (each batch removes the
  enumerated docs, surfacing the next page) until the index is empty.

Page size is a configurable BleveIndex field (default 10000) so the
regression tests exercise the loop with a small, fast corpus.

Related #756. Follow-up to MCP-3240 Codex review (MCP-3307).

Author: Dumbris

internal/index/bleve.go

@@ -14,10 +14,19 @@ import (
 	"github.com/smart-mcp-proxy/mcpproxy-go/internal/config"
 )
 
+// defaultSearchPageSize is the page size used when paginating full-coverage
+// scans (GetToolsByServer, DeleteAll). It is a generous upper bound on the
+// number of docs in a single page; pagination loops over as many pages as
+// needed, so total coverage is never bounded by this value (MCP-3319).
+const defaultSearchPageSize = 10000
+
 // BleveIndex wraps Bleve index operations
 type BleveIndex struct {
 	index  bleve.Index
 	logger *zap.Logger
+	// searchPageSize bounds a single search page during paginated full scans.
+	// Defaults to defaultSearchPageSize; overridable in tests.
+	searchPageSize int
 }
 
 // ToolDocument represents a tool document in the index
@@ -61,8 +70,9 @@ func newBleveIndexAt(indexPath string, logger *zap.Logger) (*BleveIndex, error)
 	}
 
 	return &BleveIndex{
-		index:  index,
-		logger: logger,
+		index:          index,
+		logger:         logger,
+		searchPageSize: defaultSearchPageSize,
 	}, nil
 }
 
@@ -304,21 +314,29 @@ func (b *BleveIndex) GetDocumentCount() (uint64, error) {
 // its on-disk directory.
 func (b *BleveIndex) DeleteAll() error {
 	query := bleve.NewMatchAllQuery()
-	searchReq := bleve.NewSearchRequest(query)
-	searchReq.Size = 100000 // generous upper bound on indexed tools
-	searchResult, err := b.index.Search(searchReq)
-	if err != nil {
-		return fmt.Errorf("failed to enumerate documents for delete-all: %w", err)
-	}
+	// Delete in pages until the index is empty. Each iteration enumerates a
+	// page from offset 0 and deletes exactly those docs, so the next search
+	// surfaces the following page — no From offset bookkeeping, and coverage is
+	// not bounded by a single search page (MCP-3319).
+	for {
+		searchReq := bleve.NewSearchRequest(query)
+		searchReq.Size = b.searchPageSize
+		searchResult, err := b.index.Search(searchReq)
+		if err != nil {
+			return fmt.Errorf("failed to enumerate documents for delete-all: %w", err)
+		}
+		if len(searchResult.Hits) == 0 {
+			return nil
+		}
 
-	batch := b.index.NewBatch()
-	for _, hit := range searchResult.Hits {
-		batch.Delete(hit.ID)
-	}
-	if batch.Size() == 0 {
-		return nil
+		batch := b.index.NewBatch()
+		for _, hit := range searchResult.Hits {
+			batch.Delete(hit.ID)
+		}
+		if err := b.index.Batch(batch); err != nil {
+			return fmt.Errorf("failed to delete documents for delete-all: %w", err)
+		}
 	}
-	return b.index.Batch(batch)
 }
 
 // Batch operations for efficiency
@@ -382,30 +400,40 @@ func (b *BleveIndex) GetToolsByServer(serverName string) ([]*config.ToolMetadata
 	query := bleve.NewTermQuery(serverName)
 	query.SetField("server_name")
 
-	// Create search request with high limit to get all tools
-	searchReq := bleve.NewSearchRequest(query)
-	searchReq.Size = 10000 // Maximum tools per server
-	searchReq.Fields = []string{"tool_name", "full_tool_name", "server_name", "description", "params_json", "output_schema_json", "hash"}
+	fields := []string{"tool_name", "full_tool_name", "server_name", "description", "params_json", "output_schema_json", "hash"}
 
 	b.logger.Debug("Querying tools by server", zap.String("server", serverName))
 
-	searchResult, err := b.index.Search(searchReq)
-	if err != nil {
-		return nil, fmt.Errorf("failed to query tools by server: %w", err)
-	}
-
-	// Convert results to ToolMetadata
+	// Paginate so a server exposing more than one search page of tools is fully
+	// covered — a single capped search would silently drop the overflow
+	// (MCP-3319). A short final page (fewer hits than the page size) ends the loop.
 	var tools []*config.ToolMetadata
-	for _, hit := range searchResult.Hits {
-		toolMeta := &config.ToolMetadata{
-			Name:             getStringField(hit.Fields, "full_tool_name"),
-			ServerName:       getStringField(hit.Fields, "server_name"),
-			Description:      getStringField(hit.Fields, "description"),
-			ParamsJSON:       getStringField(hit.Fields, "params_json"),
-			OutputSchemaJSON: getStringField(hit.Fields, "output_schema_json"),
-			Hash:             getStringField(hit.Fields, "hash"),
+	for from := 0; ; from += b.searchPageSize {
+		searchReq := bleve.NewSearchRequest(query)
+		searchReq.From = from
+		searchReq.Size = b.searchPageSize
+		searchReq.Fields = fields
+
+		searchResult, err := b.index.Search(searchReq)
+		if err != nil {
+			return nil, fmt.Errorf("failed to query tools by server: %w", err)
+		}
+
+		for _, hit := range searchResult.Hits {
+			toolMeta := &config.ToolMetadata{
+				Name:             getStringField(hit.Fields, "full_tool_name"),
+				ServerName:       getStringField(hit.Fields, "server_name"),
+				Description:      getStringField(hit.Fields, "description"),
+				ParamsJSON:       getStringField(hit.Fields, "params_json"),
+				OutputSchemaJSON: getStringField(hit.Fields, "output_schema_json"),
+				Hash:             getStringField(hit.Fields, "hash"),
+			}
+			tools = append(tools, toolMeta)
+		}
+
+		if len(searchResult.Hits) < b.searchPageSize {
+			break
 		}
-		tools = append(tools, toolMeta)
 	}
 
 	b.logger.Debug("Found tools for server",

internal/index/bleve_test.go

@@ -1,6 +1,7 @@
 package index
 
 import (
+	"fmt"
 	"os"
 	"testing"
 	"time"
@@ -506,3 +507,97 @@ func createTestDeFiLlamaTools() []*config.ToolMetadata {
 
 	return tools
 }
+
+// TestBleveIndex_GetToolsByServer_BeyondPageCap verifies that GetToolsByServer
+// returns every tool of a server even when the server exposes more tools than a
+// single search page can hold. Regression for MCP-3319: the prior single-search
+// implementation hard-capped at 10000 hits, silently dropping the overflow.
+func TestBleveIndex_GetToolsByServer_BeyondPageCap(t *testing.T) {
+	tmpDir, err := os.MkdirTemp("", "bleve_test_*")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	logger := zap.NewNop()
+	bleveIndex, err := NewBleveIndex(tmpDir, logger)
+	require.NoError(t, err)
+	defer bleveIndex.Close()
+
+	// Shrink the page size so the test can exercise the pagination loop with a
+	// small, fast corpus instead of indexing >10k docs.
+	bleveIndex.searchPageSize = 3
+
+	// Index 7 tools for one server (spans 3 pages: 3 + 3 + 1) plus an unrelated
+	// server to confirm the term filter is preserved across pages.
+	const total = 7
+	tools := make([]*config.ToolMetadata, 0, total+1)
+	for i := 0; i < total; i++ {
+		tools = append(tools, &config.ToolMetadata{
+			Name:        fmt.Sprintf("paged:tool_%02d", i),
+			ServerName:  "paged",
+			Description: fmt.Sprintf("Paged tool %d", i),
+			Hash:        fmt.Sprintf("hash_%02d", i),
+		})
+	}
+	tools = append(tools, &config.ToolMetadata{
+		Name:        "other:tool_x",
+		ServerName:  "other",
+		Description: "Other tool",
+		Hash:        "hash_other",
+	})
+
+	require.NoError(t, bleveIndex.BatchIndex(tools))
+
+	got, err := bleveIndex.GetToolsByServer("paged")
+	require.NoError(t, err)
+	assert.Len(t, got, total, "all tools across every page must be returned")
+
+	names := make(map[string]bool, len(got))
+	for _, tm := range got {
+		assert.Equal(t, "paged", tm.ServerName)
+		names[tm.Name] = true
+	}
+	for i := 0; i < total; i++ {
+		assert.True(t, names[fmt.Sprintf("paged:tool_%02d", i)],
+			"tool_%02d missing from paginated result", i)
+	}
+}
+
+// TestBleveIndex_DeleteAll_BeyondPageCap verifies that DeleteAll clears every
+// document even when the index holds more docs than a single search page.
+// Regression for MCP-3319: the prior single-search implementation deleted only
+// the first 100000 docs, leaving stale docs beyond the cap.
+func TestBleveIndex_DeleteAll_BeyondPageCap(t *testing.T) {
+	tmpDir, err := os.MkdirTemp("", "bleve_test_*")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpDir)
+
+	logger := zap.NewNop()
+	bleveIndex, err := NewBleveIndex(tmpDir, logger)
+	require.NoError(t, err)
+	defer bleveIndex.Close()
+
+	bleveIndex.searchPageSize = 3
+
+	// Index 10 docs (spans multiple pages of size 3).
+	const total = 10
+	tools := make([]*config.ToolMetadata, 0, total)
+	for i := 0; i < total; i++ {
+		tools = append(tools, &config.ToolMetadata{
+			Name:        fmt.Sprintf("srv:tool_%02d", i),
+			ServerName:  "srv",
+			Description: fmt.Sprintf("Tool %d", i),
+			Hash:        fmt.Sprintf("h_%02d", i),
+		})
+	}
+	require.NoError(t, bleveIndex.BatchIndex(tools))
+
+	count, err := bleveIndex.GetDocumentCount()
+	require.NoError(t, err)
+	require.Equal(t, uint64(total), count)
+
+	require.NoError(t, bleveIndex.DeleteAll())
+
+	count, err = bleveIndex.GetDocumentCount()
+	require.NoError(t, err)
+	assert.Equal(t, uint64(0), count, "DeleteAll must remove every document across all pages")
+}