Add SQLite FTS5-backed ToolStore for optimizer search (#3786)

* Add SQLite FTS5-backed ToolStore for optimizer search

Add an SQLite FTS5 ToolStore implementation as an alternative to the
existing InMemoryToolStore, and wire it into the vMCP server when
optimizer config is present.

This adds:
- pkg/vmcp/optimizer/fts5store: SQLiteToolStore implementation using
  modernc.org/sqlite (pure Go, no CGO) with FTS5 virtual tables
- BM25 ranking for search results with LIKE-based fallback
- FTS5 query sanitization for safe handling of special characters
- Thread-safe concurrent access with sync.RWMutex
- Close() method on the ToolStore interface for resource cleanup
- OptimizerStoreCloser on server.Config for store lifecycle management
- OptimizerConfig.FTSDBPath for configurable database location
  (defaults to in-memory, use emptyDir in Kubernetes)
- Wire FTS5 store into vMCP serve command when optimizer is configured
- Comprehensive tests for search, upsert, and concurrency scenarios

Part 2 of the optimizer FTS5 migration (issue #3731).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* review comments

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: aponcedeleonch

cmd/vmcp/app/commands.go

@@ -28,7 +28,6 @@ import (
 	"github.com/stacklok/toolhive/pkg/vmcp/discovery"
 	"github.com/stacklok/toolhive/pkg/vmcp/health"
 	"github.com/stacklok/toolhive/pkg/vmcp/k8s"
-	"github.com/stacklok/toolhive/pkg/vmcp/optimizer"
 	vmcprouter "github.com/stacklok/toolhive/pkg/vmcp/router"
 	vmcpserver "github.com/stacklok/toolhive/pkg/vmcp/server"
 	vmcpstatus "github.com/stacklok/toolhive/pkg/vmcp/status"
@@ -483,8 +482,7 @@ func runServe(cmd *cobra.Command, _ []string) error {
 	}
 
 	if cfg.Optimizer != nil {
-		// TODO: update this with the real optimizer.
-		serverCfg.OptimizerFactory = optimizer.NewDummyOptimizerFactory()
+		serverCfg.OptimizerEnabled = true
 	}
 
 	// Convert composite tool configurations to workflow definitions

pkg/vmcp/optimizer/dummy_optimizer.go

@@ -106,6 +106,16 @@ func (d *DummyOptimizer) toolNames() []string {
 // returned factory share the same underlying storage, enabling cross-session search.
 func NewDummyOptimizerFactory() func(context.Context, []server.ServerTool) (Optimizer, error) {
 	store := NewInMemoryToolStore()
+	return NewDummyOptimizerFactoryWithStore(store)
+}
+
+// NewDummyOptimizerFactoryWithStore returns an OptimizerFactory that creates
+// DummyOptimizer instances backed by the given ToolStore. All optimizers created
+// by the returned factory share the same store, enabling cross-session search.
+//
+// Use this when you need to provide a specific store implementation (e.g.,
+// SQLiteToolStore for FTS5-based search) instead of the default InMemoryToolStore.
+func NewDummyOptimizerFactoryWithStore(store ToolStore) func(context.Context, []server.ServerTool) (Optimizer, error) {
 	return func(ctx context.Context, tools []server.ServerTool) (Optimizer, error) {
 		return NewDummyOptimizer(ctx, store, tools)
 	}

pkg/vmcp/optimizer/dummy_optimizer_test.go

@@ -5,13 +5,136 @@ package optimizer
 
 import (
 	"context"
+	"fmt"
 	"testing"
 
 	"github.com/mark3labs/mcp-go/mcp"
 	"github.com/mark3labs/mcp-go/server"
 	"github.com/stretchr/testify/require"
 )
 
+// mockToolStore implements ToolStore for testing optimizer logic against a
+// controllable store without any database dependency.
+type mockToolStore struct {
+	upsertFunc func(ctx context.Context, tools []server.ServerTool) error
+	searchFunc func(ctx context.Context, query string, allowedTools []string) ([]ToolMatch, error)
+}
+
+func (m *mockToolStore) UpsertTools(ctx context.Context, tools []server.ServerTool) error {
+	if m.upsertFunc != nil {
+		return m.upsertFunc(ctx, tools)
+	}
+	panic("mockToolStore.UpsertTools called but not configured")
+}
+
+func (m *mockToolStore) Search(ctx context.Context, query string, allowedTools []string) ([]ToolMatch, error) {
+	if m.searchFunc != nil {
+		return m.searchFunc(ctx, query, allowedTools)
+	}
+	panic("mockToolStore.Search called but not configured")
+}
+
+func (*mockToolStore) Close() error {
+	return nil
+}
+
+// TestDummyOptimizer_MockStore tests the optimizer against a mock ToolStore,
+// verifying search delegation, scoping, and error handling without any database.
+func TestDummyOptimizer_MockStore(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name          string
+		tools         []server.ServerTool
+		searchFunc    func(ctx context.Context, query string, allowedTools []string) ([]ToolMatch, error)
+		upsertFunc    func(ctx context.Context, tools []server.ServerTool) error
+		input         FindToolInput
+		expectedNames []string
+		expectErr     bool
+		errContains   string
+		expectCreate  bool // if false, expect NewDummyOptimizer to fail
+		createErr     string
+	}{
+		{
+			name: "delegates search to store with allowedTools",
+			tools: []server.ServerTool{
+				{Tool: mcp.Tool{Name: "tool_a", Description: "Tool A"}},
+				{Tool: mcp.Tool{Name: "tool_b", Description: "Tool B"}},
+			},
+			upsertFunc: func(_ context.Context, _ []server.ServerTool) error { return nil },
+			searchFunc: func(_ context.Context, query string, allowedTools []string) ([]ToolMatch, error) {
+				require.Equal(t, "query", query)
+				require.ElementsMatch(t, []string{"tool_a", "tool_b"}, allowedTools)
+				return []ToolMatch{
+					{Name: "tool_a", Description: "Tool A", Score: 0.9},
+				}, nil
+			},
+			input:         FindToolInput{ToolDescription: "query"},
+			expectedNames: []string{"tool_a"},
+			expectCreate:  true,
+		},
+		{
+			name: "propagates store search errors",
+			tools: []server.ServerTool{
+				{Tool: mcp.Tool{Name: "tool_a", Description: "Tool A"}},
+			},
+			upsertFunc: func(_ context.Context, _ []server.ServerTool) error { return nil },
+			searchFunc: func(context.Context, string, []string) ([]ToolMatch, error) {
+				return nil, fmt.Errorf("store unavailable")
+			},
+			input:        FindToolInput{ToolDescription: "query"},
+			expectErr:    true,
+			errContains:  "tool search failed",
+			expectCreate: true,
+		},
+		{
+			name: "propagates store upsert errors at creation",
+			tools: []server.ServerTool{
+				{Tool: mcp.Tool{Name: "tool_a", Description: "Tool A"}},
+			},
+			upsertFunc: func(context.Context, []server.ServerTool) error {
+				return fmt.Errorf("upsert failed")
+			},
+			input:        FindToolInput{ToolDescription: "query"},
+			expectCreate: false,
+			createErr:    "failed to upsert tools into store",
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			store := &mockToolStore{
+				upsertFunc: tc.upsertFunc,
+				searchFunc: tc.searchFunc,
+			}
+
+			opt, err := NewDummyOptimizer(context.Background(), store, tc.tools)
+			if !tc.expectCreate {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.createErr)
+				return
+			}
+			require.NoError(t, err)
+
+			result, err := opt.FindTool(context.Background(), tc.input)
+			if tc.expectErr {
+				require.Error(t, err)
+				require.Contains(t, err.Error(), tc.errContains)
+				return
+			}
+
+			require.NoError(t, err)
+			var names []string
+			for _, m := range result.Tools {
+				names = append(names, m.Name)
+			}
+			require.ElementsMatch(t, tc.expectedNames, names)
+		})
+	}
+}
+
 func TestDummyOptimizer_FindTool(t *testing.T) {
 	t.Parallel()
 
@@ -139,7 +262,7 @@ func TestDummyOptimizerFactory_SharedStorage(t *testing.T) {
 	require.Len(t, result2.Tools, 1)
 	require.Equal(t, "tool_b", result2.Tools[0].Name)
 
-	// Both tools exist in the shared store — verify by creating an optimizer with both in scope
+	// Both tools exist in the shared store — verify by creating an optimizer with both in allowedTools
 	opt3, err := factory(ctx, []server.ServerTool{
 		{Tool: mcp.Tool{Name: "tool_a", Description: "Alpha tool"}},
 		{Tool: mcp.Tool{Name: "tool_b", Description: "Beta tool"}},
@@ -154,6 +277,80 @@ func TestDummyOptimizerFactory_SharedStorage(t *testing.T) {
 	require.ElementsMatch(t, []string{"tool_a", "tool_b"}, names)
 }
 
+func TestNewDummyOptimizerFactoryWithStore(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name           string
+		sessionATools  []server.ServerTool
+		sessionBTools  []server.ServerTool
+		searchQuery    string
+		sessionAExpect []string
+		sessionBExpect []string
+	}{
+		{
+			name: "separate sessions see only their own tools",
+			sessionATools: []server.ServerTool{
+				{Tool: mcp.Tool{Name: "tool_alpha", Description: "Alpha tool"}},
+			},
+			sessionBTools: []server.ServerTool{
+				{Tool: mcp.Tool{Name: "tool_beta", Description: "Beta tool"}},
+			},
+			searchQuery:    "tool",
+			sessionAExpect: []string{"tool_alpha"},
+			sessionBExpect: []string{"tool_beta"},
+		},
+		{
+			name: "overlapping tools are shared",
+			sessionATools: []server.ServerTool{
+				{Tool: mcp.Tool{Name: "shared_tool", Description: "Shared tool"}},
+				{Tool: mcp.Tool{Name: "tool_a_only", Description: "A only"}},
+			},
+			sessionBTools: []server.ServerTool{
+				{Tool: mcp.Tool{Name: "shared_tool", Description: "Shared tool"}},
+				{Tool: mcp.Tool{Name: "tool_b_only", Description: "B only"}},
+			},
+			searchQuery:    "tool",
+			sessionAExpect: []string{"shared_tool", "tool_a_only"},
+			sessionBExpect: []string{"shared_tool", "tool_b_only"},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			store := NewInMemoryToolStore()
+			factory := NewDummyOptimizerFactoryWithStore(store)
+			ctx := context.Background()
+
+			optA, err := factory(ctx, tc.sessionATools)
+			require.NoError(t, err)
+
+			optB, err := factory(ctx, tc.sessionBTools)
+			require.NoError(t, err)
+
+			resultA, err := optA.FindTool(ctx, FindToolInput{ToolDescription: tc.searchQuery})
+			require.NoError(t, err)
+
+			var namesA []string
+			for _, m := range resultA.Tools {
+				namesA = append(namesA, m.Name)
+			}
+			require.ElementsMatch(t, tc.sessionAExpect, namesA)
+
+			resultB, err := optB.FindTool(ctx, FindToolInput{ToolDescription: tc.searchQuery})
+			require.NoError(t, err)
+
+			var namesB []string
+			for _, m := range resultB.Tools {
+				namesB = append(namesB, m.Name)
+			}
+			require.ElementsMatch(t, tc.sessionBExpect, namesB)
+		})
+	}
+}
+
 func TestDummyOptimizer_CallTool(t *testing.T) {
 	t.Parallel()
 

pkg/vmcp/optimizer/internal/sqlite_store/schema.sql

@@ -0,0 +1,34 @@
+-- SPDX-FileCopyrightText: Copyright 2025 Stacklok, Inc.
+-- SPDX-License-Identifier: Apache-2.0
+
+-- Capabilities table stores tool/resource/prompt metadata
+CREATE TABLE IF NOT EXISTS llm_capabilities (
+    name TEXT PRIMARY KEY,
+    description TEXT NOT NULL DEFAULT ''
+);
+
+-- FTS5 virtual table for full-text search with BM25 ranking.
+-- tokenize='porter' uses the Porter stemming algorithm so that morphological
+-- variants of a word (e.g. "running", "runs", "ran") match the root form "run".
+-- This improves recall for natural-language tool descriptions.
+CREATE VIRTUAL TABLE IF NOT EXISTS llm_capabilities_fts USING fts5(
+    name,
+    description,
+    content=llm_capabilities,
+    content_rowid=rowid,
+    tokenize='porter'
+);
+
+-- Triggers to keep FTS index in sync with llm_capabilities table
+CREATE TRIGGER IF NOT EXISTS llm_capabilities_after_insert AFTER INSERT ON llm_capabilities BEGIN
+    INSERT INTO llm_capabilities_fts(rowid, name, description) VALUES (new.rowid, new.name, new.description);
+END;
+
+CREATE TRIGGER IF NOT EXISTS llm_capabilities_after_delete AFTER DELETE ON llm_capabilities BEGIN
+    INSERT INTO llm_capabilities_fts(llm_capabilities_fts, rowid, name, description) VALUES('delete', old.rowid, old.name, old.description);
+END;
+
+CREATE TRIGGER IF NOT EXISTS llm_capabilities_after_update AFTER UPDATE ON llm_capabilities BEGIN
+    INSERT INTO llm_capabilities_fts(llm_capabilities_fts, rowid, name, description) VALUES('delete', old.rowid, old.name, old.description);
+    INSERT INTO llm_capabilities_fts(rowid, name, description) VALUES (new.rowid, new.name, new.description);
+END;