Skip to content

Commit 147f1ea

Browse files
navedmerchantnavedmerchant
committed
feat(code-index): add Semble as a local on-the-fly embedding provider
1 parent b761a0a commit 147f1ea

19 files changed

Lines changed: 1443 additions & 54 deletions

File tree

‎packages/types/src/codebase-index.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ export const codebaseIndexConfigSchema = z.object({
3131
"vercel-ai-gateway",
3232
"bedrock",
3333
"openrouter",
34+
"semble",
3435
])
3536
.optional(),
3637
codebaseIndexEmbedderBaseUrl: z.string().optional(),
@@ -50,6 +51,8 @@ export const codebaseIndexConfigSchema = z.object({
5051
codebaseIndexBedrockProfile: z.string().optional(),
5152
// OpenRouter specific fields
5253
codebaseIndexOpenRouterSpecificProvider: z.string().optional(),
54+
// Semble specific fields
55+
codebaseIndexSemblePath: z.string().optional(),
5356
})
5457

5558
export type CodebaseIndexConfig = z.infer<typeof codebaseIndexConfigSchema>
@@ -67,6 +70,7 @@ export const codebaseIndexModelsSchema = z.object({
6770
"vercel-ai-gateway": z.record(z.string(), z.object({ dimension: z.number() })).optional(),
6871
openrouter: z.record(z.string(), z.object({ dimension: z.number() })).optional(),
6972
bedrock: z.record(z.string(), z.object({ dimension: z.number() })).optional(),
73+
semble: z.record(z.string(), z.object({ dimension: z.number() })).optional(),
7074
})
7175

7276
export type CodebaseIndexModels = z.infer<typeof codebaseIndexModelsSchema>

‎packages/types/src/embedding.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@ export type EmbedderProvider =
66
| "mistral"
77
| "vercel-ai-gateway"
88
| "bedrock"
9-
| "openrouter" // Add other providers as needed.
9+
| "openrouter"
10+
| "semble" // Local hybrid search via semble CLI — no API keys or Qdrant required.
1011

1112
export interface EmbeddingModelProfile {
1213
dimension: number

‎packages/types/src/vscode-extension-host.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -661,6 +661,7 @@ export interface WebviewMessage {
661661
| "vercel-ai-gateway"
662662
| "bedrock"
663663
| "openrouter"
664+
| "semble"
664665
codebaseIndexEmbedderBaseUrl?: string
665666
codebaseIndexEmbedderModelId: string
666667
codebaseIndexEmbedderModelDimension?: number // Generic dimension for all providers
@@ -670,6 +671,7 @@ export interface WebviewMessage {
670671
codebaseIndexSearchMaxResults?: number
671672
codebaseIndexSearchMinScore?: number
672673
codebaseIndexOpenRouterSpecificProvider?: string // OpenRouter provider routing
674+
codebaseIndexSemblePath?: string // Path to the semble executable
673675

674676
// Secret settings
675677
codeIndexOpenAiKey?: string

‎src/core/webview/ClineProvider.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2253,6 +2253,7 @@ export class ClineProvider
22532253
codebaseIndexBedrockRegion: codebaseIndexConfig?.codebaseIndexBedrockRegion,
22542254
codebaseIndexBedrockProfile: codebaseIndexConfig?.codebaseIndexBedrockProfile,
22552255
codebaseIndexOpenRouterSpecificProvider: codebaseIndexConfig?.codebaseIndexOpenRouterSpecificProvider,
2256+
codebaseIndexSemblePath: codebaseIndexConfig?.codebaseIndexSemblePath,
22562257
},
22572258
// Phase 1 cloud removal: do not let Cloud-auth MDM enforcement force login-only UI flows.
22582259
mdmCompliant: undefined,
@@ -2455,6 +2456,7 @@ export class ClineProvider
24552456
codebaseIndexBedrockProfile: stateValues.codebaseIndexConfig?.codebaseIndexBedrockProfile,
24562457
codebaseIndexOpenRouterSpecificProvider:
24572458
stateValues.codebaseIndexConfig?.codebaseIndexOpenRouterSpecificProvider,
2459+
codebaseIndexSemblePath: stateValues.codebaseIndexConfig?.codebaseIndexSemblePath,
24582460
},
24592461
profileThresholds: stateValues.profileThresholds ?? {},
24602462
lockApiConfigAcrossModes: this.context.workspaceState.get("lockApiConfigAcrossModes", false),

‎src/core/webview/webviewMessageHandler.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2500,6 +2500,7 @@ export const webviewMessageHandler = async (
25002500
codebaseIndexSearchMaxResults: settings.codebaseIndexSearchMaxResults,
25012501
codebaseIndexSearchMinScore: settings.codebaseIndexSearchMinScore,
25022502
codebaseIndexOpenRouterSpecificProvider: settings.codebaseIndexOpenRouterSpecificProvider,
2503+
codebaseIndexSemblePath: settings.codebaseIndexSemblePath,
25032504
}
25042505

25052506
// Save global state first

‎src/services/code-index/config-manager.ts‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ export class CodeIndexConfigManager {
2222
private vercelAiGatewayOptions?: { apiKey: string }
2323
private bedrockOptions?: { region: string; profile?: string }
2424
private openRouterOptions?: { apiKey: string; specificProvider?: string }
25+
private semblePath?: string
2526
private qdrantUrl?: string = "http://localhost:6333"
2627
private qdrantApiKey?: string
2728
private searchMinScore?: number
@@ -120,6 +121,8 @@ export class CodeIndexConfigManager {
120121
this.embedderProvider = "bedrock"
121122
} else if (codebaseIndexEmbedderProvider === "openrouter") {
122123
this.embedderProvider = "openrouter"
124+
} else if (codebaseIndexEmbedderProvider === "semble") {
125+
this.embedderProvider = "semble"
123126
} else {
124127
this.embedderProvider = "openai"
125128
}
@@ -148,6 +151,9 @@ export class CodeIndexConfigManager {
148151
this.bedrockOptions = bedrockRegion
149152
? { region: bedrockRegion, profile: bedrockProfile || undefined }
150153
: undefined
154+
155+
// Semble path (optional — defaults to "semble" in the SembleProvider)
156+
this.semblePath = codebaseIndexConfig.codebaseIndexSemblePath || undefined
151157
}
152158

153159
/**
@@ -194,6 +200,7 @@ export class CodeIndexConfigManager {
194200
openRouterSpecificProvider: this.openRouterOptions?.specificProvider ?? "",
195201
qdrantUrl: this.qdrantUrl ?? "",
196202
qdrantApiKey: this.qdrantApiKey ?? "",
203+
semblePath: this.semblePath ?? "",
197204
}
198205

199206
// Refresh secrets from VSCode storage to ensure we have the latest values
@@ -231,6 +238,11 @@ export class CodeIndexConfigManager {
231238
* Checks if the service is properly configured based on the embedder type.
232239
*/
233240
public isConfigured(): boolean {
241+
if (this.embedderProvider === "semble") {
242+
// Semble requires no API keys or Qdrant — it's always configured
243+
return true
244+
}
245+
234246
if (this.embedderProvider === "openai") {
235247
const openAiKey = this.openAiOptions?.openAiNativeApiKey
236248
const qdrantUrl = this.qdrantUrl
@@ -405,6 +417,13 @@ export class CodeIndexConfigManager {
405417
return true
406418
}
407419

420+
// Semble path change requires restart to use the new executable
421+
const prevSemblePath = prev?.semblePath ?? ""
422+
const currentSemblePath = this.semblePath ?? ""
423+
if (prevSemblePath !== currentSemblePath) {
424+
return true
425+
}
426+
408427
// Vector dimension changes (still important for compatibility)
409428
if (this._hasVectorDimensionChanged(prevProvider, prev?.modelId)) {
410429
return true
@@ -541,4 +560,12 @@ export class CodeIndexConfigManager {
541560
public get currentSearchMaxResults(): number {
542561
return this.searchMaxResults ?? DEFAULT_MAX_SEARCH_RESULTS
543562
}
563+
564+
/**
565+
* Gets the configured path to the semble executable.
566+
* Returns undefined if not explicitly configured (provider will use default "semble").
567+
*/
568+
public get currentSemblePath(): string | undefined {
569+
return this.semblePath
570+
}
544571
}

‎src/services/code-index/interfaces/config.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,4 +45,5 @@ export type PreviousConfigSnapshot = {
4545
openRouterSpecificProvider?: string
4646
qdrantUrl?: string
4747
qdrantApiKey?: string
48+
semblePath?: string
4849
}

‎src/services/code-index/interfaces/manager.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ export type EmbedderProvider =
8484
| "vercel-ai-gateway"
8585
| "bedrock"
8686
| "openrouter"
87+
| "semble"
8788

8889
export interface IndexProgressUpdate {
8990
systemStatus: IndexingState

‎src/services/code-index/manager.ts‎

Lines changed: 66 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import { CodeIndexServiceFactory } from "./service-factory"
88
import { CodeIndexSearchService } from "./search-service"
99
import { CodeIndexOrchestrator } from "./orchestrator"
1010
import { CacheManager } from "./cache-manager"
11+
import { SembleProvider } from "./semble"
1112
import { RooIgnoreController } from "../../core/ignore/RooIgnoreController"
1213
import fs from "fs/promises"
1314
import ignore from "ignore"
@@ -27,6 +28,7 @@ export class CodeIndexManager {
2728
private _orchestrator: CodeIndexOrchestrator | undefined
2829
private _searchService: CodeIndexSearchService | undefined
2930
private _cacheManager: CacheManager | undefined
31+
private _sembleProvider: SembleProvider | undefined
3032

3133
// Flag to prevent race conditions during error recovery
3234
private _isRecoveringFromError = false
@@ -125,6 +127,10 @@ export class CodeIndexManager {
125127
}
126128

127129
private assertInitialized() {
130+
if (this._sembleProvider) {
131+
// When semble is active, we don't need orchestrator/searchService
132+
return
133+
}
128134
if (!this._configManager || !this._orchestrator || !this._searchService || !this._cacheManager) {
129135
throw new Error("CodeIndexManager not initialized. Call initialize() first.")
130136
}
@@ -134,6 +140,9 @@ export class CodeIndexManager {
134140
if (!this.isFeatureEnabled) {
135141
return "Standby"
136142
}
143+
if (this._sembleProvider) {
144+
return this._sembleProvider.state
145+
}
137146
this.assertInitialized()
138147
return this._orchestrator!.state
139148
}
@@ -173,6 +182,9 @@ export class CodeIndexManager {
173182
if (this._orchestrator) {
174183
this._orchestrator.stopWatcher()
175184
}
185+
if (this._sembleProvider) {
186+
this._sembleProvider.stopIndexing()
187+
}
176188
return { requiresRestart }
177189
}
178190

@@ -196,19 +208,27 @@ export class CodeIndexManager {
196208
}
197209

198210
// 6. Determine if Core Services Need Recreation
199-
const needsServiceRecreation = !this._serviceFactory || requiresRestart
211+
const needsServiceRecreation = (!this._serviceFactory && !this._sembleProvider) || requiresRestart
200212

201213
if (needsServiceRecreation) {
202214
await this._recreateServices()
203215
}
204216

205217
// 7. Handle Indexing Start/Restart
206-
const shouldStartOrRestartIndexing =
207-
requiresRestart ||
208-
(needsServiceRecreation && (!this._orchestrator || this._orchestrator.state !== "Indexing"))
218+
if (this._sembleProvider) {
219+
// For semble, start indexing if needed
220+
const shouldStartIndexing = requiresRestart || needsServiceRecreation
221+
if (shouldStartIndexing) {
222+
await this._sembleProvider.startIndexing()
223+
}
224+
} else {
225+
const shouldStartOrRestartIndexing =
226+
requiresRestart ||
227+
(needsServiceRecreation && (!this._orchestrator || this._orchestrator.state !== "Indexing"))
209228

210-
if (shouldStartOrRestartIndexing) {
211-
this._orchestrator?.startIndexing()
229+
if (shouldStartOrRestartIndexing) {
230+
this._orchestrator?.startIndexing()
231+
}
212232
}
213233

214234
return { requiresRestart }
@@ -226,6 +246,12 @@ export class CodeIndexManager {
226246
return
227247
}
228248

249+
// Delegate to semble provider if active
250+
if (this._sembleProvider) {
251+
await this._sembleProvider.startIndexing()
252+
return
253+
}
254+
229255
// Check if we're in error state and recover if needed
230256
const currentStatus = this.getCurrentStatus()
231257
if (currentStatus.systemStatus === "Error") {
@@ -244,6 +270,10 @@ export class CodeIndexManager {
244270
* Stops any in-progress indexing operation and the file watcher.
245271
*/
246272
public stopIndexing(): void {
273+
if (this._sembleProvider) {
274+
this._sembleProvider.stopIndexing()
275+
return
276+
}
247277
if (this._orchestrator) {
248278
this._orchestrator.stopIndexing()
249279
}
@@ -295,6 +325,7 @@ export class CodeIndexManager {
295325
this._serviceFactory = undefined
296326
this._orchestrator = undefined
297327
this._searchService = undefined
328+
this._sembleProvider = undefined
298329

299330
// Reset the flag after recovery is complete
300331
this._isRecoveringFromError = false
@@ -306,6 +337,10 @@ export class CodeIndexManager {
306337
*/
307338
public dispose(): void {
308339
this.stopIndexing()
340+
if (this._sembleProvider) {
341+
this._sembleProvider.dispose()
342+
this._sembleProvider = undefined
343+
}
309344
this._stateManager.dispose()
310345
}
311346

@@ -317,6 +352,10 @@ export class CodeIndexManager {
317352
if (!this.isFeatureEnabled) {
318353
return
319354
}
355+
if (this._sembleProvider) {
356+
await this._sembleProvider.clearIndexData()
357+
return
358+
}
320359
this.assertInitialized()
321360
await this._orchestrator!.clearIndexData()
322361
await this._cacheManager!.clearCacheFile()
@@ -338,6 +377,9 @@ export class CodeIndexManager {
338377
if (!this.isFeatureEnabled) {
339378
return []
340379
}
380+
if (this._sembleProvider) {
381+
return this._sembleProvider.searchIndex(query, directoryPrefix)
382+
}
341383
this.assertInitialized()
342384
return this._searchService!.searchIndex(query, directoryPrefix)
343385
}
@@ -351,11 +393,28 @@ export class CodeIndexManager {
351393
if (this._orchestrator) {
352394
this.stopWatcher()
353395
}
396+
// Dispose existing semble provider if switching away
397+
if (this._sembleProvider) {
398+
this._sembleProvider.dispose()
399+
this._sembleProvider = undefined
400+
}
354401
// Clear existing services to ensure clean state
355402
this._orchestrator = undefined
356403
this._searchService = undefined
357404

358-
// (Re)Initialize service factory
405+
// Branch: if provider is "semble", create SembleProvider instead of external services
406+
if (this._configManager!.currentEmbedderProvider === "semble") {
407+
this._sembleProvider = new SembleProvider(
408+
this.workspacePath,
409+
this.context,
410+
this._stateManager,
411+
this._configManager!.currentSemblePath,
412+
)
413+
await this._sembleProvider.initialize()
414+
return
415+
}
416+
417+
// (Re)Initialize service factory for external providers
359418
this._serviceFactory = new CodeIndexServiceFactory(
360419
this._configManager!,
361420
this.workspacePath,

0 commit comments

Comments
 (0)