|
| 1 | +import { RuntimeLogger } from "./logger.js"; |
| 2 | + |
| 3 | +export interface SemanticRequest { |
| 4 | + taskName: string; |
| 5 | + prompt: string; |
| 6 | + maxTokens: number; |
| 7 | +} |
| 8 | + |
| 9 | +export interface SemanticResponse { |
| 10 | + text: string; |
| 11 | + provider: string; |
| 12 | + model: string; |
| 13 | + latencyMs: number; |
| 14 | + promptTokens?: number; |
| 15 | + completionTokens?: number; |
| 16 | + fallbackFrom?: string[]; |
| 17 | +} |
| 18 | + |
| 19 | +export interface SemanticProvider { |
| 20 | + readonly name: string; |
| 21 | + requestText(request: SemanticRequest): Promise<SemanticResponse | null>; |
| 22 | +} |
| 23 | + |
| 24 | +export interface LocalOpenAiProviderOptions { |
| 25 | + baseUrl: string; |
| 26 | + models: string[]; |
| 27 | + timeoutMs: number; |
| 28 | + temperature: number; |
| 29 | + allowNonLoopback: boolean; |
| 30 | +} |
| 31 | + |
| 32 | +function isLoopbackUrl(rawUrl: string): boolean { |
| 33 | + try { |
| 34 | + const hostname = new URL(rawUrl).hostname.toLowerCase(); |
| 35 | + return hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1" || hostname === "[::1]"; |
| 36 | + } catch { |
| 37 | + return false; |
| 38 | + } |
| 39 | +} |
| 40 | + |
| 41 | +function getErrorMessage(error: unknown): string { |
| 42 | + return error instanceof Error ? error.message : String(error); |
| 43 | +} |
| 44 | + |
| 45 | +export class LocalOpenAiProvider implements SemanticProvider { |
| 46 | + readonly name = "local-openai"; |
| 47 | + private readonly options: LocalOpenAiProviderOptions; |
| 48 | + |
| 49 | + constructor(options: LocalOpenAiProviderOptions) { |
| 50 | + if (!options.allowNonLoopback && !isLoopbackUrl(options.baseUrl)) { |
| 51 | + throw new Error("Local semantic provider base URL must use a loopback host unless allowNonLoopback is enabled."); |
| 52 | + } |
| 53 | + this.options = options; |
| 54 | + } |
| 55 | + |
| 56 | + async requestText(request: SemanticRequest): Promise<SemanticResponse | null> { |
| 57 | + const failedModels: string[] = []; |
| 58 | + for (const model of this.options.models) { |
| 59 | + const startedAt = Date.now(); |
| 60 | + try { |
| 61 | + const response = await fetch(`${this.options.baseUrl.replace(/\/$/, "")}/chat/completions`, { |
| 62 | + method: "POST", |
| 63 | + headers: { "content-type": "application/json" }, |
| 64 | + body: JSON.stringify({ |
| 65 | + model, |
| 66 | + messages: [{ role: "user", content: request.prompt }], |
| 67 | + stream: false, |
| 68 | + temperature: this.options.temperature, |
| 69 | + max_tokens: request.maxTokens, |
| 70 | + }), |
| 71 | + signal: AbortSignal.timeout(this.options.timeoutMs), |
| 72 | + }); |
| 73 | + |
| 74 | + if (!response.ok) { |
| 75 | + throw new Error(`HTTP ${response.status}`); |
| 76 | + } |
| 77 | + |
| 78 | + const payload = await response.json() as { |
| 79 | + choices?: Array<{ message?: { content?: unknown } }>; |
| 80 | + usage?: { prompt_tokens?: number; completion_tokens?: number }; |
| 81 | + }; |
| 82 | + const text = payload.choices?.[0]?.message?.content; |
| 83 | + if (typeof text !== "string" || text.trim().length === 0) { |
| 84 | + throw new Error("Response did not contain assistant text."); |
| 85 | + } |
| 86 | + |
| 87 | + return { |
| 88 | + text, |
| 89 | + provider: this.name, |
| 90 | + model, |
| 91 | + latencyMs: Date.now() - startedAt, |
| 92 | + promptTokens: payload.usage?.prompt_tokens, |
| 93 | + completionTokens: payload.usage?.completion_tokens, |
| 94 | + fallbackFrom: failedModels, |
| 95 | + }; |
| 96 | + } catch (error) { |
| 97 | + failedModels.push(model); |
| 98 | + RuntimeLogger.warn(`${request.taskName} local model attempt failed`, { |
| 99 | + provider: this.name, |
| 100 | + model, |
| 101 | + latencyMs: Date.now() - startedAt, |
| 102 | + error: getErrorMessage(error), |
| 103 | + }); |
| 104 | + } |
| 105 | + } |
| 106 | + |
| 107 | + return null; |
| 108 | + } |
| 109 | +} |
| 110 | + |
| 111 | +export class McpSamplingProvider implements SemanticProvider { |
| 112 | + readonly name = "mcp-sampling"; |
| 113 | + private readonly sample: (prompt: string, maxTokens: number) => Promise<string | null>; |
| 114 | + |
| 115 | + constructor(sample: (prompt: string, maxTokens: number) => Promise<string | null>) { |
| 116 | + this.sample = sample; |
| 117 | + } |
| 118 | + |
| 119 | + async requestText(request: SemanticRequest): Promise<SemanticResponse | null> { |
| 120 | + const startedAt = Date.now(); |
| 121 | + const text = await this.sample(request.prompt, request.maxTokens); |
| 122 | + if (!text) { |
| 123 | + return null; |
| 124 | + } |
| 125 | + |
| 126 | + return { |
| 127 | + text, |
| 128 | + provider: this.name, |
| 129 | + model: "client-selected", |
| 130 | + latencyMs: Date.now() - startedAt, |
| 131 | + }; |
| 132 | + } |
| 133 | +} |
| 134 | + |
| 135 | +export class SemanticProviderChain { |
| 136 | + private readonly providers: SemanticProvider[]; |
| 137 | + private readonly onSuccess?: (response: SemanticResponse, request: SemanticRequest) => void; |
| 138 | + |
| 139 | + constructor( |
| 140 | + providers: SemanticProvider[], |
| 141 | + onSuccess?: (response: SemanticResponse, request: SemanticRequest) => void, |
| 142 | + ) { |
| 143 | + this.providers = providers; |
| 144 | + this.onSuccess = onSuccess; |
| 145 | + } |
| 146 | + |
| 147 | + async requestText(request: SemanticRequest): Promise<string | null> { |
| 148 | + const unavailableProviders: string[] = []; |
| 149 | + for (const provider of this.providers) { |
| 150 | + const response = await provider.requestText(request); |
| 151 | + if (response) { |
| 152 | + response.fallbackFrom = [ |
| 153 | + ...unavailableProviders.map(name => `provider:${name}`), |
| 154 | + ...(response.fallbackFrom || []).map(model => `model:${model}`), |
| 155 | + ]; |
| 156 | + this.onSuccess?.(response, request); |
| 157 | + return response.text; |
| 158 | + } |
| 159 | + unavailableProviders.push(provider.name); |
| 160 | + } |
| 161 | + |
| 162 | + RuntimeLogger.warn(`${request.taskName} exhausted all semantic providers`); |
| 163 | + return null; |
| 164 | + } |
| 165 | +} |
0 commit comments