|
| 1 | +import { runConnectivityTest } from '@midscene/core'; |
| 2 | +import { |
| 3 | + type IModelConfig, |
| 4 | + type TIntent, |
| 5 | + globalModelConfigManager, |
| 6 | +} from '@midscene/shared/env'; |
| 7 | +import chalk from 'chalk'; |
| 8 | +import { loadDotenvConfig } from './dotenv-loader'; |
| 9 | + |
| 10 | +const DEFAULT_OPENAI_BASE_URL = 'https://api.openai.com/v1'; |
| 11 | +const MODEL_CHECK_SEPARATOR = '────────────────────────────────────────'; |
| 12 | +const MODEL_COMMAND_USAGE = `Usage: |
| 13 | + midscene model check |
| 14 | + midscene model eval |
| 15 | +`; |
| 16 | + |
| 17 | +interface ModelCommandIO { |
| 18 | + stdout: (message: string) => void; |
| 19 | + stderr: (message: string) => void; |
| 20 | +} |
| 21 | + |
| 22 | +interface ModelCommandDeps { |
| 23 | + loadDotenv: () => void; |
| 24 | + getModelConfig: (intent: TIntent) => IModelConfig; |
| 25 | + checkModel: typeof runConnectivityTest; |
| 26 | +} |
| 27 | + |
| 28 | +export interface CurlCommandItem { |
| 29 | + intents: TIntent[]; |
| 30 | + curl: string; |
| 31 | + usesDefaultBaseURL: boolean; |
| 32 | +} |
| 33 | + |
| 34 | +function assertNoModelCheckOptions(args: string[]) { |
| 35 | + for (const arg of args) { |
| 36 | + throw new Error(`Unknown option for midscene model check: ${arg}`); |
| 37 | + } |
| 38 | +} |
| 39 | + |
| 40 | +function shellSingleQuote(value: string): string { |
| 41 | + return `'${value.replace(/'/g, `'\\''`)}'`; |
| 42 | +} |
| 43 | + |
| 44 | +function buildChatCompletionsUrl(baseURL?: string): string { |
| 45 | + const normalizedBaseURL = (baseURL || DEFAULT_OPENAI_BASE_URL).replace( |
| 46 | + /\/+$/, |
| 47 | + '', |
| 48 | + ); |
| 49 | + if (normalizedBaseURL.endsWith('/chat/completions')) { |
| 50 | + return normalizedBaseURL; |
| 51 | + } |
| 52 | + return `${normalizedBaseURL}/chat/completions`; |
| 53 | +} |
| 54 | + |
| 55 | +function buildCurlCommand(modelConfig: IModelConfig): string { |
| 56 | + const payload = { |
| 57 | + model: modelConfig.modelName, |
| 58 | + messages: [{ role: 'user', content: 'What is 1+1?' }], |
| 59 | + }; |
| 60 | + |
| 61 | + return [ |
| 62 | + `curl -X POST ${shellSingleQuote(buildChatCompletionsUrl(modelConfig.openaiBaseURL))} \\`, |
| 63 | + ` -H ${shellSingleQuote(`Authorization: Bearer ${modelConfig.openaiApiKey || ''}`)} \\`, |
| 64 | + ` -H ${shellSingleQuote('Content-Type: application/json')} \\`, |
| 65 | + ` -d ${shellSingleQuote(JSON.stringify(payload, null, 2))}`, |
| 66 | + ].join('\n'); |
| 67 | +} |
| 68 | + |
| 69 | +function buildCurlDedupKey(modelConfig: IModelConfig): string { |
| 70 | + return JSON.stringify({ |
| 71 | + baseURL: modelConfig.openaiBaseURL || DEFAULT_OPENAI_BASE_URL, |
| 72 | + apiKey: modelConfig.openaiApiKey || '', |
| 73 | + modelName: modelConfig.modelName, |
| 74 | + }); |
| 75 | +} |
| 76 | + |
| 77 | +export function buildModelCheckCurlCommands( |
| 78 | + configs: Array<{ intent: TIntent; modelConfig: IModelConfig }>, |
| 79 | +): CurlCommandItem[] { |
| 80 | + const commandMap = new Map<string, CurlCommandItem>(); |
| 81 | + |
| 82 | + for (const item of configs) { |
| 83 | + const key = buildCurlDedupKey(item.modelConfig); |
| 84 | + const existing = commandMap.get(key); |
| 85 | + if (existing) { |
| 86 | + existing.intents.push(item.intent); |
| 87 | + continue; |
| 88 | + } |
| 89 | + commandMap.set(key, { |
| 90 | + intents: [item.intent], |
| 91 | + curl: buildCurlCommand(item.modelConfig), |
| 92 | + usesDefaultBaseURL: !item.modelConfig.openaiBaseURL, |
| 93 | + }); |
| 94 | + } |
| 95 | + |
| 96 | + return [...commandMap.values()]; |
| 97 | +} |
| 98 | + |
| 99 | +function formatModelCheckFailureOutput( |
| 100 | + message: string | undefined, |
| 101 | + curlCommands: CurlCommandItem[], |
| 102 | +): string { |
| 103 | + const details = message?.trim() || 'No failure details were generated.'; |
| 104 | + const curlSection = curlCommands |
| 105 | + .map((item) => { |
| 106 | + const baseUrlNote = item.usesDefaultBaseURL |
| 107 | + ? ' (base URL not configured; using OpenAI SDK default)' |
| 108 | + : ''; |
| 109 | + return chalk.gray( |
| 110 | + `# ${item.intents.join(', ')}${baseUrlNote}\n${item.curl}`, |
| 111 | + ); |
| 112 | + }) |
| 113 | + .join('\n\n'); |
| 114 | + |
| 115 | + return [ |
| 116 | + chalk.red.bold('❌ Model check failed with messages:'), |
| 117 | + MODEL_CHECK_SEPARATOR, |
| 118 | + '', |
| 119 | + details, |
| 120 | + '', |
| 121 | + MODEL_CHECK_SEPARATOR, |
| 122 | + 'Generated curl requests for basic API connectivity:', |
| 123 | + 'If the error is a basic connectivity issue, use these requests to test the base URL, API key, and model name directly.', |
| 124 | + 'These commands contain your API key. Do not share them publicly.', |
| 125 | + '', |
| 126 | + curlSection, |
| 127 | + ].join('\n'); |
| 128 | +} |
| 129 | + |
| 130 | +function getDefaultDeps(io: ModelCommandIO): ModelCommandDeps { |
| 131 | + return { |
| 132 | + loadDotenv: () => { |
| 133 | + loadDotenvConfig({ |
| 134 | + dotenvDebug: true, |
| 135 | + dotenvOverride: false, |
| 136 | + log: io.stdout, |
| 137 | + }); |
| 138 | + }, |
| 139 | + getModelConfig: (intent) => globalModelConfigManager.getModelConfig(intent), |
| 140 | + checkModel: runConnectivityTest, |
| 141 | + }; |
| 142 | +} |
| 143 | + |
| 144 | +async function runModelCheckCommand( |
| 145 | + args: string[], |
| 146 | + deps: ModelCommandDeps, |
| 147 | + io: ModelCommandIO, |
| 148 | +): Promise<number> { |
| 149 | + try { |
| 150 | + if (args.includes('--help') || args.includes('-h')) { |
| 151 | + io.stdout(MODEL_COMMAND_USAGE); |
| 152 | + return 0; |
| 153 | + } |
| 154 | + |
| 155 | + assertNoModelCheckOptions(args); |
| 156 | + io.stdout('Model check started. This usually takes about 5 seconds.\n'); |
| 157 | + deps.loadDotenv(); |
| 158 | + io.stdout(''); |
| 159 | + |
| 160 | + const defaultModelConfig = deps.getModelConfig('default'); |
| 161 | + const planningModelConfig = deps.getModelConfig('planning'); |
| 162 | + const insightModelConfig = deps.getModelConfig('insight'); |
| 163 | + const curlCommands = buildModelCheckCurlCommands([ |
| 164 | + { intent: 'default', modelConfig: defaultModelConfig }, |
| 165 | + { intent: 'planning', modelConfig: planningModelConfig }, |
| 166 | + { intent: 'insight', modelConfig: insightModelConfig }, |
| 167 | + ]); |
| 168 | + |
| 169 | + const result = await deps.checkModel({ |
| 170 | + defaultModelConfig, |
| 171 | + planningModelConfig, |
| 172 | + insightModelConfig, |
| 173 | + }); |
| 174 | + |
| 175 | + if (result.passed) { |
| 176 | + io.stdout('✅ Model check passed.'); |
| 177 | + return 0; |
| 178 | + } |
| 179 | + |
| 180 | + io.stderr(formatModelCheckFailureOutput(result.message, curlCommands)); |
| 181 | + return 1; |
| 182 | + } catch (error) { |
| 183 | + const message = error instanceof Error ? error.message : String(error); |
| 184 | + io.stderr( |
| 185 | + [ |
| 186 | + MODEL_CHECK_SEPARATOR, |
| 187 | + '❌ Model check failed with messages:', |
| 188 | + '', |
| 189 | + message, |
| 190 | + MODEL_CHECK_SEPARATOR, |
| 191 | + ].join('\n'), |
| 192 | + ); |
| 193 | + return 1; |
| 194 | + } |
| 195 | +} |
| 196 | + |
| 197 | +export async function runModelCommand( |
| 198 | + rawArgs: string[], |
| 199 | + deps?: Partial<ModelCommandDeps>, |
| 200 | + io: ModelCommandIO = { |
| 201 | + stdout: console.log, |
| 202 | + stderr: console.error, |
| 203 | + }, |
| 204 | +): Promise<number> { |
| 205 | + const [, action, ...restArgs] = rawArgs; |
| 206 | + const mergedDeps = { |
| 207 | + ...getDefaultDeps(io), |
| 208 | + ...deps, |
| 209 | + }; |
| 210 | + |
| 211 | + if (!action || action === '--help' || action === '-h') { |
| 212 | + io.stdout(MODEL_COMMAND_USAGE); |
| 213 | + return 0; |
| 214 | + } |
| 215 | + |
| 216 | + if (action === 'check') { |
| 217 | + return runModelCheckCommand(restArgs, mergedDeps, io); |
| 218 | + } |
| 219 | + |
| 220 | + if (action === 'eval') { |
| 221 | + io.stderr( |
| 222 | + 'midscene model eval is not implemented yet. It is reserved for future model evaluation suites.', |
| 223 | + ); |
| 224 | + return 1; |
| 225 | + } |
| 226 | + |
| 227 | + io.stderr( |
| 228 | + `Unknown midscene model command: ${action}\n\n${MODEL_COMMAND_USAGE}`, |
| 229 | + ); |
| 230 | + return 1; |
| 231 | +} |
0 commit comments