feat: add roll-call command for batch-testing model connectivity

cbrunnkvist · cbrunnkvist · commit a8337fd556c2 · 2026-02-19T17:03:14.000+07:00
Add a new 'roll-call' subcommand that allows users to batch-test multiple
models for connectivity and latency. This helps users discover which
provider/model pairs are available and working.

Features:
- Filter models by regex pattern (required for safety)
- Configurable prompt, timeout, and parallelization
- JSON or table output formats
- Proper provider options handling for reasoning models

Fixes applied:
- Apply ProviderTransform options (maxOutputTokens, temperature, etc.)
- Enable reasoning for dedicated thinking models via Kilo Gateway
- Prevent accidental full model list testing without explicit filter

Closes #457
diff --git a/packages/opencode/src/cli/cmd/roll-call.ts b/packages/opencode/src/cli/cmd/roll-call.ts
@@ -0,0 +1,251 @@
+import type { Argv } from "yargs"
+import { Instance } from "../../project/instance"
+import { Provider } from "../../provider/provider"
+import { ProviderTransform } from "../../provider/transform"
+import { cmd } from "./cmd"
+import { UI } from "../ui"
+import { APICallError } from "ai"
+import { ProviderError } from "../../provider/error"
+import { generateText } from "ai"
+import { randomUUID } from "crypto"
+
+export const RollCallCommand = cmd({
+  command: "roll-call <filter>",
+  describe: "batch-test models matching a filter for connectivity and latency",
+  builder: (yargs: Argv) => {
+    return yargs
+      .positional("filter", {
+        type: "string",
+        describe: "regex to filter models by provider/modelID (required)",
+        demandOption: true,
+      })
+      .option("prompt", {
+        type: "string",
+        default: "Hello",
+        describe: "Prompt to send to each model",
+      })
+      .option("timeout", {
+        type: "number",
+        default: 25000,
+        describe: "Timeout for each model call in milliseconds",
+      })
+      .option("parallel", {
+        type: "number",
+        default: 5,
+        describe: "Number of parallel model calls",
+      })
+      .option("retries", {
+        type: "number",
+        default: 0,
+        describe: "Number of additional retries for each model call",
+      })
+      .option("verbose", {
+        type: "boolean",
+        default: false,
+        describe: "Show verbose output",
+      })
+      .option("quiet", {
+        type: "boolean",
+        default: false,
+        describe: "Suppress non-error output",
+      })
+      .option("output", {
+        type: "string",
+        choices: ["table", "json"],
+        default: "table",
+        describe: "Output format",
+      })
+  },
+  handler: async (args) => {
+    await rollCallHandler(args)
+  },
+})
+
+interface RollCallResult {
+  model: string
+  access: boolean
+  snippet: string
+  latency: number | null
+  errorType: string | null
+  errorMessage: string | null
+}
+
+export async function rollCallHandler(args: any) {
+  const { prompt, timeout, filter, parallel, output, verbose, quiet } = args
+
+  if (!quiet) {
+    UI.println(`${UI.Style.TEXT_INFO}Starting roll call for models with prompt: "${prompt}"${UI.Style.TEXT_NORMAL}`)
+    UI.println(
+      `${UI.Style.TEXT_INFO}Timeout per model: ${timeout}ms, Parallel calls: ${parallel}${UI.Style.TEXT_NORMAL}`,
+    )
+  }
+
+  await Instance.provide({
+    directory: process.cwd(),
+    async fn() {
+      const providers = await Provider.list()
+      const modelsToTest: { providerID: string; modelID: string; model: Provider.Model }[] = []
+
+      for (const [providerID, provider] of Object.entries(providers)) {
+        for (const [modelID, model] of Object.entries(provider.models)) {
+          const fullName = `${providerID}/${modelID}`
+          if (filter) {
+            try {
+              const regex = new RegExp(filter, "i")
+              if (!regex.test(fullName)) continue
+            } catch (e) {
+              UI.error(`Invalid filter regex: ${filter}`)
+              return
+            }
+          }
+          modelsToTest.push({ providerID, modelID, model })
+        }
+      }
+
+      if (modelsToTest.length === 0) {
+        if (!quiet) UI.println(`${UI.Style.TEXT_WARNING}No models to test after filtering.${UI.Style.TEXT_NORMAL}`)
+        return
+      }
+
+      if (!quiet) {
+        UI.println(`${UI.Style.TEXT_INFO}Prompting ${modelsToTest.length} models...${UI.Style.TEXT_NORMAL}`)
+      }
+
+      const results: RollCallResult[] = []
+      const queue = [...modelsToTest]
+      const activePromises: Promise<void>[] = []
+
+      const processModel = async (item: (typeof modelsToTest)[0]) => {
+        const { providerID, modelID, model } = item
+        const fullName = `${providerID}/${modelID}`
+        const startTime = Date.now()
+        let access = false
+        let snippet = ""
+        let latency: number | null = null
+        let errorType: string | null = null
+        let errorMessage: string | null = null
+
+        try {
+          const languageModel = await Provider.getLanguage(model)
+
+          // Build provider options similar to how session/index.ts does it
+          const sessionID = randomUUID()
+          const baseOptions = ProviderTransform.options({ model, sessionID })
+          const providerOptions = ProviderTransform.providerOptions(model, baseOptions)
+          const maxTokens = ProviderTransform.maxOutputTokens(model)
+          const temperature = ProviderTransform.temperature(model)
+          const topP = ProviderTransform.topP(model)
+          const topK = ProviderTransform.topK(model)
+
+          const { text } = await generateText({
+            model: languageModel,
+            prompt,
+            abortSignal: AbortSignal.timeout(timeout),
+            maxOutputTokens: maxTokens,
+            temperature,
+            topP,
+            topK,
+            providerOptions,
+          })
+          access = true
+          snippet = text.substring(0, 50).replace(/\n/g, " ")
+          latency = Date.now() - startTime
+        } catch (e: any) {
+          latency = Date.now() - startTime
+          if (e instanceof APICallError) {
+            const parsedError = ProviderError.parseAPICallError({
+              providerID,
+              error: e,
+            })
+            errorType = parsedError.type
+            errorMessage = parsedError.message
+          } else {
+            errorType = "unknown"
+            errorMessage = e.message || "An unknown error occurred"
+          }
+        }
+
+        results.push({
+          model: fullName,
+          access,
+          snippet,
+          latency,
+          errorType,
+          errorMessage,
+        })
+
+        if (verbose && !quiet) {
+          if (access) {
+            UI.println(`${UI.Style.TEXT_SUCCESS}✔${UI.Style.TEXT_NORMAL} ${fullName} - ${latency}ms`)
+          } else {
+            UI.println(`${UI.Style.TEXT_DANGER}✘${UI.Style.TEXT_NORMAL} ${fullName} - ${errorType}: ${errorMessage}`)
+          }
+        }
+      }
+
+      while (queue.length > 0 || activePromises.length > 0) {
+        while (queue.length > 0 && activePromises.length < parallel) {
+          const item = queue.shift()!
+          const promise = processModel(item).finally(() => {
+            const index = activePromises.indexOf(promise)
+            if (index > -1) {
+              activePromises.splice(index, 1)
+            }
+          })
+          activePromises.push(promise)
+        }
+        if (activePromises.length > 0) {
+          await Promise.race(activePromises)
+        }
+      }
+
+      if (quiet) return
+
+      if (output === "json") {
+        console.log(JSON.stringify(results, null, 2))
+      } else {
+        const headers = ["Model", "Access", "Snippet", "Latency"]
+
+        const truncate = (text: string, maxLen: number) => {
+          if (maxLen < 10) return text.substring(0, maxLen - 3) + "..."
+          return text.length > maxLen ? text.substring(0, maxLen - 3) + "..." : text
+        }
+
+        const rows = results.map((r) => [
+          r.model,
+          r.access ? "YES" : "NO",
+          r.access ? r.snippet : r.errorMessage ? `(${r.errorMessage})` : "",
+          r.latency !== null ? `${r.latency}ms` : "N/A",
+        ])
+
+        const widths = headers.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)))
+
+        const totalWidth = widths.reduce((a, b) => a + b, 0) + 9
+        const terminalWidth = process.stdout.columns || 120
+
+        if (totalWidth > terminalWidth && widths[2] > 20) {
+          widths[2] = Math.max(20, widths[2] - (totalWidth - terminalWidth))
+        }
+
+        const headerRow = headers.map((h, i) => h.padEnd(widths[i])).join(" | ")
+        UI.println(headerRow)
+        UI.println("-".repeat(headerRow.length))
+
+        rows.forEach((row, idx) => {
+          const result = results[idx]
+          const color = result.access ? UI.Style.TEXT_SUCCESS : UI.Style.TEXT_DANGER
+          const truncatedRow = [row[0], row[1], row[2] ? truncate(row[2], widths[2]) : row[2], row[3]]
+          const line = truncatedRow.map((c, i) => c.padEnd(widths[i])).join(" | ")
+          UI.println(color + line + UI.Style.TEXT_NORMAL)
+        })
+
+        const successful = results.filter((r) => r.access).length
+        const failed = results.length - successful
+        UI.println("")
+        UI.println(
+          `${UI.Style.TEXT_SUCCESS}${successful} accessible${UI.Style.TEXT_NORMAL}, ${UI.Style.TEXT_DANGER}${failed} failed${UI.Style.TEXT_NORMAL}`,
+        )
+      }
+    },
+  })
+}
diff --git a/packages/opencode/src/index.ts b/packages/opencode/src/index.ts
@@ -8,6 +8,7 @@ import { AgentCommand } from "./cli/cmd/agent"
 import { UpgradeCommand } from "./cli/cmd/upgrade"
 import { UninstallCommand } from "./cli/cmd/uninstall"
 import { ModelsCommand } from "./cli/cmd/models"
+import { RollCallCommand } from "./cli/cmd/roll-call"
 import { UI } from "./cli/ui"
 import { Installation } from "./installation"
 import { NamedError } from "@opencode-ai/util/error"
@@ -131,6 +132,7 @@ const cli = yargs(hideBin(process.argv))
   .command(ServeCommand)
   .command(WebCommand)
   .command(ModelsCommand)
+  .command(RollCallCommand)
   .command(StatsCommand)
   .command(ExportCommand)
   .command(ImportCommand)
diff --git a/packages/opencode/src/provider/transform.ts b/packages/opencode/src/provider/transform.ts
@@ -779,6 +779,18 @@ export namespace ProviderTransform {
       result["chat_template_args"] = { enable_thinking: true }
     }
 
+    // kilocode_change - Dedicated thinking models via Kilo Gateway require reasoning to be enabled.
+    // Models with "thinking" in their ID are typically dedicated reasoning variants where
+    // thinking cannot be disabled (e.g., kimi-k2-thinking). Enable reasoning explicitly to
+    // avoid "Reasoning is mandatory for this endpoint" errors from OpenRouter.
+    if (
+      input.model.api.npm === "@kilocode/kilo-gateway" &&
+      input.model.capabilities.reasoning &&
+      input.model.api.id.includes("thinking")
+    ) {
+      result["reasoning"] = { enabled: true }
+    }
+
     if (["zai", "zhipuai"].includes(input.model.providerID) && input.model.api.npm === "@ai-sdk/openai-compatible") {
       result["thinking"] = {
         type: "enabled",
