Jlongster/simulation improvements

packages/core/script/campaign-provider-catalog-race.ts

@@ -0,0 +1,76 @@
+import fs from "node:fs/promises"
+import path from "node:path"
+import { runScenario, type Delay, type Result, type Scenario } from "./provider-catalog-race-harness"
+
+const count = numberArg("--count", 64)
+const seed = numberArg("--seed", 42000)
+const out = path.resolve(stringArg("--out", "/tmp/opencode-provider-catalog-race-campaign"))
+await fs.rm(out, { recursive: true, force: true })
+await fs.mkdir(out, { recursive: true })
+
+const results: Result[] = []
+for (let index = 0; index < count; index++) {
+  const scenario = generate(seed + index, index)
+  const directory = path.join(out, `case-${String(index + 1).padStart(3, "0")}-${seed + index}`)
+  const result = await runScenario(scenario, path.join(directory, "state"))
+  await fs.mkdir(directory, { recursive: true })
+  await fs.writeFile(path.join(directory, "scenario.json"), JSON.stringify(scenario, undefined, 2) + "\n")
+  await fs.writeFile(path.join(directory, "result.json"), JSON.stringify(result, undefined, 2) + "\n")
+  results.push(result)
+  console.log(`[${index + 1}/${count}] ${scenario.id}: ${result.reproduced ? "RACE" : "ok"}`)
+}
+
+const summary = {
+  seed,
+  count,
+  reproduced: results.filter((item) => item.reproduced).length,
+  firstAttemptFailures: results.filter((item) => item.snapshots[0]?.error).length,
+  recoveredOnLaterAttempt: results.filter(
+    (item) => item.snapshots[0]?.error && item.snapshots.slice(1).some((snapshot) => snapshot.resolved),
+  ).length,
+  byDelay: Object.fromEntries(
+    (["none", "yield", "1ms", "10ms"] as Delay[]).map((delay) => [
+      delay,
+      {
+        count: results.filter((item) => item.scenario.delay === delay).length,
+        reproduced: results.filter((item) => item.scenario.delay === delay && item.reproduced).length,
+      },
+    ]),
+  ),
+}
+await fs.writeFile(path.join(out, "summary.json"), JSON.stringify(summary, undefined, 2) + "\n")
+console.log(JSON.stringify(summary, undefined, 2))
+console.log(`Artifacts: ${out}`)
+process.exitCode = summary.reproduced > 0 ? 1 : 0
+
+function generate(value: number, index: number): Scenario {
+  const random = rng(value)
+  const delays: Delay[] = ["none", "yield", "1ms", "10ms"]
+  return {
+    id: `seed-${value}`,
+    delay: delays[index % delays.length]!,
+    providerID: `provider-${index % 8}`,
+    modelID: `model-${Math.floor(random() * 8)}`,
+    configuredDefault: random() < 0.75,
+    apiKey: true,
+    disabled: random() < 0.1,
+    repeats: 3,
+  }
+}
+
+function rng(seed: number) {
+  let state = seed >>> 0
+  return () => {
+    state = (Math.imul(state, 1664525) + 1013904223) >>> 0
+    return state / 0x1_0000_0000
+  }
+}
+
+function stringArg(name: string, fallback: string) {
+  const index = process.argv.indexOf(name)
+  return index < 0 ? fallback : (process.argv[index + 1] ?? fallback)
+}
+
+function numberArg(name: string, fallback: number) {
+  return Number(stringArg(name, String(fallback)))
+}

packages/core/script/provider-catalog-race-harness.ts

@@ -0,0 +1,150 @@
+import fs from "node:fs/promises"
+import os from "node:os"
+import path from "node:path"
+import { DateTime, Effect, Layer } from "effect"
+import { AppNodeBuilder } from "../src/effect/app-node-builder"
+import { LayerNode } from "../src/effect/layer-node"
+import { Database } from "../src/database/database"
+import { EventV2 } from "../src/event"
+import { Catalog } from "../src/catalog"
+import { Location } from "../src/location"
+import { LocationServiceMap } from "../src/location-services"
+import { ModelV2 } from "../src/model"
+import { ProjectV2 } from "../src/project"
+import { ProviderV2 } from "../src/provider"
+import { AbsolutePath } from "../src/schema"
+import { SessionV2 } from "../src/session"
+import { SessionRunnerModel } from "../src/session/runner/model"
+
+export type Delay = "none" | "yield" | "1ms" | "10ms"
+
+export interface Scenario {
+  id: string
+  delay: Delay
+  providerID: string
+  modelID: string
+  configuredDefault: boolean
+  apiKey: boolean
+  disabled: boolean
+  repeats: number
+}
+
+export interface Snapshot {
+  attempt: number
+  providers: string[]
+  models: string[]
+  availableModels: string[]
+  resolved?: { providerID: string; modelID: string }
+  error?: { tag: string; message: string }
+}
+
+export interface Result {
+  scenario: Scenario
+  directory: string
+  snapshots: Snapshot[]
+  reproduced: boolean
+}
+
+const app = AppNodeBuilder.build(LayerNode.group([Database.node, EventV2.node, LocationServiceMap.node]))
+
+export async function runScenario(scenario: Scenario, root?: string): Promise<Result> {
+  const directory = root ?? (await fs.mkdtemp(path.join(os.tmpdir(), "opencode-provider-race-")))
+  await fs.mkdir(directory, { recursive: true })
+  await fs.writeFile(path.join(directory, "opencode.json"), JSON.stringify(config(scenario), undefined, 2) + "\n")
+  const location = Location.Ref.make({ directory: AbsolutePath.make(directory) })
+
+  const program = Effect.gen(function* () {
+    const locations = yield* LocationServiceMap.Service
+    const context = yield* locations.contextEffect(location)
+    const output: Snapshot[] = []
+    for (let attempt = 0; attempt < scenario.repeats; attempt++) {
+      yield* delay(scenario.delay)
+      output.push(
+        yield* Effect.gen(function* () {
+          const catalog = yield* Catalog.Service
+          const providers = (yield* catalog.provider.all()).map((item) => item.id).sort()
+          const models = (yield* catalog.model.all()).map((item) => `${item.providerID}/${item.id}`).sort()
+          const availableModels = (yield* catalog.model.available())
+            .map((item) => `${item.providerID}/${item.id}`)
+            .sort()
+          const resolved = yield* SessionRunnerModel.Service.use((service) =>
+            service.resolve(session(scenario, location)),
+          ).pipe(
+            Effect.map((item) => ({ providerID: String(item.ref.providerID), modelID: String(item.ref.id) })),
+            Effect.catch((error) =>
+              Effect.succeed({
+                error: {
+                  tag: typeof error === "object" && error && "_tag" in error ? String(error._tag) : "Unknown",
+                  message: error instanceof Error ? error.message : String(error),
+                },
+              }),
+            ),
+          )
+          return {
+            attempt,
+            providers,
+            models,
+            availableModels,
+            ...(resolved && "error" in resolved ? { error: resolved.error } : { resolved }),
+          }
+        }).pipe(Effect.provide(context)),
+      )
+    }
+    return output
+  }).pipe(Effect.scoped, Effect.provide(app))
+  const snapshots = await Effect.runPromise(program)
+
+  const expected = `${scenario.providerID}/${scenario.modelID}`
+  return {
+    scenario,
+    directory,
+    snapshots,
+    reproduced: snapshots.some(
+      (item) => !item.models.includes(expected) || item.error?.tag === "SessionRunnerModel.ModelUnavailableError",
+    ),
+  }
+}
+
+function config(scenario: Scenario) {
+  return {
+    ...(scenario.configuredDefault ? { model: `${scenario.providerID}/${scenario.modelID}` } : {}),
+    providers: {
+      [scenario.providerID]: {
+        name: `Probe ${scenario.providerID}`,
+        api: { type: "aisdk", package: "@ai-sdk/openai-compatible", url: "https://probe.invalid/v1" },
+        request: { body: scenario.apiKey ? { apiKey: "probe-key" } : {} },
+        models: {
+          [scenario.modelID]: {
+            name: `Probe ${scenario.modelID}`,
+            disabled: scenario.disabled,
+            capabilities: { tools: true, input: ["text"], output: ["text"] },
+            limit: { context: 8192, output: 2048 },
+          },
+        },
+      },
+    },
+  }
+}
+
+function session(scenario: Scenario, location: Location.Ref) {
+  return SessionV2.Info.make({
+    id: SessionV2.ID.make(`ses_${scenario.id.replace(/[^a-zA-Z0-9_-]/g, "_")}`),
+    projectID: ProjectV2.ID.global,
+    title: scenario.id,
+    model: {
+      providerID: ProviderV2.ID.make(scenario.providerID),
+      id: ModelV2.ID.make(scenario.modelID),
+    },
+    cost: 0,
+    tokens: { input: 0, output: 0, reasoning: 0, cache: { read: 0, write: 0 } },
+    time: { created: DateTime.makeUnsafe(0), updated: DateTime.makeUnsafe(0) },
+    location,
+  })
+}
+
+function delay(value: Delay) {
+  if (value === "yield") return Effect.yieldNow
+  if (value === "1ms") return Effect.sleep("1 millis")
+  if (value === "10ms") return Effect.sleep("10 millis")
+  return Effect.void
+}

packages/core/script/reproduce-provider-catalog-race.ts

@@ -0,0 +1,37 @@
+import fs from "node:fs/promises"
+import path from "node:path"
+import { runScenario } from "./provider-catalog-race-harness"
+
+const out = path.resolve(process.env.OPENCODE_PROBE_OUT ?? "/tmp/opencode-provider-catalog-race-repro")
+await fs.rm(out, { recursive: true, force: true })
+await fs.mkdir(out, { recursive: true })
+
+const result = await runScenario(
+  {
+    id: "cold-immediate",
+    delay: "none",
+    providerID: "console-openai",
+    modelID: "gpt-5.6-sol",
+    configuredDefault: true,
+    apiKey: true,
+    disabled: false,
+    repeats: 2,
+  },
+  path.join(out, "state"),
+)
+
+await fs.writeFile(path.join(out, "result.json"), JSON.stringify(result, undefined, 2) + "\n")
+for (const snapshot of result.snapshots) {
+  console.log(
+    JSON.stringify({
+      attempt: snapshot.attempt,
+      providers: snapshot.providers,
+      models: snapshot.models,
+      resolved: snapshot.resolved,
+      error: snapshot.error,
+    }),
+  )
+}
+console.log(`${result.reproduced ? "REPRODUCED" : "NOT REPRODUCED"}: cold location provider catalog startup race`)
+console.log(`Artifacts: ${out}`)
+process.exitCode = result.reproduced ? 1 : 0

packages/core/src/config.ts

@@ -2,6 +2,7 @@ export * as Config from "./config"
 
 import { makeLocationNode } from "./effect/app-node"
 import path from "path"
+import nodeFs from "fs"
 import { type ParseError, parse } from "jsonc-parser"
 import { Context, Effect, Layer, Option, Schema } from "effect"
 import { Permission } from "@opencode-ai/schema/permission"
@@ -26,6 +27,38 @@ import { ConfigWatcher } from "./config/watcher"
 import { ConfigV1 } from "./v1/config/config"
 import { ConfigMigrateV1 } from "./v1/config/migrate"
 
+function simulationLog(type: string, data?: unknown) {
+  if (!process.env.OPENCODE_SIMULATION) return
+  try {
+    const file = process.env.OPENCODE_SIMULATION_LOG || "/tmp/opencode-simulation.log"
+    nodeFs.mkdirSync(path.dirname(file), { recursive: true })
+    nodeFs.appendFileSync(file, JSON.stringify({ time: new Date().toISOString(), pid: process.pid, type, data }) + "\n")
+  } catch {
+    return
+  }
+}
+
+function asRecord(input: unknown): Record<string, unknown> | undefined {
+  return typeof input === "object" && input !== null && !Array.isArray(input) ? (input as Record<string, unknown>) : undefined
+}
+
+function configShape(input: unknown) {
+  const record = asRecord(input)
+  const providers = asRecord(record?.providers)
+  return {
+    keys: Object.keys(record ?? {}).sort(),
+    model: typeof record?.model === "string" ? record.model : undefined,
+    default_agent: typeof record?.default_agent === "string" ? record.default_agent : undefined,
+    providers: Object.keys(providers ?? {}).sort(),
+    providerModels: Object.fromEntries(
+      Object.entries(providers ?? {}).map(([id, provider]) => [
+        id,
+        Object.keys(asRecord(asRecord(provider)?.models) ?? {}).sort(),
+      ]),
+    ),
+  }
+}
+
 export class Info extends Schema.Class<Info>("Config.Info")({
   $schema: Schema.optional(Schema.String).annotate({
     description: "JSON schema reference for configuration validation",
@@ -146,18 +179,28 @@ const layer = Layer.effect(
 
     const loadFile = Effect.fnUntraced(function* (filepath: string) {
       const text = yield* fs.readFileStringSafe(filepath)
+      simulationLog("config.file.read", { filepath, found: text !== undefined, bytes: text?.length })
       if (!text) return
 
       const errors: ParseError[] = []
       const input: unknown = parse(text, errors, { allowTrailingComma: true })
-      if (errors.length) return
-
-      const info = Option.getOrUndefined(
-        ConfigMigrateV1.isV1(input)
-          ? decodeV1Info(input).pipe(Option.map(ConfigMigrateV1.migrate), Option.flatMap(decodeInfo))
-          : decodeInfo(input),
-      )
-      if (!info) return
+      if (errors.length) {
+        simulationLog("config.file.parse.error", {
+          filepath,
+          errors: errors.map((error) => ({ error: error.error, offset: error.offset, length: error.length })),
+        })
+        return
+      }
+
+      const decoded = ConfigMigrateV1.isV1(input)
+        ? decodeV1Info(input).pipe(Option.map(ConfigMigrateV1.migrate), Option.flatMap(decodeInfo))
+        : decodeInfo(input)
+      const info = Option.getOrUndefined(decoded)
+      if (!info) {
+        simulationLog("config.file.decode.error", { filepath, shape: configShape(input) })
+        return
+      }
+      simulationLog("config.file.loaded", { filepath, v1: ConfigMigrateV1.isV1(input), shape: configShape(info) })
       return new Document({ type: "document", path: filepath, info })
     })
 
@@ -172,6 +215,12 @@ const layer = Layer.effect(
 
     const globalDirectory = AbsolutePath.make(global.config)
     const locationIsGlobal = path.resolve(location.directory) === path.resolve(global.config)
+    simulationLog("config.service.start", {
+      global: global.config,
+      location: location.directory,
+      project: location.project.directory,
+      locationIsGlobal,
+    })
     // Read configuration once when this location opens. Later calls reuse these
     // values until the location is reopened.
     const discovered = locationIsGlobal
@@ -183,6 +232,7 @@ const layer = Layer.effect(
             stop: location.project.directory,
           })
           .pipe(Effect.orDie)
+    simulationLog("config.discovered", { discovered })
     const directories = [
       globalDirectory,
       ...discovered
@@ -201,6 +251,11 @@ const layer = Layer.effect(
     // Apply general settings first and more specific settings last:
     // global config, project files, then `.opencode` files.
     const configs = [...(supplementary[0] ?? []), ...direct, ...supplementary.slice(1).flat()]
+    simulationLog("config.entries", {
+      entries: configs.map((entry) =>
+        entry.type === "document" ? { type: entry.type, path: entry.path, shape: configShape(entry.info) } : entry,
+      ),
+    })
     // Rules use the opposite order so a user-global rule can override a
     // repository rule. Statement order inside each file stays unchanged.
     yield* policy.load(