diff --git a/README.md b/README.md index ab69eff6..500bca43 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ A **TUI + CLI** for the [Runloop.ai](https://runloop.ai) platform. Use it as an **interactive TUI** (Terminal User Interface) with rich UI components, or as a **traditional CLI** for scripting and automation. -📖 **[Full Documentation](https://docs.runloop.ai/docs/tools/cli)** +📖 **[Full Documentation](https://docs.runloop.ai/docs/tools/rl-cli)**

Runloop CLI Demo diff --git a/scripts/generate-command-docs.js b/scripts/generate-command-docs.js index 148377bd..85e14aa8 100644 --- a/scripts/generate-command-docs.js +++ b/scripts/generate-command-docs.js @@ -11,7 +11,7 @@ const rootDir = join(__dirname, ".."); const readmePath = join(rootDir, "README.md"); // Default docs path - can be overridden via DOCS_PATH env var or --docs-path argument -const defaultDocsPath = join(rootDir, "..", "docs", "docs", "tools", "cli.mdx"); +const defaultDocsPath = join(rootDir, "..", "docs", "docs", "tools", "rl-cli.mdx"); /** * Generates markdown documentation for the command structure from Commander @@ -120,7 +120,7 @@ function generateCommandStructure(program) { } /** - * Generates a detailed command reference for external docs (cli.mdx) + * Generates a detailed command reference for external docs (rl-cli.mdx) * Uses Mintlify components for better presentation */ function generateDetailedCommandDocs(program) { @@ -286,7 +286,7 @@ function updateReadme(newCommandStructure) { } /** - * Generates the full cli.mdx content + * Generates the full rl-cli.mdx content */ function generateCliMdx(program) { const commandDocs = generateDetailedCommandDocs(program); @@ -508,7 +508,7 @@ The Runloop CLI is open-source. We welcome contributions! } /** - * Updates the external docs cli.mdx file + * Updates the external docs rl-cli.mdx file */ function updateDocsMdx(program, docsPath) { if (!existsSync(docsPath)) { @@ -547,9 +547,9 @@ function parseArgs() { Usage: generate-command-docs.js [options] Options: - --docs-path Path to cli.mdx file (default: ../docs/docs/tools/cli.mdx) + --docs-path Path to rl-cli.mdx file (default: ../docs/docs/tools/rl-cli.mdx) --skip-readme Skip updating README.md - --skip-docs Skip updating cli.mdx + --skip-docs Skip updating rl-cli.mdx --help, -h Show this help message Environment Variables: diff --git a/src/screens/BenchmarkJobCreateScreen.tsx b/src/screens/BenchmarkJobCreateScreen.tsx index 5fe5b466..84249f33 100644 --- a/src/screens/BenchmarkJobCreateScreen.tsx +++ b/src/screens/BenchmarkJobCreateScreen.tsx @@ -15,17 +15,25 @@ import { NavigationTips } from "../components/NavigationTips.js"; import { ResourcePicker } from "../components/ResourcePicker.js"; import { colors } from "../utils/theme.js"; import { useExitOnCtrlC } from "../hooks/useExitOnCtrlC.js"; -import { listBenchmarks } from "../services/benchmarkService.js"; +import { listBenchmarks, getBenchmark } from "../services/benchmarkService.js"; +import { + listScenarios, + getScenario, + type Scenario, +} from "../services/scenarioService.js"; import { listAgents, type Agent } from "../services/agentService.js"; import { createBenchmarkJob, type BenchmarkJob, type AgentConfig, + type OrchestratorConfig, } from "../services/benchmarkJobService.js"; import type { Benchmark } from "../store/benchmarkStore.js"; type FormField = + | "source_type" | "benchmark" + | "scenarios" | "agents" | "name" | "agent_timeout" @@ -33,8 +41,11 @@ type FormField = | "create"; interface FormData { + sourceType: "benchmark" | "scenarios"; benchmarkId: string; benchmarkName: string; + scenarioIds: string[]; + scenarioNames: string[]; agentIds: string[]; agentNames: string[]; name: string; @@ -45,6 +56,7 @@ interface FormData { type ScreenState = | "form" | "picking_benchmark" + | "picking_scenarios" | "picking_agents" | "creating" | "success" @@ -52,6 +64,17 @@ type ScreenState = interface BenchmarkJobCreateScreenProps { initialBenchmarkIds?: string; + initialScenarioIds?: string; + cloneFromJobId?: string; + cloneJobName?: string; + cloneSourceType?: "benchmark" | "scenarios"; + cloneAgentConfigs?: string; // JSON serialized AgentConfig[] + cloneOrchestratorConfig?: string; // JSON serialized OrchestratorConfig + // Legacy props for backward compatibility + cloneAgentIds?: string; + cloneAgentNames?: string; + cloneAgentTimeout?: string; + cloneConcurrentTrials?: string; } /** @@ -122,49 +145,127 @@ function SuccessScreen({ export function BenchmarkJobCreateScreen({ initialBenchmarkIds, + initialScenarioIds, + cloneFromJobId, + cloneJobName, + cloneSourceType, + cloneAgentConfigs, + cloneOrchestratorConfig, + cloneAgentIds, + cloneAgentNames, + cloneAgentTimeout, + cloneConcurrentTrials, }: BenchmarkJobCreateScreenProps) { const { navigate, goBack } = useNavigation(); + // Determine initial source type and field + const initialSourceType: "benchmark" | "scenarios" = + cloneSourceType || (initialScenarioIds ? "scenarios" : "benchmark"); + + const initialField: FormField = + initialBenchmarkIds || initialScenarioIds ? "agents" : "source_type"; + const [screenState, setScreenState] = React.useState("form"); const [currentField, setCurrentField] = - React.useState("benchmark"); + React.useState(initialField); + const [formData, setFormData] = React.useState({ + sourceType: initialSourceType, benchmarkId: initialBenchmarkIds || "", benchmarkName: "", - agentIds: [], - agentNames: [], - name: "", - agentTimeout: "", - concurrentTrials: "1", + scenarioIds: initialScenarioIds ? initialScenarioIds.split(",") : [], + scenarioNames: [], + agentIds: cloneAgentIds ? cloneAgentIds.split(",") : [], + agentNames: cloneAgentNames ? cloneAgentNames.split(",") : [], + name: cloneJobName ? `${cloneJobName} (clone)` : "", + agentTimeout: cloneAgentTimeout || "", + concurrentTrials: cloneConcurrentTrials || "1", }); + const [createdJob, setCreatedJob] = React.useState(null); const [error, setError] = React.useState(null); // Handle Ctrl+C to exit useExitOnCtrlC(); - // Field definitions + // Fetch benchmark name if we have an ID (from clone or initial selection) + React.useEffect(() => { + if (initialBenchmarkIds && !formData.benchmarkName) { + getBenchmark(initialBenchmarkIds) + .then((benchmark) => { + setFormData((prev) => ({ + ...prev, + benchmarkName: benchmark.name || benchmark.id, + })); + }) + .catch((err) => { + // Silently fail - user can re-select if needed + console.error("Failed to fetch benchmark name:", err); + }); + } + }, [initialBenchmarkIds, formData.benchmarkName]); + + // Fetch scenario names if we have IDs (from clone or initial selection) + React.useEffect(() => { + if ( + initialScenarioIds && + formData.scenarioIds.length > 0 && + formData.scenarioNames.length === 0 + ) { + // Fetch all scenarios to get their names + Promise.all( + formData.scenarioIds.map((id) => + getScenario(id).catch((err) => { + console.error(`Failed to fetch scenario ${id}:`, err); + return { id, name: id } as Scenario; + }), + ), + ).then((scenarios) => { + setFormData((prev) => ({ + ...prev, + scenarioNames: scenarios.map((s) => s.name || s.id), + })); + }); + } + }, [initialScenarioIds, formData.scenarioIds, formData.scenarioNames]); + + // Field definitions - conditionally include benchmark or scenarios based on source type const fields: Array<{ key: FormField; label: string; - type: "text" | "picker" | "action"; + type: "text" | "picker" | "action" | "toggle"; placeholder?: string; required?: boolean; description?: string; }> = [ { - key: "benchmark", - label: "Benchmark", - type: "picker", + key: "source_type", + label: "Source Type", + type: "toggle", required: true, - description: "Select a benchmark definition to run", + description: "Choose between benchmark or scenarios", }, + formData.sourceType === "benchmark" + ? { + key: "benchmark", + label: "Benchmark", + type: "picker" as const, + required: true, + description: "Select a benchmark definition to run", + } + : { + key: "scenarios", + label: "Scenarios", + type: "picker" as const, + required: true, + description: "Select one or more scenario definitions to run", + }, { key: "agents", label: "Agents", type: "picker", required: true, - description: "Select one or more agents to run the benchmark", + description: "Select one or more agents to run", }, { key: "name", @@ -200,7 +301,10 @@ export function BenchmarkJobCreateScreen({ // Check if form is valid const isFormValid = - formData.benchmarkId !== "" && formData.agentIds.length > 0; + ((formData.sourceType === "benchmark" && formData.benchmarkId !== "") || + (formData.sourceType === "scenarios" && + formData.scenarioIds.length > 0)) && + formData.agentIds.length > 0; // Memoize the fetchBenchmarksPage function const fetchBenchmarksPage = React.useCallback( @@ -245,6 +349,23 @@ export function BenchmarkJobCreateScreen({ [], ); + // Memoize the fetchScenariosPage function + const fetchScenariosPage = React.useCallback( + async (params: { limit: number; startingAt?: string; search?: string }) => { + const result = await listScenarios({ + limit: params.limit, + startingAfter: params.startingAt, + search: params.search, + }); + return { + items: result.scenarios, + hasMore: result.hasMore, + totalCount: result.totalCount, + }; + }, + [], + ); + // Memoize benchmark picker config (single-select) const benchmarkPickerConfig = React.useMemo( () => ({ @@ -268,6 +389,30 @@ export function BenchmarkJobCreateScreen({ [fetchBenchmarksPage], ); + // Memoize scenario picker config (multi-select) + const scenarioPickerConfig = React.useMemo( + () => ({ + title: "Select Scenarios", + fetchPage: fetchScenariosPage, + getItemId: (scenario: Scenario) => scenario.id, + getItemLabel: (scenario: Scenario) => scenario.name || scenario.id, + getItemStatus: (scenario: Scenario) => + scenario.is_public ? "public" : "private", + mode: "multi" as const, + minSelection: 1, + emptyMessage: "No scenarios found", + searchPlaceholder: "Search scenarios...", + breadcrumbItems: [ + { label: "Home" }, + { label: "Benchmarks" }, + { label: "Jobs" }, + { label: "Create" }, + { label: "Select Scenarios", active: true }, + ], + }), + [fetchScenariosPage], + ); + // Memoize agent picker config (multi-select) const agentPickerConfig = React.useMemo( () => ({ @@ -304,6 +449,16 @@ export function BenchmarkJobCreateScreen({ setScreenState("form"); }, []); + // Handle scenario selection (multi) + const handleScenarioSelect = React.useCallback((items: Scenario[]) => { + setFormData((prev) => ({ + ...prev, + scenarioIds: items.map((s) => s.id), + scenarioNames: items.map((s) => s.name || s.id), + })); + setScreenState("form"); + }, []); + // Handle agent selection (multi) const handleAgentSelect = React.useCallback((items: Agent[]) => { setFormData((prev) => ({ @@ -322,9 +477,14 @@ export function BenchmarkJobCreateScreen({ setError(null); try { - // Build agent configs for each selected agent - const agentConfigs: AgentConfig[] = formData.agentIds.map( - (agentId, index) => { + // Use cloned agent configs if available, otherwise build from form + let agentConfigs: AgentConfig[]; + if (cloneAgentConfigs) { + // Use the full cloned configs + agentConfigs = JSON.parse(cloneAgentConfigs); + } else { + // Build agent configs from form data (backward compatibility) + agentConfigs = formData.agentIds.map((agentId, index) => { const config: AgentConfig = { name: formData.agentNames[index], agentId: agentId, @@ -338,18 +498,31 @@ export function BenchmarkJobCreateScreen({ } return config; - }, - ); + }); + } + + // Use cloned orchestrator config if available, otherwise build from form + let orchestratorConfig: OrchestratorConfig | undefined; + if (cloneOrchestratorConfig) { + orchestratorConfig = JSON.parse(cloneOrchestratorConfig); + } else if (formData.concurrentTrials) { + orchestratorConfig = { + nConcurrentTrials: parseInt(formData.concurrentTrials, 10) || 1, + }; + } const job = await createBenchmarkJob({ name: formData.name || undefined, - benchmarkId: formData.benchmarkId, + benchmarkId: + formData.sourceType === "benchmark" + ? formData.benchmarkId + : undefined, + scenarioIds: + formData.sourceType === "scenarios" + ? formData.scenarioIds + : undefined, agentConfigs, - orchestratorConfig: formData.concurrentTrials - ? { - nConcurrentTrials: parseInt(formData.concurrentTrials, 10) || 1, - } - : undefined, + orchestratorConfig, }); setCreatedJob(job); @@ -358,12 +531,28 @@ export function BenchmarkJobCreateScreen({ setError(err as Error); setScreenState("error"); } - }, [formData, isFormValid]); + }, [formData, isFormValid, cloneAgentConfigs, cloneOrchestratorConfig]); // Handle input useInput((input, key) => { if (screenState !== "form") return; + // Handle source type toggle with left/right arrows + if (currentField === "source_type" && (key.leftArrow || key.rightArrow)) { + setFormData((prev) => ({ + ...prev, + sourceType: prev.sourceType === "benchmark" ? "scenarios" : "benchmark", + // Clear the other source when switching + benchmarkId: prev.sourceType === "scenarios" ? "" : prev.benchmarkId, + benchmarkName: + prev.sourceType === "scenarios" ? "" : prev.benchmarkName, + scenarioIds: prev.sourceType === "benchmark" ? [] : prev.scenarioIds, + scenarioNames: + prev.sourceType === "benchmark" ? [] : prev.scenarioNames, + })); + return; + } + // Navigate between fields if (key.upArrow && currentFieldIndex > 0) { setCurrentField(fieldKeys[currentFieldIndex - 1]); @@ -374,6 +563,11 @@ export function BenchmarkJobCreateScreen({ } else if (key.return) { if (currentFieldDef?.type === "picker" && currentField === "benchmark") { setScreenState("picking_benchmark"); + } else if ( + currentFieldDef?.type === "picker" && + currentField === "scenarios" + ) { + setScreenState("picking_scenarios"); } else if ( currentFieldDef?.type === "picker" && currentField === "agents" @@ -403,6 +597,18 @@ export function BenchmarkJobCreateScreen({ ); } + // Show scenario picker (multi-select) + if (screenState === "picking_scenarios") { + return ( + + config={scenarioPickerConfig} + onSelect={handleScenarioSelect} + onCancel={() => setScreenState("form")} + initialSelected={formData.scenarioIds} + /> + ); + } + // Show agent picker (multi-select) if (screenState === "picking_agents") { return ( @@ -481,8 +687,21 @@ export function BenchmarkJobCreateScreen({ // Helper to get display value for a field const getFieldValue = (fieldKey: FormField): string => { switch (fieldKey) { + case "source_type": + return formData.sourceType === "benchmark" ? "Benchmark" : "Scenarios"; case "benchmark": return formData.benchmarkName; + case "scenarios": + // Show count based on IDs even if names aren't loaded yet + if (formData.scenarioIds.length === 0) return ""; + if (formData.scenarioIds.length === 1) { + return formData.scenarioNames[0] || formData.scenarioIds[0]; + } + // If we have names, show the first name + count, otherwise show count + if (formData.scenarioNames.length > 0) { + return `${formData.scenarioNames.length} scenarios selected`; + } + return `${formData.scenarioIds.length} scenarios selected`; case "agents": if (formData.agentNames.length === 0) return ""; if (formData.agentNames.length === 1) return formData.agentNames[0]; @@ -499,6 +718,8 @@ export function BenchmarkJobCreateScreen({ }; // Main form view + const isCloning = !!cloneFromJobId; + return ( <> - {figures.pointer} Create Benchmark Job + {figures.pointer}{" "} + {isCloning ? "Clone Benchmark Job" : "Create Benchmark Job"} @@ -529,7 +751,44 @@ export function BenchmarkJobCreateScreen({ - {field.type === "action" ? ( + {field.type === "toggle" ? ( + + + {field.label} + {field.required && *} + :{" "} + + {/* Toggle between Benchmark and Scenarios */} + + {isSelected ? figures.arrowLeft : ""}{" "} + + + Benchmark + + / + + Scenarios + + + {" "} + {isSelected ? figures.arrowRight : ""} + + + ) : field.type === "action" ? ( {" "} {figures.play} {field.label}{" "} - {!isFormValid && "(select benchmark and agents)"} + {!isFormValid && "(select benchmark/scenarios and agents)"} ) : field.type === "picker" ? ( diff --git a/src/screens/BenchmarkJobDetailScreen.tsx b/src/screens/BenchmarkJobDetailScreen.tsx index 680602c0..76b49747 100644 --- a/src/screens/BenchmarkJobDetailScreen.tsx +++ b/src/screens/BenchmarkJobDetailScreen.tsx @@ -532,10 +532,10 @@ export function BenchmarkJobDetailScreen({ }); }); - // Always add create new job option + // Always add clone job option operations.push({ - key: "create-new", - label: "Create New Job", + key: "clone-job", + label: "Clone Job", color: colors.success, icon: figures.play, shortcut: "c", @@ -550,8 +550,74 @@ export function BenchmarkJobDetailScreen({ benchmarkRunId: benchmarkRunIds[idx].id, }); } - } else if (operation === "create-new") { - navigate("benchmark-job-create"); + } else if (operation === "clone-job") { + // Pass job data for cloning + const cloneParams: any = { + cloneFromJobId: resource.id, + cloneJobName: resource.name, + }; + + // Determine source type and extract IDs + if (resource.job_spec) { + const spec = resource.job_spec as any; + + // Check if it's a scenarios spec (has scenario_ids array) + if (spec.scenario_ids && Array.isArray(spec.scenario_ids)) { + cloneParams.cloneSourceType = "scenarios"; + cloneParams.initialScenarioIds = spec.scenario_ids.join(","); + } + // Check if it's a benchmark spec (has benchmark_id) + else if (spec.benchmark_id) { + cloneParams.cloneSourceType = "benchmark"; + cloneParams.initialBenchmarkIds = spec.benchmark_id; + } + // Fallback: check job_source + else if (resource.job_source) { + const source = resource.job_source as any; + if (source.scenario_ids && Array.isArray(source.scenario_ids)) { + cloneParams.cloneSourceType = "scenarios"; + cloneParams.initialScenarioIds = source.scenario_ids.join(","); + } else if (source.benchmark_id) { + cloneParams.cloneSourceType = "benchmark"; + cloneParams.initialBenchmarkIds = source.benchmark_id; + } + } + } + + // Extract agent configs - both full configs and legacy fields + if (resource.job_spec?.agent_configs) { + const agentConfigs = resource.job_spec.agent_configs.map((a: any) => ({ + agentId: a.agent_id, + name: a.name, + modelName: a.model_name, + timeoutSeconds: a.timeout_seconds, + kwargs: a.kwargs, + environmentVariables: a.agent_environment?.environment_variables, + secrets: a.agent_environment?.secrets, + })); + cloneParams.cloneAgentConfigs = JSON.stringify(agentConfigs); + + // Also extract legacy fields for form initialization + cloneParams.cloneAgentIds = resource.job_spec.agent_configs + .map((a: any) => a.agent_id) + .join(","); + cloneParams.cloneAgentNames = resource.job_spec.agent_configs + .map((a: any) => a.name) + .join(","); + } + + // Extract orchestrator config + if (resource.job_spec?.orchestrator_config) { + const orch = resource.job_spec.orchestrator_config; + cloneParams.cloneOrchestratorConfig = JSON.stringify({ + nAttempts: orch.n_attempts, + nConcurrentTrials: orch.n_concurrent_trials, + quiet: orch.quiet, + timeoutMultiplier: orch.timeout_multiplier, + }); + } + + navigate("benchmark-job-create", cloneParams); } }; diff --git a/src/screens/BenchmarkJobListScreen.tsx b/src/screens/BenchmarkJobListScreen.tsx index e5009081..83475e0b 100644 --- a/src/screens/BenchmarkJobListScreen.tsx +++ b/src/screens/BenchmarkJobListScreen.tsx @@ -155,8 +155,8 @@ export function BenchmarkJobListScreen() { icon: figures.pointer, }, { - key: "create_new", - label: "Create New Job", + key: "clone_job", + label: "Clone Job", color: colors.success, icon: figures.play, }, @@ -281,17 +281,154 @@ export function BenchmarkJobListScreen() { navigate("benchmark-job-detail", { benchmarkJobId: selectedJob.id, }); - } else if (operationKey === "create_new") { - navigate("benchmark-job-create"); + } else if (operationKey === "clone_job" && selectedJob) { + // Pass job data for cloning + const cloneParams: any = { + cloneFromJobId: selectedJob.id, + cloneJobName: selectedJob.name, + }; + + // Determine source type and extract IDs + if (selectedJob.job_spec) { + const spec = selectedJob.job_spec as any; + + // Check if it's a scenarios spec (has scenario_ids array) + if (spec.scenario_ids && Array.isArray(spec.scenario_ids)) { + cloneParams.cloneSourceType = "scenarios"; + cloneParams.initialScenarioIds = spec.scenario_ids.join(","); + } + // Check if it's a benchmark spec (has benchmark_id) + else if (spec.benchmark_id) { + cloneParams.cloneSourceType = "benchmark"; + cloneParams.initialBenchmarkIds = spec.benchmark_id; + } + // Fallback: check job_source + else if (selectedJob.job_source) { + const source = selectedJob.job_source as any; + if (source.scenario_ids && Array.isArray(source.scenario_ids)) { + cloneParams.cloneSourceType = "scenarios"; + cloneParams.initialScenarioIds = source.scenario_ids.join(","); + } else if (source.benchmark_id) { + cloneParams.cloneSourceType = "benchmark"; + cloneParams.initialBenchmarkIds = source.benchmark_id; + } + } + } + + // Extract agent configs - both full configs and legacy fields + if (selectedJob.job_spec?.agent_configs) { + const agentConfigs = selectedJob.job_spec.agent_configs.map( + (a: any) => ({ + agentId: a.agent_id, + name: a.name, + modelName: a.model_name, + timeoutSeconds: a.timeout_seconds, + kwargs: a.kwargs, + environmentVariables: + a.agent_environment?.environment_variables, + secrets: a.agent_environment?.secrets, + }), + ); + cloneParams.cloneAgentConfigs = JSON.stringify(agentConfigs); + + // Also extract legacy fields for form initialization + cloneParams.cloneAgentIds = selectedJob.job_spec.agent_configs + .map((a: any) => a.agent_id) + .join(","); + cloneParams.cloneAgentNames = selectedJob.job_spec.agent_configs + .map((a: any) => a.name) + .join(","); + } + + // Extract orchestrator config + if (selectedJob.job_spec?.orchestrator_config) { + const orch = selectedJob.job_spec.orchestrator_config; + cloneParams.cloneOrchestratorConfig = JSON.stringify({ + nAttempts: orch.n_attempts, + nConcurrentTrials: orch.n_concurrent_trials, + quiet: orch.quiet, + timeoutMultiplier: orch.timeout_multiplier, + }); + } + + navigate("benchmark-job-create", cloneParams); } } else if (input === "v" && selectedJob) { setShowPopup(false); navigate("benchmark-job-detail", { benchmarkJobId: selectedJob.id, }); - } else if (input === "n") { + } else if (input === "n" && selectedJob) { setShowPopup(false); - navigate("benchmark-job-create"); + // Clone the selected job + const cloneParams: any = { + cloneFromJobId: selectedJob.id, + cloneJobName: selectedJob.name, + }; + + // Determine source type and extract IDs + if (selectedJob.job_spec) { + const spec = selectedJob.job_spec as any; + + // Check if it's a scenarios spec (has scenario_ids array) + if (spec.scenario_ids && Array.isArray(spec.scenario_ids)) { + cloneParams.cloneSourceType = "scenarios"; + cloneParams.initialScenarioIds = spec.scenario_ids.join(","); + } + // Check if it's a benchmark spec (has benchmark_id) + else if (spec.benchmark_id) { + cloneParams.cloneSourceType = "benchmark"; + cloneParams.initialBenchmarkIds = spec.benchmark_id; + } + // Fallback: check job_source + else if (selectedJob.job_source) { + const source = selectedJob.job_source as any; + if (source.scenario_ids && Array.isArray(source.scenario_ids)) { + cloneParams.cloneSourceType = "scenarios"; + cloneParams.initialScenarioIds = source.scenario_ids.join(","); + } else if (source.benchmark_id) { + cloneParams.cloneSourceType = "benchmark"; + cloneParams.initialBenchmarkIds = source.benchmark_id; + } + } + } + + // Extract agent configs - both full configs and legacy fields + if (selectedJob.job_spec?.agent_configs) { + const agentConfigs = selectedJob.job_spec.agent_configs.map( + (a: any) => ({ + agentId: a.agent_id, + name: a.name, + modelName: a.model_name, + timeoutSeconds: a.timeout_seconds, + kwargs: a.kwargs, + environmentVariables: a.agent_environment?.environment_variables, + secrets: a.agent_environment?.secrets, + }), + ); + cloneParams.cloneAgentConfigs = JSON.stringify(agentConfigs); + + // Also extract legacy fields for form initialization + cloneParams.cloneAgentIds = selectedJob.job_spec.agent_configs + .map((a: any) => a.agent_id) + .join(","); + cloneParams.cloneAgentNames = selectedJob.job_spec.agent_configs + .map((a: any) => a.name) + .join(","); + } + + // Extract orchestrator config + if (selectedJob.job_spec?.orchestrator_config) { + const orch = selectedJob.job_spec.orchestrator_config; + cloneParams.cloneOrchestratorConfig = JSON.stringify({ + nAttempts: orch.n_attempts, + nConcurrentTrials: orch.n_concurrent_trials, + quiet: orch.quiet, + timeoutMultiplier: orch.timeout_multiplier, + }); + } + + navigate("benchmark-job-create", cloneParams); } else if (key.escape || input === "q") { setShowPopup(false); setSelectedOperation(0); @@ -329,9 +466,80 @@ export function BenchmarkJobListScreen() { } else if (input === "a" && selectedJob) { setShowPopup(true); setSelectedOperation(0); - } else if (input === "c") { - // Quick shortcut to create a new job - navigate("benchmark-job-create"); + } else if (input === "3") { + // Quick shortcut to clone the selected job, or create a new job if none selected + if (selectedJob) { + const cloneParams: any = { + cloneFromJobId: selectedJob.id, + cloneJobName: selectedJob.name, + }; + + // Determine source type and extract IDs + if (selectedJob.job_spec) { + const spec = selectedJob.job_spec as any; + + // Check if it's a scenarios spec (has scenario_ids array) + if (spec.scenario_ids && Array.isArray(spec.scenario_ids)) { + cloneParams.cloneSourceType = "scenarios"; + cloneParams.initialScenarioIds = spec.scenario_ids.join(","); + } + // Check if it's a benchmark spec (has benchmark_id) + else if (spec.benchmark_id) { + cloneParams.cloneSourceType = "benchmark"; + cloneParams.initialBenchmarkIds = spec.benchmark_id; + } + // Fallback: check job_source + else if (selectedJob.job_source) { + const source = selectedJob.job_source as any; + if (source.scenario_ids && Array.isArray(source.scenario_ids)) { + cloneParams.cloneSourceType = "scenarios"; + cloneParams.initialScenarioIds = source.scenario_ids.join(","); + } else if (source.benchmark_id) { + cloneParams.cloneSourceType = "benchmark"; + cloneParams.initialBenchmarkIds = source.benchmark_id; + } + } + } + + // Extract agent configs - both full configs and legacy fields + if (selectedJob.job_spec?.agent_configs) { + const agentConfigs = selectedJob.job_spec.agent_configs.map( + (a: any) => ({ + agentId: a.agent_id, + name: a.name, + modelName: a.model_name, + timeoutSeconds: a.timeout_seconds, + kwargs: a.kwargs, + environmentVariables: a.agent_environment?.environment_variables, + secrets: a.agent_environment?.secrets, + }), + ); + cloneParams.cloneAgentConfigs = JSON.stringify(agentConfigs); + + // Also extract legacy fields for form initialization + cloneParams.cloneAgentIds = selectedJob.job_spec.agent_configs + .map((a: any) => a.agent_id) + .join(","); + cloneParams.cloneAgentNames = selectedJob.job_spec.agent_configs + .map((a: any) => a.name) + .join(","); + } + + // Extract orchestrator config + if (selectedJob.job_spec?.orchestrator_config) { + const orch = selectedJob.job_spec.orchestrator_config; + cloneParams.cloneOrchestratorConfig = JSON.stringify({ + nAttempts: orch.n_attempts, + nConcurrentTrials: orch.n_concurrent_trials, + quiet: orch.quiet, + timeoutMultiplier: orch.timeout_multiplier, + }); + } + + navigate("benchmark-job-create", cloneParams); + } else { + navigate("benchmark-job-create"); + } } else if (input === "/") { search.enterSearchMode(); } else if (key.escape) { @@ -450,7 +658,7 @@ export function BenchmarkJobListScreen() { shortcut: op.key === "view_details" ? "v" - : op.key === "create_new" + : op.key === "clone_job" ? "n" : "", }))} @@ -470,7 +678,7 @@ export function BenchmarkJobListScreen() { condition: hasMore || hasPrev, }, { key: "Enter", label: "Details" }, - { key: "c", label: "New Job" }, + { key: "3", label: "Clone" }, { key: "a", label: "Actions" }, { key: "/", label: "Search" }, { key: "Esc", label: "Back" }, diff --git a/src/services/benchmarkJobService.ts b/src/services/benchmarkJobService.ts index 85f63618..7cfacb5f 100644 --- a/src/services/benchmarkJobService.ts +++ b/src/services/benchmarkJobService.ts @@ -43,7 +43,8 @@ export interface OrchestratorConfig { export interface CreateBenchmarkJobOptions { name?: string; - benchmarkId: string; + benchmarkId?: string; + scenarioIds?: string[]; agentConfigs: AgentConfig[]; orchestratorConfig?: OrchestratorConfig; } @@ -92,59 +93,64 @@ export async function getBenchmarkJob(id: string): Promise { } /** - * Create a benchmark job with benchmark definition spec + * Create a benchmark job with either benchmark definition spec or scenario definition spec */ export async function createBenchmarkJob( options: CreateBenchmarkJobOptions, ): Promise { const client = getClient(); + // Validate that either benchmarkId or scenarioIds is provided + if (!options.benchmarkId && !options.scenarioIds) { + throw new Error("Either benchmarkId or scenarioIds must be provided"); + } + if (options.benchmarkId && options.scenarioIds) { + throw new Error("Cannot specify both benchmarkId and scenarioIds"); + } + // Build agent configs in API format - const agentConfigs: BenchmarkJobCreateParams.BenchmarkDefinitionJobSpec["agent_configs"] = - options.agentConfigs.map((agent) => { - const config: BenchmarkJobCreateParams.BenchmarkDefinitionJobSpec.AgentConfig = - { - name: agent.name, - type: "job_agent" as const, - }; - - if (agent.agentId) { - config.agent_id = agent.agentId; - } - if (agent.modelName) { - config.model_name = agent.modelName; - } - if (agent.timeoutSeconds) { - config.timeout_seconds = agent.timeoutSeconds; - } - if (agent.kwargs && Object.keys(agent.kwargs).length > 0) { - config.kwargs = agent.kwargs; - } + // Use the same agent config type for both spec types + const agentConfigs: Array = options.agentConfigs.map((agent) => { + const config: any = { + name: agent.name, + type: "job_agent" as const, + }; + + if (agent.agentId) { + config.agent_id = agent.agentId; + } + if (agent.modelName) { + config.model_name = agent.modelName; + } + if (agent.timeoutSeconds) { + config.timeout_seconds = agent.timeoutSeconds; + } + if (agent.kwargs && Object.keys(agent.kwargs).length > 0) { + config.kwargs = agent.kwargs; + } + if ( + (agent.environmentVariables && + Object.keys(agent.environmentVariables).length > 0) || + (agent.secrets && Object.keys(agent.secrets).length > 0) + ) { + config.agent_environment = {}; if ( - (agent.environmentVariables && - Object.keys(agent.environmentVariables).length > 0) || - (agent.secrets && Object.keys(agent.secrets).length > 0) + agent.environmentVariables && + Object.keys(agent.environmentVariables).length > 0 ) { - config.agent_environment = {}; - if ( - agent.environmentVariables && - Object.keys(agent.environmentVariables).length > 0 - ) { - config.agent_environment.environment_variables = - agent.environmentVariables; - } - if (agent.secrets && Object.keys(agent.secrets).length > 0) { - config.agent_environment.secrets = agent.secrets; - } + config.agent_environment.environment_variables = + agent.environmentVariables; } + if (agent.secrets && Object.keys(agent.secrets).length > 0) { + config.agent_environment.secrets = agent.secrets; + } + } - return config; - }); + return config; + }); // Build orchestrator config if provided - let orchestratorConfig: - | BenchmarkJobCreateParams.BenchmarkDefinitionJobSpec["orchestrator_config"] - | undefined; + let orchestratorConfig: any; if (options.orchestratorConfig) { orchestratorConfig = {}; if (options.orchestratorConfig.nAttempts !== undefined) { @@ -163,14 +169,27 @@ export async function createBenchmarkJob( } } - const createParams: BenchmarkJobCreateParams = { - name: options.name, - spec: { + // Build the appropriate spec based on what's provided + let spec: BenchmarkJobCreateParams["spec"]; + if (options.benchmarkId) { + spec = { type: "benchmark" as const, benchmark_id: options.benchmarkId, agent_configs: agentConfigs, orchestrator_config: orchestratorConfig, - }, + }; + } else if (options.scenarioIds) { + spec = { + type: "scenarios" as const, + scenario_ids: options.scenarioIds, + agent_configs: agentConfigs, + orchestrator_config: orchestratorConfig, + }; + } + + const createParams: BenchmarkJobCreateParams = { + name: options.name, + spec, }; return client.benchmarkJobs.create(createParams); diff --git a/src/services/scenarioService.ts b/src/services/scenarioService.ts new file mode 100644 index 00000000..6a413263 --- /dev/null +++ b/src/services/scenarioService.ts @@ -0,0 +1,61 @@ +/** + * Scenario Service - Handles all scenario-related API calls + */ +import { getClient } from "../utils/client.js"; +import type { + ScenarioListParams, + ScenarioView, +} from "@runloop/api-client/resources/scenarios/scenarios"; + +export type Scenario = ScenarioView; + +export interface ListScenariosOptions { + limit: number; + startingAfter?: string; + search?: string; +} + +export interface ListScenariosResult { + scenarios: Scenario[]; + totalCount: number; + hasMore: boolean; +} + +/** + * List scenarios with pagination + */ +export async function listScenarios( + options: ListScenariosOptions, +): Promise { + const client = getClient(); + + const queryParams: ScenarioListParams = { + limit: options.limit, + }; + + if (options.startingAfter) { + queryParams.starting_after = options.startingAfter; + } + + // Use name filter instead of search + if (options.search) { + queryParams.name = options.search; + } + + const page = await client.scenarios.list(queryParams); + const scenarios = page.scenarios || []; + + return { + scenarios, + totalCount: page.total_count || scenarios.length, + hasMore: page.has_more || false, + }; +} + +/** + * Get scenario by ID + */ +export async function getScenario(id: string): Promise { + const client = getClient(); + return client.scenarios.retrieve(id); +} diff --git a/tests/__tests__/components/DevboxCreatePage.test.tsx b/tests/__tests__/components/DevboxCreatePage.test.tsx index 1dce46f3..94d0fa23 100644 --- a/tests/__tests__/components/DevboxCreatePage.test.tsx +++ b/tests/__tests__/components/DevboxCreatePage.test.tsx @@ -68,8 +68,9 @@ describe('DevboxCreatePage', () => { ); const frame = lastFrame() || ''; - expect(frame).toContain('Blueprint ID'); - expect(frame).toContain('Snapshot ID'); + expect(frame).toContain('Source (optional)'); + expect(frame).toContain('Blueprint'); + expect(frame).toContain('Snapshot'); expect(frame).toContain('Metadata'); });