Skip to content

Commit 73255f9

Browse files
masnwilliamsclaude
andcommitted
Add unified CUA template with multi-provider fallback
Consolidates the separate anthropic-computer-use, openai-computer-use, and gemini-computer-use templates into a single "cua" template that supports all three providers with automatic fallback. - TypeScript and Python templates with identical structure - Provider selection via CUA_PROVIDER env var - Optional fallback chain via CUA_FALLBACK_PROVIDERS - Shared browser session lifecycle with replay support - Each provider adapter is self-contained and customizable - Registered as "cua" template in templates.go Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 6d54518 commit 73255f9

22 files changed

Lines changed: 2440 additions & 0 deletions

pkg/create/templates.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ const (
1919
TemplateOpenAGIComputerUse = "openagi-computer-use"
2020
TemplateClaudeAgentSDK = "claude-agent-sdk"
2121
TemplateYutoriComputerUse = "yutori"
22+
TemplateUnifiedCUA = "cua"
2223
)
2324

2425
type TemplateInfo struct {
@@ -90,6 +91,11 @@ var Templates = map[string]TemplateInfo{
9091
Description: "Implements a Yutori n1 computer use agent",
9192
Languages: []string{LanguageTypeScript, LanguagePython},
9293
},
94+
TemplateUnifiedCUA: {
95+
Name: "Unified CUA",
96+
Description: "Multi-provider computer use agent with Anthropic/OpenAI/Gemini fallback",
97+
Languages: []string{LanguageTypeScript, LanguagePython},
98+
},
9399
}
94100

95101
// GetSupportedTemplatesForLanguage returns a list of all supported template names for a given language
@@ -213,6 +219,11 @@ var Commands = map[string]map[string]DeployConfig{
213219
NeedsEnvFile: true,
214220
InvokeCommand: `kernel invoke ts-yutori-cua cua-task --payload '{"query": "Navigate to https://example.com and describe the page"}'`,
215221
},
222+
TemplateUnifiedCUA: {
223+
EntryPoint: "index.ts",
224+
NeedsEnvFile: true,
225+
InvokeCommand: `kernel invoke ts-cua cua-task --payload '{"query": "Go to https://news.ycombinator.com and get the top 5 stories"}'`,
226+
},
216227
},
217228
LanguagePython: {
218229
TemplateSampleApp: {
@@ -260,6 +271,11 @@ var Commands = map[string]map[string]DeployConfig{
260271
NeedsEnvFile: true,
261272
InvokeCommand: `kernel invoke python-yutori-cua cua-task --payload '{"query": "Navigate to https://example.com and describe the page"}'`,
262273
},
274+
TemplateUnifiedCUA: {
275+
EntryPoint: "main.py",
276+
NeedsEnvFile: true,
277+
InvokeCommand: `kernel invoke python-cua cua-task --payload '{"query": "Go to https://news.ycombinator.com and get the top 5 stories"}'`,
278+
},
263279
},
264280
}
265281

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Copy this file to .env and fill in your API keys.
2+
# Only the key for your chosen provider is required.
3+
4+
# Primary provider: "anthropic", "openai", or "gemini"
5+
CUA_PROVIDER=anthropic
6+
7+
# Comma-separated fallback order (optional).
8+
# If the primary provider fails, these are tried in order.
9+
# CUA_FALLBACK_PROVIDERS=openai,gemini
10+
11+
# Provider API keys — set the one(s) you plan to use
12+
ANTHROPIC_API_KEY=your_anthropic_api_key_here
13+
OPENAI_API_KEY=your_openai_api_key_here
14+
GOOGLE_API_KEY=your_google_api_key_here

pkg/templates/python/cua/README.md

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Unified CUA Template
2+
3+
A multi-provider Computer Use Agent (CUA) template for [Kernel](https://kernel.sh). Supports **Anthropic**, **OpenAI**, and **Google Gemini** as interchangeable backends with automatic fallback.
4+
5+
## Quick start
6+
7+
### 1. Install dependencies
8+
9+
```bash
10+
uv sync
11+
```
12+
13+
### 2. Configure environment
14+
15+
Copy the example env file and add your API keys:
16+
17+
```bash
18+
cp .env.example .env
19+
```
20+
21+
Set `CUA_PROVIDER` to your preferred provider and add the matching API key:
22+
23+
| Provider | Env var for key | Model used |
24+
|-------------|----------------------|--------------------------------------------|
25+
| `anthropic` | `ANTHROPIC_API_KEY` | `claude-sonnet-4-6` |
26+
| `openai` | `OPENAI_API_KEY` | `gpt-5.4` |
27+
| `gemini` | `GOOGLE_API_KEY` | `gemini-2.5-computer-use-preview-10-2025` |
28+
29+
### 3. Deploy to Kernel
30+
31+
```bash
32+
kernel deploy main.py --env-file .env
33+
```
34+
35+
### 4. Invoke
36+
37+
```bash
38+
kernel invoke python-cua cua-task --payload '{"query": "Go to https://news.ycombinator.com and get the top 5 stories"}'
39+
```
40+
41+
## Multi-provider fallback
42+
43+
Set `CUA_FALLBACK_PROVIDERS` to automatically try another provider if the primary fails:
44+
45+
```env
46+
CUA_PROVIDER=anthropic
47+
CUA_FALLBACK_PROVIDERS=openai,gemini
48+
```
49+
50+
This will try Anthropic first, then OpenAI, then Gemini. Only providers with valid API keys are used.
51+
52+
## Replay recording
53+
54+
Pass `record_replay: true` in the payload to capture a video replay of the browser session:
55+
56+
```bash
57+
kernel invoke python-cua cua-task --payload '{"query": "Navigate to example.com", "record_replay": true}'
58+
```
59+
60+
The response will include a `replay_url` you can open in your browser.
61+
62+
## Project structure
63+
64+
```
65+
main.py — Kernel app entrypoint
66+
session.py — Browser session lifecycle with replay support
67+
providers/
68+
__init__.py — Provider factory and fallback logic
69+
anthropic.py — Anthropic Claude adapter
70+
openai.py — OpenAI GPT adapter
71+
gemini.py — Google Gemini adapter
72+
```
73+
74+
## Customization
75+
76+
Each provider adapter is self-contained. To customize a provider's behavior (system prompt, model, tool handling), edit the corresponding file in `providers/`.
77+
78+
To add a new provider, create a new file that implements the `CuaProvider` protocol and register it in `providers/__init__.py`.
79+
80+
## Resources
81+
82+
- [Kernel Docs](https://docs.kernel.sh)
83+
- [Anthropic Computer Use](https://docs.anthropic.com/en/docs/agents-and-tools/computer-use)
84+
- [OpenAI Computer Use](https://platform.openai.com/docs/guides/computer-use)
85+
- [Google Gemini Computer Use](https://ai.google.dev/gemini-api/docs/computer-use)
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Python
2+
__pycache__/
3+
*.py[cod]
4+
*$py.class
5+
*.so
6+
*.egg-info/
7+
dist/
8+
build/
9+
10+
# Virtual environments
11+
.venv/
12+
venv/
13+
env/
14+
15+
# Environment
16+
.env
17+
.env.local
18+
.env.*.local
19+
20+
# IDE
21+
.vscode/
22+
.idea/
23+
*.swp
24+
*.swo
25+
26+
# OS
27+
.DS_Store
28+
Thumbs.db
29+
30+
# Logs
31+
*.log

pkg/templates/python/cua/main.py

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
"""
2+
Unified CUA (Computer Use Agent) template with multi-provider support.
3+
4+
Supports Anthropic, OpenAI, and Gemini as interchangeable providers.
5+
Configure via environment variables:
6+
CUA_PROVIDER — primary provider ("anthropic", "openai", or "gemini")
7+
CUA_FALLBACK_PROVIDERS — comma-separated fallback order (optional)
8+
9+
Each provider requires its own API key:
10+
ANTHROPIC_API_KEY, OPENAI_API_KEY, GOOGLE_API_KEY
11+
"""
12+
13+
from __future__ import annotations
14+
15+
import asyncio
16+
from typing import TypedDict
17+
18+
from kernel import Kernel, KernelContext
19+
20+
from providers import resolve_providers, run_with_fallback, TaskOptions
21+
from session import KernelBrowserSession, SessionOptions
22+
23+
kernel = Kernel()
24+
app = kernel.app("python-cua")
25+
26+
27+
class CuaInput(TypedDict, total=False):
28+
query: str
29+
record_replay: bool
30+
31+
32+
class CuaOutput(TypedDict, total=False):
33+
result: str
34+
provider: str
35+
replay_url: str
36+
37+
38+
# Resolve providers at startup for fast failure on misconfiguration.
39+
providers = resolve_providers()
40+
print(f"Configured providers: {' -> '.join(p.name for p in providers)}")
41+
42+
43+
@app.action("cua-task")
44+
async def cua_task(ctx: KernelContext, payload: CuaInput | None = None) -> CuaOutput:
45+
if not payload or not payload.get("query"):
46+
raise ValueError('Query is required. Payload must include: {"query": "your task description"}')
47+
48+
session = KernelBrowserSession(
49+
kernel,
50+
SessionOptions(
51+
invocation_id=ctx.invocation_id,
52+
stealth=True,
53+
record_replay=payload.get("record_replay", False),
54+
),
55+
)
56+
57+
await session.start()
58+
print(f"Live view: {session.live_view_url}")
59+
60+
try:
61+
task_result = await run_with_fallback(
62+
providers,
63+
TaskOptions(
64+
query=payload["query"],
65+
kernel=kernel,
66+
session_id=session.session_id,
67+
viewport_width=session.opts.viewport_width,
68+
viewport_height=session.opts.viewport_height,
69+
),
70+
)
71+
72+
session_info = await session.stop()
73+
74+
output: CuaOutput = {
75+
"result": task_result.result,
76+
"provider": task_result.provider,
77+
}
78+
if session_info.replay_view_url:
79+
output["replay_url"] = session_info.replay_view_url
80+
81+
return output
82+
83+
except Exception:
84+
await session.stop()
85+
raise
Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
"""
2+
Provider factory with automatic fallback.
3+
4+
Resolution order:
5+
1. CUA_PROVIDER env var (required)
6+
2. CUA_FALLBACK_PROVIDERS env var (optional, comma-separated)
7+
"""
8+
9+
from __future__ import annotations
10+
11+
import os
12+
from dataclasses import dataclass
13+
from typing import Protocol
14+
15+
from kernel import Kernel
16+
17+
18+
@dataclass
19+
class TaskOptions:
20+
query: str
21+
kernel: Kernel
22+
session_id: str
23+
viewport_width: int = 1280
24+
viewport_height: int = 800
25+
26+
27+
@dataclass
28+
class TaskResult:
29+
result: str
30+
provider: str
31+
32+
33+
class CuaProvider(Protocol):
34+
@property
35+
def name(self) -> str: ...
36+
def is_configured(self) -> bool: ...
37+
async def run_task(self, options: TaskOptions) -> TaskResult: ...
38+
39+
40+
def _build_provider(name: str) -> CuaProvider | None:
41+
if name == "anthropic":
42+
from .anthropic import AnthropicProvider
43+
return AnthropicProvider()
44+
if name == "openai":
45+
from .openai import OpenAIProvider
46+
return OpenAIProvider()
47+
if name == "gemini":
48+
from .gemini import GeminiProvider
49+
return GeminiProvider()
50+
return None
51+
52+
53+
def resolve_providers() -> list[CuaProvider]:
54+
"""Build the ordered list of providers to try."""
55+
primary = os.environ.get("CUA_PROVIDER", "").strip().lower()
56+
fallbacks = [
57+
s.strip().lower()
58+
for s in os.environ.get("CUA_FALLBACK_PROVIDERS", "").split(",")
59+
if s.strip()
60+
]
61+
62+
order = ([primary] if primary else []) + fallbacks
63+
64+
seen: set[str] = set()
65+
providers: list[CuaProvider] = []
66+
67+
for name in order:
68+
if name in seen:
69+
continue
70+
seen.add(name)
71+
72+
provider = _build_provider(name)
73+
if provider is None:
74+
print(f'Warning: Unknown provider "{name}", skipping.')
75+
continue
76+
if not provider.is_configured():
77+
print(f'Warning: Provider "{name}" missing API key, skipping.')
78+
continue
79+
providers.append(provider)
80+
81+
if not providers:
82+
raise RuntimeError(
83+
"No CUA provider is configured. "
84+
"Set CUA_PROVIDER to one of: anthropic, openai, gemini, "
85+
"and provide the matching API key."
86+
)
87+
88+
return providers
89+
90+
91+
async def run_with_fallback(
92+
providers: list[CuaProvider],
93+
options: TaskOptions,
94+
) -> TaskResult:
95+
"""Run a CUA task, trying each provider in order until one succeeds."""
96+
errors: list[tuple[str, Exception]] = []
97+
98+
for provider in providers:
99+
try:
100+
print(f"Attempting provider: {provider.name}")
101+
return await provider.run_task(options)
102+
except Exception as exc:
103+
print(f'Provider "{provider.name}" failed: {exc}')
104+
errors.append((provider.name, exc))
105+
106+
summary = "\n".join(f" {name}: {exc}" for name, exc in errors)
107+
raise RuntimeError(f"All providers failed:\n{summary}")

0 commit comments

Comments
 (0)