Skip to content

Commit 3e78899

Browse files
committed
feat: add frontend browser live view, tool streaming, and BrowserToolDisplay
- Add BrowserToolDisplay custom renderer for browser tool with live view, action streaming, and result display - Add BrowserLiveView wrapper (React.memo) around bedrock-agentcore DCV component - Add tool renderer plugin system (useToolRenderer) for custom tool UIs - Add streamEvents and liveViewUrl to ToolCall type for tool streaming - Fix strands parser: check tool_stream_event before text data to prevent action content leaking as chat text - Add flushSync for tool_stream events to force immediate React renders - Add bedrock-agentcore npm dependency with DCV SDK Vite configuration - Add setTimeout yield in SSE reader between chunks for render flushing
1 parent b38f44d commit 3e78899

14 files changed

Lines changed: 3247 additions & 800 deletions

File tree

frontend/package-lock.json

Lines changed: 2886 additions & 797 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

frontend/package.json

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,22 @@
1313
"clean": "rm -rf build/ node_modules/ .vite/"
1414
},
1515
"dependencies": {
16+
"@babel/runtime": "^7.0.0",
17+
"@cloudscape-design/components": "^3.0.0",
18+
"@cloudscape-design/design-tokens": "^3.0.0",
19+
"@cloudscape-design/global-styles": "^1.0.0",
1620
"@radix-ui/react-alert-dialog": "^1.1.15",
1721
"@radix-ui/react-dialog": "^1.1.15",
1822
"@radix-ui/react-progress": "^1.1.7",
1923
"@radix-ui/react-select": "^2.2.5",
2024
"@radix-ui/react-slot": "^1.2.4",
2125
"@types/react-syntax-highlighter": "^15.5.13",
2226
"aws-amplify": "^6.16.2",
27+
"bedrock-agentcore": "^0.2.2",
2328
"class-variance-authority": "^0.7.1",
2429
"clsx": "^2.1.1",
2530
"lucide-react": "^0.562.0",
31+
"prop-types": "^15.0.0",
2632
"radix-ui": "^1.4.3",
2733
"react": "^19.2.1",
2834
"react-dom": "^19.2.1",
@@ -57,6 +63,7 @@
5763
"tw-animate-css": "^1.2.9",
5864
"typescript": "^5",
5965
"vite": "^7.3.2",
66+
"vite-plugin-static-copy": "^3.0.2",
6067
"vitest": "^4.0.18"
6168
}
6269
}

frontend/src/App.tsx

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@
44
import { BrowserRouter } from "react-router-dom"
55
import { AuthProvider } from "@/components/auth/AuthProvider"
66
import AppRoutes from "./routes"
7+
import { useToolRenderer } from "@/hooks/useToolRenderer"
8+
import { BrowserToolDisplay } from "@/components/chat/BrowserToolDisplay"
9+
10+
// Register browser tool renderer (runs once at module load).
11+
// Must wrap in JSX so hooks run inside the component, not inside the caller.
12+
useToolRenderer("browser", props => <BrowserToolDisplay {...props} />)
713

814
export default function App() {
915
return (
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
import React from "react"
5+
import { BrowserLiveView as AgentCoreBrowserLiveView } from "bedrock-agentcore/browser/live-view"
6+
7+
interface BrowserLiveViewProps {
8+
presignedUrl: string
9+
isActive: boolean
10+
onConnectionError?: () => void
11+
}
12+
13+
/**
14+
* Wrapper around the bedrock-agentcore BrowserLiveView component.
15+
*
16+
* Memoized to prevent DCV reconnections on parent re-renders.
17+
* The component auto-scales to fit its parent container while preserving aspect ratio.
18+
* remoteWidth and remoteHeight must match the backend viewport dimensions.
19+
*/
20+
export const BrowserLiveView = React.memo(function BrowserLiveView({
21+
presignedUrl,
22+
isActive,
23+
}: BrowserLiveViewProps) {
24+
if (!isActive || !presignedUrl) {
25+
return null
26+
}
27+
28+
return (
29+
<div style={{
30+
width: "100%",
31+
aspectRatio: "1380 / 720",
32+
background: "#0f172a",
33+
position: "relative",
34+
}}>
35+
<AgentCoreBrowserLiveView
36+
signedUrl={presignedUrl}
37+
remoteWidth={1380}
38+
remoteHeight={720}
39+
/>
40+
</div>
41+
)
42+
})
Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
"use client"
2+
3+
import { useState, useRef } from "react"
4+
import {
5+
Globe,
6+
Loader2,
7+
CheckCircle2,
8+
ChevronRight,
9+
ChevronDown,
10+
Monitor,
11+
} from "lucide-react"
12+
import type { ToolRenderProps } from "@/hooks/useToolRenderer"
13+
import { BrowserLiveView } from "./BrowserLiveView"
14+
15+
/**
16+
* Browser tool display component.
17+
*
18+
* Uses streamEvents length as a React key on the actions container
19+
* to force re-render when new events arrive.
20+
*/
21+
export function BrowserToolDisplay({
22+
name,
23+
args,
24+
status,
25+
result,
26+
liveViewUrl,
27+
streamEvents,
28+
}: ToolRenderProps) {
29+
const [expanded, setExpanded] = useState(false)
30+
const [showViewer, setShowViewer] = useState(false)
31+
const stableUrlRef = useRef<string | null>(null)
32+
const wasAutoExpanded = useRef(false)
33+
34+
// Parse the input args to extract task and URL
35+
let taskDescription = ""
36+
let startingUrl = ""
37+
try {
38+
const parsedArgs = JSON.parse(args)
39+
taskDescription = parsedArgs.task || ""
40+
startingUrl = parsedArgs.starting_url || ""
41+
} catch {
42+
taskDescription = args
43+
}
44+
45+
// Prefer streamed liveViewUrl; fall back to extracting from final result.
46+
let url: string | null = liveViewUrl ?? null
47+
if (!url && result) {
48+
try {
49+
const parsed = JSON.parse(result)
50+
if (parsed?.live_view_url && String(parsed.live_view_url).includes("bedrock-agentcore")) {
51+
url = String(parsed.live_view_url)
52+
}
53+
} catch {
54+
const match = result.match(/https:\/\/bedrock-agentcore\.[^\s"]+live-view[^\s"]+/)
55+
if (match) url = match[0]
56+
}
57+
}
58+
if (url && !stableUrlRef.current) {
59+
stableUrlRef.current = url
60+
}
61+
const stableUrl = stableUrlRef.current
62+
63+
// Parse browser tool result
64+
const parsedResult = result ? parseBrowserResult(result) : null
65+
66+
// Extract actions from stream events
67+
const evtCount = streamEvents?.length ?? 0
68+
const liveActions: string[] = []
69+
if (streamEvents) {
70+
for (const ev of streamEvents) {
71+
if (typeof ev === "object" && ev !== null) {
72+
const data = ev as Record<string, unknown>
73+
if (data.type !== "browser_live_view" && data.extracted_content) {
74+
liveActions.push(String(data.extracted_content))
75+
}
76+
}
77+
}
78+
}
79+
80+
const isInProgress = status === "streaming" || status === "executing"
81+
82+
// Auto-expand when first stream event arrives so user sees live actions
83+
if (isInProgress && liveActions.length > 0 && !wasAutoExpanded.current) {
84+
wasAutoExpanded.current = true
85+
if (!expanded) setExpanded(true)
86+
}
87+
88+
return (
89+
<div className="my-1 text-sm">
90+
{/* Header row */}
91+
<div className="flex items-center gap-1.5 px-2 py-1 rounded hover:bg-gray-200/50 transition-colors">
92+
<button
93+
onClick={() => setExpanded(!expanded)}
94+
className="flex items-center gap-1.5 flex-1 text-left"
95+
>
96+
{expanded ? (
97+
<ChevronDown size={12} className="text-gray-400" />
98+
) : (
99+
<ChevronRight size={12} className="text-gray-400" />
100+
)}
101+
<Globe size={12} className="text-blue-500" />
102+
<span className="text-blue-700 font-medium">{name}</span>
103+
{!expanded && (
104+
<span className="text-xs text-gray-500 ml-1">
105+
{isInProgress
106+
? liveActions.length > 0
107+
? `(${liveActions.length} actions)`
108+
: "(executing...)"
109+
: status === "complete"
110+
? `✓ done${liveActions.length > 0 ? ` (${liveActions.length} actions)` : ""}`
111+
: ""}
112+
</span>
113+
)}
114+
</button>
115+
{stableUrl && (
116+
<button
117+
onClick={() => setShowViewer(!showViewer)}
118+
className="flex items-center gap-1 text-xs text-blue-600 hover:text-blue-800 px-1.5 py-0.5 rounded hover:bg-blue-50 transition-colors"
119+
>
120+
<Monitor size={10} />
121+
{showViewer ? "Hide viewer" : status === "complete" ? "View session" : "Watch live"}
122+
</button>
123+
)}
124+
{status === "streaming" && <Loader2 size={12} className="animate-spin text-blue-500" />}
125+
{status === "executing" && <Loader2 size={12} className="animate-spin text-amber-500" />}
126+
{status === "complete" && <CheckCircle2 size={12} className="text-green-500" />}
127+
</div>
128+
129+
{/* Expanded content — ONLY visible when expanded */}
130+
{expanded && (
131+
<div key={`actions-${evtCount}-${status}`} className="mx-2 mt-1 space-y-2">
132+
{/* 1. Input — task and URL */}
133+
{args && (
134+
<div className="text-xs text-gray-600">
135+
{taskDescription && <div><span className="font-medium">Task:</span> {taskDescription}</div>}
136+
{startingUrl && (
137+
<div className="mt-0.5">
138+
<span className="font-medium">URL:</span>{" "}
139+
<a href={startingUrl} target="_blank" rel="noopener noreferrer" className="text-blue-600 hover:underline">{startingUrl}</a>
140+
</div>
141+
)}
142+
</div>
143+
)}
144+
145+
{/* 2. Browser actions */}
146+
{isInProgress && liveActions.length === 0 && (
147+
<div className="text-xs text-gray-400 italic">Waiting for browser actions...</div>
148+
)}
149+
{liveActions.length > 0 && (
150+
<div className="space-y-0.5">
151+
<div className="text-xs text-gray-500">Browser actions ({liveActions.length}):</div>
152+
{liveActions.slice(-5).map((action, i) => (
153+
<div
154+
key={`${evtCount}-${i}`}
155+
className={`text-xs rounded px-2 py-1 border-l-2 ${
156+
isInProgress
157+
? "text-gray-600 bg-blue-50 border-blue-300"
158+
: "text-gray-500 bg-gray-50 border-gray-300"
159+
}`}
160+
>
161+
{action}
162+
</div>
163+
))}
164+
</div>
165+
)}
166+
167+
{/* 3. Result */}
168+
{status === "complete" && parsedResult?.finalContent && (
169+
<div className="text-xs text-gray-700 bg-green-50 rounded px-2 py-1.5 border border-green-200">
170+
<span className="font-medium text-green-800"></span>
171+
<span className="line-clamp-3">{parsedResult.finalContent}</span>
172+
</div>
173+
)}
174+
</div>
175+
)}
176+
177+
{/* 4. DCV live viewer — with spacing */}
178+
{showViewer && stableUrl && (
179+
<div className="mx-2 mt-3 mb-1 rounded overflow-hidden border border-gray-200">
180+
<BrowserLiveView presignedUrl={stableUrl} isActive={showViewer} onConnectionError={() => {}} />
181+
</div>
182+
)}
183+
184+
</div>
185+
)
186+
}
187+
188+
/** Parse browser tool result to extract meaningful content. */
189+
function parseBrowserResult(result: string): {
190+
finalContent: string | null
191+
actions: string[]
192+
raw: string
193+
} | null {
194+
try {
195+
const parsed = JSON.parse(result)
196+
let finalContent: string | null = null
197+
const actions: string[] = []
198+
if (parsed.response && typeof parsed.response === "string") {
199+
const matches = Array.from(
200+
parsed.response.matchAll(/extracted_content='([^']+(?:''[^']+)*)'/g)
201+
) as RegExpMatchArray[]
202+
const contents = matches.map((m: RegExpMatchArray) => m[1].replace(/''/g, "'"))
203+
if (contents.length > 0) {
204+
finalContent = contents[contents.length - 1]
205+
for (let i = 0; i < contents.length - 1; i++) {
206+
if (contents[i] && !contents[i].includes("AgentHistoryList")) actions.push(contents[i])
207+
}
208+
}
209+
}
210+
return { finalContent, actions, raw: result }
211+
} catch {
212+
return { finalContent: null, actions: [], raw: result }
213+
}
214+
}

frontend/src/components/chat/ChatInterface.tsx

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"use client"
22

33
import { useEffect, useRef, useState } from "react"
4+
import { flushSync } from "react-dom"
45
import { ChatHeader } from "./ChatHeader"
56
import { ChatInput } from "./ChatInput"
67
import { ChatMessages } from "./ChatMessages"
@@ -116,7 +117,21 @@ export default function ChatInterface() {
116117
updated[updated.length - 1] = {
117118
...updated[updated.length - 1],
118119
content,
119-
segments: [...segments],
120+
// Deep-copy tool segments including streamEvents array
121+
// so React detects changes at every level
122+
segments: segments.map(s =>
123+
s.type === "tool"
124+
? {
125+
type: "tool" as const,
126+
toolCall: {
127+
...s.toolCall,
128+
streamEvents: s.toolCall.streamEvents
129+
? [...s.toolCall.streamEvents]
130+
: undefined,
131+
},
132+
}
133+
: { ...s }
134+
),
120135
}
121136
return updated
122137
})
@@ -175,6 +190,24 @@ export default function ChatInterface() {
175190
updateMessage()
176191
break
177192
}
193+
case "tool_stream": {
194+
const tc = toolCallMap.get(event.toolUseId)
195+
if (tc) {
196+
tc.streamEvents = [...(tc.streamEvents ?? []), event.data]
197+
if (event.data && typeof event.data === "object") {
198+
const data = event.data as Record<string, unknown>
199+
if (typeof data.live_view_url === "string") {
200+
tc.liveViewUrl = data.live_view_url
201+
}
202+
}
203+
}
204+
// flushSync forces React to render immediately instead of
205+
// batching with other state updates — critical for streaming
206+
flushSync(() => {
207+
updateMessage()
208+
})
209+
break
210+
}
178211
case "message": {
179212
if (event.role === "assistant") {
180213
for (const tc of toolCallMap.values()) {

frontend/src/components/chat/ChatMessage.tsx

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ export function ChatMessage({
5757
args: seg.toolCall.input,
5858
status: seg.toolCall.status,
5959
result: seg.toolCall.result,
60+
liveViewUrl: seg.toolCall.liveViewUrl,
61+
streamEvents: seg.toolCall.streamEvents,
6062
})}
6163
</div>
6264
)

frontend/src/components/chat/ToolCallDisplay.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ export function ToolCallDisplay({ name, args, status, result }: ToolRenderProps)
3030
</button>
3131

3232
{expanded && (
33-
<div className="ml-6 mt-1 border-l-2 border-gray-200 pl-3 space-y-2">
33+
<div className="ml-6 mt-1 border-l-2 border-gray-200 pl-3 space-y-2 max-h-60 overflow-y-auto">
3434
{args && (
3535
<div>
3636
<div className="text-xs text-gray-400">Input</div>

frontend/src/components/chat/types.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ export interface ToolCall {
99
input: string
1010
result?: string
1111
status: ToolCallStatus
12+
liveViewUrl?: string
13+
streamEvents?: unknown[]
1214
}
1315

1416
export type MessageSegment =

frontend/src/hooks/useToolRenderer.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ export interface ToolRenderProps {
66
args: string
77
status: ToolCallStatus
88
result?: string
9+
liveViewUrl?: string
10+
streamEvents?: unknown[]
911
}
1012

1113
export type ToolRenderFn = (props: ToolRenderProps) => ReactNode

0 commit comments

Comments
 (0)