Skip to content

Commit 08ac870

Browse files
theomonnomtheomonnom
authored
simulate: full per-scenario dump only in CI, report file elsewhere (#905)
1 parent 401af7b commit 08ac870

2 files changed

Lines changed: 179 additions & 171 deletions

File tree

‎cmd/lk/simulate_ci.go‎

Lines changed: 15 additions & 171 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,10 @@ import (
2020
"io"
2121
"os"
2222
"os/signal"
23-
"strings"
2423
"sync/atomic"
2524
"time"
2625

2726
"github.com/livekit/protocol/livekit"
28-
agent "github.com/livekit/protocol/livekit/agent"
2927
)
3028

3129
type toggleWriter struct {
@@ -173,7 +171,19 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error {
173171

174172
// --- Results ---
175173

176-
report.Results(run, agent)
174+
if !out.Interactive() {
175+
report.Results(run, agent)
176+
} else {
177+
// A terminal is watching; we just couldn't open the TUI (e.g. stdin
178+
// isn't a TTY). Keep it to counts and pointers, the per-scenario
179+
// transcripts go to a report file like the TUI's.
180+
dashboardURL := simulationDashboardURL(config.pc.ProjectId, runID)
181+
if path := newRunReporter().Finish(run, agent, brokenAgent, dashboardURL); path != "" {
182+
out.Statusf("Run report: %s", path)
183+
}
184+
total, _, passed, failedN := simulationJobCounts(run)
185+
fmt.Fprintf(out.ResultWriter(), "%d total, %d passed, %d failed\n", total, passed, failedN)
186+
}
177187

178188
if brokenAgent && agent != nil {
179189
writeBrokenAgentNote(out.WarnWriter(), agent)
@@ -183,13 +193,10 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error {
183193
out.Statusf("Dashboard: %s", url)
184194
}
185195

196+
// the returned error is printed by main and reports the failure; the
197+
// counts line / full dump above already carries the detail
186198
_, _, _, failed := simulationJobCounts(run)
187199
if failed > 0 || run.Status == livekit.SimulationRun_STATUS_FAILED {
188-
if failed > 0 {
189-
out.Resultf("::error::%d simulation(s) failed\n", failed)
190-
} else {
191-
out.Resultf("::error::Simulation run failed: %s\n", run.Error)
192-
}
193200
if run.Status == livekit.SimulationRun_STATUS_FAILED && len(run.Jobs) == 0 {
194201
return fmt.Errorf("simulation failed: %s", run.Error)
195202
}
@@ -198,166 +205,3 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error {
198205

199206
return nil
200207
}
201-
202-
// writeRunResults writes the per-job results and the run summary, with GitHub
203-
// group markers (a useful delimiter outside GitHub too).
204-
func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) {
205-
if run == nil {
206-
return
207-
}
208-
summary := decodeRunSummary(run)
209-
210-
if run.Status == livekit.SimulationRun_STATUS_FAILED && len(run.Jobs) == 0 {
211-
fmt.Fprintf(w, "✗ Simulation failed: %s\n", run.Error)
212-
return
213-
}
214-
215-
for i, job := range run.Jobs {
216-
icon := "⏺"
217-
switch job.Status {
218-
case livekit.SimulationRun_Job_STATUS_COMPLETED:
219-
icon = "✓"
220-
case livekit.SimulationRun_Job_STATUS_FAILED:
221-
icon = "✗"
222-
}
223-
224-
label := job.Label
225-
if label == "" {
226-
label = fmt.Sprintf("Job %d", i+1)
227-
}
228-
229-
fmt.Fprintf(w, "::group::%s %s (%s)\n", icon, label, job.Id)
230-
231-
if job.Instructions != "" {
232-
fmt.Fprintln(w, "Instructions:")
233-
for line := range strings.SplitSeq(job.Instructions, "\n") {
234-
fmt.Fprintf(w, " %s\n", line)
235-
}
236-
}
237-
238-
if job.AgentExpectations != "" {
239-
fmt.Fprintln(w, "Expected:")
240-
for line := range strings.SplitSeq(job.AgentExpectations, "\n") {
241-
fmt.Fprintf(w, " %s\n", line)
242-
}
243-
}
244-
245-
if job.Error != "" {
246-
if job.Status == livekit.SimulationRun_Job_STATUS_COMPLETED {
247-
fmt.Fprintf(w, "Result: %s\n", job.Error)
248-
} else {
249-
fmt.Fprintf(w, "Error: %s\n", job.Error)
250-
}
251-
}
252-
253-
if summary != nil && summary.ChatHistory != nil {
254-
writeChatHistory(w, summary.ChatHistory[job.Id])
255-
}
256-
257-
if ap != nil && job.RoomName != "" {
258-
logs := ap.RecentRoomLogs(0, job.RoomName)
259-
if len(logs) > 0 {
260-
fmt.Fprintln(w, "Logs:")
261-
for _, line := range logs {
262-
fmt.Fprintf(w, " %s\n", ansiEscapeRe.ReplaceAllString(line, ""))
263-
}
264-
}
265-
}
266-
267-
fmt.Fprintln(w, "::endgroup::")
268-
269-
if job.Status == livekit.SimulationRun_Job_STATUS_FAILED {
270-
firstLine, _, _ := strings.Cut(job.Error, "\n")
271-
fmt.Fprintf(w, "::error::Job %d failed: %s\n", i+1, firstLine)
272-
}
273-
}
274-
275-
if summary != nil {
276-
writeRunSummary(w, run, summary)
277-
} else {
278-
msg := "The summary for this run is not available"
279-
if run.Error != "" {
280-
msg = run.Error
281-
}
282-
fmt.Fprintln(w)
283-
fmt.Fprintln(w, "⚠ "+msg)
284-
}
285-
}
286-
287-
func writeRunSummary(w io.Writer, run *livekit.SimulationRun, summary *livekit.SimulationRunSummary) {
288-
total, _, passed, failed := simulationJobCounts(run)
289-
290-
fmt.Fprintln(w)
291-
fmt.Fprintln(w, "::group::Summary")
292-
fmt.Fprintf(w, "%d total, %d passed, %d failed\n", total, passed, failed)
293-
294-
if summary.GoingWell != "" {
295-
fmt.Fprintln(w)
296-
fmt.Fprintln(w, "Going well:")
297-
for line := range strings.SplitSeq(summary.GoingWell, "\n") {
298-
fmt.Fprintf(w, " %s\n", line)
299-
}
300-
}
301-
302-
if summary.ToImprove != "" {
303-
fmt.Fprintln(w)
304-
fmt.Fprintln(w, "To improve:")
305-
for line := range strings.SplitSeq(summary.ToImprove, "\n") {
306-
fmt.Fprintf(w, " %s\n", line)
307-
}
308-
}
309-
310-
if len(summary.Issues) > 0 {
311-
fmt.Fprintln(w)
312-
fmt.Fprintln(w, "Issues:")
313-
for i, issue := range summary.Issues {
314-
fmt.Fprintf(w, " %d. %s\n", i+1, issue.Description)
315-
if issue.Suggestion != "" {
316-
fmt.Fprintf(w, " Suggestion: %s\n", issue.Suggestion)
317-
}
318-
}
319-
}
320-
321-
fmt.Fprintln(w, "::endgroup::")
322-
}
323-
324-
func writeChatHistory(w io.Writer, chatCtx *agent.ChatContext) {
325-
if chatCtx == nil || len(chatCtx.Items) == 0 {
326-
return
327-
}
328-
fmt.Fprintln(w, "Transcript:")
329-
for _, item := range chatCtx.Items {
330-
switch v := item.Item.(type) {
331-
case *agent.ChatContext_ChatItem_Message:
332-
msg := v.Message
333-
text := chatMessageText(msg)
334-
if text == "" {
335-
continue
336-
}
337-
switch msg.Role {
338-
case agent.ChatRole_USER:
339-
fmt.Fprintf(w, " ● You\n")
340-
case agent.ChatRole_ASSISTANT:
341-
fmt.Fprintf(w, " ● Agent\n")
342-
default:
343-
fmt.Fprintf(w, " ● %s\n", msg.Role)
344-
}
345-
for tl := range strings.SplitSeq(text, "\n") {
346-
fmt.Fprintf(w, " %s\n", tl)
347-
}
348-
case *agent.ChatContext_ChatItem_FunctionCall:
349-
fc := v.FunctionCall
350-
fmt.Fprintf(w, " [call] %s(%s)\n", fc.Name, fc.Arguments)
351-
case *agent.ChatContext_ChatItem_FunctionCallOutput:
352-
fco := v.FunctionCallOutput
353-
label := "output"
354-
if fco.IsError {
355-
label = "error"
356-
}
357-
fmt.Fprintf(w, " [%s] %s -> %s\n", label, fco.Name, fco.Output)
358-
case *agent.ChatContext_ChatItem_AgentHandoff:
359-
h := v.AgentHandoff
360-
fmt.Fprintf(w, " [handoff] -> %s\n", h.NewAgentId)
361-
}
362-
}
363-
}

‎cmd/lk/simulate_report.go‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"time"
2323

2424
"github.com/livekit/protocol/livekit"
25+
agent "github.com/livekit/protocol/livekit/agent"
2526
)
2627

2728
// simLog is the single source of the plain-text simulation output, shared by
@@ -144,6 +145,169 @@ func (a asciiWriter) Write(p []byte) (int, error) {
144145
return len(p), err
145146
}
146147

148+
// writeRunResults writes the per-job results and the run summary, with GitHub
149+
// group markers (a useful delimiter outside GitHub too).
150+
func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) {
151+
if run == nil {
152+
return
153+
}
154+
summary := decodeRunSummary(run)
155+
156+
if run.Status == livekit.SimulationRun_STATUS_FAILED && len(run.Jobs) == 0 {
157+
fmt.Fprintf(w, "✗ Simulation failed: %s\n", run.Error)
158+
return
159+
}
160+
161+
for i, job := range run.Jobs {
162+
icon := "⏺"
163+
switch job.Status {
164+
case livekit.SimulationRun_Job_STATUS_COMPLETED:
165+
icon = "✓"
166+
case livekit.SimulationRun_Job_STATUS_FAILED:
167+
icon = "✗"
168+
}
169+
170+
label := job.Label
171+
if label == "" {
172+
label = fmt.Sprintf("Job %d", i+1)
173+
}
174+
175+
fmt.Fprintf(w, "::group::%s %s (%s)\n", icon, label, job.Id)
176+
177+
if job.Instructions != "" {
178+
fmt.Fprintln(w, "Instructions:")
179+
for line := range strings.SplitSeq(job.Instructions, "\n") {
180+
fmt.Fprintf(w, " %s\n", line)
181+
}
182+
}
183+
184+
if job.AgentExpectations != "" {
185+
fmt.Fprintln(w, "Expected:")
186+
for line := range strings.SplitSeq(job.AgentExpectations, "\n") {
187+
fmt.Fprintf(w, " %s\n", line)
188+
}
189+
}
190+
191+
if job.Error != "" {
192+
if job.Status == livekit.SimulationRun_Job_STATUS_COMPLETED {
193+
fmt.Fprintf(w, "Result: %s\n", job.Error)
194+
} else {
195+
fmt.Fprintf(w, "Error: %s\n", job.Error)
196+
}
197+
}
198+
199+
if summary != nil && summary.ChatHistory != nil {
200+
writeChatHistory(w, summary.ChatHistory[job.Id])
201+
}
202+
203+
if ap != nil && job.RoomName != "" {
204+
logs := ap.RecentRoomLogs(0, job.RoomName)
205+
if len(logs) > 0 {
206+
fmt.Fprintln(w, "Logs:")
207+
for _, line := range logs {
208+
fmt.Fprintf(w, " %s\n", ansiEscapeRe.ReplaceAllString(line, ""))
209+
}
210+
}
211+
}
212+
213+
fmt.Fprintln(w, "::endgroup::")
214+
215+
if job.Status == livekit.SimulationRun_Job_STATUS_FAILED {
216+
firstLine, _, _ := strings.Cut(job.Error, "\n")
217+
fmt.Fprintf(w, "::error::Job %d failed: %s\n", i+1, firstLine)
218+
}
219+
}
220+
221+
if summary != nil {
222+
writeRunSummary(w, run, summary)
223+
} else {
224+
msg := "The summary for this run is not available"
225+
if run.Error != "" {
226+
msg = run.Error
227+
}
228+
fmt.Fprintln(w)
229+
fmt.Fprintln(w, "⚠ "+msg)
230+
}
231+
}
232+
233+
func writeRunSummary(w io.Writer, run *livekit.SimulationRun, summary *livekit.SimulationRunSummary) {
234+
total, _, passed, failed := simulationJobCounts(run)
235+
236+
fmt.Fprintln(w)
237+
fmt.Fprintln(w, "::group::Summary")
238+
fmt.Fprintf(w, "%d total, %d passed, %d failed\n", total, passed, failed)
239+
240+
if summary.GoingWell != "" {
241+
fmt.Fprintln(w)
242+
fmt.Fprintln(w, "Going well:")
243+
for line := range strings.SplitSeq(summary.GoingWell, "\n") {
244+
fmt.Fprintf(w, " %s\n", line)
245+
}
246+
}
247+
248+
if summary.ToImprove != "" {
249+
fmt.Fprintln(w)
250+
fmt.Fprintln(w, "To improve:")
251+
for line := range strings.SplitSeq(summary.ToImprove, "\n") {
252+
fmt.Fprintf(w, " %s\n", line)
253+
}
254+
}
255+
256+
if len(summary.Issues) > 0 {
257+
fmt.Fprintln(w)
258+
fmt.Fprintln(w, "Issues:")
259+
for i, issue := range summary.Issues {
260+
fmt.Fprintf(w, " %d. %s\n", i+1, issue.Description)
261+
if issue.Suggestion != "" {
262+
fmt.Fprintf(w, " Suggestion: %s\n", issue.Suggestion)
263+
}
264+
}
265+
}
266+
267+
fmt.Fprintln(w, "::endgroup::")
268+
}
269+
270+
func writeChatHistory(w io.Writer, chatCtx *agent.ChatContext) {
271+
if chatCtx == nil || len(chatCtx.Items) == 0 {
272+
return
273+
}
274+
fmt.Fprintln(w, "Transcript:")
275+
for _, item := range chatCtx.Items {
276+
switch v := item.Item.(type) {
277+
case *agent.ChatContext_ChatItem_Message:
278+
msg := v.Message
279+
text := chatMessageText(msg)
280+
if text == "" {
281+
continue
282+
}
283+
switch msg.Role {
284+
case agent.ChatRole_USER:
285+
fmt.Fprintf(w, " ● You\n")
286+
case agent.ChatRole_ASSISTANT:
287+
fmt.Fprintf(w, " ● Agent\n")
288+
default:
289+
fmt.Fprintf(w, " ● %s\n", msg.Role)
290+
}
291+
for tl := range strings.SplitSeq(text, "\n") {
292+
fmt.Fprintf(w, " %s\n", tl)
293+
}
294+
case *agent.ChatContext_ChatItem_FunctionCall:
295+
fc := v.FunctionCall
296+
fmt.Fprintf(w, " [call] %s(%s)\n", fc.Name, fc.Arguments)
297+
case *agent.ChatContext_ChatItem_FunctionCallOutput:
298+
fco := v.FunctionCallOutput
299+
label := "output"
300+
if fco.IsError {
301+
label = "error"
302+
}
303+
fmt.Fprintf(w, " [%s] %s -> %s\n", label, fco.Name, fco.Output)
304+
case *agent.ChatContext_ChatItem_AgentHandoff:
305+
h := v.AgentHandoff
306+
fmt.Fprintf(w, " [handoff] -> %s\n", h.NewAgentId)
307+
}
308+
}
309+
}
310+
147311
// runReporter writes the simLog to a temp file so TUI runs leave the same
148312
// record the non-TUI mode prints.
149313
type runReporter struct {

0 commit comments

Comments
 (0)