|
| 1 | +import { OpenAIAdapter } from '@agentic-kit/openai'; |
| 2 | +import { createUserMessage, type AssistantMessage } from 'agentic-kit'; |
| 3 | + |
| 4 | +import { Agent } from '../src'; |
| 5 | + |
| 6 | +const modelId = process.env.OPENAI_LIVE_MODEL ?? 'gpt-5.4-nano'; |
| 7 | +const apiKey = process.env.OPENAI_API_KEY; |
| 8 | + |
| 9 | +if (!apiKey) { |
| 10 | + throw new Error('Missing required env var: OPENAI_API_KEY'); |
| 11 | +} |
| 12 | + |
| 13 | +const liveSuite = process.env.AGENT_LIVE_SUITE ?? 'smoke'; |
| 14 | +const runSmoke = liveSuite === 'smoke' || liveSuite === 'extended'; |
| 15 | +const runExtended = liveSuite === 'extended'; |
| 16 | +const describeSmoke = runSmoke ? describe : describe.skip; |
| 17 | +const describeExtended = runExtended ? describe : describe.skip; |
| 18 | + |
| 19 | +describeSmoke('Agent live smoke', () => { |
| 20 | + jest.setTimeout(60_000); |
| 21 | + |
| 22 | + it('single turn populates state.totalUsage from the assistant message', async () => { |
| 23 | + const adapter = new OpenAIAdapter({ apiKey }); |
| 24 | + const model = adapter.createModel(modelId); |
| 25 | + const agent = new Agent({ initialState: { model }, streamFn: adapter.stream.bind(adapter) }); |
| 26 | + |
| 27 | + await agent.prompt('Reply with the single word PONG.'); |
| 28 | + |
| 29 | + expect(agent.state.totalUsage.input).toBeGreaterThan(0); |
| 30 | + expect(agent.state.totalUsage.output).toBeGreaterThan(0); |
| 31 | + expect(agent.state.totalUsage.totalTokens).toBeGreaterThan(0); |
| 32 | + expect(agent.state.totalUsage.cost.total).toBeGreaterThan(0); |
| 33 | + |
| 34 | + const lastAssistant = agent.state.messages |
| 35 | + .filter((m): m is AssistantMessage => m.role === 'assistant') |
| 36 | + .at(-1)!; |
| 37 | + |
| 38 | + // Single turn: the per-message usage IS the cumulative total. |
| 39 | + expect(agent.state.totalUsage.input).toBe(lastAssistant.usage.input); |
| 40 | + expect(agent.state.totalUsage.output).toBe(lastAssistant.usage.output); |
| 41 | + expect(agent.state.totalUsage.reasoning).toBe(lastAssistant.usage.reasoning); |
| 42 | + expect(agent.state.totalUsage.cacheRead).toBe(lastAssistant.usage.cacheRead); |
| 43 | + expect(agent.state.totalUsage.cacheWrite).toBe(lastAssistant.usage.cacheWrite); |
| 44 | + expect(agent.state.totalUsage.totalTokens).toBe(lastAssistant.usage.totalTokens); |
| 45 | + }); |
| 46 | +}); |
| 47 | + |
| 48 | +describeExtended('Agent live extended', () => { |
| 49 | + jest.setTimeout(120_000); |
| 50 | + |
| 51 | + it('state.totalUsage equals field-wise sum across two turns', async () => { |
| 52 | + const adapter = new OpenAIAdapter({ apiKey }); |
| 53 | + const model = adapter.createModel(modelId); |
| 54 | + const agent = new Agent({ initialState: { model }, streamFn: adapter.stream.bind(adapter) }); |
| 55 | + |
| 56 | + await agent.prompt('What is 2 + 2? Reply with just the number.'); |
| 57 | + |
| 58 | + const t1Usage = { |
| 59 | + ...agent.state.totalUsage, |
| 60 | + cost: { ...agent.state.totalUsage.cost }, |
| 61 | + }; |
| 62 | + |
| 63 | + // continue() does not accept text; append the follow-up user message first. |
| 64 | + agent.appendMessage(createUserMessage('Now what is that doubled? Reply with just the number.')); |
| 65 | + await agent.continue(); |
| 66 | + |
| 67 | + const lastAssistant = agent.state.messages |
| 68 | + .filter((m): m is AssistantMessage => m.role === 'assistant') |
| 69 | + .at(-1)!; |
| 70 | + |
| 71 | + expect(agent.state.totalUsage.input).toBe(t1Usage.input + lastAssistant.usage.input); |
| 72 | + expect(agent.state.totalUsage.output).toBe(t1Usage.output + lastAssistant.usage.output); |
| 73 | + expect(agent.state.totalUsage.reasoning).toBe(t1Usage.reasoning + lastAssistant.usage.reasoning); |
| 74 | + expect(agent.state.totalUsage.cacheRead).toBe(t1Usage.cacheRead + lastAssistant.usage.cacheRead); |
| 75 | + expect(agent.state.totalUsage.cacheWrite).toBe(t1Usage.cacheWrite + lastAssistant.usage.cacheWrite); |
| 76 | + expect(agent.state.totalUsage.totalTokens).toBe(t1Usage.totalTokens + lastAssistant.usage.totalTokens); |
| 77 | + expect(agent.state.totalUsage.cost.input).toBeCloseTo( |
| 78 | + t1Usage.cost.input + lastAssistant.usage.cost.input, |
| 79 | + 10 |
| 80 | + ); |
| 81 | + expect(agent.state.totalUsage.cost.output).toBeCloseTo( |
| 82 | + t1Usage.cost.output + lastAssistant.usage.cost.output, |
| 83 | + 10 |
| 84 | + ); |
| 85 | + expect(agent.state.totalUsage.cost.total).toBeCloseTo( |
| 86 | + t1Usage.cost.total + lastAssistant.usage.cost.total, |
| 87 | + 10 |
| 88 | + ); |
| 89 | + }); |
| 90 | + |
| 91 | + it('prompt() resets totalUsage; continue() preserves it', async () => { |
| 92 | + const adapter = new OpenAIAdapter({ apiKey }); |
| 93 | + const model = adapter.createModel(modelId); |
| 94 | + const agent = new Agent({ initialState: { model }, streamFn: adapter.stream.bind(adapter) }); |
| 95 | + |
| 96 | + await agent.prompt('Reply with the single word A.'); |
| 97 | + const firstTotals = { ...agent.state.totalUsage, cost: { ...agent.state.totalUsage.cost } }; |
| 98 | + |
| 99 | + agent.appendMessage(createUserMessage('Reply with the single word B.')); |
| 100 | + await agent.continue(); |
| 101 | + const secondTotals = { ...agent.state.totalUsage, cost: { ...agent.state.totalUsage.cost } }; |
| 102 | + |
| 103 | + // continue() must not reset — totals should have grown. |
| 104 | + expect(secondTotals.input).toBeGreaterThanOrEqual(firstTotals.input); |
| 105 | + expect(secondTotals.totalTokens).toBeGreaterThanOrEqual(firstTotals.totalTokens); |
| 106 | + expect(agent.state.totalUsage.input).toBeGreaterThanOrEqual(firstTotals.input); |
| 107 | + |
| 108 | + await agent.prompt('Reply with the single word C.'); |
| 109 | + |
| 110 | + const thirdAssistant = agent.state.messages |
| 111 | + .filter((m): m is AssistantMessage => m.role === 'assistant') |
| 112 | + .at(-1)!; |
| 113 | + |
| 114 | + // prompt() resets: the new total should be one turn's worth, not cumulative |
| 115 | + // across all three. We use < rather than === because token counts vary and |
| 116 | + // we cannot pin the exact value — only that it did not carry over the prior |
| 117 | + // two turns' worth of input tokens. |
| 118 | + expect(agent.state.totalUsage.input).toBeLessThan(secondTotals.input + 100); |
| 119 | + expect(agent.state.totalUsage.totalTokens).toBe(thirdAssistant.usage.totalTokens); |
| 120 | + expect(agent.state.totalUsage.input).toBe(thirdAssistant.usage.input); |
| 121 | + expect(agent.state.totalUsage.output).toBe(thirdAssistant.usage.output); |
| 122 | + }); |
| 123 | +}); |
0 commit comments