Skip to content

Commit fa8f19a

Browse files
k1ytmyk1yt
authored andcommitted
fix(ci): increase e2e timeout and add provider totalCost tests
- Increase shell integration timeout from 5s to 15s (BaseTerminal.ts) - Increase e2e test timeout from 60s to 120s and add retry (terminal-reuse-shell-race.test.ts) - Add totalCost tests for OpenAICompatibleHandler (new spec file) - Add totalCost tests for OpenAiHandler (existing spec file)
1 parent 59ac789 commit fa8f19a

4 files changed

Lines changed: 290 additions & 2 deletions

File tree

apps/vscode-e2e/src/suite/tools/terminal-reuse-shell-race.test.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ suite("Terminal reuse after zero-chunk shell race", function () {
2525
return
2626
}
2727

28+
this.retries(1)
29+
2830
setDefaultSuiteTimeout(this)
2931

3032
setup(async () => {
@@ -73,7 +75,7 @@ suite("Terminal reuse after zero-chunk shell race", function () {
7375
},
7476
text: "TERMINAL_REUSE_SHELL_RACE_E2E",
7577
}),
76-
timeout: 60_000,
78+
timeout: 120_000,
7779
})
7880

7981
const elapsedMs = Date.now() - startedAt
Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
// npx vitest run api/providers/__tests__/openai-compatible.spec.ts
2+
3+
import type { ModelInfo } from "@roo-code/types"
4+
import type { ApiHandlerOptions } from "../../../shared/api"
5+
6+
// Mock @ai-sdk/openai-compatible
7+
vi.mock("@ai-sdk/openai-compatible", () => ({
8+
createOpenAICompatible: vi.fn().mockReturnValue(vi.fn()),
9+
}))
10+
11+
// Mock ai
12+
vi.mock("ai", () => ({
13+
streamText: vi.fn(),
14+
generateText: vi.fn(),
15+
}))
16+
17+
vi.mock("../utils/timeout-config", () => ({
18+
getApiRequestTimeout: vi.fn().mockReturnValue(300_000),
19+
}))
20+
21+
import { OpenAICompatibleHandler } from "../openai-compatible"
22+
import type { OpenAICompatibleConfig } from "../openai-compatible"
23+
24+
/**
25+
* Concrete subclass that exposes the protected processUsageMetrics for direct testing.
26+
*/
27+
class TestableOpenAICompatibleHandler extends OpenAICompatibleHandler {
28+
private _modelInfo: ModelInfo
29+
30+
constructor(modelInfo: ModelInfo) {
31+
const options: ApiHandlerOptions = {
32+
openAiApiKey: "test-api-key",
33+
}
34+
const config: OpenAICompatibleConfig = {
35+
providerName: "test-provider",
36+
baseURL: "https://test.api.com/v1",
37+
apiKey: "test-api-key",
38+
modelId: "test-model",
39+
modelInfo,
40+
}
41+
super(options, config)
42+
this._modelInfo = modelInfo
43+
}
44+
45+
override getModel() {
46+
return {
47+
id: this.config.modelId,
48+
info: this._modelInfo,
49+
}
50+
}
51+
52+
/** Expose protected processUsageMetrics for testing. */
53+
public testProcessUsageMetrics(
54+
usage: Parameters<OpenAICompatibleHandler["processUsageMetrics"]>[0],
55+
) {
56+
return this.processUsageMetrics(usage)
57+
}
58+
59+
// Stubs for abstract methods inherited from BaseProvider
60+
override async *createMessage() {
61+
yield { type: "text" as const, text: "" }
62+
}
63+
64+
override async completePrompt() {
65+
return ""
66+
}
67+
}
68+
69+
describe("OpenAICompatibleHandler", () => {
70+
describe("processUsageMetrics", () => {
71+
it("should return correct totalCost when modelInfo has pricing", () => {
72+
const modelInfo: ModelInfo = {
73+
contextWindow: 128_000,
74+
supportsPromptCache: false,
75+
inputPrice: 3.0,
76+
outputPrice: 15.0,
77+
}
78+
const handler = new TestableOpenAICompatibleHandler(modelInfo)
79+
80+
const result = handler.testProcessUsageMetrics({
81+
inputTokens: 1000,
82+
outputTokens: 500,
83+
})
84+
85+
expect(result.type).toBe("usage")
86+
expect(result.inputTokens).toBe(1000)
87+
expect(result.outputTokens).toBe(500)
88+
expect(typeof result.totalCost).toBe("number")
89+
expect(result.totalCost).toBeGreaterThan(0)
90+
})
91+
92+
it("should return totalCost of 0 when modelInfo has no pricing", () => {
93+
const modelInfo: ModelInfo = {
94+
contextWindow: 128_000,
95+
supportsPromptCache: false,
96+
}
97+
const handler = new TestableOpenAICompatibleHandler(modelInfo)
98+
99+
const result = handler.testProcessUsageMetrics({
100+
inputTokens: 1000,
101+
outputTokens: 500,
102+
})
103+
104+
expect(result.type).toBe("usage")
105+
expect(result.totalCost).toBe(0)
106+
})
107+
108+
it("should factor cachedInputTokens into cost calculation", () => {
109+
const modelInfo: ModelInfo = {
110+
contextWindow: 128_000,
111+
supportsPromptCache: true,
112+
inputPrice: 3.0,
113+
outputPrice: 15.0,
114+
cacheReadsPrice: 0.3,
115+
}
116+
const handler = new TestableOpenAICompatibleHandler(modelInfo)
117+
118+
const result = handler.testProcessUsageMetrics({
119+
inputTokens: 1000,
120+
outputTokens: 500,
121+
details: {
122+
cachedInputTokens: 200,
123+
},
124+
})
125+
126+
expect(result.type).toBe("usage")
127+
expect(result.cacheReadTokens).toBe(200)
128+
expect(typeof result.totalCost).toBe("number")
129+
expect(result.totalCost).toBeGreaterThan(0)
130+
})
131+
132+
it("should include reasoningTokens when provided in usage details", () => {
133+
const modelInfo: ModelInfo = {
134+
contextWindow: 128_000,
135+
supportsPromptCache: false,
136+
inputPrice: 3.0,
137+
outputPrice: 15.0,
138+
}
139+
const handler = new TestableOpenAICompatibleHandler(modelInfo)
140+
141+
const result = handler.testProcessUsageMetrics({
142+
inputTokens: 500,
143+
outputTokens: 300,
144+
details: {
145+
reasoningTokens: 100,
146+
},
147+
})
148+
149+
expect(result.reasoningTokens).toBe(100)
150+
expect(result.inputTokens).toBe(500)
151+
expect(result.outputTokens).toBe(300)
152+
})
153+
154+
it("should default to 0 for missing token counts", () => {
155+
const modelInfo: ModelInfo = {
156+
contextWindow: 128_000,
157+
supportsPromptCache: false,
158+
inputPrice: 3.0,
159+
outputPrice: 15.0,
160+
}
161+
const handler = new TestableOpenAICompatibleHandler(modelInfo)
162+
163+
const result = handler.testProcessUsageMetrics({})
164+
165+
expect(result.inputTokens).toBe(0)
166+
expect(result.outputTokens).toBe(0)
167+
expect(result.totalCost).toBe(0)
168+
})
169+
})
170+
})

src/api/providers/__tests__/openai.spec.ts

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1496,6 +1496,122 @@ describe("OpenAiHandler", () => {
14961496
)
14971497
})
14981498
})
1499+
1500+
describe("processUsageMetrics totalCost", () => {
1501+
it("should include totalCost in the usage chunk from streaming response", async () => {
1502+
mockCreate.mockImplementationOnce(async () => ({
1503+
[Symbol.asyncIterator]: async function* () {
1504+
yield {
1505+
choices: [{ delta: { content: "Hello" }, index: 0 }],
1506+
usage: null,
1507+
}
1508+
yield {
1509+
choices: [{ delta: {}, index: 0 }],
1510+
usage: {
1511+
prompt_tokens: 100,
1512+
completion_tokens: 50,
1513+
total_tokens: 150,
1514+
},
1515+
}
1516+
},
1517+
}))
1518+
1519+
const stream = handler.createMessage("system", [])
1520+
const chunks: any[] = []
1521+
for await (const chunk of stream) {
1522+
chunks.push(chunk)
1523+
}
1524+
1525+
const usageChunk = chunks.find((c) => c.type === "usage")
1526+
expect(usageChunk).toBeDefined()
1527+
expect(typeof usageChunk?.totalCost).toBe("number")
1528+
})
1529+
1530+
it("should return totalCost as 0 when model has no pricing", async () => {
1531+
const noPricingHandler = new OpenAiHandler({
1532+
openAiApiKey: "test-api-key",
1533+
openAiModelId: "custom-model",
1534+
openAiCustomModelInfo: {
1535+
contextWindow: 128_000,
1536+
supportsPromptCache: false,
1537+
// No inputPrice or outputPrice set
1538+
},
1539+
})
1540+
1541+
mockCreate.mockImplementationOnce(async () => ({
1542+
[Symbol.asyncIterator]: async function* () {
1543+
yield {
1544+
choices: [{ delta: { content: "Hello" }, index: 0 }],
1545+
usage: null,
1546+
}
1547+
yield {
1548+
choices: [{ delta: {}, index: 0 }],
1549+
usage: {
1550+
prompt_tokens: 1000,
1551+
completion_tokens: 500,
1552+
total_tokens: 1500,
1553+
},
1554+
}
1555+
},
1556+
}))
1557+
1558+
const stream = noPricingHandler.createMessage("system", [])
1559+
const chunks: any[] = []
1560+
for await (const chunk of stream) {
1561+
chunks.push(chunk)
1562+
}
1563+
1564+
const usageChunk = chunks.find((c) => c.type === "usage")
1565+
expect(usageChunk).toBeDefined()
1566+
expect(usageChunk?.totalCost).toBe(0)
1567+
})
1568+
1569+
it("should factor cache tokens into totalCost calculation", async () => {
1570+
const cacheHandler = new OpenAiHandler({
1571+
openAiApiKey: "test-api-key",
1572+
openAiModelId: "gpt-4",
1573+
openAiCustomModelInfo: {
1574+
contextWindow: 128_000,
1575+
supportsPromptCache: true,
1576+
inputPrice: 3.0,
1577+
outputPrice: 15.0,
1578+
cacheReadsPrice: 0.3,
1579+
},
1580+
})
1581+
1582+
mockCreate.mockImplementationOnce(async () => ({
1583+
[Symbol.asyncIterator]: async function* () {
1584+
yield {
1585+
choices: [{ delta: { content: "Hello" }, index: 0 }],
1586+
usage: null,
1587+
}
1588+
yield {
1589+
choices: [{ delta: {}, index: 0 }],
1590+
usage: {
1591+
prompt_tokens: 1000,
1592+
completion_tokens: 500,
1593+
total_tokens: 1500,
1594+
cache_creation_input_tokens: 100,
1595+
cache_read_input_tokens: 200,
1596+
},
1597+
}
1598+
},
1599+
}))
1600+
1601+
const stream = cacheHandler.createMessage("system", [])
1602+
const chunks: any[] = []
1603+
for await (const chunk of stream) {
1604+
chunks.push(chunk)
1605+
}
1606+
1607+
const usageChunk = chunks.find((c) => c.type === "usage")
1608+
expect(usageChunk).toBeDefined()
1609+
expect(typeof usageChunk?.totalCost).toBe("number")
1610+
expect(usageChunk?.totalCost).toBeGreaterThan(0)
1611+
expect(usageChunk?.cacheReadTokens).toBe(200)
1612+
expect(usageChunk?.cacheWriteTokens).toBe(100)
1613+
})
1614+
})
14991615
})
15001616

15011617
describe("getOpenAiModels", () => {

src/integrations/terminal/BaseTerminal.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ export abstract class BaseTerminal implements RooTerminal {
154154
return output
155155
}
156156

157-
public static defaultShellIntegrationTimeout = 5_000
157+
public static defaultShellIntegrationTimeout = 15_000
158158
private static shellIntegrationTimeout: number = BaseTerminal.defaultShellIntegrationTimeout
159159
private static shellIntegrationDisabled: boolean = false
160160
private static commandDelay: number = 0

0 commit comments

Comments
 (0)