Skip to content

Commit 815d315

Browse files
k1ytk1yt
committed
fix(ci): increase e2e timeout and add provider totalCost tests
- Increase shell integration timeout from 5s to 15s (BaseTerminal.ts) - Increase e2e test timeout from 60s to 120s and add retry (terminal-reuse-shell-race.test.ts) - Add totalCost tests for OpenAICompatibleHandler (new spec file) - Add totalCost tests for OpenAiHandler (existing spec file)
1 parent 22b24d9 commit 815d315

4 files changed

Lines changed: 290 additions & 2 deletions

File tree

‎apps/vscode-e2e/src/suite/tools/terminal-reuse-shell-race.test.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ suite("Terminal reuse after zero-chunk shell race", function () {
2525
return
2626
}
2727

28+
this.retries(1)
29+
2830
setDefaultSuiteTimeout(this)
2931

3032
setup(async () => {
@@ -73,7 +75,7 @@ suite("Terminal reuse after zero-chunk shell race", function () {
7375
},
7476
text: "TERMINAL_REUSE_SHELL_RACE_E2E",
7577
}),
76-
timeout: 60_000,
78+
timeout: 120_000,
7779
})
7880

7981
const elapsedMs = Date.now() - startedAt

‎src/api/providers/__tests__/openai-compatible.spec.ts‎

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
// npx vitest run api/providers/__tests__/openai-compatible.spec.ts
2+
3+
import type { ModelInfo } from "@roo-code/types"
4+
import type { ApiHandlerOptions } from "../../../shared/api"
5+
6+
// Mock @ai-sdk/openai-compatible
7+
vi.mock("@ai-sdk/openai-compatible", () => ({
8+
createOpenAICompatible: vi.fn().mockReturnValue(vi.fn()),
9+
}))
10+
11+
// Mock ai
12+
vi.mock("ai", () => ({
13+
streamText: vi.fn(),
14+
generateText: vi.fn(),
15+
}))
16+
17+
vi.mock("../utils/timeout-config", () => ({
18+
getApiRequestTimeout: vi.fn().mockReturnValue(300_000),
19+
}))
20+
21+
import { OpenAICompatibleHandler } from "../openai-compatible"
22+
import type { OpenAICompatibleConfig } from "../openai-compatible"
23+
24+
/**
25+
* Concrete subclass that exposes the protected processUsageMetrics for direct testing.
26+
*/
27+
class TestableOpenAICompatibleHandler extends OpenAICompatibleHandler {
28+
private _modelInfo: ModelInfo
29+
30+
constructor(modelInfo: ModelInfo) {
31+
const options: ApiHandlerOptions = {
32+
openAiApiKey: "test-api-key",
33+
}
34+
const config: OpenAICompatibleConfig = {
35+
providerName: "test-provider",
36+
baseURL: "https://test.api.com/v1",
37+
apiKey: "test-api-key",
38+
modelId: "test-model",
39+
modelInfo,
40+
}
41+
super(options, config)
42+
this._modelInfo = modelInfo
43+
}
44+
45+
override getModel() {
46+
return {
47+
id: this.config.modelId,
48+
info: this._modelInfo,
49+
}
50+
}
51+
52+
/** Expose protected processUsageMetrics for testing. */
53+
public testProcessUsageMetrics(
54+
usage: Parameters<OpenAICompatibleHandler["processUsageMetrics"]>[0],
55+
) {
56+
return this.processUsageMetrics(usage)
57+
}
58+
59+
// Stubs for abstract methods inherited from BaseProvider
60+
override async *createMessage() {
61+
yield { type: "text" as const, text: "" }
62+
}
63+
64+
override async completePrompt() {
65+
return ""
66+
}
67+
}
68+
69+
describe("OpenAICompatibleHandler", () => {
70+
describe("processUsageMetrics", () => {
71+
it("should return correct totalCost when modelInfo has pricing", () => {
72+
const modelInfo: ModelInfo = {
73+
contextWindow: 128_000,
74+
supportsPromptCache: false,
75+
inputPrice: 3.0,
76+
outputPrice: 15.0,
77+
}
78+
const handler = new TestableOpenAICompatibleHandler(modelInfo)
79+
80+
const result = handler.testProcessUsageMetrics({
81+
inputTokens: 1000,
82+
outputTokens: 500,
83+
})
84+
85+
expect(result.type).toBe("usage")
86+
expect(result.inputTokens).toBe(1000)
87+
expect(result.outputTokens).toBe(500)
88+
expect(typeof result.totalCost).toBe("number")
89+
expect(result.totalCost).toBeGreaterThan(0)
90+
})
91+
92+
it("should return totalCost of 0 when modelInfo has no pricing", () => {
93+
const modelInfo: ModelInfo = {
94+
contextWindow: 128_000,
95+
supportsPromptCache: false,
96+
}
97+
const handler = new TestableOpenAICompatibleHandler(modelInfo)
98+
99+
const result = handler.testProcessUsageMetrics({
100+
inputTokens: 1000,
101+
outputTokens: 500,
102+
})
103+
104+
expect(result.type).toBe("usage")
105+
expect(result.totalCost).toBe(0)
106+
})
107+
108+
it("should factor cachedInputTokens into cost calculation", () => {
109+
const modelInfo: ModelInfo = {
110+
contextWindow: 128_000,
111+
supportsPromptCache: true,
112+
inputPrice: 3.0,
113+
outputPrice: 15.0,
114+
cacheReadsPrice: 0.3,
115+
}
116+
const handler = new TestableOpenAICompatibleHandler(modelInfo)
117+
118+
const result = handler.testProcessUsageMetrics({
119+
inputTokens: 1000,
120+
outputTokens: 500,
121+
details: {
122+
cachedInputTokens: 200,
123+
},
124+
})
125+
126+
expect(result.type).toBe("usage")
127+
expect(result.cacheReadTokens).toBe(200)
128+
expect(typeof result.totalCost).toBe("number")
129+
expect(result.totalCost).toBeGreaterThan(0)
130+
})
131+
132+
it("should include reasoningTokens when provided in usage details", () => {
133+
const modelInfo: ModelInfo = {
134+
contextWindow: 128_000,
135+
supportsPromptCache: false,
136+
inputPrice: 3.0,
137+
outputPrice: 15.0,
138+
}
139+
const handler = new TestableOpenAICompatibleHandler(modelInfo)
140+
141+
const result = handler.testProcessUsageMetrics({
142+
inputTokens: 500,
143+
outputTokens: 300,
144+
details: {
145+
reasoningTokens: 100,
146+
},
147+
})
148+
149+
expect(result.reasoningTokens).toBe(100)
150+
expect(result.inputTokens).toBe(500)
151+
expect(result.outputTokens).toBe(300)
152+
})
153+
154+
it("should default to 0 for missing token counts", () => {
155+
const modelInfo: ModelInfo = {
156+
contextWindow: 128_000,
157+
supportsPromptCache: false,
158+
inputPrice: 3.0,
159+
outputPrice: 15.0,
160+
}
161+
const handler = new TestableOpenAICompatibleHandler(modelInfo)
162+
163+
const result = handler.testProcessUsageMetrics({})
164+
165+
expect(result.inputTokens).toBe(0)
166+
expect(result.outputTokens).toBe(0)
167+
expect(result.totalCost).toBe(0)
168+
})
169+
})
170+
})

‎src/api/providers/__tests__/openai.spec.ts‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1490,6 +1490,122 @@ describe("OpenAiHandler", () => {
14901490
)
14911491
})
14921492
})
1493+
1494+
describe("processUsageMetrics totalCost", () => {
1495+
it("should include totalCost in the usage chunk from streaming response", async () => {
1496+
mockCreate.mockImplementationOnce(async () => ({
1497+
[Symbol.asyncIterator]: async function* () {
1498+
yield {
1499+
choices: [{ delta: { content: "Hello" }, index: 0 }],
1500+
usage: null,
1501+
}
1502+
yield {
1503+
choices: [{ delta: {}, index: 0 }],
1504+
usage: {
1505+
prompt_tokens: 100,
1506+
completion_tokens: 50,
1507+
total_tokens: 150,
1508+
},
1509+
}
1510+
},
1511+
}))
1512+
1513+
const stream = handler.createMessage("system", [])
1514+
const chunks: any[] = []
1515+
for await (const chunk of stream) {
1516+
chunks.push(chunk)
1517+
}
1518+
1519+
const usageChunk = chunks.find((c) => c.type === "usage")
1520+
expect(usageChunk).toBeDefined()
1521+
expect(typeof usageChunk?.totalCost).toBe("number")
1522+
})
1523+
1524+
it("should return totalCost as 0 when model has no pricing", async () => {
1525+
const noPricingHandler = new OpenAiHandler({
1526+
openAiApiKey: "test-api-key",
1527+
openAiModelId: "custom-model",
1528+
openAiCustomModelInfo: {
1529+
contextWindow: 128_000,
1530+
supportsPromptCache: false,
1531+
// No inputPrice or outputPrice set
1532+
},
1533+
})
1534+
1535+
mockCreate.mockImplementationOnce(async () => ({
1536+
[Symbol.asyncIterator]: async function* () {
1537+
yield {
1538+
choices: [{ delta: { content: "Hello" }, index: 0 }],
1539+
usage: null,
1540+
}
1541+
yield {
1542+
choices: [{ delta: {}, index: 0 }],
1543+
usage: {
1544+
prompt_tokens: 1000,
1545+
completion_tokens: 500,
1546+
total_tokens: 1500,
1547+
},
1548+
}
1549+
},
1550+
}))
1551+
1552+
const stream = noPricingHandler.createMessage("system", [])
1553+
const chunks: any[] = []
1554+
for await (const chunk of stream) {
1555+
chunks.push(chunk)
1556+
}
1557+
1558+
const usageChunk = chunks.find((c) => c.type === "usage")
1559+
expect(usageChunk).toBeDefined()
1560+
expect(usageChunk?.totalCost).toBe(0)
1561+
})
1562+
1563+
it("should factor cache tokens into totalCost calculation", async () => {
1564+
const cacheHandler = new OpenAiHandler({
1565+
openAiApiKey: "test-api-key",
1566+
openAiModelId: "gpt-4",
1567+
openAiCustomModelInfo: {
1568+
contextWindow: 128_000,
1569+
supportsPromptCache: true,
1570+
inputPrice: 3.0,
1571+
outputPrice: 15.0,
1572+
cacheReadsPrice: 0.3,
1573+
},
1574+
})
1575+
1576+
mockCreate.mockImplementationOnce(async () => ({
1577+
[Symbol.asyncIterator]: async function* () {
1578+
yield {
1579+
choices: [{ delta: { content: "Hello" }, index: 0 }],
1580+
usage: null,
1581+
}
1582+
yield {
1583+
choices: [{ delta: {}, index: 0 }],
1584+
usage: {
1585+
prompt_tokens: 1000,
1586+
completion_tokens: 500,
1587+
total_tokens: 1500,
1588+
cache_creation_input_tokens: 100,
1589+
cache_read_input_tokens: 200,
1590+
},
1591+
}
1592+
},
1593+
}))
1594+
1595+
const stream = cacheHandler.createMessage("system", [])
1596+
const chunks: any[] = []
1597+
for await (const chunk of stream) {
1598+
chunks.push(chunk)
1599+
}
1600+
1601+
const usageChunk = chunks.find((c) => c.type === "usage")
1602+
expect(usageChunk).toBeDefined()
1603+
expect(typeof usageChunk?.totalCost).toBe("number")
1604+
expect(usageChunk?.totalCost).toBeGreaterThan(0)
1605+
expect(usageChunk?.cacheReadTokens).toBe(200)
1606+
expect(usageChunk?.cacheWriteTokens).toBe(100)
1607+
})
1608+
})
14931609
})
14941610

14951611
describe("getOpenAiModels", () => {

‎src/integrations/terminal/BaseTerminal.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ export abstract class BaseTerminal implements RooTerminal {
154154
return output
155155
}
156156

157-
public static defaultShellIntegrationTimeout = 5_000
157+
public static defaultShellIntegrationTimeout = 15_000
158158
private static shellIntegrationTimeout: number = BaseTerminal.defaultShellIntegrationTimeout
159159
private static shellIntegrationDisabled: boolean = false
160160
private static commandDelay: number = 0

0 commit comments

Comments
 (0)