Skip to content

Commit b8e9353

Browse files
cameroncookecodex
andcommitted
fix(benchmarks): Reject removed sequence config key
Reject the old sequence suite config key with an explicit migration message so migrated benchmarks do not silently drop sequence checks. Co-Authored-By: OpenAI Codex <noreply@openai.com>
1 parent 3ab280a commit b8e9353

2 files changed

Lines changed: 14 additions & 0 deletions

File tree

src/benchmarks/claude-ui/__tests__/claude-ui-benchmark.test.ts

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,19 @@ describe('Claude UI benchmark analysis', () => {
315315
'weather.yml',
316316
),
317317
).toThrow('weather.yml.allowedVariance: removed; baselines are observed data only');
318+
319+
expect(() =>
320+
readConfig(
321+
{
322+
name: 'weather',
323+
prompt: 'prompt.md',
324+
sequence: { mode: 'fail' },
325+
},
326+
'weather.yml',
327+
),
328+
).toThrow(
329+
'weather.yml.sequence: removed; use baselineToolSequence for observed sequence reporting',
330+
);
318331
});
319332

320333
it('rejects malformed failure pattern regexes when loading config', () => {

src/benchmarks/claude-ui/config.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -247,6 +247,7 @@ function rejectRemovedConfigKeys(raw: Record<string, unknown>, source: string):
247247
allowedVariance: 'removed; baselines are observed data only',
248248
expectedFailures: 'removed; benchmark stumbles are observed data',
249249
expectedToolSequence: 'renamed to baselineToolSequence',
250+
sequence: 'removed; use baselineToolSequence for observed sequence reporting',
250251
};
251252
for (const [key, message] of Object.entries(removedKeys)) {
252253
if (raw[key] !== undefined) throw new Error(`${source}.${key}: ${message}`);

0 commit comments

Comments
 (0)