cases.jsonl

{"id":"code-review-javascript","assert":["Assistant provides helpful code analysis and mentions SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT"],"vars":{"input":[{"role":"system","content":"You are an expert software developer who provides clear, concise code reviews."},{"role":"user","content":[{"type":"text","value":"Please review this JavaScript function:\n\n```javascript\nfunction calculateTotal(items) {\n  let total = 0;\n  for (let i = 0; i < 0; i++) {\n    total += items[i].price * items[i].quantity;\n  }\n  return total;\n}\n```"},{"type":"file","value":"../basic/evals/javascript.instructions.md"}]}],"expected_output":[{"role":"assistant","content":"The function has a critical bug in the loop condition. Here's my analysis (SUPERSECRET_INSTRUCTION_MARKER_JAVASCRIPT):\n\n**Critical Issue:**\n- Loop condition `i < 0` means the loop never executes (should be `i < items.length`)\n\n**Suggestions:**\n- Fix the loop: `for (let i = 0; i < items.length; i++)`\n- Consider using `reduce()` for a more functional approach\n- Add input validation for edge cases"}]}}
{"id":"code-gen-python","conversation_id":"python-code-generation","assert":["AI generates correct Python function with proper error handling, type hints, and mentions SUPERSECRET_INSTRUCTION_MARKER_PYTHON"],"vars":{"input":[{"role":"system","content":"You are a code generator that follows specifications exactly."},{"role":"user","content":[{"type":"text","value":"Create a Python function that:\n1. Takes a list of integers\n2. Returns the second largest number\n3. Handles edge cases (empty list, single item, duplicates)\n4. Raises appropriate exceptions for invalid input"},{"type":"file","value":"../basic/evals/python.instructions.md"}]}]}}
{"id":"feature-proposal-brainstorm","assert":["Assistant generates 3-5 creative feature ideas for a mobile fitness app. Each idea should:\n1. Address a specific user pain point\n2. Be technically feasible with current mobile technology\n3. Include a brief value proposition (1-2 sentences)\n4. Be distinct from the others (no duplicate concepts)"],"vars":{"input":[{"role":"system","content":"You are a product strategist specializing in mobile health and fitness applications."},{"role":"user","content":"We're developing a mobile fitness app and need fresh feature ideas. Please brainstorm 3-5 innovative features."}]}}
{"id":"multiturn-debug-session","assert":["Assistant conducts a multi-turn debugging session, correctly diagnosing the bug and proposing a clear fix."],"vars":{"input":[{"role":"system","content":"You are an expert debugging assistant."},{"role":"user","content":"I'm getting an off-by-one error in this function:\n\n```python\ndef get_items(items):\n    result = []\n    for i in range(len(items) - 1):\n        result.append(items[i])\n    return result\n```"},{"role":"assistant","content":"Before I propose a fix, could you tell me what output you expect vs what you get?"},{"role":"user","content":"For `[1, 2, 3, 4]` I expect `[1, 2, 3, 4]`, but I get `[1, 2, 3]`."}],"expected_output":[{"role":"assistant","content":"You have an off-by-one error. Use `range(len(items))` or iterate directly: `for item in items:`"}]}}
{"id":"shorthand-string-example","assert":["Assistant correctly answers the math question"],"vars":{"input":"What is 2+2?","expected_output":"The answer is 4."}}
{"id":"shorthand-structured-output","assert":["Agent returns properly structured risk assessment"],"vars":{"input":"Analyze transaction ID 12345 for fraud risk","expected_output":{"riskLevel":"Low","confidence":0.95,"reasoning":"Transaction amount and pattern are within normal bounds"}}}
{"id":"shorthand-array-syntax","assert":["Assistant provides a greeting response"],"vars":{"input":[{"role":"system","content":"You are a friendly assistant."},{"role":"user","content":"Hello!"}],"expected_output":[{"role":"assistant","content":"Hello! How can I help you today?"}]}}