trpc-agent-python/examples/optimization/eval_optimize_loop/evalsets/train_baseline.evalset.json at c6817806284e8812d910a5d46cde11109a00e800 · trpc-group/trpc-agent-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
{
  "eval_set_id": "train_baseline",
  "name": "Training set - baseline traces",
  "description": "3 training cases with baseline prompt traces. Contains: optimizable, always-fail, and regression cases.",
  "eval_cases": [
    {
      "eval_id": "case_train_optimizable",
      "eval_mode": "trace",
      "conversation": [
        {
          "invocation_id": "t1",
          "user_content": {
            "parts": [{"text": "小明有 5 个苹果，小红给了他 7 个苹果，小明现在一共有多少个苹果？"}],
            "role": "user"
          },
          "final_response": {
            "parts": [{"text": "答案：12 个"}],
            "role": "model"
          }
        }
      ],
      "actual_conversation": [
        {
          "invocation_id": "t1",
          "user_content": {
            "parts": [{"text": "小明有 5 个苹果，小红给了他 7 个苹果，小明现在一共有多少个苹果？"}],
            "role": "user"
          },
          "final_response": {
            "parts": [{"text": "答案：13 个"}],
            "role": "model"
          }
        }
      ],
      "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}}
    },
    {
      "eval_id": "case_train_always_fail",
      "eval_mode": "trace",
      "conversation": [
        {
          "invocation_id": "t2",
          "user_content": {
            "parts": [{"text": "一辆车开了 60 公里每小时，开了 2 小时，一共开了多少公里？"}],
            "role": "user"
          },
          "final_response": {
            "parts": [{"text": "答案：120 公里"}],
            "role": "model"
          }
        }
      ],
      "actual_conversation": [
        {
          "invocation_id": "t2",
          "user_content": {
            "parts": [{"text": "一辆车开了 60 公里每小时，开了 2 小时，一共开了多少公里？"}],
            "role": "user"
          },
          "final_response": {
            "parts": [{"text": "120 公里"}],
            "role": "model"
          }
        }
      ],
      "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}}
    },
    {
      "eval_id": "case_train_regression",
      "eval_mode": "trace",
      "conversation": [
        {
          "invocation_id": "t3",
          "user_content": {
            "parts": [{"text": "5 排座位，每排 8 个，一共多少个？"}],
            "role": "user"
          },
          "final_response": {
            "parts": [{"text": "答案：40 个"}],
            "role": "model"
          }
        }
      ],
      "actual_conversation": [
        {
          "invocation_id": "t3",
          "user_content": {
            "parts": [{"text": "5 排座位，每排 8 个，一共多少个？"}],
            "role": "user"
          },
          "final_response": {
            "parts": [{"text": "答案：40 个"}],
            "role": "model"
          }
        }
      ],
      "session_input": {"app_name": "eval_opt_loop", "user_id": "trainer", "state": {}}
    }
  ]
}