memory-path-engine/benchmarks/structured_memory/structure_ablation_benchmark.json at master · ly85206559/memory-path-engine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
{
  "dataset_id": "structure-ablation-benchmark-v1",
  "dataset_name": "Structure ablation benchmark",
  "domain_pack_name": "example_runbook_pack",
  "document_directory": "../../examples/runbook_pack/runbooks",
  "cases": [
    {
      "case_id": "ablation-001",
      "query": "If rollback does not recover service and slow queries continue, who should we escalate to?",
      "tags": ["ablation", "multi-hop", "escalation", "runbook"],
      "expectation": {
        "evidence_node_ids": [
          "01_api_incident_runbook:5",
          "01_api_incident_runbook:6"
        ],
        "minimum_evidence_matches": 2,
        "required_edge_types": ["depends_on"],
        "required_semantic_roles": ["escalation"],
        "path_scope": "best_path",
        "path": {
          "match_mode": "subsequence",
          "steps": [
            {
              "node_id": "01_api_incident_runbook:5"
            },
            {
              "node_id": "01_api_incident_runbook:6",
              "via_edge_type": "depends_on"
            }
          ]
        }
      }
    }
  ]
}