memory-path-engine/benchmarks/structured_memory/exception_override_benchmark.json at master · ly85206559/memory-path-engine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
{
  "dataset_id": "exception-override-benchmark-v1",
  "dataset_name": "Exception override benchmark",
  "domain_pack_name": "example_contract_pack",
  "document_directory": "../../examples/graph_contract_pack/contracts",
  "cases": [
    {
      "case_id": "exception-001",
      "query": "What overrides the normal 30 day payment rule when goods are defective?",
      "tags": ["exception", "override", "payment"],
      "expectation": {
        "evidence_node_ids": [
          "01_exception_override_contract:2",
          "01_exception_override_contract:3"
        ],
        "minimum_evidence_matches": 2,
        "activation_trace": {
          "match_mode": "subsequence",
          "steps": [
            {
              "node_id": "01_exception_override_contract:3",
              "is_seed": true,
              "hop": 0
            },
            {
              "node_id": "01_exception_override_contract:2",
              "edge_type": "depends_on",
              "hop": 1
            },
            {
              "node_id": "01_exception_override_contract:1",
              "edge_type": "exception_to",
              "hop": 2
            }
          ]
        },
        "required_trace_stop_reasons": ["below_threshold"],
        "min_activation_trace_length": 5,
        "max_activation_trace_length": 5,
        "required_semantic_roles": ["exception", "remedy"],
        "required_contradiction_pairs": [
          ["01_exception_override_contract:1", "01_exception_override_contract:2"]
        ]
      }
    }
  ]
}