memory-path-engine/benchmarks/structured_memory/contract_exception_priming_benchmark.json at master · ly85206559/memory-path-engine · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
{
  "dataset_id": "contract-exception-priming-benchmark-v1",
  "dataset_name": "Contract exception priming benchmark",
  "domain_pack_name": "example_contract_pack",
  "document_directory": "../../examples/contract_priming_pack/contracts",
  "cases": [
    {
      "case_id": "prime-001",
      "query": "Unless alpha service credits are disputed, Buyer must pay alpha service invoices within 30 days of receipt.",
      "tags": ["priming", "contract", "exception", "alpha"],
      "expectation": {
        "evidence_node_ids": ["dynamic_exception_priming_contract:2"],
        "minimum_evidence_matches": 1
      }
    },
    {
      "case_id": "prime-002",
      "query": "Unless alpha service credits are disputed, Buyer must pay alpha service invoices within 30 days of receipt.",
      "tags": ["priming", "contract", "exception", "alpha"],
      "expectation": {
        "evidence_node_ids": ["dynamic_exception_priming_contract:2"],
        "minimum_evidence_matches": 1
      }
    },
    {
      "case_id": "prime-003",
      "query": "Unless alpha service credits are disputed, Buyer must pay alpha service invoices within 30 days of receipt.",
      "tags": ["priming", "contract", "exception", "alpha"],
      "expectation": {
        "evidence_node_ids": ["dynamic_exception_priming_contract:2"],
        "minimum_evidence_matches": 1
      }
    },
    {
      "case_id": "prime-004",
      "query": "Unless alpha service credits are disputed, Buyer must pay alpha service invoices within 30 days of receipt.",
      "tags": ["priming", "contract", "exception", "alpha"],
      "expectation": {
        "evidence_node_ids": ["dynamic_exception_priming_contract:2"],
        "minimum_evidence_matches": 1
      }
    },
    {
      "case_id": "prime-005",
      "query": "Unless alpha service credits are disputed, Buyer must pay alpha service invoices within 30 days of receipt.",
      "tags": ["priming", "contract", "exception", "alpha"],
      "expectation": {
        "evidence_node_ids": ["dynamic_exception_priming_contract:2"],
        "minimum_evidence_matches": 1
      }
    },
    {
      "case_id": "prime-006",
      "query": "Unless alpha service credits are disputed, Buyer must pay alpha service invoices within 30 days of receipt.",
      "tags": ["priming", "contract", "exception", "alpha"],
      "expectation": {
        "evidence_node_ids": ["dynamic_exception_priming_contract:2"],
        "minimum_evidence_matches": 1
      }
    },
    {
      "case_id": "prime-007",
      "query": "Unless alpha service credits are disputed, Buyer must pay alpha service invoices within 30 days of receipt.",
      "tags": ["priming", "contract", "exception", "alpha"],
      "expectation": {
        "evidence_node_ids": ["dynamic_exception_priming_contract:2"],
        "minimum_evidence_matches": 1
      }
    },
    {
      "case_id": "prime-008",
      "query": "Unless alpha service credits are disputed, Buyer must pay alpha service invoices within 30 days of receipt.",
      "tags": ["priming", "contract", "exception", "alpha"],
      "expectation": {
        "evidence_node_ids": ["dynamic_exception_priming_contract:2"],
        "minimum_evidence_matches": 1
      }
    },
    {
      "case_id": "probe-009",
      "query": "What clause comes after beta termination evidence is collected?",
      "tags": ["priming", "contract", "termination", "beta", "probe"],
      "expectation": {
        "evidence_node_ids": ["dynamic_exception_priming_contract:7"],
        "minimum_evidence_matches": 1,
        "path_scope": "best_path",
        "path": {
          "match_mode": "prefix",
          "steps": [
            {
              "node_id": "dynamic_exception_priming_contract:7",
              "via_edge_type": null
            },
            {
              "node_id": "dynamic_exception_priming_contract:6",
              "via_edge_type": "depends_on"
            }
          ]
        }
      }
    }
  ]
}