|
| 1 | +""" |
| 2 | +Simple trace assertion - just check that expected spans exist with required attributes. |
| 3 | +Much simpler than the tree-based approach. |
| 4 | +""" |
| 5 | + |
| 6 | +import json |
| 7 | +from typing import Any |
| 8 | + |
| 9 | + |
| 10 | +def load_traces(traces_file: str) -> list[dict[str, Any]]: |
| 11 | + """Load traces from a JSONL file.""" |
| 12 | + traces = [] |
| 13 | + with open(traces_file, "r", encoding="utf-8") as f: |
| 14 | + for line in f: |
| 15 | + if line.strip(): |
| 16 | + traces.append(json.loads(line)) |
| 17 | + return traces |
| 18 | + |
| 19 | + |
| 20 | +def load_expected_traces(expected_file: str) -> list[dict[str, Any]]: |
| 21 | + """Load expected trace definitions from a JSON file.""" |
| 22 | + with open(expected_file, "r", encoding="utf-8") as f: |
| 23 | + data = json.load(f) |
| 24 | + return data.get("required_spans", []) |
| 25 | + |
| 26 | + |
| 27 | +def get_attributes(span: dict[str, Any]) -> dict[str, Any]: |
| 28 | + """ |
| 29 | + Parse attributes from a span. |
| 30 | + Supports both formats: |
| 31 | + - Old format: 'Attributes' as a JSON string |
| 32 | + - New format: 'attributes' as a dict |
| 33 | + """ |
| 34 | + # New format: attributes is already a dict |
| 35 | + if "attributes" in span and isinstance(span["attributes"], dict): |
| 36 | + return span["attributes"] |
| 37 | + # Old format: Attributes is a JSON string |
| 38 | + attributes_str = span.get("Attributes", "{}") |
| 39 | + try: |
| 40 | + return json.loads(attributes_str) |
| 41 | + except json.JSONDecodeError: |
| 42 | + return {} |
| 43 | + |
| 44 | + |
| 45 | +def matches_value(expected_value: Any, actual_value: Any) -> bool: |
| 46 | + """ |
| 47 | + Check if an actual value matches the expected value. |
| 48 | + Supports: |
| 49 | + - List of possible values: ["value1", "value2"] |
| 50 | + - Wildcard: "*" (any value accepted) |
| 51 | + - Exact match: "value" |
| 52 | + """ |
| 53 | + # Wildcard - accept any value |
| 54 | + if expected_value == "*": |
| 55 | + return True |
| 56 | + # List of possible values |
| 57 | + if isinstance(expected_value, list): |
| 58 | + return actual_value in expected_value |
| 59 | + # Exact match |
| 60 | + return expected_value == actual_value |
| 61 | + |
| 62 | + |
| 63 | +def matches_expected(span: dict[str, Any], expected: dict[str, Any]) -> bool: |
| 64 | + """ |
| 65 | + Check if a span matches the expected definition. |
| 66 | + Supports both formats: |
| 67 | + - Old format: 'Name', 'SpanType' fields |
| 68 | + - New format: 'name', 'attributes.span_type' fields |
| 69 | + """ |
| 70 | + # Check name - can be a string or list of possible names |
| 71 | + expected_name = expected.get("name") |
| 72 | + # Support both old format (Name) and new format (name) |
| 73 | + actual_name = span.get("name") or span.get("Name") |
| 74 | + if isinstance(expected_name, list): |
| 75 | + if actual_name not in expected_name: |
| 76 | + return False |
| 77 | + elif expected_name != actual_name: |
| 78 | + return False |
| 79 | + # Check span type if specified |
| 80 | + if "span_type" in expected: |
| 81 | + # Old format: SpanType field |
| 82 | + # New format: attributes.span_type field |
| 83 | + actual_span_type = span.get("SpanType") |
| 84 | + if not actual_span_type: |
| 85 | + actual_attrs = get_attributes(span) |
| 86 | + actual_span_type = actual_attrs.get("span_type") |
| 87 | + if actual_span_type != expected["span_type"]: |
| 88 | + return False |
| 89 | + # Check attributes if specified |
| 90 | + if "attributes" in expected: |
| 91 | + actual_attrs = get_attributes(span) |
| 92 | + for key, expected_value in expected["attributes"].items(): |
| 93 | + if key not in actual_attrs: |
| 94 | + return False |
| 95 | + # Use flexible value matching |
| 96 | + if not matches_value(expected_value, actual_attrs[key]): |
| 97 | + return False |
| 98 | + return True |
| 99 | + |
| 100 | + |
| 101 | +def assert_traces(traces_file: str, expected_file: str) -> None: |
| 102 | + """ |
| 103 | + Assert that all expected traces exist in the traces file. |
| 104 | + Args: |
| 105 | + traces_file: Path to the traces.jsonl file |
| 106 | + expected_file: Path to the expected_traces.json file |
| 107 | + Raises: |
| 108 | + AssertionError: If any expected trace is not found |
| 109 | + """ |
| 110 | + traces = load_traces(traces_file) |
| 111 | + expected_spans = load_expected_traces(expected_file) |
| 112 | + print(f"Loaded {len(traces)} traces from {traces_file}") |
| 113 | + print(f"Checking {len(expected_spans)} expected spans...") |
| 114 | + missing_spans = [] |
| 115 | + for expected in expected_spans: |
| 116 | + # Find a matching span |
| 117 | + found = False |
| 118 | + name = expected["name"] |
| 119 | + # Handle both string and list of names |
| 120 | + name_str = name if isinstance(name, str) else f"[{' | '.join(name)}]" |
| 121 | + |
| 122 | + for span in traces: |
| 123 | + if matches_expected(span, expected): |
| 124 | + found = True |
| 125 | + print(f"✓ Found span: {name_str}") |
| 126 | + break |
| 127 | + if not found: |
| 128 | + missing_spans.append(name_str) |
| 129 | + print(f"✗ Missing span: {name_str}") |
| 130 | + |
| 131 | + print("Traces file content:") |
| 132 | + with open(traces_file, "r", encoding="utf-8") as f: |
| 133 | + print(f.read()) |
| 134 | + if missing_spans: |
| 135 | + print(f"\n=== Dumping raw traces from {traces_file} ===") |
| 136 | + with open(traces_file, "r", encoding="utf-8") as f: |
| 137 | + print(f.read()) |
| 138 | + print("\n=== End of traces dump ===\n") |
| 139 | + raise AssertionError( |
| 140 | + f"Missing expected spans: {', '.join(missing_spans)}\n" |
| 141 | + f"Expected {len(expected_spans)} spans, found {len(expected_spans) - len(missing_spans)}" |
| 142 | + ) |
| 143 | + print(f"\n✓ All {len(expected_spans)} expected spans found!") |
0 commit comments