|
| 1 | +import re |
| 2 | +import json |
| 3 | +from pathlib import Path |
| 4 | +from collections import defaultdict, Counter |
| 5 | + |
| 6 | +ROOT = Path('.') |
| 7 | +TEST_ROOT = ROOT / 'tests' |
| 8 | +CHECKLIST_PATH = Path('artifacts/canonical_full_run/run1/generated_checklist.json') |
| 9 | +OUT_DIR = Path('artifacts/test_linkage') |
| 10 | +OUT_DIR.mkdir(parents=True, exist_ok=True) |
| 11 | + |
| 12 | +# Build test registry |
| 13 | +# For each test file, capture: file path, list of test functions, markers, raw content |
| 14 | +registry = [] |
| 15 | +hex_id_re = re.compile(r"[0-9a-f]{10,}") |
| 16 | + |
| 17 | +def extract_tests_from_file(p: Path): |
| 18 | + txt = p.read_text() |
| 19 | + functions = re.findall(r"def\s+(test_[A-Za-z0-9_]+)", txt) |
| 20 | + markers = re.findall(r"@pytest\.mark\.([A-Za-z0-9_]+)", txt) |
| 21 | + # build list of full test ids: file::function |
| 22 | + tests = [f"{str(p)}::{fn}" for fn in functions] |
| 23 | + return { |
| 24 | + 'path': str(p), |
| 25 | + 'functions': functions, |
| 26 | + 'markers': markers, |
| 27 | + 'tests_full': tests, |
| 28 | + 'content': txt, |
| 29 | + } |
| 30 | + |
| 31 | +for f in sorted(TEST_ROOT.rglob('test_*.py')): |
| 32 | + registry.append(extract_tests_from_file(f)) |
| 33 | + |
| 34 | +# Index by signals: id substrings, pointers, file names |
| 35 | +id_index = defaultdict(list) # id -> list of test_full |
| 36 | +ptr_index = defaultdict(list) |
| 37 | +file_index = defaultdict(list) |
| 38 | + |
| 39 | +for entry in registry: |
| 40 | + content = entry['content'] |
| 41 | + # find hex ids in content |
| 42 | + for m in hex_id_re.findall(content): |
| 43 | + id_index[m].extend(entry['tests_full']) |
| 44 | + # find pointer-like strings ("/something") appearing in content |
| 45 | + ptrs = re.findall(r"(/[-A-Za-z0-9_/$]+)", content) |
| 46 | + for p in ptrs: |
| 47 | + ptr_index[p].extend(entry['tests_full']) |
| 48 | + # index by file stem |
| 49 | + stem = Path(entry['path']).stem |
| 50 | + file_index[stem].extend(entry['tests_full']) |
| 51 | + |
| 52 | +# Load checklist |
| 53 | +items = json.loads(CHECKLIST_PATH.read_text()) |
| 54 | + |
| 55 | +attached = {} |
| 56 | +items_with = 0 |
| 57 | +items_without = 0 |
| 58 | +unlinked_by_category = Counter() |
| 59 | + |
| 60 | +for it in items: |
| 61 | + itid = it.get('id') |
| 62 | + ptr = it.get('ptr') |
| 63 | + category = it.get('category') or 'unknown' |
| 64 | + matches = set() |
| 65 | + # Direct ID match |
| 66 | + if itid: |
| 67 | + # hex ids may be longer/shorter; match any registry id that contains this substring |
| 68 | + for key, tests in id_index.items(): |
| 69 | + if itid in key or key in itid: |
| 70 | + matches.update(tests) |
| 71 | + # direct ptr match |
| 72 | + if ptr: |
| 73 | + if ptr in ptr_index: |
| 74 | + matches.update(ptr_index[ptr]) |
| 75 | + # also canonical pointer variations: /$schema -> $schema |
| 76 | + short = ptr.lstrip('/') |
| 77 | + for k, tests in file_index.items(): |
| 78 | + if short.startswith(k) or k.startswith(short): |
| 79 | + matches.update(tests) |
| 80 | + # Also search for exact id in any test content (substring scan) |
| 81 | + if itid: |
| 82 | + for entry in registry: |
| 83 | + if itid in entry['content']: |
| 84 | + matches.update(entry['tests_full']) |
| 85 | + # Stabilize ordering |
| 86 | + matches = sorted(set(matches)) |
| 87 | + # Rule: attach only when clear direct match exists |
| 88 | + # 'Clear' = at least one test path that contains the exact id or ptr or has function name referencing ptr |
| 89 | + authoritative = [] |
| 90 | + for t in matches: |
| 91 | + # inspect file content for exact id or ptr |
| 92 | + fpath, fn = t.split('::') |
| 93 | + content = Path(fpath).read_text() |
| 94 | + if itid and itid in content: |
| 95 | + authoritative.append(t) |
| 96 | + elif ptr and ptr in content: |
| 97 | + authoritative.append(t) |
| 98 | + else: |
| 99 | + # check if fn name contains meaningful words from ptr |
| 100 | + if ptr: |
| 101 | + short = ptr.strip('/').replace('/', '_') |
| 102 | + if short and short in fn: |
| 103 | + authoritative.append(t) |
| 104 | + authoritative = sorted(set(authoritative)) |
| 105 | + |
| 106 | + if authoritative: |
| 107 | + attached[itid] = authoritative |
| 108 | + items_with += 1 |
| 109 | + else: |
| 110 | + attached[itid] = [] |
| 111 | + items_without += 1 |
| 112 | + unlinked_by_category[category] += 1 |
| 113 | + |
| 114 | +# Write report |
| 115 | +report = { |
| 116 | + 'total_checklist_items': len(items), |
| 117 | + 'items_with_test_refs': items_with, |
| 118 | + 'items_without_test_refs': items_without, |
| 119 | + 'unlinked_by_category': dict(sorted(unlinked_by_category.items())), |
| 120 | + 'attached_refs': dict(sorted(attached.items())), |
| 121 | + 'gate_report': { |
| 122 | + 'gate': 'tests_attached', |
| 123 | + 'mode': 'report_only', |
| 124 | + 'do_not_halt': True, |
| 125 | + 'linked_count': items_with, |
| 126 | + 'total_count': len(items), |
| 127 | + 'percent_linked': round((items_with / len(items)) * 100, 2) if len(items) else 0.0, |
| 128 | + 'blocking_invariants': [] |
| 129 | + } |
| 130 | +} |
| 131 | + |
| 132 | +OUT = OUT_DIR / 'report.json' |
| 133 | +OUT.write_text(json.dumps(report, indent=2, sort_keys=True)) |
| 134 | +print(json.dumps(report, indent=2, sort_keys=True)) |
0 commit comments