|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Compare spar output against OSATE reference data. |
| 3 | +
|
| 4 | +Usage: |
| 5 | + python3 tools/osate-conformance/compare.py [--model BasicHierarchy] |
| 6 | +
|
| 7 | +Runs spar on the same test models that OSATE processed and compares: |
| 8 | +- Component counts (total, per category) |
| 9 | +- Feature counts |
| 10 | +- Connection counts |
| 11 | +- Component tree structure (names, categories, hierarchy) |
| 12 | +""" |
| 13 | + |
| 14 | +import argparse |
| 15 | +import json |
| 16 | +import os |
| 17 | +import subprocess |
| 18 | +import sys |
| 19 | +import xml.etree.ElementTree as ET |
| 20 | + |
| 21 | +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) |
| 22 | +PROJECT_ROOT = os.path.join(SCRIPT_DIR, "..", "..") |
| 23 | +REFERENCE_DIR = os.path.join(SCRIPT_DIR, "reference-data") |
| 24 | +TEST_DATA_DIR = os.path.join(PROJECT_ROOT, "test-data", "osate2") |
| 25 | + |
| 26 | +# Same models as the EASE script |
| 27 | +TEST_MODELS = [ |
| 28 | + ("BasicHierarchy.aadl", "BasicHierarchy::Top.Impl"), |
| 29 | + ("BasicBinding.aadl", "BasicBinding::Sys.Impl"), |
| 30 | + ("BasicEndToEndFlow.aadl", "BasicEndToEndFlow::Sys.Impl"), |
| 31 | + ("DigitalControlSystem.aadl", "DigitalControlSystem::DCS.Impl"), |
| 32 | + ("FlightSystem.aadl", "FlightSystem::FlightSystem.Impl"), |
| 33 | + ("GPSSystem.aadl", "GPSSystem::GPS.Impl"), |
| 34 | +] |
| 35 | + |
| 36 | + |
| 37 | +def run_spar(aadl_file, root_classifier): |
| 38 | + """Run spar instance --format json and return parsed JSON.""" |
| 39 | + cmd = [ |
| 40 | + "cargo", "run", "--release", "-p", "spar", "--", |
| 41 | + "instance", "--root", root_classifier, "--format", "json", |
| 42 | + aadl_file, |
| 43 | + ] |
| 44 | + result = subprocess.run( |
| 45 | + cmd, capture_output=True, text=True, |
| 46 | + cwd=PROJECT_ROOT, |
| 47 | + ) |
| 48 | + if result.returncode != 0: |
| 49 | + return None, result.stderr |
| 50 | + try: |
| 51 | + return json.loads(result.stdout), None |
| 52 | + except json.JSONDecodeError as e: |
| 53 | + return None, f"JSON parse error: {e}\nstdout: {result.stdout[:500]}" |
| 54 | + |
| 55 | + |
| 56 | +def run_spar_analyze(aadl_file, root_classifier): |
| 57 | + """Run spar analyze --format json and return parsed JSON.""" |
| 58 | + cmd = [ |
| 59 | + "cargo", "run", "--release", "-p", "spar", "--", |
| 60 | + "analyze", "--root", root_classifier, "--format", "json", |
| 61 | + aadl_file, |
| 62 | + ] |
| 63 | + result = subprocess.run( |
| 64 | + cmd, capture_output=True, text=True, |
| 65 | + cwd=PROJECT_ROOT, |
| 66 | + ) |
| 67 | + if result.returncode != 0: |
| 68 | + return None, result.stderr |
| 69 | + try: |
| 70 | + return json.loads(result.stdout), None |
| 71 | + except json.JSONDecodeError as e: |
| 72 | + return None, f"JSON parse error: {e}" |
| 73 | + |
| 74 | + |
| 75 | +def load_osate_reference(model_base): |
| 76 | + """Load OSATE reference JSON for a model.""" |
| 77 | + json_path = os.path.join(REFERENCE_DIR, "instances", f"{model_base}.json") |
| 78 | + if not os.path.exists(json_path): |
| 79 | + return None |
| 80 | + with open(json_path) as f: |
| 81 | + return json.load(f) |
| 82 | + |
| 83 | + |
| 84 | +def load_osate_analysis(model_base): |
| 85 | + """Load OSATE analysis reference JSON.""" |
| 86 | + json_path = os.path.join(REFERENCE_DIR, "analysis", f"{model_base}.json") |
| 87 | + if not os.path.exists(json_path): |
| 88 | + return None |
| 89 | + with open(json_path) as f: |
| 90 | + return json.load(f) |
| 91 | + |
| 92 | + |
| 93 | +def count_spar_components(node): |
| 94 | + """Count components in spar's instance JSON tree.""" |
| 95 | + if node is None: |
| 96 | + return 0 |
| 97 | + count = 1 |
| 98 | + for child in node.get("children", []): |
| 99 | + count += count_spar_components(child) |
| 100 | + return count |
| 101 | + |
| 102 | + |
| 103 | +def count_spar_features(node): |
| 104 | + """Count features in spar's instance JSON tree.""" |
| 105 | + if node is None: |
| 106 | + return 0 |
| 107 | + count = len(node.get("features", [])) |
| 108 | + for child in node.get("children", []): |
| 109 | + count += count_spar_features(child) |
| 110 | + return count |
| 111 | + |
| 112 | + |
| 113 | +def count_spar_connections(node): |
| 114 | + """Count connections in spar's instance JSON tree.""" |
| 115 | + if node is None: |
| 116 | + return 0 |
| 117 | + count = len(node.get("connections", [])) |
| 118 | + for child in node.get("children", []): |
| 119 | + count += count_spar_connections(child) |
| 120 | + return count |
| 121 | + |
| 122 | + |
| 123 | +def compare_trees(osate_tree, spar_tree, path="root"): |
| 124 | + """Compare component trees structurally. Returns list of differences.""" |
| 125 | + diffs = [] |
| 126 | + |
| 127 | + if osate_tree is None or spar_tree is None: |
| 128 | + diffs.append(f"{path}: one tree is None") |
| 129 | + return diffs |
| 130 | + |
| 131 | + # Compare name |
| 132 | + osate_name = osate_tree.get("name", "").lower() |
| 133 | + spar_name = spar_tree.get("name", "").lower() |
| 134 | + if osate_name != spar_name: |
| 135 | + diffs.append(f"{path}: name mismatch: OSATE='{osate_name}' spar='{spar_name}'") |
| 136 | + |
| 137 | + # Compare category |
| 138 | + osate_cat = osate_tree.get("category", "").lower() |
| 139 | + spar_cat = spar_tree.get("category", "").lower() |
| 140 | + if osate_cat != spar_cat: |
| 141 | + diffs.append(f"{path}: category mismatch: OSATE='{osate_cat}' spar='{spar_cat}'") |
| 142 | + |
| 143 | + # Compare child count |
| 144 | + osate_children = osate_tree.get("children", []) |
| 145 | + spar_children = spar_tree.get("children", []) |
| 146 | + if len(osate_children) != len(spar_children): |
| 147 | + diffs.append( |
| 148 | + f"{path}: child count mismatch: OSATE={len(osate_children)} " |
| 149 | + f"spar={len(spar_children)}" |
| 150 | + ) |
| 151 | + |
| 152 | + # Compare children by name |
| 153 | + osate_by_name = {c["name"].lower(): c for c in osate_children} |
| 154 | + spar_by_name = {c["name"].lower(): c for c in spar_children} |
| 155 | + |
| 156 | + for name in sorted(set(osate_by_name.keys()) | set(spar_by_name.keys())): |
| 157 | + if name not in osate_by_name: |
| 158 | + diffs.append(f"{path}/{name}: exists in spar but not OSATE") |
| 159 | + elif name not in spar_by_name: |
| 160 | + diffs.append(f"{path}/{name}: exists in OSATE but not spar") |
| 161 | + else: |
| 162 | + child_diffs = compare_trees( |
| 163 | + osate_by_name[name], spar_by_name[name], f"{path}/{name}" |
| 164 | + ) |
| 165 | + diffs.extend(child_diffs) |
| 166 | + |
| 167 | + return diffs |
| 168 | + |
| 169 | + |
| 170 | +def compare_model(filename, classifier, verbose=False): |
| 171 | + """Compare one model between OSATE and spar.""" |
| 172 | + model_base = os.path.splitext(filename)[0] |
| 173 | + aadl_path = os.path.join(TEST_DATA_DIR, filename) |
| 174 | + |
| 175 | + print(f"\n{'='*60}") |
| 176 | + print(f"Model: {filename} [{classifier}]") |
| 177 | + print(f"{'='*60}") |
| 178 | + |
| 179 | + if not os.path.exists(aadl_path): |
| 180 | + print(f" SKIP: {aadl_path} not found") |
| 181 | + return None |
| 182 | + |
| 183 | + # Load OSATE reference |
| 184 | + osate_ref = load_osate_reference(model_base) |
| 185 | + osate_analysis = load_osate_analysis(model_base) |
| 186 | + |
| 187 | + if osate_ref is None: |
| 188 | + print(" SKIP: No OSATE reference data (run generate-references.sh first)") |
| 189 | + return None |
| 190 | + |
| 191 | + # Run spar |
| 192 | + spar_instance, err = run_spar(aadl_path, classifier) |
| 193 | + if spar_instance is None: |
| 194 | + print(f" FAIL: spar instance failed: {err}") |
| 195 | + return False |
| 196 | + |
| 197 | + spar_analysis, err = run_spar_analyze(aadl_path, classifier) |
| 198 | + |
| 199 | + # Compare counts |
| 200 | + osate_comp_count = osate_analysis.get("component_count", 0) if osate_analysis else 0 |
| 201 | + osate_conn_count = osate_analysis.get("connection_count", 0) if osate_analysis else 0 |
| 202 | + osate_feat_count = osate_analysis.get("feature_count", 0) if osate_analysis else 0 |
| 203 | + |
| 204 | + spar_instance_node = spar_instance.get("instance") if spar_instance else None |
| 205 | + spar_comp_count = count_spar_components(spar_instance_node) |
| 206 | + spar_conn_count = count_spar_connections(spar_instance_node) |
| 207 | + spar_feat_count = count_spar_features(spar_instance_node) |
| 208 | + |
| 209 | + passed = True |
| 210 | + |
| 211 | + # Component count |
| 212 | + if osate_comp_count == spar_comp_count: |
| 213 | + print(f" PASS: component count = {spar_comp_count}") |
| 214 | + else: |
| 215 | + print(f" DIFF: component count: OSATE={osate_comp_count} spar={spar_comp_count}") |
| 216 | + passed = False |
| 217 | + |
| 218 | + # Connection count |
| 219 | + if osate_conn_count == spar_conn_count: |
| 220 | + print(f" PASS: connection count = {spar_conn_count}") |
| 221 | + else: |
| 222 | + print(f" DIFF: connection count: OSATE={osate_conn_count} spar={spar_conn_count}") |
| 223 | + passed = False |
| 224 | + |
| 225 | + # Feature count |
| 226 | + if osate_feat_count == spar_feat_count: |
| 227 | + print(f" PASS: feature count = {spar_feat_count}") |
| 228 | + else: |
| 229 | + print(f" DIFF: feature count: OSATE={osate_feat_count} spar={spar_feat_count}") |
| 230 | + passed = False |
| 231 | + |
| 232 | + # Structural tree comparison |
| 233 | + tree_diffs = compare_trees(osate_ref, spar_instance_node) |
| 234 | + if not tree_diffs: |
| 235 | + print(f" PASS: component tree structure matches") |
| 236 | + else: |
| 237 | + print(f" DIFF: {len(tree_diffs)} structural difference(s):") |
| 238 | + for d in tree_diffs[:10]: |
| 239 | + print(f" - {d}") |
| 240 | + if len(tree_diffs) > 10: |
| 241 | + print(f" ... and {len(tree_diffs) - 10} more") |
| 242 | + passed = False |
| 243 | + |
| 244 | + return passed |
| 245 | + |
| 246 | + |
| 247 | +def main(): |
| 248 | + parser = argparse.ArgumentParser(description="Compare spar vs OSATE reference data") |
| 249 | + parser.add_argument("--model", help="Specific model to compare (e.g., BasicHierarchy)") |
| 250 | + parser.add_argument("--verbose", "-v", action="store_true") |
| 251 | + args = parser.parse_args() |
| 252 | + |
| 253 | + results = {} |
| 254 | + |
| 255 | + for filename, classifier in TEST_MODELS: |
| 256 | + model_base = os.path.splitext(filename)[0] |
| 257 | + if args.model and args.model != model_base: |
| 258 | + continue |
| 259 | + result = compare_model(filename, classifier, args.verbose) |
| 260 | + if result is not None: |
| 261 | + results[model_base] = result |
| 262 | + |
| 263 | + # Summary |
| 264 | + print(f"\n{'='*60}") |
| 265 | + print("SUMMARY") |
| 266 | + print(f"{'='*60}") |
| 267 | + |
| 268 | + if not results: |
| 269 | + print("No models compared (missing reference data or test files)") |
| 270 | + sys.exit(2) |
| 271 | + |
| 272 | + passed = sum(1 for v in results.values() if v) |
| 273 | + failed = sum(1 for v in results.values() if not v) |
| 274 | + print(f" Passed: {passed}") |
| 275 | + print(f" Failed: {failed}") |
| 276 | + print(f" Total: {len(results)}") |
| 277 | + |
| 278 | + sys.exit(0 if failed == 0 else 1) |
| 279 | + |
| 280 | + |
| 281 | +if __name__ == "__main__": |
| 282 | + main() |
0 commit comments