Skip to content

Commit 9138e1f

Browse files
fjtrujyclaude
andcommitted
test: semantic VSM diff against Sony reference per renderer
Adds cmake/vsm_diff.py plus CTest wiring that compares openvcl's output for each VU1 renderer against the matching Sony-generated reference in vu1/sce_<X>_vcl.vsm. The comparison is deliberately permissive about pipe pairing and register-allocator choices -- it checks: - opcode histogram (same mnemonics with same counts) - flag histogram ([E]/[I]/[D]/[T] occurrences) - label set (control-flow structure preserved) - non-nop slot count + ratio (rough scheduler-progress signal) The 12 renderers with a Sony reference get one CTest each, all labelled `vsm-diff` + `known-failing` and marked WILL_FAIL. Today every renderer diverges (openvcl produces 34-73% of Sony's instruction count because the dual-pipe scheduler and the multi-variant specialization aren't implemented yet), so WILL_FAIL keeps the build green; when a renderer starts matching, the test will XPASS so we notice. Baseline ratios captured at commit time: fast_nolights 0.72 general_pv_diff 0.37 fast 0.73 general_quad 0.34 general_nospec_quad 0.38 general_tri 0.38 general_nospec_tri 0.46 general 0.38 general_nospec 0.45 indexed 0.34 general_pv_diff_quad 0.39 general_pv_diff_tri 0.38 Usage: cmake -B build-test && cmake --build build-test ctest --test-dir build-test -L vsm-diff # raw diff for a single renderer: python3 cmake/vsm_diff.py vu1/sce_general_vcl.vsm build-test/vu1/general_vcl.vsm The scei renderer is skipped because the Sony reference for it uses a different naming convention (`scei_vcl.vsm` vs `sce_<X>_vcl.vsm`). Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent a4c5e06 commit 9138e1f

2 files changed

Lines changed: 277 additions & 0 deletions

File tree

CMakeLists.txt

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -231,6 +231,50 @@ if(VU1_OBJECTS)
231231
add_custom_target(vu1_objects ALL DEPENDS ${VU1_OBJECTS})
232232
endif()
233233

234+
# ============================================================================
235+
# VSM semantic-diff tests
236+
# ============================================================================
237+
# For each renderer with a Sony reference VSM in `vu1/sce_<X>_vcl.vsm`,
238+
# add a CTest that compares its opcode histogram + label set against the
239+
# openvcl-produced `<build>/vu1/<X>_vcl.vsm`. Used as a ratchet for the
240+
# eventual dual-pipe scheduler / specialization work: tests that don't
241+
# pass yet are marked WILL_FAIL so a renderer "going green" lights up
242+
# instead of breaking the build.
243+
find_package(Python3 COMPONENTS Interpreter)
244+
if(VU1_TOOLS_AVAILABLE AND Python3_Interpreter_FOUND)
245+
enable_testing()
246+
247+
set(VSM_DIFF_SCRIPT ${CMAKE_SOURCE_DIR}/cmake/vsm_diff.py)
248+
249+
foreach(RENDERER ${RENDERERS})
250+
set(SCE_VSM "${CMAKE_SOURCE_DIR}/vu1/sce_${RENDERER}_vcl.vsm")
251+
set(OVC_VSM "${CMAKE_BINARY_DIR}/vu1/${RENDERER}_vcl.vsm")
252+
253+
if(NOT EXISTS "${SCE_VSM}")
254+
# No Sony reference for this renderer (e.g. scei.vcl ships its
255+
# own reference under a different name); skip silently.
256+
continue()
257+
endif()
258+
259+
add_test(NAME vsm_diff_${RENDERER}
260+
COMMAND ${Python3_EXECUTABLE} ${VSM_DIFF_SCRIPT}
261+
${SCE_VSM} ${OVC_VSM})
262+
# Building the openvcl output is a prerequisite. CTest doesn't
263+
# auto-build, so the user must run `cmake --build` first; the
264+
# FIXTURES_REQUIRED machinery would be overkill here.
265+
set_tests_properties(vsm_diff_${RENDERER} PROPERTIES
266+
# Today openvcl produces neither pipe-paired nor multi-
267+
# specialised output, so the strict histogram check fails
268+
# for every renderer. Mark them WILL_FAIL so a renderer
269+
# becoming equivalent shows up as XPASS instead of silently
270+
# breaking the build. Drop this property per-renderer as
271+
# the scheduler / specialization work closes each one.
272+
WILL_FAIL TRUE
273+
LABELS "vsm-diff;known-failing"
274+
)
275+
endforeach()
276+
endif()
277+
234278
# ============================================================================
235279
# Build the library
236280
# ============================================================================

cmake/vsm_diff.py

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Compare two VU1 .vsm files at the semantic level.
4+
5+
Used as a CTest target in ps2gl to verify that openvcl produces the same
6+
*set* of operations as Sony's proprietary vcl for each VU1 renderer.
7+
Differences in pipe-pairing, register-allocator choices, and whitespace
8+
are intentionally ignored -- the goal is to surface real divergences
9+
(missing instructions, wrong opcodes, missing labels) and to track how
10+
close openvcl is getting to the reference as the dual-pipe scheduler
11+
matures.
12+
13+
Usage:
14+
vsm_diff.py <reference.vsm> <openvcl.vsm>
15+
16+
Exit codes:
17+
0 = histograms and labels match.
18+
1 = real divergence (different opcode set, different label set).
19+
2 = file read / parse error.
20+
21+
The script is intentionally permissive about pipe placement: only the
22+
opcode mix matters. A separate "instruction-count delta" line is printed
23+
to track scheduler progress over time.
24+
"""
25+
26+
import re
27+
import sys
28+
from collections import Counter
29+
30+
# Lines that are not real instructions and should be skipped entirely.
31+
_DIRECTIVE_PREFIXES = (".vu", ".align", ".global", ".name", ".end")
32+
33+
# Sony's reference output includes annotation comments like
34+
# ; === __LP__ ...
35+
# ; _LNOPT_w=[...] ...
36+
# openvcl emits no such comments. Both should be dropped from the
37+
# semantic comparison.
38+
_COMMENT_RE = re.compile(r"^\s*;.*")
39+
40+
# A label line: identifier ending with ':' optionally followed by a comment.
41+
_LABEL_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*:\s*(?:;.*)?$")
42+
43+
# An instruction line carries upper-pipe + lower-pipe ops separated by a
44+
# wide whitespace gap. Sony's reference left-pads the mnemonic into a
45+
# ~14-char column and uses commas (no spaces) between operands, so within
46+
# a single pipe there's never more than ~13 contiguous spaces. The gap
47+
# between pipes is always 20+ spaces in practice. 15 is the safest
48+
# threshold that catches both styles (openvcl + reference) without
49+
# splitting through an operand list.
50+
_PIPE_SPLIT_RE = re.compile(r"\s{15,}")
51+
52+
# Flag suffixes the assembler writes after the mnemonic: NOP[E], NOP[I],
53+
# NOP[D], NOP[T]. Captured separately from the bare mnemonic so we can
54+
# verify control-flow tags independently of the surrounding ops.
55+
_FLAG_RE = re.compile(r"^([a-z0-9.]+?)(\[[A-Za-z]+\])?$")
56+
57+
58+
def _normalize_mnemonic(tok: str) -> str:
59+
"""Lowercase the mnemonic and strip any [E]/[I]/[D]/[T] flag suffix.
60+
61+
Keep dest fields (`.xyz`, `.w`) attached to the mnemonic so we can
62+
distinguish `addi.xy` from `addi.xyz` -- they're semantically
63+
different operations on different fields.
64+
"""
65+
m = _FLAG_RE.match(tok.lower())
66+
return m.group(1) if m else tok.lower()
67+
68+
69+
def _extract_flag(tok: str) -> str:
70+
"""Return the flag suffix (e.g. "[E]") if present, else ""."""
71+
m = _FLAG_RE.match(tok.lower())
72+
return m.group(2) or "" if m else ""
73+
74+
75+
def _is_mnemonic_token(tok: str) -> bool:
76+
"""True iff `tok` looks like a VU1 mnemonic (as opposed to a register
77+
or immediate operand).
78+
79+
Sony's reference output uses uppercase mnemonics with comma-joined
80+
operands and no space between them; openvcl's output uses lowercase
81+
mnemonics with space-separated operands. A consistent classifier
82+
over both is to bucket each whitespace-separated token by what it
83+
looks like:
84+
85+
mnemonic: starts with a letter, is not a register name
86+
register: V[FI]<digit>... or ACC[component] or single-letter I/Q/P/R
87+
immediate: starts with a digit (incl. 0x...)
88+
indirect: contains '(' (e.g. 62(VI00))
89+
label-ref: trailing ':' -- handled before this function gets called
90+
"""
91+
if not tok:
92+
return False
93+
# Strip any leading punctuation that the assembler emits with the
94+
# token (none expected for mnemonics, but harmless).
95+
if not tok[0].isalpha():
96+
return False
97+
# Indirect access embedded in a mnemonic isn't a thing -- those are
98+
# always operands like "62(VI00)" which start with a digit anyway,
99+
# but defend against weirdness.
100+
if "(" in tok:
101+
return False
102+
upper = tok.upper()
103+
# Register names: VF<digits> / VI<digits>, optionally with a field
104+
# suffix like VF15w.
105+
if len(tok) > 2 and upper[:2] in ("VF", "VI") and tok[2].isdigit():
106+
return False
107+
# The accumulator operand prefix (ACC, ACCxyz, ...).
108+
if upper.startswith("ACC"):
109+
return False
110+
# Single-letter pseudo-registers used as operands.
111+
if upper in ("I", "Q", "P", "R"):
112+
return False
113+
return True
114+
115+
116+
def parse_vsm(path: str):
117+
"""Return (opcode_histogram, flag_histogram, label_set, instr_count).
118+
119+
instr_count is the total number of pipe slots filled with anything
120+
other than `nop` -- a rough "work-per-cycle" signal for the scheduler.
121+
"""
122+
opcodes = Counter()
123+
flags = Counter()
124+
labels = set()
125+
instr_count = 0
126+
127+
with open(path) as f:
128+
for raw in f:
129+
line = raw.rstrip("\n")
130+
131+
if not line.strip():
132+
continue
133+
if _COMMENT_RE.match(line):
134+
continue
135+
stripped = line.strip()
136+
if stripped.startswith(_DIRECTIVE_PREFIXES):
137+
continue
138+
139+
label_match = _LABEL_RE.match(line)
140+
if label_match:
141+
labels.add(label_match.group(1))
142+
continue
143+
144+
# Real instruction line: split into upper-pipe / lower-pipe
145+
# halves on a 15+ whitespace gap. Within each half the
146+
# mnemonic is the first token; the rest is operands and
147+
# would otherwise alias as bogus "opcodes" if we treated
148+
# every token equally.
149+
halves = _PIPE_SPLIT_RE.split(line.strip(), maxsplit=1)
150+
for half in halves:
151+
if not half:
152+
continue
153+
tok = half.split()[0]
154+
if not _is_mnemonic_token(tok):
155+
continue
156+
op = _normalize_mnemonic(tok)
157+
flag = _extract_flag(tok)
158+
opcodes[op] += 1
159+
if flag:
160+
flags[flag] += 1
161+
if op != "nop":
162+
instr_count += 1
163+
164+
return opcodes, flags, labels, instr_count
165+
166+
167+
def _diff_counters(a: Counter, b: Counter):
168+
"""Return dict {key: (a, b)} for keys where a and b disagree."""
169+
diffs = {}
170+
for k in sorted(set(a) | set(b)):
171+
if a[k] != b[k]:
172+
diffs[k] = (a[k], b[k])
173+
return diffs
174+
175+
176+
def main(argv) -> int:
177+
if len(argv) != 3:
178+
print(f"usage: {argv[0]} <reference.vsm> <openvcl.vsm>", file=sys.stderr)
179+
return 2
180+
181+
ref_path, ovc_path = argv[1], argv[2]
182+
183+
try:
184+
ref_ops, ref_flags, ref_labels, ref_count = parse_vsm(ref_path)
185+
ovc_ops, ovc_flags, ovc_labels, ovc_count = parse_vsm(ovc_path)
186+
except FileNotFoundError as e:
187+
print(f"missing file: {e.filename}", file=sys.stderr)
188+
return 2
189+
190+
op_diff = _diff_counters(ref_ops, ovc_ops)
191+
flag_diff = _diff_counters(ref_flags, ovc_flags)
192+
only_in_ref = ref_labels - ovc_labels
193+
only_in_ovc = ovc_labels - ref_labels
194+
195+
histogram_ok = not op_diff
196+
flags_ok = not flag_diff
197+
labels_ok = not (only_in_ref or only_in_ovc)
198+
199+
# The scheduler-progress line: a single ratio that should approach 1.0
200+
# as openvcl learns to pair pipes. Values are non-nop pipe slots.
201+
if ref_count == 0:
202+
ratio = float("inf") if ovc_count else 1.0
203+
else:
204+
ratio = ovc_count / ref_count
205+
206+
print(f"=== vsm_diff: {ref_path} vs {ovc_path}")
207+
print(f" non-nop slots: reference={ref_count} openvcl={ovc_count} ratio={ratio:.2f}")
208+
print(f" unique opcodes: reference={len(ref_ops)} openvcl={len(ovc_ops)}")
209+
print(f" labels: reference={len(ref_labels)} openvcl={len(ovc_labels)}")
210+
print(f" histogram_ok={histogram_ok} flags_ok={flags_ok} labels_ok={labels_ok}")
211+
212+
if op_diff:
213+
print(" opcode count mismatches (op: reference -> openvcl):")
214+
for op, (ra, oa) in op_diff.items():
215+
print(f" {op:<14} {ra:>4} -> {oa}")
216+
if flag_diff:
217+
print(" flag count mismatches ([X]: reference -> openvcl):")
218+
for fl, (ra, oa) in flag_diff.items():
219+
print(f" {fl:<6} {ra} -> {oa}")
220+
if only_in_ref:
221+
print(" labels only in reference:")
222+
for l in sorted(only_in_ref):
223+
print(f" - {l}")
224+
if only_in_ovc:
225+
print(" labels only in openvcl:")
226+
for l in sorted(only_in_ovc):
227+
print(f" + {l}")
228+
229+
return 0 if (histogram_ok and labels_ok) else 1
230+
231+
232+
if __name__ == "__main__":
233+
sys.exit(main(sys.argv))

0 commit comments

Comments
 (0)