Skip to content

Commit 818afbb

Browse files
apstenku123apstenku123
committed
feat(e2e-e1): fixture generator — 4 tokenizers + 16 parquet shards
Stage E-1 of the E2E Coverage Matrix epic (cppmega-mlx-pa3.2). Lands the canonical inputs every Playwright scenario reads. tests/fixtures/build_e2e_matrix.py: - TOKENIZER_SPECS = (T1_cppmega_v3, T2_gpt2_small, T3_minimal_no_fim, T4_fim_only). T1 is a structural copy of the vendored cppmega tokenizer (vocab 65536, full FIM + SPACE/NL). T2 is HF gpt2 (50257). T3/T4 are trained on a small Python+C++ corpus via tokenizers BPE trainer (256 / 1024 vocab, with FIM specials only in T4). - PARQUET_SCHEMAS = (P1_minimal, P2_doc, P3_engram, P4_full), emitted per tokenizer so input_ids round-trip through the matching vocab. P4 carries the full enriched-corpus column set (input_ids, doc_ids, loss_mask, chunk_boundaries, call_edges, type_edges, constituent_provenance_offsets). - generate_tokenizers() is idempotent (skips writes when SHA-256 of the new content equals existing); generate_parquets() always rewrites (parquet is small, deterministic given pinned pyarrow). - validate_round_trip() decodes the first row of every shard under its tokenizer and confirms non-empty text. - write_index() emits tests/fixtures/MATRIX.json — single machine- readable index pointing the Playwright matrix at the artefacts. Outputs (all checked in): tests/fixtures/tokenizers/T1..T4.json (4.1 MB total) tests/fixtures/parquet/T*__P*.parquet (16 shards, 64 KB total) tests/fixtures/MATRIX.json (~3 KB index) Tests (+36 new pytest, 2139 total): - spec lock: 4 tokenizers / 4 schemas / distinct vocab sizes - per-tokenizer file present + loads + vocab-size expectations - T3 confirmed FIM-free, T4 confirmed FIM trio present - generate_tokenizers idempotent across runs (digests stable) - parquet shard per (tokenizer x schema) has input_ids + 32 rows - per-schema expected extra column present - P4 enriched column set - round-trip decode emits non-empty text for all 16 shards - MATRIX.json index machine-readable, all paths resolve Closes cppmega-mlx-pa3.2.
1 parent 52bfde3 commit 818afbb

24 files changed

Lines changed: 802 additions & 0 deletions

‎tests/fixtures/MATRIX.json‎

Lines changed: 309 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,309 @@
1+
{
2+
"parquets": {
3+
"T1_cppmega_v3__P1_minimal": {
4+
"columns": [
5+
"input_ids"
6+
],
7+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T1_cppmega_v3__P1_minimal.parquet",
8+
"rows": 32,
9+
"schema": "P1_minimal",
10+
"seq_len": 64,
11+
"tokenizer": "T1_cppmega_v3"
12+
},
13+
"T1_cppmega_v3__P2_doc": {
14+
"columns": [
15+
"input_ids",
16+
"doc_ids"
17+
],
18+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T1_cppmega_v3__P2_doc.parquet",
19+
"rows": 32,
20+
"schema": "P2_doc",
21+
"seq_len": 64,
22+
"tokenizer": "T1_cppmega_v3"
23+
},
24+
"T1_cppmega_v3__P3_engram": {
25+
"columns": [
26+
"input_ids",
27+
"doc_ids",
28+
"call_edges"
29+
],
30+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T1_cppmega_v3__P3_engram.parquet",
31+
"rows": 32,
32+
"schema": "P3_engram",
33+
"seq_len": 64,
34+
"tokenizer": "T1_cppmega_v3"
35+
},
36+
"T1_cppmega_v3__P4_full": {
37+
"columns": [
38+
"input_ids",
39+
"doc_ids",
40+
"call_edges",
41+
"loss_mask",
42+
"chunk_boundaries",
43+
"type_edges",
44+
"constituent_provenance_offsets"
45+
],
46+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T1_cppmega_v3__P4_full.parquet",
47+
"rows": 32,
48+
"schema": "P4_full",
49+
"seq_len": 64,
50+
"tokenizer": "T1_cppmega_v3"
51+
},
52+
"T2_gpt2_small__P1_minimal": {
53+
"columns": [
54+
"input_ids"
55+
],
56+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T2_gpt2_small__P1_minimal.parquet",
57+
"rows": 32,
58+
"schema": "P1_minimal",
59+
"seq_len": 64,
60+
"tokenizer": "T2_gpt2_small"
61+
},
62+
"T2_gpt2_small__P2_doc": {
63+
"columns": [
64+
"input_ids",
65+
"doc_ids"
66+
],
67+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T2_gpt2_small__P2_doc.parquet",
68+
"rows": 32,
69+
"schema": "P2_doc",
70+
"seq_len": 64,
71+
"tokenizer": "T2_gpt2_small"
72+
},
73+
"T2_gpt2_small__P3_engram": {
74+
"columns": [
75+
"input_ids",
76+
"doc_ids",
77+
"call_edges"
78+
],
79+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T2_gpt2_small__P3_engram.parquet",
80+
"rows": 32,
81+
"schema": "P3_engram",
82+
"seq_len": 64,
83+
"tokenizer": "T2_gpt2_small"
84+
},
85+
"T2_gpt2_small__P4_full": {
86+
"columns": [
87+
"input_ids",
88+
"doc_ids",
89+
"call_edges",
90+
"loss_mask",
91+
"chunk_boundaries",
92+
"type_edges",
93+
"constituent_provenance_offsets"
94+
],
95+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T2_gpt2_small__P4_full.parquet",
96+
"rows": 32,
97+
"schema": "P4_full",
98+
"seq_len": 64,
99+
"tokenizer": "T2_gpt2_small"
100+
},
101+
"T3_minimal_no_fim__P1_minimal": {
102+
"columns": [
103+
"input_ids"
104+
],
105+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T3_minimal_no_fim__P1_minimal.parquet",
106+
"rows": 32,
107+
"schema": "P1_minimal",
108+
"seq_len": 64,
109+
"tokenizer": "T3_minimal_no_fim"
110+
},
111+
"T3_minimal_no_fim__P2_doc": {
112+
"columns": [
113+
"input_ids",
114+
"doc_ids"
115+
],
116+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T3_minimal_no_fim__P2_doc.parquet",
117+
"rows": 32,
118+
"schema": "P2_doc",
119+
"seq_len": 64,
120+
"tokenizer": "T3_minimal_no_fim"
121+
},
122+
"T3_minimal_no_fim__P3_engram": {
123+
"columns": [
124+
"input_ids",
125+
"doc_ids",
126+
"call_edges"
127+
],
128+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T3_minimal_no_fim__P3_engram.parquet",
129+
"rows": 32,
130+
"schema": "P3_engram",
131+
"seq_len": 64,
132+
"tokenizer": "T3_minimal_no_fim"
133+
},
134+
"T3_minimal_no_fim__P4_full": {
135+
"columns": [
136+
"input_ids",
137+
"doc_ids",
138+
"call_edges",
139+
"loss_mask",
140+
"chunk_boundaries",
141+
"type_edges",
142+
"constituent_provenance_offsets"
143+
],
144+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T3_minimal_no_fim__P4_full.parquet",
145+
"rows": 32,
146+
"schema": "P4_full",
147+
"seq_len": 64,
148+
"tokenizer": "T3_minimal_no_fim"
149+
},
150+
"T4_fim_only__P1_minimal": {
151+
"columns": [
152+
"input_ids"
153+
],
154+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T4_fim_only__P1_minimal.parquet",
155+
"rows": 32,
156+
"schema": "P1_minimal",
157+
"seq_len": 64,
158+
"tokenizer": "T4_fim_only"
159+
},
160+
"T4_fim_only__P2_doc": {
161+
"columns": [
162+
"input_ids",
163+
"doc_ids"
164+
],
165+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T4_fim_only__P2_doc.parquet",
166+
"rows": 32,
167+
"schema": "P2_doc",
168+
"seq_len": 64,
169+
"tokenizer": "T4_fim_only"
170+
},
171+
"T4_fim_only__P3_engram": {
172+
"columns": [
173+
"input_ids",
174+
"doc_ids",
175+
"call_edges"
176+
],
177+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T4_fim_only__P3_engram.parquet",
178+
"rows": 32,
179+
"schema": "P3_engram",
180+
"seq_len": 64,
181+
"tokenizer": "T4_fim_only"
182+
},
183+
"T4_fim_only__P4_full": {
184+
"columns": [
185+
"input_ids",
186+
"doc_ids",
187+
"call_edges",
188+
"loss_mask",
189+
"chunk_boundaries",
190+
"type_edges",
191+
"constituent_provenance_offsets"
192+
],
193+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/parquet/T4_fim_only__P4_full.parquet",
194+
"rows": 32,
195+
"schema": "P4_full",
196+
"seq_len": 64,
197+
"tokenizer": "T4_fim_only"
198+
}
199+
},
200+
"round_trip": {
201+
"T1_cppmega_v3__P1_minimal": {
202+
"first_decoded": "add a b a b",
203+
"non_empty": "True"
204+
},
205+
"T1_cppmega_v3__P2_doc": {
206+
"first_decoded": "add a b a b",
207+
"non_empty": "True"
208+
},
209+
"T1_cppmega_v3__P3_engram": {
210+
"first_decoded": "add a b a b",
211+
"non_empty": "True"
212+
},
213+
"T1_cppmega_v3__P4_full": {
214+
"first_decoded": "add a b a b",
215+
"non_empty": "True"
216+
},
217+
"T2_gpt2_small__P1_minimal": {
218+
"first_decoded": "def add(a, b): return a + b\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",
219+
"non_empty": "True"
220+
},
221+
"T2_gpt2_small__P2_doc": {
222+
"first_decoded": "def add(a, b): return a + b\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",
223+
"non_empty": "True"
224+
},
225+
"T2_gpt2_small__P3_engram": {
226+
"first_decoded": "def add(a, b): return a + b\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",
227+
"non_empty": "True"
228+
},
229+
"T2_gpt2_small__P4_full": {
230+
"first_decoded": "def add(a, b): return a + b\n!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!",
231+
"non_empty": "True"
232+
},
233+
"T3_minimal_no_fim__P1_minimal": {
234+
"first_decoded": "def add(a, b): return a + b\n",
235+
"non_empty": "True"
236+
},
237+
"T3_minimal_no_fim__P2_doc": {
238+
"first_decoded": "def add(a, b): return a + b\n",
239+
"non_empty": "True"
240+
},
241+
"T3_minimal_no_fim__P3_engram": {
242+
"first_decoded": "def add(a, b): return a + b\n",
243+
"non_empty": "True"
244+
},
245+
"T3_minimal_no_fim__P4_full": {
246+
"first_decoded": "def add(a, b): return a + b\n",
247+
"non_empty": "True"
248+
},
249+
"T4_fim_only__P1_minimal": {
250+
"first_decoded": "def add(a, b): return a + b\n",
251+
"non_empty": "True"
252+
},
253+
"T4_fim_only__P2_doc": {
254+
"first_decoded": "def add(a, b): return a + b\n",
255+
"non_empty": "True"
256+
},
257+
"T4_fim_only__P3_engram": {
258+
"first_decoded": "def add(a, b): return a + b\n",
259+
"non_empty": "True"
260+
},
261+
"T4_fim_only__P4_full": {
262+
"first_decoded": "def add(a, b): return a + b\n",
263+
"non_empty": "True"
264+
}
265+
},
266+
"tokenizers": {
267+
"T1_cppmega_v3": {
268+
"digest": "34439c0fbe76d582",
269+
"fresh": true,
270+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/tokenizers/T1_cppmega_v3.json",
271+
"specials": [],
272+
"vocab_size": 65536
273+
},
274+
"T2_gpt2_small": {
275+
"digest": "11e818f948f43497",
276+
"fresh": true,
277+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/tokenizers/T2_gpt2_small.json",
278+
"specials": [],
279+
"vocab_size": 50257
280+
},
281+
"T3_minimal_no_fim": {
282+
"digest": "17a044a3c1bb0486",
283+
"fresh": true,
284+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/tokenizers/T3_minimal_no_fim.json",
285+
"specials": [
286+
"<PAD>",
287+
"<UNK>",
288+
"<BOS>",
289+
"<EOS>"
290+
],
291+
"vocab_size": 260
292+
},
293+
"T4_fim_only": {
294+
"digest": "892f2bab670c5cd4",
295+
"fresh": true,
296+
"path": "/Volumes/external/sources/cppmega.mlx/tests/fixtures/tokenizers/T4_fim_only.json",
297+
"specials": [
298+
"<PAD>",
299+
"<UNK>",
300+
"<BOS>",
301+
"<EOS>",
302+
"<FIM_PREFIX>",
303+
"<FIM_MIDDLE>",
304+
"<FIM_SUFFIX>"
305+
],
306+
"vocab_size": 440
307+
}
308+
}
309+
}

‎tests/fixtures/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)