Skip to content

Commit e1e453f

Browse files
RandomCoder-labclaude
RandomCoder-lab
and
claude
committed
tokenizer: +269 dict entries (single-var patterns, total 1167)
For every common single-letter variable name (a,b,c,i,j,k,m,n,p,q, r,s,t,u,v,x,y,z,d), add per-variable bigram patterns: declaration (h X = ), arithmetic (X + 1, X - 1, X = 0, X = 1), return (return X;), control conditions (if X , while X , X < , X > , X == , X !=). Plus shapes for match-arms, class self patterns, pragmas (@Harmony, @predict, @hbit, @no_heal), module imports, common print/loop/test forms, and common arr/dict access patterns. Cumulative: 1167 dict entries. Single-character variables now compress to 1 ID per occurrence instead of being escaped per byte. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 6d6467f commit e1e453f

1 file changed

Lines changed: 271 additions & 0 deletions

File tree

‎omnimcode-core/src/tokenizer.rs‎

Lines changed: 271 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -932,6 +932,277 @@ pub const TOKEN_DICT: &[&str] = &[
932932
"attractor_distance(",
933933
"fibonacci_index(",
934934
"crt_recover([",
935+
936+
// ---- Dict expansion v4 (single-var-name patterns) ----
937+
"h a = ",
938+
"a + 1",
939+
"a - 1",
940+
"a = 0",
941+
"a = 1",
942+
"return a;",
943+
"if a ",
944+
"while a ",
945+
"a < ",
946+
"a > ",
947+
"a == ",
948+
"a != ",
949+
"h b = ",
950+
"b + 1",
951+
"b - 1",
952+
"b = 0",
953+
"b = 1",
954+
"return b;",
955+
"if b ",
956+
"while b ",
957+
"b < ",
958+
"b > ",
959+
"b == ",
960+
"b != ",
961+
"h c = ",
962+
"c + 1",
963+
"c - 1",
964+
"c = 0",
965+
"c = 1",
966+
"return c;",
967+
"if c ",
968+
"while c ",
969+
"c < ",
970+
"c > ",
971+
"c == ",
972+
"c != ",
973+
"h i = ",
974+
"i + 1",
975+
"i - 1",
976+
"i = 0",
977+
"i = 1",
978+
"return i;",
979+
"if i ",
980+
"while i ",
981+
"i < ",
982+
"i > ",
983+
"i == ",
984+
"i != ",
985+
"h j = ",
986+
"j + 1",
987+
"j - 1",
988+
"j = 0",
989+
"j = 1",
990+
"return j;",
991+
"if j ",
992+
"while j ",
993+
"j < ",
994+
"j > ",
995+
"j == ",
996+
"j != ",
997+
"h k = ",
998+
"k + 1",
999+
"k - 1",
1000+
"k = 0",
1001+
"k = 1",
1002+
"return k;",
1003+
"if k ",
1004+
"while k ",
1005+
"k < ",
1006+
"k > ",
1007+
"k == ",
1008+
"k != ",
1009+
"h n = ",
1010+
"n + 1",
1011+
"n - 1",
1012+
"n = 0",
1013+
"n = 1",
1014+
"return n;",
1015+
"if n ",
1016+
"while n ",
1017+
"n < ",
1018+
"n > ",
1019+
"n == ",
1020+
"n != ",
1021+
"h m = ",
1022+
"m + 1",
1023+
"m - 1",
1024+
"m = 0",
1025+
"m = 1",
1026+
"return m;",
1027+
"if m ",
1028+
"while m ",
1029+
"m < ",
1030+
"m > ",
1031+
"m == ",
1032+
"m != ",
1033+
"h x = ",
1034+
"x + 1",
1035+
"x - 1",
1036+
"x = 0",
1037+
"x = 1",
1038+
"return x;",
1039+
"if x ",
1040+
"while x ",
1041+
"x < ",
1042+
"x > ",
1043+
"x == ",
1044+
"x != ",
1045+
"h y = ",
1046+
"y + 1",
1047+
"y - 1",
1048+
"y = 0",
1049+
"y = 1",
1050+
"return y;",
1051+
"if y ",
1052+
"while y ",
1053+
"y < ",
1054+
"y > ",
1055+
"y == ",
1056+
"y != ",
1057+
"h z = ",
1058+
"z + 1",
1059+
"z - 1",
1060+
"z = 0",
1061+
"z = 1",
1062+
"return z;",
1063+
"if z ",
1064+
"while z ",
1065+
"z < ",
1066+
"z > ",
1067+
"z == ",
1068+
"z != ",
1069+
"h r = ",
1070+
"r + 1",
1071+
"r - 1",
1072+
"r = 0",
1073+
"r = 1",
1074+
"return r;",
1075+
"if r ",
1076+
"while r ",
1077+
"r < ",
1078+
"r > ",
1079+
"r == ",
1080+
"r != ",
1081+
"h s = ",
1082+
"s + 1",
1083+
"s - 1",
1084+
"s = 0",
1085+
"s = 1",
1086+
"return s;",
1087+
"if s ",
1088+
"while s ",
1089+
"s < ",
1090+
"s > ",
1091+
"s == ",
1092+
"s != ",
1093+
"h t = ",
1094+
"t + 1",
1095+
"t - 1",
1096+
"t = 0",
1097+
"t = 1",
1098+
"return t;",
1099+
"if t ",
1100+
"while t ",
1101+
"t < ",
1102+
"t > ",
1103+
"t == ",
1104+
"t != ",
1105+
"h u = ",
1106+
"u + 1",
1107+
"u - 1",
1108+
"u = 0",
1109+
"u = 1",
1110+
"return u;",
1111+
"if u ",
1112+
"while u ",
1113+
"u < ",
1114+
"u > ",
1115+
"u == ",
1116+
"u != ",
1117+
"h v = ",
1118+
"v + 1",
1119+
"v - 1",
1120+
"v = 0",
1121+
"v = 1",
1122+
"return v;",
1123+
"if v ",
1124+
"while v ",
1125+
"v < ",
1126+
"v > ",
1127+
"v == ",
1128+
"v != ",
1129+
"h d = ",
1130+
"d + 1",
1131+
"d - 1",
1132+
"d = 0",
1133+
"d = 1",
1134+
"return d;",
1135+
"if d ",
1136+
"while d ",
1137+
"d < ",
1138+
"d > ",
1139+
"d == ",
1140+
"d != ",
1141+
"h p = ",
1142+
"p + 1",
1143+
"p - 1",
1144+
"p = 0",
1145+
"p = 1",
1146+
"return p;",
1147+
"if p ",
1148+
"while p ",
1149+
"p < ",
1150+
"p > ",
1151+
"p == ",
1152+
"p != ",
1153+
"h q = ",
1154+
"q + 1",
1155+
"q - 1",
1156+
"q = 0",
1157+
"q = 1",
1158+
"return q;",
1159+
"if q ",
1160+
"while q ",
1161+
"q < ",
1162+
"q > ",
1163+
"q == ",
1164+
"q != ",
1165+
" => return ",
1166+
" => {",
1167+
"match v {",
1168+
"match arg {",
1169+
"_ => null",
1170+
"Wildcard",
1171+
"(self, ",
1172+
"(self) {",
1173+
"self, ",
1174+
"@harmony",
1175+
"@predict",
1176+
"@hbit",
1177+
"@no_heal",
1178+
"import \"",
1179+
"from \"",
1180+
"} import ",
1181+
"print(concat_many(",
1182+
"print(to_string(",
1183+
"print(\"",
1184+
"while i < arr_len(xs)",
1185+
"while j < arr_len(",
1186+
"i = i + 1; }",
1187+
"j = j + 1; }",
1188+
"i = i - 1; }",
1189+
"if !cond { test_record_failure(msg); }",
1190+
"if cond { return 1; }",
1191+
"if !cond { return 0; }",
1192+
"fn assert_eq(actual",
1193+
"fn assert_true(cond",
1194+
"test_record_failure(msg",
1195+
"arr_get(arr, ",
1196+
"arr_set(arr, ",
1197+
"arr_push(arr, ",
1198+
"dict_get(dict, ",
1199+
"dict_set(dict, ",
1200+
"str_split(s, ",
1201+
"str_join(arr, ",
1202+
"concat_many(\"",
1203+
"type_of(value)",
1204+
"type_of(v) == \"",
1205+
"is_defined(\"",
9351206
];
9361207

9371208
/// Substrate distance between two token IDs. Returns the absolute

0 commit comments

Comments
 (0)