tokenizer: expand TOKEN_DICT by 285 builtins + 53 common idioms

Bulk-add every interpreter builtin not yet in TOKEN_DICT plus
~50 high-frequency OMC code idioms (h x = , i = i + 1;,
arr_push(out, , assert_eq(, } else {, etc.).

Greedy longest-match is order-stable so existing encoded streams
keep round-tripping. New entries get high IDs (they don't displace
the attractor-aligned 1..19 slots). Net effect: any OMC code using
ANY of the 368 builtins now compresses; idiom-heavy code drops to
~1/3 its raw size.

Measured compression on the demo workloads:
  arr_softmax([1.0...])        1.75 -> 1.75  (already in dict)
  h x = arr_matmul(A, B);      1.77 -> 2.30
  tape_reset();...             2.41 -> 2.79
  if i < arr_len(xs)...        1.80 -> 2.14

Roundtrip exact on all 15 tokenizer tests after the change.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: RandomCoder-lab

omnimcode-core/src/tokenizer.rs

@@ -218,6 +218,346 @@ pub const TOKEN_DICT: &[&str] = &[
     // Common 2-char operators / openers
     "==", "!=", "<=", ">=", "&&", "||", "<<", ">>",
     "//", "/*", "*/",
+
+    // ---- Auto-appended bulk dict expansion (Phase 2) ----
+    "abs",
+    "acos",
+    "arr_all",
+    "arr_any",
+    "arr_argmax",
+    "arr_argmin",
+    "arr_avg_distance",
+    "arr_chunk",
+    "arr_concat",
+    "arr_contains",
+    "arr_count",
+    "arr_cumsum",
+    "arr_diff",
+    "arr_drop",
+    "arr_enumerate",
+    "arr_filter",
+    "arr_find",
+    "arr_first",
+    "arr_flatten",
+    "arr_fold_elements",
+    "arr_from_range",
+    "arr_gcd",
+    "arr_geometric_mean",
+    "arr_harmonic_mean",
+    "arr_index_of",
+    "arr_is_sorted",
+    "arr_join",
+    "arr_last",
+    "arr_map",
+    "arr_max",
+    "arr_max_float",
+    "arr_max_int",
+    "arr_mean",
+    "arr_median",
+    "arr_min",
+    "arr_min_float",
+    "arr_min_int",
+    "arr_neg",
+    "arr_new",
+    "arr_norm",
+    "arr_ones",
+    "arr_outer",
+    "arr_partition_by",
+    "arr_product",
+    "arr_range",
+    "arr_reduce",
+    "arr_repeat",
+    "arr_resonance",
+    "arr_reverse",
+    "arr_slice",
+    "arr_sort",
+    "arr_sort_int",
+    "arr_stddev",
+    "arr_sum",
+    "arr_sum_int",
+    "arr_sum_sq",
+    "arr_take",
+    "arr_unique",
+    "arr_unique_count",
+    "arr_variance",
+    "arr_window",
+    "arr_zeros",
+    "arr_zip",
+    "asin",
+    "atan",
+    "atan2",
+    "attractor_bucket",
+    "attractor_table",
+    "bit_count",
+    "bit_length",
+    "call",
+    "ceil",
+    "clamp",
+    "classify_resonance",
+    "cleanup_array",
+    "collapse",
+    "cos",
+    "crt_residues",
+    "csv_parse",
+    "cube",
+    "defined_functions",
+    "dict_clear",
+    "dict_del",
+    "dict_get_or",
+    "dict_has",
+    "dict_items",
+    "dict_keys",
+    "dict_len",
+    "dict_merge",
+    "dict_new",
+    "dict_pop",
+    "dict_size",
+    "dict_values",
+    "digit_count",
+    "digit_sum",
+    "dual_cos",
+    "dual_exp",
+    "dual_neg",
+    "dual_pow_int",
+    "dual_relu",
+    "dual_sigmoid",
+    "dual_sin",
+    "dual_tanh",
+    "dual_v",
+    "e",
+    "ensure_clean",
+    "erf",
+    "error",
+    "even",
+    "exp",
+    "factorial",
+    "fib",
+    "fib_chunks",
+    "fibonacci",
+    "file_exists",
+    "filter_by_resonance",
+    "floor",
+    "fnv1a_hash",
+    "fold",
+    "fold_escape",
+    "frac",
+    "from_zeckendorf",
+    "gcd",
+    "harmonic_align",
+    "harmonic_checksum",
+    "harmonic_dedupe",
+    "harmonic_diff",
+    "harmonic_hash",
+    "harmonic_interfere",
+    "harmonic_partition",
+    "harmonic_partition_3",
+    "harmonic_read_file",
+    "harmonic_resample",
+    "harmonic_score",
+    "harmonic_sort",
+    "harmonic_split",
+    "harmonic_unalign",
+    "harmonic_write_file",
+    "harmony_value",
+    "hbit_tension",
+    "hypot",
+    "int_binary_search",
+    "int_lower_bound",
+    "int_upper_bound",
+    "interfere",
+    "is_even",
+    "is_fibonacci",
+    "is_instance",
+    "is_odd",
+    "is_phi_resonant",
+    "is_prime",
+    "is_singularity",
+    "is_zeckendorf_valid",
+    "largest_attractor_at_most",
+    "lcm",
+    "len",
+    "lerp",
+    "ln_2",
+    "log",
+    "log10",
+    "log2",
+    "log_phi_pi_fibonacci",
+    "max",
+    "mean_omni_weight",
+    "measure_coherence",
+    "min",
+    "mod_pow",
+    "nearest_attractor",
+    "now_ms",
+    "nth_fibonacci",
+    "odd",
+    "omc_code_canonical",
+    "omc_code_equivalent",
+    "omc_error_categories",
+    "omc_error_count",
+    "omc_token_vocab_size",
+    "phi",
+    "phi_inv",
+    "phi_pi_bin_search",
+    "phi_pi_fib_nearest",
+    "phi_pi_fib_nearest_traced",
+    "phi_pi_fib_nearest_v2",
+    "phi_pi_fib_reset",
+    "phi_pi_fib_search",
+    "phi_pi_fib_search_traced",
+    "phi_pi_fib_search_v2",
+    "phi_pi_fib_stats",
+    "phi_pi_fib_stats_all",
+    "phi_pi_fib_stats_bg",
+    "phi_pi_log_distance",
+    "phi_pi_pow",
+    "phi_pow",
+    "phi_shadow",
+    "phi_sq",
+    "phi_squared",
+    "pi",
+    "pow",
+    "pow_int",
+    "print_raw",
+    "println",
+    "quantization_ratio",
+    "quantize",
+    "random_float",
+    "random_int",
+    "random_seed",
+    "re_find",
+    "re_split",
+    "read_file",
+    "res",
+    "resolve_singularity",
+    "resonance_band",
+    "resonance_band_histogram",
+    "round",
+    "safe_add",
+    "safe_arr_get",
+    "safe_arr_set",
+    "safe_divide",
+    "safe_log",
+    "safe_mod",
+    "safe_mul",
+    "safe_sqrt",
+    "safe_sub",
+    "sigmoid",
+    "sign",
+    "sin",
+    "sorted_dedupe",
+    "sorted_merge",
+    "sorted_union",
+    "sqrt",
+    "sqrt_2",
+    "sqrt_5",
+    "square",
+    "str_capitalize",
+    "str_chars",
+    "str_concat",
+    "str_contains",
+    "str_count",
+    "str_ends_with",
+    "str_index_of",
+    "str_is_empty",
+    "str_lowercase",
+    "str_pad_left",
+    "str_pad_right",
+    "str_repeat",
+    "str_replace",
+    "str_reverse",
+    "str_split_lines",
+    "str_starts_with",
+    "str_to_float",
+    "str_to_int",
+    "str_trim",
+    "str_uppercase",
+    "substrate_count_range",
+    "substrate_difference",
+    "substrate_hash",
+    "substrate_insert",
+    "substrate_intersect",
+    "substrate_lower_bound",
+    "substrate_min_distance",
+    "substrate_nearest",
+    "substrate_quantile",
+    "substrate_rank",
+    "substrate_search",
+    "substrate_select_k",
+    "substrate_slice_range",
+    "substrate_upper_bound",
+    "tan",
+    "tanh",
+    "tape_neg",
+    "tape_pow_int",
+    "tau",
+    "test_clear_failures",
+    "test_failure_count",
+    "test_get_current",
+    "test_get_failures",
+    "test_record_failure",
+    "test_set_current",
+    "to_float",
+    "to_int",
+    "type_of",
+    "value_danger",
+    "write_file",
+    "zeckendorf",
+    "zeckendorf_bit",
+    "zeckendorf_weight",
+    " 0;\n",
+    " 1;\n",
+    " 2;\n",
+    " -1;\n",
+    "h x = ",
+    "h y = ",
+    "h i = ",
+    "h s = ",
+    "h n = ",
+    "h r = ",
+    "h sum = 0",
+    "h count = 0",
+    "h result = ",
+    "i = i + 1;",
+    "j = j + 1;",
+    "k = k + 1;",
+    " < n {",
+    " < arr_len(",
+    "} else {",
+    "} else if ",
+    "while i < ",
+    "for x in ",
+    "for v in ",
+    "fn test_",
+    "test_record_failure(",
+    "assert_eq(",
+    "assert_true(",
+    "assert_true(arr_len(",
+    " == 1, \"",
+    " == 0, \"",
+    "approx_eq(",
+    "to_string(",
+    ".items.borrow()",
+    "if arr_get(",
+    "return arr_get(",
+    "arr_push(out, ",
+    "h out = [];",
+    "h out = arr_new()",
+    "h xs = [",
+    "h ys = [",
+    "if condition",
+    "is empty",
+    "out of bounds",
+    "shape mismatch",
+    " }\n",
+    " {\n    ",
+    " {\n",
+    ");\n",
+    ", ",
+    " + 1",
+    " - 1",
+    " * 2",
+    " / 2",
 ];
 
 /// Substrate distance between two token IDs. Returns the absolute