|
| 1 | +# ============================================================================= |
| 2 | +# Experiment 2 — Positional encoding under length extrapolation. |
| 3 | +# |
| 4 | +# Question: when a position-encoding scheme is "designed" for sequences |
| 5 | +# of length N, how does it behave at length 2N? This is the classic |
| 6 | +# length-extrapolation problem in transformers — sinusoidal PE |
| 7 | +# extrapolates poorly because higher positions fall outside the |
| 8 | +# frequency basis the model has effectively seen. |
| 9 | +# |
| 10 | +# We compare two single-channel encodings: |
| 11 | +# |
| 12 | +# harmonic_pe(pos) = phi.fold(pos * 7 + 1) |
| 13 | +# Maps a position to its nearest Fibonacci attractor. |
| 14 | +# The attractor set is FINITE (~14 attractors in the int range |
| 15 | +# we use), so positions WILL collide once the sequence is long |
| 16 | +# enough. The question is whether the collisions are "structured" |
| 17 | +# (positions K apart collide predictably) or "chaotic". |
| 18 | +# |
| 19 | +# sin_pe(pos) = to_int(1000.0 * sin(pos * 2*pi / 17.0)) + 1000 |
| 20 | +# Single-channel sinusoidal with prime period 17 and quantised |
| 21 | +# to integer for fair comparison. The period bounds the |
| 22 | +# number of distinct values to ~17 (modulo quantisation). |
| 23 | +# Extrapolation past pos=17 produces straight-up wraparound. |
| 24 | +# |
| 25 | +# Both encodings produce a single integer per position. Distinctness |
| 26 | +# at sequence length L is measured by: |
| 27 | +# |
| 28 | +# 1. unique values — how many of L positions map to different codes |
| 29 | +# 2. max collisions — the largest group of positions sharing one code |
| 30 | +# 3. first collision — the smallest pos at which a code repeats from |
| 31 | +# an earlier position |
| 32 | +# |
| 33 | +# A real positional encoding needs `unique values == L`. The point at |
| 34 | +# which `first collision < L` is the point at which the scheme breaks |
| 35 | +# for "addressable" memory. |
| 36 | +# |
| 37 | +# Run: |
| 38 | +# ./target/release/omnimcode-standalone experiments/hybrid_llm/experiment_2_positional_encoding.omc |
| 39 | +# ============================================================================= |
| 40 | + |
| 41 | +# --------------------------------------------------------------------------- |
| 42 | +# Encodings |
| 43 | +# --------------------------------------------------------------------------- |
| 44 | + |
| 45 | +fn harmonic_pe(pos) -> int { |
| 46 | + return phi.fold(pos * 7 + 1); |
| 47 | +} |
| 48 | + |
| 49 | +fn sin_pe(pos) -> int { |
| 50 | + h two_pi = 2.0 * pi(); |
| 51 | + h period = 17.0; |
| 52 | + h theta = to_float(pos) * two_pi / period; |
| 53 | + return to_int(1000.0 * sin(theta)) + 1000; |
| 54 | +} |
| 55 | + |
| 56 | +# --------------------------------------------------------------------------- |
| 57 | +# Distinctness over positions [0, L). |
| 58 | +# |
| 59 | +# We compute the PE for each position, then naive O(L^2) scan to count: |
| 60 | +# - distinct codes |
| 61 | +# - max group size |
| 62 | +# - first collision position (or L if none) |
| 63 | +# |
| 64 | +# Returns [unique_codes, max_group, first_collision]. |
| 65 | +# --------------------------------------------------------------------------- |
| 66 | +fn analyse_pe(L, which) -> array { |
| 67 | + h codes = arr_new(L, 0); |
| 68 | + h i = 0; |
| 69 | + while i < L { |
| 70 | + h c = 0; |
| 71 | + if which == 0 { c = harmonic_pe(i); } |
| 72 | + if which == 1 { c = sin_pe(i); } |
| 73 | + arr_set(codes, i, c); |
| 74 | + i = i + 1; |
| 75 | + } |
| 76 | + |
| 77 | + # Count unique codes via O(L^2) — fine for L <= 64. |
| 78 | + h unique = 0; |
| 79 | + h max_group = 0; |
| 80 | + h first_collision = L; |
| 81 | + h ii = 0; |
| 82 | + while ii < L { |
| 83 | + h is_unique = 1; |
| 84 | + h group = 1; |
| 85 | + h jj = 0; |
| 86 | + while jj < L { |
| 87 | + if jj != ii { |
| 88 | + if arr_get(codes, jj) == arr_get(codes, ii) { |
| 89 | + group = group + 1; |
| 90 | + if jj < ii { |
| 91 | + # ii is a collision against an earlier position. |
| 92 | + is_unique = 0; |
| 93 | + if ii < first_collision { first_collision = ii; } |
| 94 | + } |
| 95 | + } |
| 96 | + } |
| 97 | + jj = jj + 1; |
| 98 | + } |
| 99 | + if is_unique == 1 { unique = unique + 1; } |
| 100 | + if group > max_group { max_group = group; } |
| 101 | + ii = ii + 1; |
| 102 | + } |
| 103 | + |
| 104 | + h out = arr_new(3, 0); |
| 105 | + arr_set(out, 0, unique); |
| 106 | + arr_set(out, 1, max_group); |
| 107 | + arr_set(out, 2, first_collision); |
| 108 | + return out; |
| 109 | +} |
| 110 | + |
| 111 | +# --------------------------------------------------------------------------- |
| 112 | +# Sweep sequence length and tabulate distinctness. |
| 113 | +# --------------------------------------------------------------------------- |
| 114 | +print("== Experiment 2: positional-encoding distinctness vs sequence length =="); |
| 115 | +print(""); |
| 116 | +print("L | harmonic (unique / max_group / first_collision) | sinusoidal (unique / max_group / first_collision)"); |
| 117 | +print("------+--------------------------------------------------+----------------------------------------------------"); |
| 118 | + |
| 119 | +h lengths = [8, 14, 17, 22, 32, 48, 64]; |
| 120 | +h n_lens = arr_len(lengths); |
| 121 | +h li = 0; |
| 122 | +while li < n_lens { |
| 123 | + h L = arr_get(lengths, li); |
| 124 | + h h_stats = analyse_pe(L, 0); |
| 125 | + h s_stats = analyse_pe(L, 1); |
| 126 | + print(concat_many( |
| 127 | + " ", L, |
| 128 | + " | ", arr_get(h_stats, 0), " / ", arr_get(h_stats, 1), " / ", arr_get(h_stats, 2), |
| 129 | + " | ", arr_get(s_stats, 0), " / ", arr_get(s_stats, 1), " / ", arr_get(s_stats, 2) |
| 130 | + )); |
| 131 | + li = li + 1; |
| 132 | +} |
| 133 | +print(""); |
| 134 | + |
| 135 | +# --------------------------------------------------------------------------- |
| 136 | +# Practical task: positional lookup. |
| 137 | +# Given a sequence of 8 tokens and a query position k, retrieve token[k]. |
| 138 | +# "Retrieval" works by combining each token with its PE, then matching |
| 139 | +# against (sentinel_token, pe(k)). Whichever stored pair matches the |
| 140 | +# target PE wins. If two positions share a PE, the wrong token can win. |
| 141 | +# |
| 142 | +# We run this at L=8 (where both PEs should be distinct) and L=24 |
| 143 | +# (well past sin's period and within harmonic's small attractor set). |
| 144 | +# --------------------------------------------------------------------------- |
| 145 | + |
| 146 | +fn vocab_token(i) -> int { |
| 147 | + # Small fixed vocab for the lookup task — Fibonacci attractors again. |
| 148 | + if i == 0 { return 1; } |
| 149 | + if i == 1 { return 2; } |
| 150 | + if i == 2 { return 3; } |
| 151 | + if i == 3 { return 5; } |
| 152 | + if i == 4 { return 8; } |
| 153 | + if i == 5 { return 13; } |
| 154 | + if i == 6 { return 21; } |
| 155 | + if i == 7 { return 34; } |
| 156 | + if i == 8 { return 55; } |
| 157 | + if i == 9 { return 89; } |
| 158 | + if i == 10 { return 144; } |
| 159 | + if i == 11 { return 233; } |
| 160 | + return 1; |
| 161 | +} |
| 162 | + |
| 163 | +# A "lookup" head: for each stored (token, pe(stored_pos)), pick the one |
| 164 | +# whose pe(stored_pos) is closest to pe(query). Ties broken by first. |
| 165 | +fn lookup(tokens, n, query_pos, which_pe) -> int { |
| 166 | + h target_code = 0; |
| 167 | + if which_pe == 0 { target_code = harmonic_pe(query_pos); } |
| 168 | + if which_pe == 1 { target_code = sin_pe(query_pos); } |
| 169 | + |
| 170 | + h best_idx = 0; |
| 171 | + h best_diff = 999999; |
| 172 | + h i = 0; |
| 173 | + while i < n { |
| 174 | + h stored_code = 0; |
| 175 | + if which_pe == 0 { stored_code = harmonic_pe(i); } |
| 176 | + if which_pe == 1 { stored_code = sin_pe(i); } |
| 177 | + h d = stored_code - target_code; |
| 178 | + if d < 0 { d = 0 - d; } |
| 179 | + if d < best_diff { |
| 180 | + best_diff = d; |
| 181 | + best_idx = i; |
| 182 | + } |
| 183 | + i = i + 1; |
| 184 | + } |
| 185 | + return arr_get(tokens, best_idx); |
| 186 | +} |
| 187 | + |
| 188 | +fn run_lookup_suite(L) { |
| 189 | + print(concat_many("-- Lookup task at L=", L, " (token = vocab[i]) --")); |
| 190 | + h tokens = arr_new(L, 0); |
| 191 | + h i = 0; |
| 192 | + while i < L { |
| 193 | + arr_set(tokens, i, vocab_token(i)); # deterministic: token[i] = vocab[i] |
| 194 | + i = i + 1; |
| 195 | + } |
| 196 | + |
| 197 | + h h_correct = 0; |
| 198 | + h s_correct = 0; |
| 199 | + h k = 0; |
| 200 | + while k < L { |
| 201 | + h truth = arr_get(tokens, k); |
| 202 | + h h_got = lookup(tokens, L, k, 0); |
| 203 | + h s_got = lookup(tokens, L, k, 1); |
| 204 | + if h_got == truth { h_correct = h_correct + 1; } |
| 205 | + if s_got == truth { s_correct = s_correct + 1; } |
| 206 | + k = k + 1; |
| 207 | + } |
| 208 | + print(concat_many(" harmonic_pe: ", h_correct, " / ", L, " positions retrieved correctly")); |
| 209 | + print(concat_many(" sin_pe: ", s_correct, " / ", L, " positions retrieved correctly")); |
| 210 | + print(""); |
| 211 | +} |
| 212 | + |
| 213 | +run_lookup_suite(8); |
| 214 | +run_lookup_suite(14); |
| 215 | +run_lookup_suite(24); |
| 216 | +run_lookup_suite(48); |
| 217 | + |
| 218 | +print("== Interpretation =="); |
| 219 | +print(""); |
| 220 | +print("Distinctness table:"); |
| 221 | +print(" - harmonic_pe collides EARLY. phi.fold has ~14 attractors in this"); |
| 222 | +print(" range, so once we ask for >14 positions, codes must repeat."); |
| 223 | +print(" first_collision tells us at what L this kicks in."); |
| 224 | +print(" - sin_pe with period 17 should be roughly distinct up to L=17"); |
| 225 | +print(" then wrap. Quantisation can cause earlier collisions if two"); |
| 226 | +print(" sin values land in the same int bucket."); |
| 227 | +print(""); |
| 228 | +print("Lookup table:"); |
| 229 | +print(" - At L=8, both PEs should retrieve perfectly (one code per position)."); |
| 230 | +print(" - At L=14, harmonic is at the edge of its attractor budget."); |
| 231 | +print(" - At L=24, sin starts wrapping (24 > 17) — positions k and k+17"); |
| 232 | +print(" collide and the wrong token wins."); |
| 233 | +print(" - At L=48, both should be heavily degraded but in different ways:"); |
| 234 | +print(" sin wraps cleanly, harmonic collapses to a few attractor buckets."); |
| 235 | +print(""); |
| 236 | +print("If harmonic_pe collides badly at modest L, the harmonic positional"); |
| 237 | +print("scheme is NOT a drop-in for sinusoidal — you'd need a multi-channel"); |
| 238 | +print("variant (several phi.fold(pos*k_i)) to get a unique per-position"); |
| 239 | +print("vector. That extension is experiment 3."); |
| 240 | +print("== End =="); |
0 commit comments