fix(tokenizer): Qwen3.6 BOS = <|endoftext|> (248044), not <|im_start|>

ROOT CAUSE FOUND for Qwen3.6-27B Tier 3 forward-pass divergence.

Investigation chain:
1. basin_compat showed L0 element-level sign flip (ours +0.25 vs llama -0.29)
2. Pre-norm input also sign-flipped (ours +0.0035 vs inferred llama -0.003)
3. Embedding lookup itself diverged at supposed-same token
4. Token IDs traced via TQ_DEBUG_TOKENS env: ours=[248045, 9419] but
   GGUF bos_token_id metadata = 248044 (<|endoftext|>)
5. vocab[248044] = '<|endoftext|>', vocab[248045] = '<|im_start|>'
6. tq_encode str_lookup chain hits <|im_start|> first (id 248045)
   before <|endoftext|> (id 248044) is checked → wrong BOS

Fix:
- src/engine/tq_tokenizer.c: append <|endoftext|> to BOS str_lookup chain
  (still preferred AFTER <|im_start|> for backward compat with smaller
  Qwen models that use <|im_start|> as functional BOS)
- src/engine/tq_generate.c: for Qwen3.6 family (vocab > 240K), detect
  presence of <|endoftext|> and override prompt_tokens[0] to that id.
  Bypasses the str_lookup ordering issue without breaking Qwen3-0.6B,
  Qwen3.5-4B, etc. (which have smaller vocab and use older convention).
- src/engine/tq_transformer.c: enhanced [dn-trace] output to include
  attn_norm first3+last3 and pre-norm input for paired-diff debugging.

Verified after fix:
  Tokens: [248044, 9419] ✓ matches llama
  L0 attn_norm pos=0 (BOS): first3 = [-0.2891, -0.6430, 0.4991]
  llama row 0 first3:                [-0.2891, -0.6430, 0.4991] ✓ BIT-EXACT

Remaining issue: pos=1 ("Hello" token id 9419) doesn't match llama,
suggesting llama tokenizes start-of-prompt with implicit space prefix
("ĠHello" id 21251). This is BPE pre-tokenizer behavior — separate
fix needed in pre_tokenize_gpt2_bpe path, not blocking BOS fix.

Earlier verdict "Qwen3.6-27B is Tier 3, fundamental forward-pass bug"
was WRONG. The forward pass is correct; tokenization was the issue.
With BOS fix, L0 BOS row is bit-exact to llama. Real tier classification
requires re-running coh_bench after pre-tokenizer fix lands.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: unamedkr

src/engine/tq_generate.c

@@ -356,10 +356,51 @@ int tq_generate(tq_model_t* model, tq_tokenizer_t* tokenizer,
                 }
                 if (bos_id >= 0) add_bos = 1;
             }
+            /* Qwen3.6 family (27B dense, 35B-A3B): GGUF metadata sets
+             * BOS=<|endoftext|> id 248044. tokenizer.ggml.add_bos_token=false
+             * but llama-cli adds BOS by default in main, and our basin_compat
+             * measurements showed missing BOS causes 100× outlier divergence
+             * at L0 (tokenization mismatch with reference). Detect by
+             * presence of <|endoftext|> in vocab. */
+            if (!add_bos) {
+                /* <|endoftext|> for Qwen3.6 lives in 248040-248050 range (vocab=248320) */
+                int lo = 248040, hi = 248060;
+                if (hi > tokenizer->vocab_size) hi = tokenizer->vocab_size;
+                for (int i = lo; i < hi; i++) {
+                    if (tokenizer->vocab[i] && strcmp(tokenizer->vocab[i], "<|endoftext|>") == 0) {
+                        add_bos = 1; break;
+                    }
+                }
+            }
+        }
+        /* Qwen3.6 BOS-id fix: tq_encode str_lookup chain checks <|im_start|>
+         * before <|endoftext|>, picking id 248045 instead of correct 248044
+         * for Qwen3.6 family (27B, 35B-A3B). For these models, override the
+         * BOS to <|endoftext|> directly. Detected by large vocab (>240K) +
+         * presence of <|endoftext|>. */
+        int qwen36_bos_override = -1;
+        if (add_bos && tokenizer->vocab_size > 240000) {
+            int lo = 248040, hi = 248060;
+            if (hi > tokenizer->vocab_size) hi = tokenizer->vocab_size;
+            for (int i = lo; i < hi; i++) {
+                if (tokenizer->vocab[i] && strcmp(tokenizer->vocab[i], "<|endoftext|>") == 0) {
+                    qwen36_bos_override = i; break;
+                }
+            }
         }
         n_prompt = tq_encode(tokenizer, prompt, prompt_tokens,
                               (int)(sizeof(prompt_tokens)/sizeof(prompt_tokens[0])),
                               add_bos);
+        /* Qwen3.6 BOS override: tq_encode picked <|im_start|> (248045) but
+         * GGUF metadata BOS = <|endoftext|> (248044). Replace at index 0. */
+        if (qwen36_bos_override >= 0 && n_prompt > 0 && add_bos) {
+            prompt_tokens[0] = qwen36_bos_override;
+        }
+        if (getenv("TQ_DEBUG_TOKENS")) {
+            fprintf(stderr, "[tq_encode] add_bos=%d n_prompt=%d tokens=[", add_bos, n_prompt);
+            for (int i = 0; i < n_prompt && i < 20; i++) fprintf(stderr, "%d%s", prompt_tokens[i], i+1<n_prompt?",":"");
+            fprintf(stderr, "]\n");
+        }
     } else {
         prompt_tokens[0] = (model->config.model_type == 1) ? 2 : 1;
         n_prompt = 1;

src/engine/tq_tokenizer.c

@@ -1203,13 +1203,16 @@ int tq_encode(const tq_tokenizer_t* tok, const char* text,
     int n_tokens = 0;
 
     /* Add BOS token if requested.
-     * Gemma: BOS=2, Qwen: no BOS (uses <|im_start|> instead) */
+     * Gemma: BOS=2, Qwen: no BOS (uses <|im_start|> instead).
+     * Qwen3.6 (27B dense, 35B-A3B): GGUF metadata has BOS=<|endoftext|> id 248044.
+     * Added for Qwen3.6 family to match llama.cpp tokenization. */
     if (add_bos) {
         /* Look up <bos> token in vocab; default to id 2 (Gemma convention) */
         int bos_id = str_lookup(tok, "<bos>");
         if (bos_id < 0) { bos_id = str_lookup(tok, "<s>"); }
         if (bos_id < 0) { bos_id = str_lookup(tok, "<|begin_of_text|>"); }
         if (bos_id < 0) { bos_id = str_lookup(tok, "<|im_start|>"); }
+        if (bos_id < 0) { bos_id = str_lookup(tok, "<|endoftext|>"); }
         if (bos_id >= 0) {
             tokens[n_tokens++] = bos_id;
         }

src/engine/tq_transformer.c

@@ -702,7 +702,14 @@ static void deltanet_forward(tq_model_t* model, tq_state_t* s, int l) {
     if (dn_trace) {
         double xb_sum = 0;
         for (int i = 0; i < dim; i++) xb_sum += s->xb[i];
-        fprintf(stderr, "[dn-trace] L%d attn_norm_out sum=%.6f\n", l, xb_sum);
+        fprintf(stderr, "[dn-trace] L%d attn_norm_out sum=%.6f first3=%.4f,%.4f,%.4f last3=%.4f,%.4f,%.4f\n",
+                l, xb_sum, s->xb[0], s->xb[1], s->xb[2],
+                s->xb[dim-3], s->xb[dim-2], s->xb[dim-1]);
+        /* Also dump pre-norm input (s->x) for embedding probe */
+        double xs = 0;
+        for (int i = 0; i < dim; i++) xs += s->x[i];
+        fprintf(stderr, "[dn-trace] L%d pre_norm_input sum=%.6f first3=%.4f,%.4f,%.4f\n",
+                l, xs, s->x[0], s->x[1], s->x[2]);
     }
 
     /* Pre-quantize activation to Q8 once for all Q2/Q4 projections in this layer.