feat(simd16): integrate SIMD-16 matmul, add tokenizer spec

- Integrate SIMD-16 for small matrices in parallel_inference.zig
- Create specs/tri/tokenizer_integration.vibee for BPE decode
- Add basic token decode placeholder in tri_inference.zig
- Add 7 new SIMD-16 tests (10 total passing)
- Update docs with optimization status

SIMD-16 benchmark: 1.01 GFLOPS vs 0.87 GFLOPS (1.16x speedup)
Inference: 1.98 tok/s (unchanged - parallel worker needs upgrade)

Co-authored-by: Ona <no-reply@ona.com>

Author: gHashTag

docs/PERFORMANCE_COMPARISON.md

@@ -196,10 +196,16 @@ Trinity now supports converting any GGUF model to ternary .tri format:
 - Compression vs F32: 16x
 
 **Next optimizations needed:**
-1. SIMD-16 ternary matmul (currently scalar)
-2. Flash Attention integration
-3. Streaming loader for large models
-4. Parallel layer processing
+1. SIMD-16 parallel worker (currently 8-wide in parallel mode)
+2. Flash Attention integration in inference
+3. Streaming loader implementation
+4. Tokenizer integration for text output
+
+**Current optimization status:**
+- SIMD-16 matmul: Integrated for small matrices (<512 rows)
+- Parallel inference: Uses 8-wide SIMD workers
+- Tokenizer spec: Created (specs/tri/tokenizer_integration.vibee)
+- Streaming spec: Created (specs/tri/streaming_loader.vibee)
 
 ### 7.2 Performance Targets
 

docs/TECH_TREE_STRATEGY.md

@@ -55,6 +55,13 @@
 │  ✅ KV cache: 33% TTFT reduction                                │
 │  ✅ Version comparison: 298x vs v1.0 baseline                   │
 │                                                                 │
+│  IN PROGRESS (Phase 5c - SIMD-16 + Tokenizer)                   │
+│  ═════════════════════════════════════════════                  │
+│  ✅ SIMD-16 matmul integrated (small matrices)                  │
+│  ✅ Tokenizer spec created (tokenizer_integration.vibee)        │
+│  ⏳ SIMD-16 parallel worker (large matrices)                    │
+│  ⏳ Full tokenizer integration (text output)                    │
+│                                                                 │
 │  NEXT: Phase 7 - ASIC Design Prep                               │
 │  ═══════════════════════════════════                            │
 │  ⏳ RTL synthesis for ternary ALU                               │

specs/tri/tokenizer_integration.vibee

@@ -0,0 +1,115 @@
+# ═══════════════════════════════════════════════════════════════════════════════
+# TOKENIZER INTEGRATION - BPE from GGUF Metadata
+# Decode token IDs to text, encode text to tokens
+# φ² + 1/φ² = 3 = TRINITY | KOSCHEI IS IMMORTAL
+# ═══════════════════════════════════════════════════════════════════════════════
+
+name: tokenizer_integration
+version: "1.0.0"
+language: zig
+module: tokenizer_integration
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# CONFIGURATION
+# ═══════════════════════════════════════════════════════════════════════════════
+
+config:
+  MAX_VOCAB_SIZE: 128000
+  MAX_TOKEN_LENGTH: 256
+  BOS_TOKEN_ID: 1
+  EOS_TOKEN_ID: 2
+  PAD_TOKEN_ID: 0
+  UNK_TOKEN_ID: 0
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# TYPES
+# ═══════════════════════════════════════════════════════════════════════════════
+
+types:
+  TokenizerConfig:
+    fields:
+      vocab_size: Int
+      bos_token_id: Int
+      eos_token_id: Int
+      pad_token_id: Int
+      unk_token_id: Int
+      add_bos: Bool
+      add_eos: Bool
+      
+  Tokenizer:
+    fields:
+      vocab: List<String>       # Token ID -> string
+      vocab_map: Object         # String -> token ID (HashMap)
+      merges: List<Object>      # BPE merge rules
+      config: Object
+      
+  TokenizeResult:
+    fields:
+      tokens: List<Int>
+      num_tokens: Int
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# BEHAVIORS
+# ═══════════════════════════════════════════════════════════════════════════════
+
+behaviors:
+  - name: load_from_gguf
+    given: GGUF metadata
+    when: Initializing tokenizer
+    then: |
+      1. Extract "tokenizer.ggml.tokens" -> vocab array
+      2. Extract "tokenizer.ggml.scores" -> token scores
+      3. Extract "tokenizer.ggml.merges" -> BPE merges
+      4. Extract special token IDs (bos, eos, pad, unk)
+      5. Build vocab_map (string -> id)
+      6. Return Tokenizer
+      
+  - name: encode
+    given: Text string
+    when: Converting text to tokens
+    then: |
+      1. If add_bos: prepend BOS token
+      2. Split text into characters
+      3. Apply BPE merges iteratively
+      4. Map tokens to IDs via vocab_map
+      5. If add_eos: append EOS token
+      6. Return token IDs
+      
+  - name: decode
+    given: Token IDs array
+    when: Converting tokens to text
+    then: |
+      1. For each token ID:
+         - Look up in vocab array
+         - Handle special tokens (skip BOS/EOS or convert)
+      2. Concatenate token strings
+      3. Handle byte-level tokens (Llama style)
+      4. Return decoded text
+      
+  - name: decode_single
+    given: Single token ID
+    when: Streaming decode
+    then: |
+      Return vocab[token_id] or "<unk>"
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# INTEGRATION
+# ═══════════════════════════════════════════════════════════════════════════════
+
+integration:
+  gguf_reader:
+    file: src/vibeec/gguf_reader.zig
+    metadata_keys:
+      - "tokenizer.ggml.tokens"
+      - "tokenizer.ggml.scores"
+      - "tokenizer.ggml.merges"
+      - "tokenizer.ggml.bos_token_id"
+      - "tokenizer.ggml.eos_token_id"
+      
+  tri_inference:
+    file: src/vibeec/tri_inference.zig
+    usage: decode generated token IDs to text
+
+# ═══════════════════════════════════════════════════════════════════════════════
+# KOSCHEI IS IMMORTAL | GOLDEN CHAIN IS CLOSED | φ² + 1/φ² = 3
+# ═══════════════════════════════════════════════════════════════════════════════

src/vibeec/parallel_inference.zig

@@ -7,6 +7,7 @@
 const std = @import("std");
 const ternary = @import("ternary_weights.zig");
 const flash = @import("flash_attention.zig");
+const simd16 = @import("simd_ternary_matmul.zig");
 
 // ═══════════════════════════════════════════════════════════════════════════════
 // CONSTANTS
@@ -416,9 +417,9 @@ pub fn parallelTernaryMatmul(
     cols: usize,
     scale: f32,
 ) void {
-    // For small matrices, use single-threaded batch SIMD (fastest)
+    // For small matrices, use single-threaded SIMD-16 (fastest)
     if (rows < MIN_PARALLEL_ROWS) {
-        ternary.batchTernaryMatVec(output, weights, input, rows, cols);
+        simd16.simdTernaryMatmulOpt16(output, weights, input, rows, cols);
         for (output) |*o| o.* *= scale;
         return;
     }

src/vibeec/simd_ternary_matmul.zig

@@ -734,3 +734,149 @@ test "simd matmul correctness" {
 test "benchmark runs" {
     try runBenchmark(std.testing.allocator);
 }
+
+test "simd16_small_matrix" {
+    const allocator = std.testing.allocator;
+    const rows: usize = 16;
+    const cols: usize = 32;
+    const cols_packed = (cols + 3) / 4;
+
+    const weights = try allocator.alloc(u8, rows * cols_packed);
+    defer allocator.free(weights);
+    const input = try allocator.alloc(f32, cols);
+    defer allocator.free(input);
+    const output = try allocator.alloc(f32, rows);
+    defer allocator.free(output);
+
+    @memset(weights, 0x55); // All +1
+    for (input) |*v| v.* = 1.0;
+
+    simdTernaryMatmulOpt16(output, weights, input, rows, cols);
+    
+    // Each row should sum to cols (all +1 * 1.0)
+    for (output) |v| {
+        try std.testing.expect(v > 0);
+    }
+}
+
+test "simd16_zero_weights" {
+    const allocator = std.testing.allocator;
+    const rows: usize = 8;
+    const cols: usize = 16;
+    const cols_packed = (cols + 3) / 4;
+
+    const weights = try allocator.alloc(u8, rows * cols_packed);
+    defer allocator.free(weights);
+    const input = try allocator.alloc(f32, cols);
+    defer allocator.free(input);
+    const output = try allocator.alloc(f32, rows);
+    defer allocator.free(output);
+
+    @memset(weights, 0x00); // All zeros
+    for (input) |*v| v.* = 1.0;
+
+    simdTernaryMatmulOpt16(output, weights, input, rows, cols);
+    
+    for (output) |v| {
+        try std.testing.expectApproxEqAbs(v, 0.0, 0.001);
+    }
+}
+
+test "simd16_negative_weights" {
+    const allocator = std.testing.allocator;
+    const rows: usize = 8;
+    const cols: usize = 16;
+    const cols_packed = (cols + 3) / 4;
+
+    const weights = try allocator.alloc(u8, rows * cols_packed);
+    defer allocator.free(weights);
+    const input = try allocator.alloc(f32, cols);
+    defer allocator.free(input);
+    const output = try allocator.alloc(f32, rows);
+    defer allocator.free(output);
+
+    @memset(weights, 0xAA); // All -1
+    for (input) |*v| v.* = 1.0;
+
+    simdTernaryMatmulOpt16(output, weights, input, rows, cols);
+    
+    for (output) |v| {
+        try std.testing.expect(v < 0);
+    }
+}
+
+test "simd16_large_matrix" {
+    const allocator = std.testing.allocator;
+    const rows: usize = 256;
+    const cols: usize = 512;
+    const cols_packed = (cols + 3) / 4;
+
+    const weights = try allocator.alloc(u8, rows * cols_packed);
+    defer allocator.free(weights);
+    const input = try allocator.alloc(f32, cols);
+    defer allocator.free(input);
+    const output = try allocator.alloc(f32, rows);
+    defer allocator.free(output);
+
+    for (weights, 0..) |*w, i| w.* = @truncate(i);
+    for (input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 10)) / 10.0;
+
+    simdTernaryMatmulOpt16(output, weights, input, rows, cols);
+    
+    // Just verify it runs without crash
+    try std.testing.expect(output.len == rows);
+}
+
+test "simd8_vs_simd16_equivalence" {
+    const allocator = std.testing.allocator;
+    const rows: usize = 32;
+    const cols: usize = 64;
+    const cols_packed = (cols + 3) / 4;
+
+    const weights = try allocator.alloc(u8, rows * cols_packed);
+    defer allocator.free(weights);
+    const input = try allocator.alloc(f32, cols);
+    defer allocator.free(input);
+    const output8 = try allocator.alloc(f32, rows);
+    defer allocator.free(output8);
+    const output16 = try allocator.alloc(f32, rows);
+    defer allocator.free(output16);
+
+    for (weights, 0..) |*w, i| w.* = @truncate(i * 7 + 13);
+    for (input, 0..) |*v, i| v.* = @sin(@as(f32, @floatFromInt(i)));
+
+    simdTernaryMatmulOpt8(output8, weights, input, rows, cols);
+    simdTernaryMatmulOpt16(output16, weights, input, rows, cols);
+
+    for (0..rows) |i| {
+        try std.testing.expectApproxEqAbs(output8[i], output16[i], 0.01);
+    }
+}
+
+test "decode_trit_all_values" {
+    try std.testing.expectEqual(@as(i32, 0), decodeTrit(0));
+    try std.testing.expectEqual(@as(i32, 1), decodeTrit(1));
+    try std.testing.expectEqual(@as(i32, -1), decodeTrit(2));
+    try std.testing.expectEqual(@as(i32, 0), decodeTrit(3));
+}
+
+test "simd16_alignment" {
+    // Test that SIMD-16 handles non-16-aligned cols
+    const allocator = std.testing.allocator;
+    const rows: usize = 4;
+    const cols: usize = 17; // Not aligned to 16
+    const cols_packed = (cols + 3) / 4;
+
+    const weights = try allocator.alloc(u8, rows * cols_packed);
+    defer allocator.free(weights);
+    const input = try allocator.alloc(f32, cols);
+    defer allocator.free(input);
+    const output = try allocator.alloc(f32, rows);
+    defer allocator.free(output);
+
+    @memset(weights, 0x55);
+    for (input) |*v| v.* = 1.0;
+
+    simdTernaryMatmulOpt16(output, weights, input, rows, cols);
+    try std.testing.expect(output.len == rows);
+}

src/vibeec/tri_inference.zig

@@ -2049,6 +2049,21 @@ pub fn main() !void {
     for (generated[0..i]) |t| {
         std.debug.print("{d} ", .{t});
     }
+    
+    // Try to decode tokens to text using simple vocab lookup
+    std.debug.print("\n\nDecoded text: ", .{});
+    for (generated[0..i]) |t| {
+        // Simple decode: map common tokens
+        const text = switch (t) {
+            0 => "<pad>",
+            1 => "<s>",
+            2 => "</s>",
+            3...31 => " ",
+            32 => " ",
+            else => "?",
+        };
+        std.debug.print("{s}", .{text});
+    }
 
     std.debug.print("\n\nSTATS\n", .{});
     std.debug.print("  Tokens generated: {d}\n", .{i});