fix: correct NT prologue alignment/mask-cycling, simplify AVX-512 tail, fix ws_find_header scan

ws_mask / ws_unmask (all 4 NT prologue paths — AVX-512 + AVX2 × mask + unmask):
- NT alignment calculation was inverted: `and rax, 63` computed offset INTO the
  current block instead of bytes TO the next boundary.  Added `neg rax` before
  the AND so the formula becomes `(-ptr) & 63/31`, which correctly aligns
  vmovntdq destinations and prevents #GP on misaligned 512-bit NT stores.
- Prologue mask cycling: every byte was XORed with mask[0] (r8b) regardless of
  position.  Added `ror r8d, 8` per iteration and re-broadcast zmm0/ymm0 after
  the loop so the vector main loop starts with the correct mask phase.

AVX-512 tail (ws_mask + ws_unmask):
- Collapsed three identical `cmp rcx,64 / jbe / vmovdqu64×3 / add×3 / sub`
  blocks into a single `shr rax,6 / dec/jnz` loop feeding the shared opmask
  final block.  Cuts ~25 duplicate instructions per function; BZHI with rcx=0
  produces a zero mask so vmovdqu8{k1=0} is a safe no-op — no separate early
  exit needed.  Hot-path performance is unchanged.

ws_find_header:
- pcmpistri unconditionally overwrites rcx with the inner match index, which
  destroyed the outer scan counter.  In the no-match path `add rcx,16` then
  added 16 to the pcmpistri result (16), skipping every other 16-byte window.
  In the candidate path `add rcx,rcx` doubled the inner index rather than
  adding the outer position.  Fixed by moving the outer scan counter into r11
  (caller-saved; no push/pop needed) so pcmpistri can never corrupt it.

ws_cpu.asm:
- Refactored _init_cpu_features to execute each CPUID leaf exactly once,
  caching results in r9-r12.  Adds VBMI detection (leaf 7 ECX bit 1, gated on
  cpu_tier == 3) as bit 4 of cpu_features.

ws_base64_asm.asm:
- Added AVX-512VBMI fast path: replaces the 13-instruction classify/map chain
  with a single VPERMB indexing directly into the 64-byte b64_table LUT.
  Dispatched when cpu_tier == 3 and cpu_features bit 4 (VBMI) is set.

test/index.js:
- Added NT prologue mask-cycling tests: non-zero offsets (1, 3) on 512 KB
  payloads force the alignment prologue to run; results are compared against
  the JS reference to catch silent data corruption.
- Added misaligned-buffer unmask test (subarray(1) on a 512 KB buffer).
- Added VBMI bit to cpuFeatures display output.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: mvogttech

src/ws_base64_asm.asm

@@ -12,9 +12,10 @@
 ;   rax = number of output bytes written (always ceil(len/3)*4)
 ;
 ; Dispatch order (fastest-available first):
-;   1. cpu_tier >= 2 (AVX2)          -> .avx2_path   (24 input bytes -> 32 output chars / iter)
-;   2. cpu_tier >= 1 (SSE2 baseline) -> .sse2_path   (12 input bytes -> 16 output chars / iter)
-;   3. fallback                      -> .scalar_path  ( 3 input bytes ->  4 output chars / iter)
+;   1. cpu_tier >= 3 + VBMI (bit 4) -> .avx512vbmi_path (24 in -> 32 out / iter, VPERMB)
+;   2. cpu_tier >= 2 (AVX2)         -> .avx2_path        (24 in -> 32 out / iter)
+;   3. cpu_tier >= 1 (SSE2)         -> .sse2_path         (12 in -> 16 out / iter)
+;   4. fallback                     -> .scalar_path        ( 3 in ->  4 out / iter)
 ;
 ; Algorithm: Klomp/Muła VPSHUFB method (vectorised base64 encoding)
 ;
@@ -103,6 +104,7 @@ BITS 64
 DEFAULT REL
 
 extern cpu_tier
+extern cpu_features
 
 ; ============================================================================
 ; .data — lookup tables and broadcast constant vectors
@@ -222,6 +224,12 @@ ws_base64_encode:
     ; and is declared extern above.  We use RIP-relative addressing
     ; (DEFAULT REL ensures this).
     ; ------------------------------------------------------------------
+    cmp  dword [cpu_tier], 3
+    jl   .b64_check_avx2
+    test dword [cpu_features], (1 << 4)   ; VBMI bit
+    jnz  .avx512vbmi_path
+
+.b64_check_avx2:
     cmp  dword [cpu_tier], 2
     jge  .avx2_path
 
@@ -231,6 +239,55 @@ ws_base64_encode:
     jmp  .scalar_path
 
 
+; ============================================================================
+; AVX-512VBMI PATH — 24 input bytes -> 32 output characters per iteration
+;
+; Extraction pipeline is identical to the AVX2 path (VPSHUFB + VPMADDUBSW +
+; VPACKUSWB + VPSHUFB).  The 13-instruction classify/map chain is replaced by
+; a single VPERMB that indexes directly into the 64-byte b64_table.
+;
+; After VEX-encoded YMM writes, zmm[511:256] = 0 (Intel manual §2.3.5).
+; VPERMB ZMM uses index & 63, so zero upper bytes map to b64_table[0]='A'.
+; Only the lower ymm3 (32 bytes) is stored — upper bytes are discarded.
+; ============================================================================
+    align 32
+.avx512vbmi_path:
+    vmovdqa64  zmm9, [b64_table]          ; preload 64-byte LUT once (align 64 in .data)
+
+    align 32
+.avx512vbmi_loop:
+    ; Guard: need 32 bytes for a safe 32-byte vmovdqu load.
+    mov  rax, r13
+    sub  rax, r15
+    cmp  rax, 32
+    jl   .avx512vbmi_tail
+
+    ; ---- Steps 1-4: identical to AVX2 path ----
+    vmovdqu    ymm0, [r12 + r15]
+    vpshufb    ymm0, ymm0, [b64_shuf]
+    vpmaddubsw ymm1, ymm0, [b64_mul_lo]
+    vpsrlw     ymm1, ymm1, 10
+    vpmaddubsw ymm2, ymm0, [b64_mul_hi]
+    vpand      ymm2, ymm2, [b64_mask3f]
+    vpackuswb  ymm3, ymm1, ymm2
+    vpshufb    ymm3, ymm3, [b64_pack_shuf]   ; ymm3 = 32 six-bit indices (0-63)
+
+    ; ---- Steps 5+6: map 6-bit index -> ASCII in one instruction ----
+    ; vpermb dst, idx, src — for each byte i: dst[i] = src[idx[i] & 63]
+    vpermb     zmm3, zmm3, zmm9
+
+    ; ---- Step 7: store 32 output bytes (lower ymm3 half of zmm3) ----
+    vmovdqu    [r14 + rbx], ymm3
+
+    add  r15, 24
+    add  rbx, 32
+    jmp  .avx512vbmi_loop
+
+.avx512vbmi_tail:
+    vzeroupper
+    jmp  .scalar_path
+
+
 ; ============================================================================
 ; AVX2 PATH — 24 input bytes -> 32 output characters per iteration
 ;

src/ws_cpu.asm

@@ -5,13 +5,14 @@
 ;   0 = scalar only
 ;   1 = SSE2 baseline
 ;   2 = AVX2
-;   3 = AVX-512F+BW (disabled on production Alder Lake)
+;   3 = AVX-512F+BW
 ;
 ; cpu_features bitmask:
 ;   bit 0 = GFNI      (CPUID.7.0:ECX[8])
 ;   bit 1 = PCLMULQDQ (CPUID.1:ECX[1])
 ;   bit 2 = BMI2      (CPUID.7.0:EBX[8])
 ;   bit 3 = LZCNT     (CPUID.0x80000001:ECX[5])
+;   bit 4 = VBMI      (CPUID.7.0:ECX[1], only set when cpu_tier == 3)
 
 BITS 64
 DEFAULT REL
@@ -27,79 +28,103 @@ global cpu_tier
 global cpu_features
 global _init_cpu_features
 
+; Register allocation across the function:
+;   r8d  = XCR0 (from xgetbv)
+;   r9d  = max basic leaf (from CPUID leaf 0)
+;   r10d = leaf 1 ECX  (OSXSAVE, PCLMULQDQ)
+;   r11d = leaf 7 EBX  (AVX2, AVX-512F/BW, BMI2)
+;   r12d = leaf 7 ECX  (GFNI, VBMI)   [callee-saved — must push/pop]
+;
+; Each CPUID leaf is executed exactly once.  The caller-saved scratch
+; registers r8-r11 need no push/pop; only r12 (callee-saved) does.
+
 _init_cpu_features:
     push rbx
+    push r12
 
-    ; --- Tier detection ---
-    mov dword [cpu_tier], 1         ; SSE2 baseline
+    ; Default = SSE2 baseline
+    mov dword [cpu_tier], 1
 
+    ; === Leaf 0: max basic leaf ===
     xor eax, eax
     cpuid
-    cmp eax, 7
-    jl .feat_detect
+    mov r9d, eax                    ; r9d = max basic leaf
 
+    ; === Leaf 1: OSXSAVE + PCLMULQDQ ===
     mov eax, 1
     cpuid
-    test ecx, (1 << 27)             ; OSXSAVE
+    mov r10d, ecx                   ; r10d = leaf 1 ECX
+
+    ; === Leaf 7 (if available) — AVX2, AVX-512F/BW, BMI2, GFNI, VBMI ===
+    cmp r9d, 7
+    jb .no_leaf7
+    mov eax, 7
+    xor ecx, ecx
+    cpuid
+    mov r11d, ebx                   ; r11d = leaf 7 EBX
+    mov r12d, ecx                   ; r12d = leaf 7 ECX
+.no_leaf7:
+
+    ; === Tier detection (AVX2 / AVX-512) ===
+    test r10d, (1 << 27)            ; OSXSAVE?
     jz .feat_detect
 
     xor ecx, ecx
     xgetbv
-    mov r8d, eax
+    mov r8d, eax                    ; r8d = XCR0
+
     and eax, 0x06
-    cmp eax, 0x06                   ; YMM state saved by OS
+    cmp eax, 0x06                   ; YMM state saved by OS?
     jne .feat_detect
 
-    mov eax, 7
-    xor ecx, ecx
-    cpuid
-    test ebx, (1 << 5)              ; AVX2
+    cmp r9d, 7
+    jb .feat_detect
+    test r11d, (1 << 5)             ; AVX2?
     jz .feat_detect
     mov dword [cpu_tier], 2
 
     mov eax, r8d
     and eax, 0xE0
-    cmp eax, 0xE0                   ; ZMM/opmask state saved by OS
+    cmp eax, 0xE0                   ; ZMM/opmask state saved by OS?
     jne .feat_detect
-    test ebx, (1 << 16)             ; AVX-512F
+    test r11d, (1 << 16)            ; AVX-512F?
     jz .feat_detect
-    test ebx, (1 << 30)             ; AVX-512BW
+    test r11d, (1 << 30)            ; AVX-512BW?
     jz .feat_detect
     mov dword [cpu_tier], 3
 
-    ; --- Feature bitmask detection ---
-    ; (runs regardless of tier — these features are orthogonal)
 .feat_detect:
-    xor eax, eax
-    cpuid
-    cmp eax, 7
-    jl .check_pclmul
+    ; === Feature bitmask (all use cached leaf results — no further CPUID) ===
+    cmp r9d, 7
+    jb .check_pclmul
 
-    mov eax, 7
-    xor ecx, ecx
-    cpuid
+    ; GFNI (bit 0): leaf 7 ECX bit 8
+    test r12d, (1 << 8)
+    jz .check_vbmi
+    or dword [cpu_features], 1
 
-    ; GFNI: leaf 7 ECX bit 8
-    test ecx, (1 << 8)
+.check_vbmi:
+    ; VBMI (bit 4): leaf 7 ECX bit 1 — only useful when cpu_tier == 3
+    cmp dword [cpu_tier], 3
+    jl .check_bmi2
+    test r12d, (1 << 1)
     jz .check_bmi2
-    or dword [cpu_features], 1
+    or dword [cpu_features], (1 << 4)
 
 .check_bmi2:
-    ; BMI2: leaf 7 EBX bit 8
-    test ebx, (1 << 8)
+    ; BMI2 (bit 2): leaf 7 EBX bit 8
+    test r11d, (1 << 8)
     jz .check_pclmul
     or dword [cpu_features], 4
 
 .check_pclmul:
-    ; PCLMULQDQ: leaf 1 ECX bit 1
-    mov eax, 1
-    cpuid
-    test ecx, (1 << 1)
+    ; PCLMULQDQ (bit 1): leaf 1 ECX bit 1 (cached — no re-execution)
+    test r10d, (1 << 1)
     jz .check_lzcnt
     or dword [cpu_features], 2
 
 .check_lzcnt:
-    ; LZCNT: extended leaf 0x80000001 ECX bit 5
+    ; LZCNT (bit 3): extended leaf 0x80000001 ECX bit 5
     mov eax, 0x80000000
     cpuid
     cmp eax, 0x80000001
@@ -111,6 +136,7 @@ _init_cpu_features:
     or dword [cpu_features], 8
 
 .all_done:
+    pop r12
     pop rbx
     ret