CRC SIMD-related updates

Author: Xor-el

HashLib/src/Checksum/HlpCRC.pas

@@ -1468,9 +1468,10 @@ procedure TCRC.Initialize;
 
     if not FHasPclmulConstants then
     begin
-      if (Width >= 8) and (Width <= 32) and IsInputReflected then
+      if (Width >= 8) and (Width <= 64) then
       begin
-        TGF2.GenerateFoldConstants(Polynomial, Width, True, FPclmulConstants);
+        TGF2.GenerateFoldConstants(Polynomial, Width, IsInputReflected,
+          FPclmulConstants);
         FHasPclmulConstants := True;
       end;
     end;
@@ -1515,17 +1516,35 @@ procedure TCRC.TransformBytes(const AData: THashLibByteArray;
 
   if Width > Delta then
   begin
-    if FHasPclmulConstants and IsInputReflected and (ALength >= 64)
-      and Assigned(CRC_Fold_Lsb) then
+    if FHasPclmulConstants and (ALength >= 64) then
     begin
-      LState[0] := FHash;
-      LState[1] := 0;
-      LProcessed := ALength and (not Int32(15));
-      FHash := CRC_Fold_Lsb(LPtrAData + AIndex, UInt32(LProcessed),
-        @LState[0], @FPclmulConstants) and FCRCMask;
-      LTail := ALength - LProcessed;
-      if LTail > 0 then
-        CalculateCRCbyTable(LPtrAData, LTail, AIndex + LProcessed);
+      if IsInputReflected and Assigned(CRC_Fold_Lsb) then
+      begin
+        LState[0] := FHash;
+        LState[1] := 0;
+        LProcessed := ALength and (not Int32(15));
+        FHash := CRC_Fold_Lsb(LPtrAData + AIndex, UInt32(LProcessed),
+          @LState[0], @FPclmulConstants) and FCRCMask;
+        LTail := ALength - LProcessed;
+        if LTail > 0 then
+          CalculateCRCbyTable(LPtrAData, LTail, AIndex + LProcessed);
+      end
+      else if (not IsInputReflected) and Assigned(CRC_Fold_Msb) then
+      begin
+        if Width < 64 then
+          LState[0] := FHash shl (64 - Width)
+        else
+          LState[0] := FHash;
+        LState[1] := 0;
+        LProcessed := ALength and (not Int32(15));
+        FHash := CRC_Fold_Msb(LPtrAData + AIndex, UInt32(LProcessed),
+          @LState[0], @FPclmulConstants) and FCRCMask;
+        LTail := ALength - LProcessed;
+        if LTail > 0 then
+          CalculateCRCbyTable(LPtrAData, LTail, AIndex + LProcessed);
+      end
+      else
+        CalculateCRCbyTable(LPtrAData, ALength, AIndex);
     end
     else
       CalculateCRCbyTable(LPtrAData, ALength, AIndex);

HashLib/src/Checksum/HlpCRCDispatch.pas

@@ -13,6 +13,7 @@ interface
 
 var
   CRC_Fold_Lsb: TCRCFoldFunc;
+  CRC_Fold_Msb: TCRCFoldFunc;
 
 implementation
 
@@ -33,16 +34,35 @@ function CRC_Fold_Vpclmul(AData: PByte; ALength: UInt32;
   {$I ..\Include\Simd\CRC\CRCFoldVpclmul.inc}
 end;
 
+function CRC_Fold_Pclmul_Msb(AData: PByte; ALength: UInt32;
+  AState: Pointer; AConstants: Pointer): UInt64;
+  {$I ..\Include\Simd\Common\SimdProc4Begin.inc}
+  {$I ..\Include\Simd\CRC\CRCFoldPclmulMsb.inc}
+end;
+
+function CRC_Fold_Vpclmul_Msb(AData: PByte; ALength: UInt32;
+  AState: Pointer; AConstants: Pointer): UInt64;
+  {$I ..\Include\Simd\Common\SimdProc4Begin.inc}
+  {$I ..\Include\Simd\CRC\CRCFoldVpclmulMsb.inc}
+end;
+
 {$ENDIF HASHLIB_X86_64}
 
 procedure InitDispatch();
 begin
   CRC_Fold_Lsb := nil;
+  CRC_Fold_Msb := nil;
 {$IFDEF HASHLIB_X86_64}
   if TSimd.HasVPCLMULQDQ() then
-    CRC_Fold_Lsb := @CRC_Fold_Vpclmul
+  begin
+    CRC_Fold_Lsb := @CRC_Fold_Vpclmul;
+    CRC_Fold_Msb := @CRC_Fold_Vpclmul_Msb;
+  end
   else if TSimd.HasPCLMULQDQ() then
+  begin
     CRC_Fold_Lsb := @CRC_Fold_Pclmul;
+    CRC_Fold_Msb := @CRC_Fold_Pclmul_Msb;
+  end;
 {$ENDIF}
 end;
 

HashLib/src/Checksum/HlpGF2.pas

@@ -10,13 +10,15 @@   TUInt128 = record
   end;
 
   // PCLMULQDQ / VPCLMULQDQ CRC folding and Barrett reduction constants.
-  // Layout must match the assembly expectations in CRCFoldPclmul.inc
-  // and CRCFoldVpclmul.inc.
+  // Layout must match the assembly expectations in the CRCFold*.inc files.
   TCRCFoldConstants = packed record
     Fold_4x128: array [0 .. 1] of UInt64;   // offset  0: fold-by-4 constants (stride 512)
     Fold_1x128: array [0 .. 1] of UInt64;   // offset 16: fold-by-1 constants (stride 128)
     Barrett: array [0 .. 1] of UInt64;       // offset 32: Barrett reduction constants
     Fold_8x128: array [0 .. 1] of UInt64;   // offset 48: fold-by-8 constants (stride 1024)
+    BswapMask: array [0 .. 15] of Byte;      // offset 64: byte-reverse mask for MSB-first CRCs
+    CrcBits: UInt64;                          // offset 80: CRC width (8..64)
+    BarrettShift: UInt64;                     // offset 88: = 64 - CrcBits (MSB psrlq alignment)
   end;
 
   TGF2 = class sealed
@@ -214,8 +216,8 @@ class function TGF2.BitReverse(AValue: UInt64; ANumBits: Int32): UInt64;
 class procedure TGF2.GenerateFoldConstants(APoly: UInt64; ABits: Int32;
   AReflected: Boolean; out AConstants: TCRCFoldConstants);
 var
-  LK, LPowOfX: Int32;
-  LConst0, LConst1, LBarrett0, LBarrett1, LGMinusXn: UInt64;
+  LK, LPowOfX, I: Int32;
+  LConst0, LConst1, LBarrett0, LGMinusXn: UInt64;
   LDiv128: TUInt128;
 begin
   // Following the Linux kernel gen-crc-consts.py algorithm.
@@ -271,40 +273,61 @@ class procedure TGF2.GenerateFoldConstants(APoly: UInt64; ABits: Int32;
   end;
 
   // --- Barrett reduction constants ---
-  // barrett[0] = floor(x^(63+n) / G)
-  LDiv128.Lo := 0;
-  LDiv128.Hi := 0;
-  if (63 + ABits) < 64 then
-    LDiv128.Lo := UInt64(1) shl (63 + ABits)
-  else if (63 + ABits) = 64 then
-    LDiv128.Hi := 1
-  else
-    LDiv128.Hi := UInt64(1) shl ((63 + ABits) - 64);
-  LBarrett0 := DivPoly(LDiv128, APoly, ABits);
-
-  // barrett[1] = (G - x^n) * x^(64-n-1) for n < 64
-  //            = ((G - x^n) - x^0) / x   for n = 64
   LGMinusXn := APoly;
-  if ABits < 64 then
-  begin
-    LPowOfX := 64 - ABits - 1;
-    LBarrett1 := LGMinusXn shl LPowOfX;
-  end
-  else
-  begin
-    LBarrett1 := LGMinusXn shr 1;
-  end;
 
   if AReflected then
   begin
+    // LSB-first: m = 63.
+    // Barrett[0] = bitreflect(floor(x^(63+n) / G), 64)
+    // Barrett[1] = bitreflect(G, n+1)  (truncated: for n=64, x^0 removed)
+    LDiv128.Lo := 0;
+    LDiv128.Hi := 0;
+    if (63 + ABits) < 64 then
+      LDiv128.Lo := UInt64(1) shl (63 + ABits)
+    else if (63 + ABits) = 64 then
+      LDiv128.Hi := 1
+    else
+      LDiv128.Hi := UInt64(1) shl ((63 + ABits) - 64);
+    LBarrett0 := DivPoly(LDiv128, APoly, ABits);
+
     AConstants.Barrett[0] := BitReverse(LBarrett0, 64);
-    AConstants.Barrett[1] := BitReverse(LBarrett1, 64);
+    if ABits < 64 then
+    begin
+      LPowOfX := 64 - ABits - 1;
+      AConstants.Barrett[1] := BitReverse(LGMinusXn shl LPowOfX, 64);
+    end
+    else
+      AConstants.Barrett[1] := BitReverse(LGMinusXn shr 1, 64);
   end
   else
   begin
-    AConstants.Barrett[0] := LBarrett1;
-    AConstants.Barrett[1] := LBarrett0;
+    // MSB-first: m = 64.  Two-round Barrett per Linux kernel.
+    // Barrett[0] = floor(x^(64+n) / G) - x^64  (mu, low 64 bits)
+    //   Since floor(x^(64+n)/G) = x^64 + floor(APoly*x^64 / G),
+    //   we compute Barrett[0] = DivPoly(APoly*x^64, G) to avoid the x^64 overflow.
+    // Barrett[1] = G  (full generator polynomial; for n=64, x^64 term implicit)
+    LDiv128.Hi := APoly;
+    LDiv128.Lo := 0;
+    AConstants.Barrett[0] := DivPoly(LDiv128, APoly, ABits);
+    if ABits < 64 then
+      AConstants.Barrett[1] := (UInt64(1) shl ABits) or APoly
+    else
+      AConstants.Barrett[1] := APoly;
   end;
+
+  // --- Byte-swap mask for MSB-first CRCs ---
+  if AReflected then
+    FillChar(AConstants.BswapMask[0], 16, 0)
+  else
+    for I := 0 to 15 do
+      AConstants.BswapMask[I] := Byte(15 - I);
+
+  // --- Metadata ---
+  AConstants.CrcBits := UInt64(ABits);
+  if ABits < 64 then
+    AConstants.BarrettShift := UInt64(64 - ABits)
+  else
+    AConstants.BarrettShift := 0;
 end;
 
 end.

HashLib/src/Include/Simd/CRC/CRCFoldPclmul.inc

@@ -1,5 +1,5 @@
 // SSE2 + PCLMULQDQ CRC folding + Barrett reduction for reflected (LSB-first)
-// CRCs with width 8..32 (CRC-64 requires a separate path).
+// CRCs with width 8..64. CRC-64 uses a special Barrett path for the 65-bit G.
 // PCLMULQDQ instructions are db-encoded for broad assembler compatibility.
 //
 // Function signature (included after SimdProc4Begin.inc):
@@ -14,6 +14,8 @@
 //     [16..31] = Fold_1x128
 //     [32..47] = Barrett
 //     [48..63] = Fold_8x128 (unused by this path)
+//     [64..79] = BswapMask  (unused by this path)
+//     [80..87] = CrcBits
 //   Returns: final CRC in RAX.
 //
 // Reference: Linux kernel crc-pclmul-template.S by Eric Biggers (Google).
@@ -143,19 +145,36 @@
   // Step 2: Barrett reduction.
   // Load Barrett constants into xmm6:
   //   xmm6.lo = bitreflect(floor(x^(63+n)/G), 64)
-  //   xmm6.hi = bitreflect((G - x^n) * x^(64-n-1), 64)
+  //   xmm6.hi:
+  //     n < 64: bitreflect((G - x^n) * x^(64-n-1), 64)
+  //     n = 64: bitreflect(((G - x^n) - x^0) / x, 64)
   movdqu xmm6, dqword ptr [r9 + 32]
 
-  // t1 = clmul(t0.lo, barrett[0])
+  // t1 = clmul(t0.lo, barrett[0])  =>  q = floor(t0 / G)
   movdqa xmm1, xmm0
   db $66, $0F, $3A, $44, $CE, $00              // pclmulqdq xmm1, xmm6, $00
 
-  // t2 = clmul(t1.lo, barrett[1]) = G * floor(t0 / G)
+  // Check if CRC-64: G is 65-bit, needs split multiplication
+  cmp qword ptr [r9 + 80], 64
+  je @barrett_64
+
+  // Standard Barrett (width < 64): single pclmulqdq covers full G
   db $66, $0F, $3A, $44, $CE, $10              // pclmulqdq xmm1, xmm6, $10
+  pxor xmm0, xmm1
+  jmp @extract_crc
 
-  // xmm0 = t0 XOR t2, CRC is in bits 64..(64+n-1)
+@barrett_64:
+  // CRC-64 Barrett: G is 65-bit, split into 64-bit pclmulqdq + x^0 correction.
+  // Barrett[1] stores ((G - x^64) - x^0) / x; the x^0 term is handled
+  // separately by duplicating q.lo and XORing it into the result.
+  movdqa xmm2, xmm1                           // save q
+  db $66, $0F, $3A, $44, $CE, $10              // pclmulqdq xmm1, xmm6, $10
+  // punpcklqdq xmm2, xmm2: 66 0F 6C D2
+  db $66, $0F, $6C, $D2                        // xmm2 = {q.lo, q.lo}
   pxor xmm0, xmm1
+  pxor xmm0, xmm2
 
+@extract_crc:
   // Extract high qword (bits 64..127) into rax.
   // pextrq rax, xmm0, 1: 66 48 0F 3A 16 C0 01
   db $66, $48, $0F, $3A, $16, $C0, $01