Merged PR 11273883: Implement ML-KEM

## Description:

+ Adds ML-KEM API surface
+ Implements the API with initial C implementation, with sprinkling of SSE/NEON for (I)NTT
+ Adds low level ML-KEM polynomial arithmetic testing which tests self-consistency and exercises internal assertion in debug builds
+ Adds multi-implementation functionality testing to enable comparative functionality and performance testing on E2E functionality
  + For now, comparison testing is just between SymCrypt static and dynamic, with 3rd party comparison with libcrux disabled until they publish a final ML-KEM implementation
+ Adds ML-KEM KATs from NIST

Related work items: #50913735

Author: samuel-lee-msft

.config/CredScanSuppressions.json

@@ -0,0 +1,16 @@
+{
+  "tool": "Credential Scanner",
+  "suppressions": [
+    {
+      "file": [
+        "3rdparty/hacl-packages/rust/tests/wycheproof/ecdh_secp256r1_ecpoint_test.json",
+        "3rdparty/hacl-packages/rust/tests/wycheproof/x25519_test.json",
+        "3rdparty/hacl-packages/tests/k256_ecdh/ecdh_secp256k1_test.json",
+        "3rdparty/hacl-packages/tests/kyber/kyber768_nistkats.json",
+        "3rdparty/hacl-packages/tests/p256_ecdh/ecdh_secp256r1_ecpoint_test.json",
+        "3rdparty/hacl-packages/tests/x25519/x25519_test.json"
+      ], 
+      "_justification": "Known answer tests included in hacl-packages submodule are incorrectly identified as secrets"
+    }
+  ]
+}

.gitmodules

@@ -6,3 +6,9 @@
 	path = 3rdparty/jitterentropy-library
 	url = https://github.com/smuellerDD/jitterentropy-library
 	ignore = untracked
+
+# To-be updated with final ML-KEM in hacl-packages
+# [submodule "libcrux"]
+# 	path = 3rdparty/hacl-packages
+# 	url = https://github.com/cryspen/hacl-packages
+# 	ignore = untracked

.pipelines/OneBranch.PullRequest.yml

@@ -37,6 +37,7 @@ extends:
         break: true # always break the build on policheck issues.
       binskim:
         enabled: false # Temporarily disable binskim until we sort out BA2018 errors
+      credscanSuppressionsPath: $(Build.SourcesDirectory)/.config/CredScanSuppressions.json
 
     stages:
     - stage: Set_Version
@@ -74,6 +75,7 @@ extends:
           arch: 'AMD64'
           config: 'Release'
           additionalArgs: '--test-legacy-impl'
+          libcrux: true
       - template: .pipelines/templates/build-windows-cmake.yml@self
         parameters:
           arch: 'ARM64'
@@ -88,6 +90,7 @@ extends:
           skipTests: true
           additionalArgs: '--no-asm'
           identifier: 'NoAsm'
+          libcrux: true
       - template: .pipelines/templates/build-windows-cmake.yml@self
         parameters:
           arch: 'X86'
@@ -98,6 +101,7 @@ extends:
           arch: 'X86'
           config: 'Release'
           additionalArgs: '--test-legacy-impl'
+          libcrux: true
       - template: .pipelines/templates/build-windows-cmake.yml@self
         parameters:
           arch: 'AMD64'
@@ -137,6 +141,7 @@ extends:
           cc: 'gcc'
           cxx: 'g++'
           openssl: true
+          libcrux: true
       - template: .pipelines/templates/build-linux.yml@self
         parameters:
           arch: 'AMD64'
@@ -158,6 +163,7 @@ extends:
           cc: 'clang'
           cxx: 'clang++'
           openssl: true
+          libcrux: true
       - template: .pipelines/templates/build-linux.yml@self
         parameters:
           arch: 'AMD64'
@@ -210,6 +216,7 @@ extends:
           cc: 'clang'
           cxx: 'clang++'
           additionalArgs: '--toolchain=cmake-configs/Toolchain-Clang-ARM64.cmake'
+          libcrux: true
       - template: .pipelines/templates/build-linux.yml@self
         parameters:
           arch: 'ARM'
@@ -224,6 +231,7 @@ extends:
           cc: 'gcc'
           cxx: 'g++'
           additionalArgs: '--toolchain=cmake-configs/Toolchain-GCC-ARM.cmake'
+          libcrux: true
 
     - stage: Build_macOS
       displayName: Build macOS
@@ -249,4 +257,4 @@ extends:
           arch: 'ARM64'
           config: 'Release'
           additionalArgs: '--no-asm'
-          skipTests: true
+          skipTests: true

.pipelines/OneBranch.WindowsUndocked.PullRequest.yml

@@ -37,6 +37,7 @@ extends:
     globalSdl:
       tsa:
         enabled: false # Disable TSA to force build breaks
+      credscanSuppressionsPath: $(Build.SourcesDirectory)/.config/CredScanSuppressions.json
     featureFlags:
       EnableCDPxPAT: false
       WindowsHostVersion: '1ESWindows2022'

.pipelines/templates/build-linux.yml

@@ -31,7 +31,10 @@ parameters:
 - name: identifier # Additional identifier for job name
   type: string
   default: ''
-- name: openssl # Build with OpenSSL
+- name: openssl # Build with OpenSSL testing
+  type: boolean
+  default: false
+- name: libcrux # Build with libcrux testing
   type: boolean
   default: false
 
@@ -53,6 +56,10 @@ jobs:
       openssl_build_flag: '--openssl-build-from-source'
     ${{ else }}:
       openssl_build_flag: ''
+    ${{ if eq(parameters.libcrux, true) }}:
+      libcrux_build_flag: '--libcrux'
+    ${{ else }}:
+      libcrux_build_flag: ''
 
   steps:
     - script: |
@@ -107,7 +114,7 @@ jobs:
       inputs:
         scriptSource: 'filePath'
         scriptPath: scripts/build.py
-        arguments: 'cmake bin --arch ${{ parameters.arch }} --config ${{ parameters.config }} --cc ${{ parameters.cc }} --cxx ${{ parameters.cxx }} ${{ parameters.additionalArgs }} $(verbose_build_flag) $(openssl_build_flag)'
+        arguments: 'cmake bin --arch ${{ parameters.arch }} --config ${{ parameters.config }} --cc ${{ parameters.cc }} --cxx ${{ parameters.cxx }} ${{ parameters.additionalArgs }} $(verbose_build_flag) $(openssl_build_flag) $(libcrux_build_flag)'
         workingDirectory: $(Build.SourcesDirectory)
 
     # Overwrite default artifact publishing with our copy (enables publishing binaries for failed runs)

.pipelines/templates/build-windows-cmake.yml

@@ -23,6 +23,9 @@ parameters:
 - name: sign # Sign user-mode binaries. Required by Guardian, even though we don't publish Windows binaries from CMake
   type: boolean
   default: false
+- name: libcrux # Build with libcrux testing
+  type: boolean
+  default: false
 
 jobs:
 
@@ -39,6 +42,10 @@ jobs:
       verbose_build_flag: '--verbose'
     ${{ else }}:
       verbose_build_flag: ''
+    ${{ if eq(parameters.libcrux, true) }}:
+      libcrux_build_flag: '--libcrux'
+    ${{ else }}:
+      libcrux_build_flag: ''
 
   steps:
 
@@ -47,7 +54,7 @@ jobs:
     inputs:
       scriptSource: 'filePath'
       scriptPath: scripts\build.py
-      arguments: 'cmake bin --arch ${{ parameters.arch }} --config ${{ parameters.config }} ${{ parameters.additionalArgs }} $(verbose_build_flag)'
+      arguments: 'cmake bin --arch ${{ parameters.arch }} --config ${{ parameters.config }} ${{ parameters.additionalArgs }} $(verbose_build_flag) $(libcrux_build_flag)'
       workingDirectory: $(Build.SourcesDirectory)
 
   # Overwrite default artifact publishing with our copy (enables publishing binaries for failed runs)