diff --git a/.github/workflows/ct-tooling.yml b/.github/workflows/ct-tooling.yml new file mode 100644 index 0000000000..61e98ed431 --- /dev/null +++ b/.github/workflows/ct-tooling.yml @@ -0,0 +1,125 @@ +name: ct-tooling + +permissions: + contents: read + +on: + workflow_dispatch: + inputs: + algorithms: + description: 'Algorithms to execute CT testing on (comma-separated)' + required: true + type: string + default: 'ML-KEM-512,ML-KEM-768,ML-KEM-1024' + + workflow_call: + inputs: + algorithms: + required: true + type: string + +jobs: + resolve-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.algs.outputs.matrix }} + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4 + - run: python -m pip install --require-hashes -r .github/workflows/requirements.txt + - name: Resolve algorithm list + id: algs + env: + ALGORITHMS_INPUT: ${{ inputs.algorithms }} + run: | + python3 - <<'PY' + import os, sys, json + sys.path.insert(0, 'tests') + import helpers + requested = os.environ["ALGORITHMS_INPUT"].strip() + all_algs = helpers.available_kems_by_name() + helpers.available_sigs_by_name() + requested_set = { a.strip() for a in requested.split(',')} + invalid_algorithms = requested_set - set(all_algs) + if invalid_algorithms: + print(f"Invalid algorithms: {invalid_algorithms}", file=sys.stderr) + sys.exit(1) + selected_algorithms = [a for a in all_algs if a in requested_set] + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f"matrix={json.dumps(selected_algorithms)}\n") + PY + + valgrind-varlat: + needs: [resolve-matrix] + runs-on: ubuntu-latest + container: + image: openquantumsafe/ci-ubuntu-latest:latest + strategy: + matrix: + algorithm: ${{ fromJson(needs.resolve-matrix.outputs.matrix) }} + compiler: [gcc, clang] + liboqs_build: [generic, auto] + opt_flag: [-O0, -O1, -O2, -O3, -Os, -Ofast, "-O2 -fno-tree-vectorize", "-O3 -fno-tree-vectorize", "-O2 -fno-vectorize", "-O3 -fno-vectorize"] + exclude: + - compiler: clang + opt_flag: "-O2 -fno-tree-vectorize" + - compiler: clang + opt_flag: "-O3 -fno-tree-vectorize" + - compiler: gcc + opt_flag: "-O2 -fno-vectorize" + - compiler: gcc + opt_flag: "-O3 -fno-vectorize" + max-parallel: 5 + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4 + + - name: Run valgrind_varlat tests + shell: bash + env: + OPT_FLAG: ${{ matrix.opt_flag }} + ALGORITHM: ${{ matrix.algorithm }} + run: | + set -eu -o pipefail + cd "$GITHUB_WORKSPACE/tests/ct_tooling" + chmod +x ct_test.sh + ./ct_test.sh valgrind-varlat ${{ matrix.compiler }} ${{ matrix.liboqs_build }} "$OPT_FLAG" "$ALGORITHM" + + - name: Upload valgrind_varlat logs + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # pin@v4 + with: + name: valgrind_varlat_${{ matrix.compiler }}_${{ matrix.liboqs_build }}_${{ matrix.opt_flag }}_logs + path: | + tests/ct_tooling/tools/valgrind_varlat/logs/** + + memsan: + needs: [resolve-matrix] + runs-on: ubuntu-latest + container: + image: openquantumsafe/ci-ubuntu-latest:latest + strategy: + matrix: + algorithm: ${{ fromJson(needs.resolve-matrix.outputs.matrix) }} + compiler: [clang] + liboqs_build: [generic, auto] + opt_flag: [-O1, -O2, -O3, -Os, -Ofast, "-O2 -fno-vectorize", "-O3 -fno-vectorize"] + max-parallel: 5 + steps: + - name: Checkout code + uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4 + + - name: Run memsan tests + shell: bash + env: + OPT_FLAG: ${{ matrix.opt_flag }} + ALGORITHM: ${{ matrix.algorithm }} + run: | + set -eu -o pipefail + cd "$GITHUB_WORKSPACE/tests/ct_tooling" + chmod +x ct_test.sh + ./ct_test.sh memsan ${{ matrix.compiler }} ${{ matrix.liboqs_build }} "$OPT_FLAG" "$ALGORITHM" + + - name: Upload memsan logs + uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # pin@v4 + with: + name: memsan_${{ matrix.compiler }}_${{matrix.liboqs_build}}_${{ matrix.opt_flag }}_logs + path: | + tests/ct_tooling/tools/memsan/logs/** \ No newline at end of file diff --git a/.github/workflows/weekly.yml b/.github/workflows/weekly.yml index 1d6a5993b3..a073be59cc 100644 --- a/.github/workflows/weekly.yml +++ b/.github/workflows/weekly.yml @@ -30,4 +30,11 @@ jobs: sig-continuous-benchmarking: uses: ./.github/workflows/sig-bench.yml permissions: - contents: write \ No newline at end of file + contents: write + + ct-tooling: + uses: ./.github/workflows/ct-tooling.yml + permissions: + contents: read + with: + algorithms: 'ML-KEM-512,ML-KEM-768,ML-KEM-1024' \ No newline at end of file diff --git a/.gitignore b/.gitignore index c8faac9f0f..8aab8ad468 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ tags # CMake & testing /build* /tmp* +/CMakeFiles/ # MSVC .vs @@ -38,4 +39,4 @@ __pycache__ compile_commands.json # Generated by Nix flake -result/ +result/ \ No newline at end of file diff --git a/CONFIGURE.md b/CONFIGURE.md index 753392bd87..4d8a89cd7d 100644 --- a/CONFIGURE.md +++ b/CONFIGURE.md @@ -19,7 +19,8 @@ The following options can be passed to CMake before the build file generation pr - [OQS_SPEED_USE_ARM_PMU](#OQS_SPEED_USE_ARM_PMU) - [USE_COVERAGE](#USE_COVERAGE) - [USE_SANITIZER](#USE_SANITIZER) -- [OQS_ENABLE_TEST_CONSTANT_TIME](#OQS_ENABLE_TEST_CONSTANT_TIME) +- [OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND](#OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) +- [OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN](#OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) - [OQS_STRICT_WARNINGS](#OQS_STRICT_WARNINGS) - [OQS_EMBEDDED_BUILD](#OQS_EMBEDDED_BUILD) - [OQS_MEMOPT_BUILD](#OQS_MEMOPT_BUILD) @@ -238,16 +239,20 @@ This has an effect when the compiler is Clang and when [CMAKE_BUILD_TYPE](#CMAKE **Default**: Unset. -## OQS_ENABLE_TEST_CONSTANT_TIME +## OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND -This is used in conjunction with `tests/test_constant_time.py` to use Valgrind to look for instances of secret-dependent control flow. liboqs must also be compiled with [CMAKE_BUILD_TYPE](#CMAKE_BUILD_TYPE) set to `Debug`. - -See the documentation in [`tests/test_constant_time.py`](https://github.com/open-quantum-safe/liboqs/blob/main/tests/test_constant_time.py) for more usage information. +This is used in conjunction with `tests/test_constant_time.py` to use Valgrind to look for instances of secret-dependent control flow. liboqs must also be compiled with [CMAKE_BUILD_TYPE](#CMAKE_BUILD_TYPE) set to `Debug`. When this option is set to `ON`, the additional option `OQS_ENABLE_TEST_CONSTANT_TIME_OPTIMIZED` is made available to control whether liboqs is built using `-O3` optimization, as in a release build, or using the default "Debug" profile. By default, `OQS_ENABLE_TEST_CONSTANT_TIME_OPTIMIZED` is `OFF`. **Default**: `OFF`. +## OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN + +Similar to [`OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND`](#OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND), this option enables constant-time testing using Clang's MemorySanitizer. + +**Default**: `OFF`. + ## OQS_STRICT_WARNINGS Can be `ON` or `OFF`. When `ON`, all compiler warnings are enabled and treated as errors. This setting is recommended to be enabled prior to submission of a Pull Request as CI runs with this setting active. When `OFF`, significantly fewer compiler warnings are enabled such as to avoid undue build errors triggered by (future) compiler warning features/unknown at the development time of this library. diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e777f75321..cea7b8fdcc 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,6 +1,7 @@ # SPDX-License-Identifier: MIT -option(OQS_ENABLE_TEST_CONSTANT_TIME "Build test suite with support for Valgrind-based detection of non-constant time behaviour." OFF) +option(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND "Build test suite with support for Valgrind-based detection of non-constant time behaviour." OFF) +option(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN "Build test suite with support for MemorySanitizer-based detection of non-constant time behaviour." OFF) if(CMAKE_C_COMPILER_ID STREQUAL "GNU" OR CMAKE_C_COMPILER_ID MATCHES "Clang") @@ -107,6 +108,7 @@ target_link_libraries(speed_kem PRIVATE ${TEST_DEPS}) set(KEM_TESTS example_kem kat_kem test_kem test_kem_mem speed_kem vectors_kem) + # SIG API tests add_executable(example_sig example_sig.c) target_link_libraries(example_sig PRIVATE ${TEST_DEPS}) @@ -225,9 +227,15 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Windows" AND BUILD_SHARED_LIBS) endif() # Enable Valgrind-based timing side-channel analysis for test_kem and test_sig -if(OQS_ENABLE_TEST_CONSTANT_TIME AND NOT OQS_DEBUG_BUILD) - message(WARNING "OQS_ENABLE_TEST_CONSTANT_TIME is incompatible with CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}.") - set(OQS_ENABLE_TEST_CONSTANT_TIME OFF) +if(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND AND NOT OQS_DEBUG_BUILD) + message(WARNING "OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND is incompatible with CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}.") + set(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND OFF) +endif() + +# Enable MemSan-based timing side-channel analysis for test_kem and test_sig +if(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN AND NOT OQS_DEBUG_BUILD) + message(WARNING "OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN is incompatible with CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}.") + set(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN OFF) endif() # Record compile options -- from target speed_kem - don't set any options only for speed_kem! diff --git a/tests/constant_time/kem/issues.json b/tests/constant_time/kem/issues.json deleted file mode 100644 index 038b87877e..0000000000 --- a/tests/constant_time/kem/issues.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "BIKE-L1": [], - "BIKE-L3": [], - "BIKE-L5": [], - "Classic-McEliece-348864": ["classic-mceliece-348864"], - "Classic-McEliece-348864f": ["classic-mceliece-348864f"], - "Classic-McEliece-460896": ["classic-mceliece-460896"], - "Classic-McEliece-460896f": ["classic-mceliece-460896f"], - "Classic-McEliece-6688128": ["classic-mceliece-6688128"], - "Classic-McEliece-6688128f": ["classic-mceliece-6688128f"], - "Classic-McEliece-6960119": ["classic-mceliece-6960119"], - "Classic-McEliece-6960119f": ["classic-mceliece-6960119f"], - "Classic-McEliece-8192128": ["classic-mceliece-8192128"], - "Classic-McEliece-8192128f": ["classic-mceliece-8192128f"], - "FrodoKEM-1344-AES": [], - "FrodoKEM-1344-SHAKE": [], - "FrodoKEM-640-AES": [], - "FrodoKEM-640-SHAKE": [], - "FrodoKEM-976-AES": [], - "FrodoKEM-976-SHAKE": [], - "HQC-128": [], - "HQC-192": [], - "HQC-256": [], - "Kyber1024": [], - "Kyber512": [], - "Kyber768": [], - "ML-KEM-512": [], - "ML-KEM-768": [], - "ML-KEM-1024": [], - "NTRU-HPS-2048-509": [], - "NTRU-HPS-2048-677": [], - "NTRU-HPS-4096-821": [], - "NTRU-HRSS-701": [], - "sntrup761": [] -} diff --git a/tests/constant_time/kem/passes.json b/tests/constant_time/kem/passes.json deleted file mode 100644 index 3b1dcd18ef..0000000000 --- a/tests/constant_time/kem/passes.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "BIKE-L1": ["bike"], - "BIKE-L3": ["bike"], - "BIKE-L5": ["bike"], - "Classic-McEliece-348864": [], - "Classic-McEliece-348864f": [], - "Classic-McEliece-460896": [], - "Classic-McEliece-460896f": [], - "Classic-McEliece-6688128": [], - "Classic-McEliece-6688128f": [], - "Classic-McEliece-6960119": [], - "Classic-McEliece-6960119f": [], - "Classic-McEliece-8192128": [], - "Classic-McEliece-8192128f": [], - "FrodoKEM-1344-AES": [], - "FrodoKEM-1344-SHAKE": [], - "FrodoKEM-640-AES": [], - "FrodoKEM-640-SHAKE": [], - "FrodoKEM-976-AES": [], - "FrodoKEM-976-SHAKE": [], - "HQC-128": [], - "HQC-192": [], - "HQC-256": [], - "Kyber1024": ["kyber"], - "Kyber512": ["kyber"], - "Kyber768": ["kyber"], - "ML-KEM-512": [], - "ML-KEM-768": [], - "ML-KEM-1024": [], - "NTRU-HPS-2048-509": [], - "NTRU-HPS-2048-677": [], - "NTRU-HPS-4096-821": [], - "NTRU-HRSS-701": [], - "sntrup761": ["sntrup"] -} diff --git a/tests/constant_time/kem/passes/kyber b/tests/constant_time/kem/passes/kyber deleted file mode 100644 index 63a11707cb..0000000000 --- a/tests/constant_time/kem/passes/kyber +++ /dev/null @@ -1,35 +0,0 @@ -{ - Rejection sampling to produce public "A" matrix - Memcheck:Cond - fun:rej_uniform - fun:pqcrystals_kyber*_ref_gen_matrix - fun:pqcrystals_kyber*_ref_indcpa_* -} -{ - Rejection sampling to produce public "A" matrix - Memcheck:Cond - ... - fun:pqcrystals_kyber*_avx2_gen_matrix - fun:pqcrystals_kyber*_avx2_indcpa_* -} -{ - Rejection sampling to produce public "A" matrix - Memcheck:Value8 - ... - fun:pqcrystals_kyber*_avx2_gen_matrix - fun:pqcrystals_kyber*_avx2_indcpa_* -} -{ - Rejection sampling to produce public "A" matrix - Memcheck:Value8 - ... - fun:PQCLEAN_KYBER*_AARCH64_gen_matrix - fun:PQCLEAN_KYBER*_AARCH64_indcpa_* -} -{ - Rejection sampling to produce public "A" matrix - Memcheck:Cond - ... - fun:PQCLEAN_KYBER*_AARCH64_gen_matrix - fun:PQCLEAN_KYBER*_AARCH64_indcpa_* -} diff --git a/tests/constant_time/kem/passes/sntrup b/tests/constant_time/kem/passes/sntrup deleted file mode 100644 index f3ee77aed2..0000000000 --- a/tests/constant_time/kem/passes/sntrup +++ /dev/null @@ -1,5 +0,0 @@ -{ - Rejection sampling on non-invertible g - Memcheck:Cond - src:sntrup761.c:1985 # fun:KeyGen -} diff --git a/tests/constant_time/sig/issues.json b/tests/constant_time/sig/issues.json deleted file mode 100644 index 1298c2143d..0000000000 --- a/tests/constant_time/sig/issues.json +++ /dev/null @@ -1,199 +0,0 @@ - -{ - "cross-rsdp-128-balanced": [], - "cross-rsdp-128-fast": [], - "cross-rsdp-128-small": [], - "cross-rsdp-192-balanced": [], - "cross-rsdp-192-fast": [], - "cross-rsdp-192-small": [], - "cross-rsdp-256-balanced": [], - "cross-rsdp-256-fast": [], - "cross-rsdp-256-small": [], - "cross-rsdpg-128-balanced": [], - "cross-rsdpg-128-fast": [], - "cross-rsdpg-128-small": [], - "cross-rsdpg-192-balanced": [], - "cross-rsdpg-192-fast": [], - "cross-rsdpg-192-small": [], - "cross-rsdpg-256-balanced": [], - "cross-rsdpg-256-fast": [], - "cross-rsdpg-256-small": [], - "Falcon-1024": ["falcon"], - "Falcon-512": ["falcon"], - "Falcon-padded-1024": ["falcon"], - "Falcon-padded-512": ["falcon"], - "MAYO_1": [], - "MAYO_2": [], - "MAYO_3": [], - "ML-DSA-44": [], - "ML-DSA-65": [], - "ML-DSA-87": [], - "SNOVA_24_5_4": [], - "SNOVA_24_5_4_SHAKE": [], - "SNOVA_24_5_4_SHAKE_esk": [], - "SNOVA_24_5_4_esk": [], - "SNOVA_24_5_5": [], - "SNOVA_25_8_3": [], - "SNOVA_29_6_5": [], - "SNOVA_37_17_2": [], - "SNOVA_37_8_4": [], - "SNOVA_49_11_3": [], - "SNOVA_56_25_2": [], - "SNOVA_60_10_4": [], - "SLH_DSA_PURE_SHA2_128S": ["slh_dsa"], - "SLH_DSA_PURE_SHA2_128F": ["slh_dsa"], - "SLH_DSA_PURE_SHA2_192S": ["slh_dsa"], - "SLH_DSA_PURE_SHA2_192F": ["slh_dsa"], - "SLH_DSA_PURE_SHA2_256S": ["slh_dsa"], - "SLH_DSA_PURE_SHA2_256F": ["slh_dsa"], - "SLH_DSA_PURE_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_PURE_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_PURE_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_PURE_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_PURE_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_PURE_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHA2_224_PREHASH_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHA2_256_PREHASH_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHA2_384_PREHASH_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHA2_512_PREHASH_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHA2_512_224_PREHASH_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHA2_512_256_PREHASH_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHA3_224_PREHASH_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHA3_256_PREHASH_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHA3_384_PREHASH_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHA3_512_PREHASH_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHAKE_128_PREHASH_SHAKE_256F": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHA2_128S": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHA2_128F": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHA2_192S": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHA2_192F": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHA2_256S": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHA2_256F": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHAKE_128S": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHAKE_128F": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHAKE_192S": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHAKE_192F": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHAKE_256S": ["slh_dsa"], - "SLH_DSA_SHAKE_256_PREHASH_SHAKE_256F": ["slh_dsa"] -} diff --git a/tests/constant_time/sig/passes.json b/tests/constant_time/sig/passes.json deleted file mode 100644 index 9a9099115b..0000000000 --- a/tests/constant_time/sig/passes.json +++ /dev/null @@ -1,56 +0,0 @@ - -{ - "cross-rsdp-128-balanced": ["cross"], - "cross-rsdp-128-fast": ["cross"], - "cross-rsdp-128-small": ["cross"], - "cross-rsdp-192-balanced": ["cross"], - "cross-rsdp-192-fast": ["cross"], - "cross-rsdp-192-small": ["cross"], - "cross-rsdp-256-balanced": ["cross"], - "cross-rsdp-256-fast": ["cross"], - "cross-rsdp-256-small": ["cross"], - "cross-rsdpg-128-balanced": ["cross"], - "cross-rsdpg-128-fast": ["cross"], - "cross-rsdpg-128-small": ["cross"], - "cross-rsdpg-192-balanced": ["cross"], - "cross-rsdpg-192-fast": ["cross"], - "cross-rsdpg-192-small": ["cross"], - "cross-rsdpg-256-balanced": ["cross"], - "cross-rsdpg-256-fast": ["cross"], - "cross-rsdpg-256-small": ["cross"], - "Falcon-1024": ["falcon_keygen", "falcon_sign"], - "Falcon-512": ["falcon_keygen", "falcon_sign"], - "Falcon-padded-1024": ["falcon_keygen", "falcon_sign"], - "Falcon-padded-512": ["falcon_keygen", "falcon_sign"], - "MAYO-1": ["mayo"], - "MAYO-2": ["mayo"], - "MAYO-3": ["mayo"], - "MAYO-5": ["mayo"], - "ML-DSA-44": [], - "ML-DSA-65": [], - "ML-DSA-87": [], - "SNOVA_24_5_4": ["snova"], - "SNOVA_24_5_4_SHAKE": ["snova"], - "SNOVA_24_5_4_SHAKE_esk": ["snova"], - "SNOVA_24_5_4_esk": ["snova"], - "SNOVA_24_5_5": ["snova"], - "SNOVA_25_8_3": ["snova"], - "SNOVA_29_6_5": ["snova"], - "SNOVA_37_17_2": ["snova"], - "SNOVA_37_8_4": ["snova"], - "SNOVA_49_11_3": ["snova"], - "SNOVA_56_25_2": ["snova"], - "SNOVA_60_10_4": ["snova"], - "mqom2_cat1_gf16_fast_r3": ["mqom"], - "mqom2_cat1_gf16_fast_r5": ["mqom"], - "mqom2_cat1_gf16_short_r3": ["mqom"], - "mqom2_cat1_gf16_short_r5": ["mqom"], - "mqom2_cat3_gf16_fast_r3": ["mqom"], - "mqom2_cat3_gf16_fast_r5": ["mqom"], - "mqom2_cat3_gf16_short_r3": ["mqom"], - "mqom2_cat3_gf16_short_r5": ["mqom"], - "mqom2_cat5_gf16_fast_r3": ["mqom"], - "mqom2_cat5_gf16_fast_r5": ["mqom"], - "mqom2_cat5_gf16_short_r3": ["mqom"], - "mqom2_cat5_gf16_short_r5": ["mqom"] -} diff --git a/tests/constant_time/sig/passes/falcon_sign b/tests/constant_time/sig/passes/falcon_sign deleted file mode 100644 index 210c37c071..0000000000 --- a/tests/constant_time/sig/passes/falcon_sign +++ /dev/null @@ -1,79 +0,0 @@ -# Suppressions that work for both CLEAN and AVX2 implementations - -{ - Exception while reading secret key. No signature is produced. - Memcheck:Cond - fun:PQCLEAN_FALCON*_*_trim_i8_decode - fun:do_sign - fun:PQCLEAN_FALCON*_*_crypto_sign_signature -} -{ - Exception while re-computing private basis (non-invertible f). No signature is produced. - Memcheck:Cond - src:vrfy.c:726 # fun:PQCLEAN_FALCON*_*_complete_private - fun:do_sign - fun:PQCLEAN_FALCON*_*_crypto_sign_signature -} -{ - Exception while re-computing private basis (large G coefficients). No signature is produced. - Memcheck:Cond - src:vrfy.c:739 # fun:PQCLEAN_FALCON*_*_complete_private - fun:do_sign - fun:PQCLEAN_FALCON*_*_crypto_sign_signature -} -{ - Rejection sampling on encoded signature size - Memcheck:Cond - fun:PQCLEAN_FALCON*_*_comp_encode - fun:do_sign - fun:PQCLEAN_FALCON*_*_crypto_sign_signature -} - - -# CLEAN implementation of sign.c - -{ - Rejection sampling to re-center discrete Gaussian distribution - Memcheck:Cond - # Wildcard to catch the while loop in fun:BerExp as well as line the if on src:sign.c:1155 - ... - src:sign.c:1155 # fun:PQCLEAN_FALCON*_CLEAN_sampler - ... - fun:PQCLEAN_FALCON*_CLEAN_crypto_sign_signature -} -{ - Rejection sampling on signature length - Memcheck:Cond - src:sign.c:946 # fun:do_sign_dyn - fun:PQCLEAN_FALCON*_CLEAN_sign_dyn - fun:do_sign - fun:PQCLEAN_FALCON*_CLEAN_crypto_sign_signature -} -{ - Rejection sampling on encoded signature size - Memcheck:Cond - fun:PQCLEAN_FALCON*_CLEAN_comp_encode - fun:do_sign - fun:PQCLEAN_FALCON*_CLEAN_crypto_sign_signature -} - - -# AVX2 implementation of sign.c - -{ - Rejection sampling to re-center discrete Gaussian distribution - Memcheck:Cond - # Wildcard to catch the while loop in fun:BerExp as well as line the if on src:sign.c:1153 - ... - src:sign.c:1213 # fun:PQCLEAN_FALCON*_AVX2_sampler - ... - fun:PQCLEAN_FALCON*_AVX2_crypto_sign_signature -} -{ - Rejection sampling on signature length - Memcheck:Cond - src:sign.c:939 # fun:do_sign_dyn - fun:PQCLEAN_FALCON*_AVX2_sign_dyn - fun:do_sign - fun:PQCLEAN_FALCON*_AVX2_crypto_sign_signature -} diff --git a/tests/ct_tooling/README.md b/tests/ct_tooling/README.md new file mode 100644 index 0000000000..1c67956364 --- /dev/null +++ b/tests/ct_tooling/README.md @@ -0,0 +1,169 @@ +# Constant-Time Tooling +Framework for constant-time testing of liboqs across compilers, optimization flags, and `OQS_OPT_TARGET` build modes. + +## Repository Structure +``` +tests/ct_tooling/ +├── README.md +├── ct_test.sh # Unified shell script handling CT test execution for all tools +├── local_testing_example.sh # Example script for running CT tests locally +├── analyze_results.py # Parse log output and generate summary CSVs and graphs +└── tools/ + ├── memsan/ + │ ├── false_positives/ # Directory containing false-positives suppression files + │ ├── issues/ # Directory containing possible constant-time isues detected with Valgrind-Varlat + │ └── README.md + └── valgrind_varlat/ + ├── false_positives/ # Directory containing false-positives suppression files + ├── issues/ # Directory containing possible constant-time isues detected with MemSan + └── README.md +``` + +## Tools +### 1. Valgrind-Varlat (`valgrind_varlat/`) +- **Purpose**: Uninitialized memory error detection for constant-time analysis using Kyberslash patch for Valgrind +- **Output**: Directory containing all unique suppression blocks for each warning output + +### 2. MemSan (`memsan/`) +- **Purpose**: LLVM-based uninitialized memory error detection for constant-time analysis using MemorySanitizer +- **Output**: Unique `SUMMARY: MemorySanitizer` lines for each warning output + +Both tools are driven by the single unified `ct_test.sh` script located at the root of `tests/ct_tooling/` and enforce a warning cap of 100,000 unique warnings per algorithm run. Once the cap is reached, further warnings are suppressed. All SLH-DSA signature variants are currently skipped during SIG tests due to the excessive time they require to execute. The tool used for testing is selected via the first argument: + +```bash +./ct_test.sh +``` + +- `tool`: `valgrind-varlat` or `memsan` +- `compiler_version`: clang, clang-20, gcc, gcc-14, ... +- `liboqs_build`: `generic` or `auto` +- `input`: `all`, `kems`, `sigs`, or a specific enabled algorithm variant (`all` excludes SLH-DSA variants for the aforementioned reason). +- `opt_flags`: All arguments from position 4 up to position N-1 are treated as compiler optimization flags (including multi-flag combinations such as `-O3 -fno-tree-vectorize`). + +Examples: + +```bash +./ct_test.sh valgrind-varlat clang generic -O2 all +./ct_test.sh valgrind-varlat clang-20 auto -O3 -fno-tree-vectorize kems +./ct_test.sh valgrind-varlat gcc-14 generic -O2 -fno-tree-vectorize Kyber768 +./ct_test.sh memsan clang generic -O1 ML-DSA-44 +./ct_test.sh memsan clang-20 auto -O3 all +``` + +The `local_testing_example.sh` script demonstrates how to use `ct_test.sh` to run CT tests locally across a variety of compilers, compiler versions, liboqs target builds, and optimization flags. + +The `ct-tooling-valgrind-varlat.yml` and `ct-tooling-memsan.yml` workflows also use `ct_test.sh` to execute CT tests in CI on user-selected algorithms (using [interactive-inputs](https://github.com/marketplace/actions/interactive-inputs)) when the workflows are manually triggered on GitHub Actions. + +### Configuration +For Valgrind-Varlat configuration, see: [Valgrind-Varlat's README](tools/valgrind_varlat/README.md) +For MemSan configuration, see: [MemSan's README](tools/memsan/README.md) + +## Workflow + +### Running Tests + +Each tool follows the same testing workflow, implemented through two functions in the unified `ct_test.sh` script: + +1. **Building liboqs** +First, the `build()` function builds liboqs with the compiler options desired. In the case of `local_testing_example.sh`, these options are set to: + +- Compiler versions: gcc, gcc-14, clang, and clang-20 +- liboqs build: auto (with optimizations), and generic +- Optimization flags: -O0, -O1, -O2, -O3, -Os, -Ofast, -O2 -fno-tree-vectorize, and -O3 -fno-tree-vectorize + +When `ct_test.sh` is executed on a single algorithm variant, `build()` builds liboqs with the `OQS_MINIMAL_BUILD` option, minimizing the time and resources required for building. For the other input options, liboqs is built entirely. Each build is placed into a dedicated directory at the liboqs root, named using the pattern. + +Note that Valgrind-Varlat tests can be compiled using both gcc and clang, while MemSan is only native to the clang compiler. This leaves the following possible configurations for each of the optimization flags in the case of `local_testing_example.sh`: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Valgrind-VarlatMemSan
gccgcc-14clangclang-20clangclang-20
genericAll opt flagsAll opt flagsAll opt flagsAll opt flagsAll opt flagsAll opt flags
autoAll opt flagsAll opt flagsAll opt flagsAll opt flagsAll opt flagsAll opt flags
+ +Once the script builds each configuration into a build folder, it calls the test execution function (`test()`) on the build folder generated. + +2. **Test execution** +Then, `test()` is tasked with executing the tool's test on selected liboqs algorithms. Each tool has a different process through which it parses the tool's output to keep unique instances of the warnings, which are further detailed in their respective README files: [Valgrind-Varlat's README](tools/valgrind_varlat/README.md) and [MemSan's README](tools/memsan/README.md). + +### Output Structure +The workflow organizes test outputs into log files that capture unique warnings for each algorithm. These logs are written inside `tests/ct_tooling/tools//logs/`, categorized into concrete subdirectories based on the compiler and build configuration (`gcc_14_auto`, `clang_generic`, ...), which then contain further subdivisions by optimization levels (`O0`, `O1`, ...) and algorithm types (`kem` or `sig`). The structure is as follows: + +``` +tests/ct_tooling/tools//logs/ +├── clang_generic/ +│ ├── O0/ +│ │ ├── kem/ +│ │ │ ├── kem_summary.txt # Pass/fail summary with compiler info +│ │ │ ├── _.log # Unique warnings for the algorithm +│ │ └── sig/ +│ │ └── ... +│ ├── O1/ +│ └── ... +├── _/ +│ ├── O0/ +│ └── ... +``` + +The summary file for each run includes the compiler path, compiler version, target architecture, and compilation flags used, followed by a pass/fail line for each algorithm tested. + +### Cathegorizing warnings and false positive handling +This framework does not directly differentiate between false positives and issues directly. It must be manually maintained by contributors, who carefully analyze the framework's output and label them. The label is for auditors. We assign "false positive" to a warning that is known not to be a security threat, and we store its suppression file in the corresponding file and subdirectory. We "raise an issue" about any other error, and we store the corresponding suppression file in the "issues" subdirectory. These can be found in `liboqs/tests/ct_tooling/tools/{valgrind_varlat, memsan}/{false_positives, issues}`. If you are unsure where your suppression file belongs, then save it to the "issues" subdirectory. + +The handling of false positives slightly differs between tools, with both using the output returned by the tool during execution to generate the final suppression files. This process is outlined in the READMEs within each tool's subdirectory. + +### Analyzing Results +Use `analyze_results.py` to parse the warnings data from the log files and generate graphs describing the results. + +```bash +python3 analyze_results.py \ + --tool \ + --input tests/ct_tooling/tools//logs \ + --output results_ +``` + +**Generates**: +- `KEM_warnings_per_opt_level.csv` - Warning counts per algorithm per optimization +- `SIG_warnings_per_opt_level.csv` - Same for signature schemes +- `*.png` - 4 visualization graphs regarding the total and average number of warnings per KEM and signature + +### Credits +The observation that Valgrind can be used to identify non-constant time behaviour is due to Adam Langley [1, 2]. Mortiz Neikes' TIMECOP project applies Langley's idea to the SUPERCOP benchmarking suite [3]. Versions of SUPERCOP starting with 20200816 include TIMECOP and apply Langley's idea to randombytes calls in particular [4]. We have borrowed the idea of instrumenting randombytes calls from SUPERCOP. + +[1] https://github.com/agl/ctgrind +[2] https://boringssl.googlesource.com/boringssl/+/a6a049a6fb51a052347611d41583a0622bc89d60 +[2] https://post-apocalyptic-crypto.org/timecop/index.html +[3] http://bench.cr.yp.to/tips.html#timecop \ No newline at end of file diff --git a/tests/ct_tooling/analyze_results.py b/tests/ct_tooling/analyze_results.py new file mode 100644 index 0000000000..bac27328f4 --- /dev/null +++ b/tests/ct_tooling/analyze_results.py @@ -0,0 +1,258 @@ +# SPDX-License-Identifier: MIT + +#!/usr/bin/env python3 + +""" +Analyze constant-time results across different optimization levels. +Parses summary files to compare warning counts for KEMs and SIGs. +Generates CSV reports and visualization graphs. +""" + +import os +import re +import glob +from collections import defaultdict +import csv +import matplotlib.pyplot as plt +import matplotlib +matplotlib.use('Agg') +import numpy as np +import argparse +import sys + +parser = argparse.ArgumentParser(description='Analyze constant-time test results') +parser.add_argument('--tool', '-t', type=str, help='Constant-time tool used for analysis') +parser.add_argument('--input', '-i', type=str, help='General input directory containing build information') +parser.add_argument('--output', '-o', type=str, default='.', help='Output directory for analysis results') +args = parser.parse_args() + +cwd = os.getcwd() +LOG_BASE_DIR = os.path.abspath(os.path.join(cwd, args.input)) if args.input else cwd +OUTPUT_DIR = os.path.abspath(os.path.join(cwd, args.output)) if args.output else cwd + +def parse_summary_file(filepath): + """ + Pattern to match Valgrind test results: + Example: "Testing KEM: Kyber512 ... FAIL (Valgrind/MemSan warnings)" + " → Found 150 Valgrind warnings" + """ + results = {} + + with open(filepath, 'r') as f: + content = f.read() + + header_re = re.compile(r'^Testing (?:KEM:|SIG:|)\s*(\S+) \.\.\. (?:PASS|FAIL)', re.MULTILINE) + count_re = re.compile(r'Found (\d+) (?:Valgrind|MemSan|)?\s*warnings') + + matches = list(header_re.finditer(content)) + for idx, m in enumerate(matches): + alg_name = m.group(1) + # Skip SLH_DSA variants + if 'SLH_DSA' in alg_name: + continue + + start = m.end() + end = matches[idx+1].start() if idx+1 < len(matches) else len(content) + block = content[start:end] + + c = count_re.search(block) + if c: + error_count = int(c.group(1)) + else: + error_count = 0 + + results[alg_name] = error_count + + return results + +# Modify analyze_logs to dynamically process all builds and optimization levels +def analyze_logs(): + """Walk input log directories and aggregate warning counts dynamically. + """ + kem_data = defaultdict(lambda: defaultdict(dict)) + sig_data = defaultdict(lambda: defaultdict(dict)) + + if not os.path.isdir(LOG_BASE_DIR): + print(f"Error: Input directory '{LOG_BASE_DIR}' does not exist.", file=sys.stderr) + sys.exit(1) + + for build_dir in sorted(glob.glob(os.path.join(LOG_BASE_DIR, '*'))): + build_name = os.path.basename(build_dir) + for opt_dir in sorted(glob.glob(os.path.join(build_dir, '*'))): + opt_level = os.path.basename(opt_dir) + for test_type in ['kem', 'sig']: + test_dir = os.path.join(opt_dir, test_type) + if not os.path.isdir(test_dir): + continue + + for summary_file in glob.glob(os.path.join(test_dir, f'{test_type}_summary*.txt')): + results = parse_summary_file(summary_file) + if test_type == 'kem': + kem_data[build_name][opt_level].update(results) + else: + sig_data[build_name][opt_level].update(results) + + return kem_data, sig_data + +def sort_optimization_levels(opt_levels): + priority = { + 'O0': 0, + 'O1': 1, + 'O2-NOVEC': 2, + 'O2': 3, + 'OS': 4, + 'O3-NOVEC': 5, + 'O3': 6, + 'OFAST': 7 + } + return sorted(opt_levels, key=lambda x: priority.get(x.upper(), 99)) + +def generate_csv_file(data, alg_type, opt_levels, output_dir): + + csv_path = os.path.join(output_dir, f'{alg_type}_warnings_per_opt_level.csv') + with open(csv_path, 'w', newline='') as csvfile: + all_algs = set() + for opt_data in data.values(): + all_algs.update(opt_data.keys()) + all_algs = sorted(all_algs) + + writer = csv.writer(csvfile) + writer.writerow(['Algorithm'] + opt_levels) + + for alg in all_algs: + row = [alg] + for opt in opt_levels: + row.append(data.get(opt, {}).get(alg, 'N/A')) + writer.writerow(row) + +def heatmap_warnings_per_alg_and_opt_level(data, opt_levels, alg_type, tool, output_dir): + + # Sort algorithms by total warnings (descending). For ties, sort alphabetically ascending + def _norm_name(n): + return re.sub(r'[^0-9a-z]', '', n.lower()) + + all_algs = set() + for opt in opt_levels: + all_algs.update(data.get(opt, {}).keys()) + alg_total = {alg: sum([data.get(opt, {}).get(alg, 0) for opt in opt_levels]) for alg in all_algs} + alg_names = [a for a, _ in sorted(alg_total.items(), key=lambda x: (-x[1], _norm_name(x[0])))] + + matrix = np.zeros((len(alg_names), len(opt_levels)), dtype=int) + for i, alg in enumerate(alg_names): + for j, opt in enumerate(opt_levels): + matrix[i, j] = data.get(opt, {}).get(alg, 0) + + CMAP_BY_TYPE = {'KEM': 'Blues', 'SIG': 'Oranges'} + cmap = CMAP_BY_TYPE.get(alg_type.upper(), 'YlOrRd') + + fig_height = max(8, len(alg_names) * 0.25) + fig, ax = plt.subplots(figsize=(12, fig_height)) + + max_val = int(matrix.max()) if matrix.size else 0 + vmax = max(1, max_val) + + raster = True if len(alg_names) > 200 else False + im = ax.imshow(matrix, aspect='auto', cmap=cmap, vmin=0, vmax=vmax, rasterized=raster) + + ax.set_xticks(np.arange(len(opt_levels))) + ax.set_xticklabels(opt_levels) + plt.setp(ax.get_xticklabels(), rotation=45, ha='right', rotation_mode='anchor') + + ax.set_yticks(np.arange(len(alg_names))) + ax.set_yticklabels(alg_names) + + cbar = plt.colorbar(im, ax=ax) + cbar.set_label('Number of Warnings', rotation=270, labelpad=20, fontsize=12) + + for i in range(len(alg_names)): + for j in range(len(opt_levels)): + ax.text(j, i, int(matrix[i, j]), ha='center', va='center', color='black', fontsize=8) + + ax.set_title(f'{tool.upper()} {alg_type.upper()} Warnings per Algorithm by Optimization Level', + fontsize=14, fontweight='bold', pad=20) + ax.set_xlabel('Optimization Level', fontsize=12) + ax.set_ylabel(f'{alg_type.upper()} Algorithms', fontsize=12) + plt.tight_layout() + + graph_path = os.path.join(output_dir, f"{alg_type.upper()}_total_warnings_per_alg_by_opt_level.png") + plt.savefig(graph_path, dpi=300, bbox_inches='tight') + plt.close() + +def bar_chart_total_warnings_per_opt_level(kem_data, sig_data, opt_levels, tool, output_dir): + + fig, ax = plt.subplots(figsize=(14, 6)) + + kem_totals = [sum(kem_data.get(opt, {}).values()) for opt in opt_levels] + sig_totals = [sum(sig_data.get(opt, {}).values()) for opt in opt_levels] + + x = np.arange(len(opt_levels)) + width = 0.35 + ax.bar(x - width/2, kem_totals, width, label='KEM', color='#2b8cbe') + ax.bar(x + width/2, sig_totals, width, label='SIG', color='#f16913') + ax.set_xlabel('Optimization Level', fontsize=12) + ax.set_ylabel('Total Warnings', fontsize=12) + ax.set_title(f'{tool.upper()} Total Warnings by Optimization Level', fontsize=14, fontweight='bold', pad=20) + ax.set_xticks(x) + ax.set_xticklabels(opt_levels, rotation=45, ha='right') + ax.legend() + ax.grid(axis='y', alpha=0.3) + plt.tight_layout() + outpath = os.path.join(output_dir, f'{tool}_total_warnings_kem_vs_sig.png') + plt.savefig(outpath, dpi=300, bbox_inches='tight') + plt.close() + +def line_chart_avg_warnings_per_opt_level(kem_data, sig_data, opt_levels, tool, output_dir): + + kem_avgs = [np.mean(list(kem_data.get(opt, {}).values())) if kem_data.get(opt) else 0 for opt in opt_levels] + sig_avgs = [np.mean(list(sig_data.get(opt, {}).values())) if sig_data.get(opt) else 0 for opt in opt_levels] + + fig, ax = plt.subplots(figsize=(10, 6)) + ax.plot(opt_levels, kem_avgs, marker='o', label='KEM', color='#2b8cbe') + ax.plot(opt_levels, sig_avgs, marker='o', label='SIG', color='#f16913') + ax.set_xlabel('Optimization Level',fontsize=12) + ax.set_ylabel('Average Warnings', fontsize=12) + ax.set_title(f'{tool.upper()} Average Warnings per Optimization Level', fontsize=14, fontweight='bold', pad=20) + ax.legend() + ax.grid(True, alpha=0.3) + plt.tight_layout() + outpath = os.path.join(output_dir, f'{tool}_avg_warnings_kem_vs_sig.png') + plt.savefig(outpath, dpi=300, bbox_inches='tight') + plt.close() + +def generate_reports(kem_data, sig_data): + all_builds = set(kem_data.keys()).union(sig_data.keys()) + for build_name in all_builds: + build_output_dir = os.path.join(OUTPUT_DIR, build_name) + os.makedirs(build_output_dir, exist_ok=True) + + opt_levels = set(kem_data.get(build_name, {}).keys()).union(sig_data.get(build_name, {}).keys()) + opt_levels = sort_optimization_levels(opt_levels) + + if build_name in kem_data and kem_data[build_name]: + generate_csv_file(kem_data[build_name], 'KEM', opt_levels, build_output_dir) + heatmap_warnings_per_alg_and_opt_level(kem_data[build_name], opt_levels, 'KEM', args.tool, build_output_dir) + if build_name in sig_data and sig_data[build_name]: + generate_csv_file(sig_data[build_name], 'SIG', opt_levels, build_output_dir) + heatmap_warnings_per_alg_and_opt_level(sig_data[build_name], opt_levels, 'SIG', args.tool, build_output_dir) + + bar_chart_total_warnings_per_opt_level( + kem_data.get(build_name, {}), + sig_data.get(build_name, {}), + opt_levels, + args.tool, + build_output_dir + ) + line_chart_avg_warnings_per_opt_level( + kem_data.get(build_name, {}), + sig_data.get(build_name, {}), + opt_levels, + args.tool, + build_output_dir + ) + +def main(): + kem_data, sig_data = analyze_logs() + generate_reports(kem_data, sig_data) + +if __name__ == "__main__": + main() diff --git a/tests/ct_tooling/ct_test.sh b/tests/ct_tooling/ct_test.sh new file mode 100755 index 0000000000..8355085a98 --- /dev/null +++ b/tests/ct_tooling/ct_test.sh @@ -0,0 +1,404 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LIBOQS_DIR="$(realpath "$SCRIPT_DIR/../..")" + +notify() { + printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$*" +} + +build() { + local TOOL=$1 + local COMP_V=$2 + local LIBOQS_BUILD=$3 + local OPT_FLAG=$4 + local BUILD_DIR=$5 + local INPUT=$6 + + # Remove build dir if OQS_MINIMAL_BUILD or input changed + if [ -f "$BUILD_DIR/CMakeCache.txt" ]; then + grep -q "OQS_MINIMAL_BUILD" "$BUILD_DIR/CMakeCache.txt" && rm -rf "$BUILD_DIR" + fi + + # Handle a minimal liboqs build only if a single algorithm is passed as input, otherwise build the complete library + local MINIMAL_BUILD_ARG=() + if [[ -n "$INPUT" && "$INPUT" != "all" && "$INPUT" != "kems" && "$INPUT" != "sigs" ]]; then + MINIMAL_BUILD_ARG=(-DOQS_MINIMAL_BUILD="$INPUT") + fi + + mkdir -p "$BUILD_DIR" + cd "$BUILD_DIR" + local CMAKE_ARGS=("-S .." "-G Ninja" "${MINIMAL_BUILD_ARG[@]}" "-DCMAKE_C_COMPILER=$COMP_V" "-DOQS_OPT_TARGET=$LIBOQS_BUILD" "-DCMAKE_BUILD_TYPE=Debug" "-DOQS_USE_OPENSSL=OFF" "-DOQS_DIST_BUILD=OFF") + + case "$TOOL" in + valgrind-varlat) + + cmake "${CMAKE_ARGS[@]}" \ + -DCMAKE_C_FLAGS="$OPT_FLAG" \ + -DOQS_ENABLE_TEST_CONSTANT_TIME=ON > /dev/null + cmake --build . -j$(nproc) > /dev/null + ;; + memsan) + + # Generate suppression flags for all suppression files containing false positives + SUP_DIR="$SCRIPT_DIR/tools/memsan/false_positives" + SUP_FLAGS=() + for f in "$SUP_DIR"/*; do + [ -f "$f" ] || continue + SUP_FLAGS+=( "-fsanitize-ignorelist=$f" ) + done + + cmake "${CMAKE_ARGS[@]}" \ + -DBUILD_SHARED_LIBS=ON \ + -DOQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN=ON \ + -DCMAKE_C_FLAGS="-fsanitize=memory -fsanitize-recover=all ${SUP_FLAGS[*]} $OPT_FLAG -g" \ + -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=memory" \ + -DCMAKE_SHARED_LINKER_FLAGS="-fsanitize=memory" > /dev/null + cmake --build . -j$(nproc) > /dev/null + ;; + *) + echo "Unknown tool: $TOOL"; return 1 + ;; + esac +} + +test() { + local TOOL=$1 + local BUILD_DIR=$2 + local TEST_TYPE=$3 + local COMP_V="${4//-/_}" + local TARGET=$5 + local ALGORITHM=$6 + local SCRIPT_DIR=$7 + + if [[ "$TEST_TYPE" == "kem" ]]; then + TEST_BINARY="test_kem" + UPPER_TYPE="KEM" + elif [[ "$TEST_TYPE" == "sig" ]]; then + TEST_BINARY="test_sig" + UPPER_TYPE="SIG" + fi + + # Skip SLH-DSA for SIG tests + if [[ "$ALGORITHM" == *SLH_DSA* ]]; then + echo "Skipping ${UPPER_TYPE} ${ALGORITHM}" + return 0 + fi + + LOG_DIR="${SCRIPT_DIR}/tools/${TOOL//-/_}/logs/${COMP_V}_${TARGET}" + mkdir -p "$LOG_DIR" + CURRENT_RUN_DIR="$LOG_DIR/$SANITIZED_OPT_FLAG" + mkdir -p "$CURRENT_RUN_DIR" + OUTPUT_DIR="$CURRENT_RUN_DIR/$TEST_TYPE" + mkdir -p "$OUTPUT_DIR" + SUMMARY_FILE="$OUTPUT_DIR/${TEST_TYPE}_summary.txt" + + COMPILER_PATH=$(grep -E '^CMAKE_C_COMPILER:.*=' "$BUILD_DIR"/CMakeCache.txt | head -n1 | cut -d'=' -f2- | tr -d '\r') + COMPILER_VERSION=$("$COMPILER_PATH" --version 2>&1 | head -n1) + ARCH="$(uname -m)" + COMP_FLAGS=$(grep "CMAKE_C_FLAGS:" "$BUILD_DIR/CMakeCache.txt" | cut -d'=' -f2-) + MAX_WARNINGS=100000 + + # Check if this is first algorithm (header only once) + if [[ ! -s "$SUMMARY_FILE" ]]; then + { + echo "========================================" + echo "Compiled with: $COMP_FLAGS" + echo "Compiler version: ${COMPILER_VERSION}" + echo "Architecture: ${ARCH}" + echo "========================================" + echo "" + } > "$SUMMARY_FILE" + else + echo "" >> "$SUMMARY_FILE" + fi + + PASS_COUNT=0 + FAIL_COUNT=0 + LOG_FILE="$OUTPUT_DIR/${ALGORITHM}_${TIMESTAMP}.log" + echo -n "Testing $ALGORITHM ... " | tee -a "$SUMMARY_FILE" + + cd "$LIBOQS_DIR" + + # Execute CT tests based on the tool selected + case "$TOOL" in + valgrind-varlat) + # Generate suppression flags for all suppression files containing false positives + SUP_DIR="$SCRIPT_DIR/tools/valgrind_varlat/false_positives" + SUP_FLAGS=() + for f in "$SUP_DIR"/*.txt; do + [ -f "$f" ] || continue + SUP_FLAGS+=( "--suppressions=$f" ) + done + + VALGRIND_OPTS=( + valgrind_varlat + --tool=memcheck + --gen-suppressions=all + "${SUP_FLAGS[@]}" # Include all suppression files + --error-exitcode=123 + --max-stackframe=20480000 + --num-callers=20 + --variable-latency-errors=yes # Enable the KyberSlash patch + ) + + : > "$LOG_FILE" ; : > "$LOG_FILE.hashes"; : > "$LOG_FILE.count" + + "${VALGRIND_OPTS[@]}" "$BUILD_DIR"/tests/$TEST_BINARY "$ALGORITHM" 2>&1 | awk \ + -v log_file="$LOG_FILE" \ + -v tmp_file="$LOG_FILE.tmp" \ + -v hash_file="$LOG_FILE.hashes" \ + -v count_file="$LOG_FILE.count" \ + -v max_warnings="$MAX_WARNINGS" ' + # Extract unique suppression blocks from Valgrind output + BEGIN { + unique_warnings_count = 0; + in_block = 0; # Whether we are inside a { ... } block + block = ""; # Current block content (including braces) + suppress = 0; # reached max_warnings + + # Preload known hashes if present + while ((getline line < hash_file) > 0) { + gsub(/\r$/, "", line); + if (length(line) > 0) { + seen[line] = 1; + } + } + close(hash_file); + } + + { + if (suppress) { + if (in_block) { + if ($0 ~ /^\}$/) { in_block = 0 } + } else if ($0 ~ /^\{$/) { + in_block = 1 + } + next + } + + if (in_block) { + block = block $0 "\n"; + + # When } is encountered, it is the end of block: compute hash via tmp file + if ($0 ~ /^\}$/) { + print block > tmp_file; close(tmp_file); + cmd = "sha256sum \"" tmp_file "\""; + cmd | getline line; close(cmd); + hash = line; sub(/ .*/, "", hash); + + # If the hash is new, store it in seen[] and increase the count + if (!(hash in seen)) { + print block >> log_file; close(log_file); + print "" >> log_file; # spacer line between blocks + print hash >> hash_file; close(hash_file); + seen[hash] = 1; + unique_warnings_count++; + + if (unique_warnings_count >= max_warnings) { + suppress = 1; + } + } + + in_block = 0; + block = ""; + } + next + } + + # When { is detected, start a new block + if ($0 ~ /^\{$/) { + in_block = 1; + block = $0 "\n"; + } + } + + END { + print unique_warnings_count > count_file; close(count_file); + } + ' + EXIT_CODE=${PIPESTATUS[0]} + ERROR_COUNT=$(cat "$LOG_FILE.count" 2>/dev/null); ERROR_COUNT=${ERROR_COUNT:-0} + rm -f "$OUTPUT_DIR"/*.hashes "$OUTPUT_DIR"/*log.tmp + ;; + + memsan) + touch "$LOG_FILE" + + "$BUILD_DIR"/tests/$TEST_BINARY "$ALGORITHM" 2>&1 | awk \ + -v log_file="$LOG_FILE" \ + -v max_warnings="$MAX_WARNINGS" ' + /^SUMMARY: MemorySanitizer:/ { + # memcmp/bcmp warnings are known false positives but -fsanitize-ignorelist has no effect because they are not compiled by clang + # Skip their warnings here instead. + if ($0 ~ / in (memcmp|bcmp)$/) next + + # Check if this exact SUMMARY was already logged and store it if not + cmd = "grep -Fxq \"" $0 "\" " log_file + if (system(cmd) != 0) { + warnings++ + print >> log_file + fflush(log_file) + } + + if (warnings >= max_warnings) { + print warnings > log_file ".count" + print "TERMINATED: Exceeded " max_warnings " warnings" >> log_file + fflush(log_file) + terminated = 1 + exit 1 + } + } + # Count and store the number of unique warnings obtained and store it + END { + if (!terminated && warnings > 0) { + print warnings > log_file ".count" + } else if (!terminated) { + print 0 > log_file ".count" + } + } + ' + EXIT_CODE=$? + ERROR_COUNT=$(cat "$LOG_FILE.count" 2>/dev/null); ERROR_COUNT=${ERROR_COUNT:-0} + ;; + *) + echo "Unknown tool: $TOOL"; return 1 + ;; + esac + + if [ "$ERROR_COUNT" -eq "$MAX_WARNINGS" ]; then + echo "FAIL" | tee -a "$SUMMARY_FILE" + echo " → Found $ERROR_COUNT warnings (warning cap reached — further warnings suppressed)" \ + | tee -a "$SUMMARY_FILE" + ((++FAIL_COUNT)) + + elif [ "$ERROR_COUNT" -gt 0 ]; then + echo "FAIL" | tee -a "$SUMMARY_FILE" + echo " → Found $ERROR_COUNT warnings" \ + | tee -a "$SUMMARY_FILE" + ((++FAIL_COUNT)) + + elif [ $EXIT_CODE -ne 0 ]; then + echo "FAIL (Exit code: $EXIT_CODE)" | tee -a "$SUMMARY_FILE" + ((++FAIL_COUNT)) + + else + echo "PASS" | tee -a "$SUMMARY_FILE" + ((++PASS_COUNT)) + + fi + + rm -f "$OUTPUT_DIR"/*.count +} + +get_available_algs() { + local ALG_TYPE="$1" + local LIBOQS_DIR="$2" + + (cd "$LIBOQS_DIR" && python3 -c "import sys +sys.path.insert(0, 'tests') +import helpers +for alg in helpers.available_"$ALG_TYPE"s_by_name(): + print(alg)") +} + +# Read inputs from arguments +if [ "$#" -lt 4 ]; then + echo "Usage: $0 " + echo "Example: $0 clang-20 generic -O2 -fno-tree-vectorize all" + exit 1 +fi + +TOOL="$1" +COMPILER="$2" +TARGET="$3" + +# The last argument is the input (all/kems/sigs/) +INPUT="${!#}" + +# Collect all optimization flags between position 4 and the last-1 argument +if [ "$#" -gt 4 ]; then + NUM_OPT_ARGS=$(( $# - 4 )) + OPT_FLAG="${@:4:$NUM_OPT_ARGS}" +else + OPT_FLAG="" +fi + +SANITIZED_OPT_FLAG=$(printf "%s" "$OPT_FLAG" | tr ' ' '_' | tr -d '-' | tr -c '[:alnum:]_' '_') +if [ -z "$SANITIZED_OPT_FLAG" ]; then + SANITIZED_OPT_FLAG=default +fi + +BUILD_NAME="${TOOL//-/_}_${COMPILER//-/_}_${TARGET}_${SANITIZED_OPT_FLAG}" +BUILD_DIR="$LIBOQS_DIR/build_$BUILD_NAME" + +# Export build dir for tests/helpers.py to find generated headers +export OQS_BUILD_DIR="$BUILD_DIR" +KEMS=$(get_available_algs kem "$LIBOQS_DIR") +SIGS=$(get_available_algs sig "$LIBOQS_DIR") + +# Convert user-facing algorithm name to CMake OQS_MINIMAL_BUILD key +# Some OQS API names don't match the CMake variable suffix directly (e.g. sntrup761 maps to OQS_ENABLE_KEM_ntruprime_sntrup761, OV-Is to OQS_ENABLE_SIG_uov_ov_Is) +# Detect the type (KEM/SIG), then look up the correct suffix from alg_support.cmake +BUILD_INPUT="$INPUT" +ALG_TYPE="" +echo "$KEMS" | grep -Fxq "$INPUT" && ALG_TYPE="KEM" +echo "$SIGS" | grep -Fxq "$INPUT" && ALG_TYPE="SIG" + +if [[ -n "$ALG_TYPE" ]]; then + NORMALIZED_INPUT="$(printf '%s' "$INPUT" | tr '[:upper:]-' '[:lower:]_')" + CMAKE_SUFFIX=$(grep -oE "OQS_ENABLE_${ALG_TYPE}_[a-zA-Z0-9_]+" "$LIBOQS_DIR/.CMake/alg_support.cmake" \ + | sed "s/OQS_ENABLE_${ALG_TYPE}_//" \ + | grep -vE "(_avx2|_avx|_aesni|_x86_64|_aarch64|_icicle_cuda|_cuda)$" \ + | grep -iE "_${NORMALIZED_INPUT}$|^${NORMALIZED_INPUT}$" | head -1) + BUILD_INPUT="${ALG_TYPE}_${CMAKE_SUFFIX:-$NORMALIZED_INPUT}" +fi + +# Build liboqs with the specified compilation parameters +notify "Preparing liboqs build (compiler=${COMPILER}, target=${TARGET}, flags=${OPT_FLAG})" +build "$TOOL" "$COMPILER" "$TARGET" "$OPT_FLAG" "$BUILD_DIR" "$BUILD_INPUT" + +cd "$LIBOQS_DIR" + +TIMESTAMP="$(date '+%Y%m%d_%H%M%S')" +notify "Setting up ${TOOL} CT testing" + +RUN_KEMS=() +RUN_SIGS=() + +case "$INPUT" in + all) + RUN_KEMS=($KEMS) + RUN_SIGS=($SIGS) + ;; + kems) + RUN_KEMS=($KEMS) + ;; + sigs) + RUN_SIGS=($SIGS) + ;; + *) + if echo "$KEMS" | grep -Fxq "$INPUT"; then + RUN_KEMS=("$INPUT") + elif echo "$SIGS" | grep -Fxq "$INPUT"; then + RUN_SIGS=("$INPUT") + else + echo "Enter a valid input: all/kems/sigs/" + exit 1 + fi + ;; +esac + +for KEM in "${RUN_KEMS[@]}"; do + test "$TOOL" "$BUILD_DIR" kem "$COMPILER" "$TARGET" "$KEM" "$SCRIPT_DIR" +done + +for SIG in "${RUN_SIGS[@]}"; do + test "$TOOL" "$BUILD_DIR" sig "$COMPILER" "$TARGET" "$SIG" "$SCRIPT_DIR" +done + +notify "Finished ${TOOL} CT testing" +echo "" \ No newline at end of file diff --git a/tests/ct_tooling/local_testing_example.sh b/tests/ct_tooling/local_testing_example.sh new file mode 100755 index 0000000000..80b8b5a2cb --- /dev/null +++ b/tests/ct_tooling/local_testing_example.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT + +# Iterate through the compilation options provided by the framework to execute constant time tests +# Adjust valgrind-varlat/memsan accordingly (note that gcc uses -fno-tree-vectorize while clang uses -fno-vectorize) + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +for compiler_version in gcc gcc-14 clang clang-20; do + for liboqs_build in generic auto; do + for opt_flag in -O0 -O1 -O2 -O3 -Os -Ofast "-O2 -fno-tree-vectorize" "-O3 -fno-tree-vectorize"; do + ./ct_test.sh valgrind-varlat "$compiler_version" "$liboqs_build" "$opt_flag" all + done + done +done \ No newline at end of file diff --git a/tests/ct_tooling/tools/memsan/README.md b/tests/ct_tooling/tools/memsan/README.md new file mode 100644 index 0000000000..cb8d53ee5f --- /dev/null +++ b/tests/ct_tooling/tools/memsan/README.md @@ -0,0 +1,56 @@ +# MemSan +This directory contains the files required to execute MemSan's tooling for liboqs constant-time testing. + +MemSan handles false-positive warnings by storing specific functions into `.txt` files within the `false_positives/` directory. These files are passed as parameters of the `-fsanitize-ignorelist` flag during compilation, successfully disregarding those warnings that are cathegorized as not constant-time issues after review. + +## Compiling liboqs with MemSan +MemSan is inherently included with the clang compiler, so no requirement besides installing clang is needed. However, it does require certain workarounds to mark memory as uninitialized when building liboqs. Nonetheless, this process is directly implemented by using the `build()` function within the `ct_test.sh` script. + +The `rng_poison_msan.c` file is used to overwrite the original `OQS_randombytes()` and mark secret variables as uninitialized. Note that the actual value is filled with a non-zero buffer (0xA5) to prevent masking of bugs, as well as eliminating any random noise in the heap memory. + +For MemSan liboqs testing, it is necessary to compile liboqs with new versions of `tests/CMakeLists.txt`, `tests/test_kem.c`, `tests_sig.c`, which can be found under the repository ct-tools/memsan. These new versions allow for memory "poisoning" during the "randombytes" function in `CMakeLists.txt`, and memory "unpoisioning" of public keys in `test_kem.c` and `test_sig.c`. + +Therefore, `build()` replaces the original files with the "poisoned" ones during compilation, so that MemSan testing can successfully take place. Once liboqs compilation is ready, the script replaces the original files with a backup that was temporarily stored so that liboqs is unchanged after constant-time testing with MemSan is finished. + +## Algorithms Testing +Because of how many warnings are output, it is not feasible to store all the warnings in terms of memory and runtime. Therefore, the `test()` function in `ct_test.sh` handles MemSan's output as follows: +- It captures the first SUMMARY line of each warning, which contains key details (file, line, issue type), and stores these in log files. +- Only unique SUMMARY lines are retained, avoiding duplication from repeated warnings during execution. + +## False positive handling +MemSan follows a similar suppression mechanism to that of Valgrind-Varlat. Users can specify entities to ignore during testing by listing them in a suppression file, using a prefix that defines the entity's type. For this framework, the `fun:` prefix is used (although there are others too), since the observed false-positives originate from specific functions. The suppression file is then passed to clang at compile-time using the `-fsanitize-ignorelist` flag. + +MemSan's output also includes a full stack trace leading to the root cause. To successfully suppress a warning, the suppression file must target the exact function listed in the report's SUMMARY line. For example, given an output of the form: + +```text +==9793==WARNING: MemorySanitizer: use-of-uninitialized-value + #0 0x7eb79d2bf3f2 in sampling.c:62:9 + #1 0x7eb79d2bf3f2 in sampling.c:138:10 + #2 0x7eb79d2bf3f2 in sampling.c:174:12 + #3 0x7eb79d2bd57d in indcpa.c:278:5 + #4 0x7eb79d2bd57d in indcpa.c:508:3 + #5 0x7eb79d2be04a in kem.c:416:9 + #6 0x5d25bce8bce2 in test_kem.c:63:7 + #7 0x5d25bce8bce2 in test_kem.c:293:15 + #8 0x5d25bce8b4a5 in test_kem.c:391:12 + #9 0x7eb79ce9caa3 in pthread_create.c:447:8 + #10 0x7eb79cf29c6b in clone3.S:78 + +SUMMARY: MemorySanitizer: use-of-uninitialized-value /home/pablogf/liboqs/src/kem/ml_kem/mlkem-native_ml-kem-512_ref/mlkem/src/sampling.c:62:9 in mlk_rej_uniform_c +==9793==WARNING: MemorySanitizer: use-of-uninitialized-value +``` + +The framework will disregard this warning on future executions by including the following line in the suppression file: +```text +fun:mlk_rej_uniform_c +``` +MemSan also enables the use of the wildcard (*) within the suppression files. + +Each family of algorithms will have a specific suppression block listing the functions that output false-positives. The framework automatically includes all suppression files within the respective subdirectories so that known false positives are not returned during testing. + +For further information see https://clang.llvm.org/docs/MemorySanitizer.html and https://clang.llvm.org/docs/SanitizerSpecialCaseList.html + +## Dependencies +Remember to install the required dependencies before testing: + +`sudo apt install -y clang clang-tools` \ No newline at end of file diff --git a/tests/ct_tooling/tools/memsan/false_positives/ml_kem b/tests/ct_tooling/tools/memsan/false_positives/ml_kem new file mode 100644 index 0000000000..6b1fb56592 --- /dev/null +++ b/tests/ct_tooling/tools/memsan/false_positives/ml_kem @@ -0,0 +1,15 @@ +fun:PQCP_MLKEM_NATIVE_MLKEM*_C_poly_decompress_d* +fun:PQCP_MLKEM_NATIVE_MLKEM*_C_poly_reduce +fun:PQCP_MLKEM_NATIVE_MLKEM*_X86_64_poly_frommsg +fun:mlk_ct_memcmp +fun:mlk_ct_cmov_zero +fun:mlk_rej_uniform_scalar +fun:mlk_load32_littleendian +fun:PQCP_MLKEM_NATIVE_MLKEM*_C_poly_frommsg +fun:mlk_value_barrier_* +fun:PQCP_MLKEM_NATIVE_MLKEM*_X86_64_rej_uniform_avx2 +fun:PQCP_MLKEM_NATIVE_MLKEM*_X86_64_enc_derand +fun:mlk_ct_cmask_neg_i16 +fun:mlk_scalar_signed_to_unsigned_q +fun:mlk_ct_cmask_nonzero_u* +fun:mlk_ct_sel_int* \ No newline at end of file diff --git a/tests/ct_tooling/tools/valgrind_varlat/README.md b/tests/ct_tooling/tools/valgrind_varlat/README.md new file mode 100644 index 0000000000..7e4a805902 --- /dev/null +++ b/tests/ct_tooling/tools/valgrind_varlat/README.md @@ -0,0 +1,167 @@ +# Valgrind-Varlat +This directory uses [Daniel Bernstein's Kyberslash patches](https://kyberslash.cr.yp.to/papers.html) (valgrind-try-patch-20250805.txt and valgrind-varlat-patch-20250805.txt) and another patch including variable latency warnings in the suppression block (valgrind_varlat_sup_block.txt). These patches can also be found under the [ubuntu-latest directory of the ci-containers](https://github.com/open-quantum-safe/ci-containers/tree/main/ubuntu-latest) repository. + +Valgrind-Varlat handles false-positive warnings by storing their suppression block into `.txt` files within the `false_positives/` directory. These files are passed during the tools execution, successfully disregarding those warnings that are cathegorized as not constant-time issues after review. + +## Valgrind-Varlat Install Requirements +In order to successfully execute Valgrind-Varlat's test using the tooling developed in this subrepository follow the next steps: + +- Install valgrind using the official git repository. Go to [Daniel Bernstein's suggested](https://sourceforge.net/p/valgrind/mailman/message/59216875/) [commit](https://sourceware.org/git/?p=valgrind.git;a=commit;h=112f1080b7c21e37dfce0a2e589d0dc7aa115afa) to cleanly apply Kyberslash patches to Valgrind without encountering dependency issues: 112f1080b7c21e37dfce0a2e589d0dc7aa115afa. + +``` +VALGRIND_REPO="https://sourceware.org/git/valgrind.git" +TRY_PATCH= +VARLAT_PATCH= +SUP_BLOCK_PATCH= +INSTALL_DIR="$HOME/valgrind_varlat" + +# Clone the Valgrind repository +git clone "$VALGRIND_REPO" valgrind_varlat +git checkout 112f1080b7c21e37dfce0a2e589d0dc7aa115afa +cd valgrind_varlat +``` + +- Apply Bernstein's patches. + +``` +git apply $TRY_PATCH +git apply $VARLAT_PATCH +``` + +- Apply the suppression block patch. + +``` +git apply $SUP_BLOCK_PATCH +``` + +- Include the resultant version of valgrind into PATH under . + +``` +# Build and install valgrind_varlat +./autogen.sh +./configure --prefix="$INSTALL_DIR" +make -j$(nproc) +sudo make install + +# Rename the executable +sudo mv "$INSTALL_DIR/bin/valgrind" "$INSTALL_DIR/bin/valgrind_varlat" + +# Add valgrind_varlat to PATH +echo "export PATH=\"$INSTALL_DIR/bin:\$PATH\"" >> ~/.bashrc +source ~/.bashrc +``` + +To check whether the installation was successful, you can use the varlat tests provided in the Kyberslash patch. Compile valgrind/memcheck/tests/varlat.c with `gcc -o varlat varlat.c` and execute `valgrind_varlat --tool=memcheck --variable-latency-errors=yes --gen-suppressions=all ./varlat`. If the output ressembles something like the following output, valgrind_varlat was installed successfully: + +``` +==5335== Memcheck, a memory error detector +==5335== Copyright (C) 2002-2024, and GNU GPL'd, by Julian Seward et al. +==5335== Using Valgrind-3.26.0.GIT and LibVEX; rerun with -h for copyright info +==5335== Command: ./varlat +==5335== +==5335== Variable-latency instruction operand of size 4 is secret/uninitialised +==5335== at 0x4001176: storage_init (in /home/.../valgrind_varlat/memcheck/tests/varlat) +==5335== by 0x40011BB: main (in /home/.../valgrind_varlat/memcheck/tests/varlat) +==5335== +{ + + Memcheck:Value4 + variable-latency: yes + fun:storage_init + fun:main +} +==5335== +==5335== HEAP SUMMARY: +==5335== in use at exit: 0 bytes in 0 blocks +==5335== total heap usage: 1 allocs, 1 frees, 1 bytes allocated +==5335== +==5335== All heap blocks were freed -- no leaks are possible +==5335== +==5335== Use --track-origins=yes to see where uninitialised values come from +==5335== For lists of detected and suppressed errors, rerun with: -s +==5335== ERROR SUMMARY: 1 errors from 1 contexts (suppressed: 0 from 0) + +``` + +## Algorithm Testing +Valgrind-Varlat produces numerous warnings, making complete storage impractical in terms of memory and runtime. The test script therefore processes output as follows: +- It captures each unique suppression block (the {...} content with full warning details) and logs it to a file. +- Only these unique blocks are counted as warnings, avoiding duplication from repeated occurrences during execution. + +## False positive handling +Here is an example of a suppression file of a known false-positive for the Kyber algorithm: + +```text + { + Rejection sampling to produce public "A" matrix + Memcheck:Cond + fun:rej_uniform + fun:PQCLEAN_KYBER*_CLEAN_gen_matrix + } +``` + +The brackets wrap a single error that is to be suppressed. Within the brackets, the first line is a comment. The remaining lines tell Valgrind to ignore any "Memcheck:Cond" errors that occur when a function named rej_uniform is called from a function whose name matches the glob pattern PQCLEAN_KYBER*_CLEAN_gen_matrix. + +Before this suppression file was written, a run of this script produced the following output. +```text + ==594== Conditional jump or move depends on uninitialised value(s) + ==594== at 0x22550D: rej_uniform (indcpa.c:133) + ==594== by 0x225654: PQCLEAN_KYBER512_CLEAN_gen_matrix (indcpa.c:177) + ==594== by 0x2257D1: PQCLEAN_KYBER512_CLEAN_indcpa_keypair (indcpa.c:216) + ==594== by 0x1B6C1E: PQCLEAN_KYBER512_CLEAN_crypto_kem_keypair (kem.c:26) + ==594== by 0x1B6B9F: OQS_KEM_kyber_512_keypair (kem_kyber_512.c:56) + ==594== by 0x10D123: OQS_KEM_keypair (kem.c:818) + ==594== by 0x10AD07: kem_test_correctness (test_kem.c:103) + ==594== by 0x10B4E7: test_wrapper (test_kem.c:186) + ==594== by 0x4CDAFA2: start_thread (pthread_create.c:486) + ==594== by 0x4DED4CE: clone (clone.S:95) + ==594== + { + + Memcheck:Cond + fun:rej_uniform + fun:PQCLEAN_KYBER512_CLEAN_gen_matrix + fun:PQCLEAN_KYBER512_CLEAN_indcpa_keypair + fun:PQCLEAN_KYBER512_CLEAN_crypto_kem_keypair + fun:OQS_KEM_kyber_512_keypair + fun:OQS_KEM_keypair + fun:kem_test_correctness + fun:test_wrapper + fun:start_thread + fun:clone + } +``` +The lines beginning with "==" are a Valgrind error message. The bracketed text is a suppression file template. To produce the final suppression file we added a comment, replaced "512" with a wildcard (since an identical error occurs in other Kyber parameter sets), and truncated the backtrace (since the extra lines provide no interesting information to auditors). + +The "fun:rej_uniform" line says to ignore _all_ Memcheck:Cond errors in rej_uniform, but Valgrind told us that line 133 was the problem. Any "fun:name" line in the backtrace can be replaced by an equivalent "src:file:line", so we could have narrowed the scope of our suppression: +```text + { + Rejection sampling to produce public "A" matrix + Memcheck:Cond + src:indcpa.c:133 # fun:rej_uniform + fun:PQCLEAN_KYBER*_CLEAN_gen_matrix + } +``` +Here "# fun:rej_uniform" is a comment. An update to the Kyber source code might break our suppression file by changing the line number, and leaving the function name as a comment might help a future reviewer. + +An ellipsis (...) can serve as a wildcard for a portion of the backtrace. We could have written: +```text + { + Rejection sampling to produce public "A" matrix + Memcheck:Cond + ... + fun:PQCLEAN_KYBER*_CLEAN_gen_matrix + } +``` +But this is perhaps too concise. Remember that the goal here is to help auditors. + +Further information can be found in Valgrind's manual. See + https://www.valgrind.org/docs/manual/manual-core.html#manual-core.suppress +and + https://www.valgrind.org/docs/manual/mc-manual.html#mc-manual.suppfiles + +## Dependencies +Remember to install the prerequisites for building liboqs: + +`sudo apt install -y build-essential cmake ninja-build gcc g++ clang python3 python3-pip valgrind libssl-dev realpath python3 python3-pip` +`pip3 install pytest` diff --git a/tests/constant_time/kem/passes/bike b/tests/ct_tooling/tools/valgrind_varlat/false_positives/bike similarity index 100% rename from tests/constant_time/kem/passes/bike rename to tests/ct_tooling/tools/valgrind_varlat/false_positives/bike diff --git a/tests/constant_time/sig/passes/cross b/tests/ct_tooling/tools/valgrind_varlat/false_positives/cross similarity index 100% rename from tests/constant_time/sig/passes/cross rename to tests/ct_tooling/tools/valgrind_varlat/false_positives/cross diff --git a/tests/constant_time/sig/passes/falcon_keygen b/tests/ct_tooling/tools/valgrind_varlat/false_positives/falcon similarity index 67% rename from tests/constant_time/sig/passes/falcon_keygen rename to tests/ct_tooling/tools/valgrind_varlat/false_positives/falcon index deca2f9e3b..0364a95e03 100644 --- a/tests/constant_time/sig/passes/falcon_keygen +++ b/tests/ct_tooling/tools/valgrind_varlat/false_positives/falcon @@ -168,3 +168,83 @@ fun:PQCLEAN_FALCON*_*_modq_encode fun:PQCLEAN_FALCON*_*_crypto_sign_keypair } + +# Suppressions that work for both CLEAN and AVX2 implementations + +{ + Exception while reading secret key. No signature is produced. + Memcheck:Cond + fun:PQCLEAN_FALCON*_*_trim_i8_decode + fun:do_sign + fun:PQCLEAN_FALCON*_*_crypto_sign_signature +} +{ + Exception while re-computing private basis (non-invertible f). No signature is produced. + Memcheck:Cond + src:vrfy.c:726 # fun:PQCLEAN_FALCON*_*_complete_private + fun:do_sign + fun:PQCLEAN_FALCON*_*_crypto_sign_signature +} +{ + Exception while re-computing private basis (large G coefficients). No signature is produced. + Memcheck:Cond + src:vrfy.c:739 # fun:PQCLEAN_FALCON*_*_complete_private + fun:do_sign + fun:PQCLEAN_FALCON*_*_crypto_sign_signature +} +{ + Rejection sampling on encoded signature size + Memcheck:Cond + fun:PQCLEAN_FALCON*_*_comp_encode + fun:do_sign + fun:PQCLEAN_FALCON*_*_crypto_sign_signature +} + + +# CLEAN implementation of sign.c + +{ + Rejection sampling to re-center discrete Gaussian distribution + Memcheck:Cond + # Wildcard to catch the while loop in fun:BerExp as well as line the if on src:sign.c:1155 + ... + src:sign.c:1155 # fun:PQCLEAN_FALCON*_CLEAN_sampler + ... + fun:PQCLEAN_FALCON*_CLEAN_crypto_sign_signature +} +{ + Rejection sampling on signature length + Memcheck:Cond + src:sign.c:946 # fun:do_sign_dyn + fun:PQCLEAN_FALCON*_CLEAN_sign_dyn + fun:do_sign + fun:PQCLEAN_FALCON*_CLEAN_crypto_sign_signature +} +{ + Rejection sampling on encoded signature size + Memcheck:Cond + fun:PQCLEAN_FALCON*_CLEAN_comp_encode + fun:do_sign + fun:PQCLEAN_FALCON*_CLEAN_crypto_sign_signature +} + + +# AVX2 implementation of sign.c + +{ + Rejection sampling to re-center discrete Gaussian distribution + Memcheck:Cond + # Wildcard to catch the while loop in fun:BerExp as well as line the if on src:sign.c:1153 + ... + src:sign.c:1213 # fun:PQCLEAN_FALCON*_AVX2_sampler + ... + fun:PQCLEAN_FALCON*_AVX2_crypto_sign_signature +} +{ + Rejection sampling on signature length + Memcheck:Cond + src:sign.c:939 # fun:do_sign_dyn + fun:PQCLEAN_FALCON*_AVX2_sign_dyn + fun:do_sign + fun:PQCLEAN_FALCON*_AVX2_crypto_sign_signature +} diff --git a/tests/constant_time/sig/passes/mayo b/tests/ct_tooling/tools/valgrind_varlat/false_positives/mayo similarity index 100% rename from tests/constant_time/sig/passes/mayo rename to tests/ct_tooling/tools/valgrind_varlat/false_positives/mayo diff --git a/tests/ct_tooling/tools/valgrind_varlat/false_positives/ml-kem b/tests/ct_tooling/tools/valgrind_varlat/false_positives/ml-kem new file mode 100644 index 0000000000..11007e5702 --- /dev/null +++ b/tests/ct_tooling/tools/valgrind_varlat/false_positives/ml-kem @@ -0,0 +1,83 @@ +{ + + Memcheck:Cond + ... + fun:mlk_rej_uniform + ... + fun:kem_test_correctness + fun:test_wrapper + fun:start_thread + fun:clone +} + +{ + + Memcheck:Value8 + ... + fun:mlk_rej_uniform + ... + fun:kem_test_correctness + fun:test_wrapper + fun:start_thread + fun:clone +} + +{ + + Memcheck:Cond + ... + fun:PQCP_MLKEM_NATIVE_MLKEM*_dec + ... + fun:kem_test_correctness + fun:test_wrapper + fun:start_thread + fun:clone +} + +{ + + Memcheck:Cond + ... + fun:PQCP_MLKEM_NATIVE_MLKEM*_X86_64_poly_rej_uniform* + ... + fun:kem_test_correctness + fun:test_wrapper + fun:start_thread + fun:clone +} + +{ + + Memcheck:Value8 + ... + fun:PQCP_MLKEM_NATIVE_MLKEM*_X86_64_poly_rej_uniform* + ... + fun:kem_test_correctness + fun:test_wrapper + fun:start_thread + fun:clone +} + +{ + + Memcheck:Cond + ... + fun:mlk_check_sk + ... + fun:kem_test_correctness + fun:test_wrapper + fun:start_thread + fun:clone +} + +{ + + Memcheck:Value8 + ... + fun:_mm_set_epi64x + ... + fun:kem_test_correctness + fun:test_wrapper + fun:start_thread + fun:clone +} \ No newline at end of file diff --git a/tests/constant_time/sig/passes/mqom b/tests/ct_tooling/tools/valgrind_varlat/false_positives/mqom similarity index 100% rename from tests/constant_time/sig/passes/mqom rename to tests/ct_tooling/tools/valgrind_varlat/false_positives/mqom diff --git a/tests/constant_time/sig/passes/snova b/tests/ct_tooling/tools/valgrind_varlat/false_positives/snova similarity index 100% rename from tests/constant_time/sig/passes/snova rename to tests/ct_tooling/tools/valgrind_varlat/false_positives/snova diff --git a/tests/constant_time/kem/issues/classic-mceliece-348864 b/tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-348864 similarity index 100% rename from tests/constant_time/kem/issues/classic-mceliece-348864 rename to tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-348864 diff --git a/tests/constant_time/kem/issues/classic-mceliece-348864f b/tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-348864f similarity index 100% rename from tests/constant_time/kem/issues/classic-mceliece-348864f rename to tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-348864f diff --git a/tests/constant_time/kem/issues/classic-mceliece-460896 b/tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-460896 similarity index 100% rename from tests/constant_time/kem/issues/classic-mceliece-460896 rename to tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-460896 diff --git a/tests/constant_time/kem/issues/classic-mceliece-460896f b/tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-460896f similarity index 100% rename from tests/constant_time/kem/issues/classic-mceliece-460896f rename to tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-460896f diff --git a/tests/constant_time/kem/issues/classic-mceliece-6688128 b/tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-6688128 similarity index 100% rename from tests/constant_time/kem/issues/classic-mceliece-6688128 rename to tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-6688128 diff --git a/tests/constant_time/kem/issues/classic-mceliece-6688128f b/tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-6688128f similarity index 100% rename from tests/constant_time/kem/issues/classic-mceliece-6688128f rename to tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-6688128f diff --git a/tests/constant_time/kem/issues/classic-mceliece-6960119 b/tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-6960119 similarity index 100% rename from tests/constant_time/kem/issues/classic-mceliece-6960119 rename to tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-6960119 diff --git a/tests/constant_time/kem/issues/classic-mceliece-6960119f b/tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-6960119f similarity index 100% rename from tests/constant_time/kem/issues/classic-mceliece-6960119f rename to tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-6960119f diff --git a/tests/constant_time/kem/issues/classic-mceliece-8192128 b/tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-8192128 similarity index 100% rename from tests/constant_time/kem/issues/classic-mceliece-8192128 rename to tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-8192128 diff --git a/tests/constant_time/kem/issues/classic-mceliece-8192128f b/tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-8192128f similarity index 100% rename from tests/constant_time/kem/issues/classic-mceliece-8192128f rename to tests/ct_tooling/tools/valgrind_varlat/issues/classic-mceliece-8192128f diff --git a/tests/constant_time/sig/issues/falcon b/tests/ct_tooling/tools/valgrind_varlat/issues/falcon similarity index 100% rename from tests/constant_time/sig/issues/falcon rename to tests/ct_tooling/tools/valgrind_varlat/issues/falcon diff --git a/tests/constant_time/sig/issues/slh_dsa b/tests/ct_tooling/tools/valgrind_varlat/issues/slh_dsa similarity index 100% rename from tests/constant_time/sig/issues/slh_dsa rename to tests/ct_tooling/tools/valgrind_varlat/issues/slh_dsa diff --git a/tests/test_constant_time.py b/tests/test_constant_time.py deleted file mode 100644 index c31436480d..0000000000 --- a/tests/test_constant_time.py +++ /dev/null @@ -1,282 +0,0 @@ -# SPDX-License-Identifier: MIT - -""" test_constant_time.py - -The goal of this script is to ensure that every instance of secret-dependant -control flow in liboqs is documented. This script does not ensure that all of -the software in liboqs is constant time. Rather, it is intended to aid auditors -in their search for non-constant time behaviour. - -WARNING: This script currently runs test_kem and test_sig on random seeds. -It is not coverage guided. It will miss instances of non-constant time -behaviour in code paths that are rarely executed. - -This script requires Valgrind version >= 3.14.0, and it only gives meaningful -results if test_kem and test_sig have been compiled with CMAKE_BUILD_TYPE=Debug -and OQS_ENABLE_TEST_CONSTANT_TIME. - - -How this script works ---------------------- -This script runs test_kem (and/or test_sig) through Valgrind's Memcheck tool. -Valgrind executes the test program and issues an error message if/whenever the -program's control flow depends on uninitialized data. As observed by Adam -Langley [1], if we tell Valgrind that secrets are uninitialized, then Valgrind -will tell us about secret-dependant control flow. - -Assuming that each scheme in liboqs passes test_kat, our test_kem and test_sig -programs are structured such that all secret data can be traced back to a call -to OQS_randombytes. The tests intercept calls to OQS_randombytes and tell -Valgrind that every random byte is uninitialized. Hence, Valgrind will issue an -error if (but not only if!) our tests branch on secret data. - -Since there may be false positives, we say that Valgrind identifies "suspected -non-constant time behaviour". - -Again, the purpose of this script is to ensure that suspected non-constant time -behaviour is documented. This script ships with a collection of Valgrind -"suppression files". Each suppression file documents one or more instances of -suspected non-constant time behaviour in liboqs. - -The suppression files are also used to silence errors from Valgrind. If this -script runs without error, then all of the suspected non-constant time behaviour -in liboqs has been documented. If this script fails, then a new suppression -file should be written. - - -How to write suppression files ------------------------------- -Valgrind will output a suppression file template along with its error message. -It's your job to copy this template to the correct location, edit it, and tell -this script about the new file. - -Suppression files for KEMs are stored in - liboqs/tests/constant_time/kem/{passes,issues}/. -Suppression files for signature schemes are stored in - liboqs/tests/constant_time/sig/{passes,issues}/. - -This script does not differentiate between the passes and issues -subdirectories. The label is for auditors. We "give a pass" to an error that -is known not to be a security threat, and we store the corresponding -suppression file in the "passes" subdirectory. We "raise an issue" about any -other error, and we store the corresponding suppression file in the "issues" -subdirectory. - -If you are unsure where your suppression file belongs, then save it to the -"issues" subdirectory. - -Once you've written a suppression file, give it a descriptive file name and -tell this script about it. There are json files called passes.json and -issues.json in - liboqs/tests/constant_time/{kem,sig}/ -These json files contain dictionaries of the form - { "Scheme name" : ["list", "of", "suppression", "files"], ... } -Add the name of your suppression file to the appropriate list to suppress -the errors that you have documented. - - -How to write a good suppression file ------------------------------------- -Here is an example of a suppression file: - { - Rejection sampling to produce public "A" matrix - Memcheck:Cond - fun:rej_uniform - fun:PQCLEAN_KYBER*_CLEAN_gen_matrix - } - -The brackets wrap a single error that is to be suppressed. Within the brackets, -the first line is a comment. The remaining lines tell Valgrind to ignore -any "Memcheck:Cond" errors that occur when a function named rej_uniform is -called from a function whose name matches the glob pattern -PQCLEAN_KYBER*_CLEAN_gen_matrix. - -Before this suppression file was written, a run of this script produced the -following output. - - ==594== Conditional jump or move depends on uninitialised value(s) - ==594== at 0x22550D: rej_uniform (indcpa.c:133) - ==594== by 0x225654: PQCLEAN_KYBER512_CLEAN_gen_matrix (indcpa.c:177) - ==594== by 0x2257D1: PQCLEAN_KYBER512_CLEAN_indcpa_keypair (indcpa.c:216) - ==594== by 0x1B6C1E: PQCLEAN_KYBER512_CLEAN_crypto_kem_keypair (kem.c:26) - ==594== by 0x1B6B9F: OQS_KEM_kyber_512_keypair (kem_kyber_512.c:56) - ==594== by 0x10D123: OQS_KEM_keypair (kem.c:818) - ==594== by 0x10AD07: kem_test_correctness (test_kem.c:103) - ==594== by 0x10B4E7: test_wrapper (test_kem.c:186) - ==594== by 0x4CDAFA2: start_thread (pthread_create.c:486) - ==594== by 0x4DED4CE: clone (clone.S:95) - ==594== - { - - Memcheck:Cond - fun:rej_uniform - fun:PQCLEAN_KYBER512_CLEAN_gen_matrix - fun:PQCLEAN_KYBER512_CLEAN_indcpa_keypair - fun:PQCLEAN_KYBER512_CLEAN_crypto_kem_keypair - fun:OQS_KEM_kyber_512_keypair - fun:OQS_KEM_keypair - fun:kem_test_correctness - fun:test_wrapper - fun:start_thread - fun:clone - } - -The lines beginning with "==" are a Valgrind error message. The bracketed text -is a suppression file template. To produce the final suppression file we -added a comment, replaced "512" with a wildcard (since an identical error occurs -in other Kyber parameter sets), and truncated the backtrace (since the extra lines -provide no interesting information to auditors). - -The "fun:rej_uniform" line says to ignore _all_ Memcheck:Cond errors in -rej_uniform, but Valgrind told us that line 133 was the problem. Any -"fun:name" line in the backtrace can be replaced by an equivalent -"src:file:line", so we could have narrowed the scope of our suppression: - { - Rejection sampling to produce public "A" matrix - Memcheck:Cond - src:indcpa.c:133 # fun:rej_uniform - fun:PQCLEAN_KYBER*_CLEAN_gen_matrix - } -Here "# fun:rej_uniform" is a comment. An update to the Kyber source code might -break our suppression file by changing the line number, and leaving the function -name as a comment might help a future reviewer. - -An ellipsis (...) can serve as a wildcard for a portion of the backtrace. -We could have written: - { - Rejection sampling to produce public "A" matrix - Memcheck:Cond - ... - fun:PQCLEAN_KYBER*_CLEAN_gen_matrix - } -But this is perhaps too concise. Remember that the goal here is to help auditors. - -Further information can be found in Valgrind's manual. See - https://www.valgrind.org/docs/manual/manual-core.html#manual-core.suppress -and - https://www.valgrind.org/docs/manual/mc-manual.html#mc-manual.suppfiles - -Credits -------- -The observation that Valgrind can be used to identify non-constant time -behaviour is due to Adam Langley [1, 2]. Mortiz Neikes' TIMECOP project applies -Langley's idea to the SUPERCOP benchmarking suite [3]. Versions of SUPERCOP -starting with 20200816 include TIMECOP and apply Langley's idea to randombytes -calls in particular [4]. We have borrowed the idea of instrumenting randombytes -calls from SUPERCOP. - -[1] https://github.com/agl/ctgrind -[2] https://boringssl.googlesource.com/boringssl/+/a6a049a6fb51a052347611d41583a0622bc89d60 -[2] https://post-apocalyptic-crypto.org/timecop/index.html -[3] http://bench.cr.yp.to/tips.html#timecop -""" - - -import helpers -import json -import os -import pytest -import sys -import re - -REQ_LIBOQS_BUILD_OPTS = ['OQS_ENABLE_TEST_CONSTANT_TIME', - 'OQS_DEBUG_BUILD'] - -# Error suppression based on file and line number was introduced in -# Valgrind 3.14.0 (9 October 2018). -# https://www.valgrind.org/docs/manual/dist.news.html -MIN_VALGRIND_VERSION = [3, 14, 0] - -VALGRIND = ['valgrind', - # '-v', # Turn on -v to see which suppression files are used - '--tool=memcheck', - '--gen-suppressions=all', - '--error-exitcode=1', - '--max-stackframe=20480000', - '--num-callers=20', - ] - -# The following two functions read the json files -# liboqs/tests/constant_time/{kem,sig}/{passes,issues}.json -# into python dictionaries `ct_passes' and `ct_issues', which -# are of the form -# { 'kem' : { 'Kem Name' : ['list', 'of', 'filenames'], ... }, -# 'sig' : { 'Sig Name' : ['list', 'of', 'filenames'], ... } -# } - -ct_passes = {'kem': None, 'sig': None} -ct_issues = {'kem': None, 'sig': None} - -def get_ct_passes(t, name): - ct_t = os.path.join('tests', 'constant_time', t) - if ct_passes[t] is None: - with open(os.path.join(ct_t, 'passes.json'), 'r') as fp: - ct_passes[t] = json.load(fp) - passes = ct_passes[t].get(name,[]) - return [os.path.join(ct_t, 'passes', f) for f in passes] - -def get_ct_issues(t, name): - ct_t = os.path.join('tests', 'constant_time', t) - if ct_issues[t] is None: - with open(os.path.join(ct_t, 'issues.json'), 'r') as fp: - ct_issues[t] = json.load(fp) - issues = ct_issues[t].get(name,[]) - return [os.path.join(ct_t, 'issues', f) for f in issues] - - -@helpers.filtered_test -@helpers.test_requires_build_options(*REQ_LIBOQS_BUILD_OPTS) -@helpers.test_requires_valgrind_version_at_least(*MIN_VALGRIND_VERSION) -@pytest.mark.parametrize('kem_name', helpers.available_kems_by_name()) -def test_constant_time_kem(kem_name): - if not(helpers.is_kem_enabled_by_name(kem_name)): pytest.skip('Not enabled') - if ('SKIP_ALGS' in os.environ) and len(os.environ['SKIP_ALGS'])>0: - for algexp in os.environ['SKIP_ALGS'].split(','): - if len(re.findall(algexp, kem_name))>0: - pytest.skip("Test disabled by alg filter") - passes = get_ct_passes('kem', kem_name) - issues = get_ct_issues('kem', kem_name) - output = helpers.run_subprocess( - VALGRIND + [ - *(['--suppressions='+f for f in passes]), - *(['--suppressions='+f for f in issues]), - helpers.path_to_executable('test_kem'), - kem_name - ] - ) - -@helpers.filtered_test -@helpers.test_requires_build_options(*REQ_LIBOQS_BUILD_OPTS) -@helpers.test_requires_valgrind_version_at_least(*MIN_VALGRIND_VERSION) -@pytest.mark.parametrize('sig_name', helpers.available_sigs_by_name()) -def test_constant_time_sig(sig_name): - if not(helpers.is_sig_enabled_by_name(sig_name)): pytest.skip('Not enabled') - if ('SKIP_ALGS' in os.environ) and len(os.environ['SKIP_ALGS'])>0: - for algexp in os.environ['SKIP_ALGS'].split(','): - if len(re.findall(algexp, sig_name))>0: - pytest.skip("Test disabled by alg filter") - passes = get_ct_passes('sig', sig_name) - issues = get_ct_issues('sig', sig_name) - output = helpers.run_subprocess( - VALGRIND + [ - *(['--suppressions='+f for f in passes]), - *(['--suppressions='+f for f in issues]), - helpers.path_to_executable('test_sig'), - sig_name - ] - ) - -if __name__ == '__main__': - pytest.main(sys.argv) - -# Unused/obsolete suppressions are a burden on reviewers. You can find out which suppressions -# are used by passing the -v flag to valgrind. To find unused suppressions we have to extract -# a list of available suppressions first. You can use awk to find lines that contain only a '{'. -# Increment these line numbers by 1 to match the output of valgrind -v, then compare against -# the used suppressions. -# -# awk '$0 ~ /^{$/{print FILENAME ":" NR+1}' suppression files > /tmp/available_suppressions -# valgrind -v --suppressions=[...] ./build/tests/test_kem KEM_NAME 2>&1 \ -# | grep used_suppression \ -# | awk '{ print $NF }' > /tmp/used_suppressions -# cat /tmp/used_suppressions /tmp/available_suppressions | sort | uniq -u diff --git a/tests/test_helpers.h b/tests/test_helpers.h index ad73fb4176..85e64f04ea 100644 --- a/tests/test_helpers.h +++ b/tests/test_helpers.h @@ -9,10 +9,14 @@ #include #include -#ifdef OQS_ENABLE_TEST_CONSTANT_TIME +#ifdef OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND #include -#define OQS_TEST_CT_CLASSIFY(addr, len) VALGRIND_MAKE_MEM_UNDEFINED(addr, len) +#define OQS_TEST_CT_CLASSIFY(addr, len) VALGRIND_MAKE_MEM_UNDEFINED(addr, len) #define OQS_TEST_CT_DECLASSIFY(addr, len) VALGRIND_MAKE_MEM_DEFINED(addr, len) +#elif defined(__SANITIZE_MEMORY__) || defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) +#include +#define OQS_TEST_CT_CLASSIFY(addr, len) __msan_poison(addr, len) +#define OQS_TEST_CT_DECLASSIFY(addr, len) __msan_unpoison(addr, len) #else #define OQS_TEST_CT_CLASSIFY(addr, len) #define OQS_TEST_CT_DECLASSIFY(addr, len) diff --git a/tests/test_kem.c b/tests/test_kem.c index 2d6819a75c..1eacae627e 100644 --- a/tests/test_kem.c +++ b/tests/test_kem.c @@ -23,6 +23,10 @@ #include "system_info.c" #include "test_helpers.h" +#if defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) +#include +#endif + #ifdef OQS_ENABLE_KEM_ML_KEM /* mlkem rejection key testcase */ static bool mlkem_rej_testcase(OQS_KEM *kem, uint8_t *ciphertext, uint8_t *secret_key) { @@ -296,7 +300,7 @@ static OQS_STATUS kem_test_correctness(const char *method_name, bool derand) { goto err; } -#ifndef OQS_ENABLE_TEST_CONSTANT_TIME +#if defined(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) || defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) rv = memcmp(public_key + kem->length_public_key, magic.val, sizeof(magic_t)); rv |= memcmp(secret_key + kem->length_secret_key, magic.val, sizeof(magic_t)); rv |= memcmp(ciphertext + kem->length_ciphertext, magic.val, sizeof(magic_t)); @@ -352,7 +356,7 @@ static OQS_STATUS kem_test_correctness(const char *method_name, bool derand) { return ret; } -#ifdef OQS_ENABLE_TEST_CONSTANT_TIME +#if defined(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) || defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) static void TEST_KEM_randombytes(uint8_t *random_array, size_t bytes_to_read) { // We can't make direct calls to the system randombytes on some platforms, // so we have to swap out the OQS_randombytes provider. @@ -414,7 +418,7 @@ int main(int argc, char **argv) { return EXIT_FAILURE; } -#ifdef OQS_ENABLE_TEST_CONSTANT_TIME +#if defined(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) || defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) OQS_randombytes_custom_algorithm(&TEST_KEM_randombytes); #else rc = OQS_randombytes_switch_algorithm("system"); diff --git a/tests/test_sig.c b/tests/test_sig.c index 91181b48d4..1ba570644a 100644 --- a/tests/test_sig.c +++ b/tests/test_sig.c @@ -17,6 +17,10 @@ #include "system_info.c" #include "test_helpers.h" +#if defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) +#include +#endif + typedef struct magic_s { uint8_t val[31]; } magic_t; @@ -98,6 +102,7 @@ static OQS_STATUS sig_test_correctness(const char *method_name, bool bitflips_al OQS_TEST_CT_DECLASSIFY(message, message_len); rc = OQS_SIG_keypair(sig, public_key, secret_key); + OQS_TEST_CT_DECLASSIFY(&rc, sizeof rc); if (rc != OQS_SUCCESS) { fprintf(stderr, "ERROR: OQS_SIG_keypair failed\n"); @@ -111,9 +116,11 @@ static OQS_STATUS sig_test_correctness(const char *method_name, bool bitflips_al goto err; } +#if !defined(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) && !defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) OQS_TEST_CT_DECLASSIFY(public_key, sig->length_public_key); OQS_TEST_CT_DECLASSIFY(signature, signature_len); rc = OQS_SIG_verify(sig, message, message_len, signature, signature_len, public_key); + OQS_TEST_CT_DECLASSIFY(&rc, sizeof rc); if (rc != OQS_SUCCESS) { fprintf(stderr, "ERROR: OQS_SIG_verify failed\n"); @@ -127,6 +134,7 @@ static OQS_STATUS sig_test_correctness(const char *method_name, bool bitflips_al goto err; } } +#endif /* testing signing with context, if supported */ OQS_randombytes(ctx, 257); @@ -145,6 +153,7 @@ static OQS_STATUS sig_test_correctness(const char *method_name, bool bitflips_al goto err; } +#if !defined(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) && !defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) OQS_TEST_CT_DECLASSIFY(public_key, sig->length_public_key); OQS_TEST_CT_DECLASSIFY(signature, signature_len); rc = OQS_SIG_verify_with_ctx_str(sig, message, message_len, signature, signature_len, ctx, i, public_key); @@ -161,6 +170,7 @@ static OQS_STATUS sig_test_correctness(const char *method_name, bool bitflips_al goto err; } } +#endif } } @@ -187,6 +197,7 @@ static OQS_STATUS sig_test_correctness(const char *method_name, bool bitflips_al fprintf(stderr, "ERROR: OQS_SIG_sign_with_ctx_str should always succeed when providing a NULL context string\n"); goto err; } +#if !defined(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) && !defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) OQS_TEST_CT_DECLASSIFY(public_key, sig->length_public_key); OQS_TEST_CT_DECLASSIFY(signature, signature_len); rc = OQS_SIG_verify_with_ctx_str(sig, message, message_len, signature, signature_len, NULL, 0, public_key); @@ -201,9 +212,10 @@ static OQS_STATUS sig_test_correctness(const char *method_name, bool bitflips_al if (rc != OQS_SUCCESS) { goto err; } +#endif } -#ifndef OQS_ENABLE_TEST_CONSTANT_TIME +#if defined(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) || defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) /* check magic values */ int rv = memcmp(public_key + sig->length_public_key, magic.val, sizeof(magic_t)); rv |= memcmp(secret_key + sig->length_secret_key, magic.val, sizeof(magic_t)); @@ -244,7 +256,7 @@ static OQS_STATUS sig_test_correctness(const char *method_name, bool bitflips_al return ret; } -#ifdef OQS_ENABLE_TEST_CONSTANT_TIME +#if defined(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) || defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) static void TEST_SIG_randombytes(uint8_t *random_array, size_t bytes_to_read) { // We can't make direct calls to the system randombytes on some platforms, // so we have to swap out the OQS_randombytes provider. @@ -279,7 +291,7 @@ void *test_wrapper(void *arg) { int main(int argc, char **argv) { OQS_STATUS rc; OQS_init(); -#if defined(OQS_ENABLE_TEST_CONSTANT_TIME) || defined(USE_SANITIZER) +#if defined(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) || defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) || defined(USE_SANITIZER) long int extended_tests = 0; #else long int extended_tests = 1; @@ -338,7 +350,7 @@ int main(int argc, char **argv) { } } -#ifdef OQS_ENABLE_TEST_CONSTANT_TIME +#if defined(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) || defined(OQS_ENABLE_TEST_CONSTANT_TIME_MEMSAN) OQS_randombytes_custom_algorithm(&TEST_SIG_randombytes); #else rc = OQS_randombytes_switch_algorithm("system"); @@ -349,7 +361,7 @@ int main(int argc, char **argv) { } #endif -#if OQS_USE_PTHREADS && !defined(OQS_ENABLE_TEST_CONSTANT_TIME) +#if OQS_USE_PTHREADS && !defined(OQS_ENABLE_TEST_CONSTANT_TIME_VALGRIND) #define MAX_LEN_SIG_NAME_ 64 // don't run algorithms with large stack usage in threads char no_thread_sig_patterns[][MAX_LEN_SIG_NAME_] = {"MAYO-5", "cross-rsdp-128-small", "cross-rsdp-192-small", "cross-rsdp-256-balanced", "cross-rsdp-256-small", "cross-rsdpg-192-small", "cross-rsdpg-256-small", "SNOVA_37_17_2", "SNOVA_56_25_2", "SNOVA_49_11_3", "SNOVA_37_8_4", "SNOVA_24_5_5", "SNOVA_60_10_4", "SNOVA_29_6_5", "mqom2_cat1_gf16_fast_r3", "mqom2_cat1_gf16_fast_r5", "mqom2_cat1_gf16_short_r3", "mqom2_cat1_gf16_short_r5", "mqom2_cat3_gf16_fast_r3", "mqom2_cat3_gf16_fast_r5", "mqom2_cat3_gf16_short_r3", "mqom2_cat3_gf16_short_r5", "mqom2_cat5_gf16_fast_r3", "mqom2_cat5_gf16_fast_r5", "mqom2_cat5_gf16_short_r3", "mqom2_cat5_gf16_short_r5"};