ACVP: Filter unsupported functions from test cases

hanno-becker · hanno-becker · commit 1f3f0cdc7b00 · 2026-07-28T05:02:14.000+01:00
The previous encapDecap-file gate only loaded the file when all four sub-functions (encapsulation, decapsulation, encapsulationKeyCheck, decapsulationKeyCheck) were compiled in. Under encaps-only, decaps-only, keygen-encaps, and keygen-decaps configurations that skipped the entire prompt file, so ACVP reported OK while driving zero test cases through the binary. This commit intead loads the encapDecap file whenever _any_ of its sub-functions are supported, and drops unsupported test cases via a filter that mutates both the prompt and the expectedResults data consistently. This mirrors the filter_test_cases machinery introduced for the same reason in mldsa-native (PR #1140). Reduced-API runs also now emit a labelled stderr summary block after "ALL GOOD!" for both whole-prompt skips and per-test drops, so coverage gaps aren't buried in scrollback. Full-API runs stay silent. Signed-off-by: Hanno Becker <beckphan@amazon.co.uk>
diff --git a/examples/basic_deterministic/mlkem_native/mlkem_native_config.h b/examples/basic_deterministic/mlkem_native/mlkem_native_config.h
@@ -151,7 +151,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/examples/bring_your_own_fips202/mlkem_native/mlkem_native_config.h b/examples/bring_your_own_fips202/mlkem_native/mlkem_native_config.h
@@ -151,7 +151,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/examples/bring_your_own_fips202_static/mlkem_native/mlkem_native_config.h b/examples/bring_your_own_fips202_static/mlkem_native/mlkem_native_config.h
@@ -152,7 +152,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/examples/custom_backend/mlkem_native/mlkem_native_config.h b/examples/custom_backend/mlkem_native/mlkem_native_config.h
@@ -153,7 +153,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/examples/monolithic_build/mlkem_native/mlkem_native_config.h b/examples/monolithic_build/mlkem_native/mlkem_native_config.h
@@ -150,7 +150,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/examples/monolithic_build_multilevel/mlkem_native/mlkem_native_config.h b/examples/monolithic_build_multilevel/mlkem_native/mlkem_native_config.h
@@ -152,7 +152,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/examples/monolithic_build_multilevel_native/mlkem_native/mlkem_native_config.h b/examples/monolithic_build_multilevel_native/mlkem_native/mlkem_native_config.h
@@ -156,7 +156,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/examples/monolithic_build_native/mlkem_native/mlkem_native_config.h b/examples/monolithic_build_native/mlkem_native/mlkem_native_config.h
@@ -150,7 +150,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/examples/multilevel_build/mlkem_native/mlkem_native_config.h b/examples/multilevel_build/mlkem_native/mlkem_native_config.h
@@ -151,7 +151,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/examples/multilevel_build_native/mlkem_native/mlkem_native_config.h b/examples/multilevel_build_native/mlkem_native/mlkem_native_config.h
@@ -153,7 +153,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/mlkem/mlkem_native.h b/mlkem/mlkem_native.h
@@ -453,7 +453,7 @@ int MLK_API_NAMESPACE(check_sk)(
 #if !defined(MLK_CONFIG_NO_DECAPS_API)
 #define crypto_kem_dec MLK_API_NAMESPACE(dec)
 #define crypto_kem_check_sk MLK_API_NAMESPACE(check_sk)
-#endif /* !MLK_CONFIG_NO_DECAPS_API */
+#endif
 
 #else /* !MLK_CONFIG_API_NO_SUPERCOP */
 
diff --git a/proofs/cbmc/mlkem_native_config_cbmc.h b/proofs/cbmc/mlkem_native_config_cbmc.h
@@ -152,7 +152,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/acvp/acvp_client.py b/test/acvp/acvp_client.py
@@ -105,47 +105,96 @@ def detect_supported_functions():
         return set(ALL_ACVP_FUNCTIONS)
 
 
+ENCAP_DECAP_FUNCTIONS = {
+    "encapsulation",
+    "encapsulationKeyCheck",
+    "decapsulation",
+    "decapsulationKeyCheck",
+}
+
+
 def loadDefaultAcvpData(version, supported_functions=None):
     if supported_functions is None:
         supported_functions = set(ALL_ACVP_FUNCTIONS)
 
-    # The ACVP encapDecap file bundles encapsulation, decapsulation, and both
-    # *KeyCheck helpers. To keep result comparison byte-identical against the
-    # published expectedResults.json, we only load the file if all four
-    # sub-functions are compiled in.
-    encapDecap_functions = {
-        "encapsulation",
-        "encapsulationKeyCheck",
-        "decapsulation",
-        "decapsulationKeyCheck",
-    }
-    encapDecap_supported = encapDecap_functions.issubset(supported_functions)
-    keyGen_supported = "keyGen" in supported_functions
+    # Load a prompt whenever any of its sub-functions is compiled in.
+    # Unsupported sub-functions are filtered out per test case (see
+    # filter_test_cases) so result comparison stays byte-identical.
+    keyGen_any = "keyGen" in supported_functions
+    encapDecap_any = bool(ENCAP_DECAP_FUNCTIONS & supported_functions)
 
     data_dir = f"test/acvp/.acvp-data/{version}/files"
     acvp_jsons_for_version = [
         (
             "keyGen",
-            keyGen_supported,
+            keyGen_any,
             f"{data_dir}/ML-KEM-keyGen-FIPS203/prompt.json",
             f"{data_dir}/ML-KEM-keyGen-FIPS203/expectedResults.json",
         ),
         (
             "encapDecap",
-            encapDecap_supported,
+            encapDecap_any,
             f"{data_dir}/ML-KEM-encapDecap-FIPS203/prompt.json",
             f"{data_dir}/ML-KEM-encapDecap-FIPS203/expectedResults.json",
         ),
     ]
     acvp_data = []
     for mode, is_supported, prompt, expectedResults in acvp_jsons_for_version:
         if not is_supported:
-            info(f"Skipping {mode} tests (mode not supported in this build)")
+            warn(
+                f"WARNING: Skipped {mode} ACVP prompt "
+                f"(no sub-function supported by this build)."
+            )
             continue
         acvp_data.append(loadAcvpData(prompt, expectedResults))
     return acvp_data
 
 
+def unwrap_acvts(data):
+    # ACVTS files wrap the payload as [{"acvVersion": ...}, {...}].
+    return data[1] if isinstance(data, list) else data
+
+
+def filter_test_cases(acvp_data, should_drop):
+    """Drop cases for which should_drop(tg, tc) returns a reason (None keeps).
+    Reasons are computed from the prompt but applied consistently to both
+    prompt and expected data, so downstream byte-identical result comparison
+    still works. Returns the list of drop reasons for reporting."""
+    reasons = []
+    for _, promptData, _, expectedData in acvp_data:
+        drop = {}
+        for tg in unwrap_acvts(promptData).get("testGroups", []):
+            for tc in tg["tests"]:
+                reason = should_drop(tg, tc)
+                if reason is not None:
+                    drop[tg["tgId"], tc["tcId"]] = reason
+        for data in (promptData, expectedData):
+            if data is None:
+                continue
+            for tg in unwrap_acvts(data).get("testGroups", []):
+                tg["tests"] = [
+                    tc for tc in tg["tests"] if (tg["tgId"], tc["tcId"]) not in drop
+                ]
+        reasons += drop.values()
+    return reasons
+
+
+def unsupported_function(supported_functions):
+    """Return a should_drop predicate that skips encapDecap test cases whose
+    tg['function'] is not in supported_functions. keyGen prompts have no
+    tg['function'] field, so they are never dropped by this predicate.
+    The predicate returns the bare function name so the caller can format
+    a compact 'Unsupported functions: A, B, C' summary."""
+
+    def should_drop(tg, tc):
+        fn = tg.get("function")
+        if fn is None or fn in supported_functions:
+            return None
+        return fn
+
+    return should_drop
+
+
 def err(msg, **kwargs):
     print(msg, file=sys.stderr, **kwargs)
 
@@ -154,6 +203,23 @@ def info(msg, **kwargs):
     print(msg, **kwargs)
 
 
+# Warnings deferred to the end of the run so they stand out after all the
+# per-test-case chatter. Collected by warn(), emitted once by flush_warnings().
+_warnings = []
+
+
+def warn(msg):
+    _warnings.append(msg)
+
+
+def flush_warnings():
+    if not _warnings:
+        return
+    err("")
+    for msg in _warnings:
+        err(msg)
+
+
 def get_acvp_binary(tg):
     """Convert JSON dict for ACVP test group to suitable ACVP binary."""
     parameterSetToLevel = {
@@ -374,11 +440,23 @@ def test(prompt, expected, output, version, supported_functions=None):
         data = loadDefaultAcvpData(version, supported_functions)
 
     if not data:
-        info("No ACVP tests supported by this build")
+        warn("WARNING: No ACVP tests supported by this build; no cases were run.")
         info("ALL GOOD!")
+        flush_warnings()
         return
 
+    # Filter out test cases whose ACVP function isn't compiled in.
+    if supported_functions is not None:
+        reasons = filter_test_cases(data, unsupported_function(supported_functions))
+        if reasons:
+            fns = ", ".join(sorted(set(reasons)))
+            warn(
+                f"WARNING: Dropped {len(reasons)} ACVP test case(s). "
+                f"Unsupported functions: {fns}."
+            )
+
     runTest(data, output)
+    flush_warnings()
 
 
 parser = argparse.ArgumentParser()
diff --git a/test/configs/break_pct_config.h b/test/configs/break_pct_config.h
@@ -151,7 +151,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/custom_heap_alloc_config.h b/test/configs/custom_heap_alloc_config.h
@@ -150,7 +150,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/custom_memcpy_config.h b/test/configs/custom_memcpy_config.h
@@ -150,7 +150,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/custom_memset_config.h b/test/configs/custom_memset_config.h
@@ -150,7 +150,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/custom_native_capability_config_0.h b/test/configs/custom_native_capability_config_0.h
@@ -151,7 +151,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/custom_native_capability_config_1.h b/test/configs/custom_native_capability_config_1.h
@@ -151,7 +151,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/custom_native_capability_config_CPUID_AVX2.h b/test/configs/custom_native_capability_config_CPUID_AVX2.h
@@ -151,7 +151,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/custom_native_capability_config_ID_AA64PFR1_EL1.h b/test/configs/custom_native_capability_config_ID_AA64PFR1_EL1.h
@@ -151,7 +151,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/custom_randombytes_config.h b/test/configs/custom_randombytes_config.h
@@ -150,7 +150,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/custom_stdlib_config.h b/test/configs/custom_stdlib_config.h
@@ -151,7 +151,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/custom_zeroize_config.h b/test/configs/custom_zeroize_config.h
@@ -150,7 +150,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/no_asm_config.h b/test/configs/no_asm_config.h
@@ -151,7 +151,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/serial_fips202_config.h b/test/configs/serial_fips202_config.h
@@ -150,7 +150,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *
diff --git a/test/configs/test_alloc_config.h b/test/configs/test_alloc_config.h
@@ -153,7 +153,6 @@
  */
 /* #define MLK_CONFIG_NO_DECAPS_API */
 
-
 /**
  * MLK_CONFIG_NO_RANDOMIZED_API
  *

Original file line number	Diff line number	Diff line change
`@@ -151,7 +151,6 @@`
`151`	`151`	`*/`
`152`	`152`	`/* #define MLK_CONFIG_NO_DECAPS_API */`
`153`	`153`
`154`		`-`
`155`	`154`	`/**`
`156`	`155`	`* MLK_CONFIG_NO_RANDOMIZED_API`
`157`	`156`	`*`
Original file line number	Diff line number	Diff line change
`@@ -152,7 +152,6 @@`
`152`	`152`	`*/`
`153`	`153`	`/* #define MLK_CONFIG_NO_DECAPS_API */`
`154`	`154`
`155`		`-`
`156`	`155`	`/**`
`157`	`156`	`* MLK_CONFIG_NO_RANDOMIZED_API`
`158`	`157`	`*`
Original file line number	Diff line number	Diff line change
`@@ -153,7 +153,6 @@`
`153`	`153`	`*/`
`154`	`154`	`/* #define MLK_CONFIG_NO_DECAPS_API */`
`155`	`155`
`156`		`-`
`157`	`156`	`/**`
`158`	`157`	`* MLK_CONFIG_NO_RANDOMIZED_API`
`159`	`158`	`*`
Original file line number	Diff line number	Diff line change
`@@ -150,7 +150,6 @@`
`150`	`150`	`*/`
`151`	`151`	`/* #define MLK_CONFIG_NO_DECAPS_API */`
`152`	`152`
`153`		`-`
`154`	`153`	`/**`
`155`	`154`	`* MLK_CONFIG_NO_RANDOMIZED_API`
`156`	`155`	`*`
Original file line number	Diff line number	Diff line change
`@@ -156,7 +156,6 @@`
`156`	`156`	`*/`
`157`	`157`	`/* #define MLK_CONFIG_NO_DECAPS_API */`
`158`	`158`
`159`		`-`
`160`	`159`	`/**`
`161`	`160`	`* MLK_CONFIG_NO_RANDOMIZED_API`
`162`	`161`	`*`