Merge pull request #278 from gilles-peskine-arm/generate_mldsa_tests-create

Support committed generated test data and generate PQCP test data

Author: valeriosetti

scripts/mbedtls_framework/config_macros.py

@@ -6,9 +6,10 @@
 import glob
 import os
 import re
-from typing import FrozenSet, Iterable, Iterator
+from typing import FrozenSet, Iterable, Iterator, List
 
 from . import build_tree
+from . import generate_files_helper
 
 
 class ConfigMacros:
@@ -34,7 +35,7 @@ def _load_file(filename: str) -> FrozenSet[str]:
                              for line in input_)
 
 
-class Current(ConfigMacros):
+class Current(ConfigMacros, generate_files_helper.Generator):
     """Information about config-like macros parsed from the source code."""
 
     _SHADOW_FILE = 'scripts/data_files/config-options-current.txt'
@@ -136,6 +137,25 @@ def update_shadow_file(self, always_update: bool) -> None:
             for name in sorted(self.live_config_options()):
                 out.write(name + '\n')
 
+    # Implement the generate_files_helper.Generator interface
+    def generator_name(self) -> str:
+        """Name as a generate_files_helper.Generator."""
+        return 'options'
+
+    def target_files(self) -> List[str]:
+        """List the (single) generated file name."""
+        return [os.path.join(self._submodule, self._SHADOW_FILE)]
+
+    def outdated_files(self) -> List[str]:
+        """List the (single) generated file name if it is out of date."""
+        if self.is_shadow_file_up_to_date():
+            return []
+        else:
+            return self.target_files()
+
+    def update(self, always: bool) -> None:
+        """Update the shadow file from the live config file."""
+        self.update_shadow_file(always)
 
 
 class History(ConfigMacros):

scripts/mbedtls_framework/generate_files_helper.py

@@ -0,0 +1,182 @@
+"""Utilities for intermediate files that are generated, but platform-independent
+and configuration-independent.
+"""
+
+# Copyright The Mbed TLS Contributors
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+
+import argparse
+import os
+import subprocess
+import sys
+from typing import Dict, Iterable, List, Sequence, Set
+
+
+class Generator:
+    """An abstract base class for generators of intermediate files."""
+
+    def generator_name(self) -> str:
+        """A name for this generator.
+
+        Generator names must be unique and should not be identical to
+        the name of any target.
+        """
+        raise NotImplementedError
+
+    def target_files(self) -> List[str]:
+        """The list of files targeted by this generator.
+
+        File names are relative to the project root.
+        """
+        raise NotImplementedError
+
+    def outdated_files(self) -> Iterable[str]:
+        """Return the list of targets that are out of date.
+
+        This is empty after running update().
+        Missing targets are considered out of date.
+        """
+        raise NotImplementedError
+
+    def update(self, always: bool) -> None:
+        """Update the target(s) of this generator.
+
+        If always is false, avoid changing the output file if it already has
+        the desired content. If always is true, make sure to update the
+        time stamp on the output file even if it already has the desired content.
+        """
+        raise NotImplementedError
+
+
+class TestDataGenerator(Generator):
+    """A test data generator script.
+
+    Even though the test data generator scripts are written in Python, we
+    run them as a separate process, because their output depends on the
+    program name (they write sys.argv[0] in a comment in the .data file).
+    """
+
+    def __init__(self, script: str) -> None:
+        """Run the specified test generator to generate files.
+
+        Assume that the script is written in Python and has the command line
+        interface of test_data_generation.py.
+        """
+        self.script = script
+
+    def generator_name(self) -> str:
+        return os.path.basename(self.script)
+
+    def target_files(self) -> List[str]:
+        output = subprocess.check_output([sys.executable, self.script, '--list'],
+                                         encoding='utf-8')
+        return output.splitlines()
+
+    def outdated_files(self) -> List[str]:
+        output = subprocess.check_output([sys.executable, self.script, '--list-outdated'],
+                                         encoding='utf-8')
+        return output.splitlines()
+
+    def update(self, _always) -> None:
+        subprocess.check_call([sys.executable, self.script])
+
+
+def assemble(available: Iterable[Generator]) -> Dict[str, Generator]:
+    """Assemble the generators into a dictionary with both names and targets as keys."""
+    by_ident = {} #type: Dict[str, Generator]
+    for generator in available:
+        ident = generator.generator_name()
+        if ident in by_ident:
+            raise Exception(f'Generator conflict: name "{ident}" of {generator} '
+                            f'already recorded for {by_ident[ident]}')
+        by_ident[ident] = generator
+        for ident in generator.target_files():
+            if ident in by_ident:
+                raise Exception(f'Generator conflict: target "{ident}" of {generator} '
+                                f'already recorded for {by_ident[ident]}')
+            by_ident[ident] = generator
+    return by_ident
+
+def list_names(available: Iterable[Generator]) -> List[str]:
+    """Return the list of generator names."""
+    return sorted(generator.generator_name() for generator in available)
+
+def list_targets(available: Iterable[Generator]) -> List[str]:
+    """Return the list of generator targets."""
+    return sorted(target
+                  for generator in available
+                  for target in generator.target_files())
+
+def select(available: Dict[str, Generator],
+           wanted: Iterable[str]) -> List[Generator]:
+    """Select generators by name or target."""
+    wanted_names = set() #type: Set[str]
+    for ident in wanted:
+        if ident not in available:
+            raise Exception(f'No generator found for {ident}')
+        wanted_names.add(ident)
+    return [available[name] for name in sorted(wanted_names)]
+
+def main(generators: Sequence[Generator],
+         description: str) -> None:
+    #pylint: disable=too-many-branches
+    """Command line entry point.
+    """
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('--always-update', '-U',
+                        action='store_true',
+                        help=('Update target files unconditionally '
+                              '(overrides --update)'))
+    parser.add_argument('--list',
+                        action='store_true',
+                        help='List generator names and targets and exit')
+    parser.add_argument('--list-names',
+                        action='store_true',
+                        help='List generator names and exit')
+    parser.add_argument('--list-targets',
+                        action='store_true',
+                        help='List generator targets and exit')
+    parser.add_argument('--update', '-u',
+                        action='store_true',
+                        help='Update target files if needed')
+    parser.add_argument('--verbose', '-v',
+                        action='store_true',
+                        help='Be more verbose')
+    parser.add_argument('idents', nargs='*', metavar='NAME|TARGET',
+                        help='List of generator names or targets (all targets if empty)')
+    args = parser.parse_args()
+
+    if args.list:
+        args.list_names = True
+        args.list_targets = True
+    if args.list_names:
+        for name in list_names(generators):
+            print(name)
+    if args.list_targets:
+        for target in list_targets(generators):
+            print(target)
+    if args.list_names or args.list_targets:
+        return
+
+    if args.idents:
+        available = assemble(generators)
+        wanted = select(available, args.idents) #type: Sequence[Generator]
+    else:
+        wanted = generators
+    if args.update or args.always_update:
+        for generator in wanted:
+            if args.verbose:
+                sys.stderr.write(f'Running generator {generator.generator_name()}...\n')
+            generator.update(args.always_update)
+    else:
+        outdated = [] #type: List[str]
+        for generator in wanted:
+            if args.verbose:
+                sys.stderr.write(f'Checking targets of generator {generator.generator_name()}...\n')
+            outdated += generator.outdated_files()
+        if outdated:
+            sys.stderr.write(f'Some targets are missing or out of date.\n')
+            for target in outdated:
+                print(target)
+            sys.stderr.write(f'Run {sys.argv[0]} -u and commit the result.')
+            sys.exit(1)

scripts/mbedtls_framework/test_case.py

@@ -127,22 +127,28 @@ def write(self, out: typing_util.Writable) -> None:
         out.write(prefix + self.function + ':' +
                   ':'.join(self.arguments) + '\n')
 
+def write_data_stream(out,
+                      test_cases: Iterable[TestCase],
+                      caller: Optional[str] = None) -> None:
+    """Write the test cases to the specified output stream."""
+    if caller is None:
+        caller = os.path.basename(sys.argv[0])
+    out.write('# Automatically generated by {}. Do not edit!\n'
+              .format(caller))
+    for tc in test_cases:
+        tc.write(out)
+    out.write('\n# End of automatically generated file.\n')
+
 def write_data_file(filename: str,
                     test_cases: Iterable[TestCase],
                     caller: Optional[str] = None) -> None:
     """Write the test cases to the specified file.
 
     If the file already exists, it is overwritten.
     """
-    if caller is None:
-        caller = os.path.basename(sys.argv[0])
     tempfile = filename + '.new'
     with open(tempfile, 'w') as out:
-        out.write('# Automatically generated by {}. Do not edit!\n'
-                  .format(caller))
-        for tc in test_cases:
-            tc.write(out)
-        out.write('\n# End of automatically generated file.\n')
+        write_data_stream(out, test_cases, caller)
     os.replace(tempfile, filename)
 
 def psa_or_3_6_feature_macro(psa_name: str,

scripts/mbedtls_framework/test_data_generation.py

@@ -11,6 +11,7 @@
 #
 
 import argparse
+import io
 import os
 import posixpath
 import re
@@ -139,6 +140,11 @@ def generate_tests(cls) -> Iterator[test_case.TestCase]:
 
 class TestGenerator:
     """Generate test cases and write to data files."""
+
+    # Note that targets whose names contain 'test_format' have their content
+    # validated by `abi_check.py`.
+    targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]]
+
     def __init__(self, options) -> None:
         self.test_suite_directory = options.directory
         # Update `targets` with an entry for each child class of BaseTarget.
@@ -163,10 +169,6 @@ def write_test_data_file(self, basename: str,
         filename = self.filename_for(basename)
         test_case.write_data_file(filename, test_cases)
 
-    # Note that targets whose names contain 'test_format' have their content
-    # validated by `abi_check.py`.
-    targets = {} # type: Dict[str, Callable[..., Iterable[test_case.TestCase]]]
-
     def generate_target(self, name: str, *target_args) -> None:
         """Generate cases and write to data file for a target.
 
@@ -176,13 +178,32 @@ def generate_target(self, name: str, *target_args) -> None:
         test_cases = self.targets[name](*target_args)
         self.write_test_data_file(name, test_cases)
 
+    def is_up_to_date(self, target) -> bool:
+        """Check if the given target already has the expected content."""
+        filename = self.filename_for(target)
+        if not os.path.exists(filename):
+            return False
+        test_cases = self.targets[target]()
+        out = io.StringIO()
+        test_case.write_data_stream(out, test_cases)
+        out.seek(0)
+        new_content = out.read()
+        out.close()
+        with open(filename) as current_file:
+            old_content = current_file.read()
+        return new_content == old_content
+
+
 def main(args, description: str, generator_class: Type[TestGenerator] = TestGenerator):
     """Command line entry point."""
     parser = argparse.ArgumentParser(description=description)
     parser.add_argument('--list', action='store_true',
                         help='List available targets and exit')
     parser.add_argument('--list-for-cmake', action='store_true',
                         help='Print \';\'-separated list of available targets and exit')
+    parser.add_argument('--list-outdated', action='store_true',
+                        help=('List outdated targets and exit '
+                              '(succeeds even if there are outdated or missing targets)'))
     # If specified explicitly, this option may be a path relative to the
     # current directory when the script is invoked. The default value
     # is relative to the mbedtls root, which we don't know yet. So we
@@ -221,4 +242,8 @@ def main(args, description: str, generator_class: Type[TestGenerator] = TestGene
     else:
         options.targets = sorted(generator.targets)
     for target in options.targets:
-        generator.generate_target(target)
+        if options.list_outdated:
+            if not generator.is_up_to_date(target):
+                print(generator.filename_for(target))
+        else:
+            generator.generate_target(target)