postprocess: Preprocess the C source code passed to transforms

The CommentsTransform can fail if presented with a C and Rust function
pair where some comments are inside directives that are compiled out
during preprocessing and thus should not be transferred to the Rust.

We can simply run the C compiler on the C snippet and it will pick up
compile commands (if any) and give us the preprocessed version back.

Author: thedataking

c2rust-postprocess/postprocess/transforms/base.py

@@ -8,7 +8,7 @@
     get_rust_definitions,
 )
 from postprocess.exclude_list import IdentifierExcludeList
-from postprocess.utils import get_highlighted_c
+from postprocess.utils import get_highlighted_c, get_preprocessed_c
 
 
 class AbstractTransform(ABC):
@@ -31,7 +31,7 @@ def apply_ident(
         c_definition: str,
         identifier: str,
         update_rust: bool = True,
-    ) -> None:
+    ) -> str:
         """
         Implementations should apply transform to a single Rust definition
         with the given identifier.
@@ -99,15 +99,17 @@ def apply_file(
 
             c_definition = c_definitions[identifier]
 
-            highlighted_c_definition = get_highlighted_c(c_definition)
+            preprocessed_c_definition = get_preprocessed_c(c_definition, identifier)
+
+            highlighted_c_definition = get_highlighted_c(preprocessed_c_definition)
             logging.debug(
                 f"C function {identifier} definition:\n{highlighted_c_definition}\n"
             )
 
             self.apply_ident(
                 rust_source_file=rust_source_file,
                 rust_definition=rust_definition,
-                c_definition=c_definition,
+                c_definition=preprocessed_c_definition,
                 identifier=identifier,
                 update_rust=update_rust,
             )

c2rust-postprocess/postprocess/transforms/comments.py

@@ -65,8 +65,7 @@ def apply_ident(
         if rust_comments:
             logging.info(
                 f"Skipping Rust fn {identifier} with existing comments:\
-                \n{rust_comments} in\
-                \n{rust_definition}"
+                \n{get_highlighted_rust(rust_definition)}"
             )
             return
 

c2rust-postprocess/postprocess/utils.py

@@ -1,8 +1,10 @@
 import argparse
 import json
+import logging
 import os
 from pathlib import Path
 from shutil import which
+import subprocess
 from typing import Any
 
 from pygments import highlight
@@ -116,3 +118,41 @@ def get_highlighted_rust(rust_code: str, bg="dark") -> str:
 def get_highlighted_code(code: str, lexer: RegexLexer, bg: str) -> str:
     # TODO: detect when terminal supports colors
     return highlight(code, lexer, TerminalFormatter(bg=bg))
+
+
+# Flags that resolve only #if/#elif/#else/#endif: -fdirectives-only leaves
+# #define/macro bodies untouched, -P drops line markers, -C keeps comments,
+# and -undef/-nostdinc avoid pulling in any predefined or system macros.
+CLANG_PREPROCESS_FLAGS = ["-E", "-fdirectives-only", "-P", "-C", "-undef", "-nostdinc"]
+
+
+def get_preprocessed_c(c_function: str, identifier: str) -> str:
+    """
+    Runs `clang -E -fdirectives-only -P -undef -nostdinc`, reading the
+    function from stdin and returning the preprocessed text; returns None
+    if clang fails.
+    """
+
+    # skip functions without any preprocessor directives
+    if "#" not in c_function or not any(
+        line.lstrip().startswith("#") for line in c_function.splitlines()
+    ):
+        return c_function
+
+    # TODO: check that current directory or parent has compile_commands.json
+
+    # NOTE: clang locates compile_commands.json automatically if it's in the
+    # current directory or any parent directory, no need to pass it explicitly
+    cmd = ["clang", *CLANG_PREPROCESS_FLAGS, "-x", "c", "-", "-o", "-"]
+    try:
+        result = subprocess.run(
+            cmd, input=c_function, capture_output=True, text=True, check=True
+        )
+    except subprocess.CalledProcessError as e:
+        logging.error(
+            "clang failed to preprocess %s: %s",
+            identifier,
+            e.stderr,
+        )
+        raise e
+    return result.stdout