Fixing embedding script

Author: thirtytwobits

.vscode/settings.json

@@ -24,6 +24,7 @@
         "fsanitize",
         "ifndef",
         "invertibility",
+        "iwyu",
         "kirienko",
         "libudpard",
         "llms",

CONTRIBUTING.md

@@ -430,7 +430,7 @@ Behaviour worth knowing:
 - The image is built if it is missing, so `build` is only needed to pick up a dependency change.
 - `serve` mounts read-only and live-reloads on edit, exactly like a host `mkdocs serve`; the same
   `/llvm-dsdl/` base path applies.
-- `check` needs a writable mount because it renders into `site/`, and runs the container as your own
+- `check` needs a writeable mount because it renders into `site/`, and runs the container as your own
   UID/GID so the output is not left root-owned.
 - Overrides: `LLVMDSDL_DOCS_PORT` (default 8000), `LLVMDSDL_DOCS_IMAGE`, and `DOCKER` (set it to
   `podman` if that is what you run).

include/llvmdsdl/CodeGen/UavcanEmbeddedCatalog.h

@@ -66,6 +66,21 @@ struct UavcanEmbeddedCatalog final
 /// @return True when the embedded MLIR text matches its recorded SHA-256.
 [[nodiscard]] bool embeddedUavcanCatalogIntegrityOk();
 
+/// @brief Returns the SHA-256 recorded next to the catalog blob, as lowercase hex.
+[[nodiscard]] llvm::StringRef embeddedUavcanCatalogRecordedSha256();
+
+/// @brief Hashes the catalog blob compiled into this binary, as lowercase hex.
+///
+/// Computed from the text rather than read from the recorded constant, so a failing integrity check
+/// can name both halves of the mismatch.
+[[nodiscard]] std::string embeddedUavcanCatalogComputedSha256();
+
+/// @brief Repository-relative path of the generated file carrying the catalog text and its hash.
+///
+/// Named in the integrity-failure diagnostic: both values live in this one generated file, so a
+/// mismatch is always a defect in it rather than in the two sources disagreeing.
+inline constexpr const char* kEmbeddedUavcanCatalogSourceFile = "lib/CodeGen/UavcanEmbeddedMlir.inc";
+
 /// @brief Loads and parses embedded UAVCAN catalog artifacts.
 ///
 /// Fails (without parsing) when the embedded MLIR blob does not match its recorded SHA-256.

lib/CodeGen/UavcanEmbeddedCatalog.cpp

@@ -677,15 +677,42 @@ bool embeddedUavcanCatalogIntegrityOk()
                                           uavcan_embedded_mlir::kEmbeddedUavcanMlirSha256);
 }
 
+llvm::StringRef embeddedUavcanCatalogRecordedSha256()
+{
+    return uavcan_embedded_mlir::kEmbeddedUavcanMlirSha256;
+}
+
+std::string embeddedUavcanCatalogComputedSha256()
+{
+    const llvm::StringRef        text = uavcan_embedded_mlir::kEmbeddedUavcanMlirText;
+    const std::array<uint8_t, 32> digest =
+        llvm::SHA256::hash(llvm::ArrayRef<uint8_t>(reinterpret_cast<const uint8_t*>(text.data()), text.size()));
+    return llvm::toHex(digest, /*LowerCase=*/true);
+}
+
 llvm::Expected<UavcanEmbeddedCatalog> loadUavcanEmbeddedCatalog(mlir::MLIRContext& context,
                                                                 DiagnosticEngine&  diagnostics)
 {
     // Integrity gate: refuse to parse a blob that does not match its recorded SHA-256 (corruption,
     // tampering, or a text/hash mismatch introduced during the build).
     if (!embeddedUavcanCatalogIntegrityOk())
     {
-        diagnostics.error({"<embedded-uavcan>", 1, 1},
-                          "embedded UAVCAN catalog failed SHA-256 integrity verification");
+        const SourceLocation here{"<embedded-uavcan>", 1, 1};
+        diagnostics.error(here, "embedded UAVCAN catalog failed SHA-256 integrity verification");
+        diagnostics.note(here,
+                         "recorded " + embeddedUavcanCatalogRecordedSha256().str() + ", computed " +
+                             embeddedUavcanCatalogComputedSha256());
+        diagnostics.note(here,
+                         std::string("both values are generated into ") + kEmbeddedUavcanCatalogSourceFile +
+                             ", so they disagree only when that file was hand-edited, partially regenerated, "
+                             "or merged badly");
+        diagnostics.note(here,
+                         std::string("restore it with 'git checkout -- ") + kEmbeddedUavcanCatalogSourceFile +
+                             "', or regenerate it with 'python3 tools/dsdlc/generate_embedded_uavcan_mlir.py "
+                             "--dsdlc <path-to-dsdlc>', then rebuild");
+        diagnostics.note(here,
+                         "to build without the compiled-in catalog, pass --no-embedded-uavcan and add "
+                         "--lookup-dir <checkout>/public_regulated_data_types/uavcan");
         return llvm::createStringError(llvm::inconvertibleErrorCode(),
                                        "embedded UAVCAN catalog failed SHA-256 integrity verification");
     }

lib/CodeGen/UavcanEmbeddedMlir.inc

@@ -66,6 +66,15 @@ bool runUavcanEmbeddedCatalogTests()
         std::cerr << "verifyEmbeddedCatalogIntegrity accepted a mismatched hash\n";
         return false;
     }
+    // (4) The two accessors the integrity-failure diagnostic quotes must agree on a healthy binary,
+    //     and must be the same pair the check itself compares.
+    if (llvmdsdl::embeddedUavcanCatalogComputedSha256() != llvmdsdl::embeddedUavcanCatalogRecordedSha256())
+    {
+        std::cerr << "embedded UAVCAN catalog hash accessors disagree: computed "
+                  << llvmdsdl::embeddedUavcanCatalogComputedSha256() << ", recorded "
+                  << llvmdsdl::embeddedUavcanCatalogRecordedSha256().str() << "\n";
+        return false;
+    }
 
     mlir::DialectRegistry registry;
     registry.insert<mlir::dsdl::DSDLDialect,

test/unit/UavcanEmbeddedCatalogTests.cpp

@@ -71,7 +71,10 @@ def _find_default_dsdlc(repo_root: Path) -> str:
 
 
 def _run_dsdlc(dsdlc: str, uavcan_root: Path) -> str:
-    command = [dsdlc, "--target-language", "mlir", str(uavcan_root)]
+    # --no-embedded-uavcan keeps this hermetic: the catalog is derived from uavcan_root alone, never
+    # from the copy compiled into the dsdlc doing the deriving. It also means a dsdlc whose embedded
+    # catalog is corrupt -- the one case where regeneration is the fix -- can still run this.
+    command = [dsdlc, "--target-language", "mlir", "--no-embedded-uavcan", str(uavcan_root)]
     process = subprocess.run(command, capture_output=True, text=True, check=False)
     if process.returncode != 0:
         sys.stderr.write(process.stderr)
@@ -88,7 +91,7 @@ def _run_dsdlc(dsdlc: str, uavcan_root: Path) -> str:
 
 
 def _render(mlir_text: str) -> str:
-    schema_count = len(re.findall(r"\\bdsdl\\.schema\\b", mlir_text))
+    schema_count = len(re.findall(r"\bdsdl\.schema\b", mlir_text))
     # The raw-string literal embeds this exact content (a leading newline keeps the generated file
     # readable). Hash the same bytes so the runtime SHA-256 check verifies what is actually embedded,
     # not a differently-normalized copy.

tools/dsdlc/generate_embedded_uavcan_mlir.py

@@ -1403,7 +1403,17 @@ int main(int argc, char** argv)
         auto loadedCatalog = llvmdsdl::loadUavcanEmbeddedCatalog(context, diagnostics);
         if (!loadedCatalog)
         {
-            llvm::errs() << llvm::toString(loadedCatalog.takeError()) << "\n";
+            // Every failure path in the loader files a diagnostic first, and that one carries the
+            // remediation notes. Printing the Error's text too would only repeat its first line.
+            auto loadError = loadedCatalog.takeError();
+            if (diagnostics.hasErrors())
+            {
+                llvm::consumeError(std::move(loadError));
+            }
+            else
+            {
+                llvm::errs() << llvm::toString(std::move(loadError)) << "\n";
+            }
             printDiagnostics(diagnostics);
             return 1;
         }