Add -fdebug-prefix-map and follow clang's behavior for debuginfo file/dir names (#5039)

This is required to have reproducible builds without remapping the source code directory to a fixed location (for example in Bazel, where the source code for every build action will end up in a sandbox directory with some hash in the path name).

Try to keep the implementation as close as possible to what clang does.

---------

Co-authored-by: Ilya Yanok <ilya.yanok@weka.io>
Co-authored-by: Martin Kinkelin <mkinkelin@symmetryinvestments.com>

Author: 3 people

CHANGELOG.md

@@ -15,6 +15,7 @@
   - `aarch64-linux-gnu-clang`
   - `clang --target=aarch64-linux-gnu`
 - The prebuilt arm64/universal macOS packages additionally bundle the arm64 iOS-*simulator* libraries, for out-of-the-box cross-compilation support via e.g. `-mtriple=arm64-apple-ios12.0-simulator`. (#4974)
+- New `--fdebug-prefix-map` command-line option and changed debuginfo file/directory name splitting logic (both now similar to clang), to aid reproducible builds. (#5039)
 
 #### Platform support
 - Supports LLVM 15 - 21.

driver/cl_options.cpp

@@ -223,6 +223,28 @@ cl::opt<bool> emitDwarfDebugInfo(
     "gdwarf", cl::ZeroOrMore,
     cl::desc("Emit DWARF debuginfo (instead of CodeView) for MSVC targets"));
 
+// Prefix map for filenames in DWARF debuginfo
+llvm::SmallVector<std::pair<std::string, std::string>, 0> debugPrefixMap;
+
+struct DwarfPrefixParser : public cl::parser<std::string> {
+  explicit DwarfPrefixParser(cl::Option &O) : cl::parser<std::string>(O) {}
+
+  bool parse(cl::Option &O, llvm::StringRef /*ArgName*/, llvm::StringRef Arg,
+             std::string & /*Val*/) {
+    auto [from, to] = Arg.split('=');
+    if (from.empty() || to.empty()) {
+      return O.error("invalid debug prefix map: " + Arg);
+    }
+    debugPrefixMap.emplace_back(from.str(), to.str());
+    return false;
+  }
+};
+
+static cl::opt<std::string, false, DwarfPrefixParser> fdebugPrefixMap(
+    "fdebug-prefix-map", cl::ZeroOrMore,
+    cl::desc("Prefix map for filenames in DWARF debuginfo"),
+    cl::value_desc("<old>=<new>"));
+
 cl::opt<bool> noAsm("noasm", cl::desc("Disallow use of inline assembler"),
                     cl::ZeroOrMore);
 

driver/cl_options.h

@@ -46,6 +46,7 @@ extern cl::list<std::string> runargs;
 extern cl::opt<bool> invokedByLDMD;
 extern cl::opt<bool> compileOnly;
 extern cl::opt<bool> emitDwarfDebugInfo;
+extern llvm::SmallVector<std::pair<std::string, std::string>, 0> debugPrefixMap; // Prefix map for filenames in DWARF debuginfo
 extern cl::opt<bool> noAsm;
 extern cl::opt<bool> dontWriteObj;
 extern cl::opt<std::string> objectFile;

gen/dibuilder.cpp

@@ -34,6 +34,7 @@
 #include "ir/irfuncty.h"
 #include "ir/irmodule.h"
 #include "ir/irtypeaggr.h"
+#include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
@@ -218,27 +219,71 @@ void DIBuilder::SetValue(Loc loc, llvm::Value *value,
                                    IR->scopebb());
 }
 
+std::string DIBuilder::remapDIPath(llvm::StringRef path) {
+  llvm::SmallString<256> P = path;
+  for (auto &[from, to] : llvm::reverse(opts::debugPrefixMap))
+    if (llvm::sys::path::replace_path_prefix(P, from, to))
+      break;
+  return P.str().str();
+}
+
 DIFile DIBuilder::CreateFile(const char *filename) {
   if (!filename)
     filename = IR->dmodule->srcfile.toChars();
 
-  // clang appears to use the curent working dir as 'directory' for relative
-  // source paths, and the root path for absolute ones:
-  // clang -g -emit-llvm -S ..\blub.c =>
-  //   !DIFile(filename: "..\\blub.c", directory: "C:\\LDC\\ninja-ldc", ...)
-  //   !DIFile(filename: "Program
-  //   Files\\LLVM\\lib\\clang\\11.0.1\\include\\stddef.h", directory: "C:\\",
-  //   ...)
-
-  if (llvm::sys::path::is_absolute(filename)) {
-    return DBuilder.createFile(llvm::sys::path::relative_path(filename),
-                               llvm::sys::path::root_path(filename));
+  // Mimic clang's behavior as much as possible, including fdebug-prefix-map
+  // remap behavior. See LLVM source clang/lib/CodeGen/CGDebugInfo.cpp
+
+  // First check cache
+  auto iter = filenameToDIFileCache.find(filename);
+  if (iter != filenameToDIFileCache.end()) {
+    // Verify that the information still exists.
+    if (llvm::Metadata *value = iter->second)
+      return llvm::cast<llvm::DIFile>(value);
   }
 
+  // Cache miss, create a new DIFile
+
+  auto remappedFile = remapDIPath(filename);
+
   llvm::SmallString<128> cwd;
   llvm::sys::fs::current_path(cwd);
+  auto currentDir = remapDIPath(cwd);
+
+  llvm::StringRef debuginfoDir;
+  llvm::StringRef debuginfoFile;
+
+  llvm::SmallString<128> DirBuf;
+  llvm::SmallString<128> FileBuf;
+  if (llvm::sys::path::is_absolute(remappedFile)) {
+    // Strip the common prefix (if it is more than just "/" or "C:\") from
+    // current directory and filename for a more space-efficient encoding.
+    auto FileIt = llvm::sys::path::begin(remappedFile);
+    auto FileE = llvm::sys::path::end(remappedFile);
+    auto CurDirIt = llvm::sys::path::begin(currentDir);
+    auto CurDirE = llvm::sys::path::end(currentDir);
+    for (; CurDirIt != CurDirE && *CurDirIt == *FileIt; ++CurDirIt, ++FileIt)
+      llvm::sys::path::append(DirBuf, *CurDirIt);
+    if (llvm::sys::path::root_path(DirBuf) == DirBuf) {
+      // Don't strip the common prefix if it is only the root ("/" or "C:\")
+      // since that would make diagnostic locations confusing.
+      debuginfoDir = {};
+      debuginfoFile = remappedFile;
+    } else {
+      for (; FileIt != FileE; ++FileIt)
+        llvm::sys::path::append(FileBuf, *FileIt);
+      debuginfoDir = DirBuf;
+      debuginfoFile = FileBuf;
+    }
+  } else {
+    if (!llvm::sys::path::is_absolute(filename))
+      debuginfoDir = currentDir;
+    debuginfoFile = remappedFile;
+  }
 
-  return DBuilder.createFile(filename, cwd);
+  auto difile = DBuilder.createFile(debuginfoFile, debuginfoDir);
+  filenameToDIFileCache[filename].reset(difile);
+  return difile;
 }
 
 DIFile DIBuilder::CreateFile(Loc loc) {

gen/dibuilder.h

@@ -67,6 +67,7 @@ class DIBuilder {
   const bool emitColumnInfo;
 
   llvm::DenseMap<Declaration*, llvm::TypedTrackingMDRef<llvm::MDNode>> StaticDataMemberCache;
+  llvm::DenseMap<const char *, llvm::TrackingMDRef> filenameToDIFileCache;
 
   DICompileUnit GetCU() {
     return CUNode;
@@ -163,6 +164,7 @@ class DIBuilder {
                  llvm::SmallVector<llvm::Metadata *, 16> &elems);
   void AddStaticMembers(AggregateDeclaration *sd, ldc::DIFile file,
                  llvm::SmallVector<llvm::Metadata *, 16> &elems);
+  std::string remapDIPath(llvm::StringRef path);
   DIFile CreateFile(const char *filename = nullptr);
   DIFile CreateFile(Loc loc);
   DIFile CreateFile(Dsymbol *decl);

tests/debuginfo/fdebug_prefix_map.d

@@ -0,0 +1,9 @@
+// RUN: %ldc -g -fdebug-prefix-map=%S=/blablabla -output-ll -of=%t.ll %s
+// RUN: FileCheck %s < %t.ll
+
+// Check that the substitution took place. Could be as `filename: "/blablabla...` or as `directory: "/blablabla...` depending on the invoke path for lit testsuite.
+// CHECK: !DIFile({{.*}}: "/blablabla
+
+void foo()
+{
+}