Upload crash info to Sentry

.github/workflows/build.yml

@@ -38,6 +38,12 @@ on:
         required: false
       manual_netlify_site_id:
         required: false
+      sentry_auth_token:
+        required: false
+      sentry_org:
+        required: false
+      sentry_project:
+        required: false
 
 jobs:
   build:
@@ -52,6 +58,14 @@ jobs:
       - uses: DeterminateSystems/flakehub-cache-action@main
       - run: nix build .#packages.${{ inputs.system }}.default .#packages.${{ inputs.system }}.binaryTarball --no-link -L
       - run: nix build .#packages.${{ inputs.system }}.binaryTarball --out-link tarball
+      - run: nix build .#^debug,out
+      - name: Upload debug info to Sentry
+        run: ./maintainers/upload-debug-info-to-sentry.py --debug-dir ./result-debug ./result/bin/nix
+        if: env.SENTRY_AUTH_TOKEN != ''
+        env:
+          SENTRY_AUTH_TOKEN: ${{ secrets.sentry_auth_token }}
+          SENTRY_ORG: ${{ secrets.sentry_org }}
+          SENTRY_PROJECT: ${{ secrets.sentry_project }}
       - uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.system }}

.github/workflows/ci.yml

@@ -44,6 +44,9 @@ jobs:
     secrets:
       manual_netlify_auth_token: ${{ secrets.NETLIFY_AUTH_TOKEN }}
       manual_netlify_site_id: ${{ secrets.NETLIFY_SITE_ID }}
+      sentry_auth_token: ${{ secrets.SENTRY_AUTH_TOKEN }}
+      sentry_org: ${{ secrets.SENTRY_ORG }}
+      sentry_project: ${{ secrets.SENTRY_PROJECT }}
 
   build_aarch64-linux:
     uses: ./.github/workflows/build.yml
@@ -62,6 +65,10 @@ jobs:
       runner: UbuntuLatest32Cores128GArm
       runner_for_virt: UbuntuLatest32Cores128GArm
       runner_small: UbuntuLatest32Cores128GArm
+    secrets:
+      sentry_auth_token: ${{ secrets.SENTRY_AUTH_TOKEN }}
+      sentry_org: ${{ secrets.SENTRY_ORG }}
+      sentry_project: ${{ secrets.SENTRY_PROJECT }}
 
   build_aarch64-darwin:
     uses: ./.github/workflows/build.yml
@@ -70,6 +77,10 @@ jobs:
       runner: namespace-profile-mac-m2-12c28g
       runner_for_virt: namespace-profile-mac-m2-12c28g
       runner_small: macos-latest-xlarge
+    secrets:
+      sentry_auth_token: ${{ secrets.SENTRY_AUTH_TOKEN }}
+      sentry_org: ${{ secrets.SENTRY_ORG }}
+      sentry_project: ${{ secrets.SENTRY_PROJECT }}
 
   success:
     runs-on: ubuntu-latest

doc/manual/generate-manpage.nix

@@ -67,24 +67,30 @@ let
           `${command}` [*option*...] ${arguments}
         '';
 
-      maybeSubcommands = optionalString (details ? commands && details.commands != { }) ''
+      filteredCommands = filterAttrs (n: v: v.category.id != 103) (details.commands or { });
+
+      maybeSubcommands = optionalString (filteredCommands != { }) ''
         where *subcommand* is one of the following:
 
         ${subcommands}
       '';
 
-      subcommands = if length categories > 1 then listCategories else listSubcommands details.commands;
+      subcommands = if length categories > 1 then listCategories else listSubcommands filteredCommands;
 
       categories = sort (x: y: x.id < y.id) (
-        unique (map (cmd: { inherit (cmd.category) id description; }) (attrValues details.commands))
+        unique (
+          map (cmd: { inherit (cmd.category) id description; }) (
+            attrValues filteredCommands
+          )
+        )
       );
 
       listCategories = concatStrings (map showCategory categories);
 
       showCategory = cat: ''
         **${toString cat.description}:**
 
-        ${listSubcommands (filterAttrs (n: v: v.category.id == cat.id) details.commands)}
+        ${listSubcommands (filterAttrs (n: v: v.category.id == cat.id) filteredCommands)}
       '';
 
       listSubcommands = cmds: concatStrings (attrValues (mapAttrs showSubcommand cmds));
@@ -202,6 +208,8 @@ let
             ;
         };
       };
+      filteredCommands = filterAttrs (n: v: v.category.id != 103) (details.commands or { });
+
       subcommand =
         subCmd:
         processCommand {
@@ -211,7 +219,7 @@ let
           inherit toplevel;
         };
     in
-    [ cmd ] ++ concatMap subcommand (attrNames details.commands or { });
+    [ cmd ] ++ concatMap subcommand (attrNames filteredCommands);
 
   manpages = processCommand {
     command = "nix";

maintainers/upload-debug-info-to-sentry.py

@@ -0,0 +1,161 @@
+#!/usr/bin/env nix
+#!nix shell --inputs-from . nixpkgs#sentry-cli --command python3
+
+import argparse
+import json
+import os
+import platform
+import re
+import subprocess
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+NAR_DIR = "/tmp/nars"
+DEBUG_INFO_DIR = "/tmp/debug-info"
+
+
+def get_dynamic_libraries(executable: str) -> list[str]:
+    if platform.system() == "Darwin":
+        result = subprocess.run(["otool", "-L", executable], capture_output=True, text=True, check=True)
+        libs = []
+        for line in result.stdout.splitlines()[1:]:  # skip first line (the binary path itself)
+            # otool -L output lines look like:
+            #   /nix/store/.../libfoo.dylib (compatibility version X.Y.Z, current version A.B.C)
+            m = re.match(r"\s+(\S+)\s+\(", line)
+            if m:
+                libs.append(m.group(1))
+        return libs
+    else:
+        result = subprocess.run(["ldd", executable], capture_output=True, text=True, check=True)
+        libs = []
+        for line in result.stdout.splitlines():
+            # ldd output lines look like:
+            #   libfoo.so.1 => /nix/store/.../libfoo.so.1 (0x...)
+            #   /lib64/ld-linux-x86-64.so.2 (0x...)
+            m = re.search(r"=> (/\S+)", line)
+            if m:
+                libs.append(m.group(1))
+            elif line.strip().startswith("/"):
+                path = line.strip().split()[0]
+                libs.append(path)
+        return libs
+
+
+def get_build_id(path: str) -> str | None:
+    result = subprocess.run(["readelf", "-n", path], capture_output=True, text=True)
+    m = re.search(r"Build ID:\s+([0-9a-f]+)", result.stdout)
+    return m.group(1) if m else None
+
+
+def download_nar(build_id: str, archive: str) -> str:
+    """Download a NAR to /tmp/nars and return the local path. Skips if already present."""
+    base_url = f"https://cache.nixos.org/debuginfo/{build_id}"
+    nar_url = urllib.parse.urljoin(base_url, archive)
+    filename = nar_url.split("/")[-1]
+    local_path = os.path.join(NAR_DIR, filename)
+    if not os.path.exists(local_path):
+        os.makedirs(NAR_DIR, exist_ok=True)
+        print(f"    downloading {nar_url} ...", file=sys.stderr)
+        urllib.request.urlretrieve(nar_url, local_path)
+    else:
+        print(f"    already have {filename}", file=sys.stderr)
+    return local_path
+
+
+def extract_debug_symbols(nar_path: str, member: str, build_id: str) -> str:
+    """Extract a member from a .nar.xz into /tmp/debug-info/<build_id>.debug. Returns the output path."""
+    out_path = os.path.join(DEBUG_INFO_DIR, f"{build_id}.debug")
+    if os.path.exists(out_path):
+        print(f"    already extracted {out_path}", file=sys.stderr)
+        return out_path
+    os.makedirs(DEBUG_INFO_DIR, exist_ok=True)
+    print(f"    extracting {member} -> {out_path} ...", file=sys.stderr)
+    xz = subprocess.Popen(["xz", "-d"], stdin=open(nar_path, "rb"), stdout=subprocess.PIPE)
+    nar_cat = subprocess.run(
+        ["nix", "nar", "cat", "/dev/stdin", member],
+        stdin=xz.stdout,
+        capture_output=True,
+        check=True,
+    )
+    xz.wait()
+    with open(out_path, "wb") as f:
+        f.write(nar_cat.stdout)
+    return out_path
+
+
+def find_debug_file_in_dirs(build_id: str, debug_dirs: list[str]) -> str | None:
+    """Look for a .debug file by build ID under <dir>/lib/debug/.build-id/NN/NNN.debug."""
+    subpath = os.path.join("lib", "debug", ".build-id", build_id[:2], build_id[2:] + ".debug")
+    for d in debug_dirs:
+        candidate = os.path.join(d, subpath)
+        if os.path.exists(candidate):
+            return candidate
+    return None
+
+
+def fetch_debuginfo(build_id: str) -> dict | None:
+    url = f"https://cache.nixos.org/debuginfo/{build_id}"
+    try:
+        with urllib.request.urlopen(url) as resp:
+            return json.loads(resp.read())
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return None
+        raise
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Upload debug symbols to Sentry."
+    )
+    parser.add_argument("executable", help="Path to the executable (e.g. ./result/bin/nix)")
+    parser.add_argument("--project", help="Sentry project ID")
+    parser.add_argument("--debug-dir", action="append", default=[], metavar="DIR",
+                        help="Directory to search for debug files (may be repeated, Linux only)")
+    args = parser.parse_args()
+
+    libs = [args.executable] + get_dynamic_libraries(args.executable)
+
+    if platform.system() == "Darwin":
+        # On macOS there are no separate debug info files; upload the binaries directly.
+        print("Files to upload:", file=sys.stderr)
+        for lib in libs:
+            print(f"  {lib}", file=sys.stderr)
+        files_to_upload = libs
+    else:
+        debug_files = []
+        print("ELF files to process:", file=sys.stderr)
+        for lib in libs:
+            build_id = get_build_id(lib)
+            if build_id is None:
+                print(f"  {lib} (no build ID)", file=sys.stderr)
+                continue
+
+            local = find_debug_file_in_dirs(build_id, args.debug_dir)
+            if local:
+                print(f"  {lib} ({build_id}): found locally at {local}", file=sys.stderr)
+                debug_files.append(local)
+                continue
+
+            debuginfo = fetch_debuginfo(build_id)
+            if debuginfo is None:
+                print(f"  {lib} ({build_id}, no debug info in cache)", file=sys.stderr)
+                continue
+            print(f"  {lib} ({build_id}): member={debuginfo['member']}", file=sys.stderr)
+            nar_path = download_nar(build_id, debuginfo["archive"])
+            debug_file = extract_debug_symbols(nar_path, debuginfo["member"], build_id)
+            debug_files.append(debug_file)
+        files_to_upload = debug_files
+
+    if files_to_upload:
+        print(f"Uploading {len(files_to_upload)} file(s) to Sentry...", file=sys.stderr)
+        cmd = ["sentry-cli", "debug-files", "upload"]
+        if args.project:
+            cmd += ["--project", args.project]
+        subprocess.run(cmd + files_to_upload, check=True)
+
+
+if __name__ == "__main__":
+    main()

packaging/dependencies.nix

@@ -147,4 +147,9 @@ scope: {
     pslSupport = !stdenv.hostPlatform.isStatic;
     idnSupport = !stdenv.hostPlatform.isStatic;
   };
+
+  sentry-native = (pkgs.callPackage ./sentry-native.nix { }).override {
+    # Avoid having two curls in our closure.
+    inherit (scope) curl;
+  };
 }

packaging/dev-shell.nix

@@ -290,7 +290,8 @@ pkgs.nixComponents2.nix-util.overrideAttrs (
       map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents2.nix-perl-bindings.mesonFlags)
     )
     ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents2.nix-expr.mesonFlags)
-    ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags);
+    ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents2.nix-cmd.mesonFlags)
+    ++ map (transformFlag "nix") (ignoreCrossFile pkgs.nixComponents2.nix-cli.mesonFlags);
 
     nativeBuildInputs =
       let

packaging/everything.nix

@@ -44,6 +44,10 @@
   testers,
 
   patchedSrc ? null,
+
+  curl,
+  boehmgc,
+  sentry-native,
 }:
 
 let
@@ -102,6 +106,7 @@ stdenv.mkDerivation (finalAttrs: {
     "dev"
     "doc"
     "man"
+    "debug"
   ];
 
   /**
@@ -153,9 +158,18 @@ stdenv.mkDerivation (finalAttrs: {
   installPhase =
     let
       devPaths = lib.mapAttrsToList (_k: lib.getDev) finalAttrs.finalPackage.libs;
+      debugPaths = lib.map (lib.getOutput "debug") (
+        lib.attrValues finalAttrs.finalPackage.libs
+        ++ [
+          nix-cli
+          curl
+          boehmgc
+        ]
+        ++ lib.optional (stdenv.hostPlatform.isLinux && !stdenv.hostPlatform.isStatic) sentry-native
+      );
     in
     ''
-      mkdir -p $out $dev/nix-support
+      mkdir -p $out $dev/nix-support $debug/lib/debug
 
       # Custom files
       echo $libs >> $dev/nix-support/propagated-build-inputs
@@ -168,6 +182,12 @@ stdenv.mkDerivation (finalAttrs: {
         lndir $lib $dev
       done
 
+      for d in ${lib.escapeShellArgs debugPaths}; do
+        if [[ -d $d/lib/debug ]]; then
+          lndir $d/lib/debug $debug/lib/debug
+        fi
+      done
+
       # Forwarded outputs
       ln -sT ${nix-manual} $doc
       ln -sT ${nix-manual.man} $man

packaging/sentry-native.nix

@@ -0,0 +1,52 @@
+{
+  lib,
+  stdenv,
+  fetchgit,
+  cmake,
+  curl,
+  pkg-config,
+  python3,
+  darwin,
+}:
+
+stdenv.mkDerivation rec {
+  pname = "sentry-native";
+  version = "0.13.5";
+
+  src = fetchgit {
+    url = "https://github.com/getsentry/sentry-native";
+    tag = version;
+    hash = "sha256-vDBI6lB1DMLleAgRCfsHvTSdtmXOzvJSaNAt+NwOd3c=";
+    fetchSubmodules = true;
+  };
+
+  dontFixCmake = true;
+
+  nativeBuildInputs = [
+    cmake
+    pkg-config
+  ]
+  ++ lib.optionals stdenv.hostPlatform.isDarwin [
+    python3
+    darwin.bootstrap_cmds
+  ];
+
+  postPatch = ''
+    # Borrowed from psutil: stick to the old SDK name for now.
+    substituteInPlace external/crashpad/util/mac/mac_util.cc \
+      --replace-fail kIOMainPortDefault kIOMasterPortDefault
+  '';
+
+  buildInputs = [
+    curl
+  ];
+
+  cmakeBuildType = "RelWithDebInfo";
+
+  cmakeFlags = [ ];
+
+  outputs = [
+    "out"
+    "dev"
+  ];
+}