Build the dep-graph reverse index lazily, per DepKind

When a SerializedDepGraph is decoded, it built a fingerprint to index
map for every DepKind covering every node. That inverse index is only
consulted by node_to_index_opt, which runs for the nodes a session
queries directly; the bulk of the graph is reached as edge targets by
index and is never looked up by fingerprint, so most of those maps are
never read.

Replace the eager build with a counting sort that groups node indices
into a contiguous range per DepKind, and build the fingerprint map for a
kind only the first time a node of that kind is looked up. Decode no
longer pays a hash-map insert per node, and kinds that are never looked
up never build a map. The on-disk format is unchanged.

Author: xmakro

compiler/rustc_middle/src/dep_graph/serialized.rs

@@ -41,8 +41,8 @@
 
 use std::cell::RefCell;
 use std::cmp::max;
-use std::sync::Arc;
 use std::sync::atomic::Ordering;
+use std::sync::{Arc, OnceLock};
 use std::{iter, mem};
 
 use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint};
@@ -113,14 +113,51 @@ pub struct SerializedDepGraph {
     /// A flattened list of all edge targets in the graph, stored in the same
     /// varint encoding that we use on disk. Edge sources are implicit in edge_list_indices.
     edge_list_data: Vec<u8>,
-    /// For each dep kind, stores a map from key fingerprints back to the index
-    /// of the corresponding node. This is the inverse of `nodes`.
-    index: Vec<UnhashMap<PackedFingerprint, SerializedDepNodeIndex>>,
+    /// The lazily-built inverse of `nodes`: maps a [`DepNode`] back to its
+    /// [`SerializedDepNodeIndex`] via the node's key fingerprint. See
+    /// [`LazyNodeIndex`].
+    index: LazyNodeIndex,
     /// The number of previous compilation sessions. This is used to generate
     /// unique anon dep nodes per session.
     session_count: u64,
 }
 
+/// The inverse of [`SerializedDepGraph::nodes`], built lazily per [`DepKind`].
+///
+/// The node indices are grouped by `DepKind` into contiguous ranges of
+/// `nodes_by_kind` when the graph is decoded (a cheap counting sort). The actual
+/// fingerprint -> index map for a given `DepKind` is only built the first time a
+/// node of that kind is looked up by fingerprint.
+///
+/// This matters for incremental performance: marking the graph green in an
+/// unchanged session reaches the overwhelming majority of nodes by walking
+/// edges, which reference nodes by index (see `try_mark_previous_green`), not by
+/// fingerprint. Only the comparatively few nodes that the session queries
+/// directly are ever looked up here, and those cluster into a handful of
+/// `DepKind`s. Building a map for every kind up front (the bulk being per-`DefId`
+/// foreign/metadata reads that are never looked up) is therefore mostly wasted
+/// work.
+#[derive(Debug, Default)]
+struct LazyNodeIndex {
+    /// All (non-`Null`) node indices, grouped into contiguous per-`DepKind`
+    /// ranges described by `kinds`.
+    nodes_by_kind: Vec<SerializedDepNodeIndex>,
+    /// For each `DepKind`, the range of `nodes_by_kind` holding its node indices
+    /// and the lazily-built fingerprint map over that range.
+    kinds: Vec<LazyKindIndex>,
+}
+
+#[derive(Debug, Default)]
+struct LazyKindIndex {
+    /// Offset into `LazyNodeIndex::nodes_by_kind` of this kind's first node.
+    start: u32,
+    /// Number of nodes of this kind.
+    len: u32,
+    /// `key_fingerprint -> node index`, built from this kind's range on first
+    /// lookup. Empty kinds (and kinds never looked up) never build a map.
+    map: OnceLock<UnhashMap<PackedFingerprint, SerializedDepNodeIndex>>,
+}
+
 impl SerializedDepGraph {
     #[inline]
     pub fn edge_targets_from(
@@ -151,7 +188,29 @@ impl SerializedDepGraph {
 
     #[inline]
     pub fn node_to_index_opt(&self, dep_node: &DepNode) -> Option<SerializedDepNodeIndex> {
-        self.index.get(dep_node.kind.as_usize())?.get(&dep_node.key_fingerprint).copied()
+        let kind = self.index.kinds.get(dep_node.kind.as_usize())?;
+        let map = kind.map.get_or_init(|| {
+            let range = (kind.start as usize)..(kind.start as usize + kind.len as usize);
+            let mut map = UnhashMap::with_capacity_and_hasher(kind.len as usize, Default::default());
+            for &idx in &self.index.nodes_by_kind[range] {
+                let node = &self.nodes[idx];
+                if map.insert(node.key_fingerprint, idx).is_some()
+                    // Side effect nodes can legitimately share a fingerprint.
+                    && node.kind != DepKind::SideEffect
+                {
+                    panic!(
+                        "Error: A dep graph node ({:?}) does not have an unique index. \
+                         Running a clean build on a nightly compiler with \
+                         `-Z incremental-verify-ich` can help narrow down the issue for reporting. \
+                         A clean build may also work around the issue.\n
+                         DepNode: {node:?}",
+                        node.kind
+                    )
+                }
+            }
+            map
+        });
+        map.get(&dep_node.key_fingerprint).copied()
     }
 
     #[inline]
@@ -286,28 +345,36 @@ impl SerializedDepGraph {
         // end of the array. This padding ensure it doesn't.
         edge_list_data.extend(&[0u8; DEP_NODE_PAD]);
 
-        // Read the number of each dep kind and use it to create an hash map with a suitable size.
-        let mut index: Vec<_> = (0..(DepKind::MAX + 1))
-            .map(|_| UnhashMap::with_capacity_and_hasher(d.read_u32() as usize, Default::default()))
-            .collect();
+        // Read the number of nodes of each dep kind. These per-kind counts let us
+        // group node indices by kind below; the fingerprint -> index maps
+        // themselves are built lazily, per kind, on first lookup (see
+        // `LazyNodeIndex`).
+        let kind_counts: Vec<u32> = (0..(DepKind::MAX + 1)).map(|_| d.read_u32()).collect();
 
         let session_count = d.read_u64();
 
+        // Counting sort: place every (non-`Null`) node index into a contiguous
+        // per-kind range of `nodes_by_kind`, using the per-kind counts to compute
+        // each kind's start offset. Padding gaps (`DepKind::Null`) are never
+        // looked up by fingerprint, so we skip them.
+        let mut kinds = Vec::with_capacity(kind_counts.len());
+        let mut offset = 0u32;
+        for &len in &kind_counts {
+            kinds.push(LazyKindIndex { start: offset, len, map: OnceLock::new() });
+            offset += len;
+        }
+        debug_assert_eq!(offset as usize, node_count);
+        let mut nodes_by_kind = vec![SerializedDepNodeIndex::from_u32(0); offset as usize];
+        let mut fill: Vec<u32> = kinds.iter().map(|k| k.start).collect();
         for (idx, node) in nodes.iter_enumerated() {
-            if index[node.kind.as_usize()].insert(node.key_fingerprint, idx).is_some() {
-                // Empty nodes and side effect nodes can have duplicates
-                if node.kind != DepKind::Null && node.kind != DepKind::SideEffect {
-                    let kind = node.kind;
-                    panic!(
-                        "Error: A dep graph node ({kind:?}) does not have an unique index. \
-                         Running a clean build on a nightly compiler with \
-                         `-Z incremental-verify-ich` can help narrow down the issue for reporting. \
-                         A clean build may also work around the issue.\n
-                         DepNode: {node:?}"
-                    )
-                }
+            if node.kind == DepKind::Null {
+                continue;
             }
+            let k = node.kind.as_usize();
+            nodes_by_kind[fill[k] as usize] = idx;
+            fill[k] += 1;
         }
+        let index = LazyNodeIndex { nodes_by_kind, kinds };
 
         Arc::new(SerializedDepGraph {
             nodes,