Make create_def a query.

Author: cjgillot

compiler/rustc_hir/src/definitions.rs

@@ -11,7 +11,7 @@ use rustc_data_structures::fx::FxHashMap;
 use rustc_data_structures::stable_hasher::StableHasher;
 use rustc_hashes::Hash64;
 use rustc_index::IndexVec;
-use rustc_macros::{BlobDecodable, Decodable, Encodable, extension};
+use rustc_macros::{BlobDecodable, Decodable, Encodable, HashStable_Generic, extension};
 use rustc_span::def_id::LocalDefIdMap;
 use rustc_span::{Symbol, kw, sym};
 use tracing::{debug, instrument};
@@ -277,7 +277,7 @@ impl DefPath {
 }
 
 /// New variants should only be added in synchronization with `enum DefKind`.
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Encodable, BlobDecodable)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Encodable, BlobDecodable, HashStable_Generic)]
 pub enum DefPathData {
     // Root: these should only be used for the root nodes, because
     // they are treated specially by the `def_path` function.

compiler/rustc_middle/src/dep_graph/dep_node.rs

@@ -51,14 +51,15 @@ use std::fmt;
 use std::hash::Hash;
 
 use rustc_data_structures::fingerprint::{Fingerprint, PackedFingerprint};
-use rustc_data_structures::stable_hasher::{StableHasher, StableOrd, ToStableHashKey};
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher, StableOrd, ToStableHashKey};
 use rustc_hir::def_id::DefId;
 use rustc_hir::definitions::DefPathHash;
 use rustc_macros::{Decodable, Encodable, HashStable};
 use rustc_span::Symbol;
 
 use super::{KeyFingerprintStyle, SerializedDepNodeIndex};
 use crate::dep_graph::DepNodeKey;
+use crate::ich::StableHashingContext;
 use crate::mono::MonoItem;
 use crate::ty::{TyCtxt, tls};
 
@@ -152,6 +153,14 @@ impl fmt::Debug for DepNode {
     }
 }
 
+impl HashStable<StableHashingContext<'_>> for DepNode {
+    fn hash_stable(&self, hcx: &mut StableHashingContext<'_>, hasher: &mut StableHasher) {
+        let DepNode { kind, key_fingerprint } = self;
+        kind.hash_stable(hcx, hasher);
+        Fingerprint::from(*key_fingerprint).hash_stable(hcx, hasher);
+    }
+}
+
 /// This struct stores function pointers and other metadata for a particular DepKind.
 ///
 /// Information is retrieved by indexing the `DEP_KINDS` array using the integer value
@@ -269,7 +278,7 @@ macro_rules! define_dep_nodes {
         // encoding. The derived Encodable/Decodable uses leb128 encoding which is
         // dense when only considering this enum. But DepKind is encoded in a larger
         // struct, and there we can take advantage of the unused bits in the u16.
-        #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+        #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, HashStable)]
         #[allow(non_camel_case_types)]
         #[repr(u16)] // Must be kept in sync with the rest of `DepKind`.
         pub enum DepKind {

compiler/rustc_middle/src/dep_graph/graph.rs

@@ -335,11 +335,7 @@ impl DepGraphData {
         let (result, edges) = if tcx.is_eval_always(dep_node.kind) {
             (with_deps(TaskDepsRef::EvalAlways), EdgesVec::new())
         } else {
-            let task_deps = Lock::new(TaskDeps::new(
-                #[cfg(debug_assertions)]
-                Some(dep_node),
-                0,
-            ));
+            let task_deps = Lock::new(TaskDeps::new(Some(dep_node), 0));
             (with_deps(TaskDepsRef::Allow(&task_deps)), task_deps.into_inner().reads)
         };
 
@@ -372,11 +368,7 @@ impl DepGraphData {
 
         // Large numbers of reads are common enough here that pre-sizing `read_set`
         // to 128 actually helps perf on some benchmarks.
-        let task_deps = Lock::new(TaskDeps::new(
-            #[cfg(debug_assertions)]
-            None,
-            128,
-        ));
+        let task_deps = Lock::new(TaskDeps::new(None, 128));
         let result = with_deps(TaskDepsRef::Allow(&task_deps), op);
         let task_deps = task_deps.into_inner();
         let reads = task_deps.reads;
@@ -486,7 +478,7 @@ impl DepGraph {
 
                     #[cfg(debug_assertions)]
                     {
-                        if let Some(target) = task_deps.node
+                        if let Some(target) = task_deps.current_dep_node
                             && let Some(ref forbidden_edge) = data.current.forbidden_edge
                         {
                             let src = forbidden_edge.index_to_node.lock()[&dep_node_index];
@@ -1220,8 +1212,10 @@ pub enum TaskDepsRef<'a> {
 
 #[derive(Debug)]
 pub struct TaskDeps {
-    #[cfg(debug_assertions)]
-    node: Option<DepNode>,
+    current_dep_node: Option<DepNode>,
+
+    /// Counter inside this query.
+    local_index: usize,
 
     /// A vector of `DepNodeIndex`, basically.
     reads: EdgesVec,
@@ -1238,16 +1232,25 @@ impl TaskDeps {
     const LINEAR_SCAN_MAX: usize = 16;
 
     #[inline]
-    fn new(#[cfg(debug_assertions)] node: Option<DepNode>, read_set_capacity: usize) -> Self {
+    fn new(current_dep_node: Option<DepNode>, read_set_capacity: usize) -> Self {
         TaskDeps {
-            #[cfg(debug_assertions)]
-            node,
+            current_dep_node,
+            local_index: 0,
             reads: EdgesVec::new(),
             read_set: FxHashSet::with_capacity_and_hasher(read_set_capacity, Default::default()),
         }
     }
 }
 
+impl TaskDeps {
+    pub fn next_query_local_index(&mut self) -> Option<(DepNode, usize)> {
+        let node = self.current_dep_node?;
+        let index = self.local_index;
+        self.local_index = index + 1;
+        Some((node, index))
+    }
+}
+
 // A data structure that stores Option<DepNodeColor> values as a contiguous
 // array, using one u32 per entry.
 pub(super) struct DepNodeColorMap {

compiler/rustc_middle/src/queries.rs

@@ -67,6 +67,7 @@ use rustc_hir::def::{DefKind, DocLinkResMap};
 use rustc_hir::def_id::{
     CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap, LocalDefIdSet, LocalModDefId,
 };
+use rustc_hir::definitions::DefPathData;
 use rustc_hir::lang_items::{LangItem, LanguageItems};
 use rustc_hir::{ItemLocalId, ItemLocalMap, PreciseCapturingArgKind, TraitCandidate};
 use rustc_index::IndexVec;
@@ -82,6 +83,7 @@ use rustc_span::def_id::LOCAL_CRATE;
 use rustc_span::{DUMMY_SP, LocalExpnId, Span, Spanned, Symbol};
 use rustc_target::spec::PanicStrategy;
 
+use crate::dep_graph::DepNode;
 use crate::hir::Crate;
 use crate::infer::canonical::{self, Canonical};
 use crate::lint::LintExpectation;
@@ -198,6 +200,23 @@ rustc_queries! {
         desc { "getting the source span" }
     }
 
+    /// Create a new definition.
+    query create_def_raw(key: (
+        LocalDefId, // parent
+        DefPathData, // def_path_data
+        Option<DepNode>, // caller query
+        usize, // counter of calls to `create_def_raw` by the caller query
+    )) -> LocalDefId {
+        // Accesses untracked data
+        eval_always
+        desc {
+            "create a new definition for `{}::{:?}`, {}th call",
+            tcx.def_path_str(key.0),
+            key.1,
+            key.3,
+        }
+    }
+
     /// Represents crate as a whole (as distinct from the top-level crate module).
     ///
     /// If you call `tcx.hir_crate(())` we will have to assume that any change

compiler/rustc_middle/src/query/erase.rs

@@ -214,6 +214,7 @@ impl_erasable_for_types_with_no_type_params! {
     rustc_hir::OpaqueTyOrigin<rustc_hir::def_id::DefId>,
     rustc_hir::def::DefKind,
     rustc_hir::def_id::DefId,
+    rustc_hir::def_id::LocalDefId,
     rustc_middle::middle::codegen_fn_attrs::SanitizerFnAttrs,
     rustc_middle::middle::resolve_bound_vars::ObjectLifetimeDefault,
     rustc_middle::mir::ConstQualifs,

compiler/rustc_middle/src/query/keys.rs

@@ -7,10 +7,11 @@ use std::hash::Hash;
 use rustc_ast::tokenstream::TokenStream;
 use rustc_data_structures::stable_hasher::HashStable;
 use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE, LocalDefId, LocalModDefId};
+use rustc_hir::definitions::DefPathData;
 use rustc_hir::hir_id::OwnerId;
 use rustc_span::{DUMMY_SP, Ident, LocalExpnId, Span, Symbol};
 
-use crate::dep_graph::DepNodeIndex;
+use crate::dep_graph::{DepNode, DepNodeIndex};
 use crate::ich::StableHashingContext;
 use crate::infer::canonical::CanonicalQueryInput;
 use crate::mono::CollectionMode;
@@ -361,3 +362,11 @@ impl<'tcx> QueryKey for (ty::Instance<'tcx>, CollectionMode) {
         self.0.default_span(tcx)
     }
 }
+
+impl QueryKey for (LocalDefId, DefPathData, Option<DepNode>, usize) {
+    type Cache<V> = DefaultCache<Self, V>;
+
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}

compiler/rustc_middle/src/ty/context.rs

@@ -59,7 +59,7 @@ use tracing::{debug, instrument};
 
 use crate::arena::Arena;
 use crate::dep_graph::dep_node::make_metadata;
-use crate::dep_graph::{DepGraph, DepKindVTable, DepNodeIndex};
+use crate::dep_graph::{DepGraph, DepKindVTable, DepNode, TaskDepsRef};
 use crate::ich::StableHashingContext;
 use crate::infer::canonical::{CanonicalParamEnvCache, CanonicalVarKind};
 use crate::lint::emit_lint_base;
@@ -1428,6 +1428,34 @@ impl<'tcx> TyCtxt<'tcx> {
     }
 }
 
+#[instrument(level = "trace", skip(tcx), ret)]
+fn create_def_raw_provider<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    (parent, data, query, index): (LocalDefId, DefPathData, Option<DepNode>, usize),
+) -> LocalDefId {
+    // `query` and `index` are guaranteed to change for each successive call to
+    // `create_def_raw`, but in a predictable manner.
+    let _ = (query, index);
+
+    // This query is `eval_always`, so we can access untracked data.
+    let mut disambiguator_state = tcx.untracked_disambiguator_state.lock();
+
+    // The following call has the side effect of modifying the tables inside `definitions`.
+    // These very tables are relied on by the incr. comp. engine to decode DepNodes and to
+    // decode the on-disk cache.
+    //
+    // Any LocalDefId which is used within queries, either as key or result, either:
+    // - has been created before the construction of the TyCtxt;
+    // - has been created by this call to `create_def`.
+    // As a consequence, this LocalDefId is always re-created before it is needed by the incr.
+    // comp. engine itself.
+    tcx.untracked.definitions.write().create_def(
+        parent,
+        data,
+        disambiguator_state.get_or_create(parent),
+    )
+}
+
 impl<'tcx> TyCtxtAt<'tcx> {
     /// Create a new definition within the incr. comp. engine.
     pub fn create_def(
@@ -1437,9 +1465,7 @@ impl<'tcx> TyCtxtAt<'tcx> {
         def_kind: DefKind,
         override_def_path_data: Option<DefPathData>,
     ) -> TyCtxtFeed<'tcx, LocalDefId> {
-        let feed =
-            self.tcx.create_def(parent, name, def_kind, override_def_path_data);
-
+        let feed = self.tcx.create_def(parent, name, def_kind, override_def_path_data);
         feed.def_span(self.span);
         feed
     }
@@ -1455,28 +1481,36 @@ impl<'tcx> TyCtxt<'tcx> {
         override_def_path_data: Option<DefPathData>,
     ) -> TyCtxtFeed<'tcx, LocalDefId> {
         let data = override_def_path_data.unwrap_or_else(|| def_kind.def_path_data(name));
-        let mut disambiguators = self.untracked_disambiguator_state.lock();
-        let disambiguator = disambiguators.get_or_create(parent);
-
-        // The following call has the side effect of modifying the tables inside `definitions`.
-        // These very tables are relied on by the incr. comp. engine to decode DepNodes and to
-        // decode the on-disk cache.
-        //
-        // Any LocalDefId which is used within queries, either as key or result, either:
-        // - has been created before the construction of the TyCtxt;
-        // - has been created by this call to `create_def`.
-        // As a consequence, this LocalDefId is always re-created before it is needed by the incr.
-        // comp. engine itself.
-        let def_id = self.untracked.definitions.write().create_def(parent, data, disambiguator);
-
-        // This function modifies `self.definitions` using a side-effect.
-        // We need to ensure that these side effects are re-run by the incr. comp. engine.
-        // Depending on the forever-red node will tell the graph that the calling query
-        // needs to be re-evaluated.
-        self.dep_graph.read_index(DepNodeIndex::FOREVER_RED_NODE);
+
+        // `create_def_raw` is a query, so it can be replayed by the dep-graph engine. However, we
+        // may invoke it multiple times with the same `(parent, data)` pair, and we expect to
+        // create *different* defintions from them. In order to make this compatible with the
+        // general model of queries, we add additional information which must change at each call.
+        let (dep_node, query_local_index) = ty::tls::with_context(|icx| match icx.task_deps {
+            // If we are inside a query, we can only use local information, and no global
+            // mutable state. The current query's name and the number of calls to `create_def`
+            // are local to the current query, so are ok to use.
+            TaskDepsRef::Allow(deps) => {
+                let opt_dep_node_and_index = deps.lock().next_query_local_index();
+                if let Some((dep_node, index)) = opt_dep_node_and_index {
+                    (Some(dep_node), index)
+                } else {
+                    // No idea how to support this for now...
+                    bug!("trying to create a definition from an anonymous query")
+                }
+            }
+            // If we are not tracking dependencies, we can use global mutable state,
+            // so we use the total number of definitions as a proxy.
+            TaskDepsRef::EvalAlways | TaskDepsRef::Forbid | TaskDepsRef::Ignore => {
+                let global_count = self.untracked.definitions.read().def_index_count();
+                (None, global_count)
+            }
+        });
+        let def_id = self.create_def_raw((parent, data, dep_node, query_local_index));
 
         let feed = TyCtxtFeed { tcx: self, key: def_id };
         feed.def_kind(def_kind);
+
         // Unique types created for closures participate in type privacy checking.
         // They have visibilities inherited from the module they are defined in.
         // Visibilities for opaque types are meaningless, but still provided
@@ -2914,4 +2948,5 @@ pub fn provide(providers: &mut Providers) {
         tcx.lang_items().panic_impl().is_some_and(|did| did.is_local())
     };
     providers.source_span = |tcx, def_id| tcx.untracked.source_span.get(def_id).unwrap_or(DUMMY_SP);
+    providers.create_def_raw = create_def_raw_provider;
 }