moar checkpointing

Daniel-B-Smith · Daniel-B-Smith · commit 1fe48e6b1ade · 2026-04-08T10:22:28.000-04:00
diff --git a/Cargo.lock b/Cargo.lock
@@ -4288,7 +4288,6 @@ dependencies = [
  "rustc_macros",
  "rustc_middle",
  "rustc_proc_macro",
- "rustc_query_system",
  "rustc_serialize",
  "rustc_session",
  "rustc_span",
diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs
@@ -951,8 +951,13 @@ pub fn create_and_enter_global_ctxt<T, F: for<'tcx> FnOnce(TyCtxt<'tcx>) -> T>(
     let definitions = FreezeLock::new(Definitions::new(stable_crate_id));
 
     let stable_crate_ids = FreezeLock::new(StableCrateIdMap::default());
-    let untracked =
-        Untracked { cstore, source_span: AppendOnlyIndexVec::new(), definitions, stable_crate_ids };
+    let untracked = Untracked {
+        cstore,
+        source_span: AppendOnlyIndexVec::new(),
+        definitions,
+        stable_crate_ids,
+        local_crate_hash: OnceLock::new(),
+    };
 
     // We're constructing the HIR here; we don't care what we will
     // read, since we haven't even constructed the *input* to
diff --git a/compiler/rustc_metadata/Cargo.toml b/compiler/rustc_metadata/Cargo.toml
@@ -25,7 +25,6 @@ rustc_middle = { path = "../rustc_middle" }
 # We must use the proc_macro version that we will compile proc-macros against,
 # not the one from our own sysroot.
 rustc_proc_macro = { path = "../rustc_proc_macro" }
-rustc_query_system = { path = "../rustc_query_system" }
 rustc_serialize = { path = "../rustc_serialize" }
 rustc_session = { path = "../rustc_session" }
 rustc_span = { path = "../rustc_span" }
diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs
@@ -740,4 +740,101 @@ fn provide_cstore_hooks(providers: &mut Providers) {
             cdata.imported_source_file(tcx, file_index as u32);
         }
     };
+
+    providers.queries.crate_hash = |tcx: TyCtxt<'_>, _: LocalCrate| {
+        *tcx.untracked()
+            .local_crate_hash
+            .get()
+            .expect("crate_hash(LOCAL_CRATE) called before metadata encoding")
+    };
 }
+
+/*pub(super) fn crate_hash(tcx: TyCtxt<'_>, cnum: rustc_hir::def_id::CrateNum) -> Svh {
+    let cstore = CStore::from_tcx(tcx);
+    let crate_data = cstore.get_crate_data(cnum);
+    crate_data.root.header.hash
+
+    let upstream_crates = upstream_crates(tcx);
+
+    let resolutions = tcx.resolutions(());
+
+    // We hash the final, remapped names of all local source files so we
+    // don't have to include the path prefix remapping commandline args.
+    // If we included the full mapping in the SVH, we could only have
+    // reproducible builds by compiling from the same directory. So we just
+    // hash the result of the mapping instead of the mapping itself.
+    let mut source_file_names: Vec<_> = tcx
+        .sess
+        .source_map()
+        .files()
+        .iter()
+        .filter(|source_file| source_file.cnum == LOCAL_CRATE)
+        .map(|source_file| source_file.stable_id)
+        .collect();
+
+    source_file_names.sort_unstable();
+
+    // We have to take care of debugger visualizers explicitly. The HIR (and
+    // thus `hir_body_hash`) contains the #[debugger_visualizer] attributes but
+    // these attributes only store the file path to the visualizer file, not
+    // their content. Yet that content is exported into crate metadata, so any
+    // changes to it need to be reflected in the crate hash.
+    let debugger_visualizers: Vec<_> = tcx
+        .debugger_visualizers(LOCAL_CRATE)
+        .iter()
+        // We ignore the path to the visualizer file since it's not going to be
+        // encoded in crate metadata and we already hash the full contents of
+        // the file.
+        .map(DebuggerVisualizerFile::path_erased)
+        .collect();
+
+    let crate_hash: Fingerprint = tcx.with_stable_hashing_context(|mut hcx| {
+        let mut stable_hasher = StableHasher::new();
+        metadata_hash.hash_stable(&mut hcx, &mut stable_hasher);
+        upstream_crates.hash_stable(&mut hcx, &mut stable_hasher);
+        source_file_names.hash_stable(&mut hcx, &mut stable_hasher);
+        debugger_visualizers.hash_stable(&mut hcx, &mut stable_hasher);
+        if tcx.sess.opts.incremental.is_some() {
+            let definitions = tcx.untracked().definitions.freeze();
+            let mut owner_spans: Vec<_> = tcx
+                .hir_crate_items(())
+                .definitions()
+                .map(|def_id| {
+                    let def_path_hash = definitions.def_path_hash(def_id);
+                    let span = tcx.source_span(def_id);
+                    debug_assert_eq!(span.parent(), None);
+                    (def_path_hash, span)
+                })
+                .collect();
+            owner_spans.sort_unstable_by_key(|bn| bn.0);
+            owner_spans.hash_stable(&mut hcx, &mut stable_hasher);
+        }
+        tcx.sess.opts.dep_tracking_hash(true).hash_stable(&mut hcx, &mut stable_hasher);
+        tcx.stable_crate_id(LOCAL_CRATE).hash_stable(&mut hcx, &mut stable_hasher);
+        // Hash visibility information since it does not appear in HIR.
+        // FIXME: Figure out how to remove `visibilities_for_hashing` by hashing visibilities on
+        // the fly in the resolver, storing only their accumulated hash in `ResolverGlobalCtxt`,
+        // and combining it with other hashes here.
+        resolutions.visibilities_for_hashing.hash_stable(&mut hcx, &mut stable_hasher);
+        with_metavar_spans(|mspans| {
+            mspans.freeze_and_get_read_spans().hash_stable(&mut hcx, &mut stable_hasher);
+        });
+        stable_hasher.finish()
+    });
+
+    Svh::new(crate_hash)
+}
+
+fn upstream_crates(tcx: TyCtxt<'_>) -> Vec<(StableCrateId, Svh)> {
+    let mut upstream_crates: Vec<_> = tcx
+        .crates(())
+        .iter()
+        .map(|&cnum| {
+            let stable_crate_id = tcx.stable_crate_id(cnum);
+            let hash = tcx.crate_hash(cnum);
+            (stable_crate_id, hash)
+        })
+        .collect();
+    upstream_crates.sort_unstable_by_key(|&(stable_crate_id, _)| stable_crate_id);
+    upstream_crates
+}*/
diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs
@@ -5,10 +5,12 @@ use std::io::{Read, Seek, Write};
 use std::path::{Path, PathBuf};
 use std::sync::Arc;
 
+use rustc_data_structures::fingerprint::Fingerprint;
+//use rustc_data_structures::Svh;
 use rustc_data_structures::fx::{FxIndexMap, FxIndexSet};
 use rustc_data_structures::memmap::{Mmap, MmapMut};
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
-use rustc_data_structures::sync::{join, par_for_each_in};
+use rustc_data_structures::sync::{par_for_each_in, par_join};
 use rustc_data_structures::temp_dir::MaybeTempDir;
 use rustc_data_structures::thousands::usize_with_underscores;
 use rustc_feature::Features;
@@ -19,6 +21,7 @@ use rustc_hir::definitions::DefPathData;
 use rustc_hir::find_attr;
 use rustc_hir_pretty::id_to_string;
 use rustc_middle::dep_graph::WorkProductId;
+use rustc_middle::ich::StableHashingContext;
 use rustc_middle::middle::dependency_format::Linkage;
 use rustc_middle::mir::interpret;
 use rustc_middle::query::Providers;
@@ -27,7 +30,6 @@ use rustc_middle::ty::AssocContainer;
 use rustc_middle::ty::codec::TyEncoder;
 use rustc_middle::ty::fast_reject::{self, TreatParams};
 use rustc_middle::{bug, span_bug};
-use rustc_query_system::ich::StableHashingContext;
 use rustc_serialize::{Decodable, Decoder, Encodable, Encoder, opaque};
 use rustc_session::config::{CrateType, OptLevel, TargetModifier};
 use rustc_span::hygiene::HygieneEncodeContext;
@@ -42,7 +44,7 @@ use crate::errors::{FailCreateFileEncoder, FailWriteFile};
 use crate::rmeta::*;
 
 // Struct to enable split borrows.
-struct ContextEncoder<'a> {
+pub(super) struct ContextEncoder<'a> {
     opaque: opaque::FileEncoder,
     stable_hasher: StableHasher,
     hcx: StableHashingContext<'a>,
@@ -120,9 +122,17 @@ impl<'a> Encoder for ContextEncoder<'a> {
 
 impl<'a> ContextEncoder<'a> {
     #[inline]
-    fn position(&self) -> usize {
+    pub(super) fn position(&self) -> usize {
         self.opaque.position()
     }
+
+    pub(super) fn write_m_with<const N: usize>(&mut self, b: &[u8; N], m: usize) {
+        (b[..m]).hash_stable(&mut self.hcx, &mut self.stable_hasher);
+        self.opaque.write_with(|dest| {
+            *dest = *b;
+            m
+        });
+    }
 }
 
 macro_rules! encoder_methods {
@@ -194,11 +204,6 @@ impl<'a, 'tcx> SpanEncoder for EncodeContext<'a, 'tcx> {
     }
 
     fn encode_def_id(&mut self, def_id: DefId) {
-        HashStable::<StableHashingContext<'_>>::hash_stable(
-            &def_id,
-            &mut self.encoder.hcx,
-            &mut self.encoder.stable_hasher,
-        );
         def_id.krate.encode(self);
         def_id.index.encode(self);
     }
@@ -231,17 +236,15 @@ impl<'a, 'tcx> SpanEncoder for EncodeContext<'a, 'tcx> {
                 if offset < last_location {
                     let needed = bytes_needed(offset);
                     SpanTag::indirect(true, needed as u8).encode(self);
-                    self.encoder.opaque.write_with(|dest| {
-                        *dest = offset.to_le_bytes();
-                        needed
-                    });
+                    self.encoder.write_m_with(&offset.to_le_bytes(), needed);
+                    offset.hash_stable(&mut self.encoder.hcx, &mut self.encoder.stable_hasher);
                 } else {
                     let needed = bytes_needed(last_location);
                     SpanTag::indirect(false, needed as u8).encode(self);
-                    self.encoder.opaque.write_with(|dest| {
-                        *dest = last_location.to_le_bytes();
-                        needed
-                    });
+                    self.encoder.write_m_with(&last_location.to_le_bytes(), needed);
+
+                    last_location
+                        .hash_stable(&mut self.encoder.hcx, &mut self.encoder.stable_hasher);
                 }
             }
             Entry::Vacant(v) => {
@@ -648,7 +651,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             adapted.set_some(on_disk_index, self.lazy(adapted_source_file));
         }
 
-        adapted.encode(&mut self.encoder.opaque)
+        adapted.encode(&mut self.encoder)
     }
 
     fn encode_crate_root(&mut self) -> LazyValue<CrateRoot> {
@@ -732,7 +735,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         // encode_def_path_table.
         let proc_macro_data = stat!("proc-macro-data", || self.encode_proc_macros());
 
-        let tables = stat!("tables", || self.tables.encode(&mut self.encoder.opaque));
+        let tables = stat!("tables", || self.tables.encode(&mut self.encoder));
 
         let debugger_visualizers =
             stat!("debugger-visualizers", || self.encode_debugger_visualizers());
@@ -768,11 +771,26 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
 
         let root = stat!("final", || {
             let attrs = tcx.hir_krate_attrs();
+            let new_hash = Svh::new(self.encoder.stable_hasher.clone().finish());
+
+            /*eprintln!("crate: {:?}", tcx.crate_name(LOCAL_CRATE));
+            eprintln!("crate HASH: {:?}", new_hash);
+            if let Some(hash) = tcx.untracked().local_crate_hash.get() {
+                eprintln!("resetting hash: {:?}", hash);
+            }*/
+
+            tcx.untracked().local_crate_hash.set(new_hash).expect("local_crate_hash set twice");
+
+            /*let old_hash = tcx.crate_hash(new_hash);
+            eprintln!("OLD HASH: {:?}", old_hash);
+            eprintln!("NEW HASH: {:?}", new_hash);
+            assert_eq!(old_hash, new_hash, "Hash mismatch!");*/
+
             self.lazy(CrateRoot {
                 header: CrateHeader {
                     name: tcx.crate_name(LOCAL_CRATE),
                     triple: tcx.sess.opts.target_triple.clone(),
-                    hash: tcx.crate_hash(LOCAL_CRATE),
+                    hash: new_hash,
                     is_proc_macro_crate: proc_macro_data.is_some(),
                     is_stub: false,
                 },
@@ -2018,9 +2036,9 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
         );
 
         (
-            syntax_contexts.encode(&mut self.encoder.opaque),
-            expn_data_table.encode(&mut self.encoder.opaque),
-            expn_hash_table.encode(&mut self.encoder.opaque),
+            syntax_contexts.encode(&mut self.encoder),
+            expn_data_table.encode(&mut self.encoder),
+            expn_hash_table.encode(&mut self.encoder),
         )
     }
 
@@ -2475,10 +2493,25 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path, ref_path: Option<&Path>) {
         let _prof_timer = tcx.prof.verbose_generic_activity("generate_crate_metadata_stub");
 
         with_encode_metadata_header(tcx, ref_path, |ecx| {
+            //let new_hash = Svh::new(ecx.encoder.stable_hasher.clone().finish());
+
+            /*eprintln!("crate: {:?}", tcx.crate_name(LOCAL_CRATE));
+            eprintln!("encoding hash HASH: {:?}", new_hash);
+            if let Some(hash) = tcx.untracked().local_crate_hash.get() {
+                eprintln!("resetting hash: {:?}", hash);
+            }
+
+
+            tcx.untracked().local_crate_hash.set(new_hash).expect("local_crate_hash set twice");*/
+
             let header: LazyValue<CrateHeader> = ecx.lazy(CrateHeader {
                 name: tcx.crate_name(LOCAL_CRATE),
                 triple: tcx.sess.opts.target_triple.clone(),
-                hash: tcx.crate_hash(LOCAL_CRATE),
+                hash: *tcx
+                    .untracked()
+                    .local_crate_hash
+                    .get()
+                    .expect("The hash should have been calculated during metadataencoding"),
                 is_proc_macro_crate: false,
                 is_stub: true,
             });
diff --git a/compiler/rustc_metadata/src/rmeta/mod.rs b/compiler/rustc_metadata/src/rmeta/mod.rs
@@ -35,7 +35,6 @@ use rustc_middle::mir::ConstValue;
 use rustc_middle::ty::fast_reject::SimplifiedType;
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_middle::util::Providers;
-use rustc_serialize::opaque::FileEncoder;
 use rustc_session::config::{SymbolManglingVersion, TargetModifier};
 use rustc_session::cstore::{CrateDepKind, ForeignModule, LinkagePreference, NativeLib};
 use rustc_span::edition::Edition;
@@ -46,6 +45,7 @@ use table::TableBuilder;
 
 use crate::creader::CrateMetadataRef;
 use crate::eii::EiiMapEncodedKeyValue;
+use crate::rmeta::encoder::ContextEncoder;
 
 mod decoder;
 mod def_path_hash_map;
@@ -367,7 +367,7 @@ macro_rules! define_tables {
         }
 
         impl TableBuilders {
-            fn encode(&self, buf: &mut FileEncoder) -> LazyTables {
+            fn encode(&self, buf: &mut ContextEncoder<'_>) -> LazyTables {
                 LazyTables {
                     $($name1: self.$name1.encode(buf),)+
                     $($name2: self.$name2.encode(buf),)+
diff --git a/compiler/rustc_metadata/src/rmeta/table.rs b/compiler/rustc_metadata/src/rmeta/table.rs
@@ -2,6 +2,7 @@ use rustc_hir::def::CtorOf;
 use rustc_index::Idx;
 
 use crate::rmeta::decoder::Metadata;
+use crate::rmeta::encoder::ContextEncoder;
 use crate::rmeta::*;
 
 pub(super) trait IsDefault: Default {
@@ -486,15 +487,12 @@ impl<I: Idx, const N: usize, T: FixedSizeEncoding<ByteArray = [u8; N]>> TableBui
         }
     }
 
-    pub(crate) fn encode(&self, buf: &mut FileEncoder) -> LazyTable<I, T> {
+    pub(crate) fn encode(&self, buf: &mut ContextEncoder<'_>) -> LazyTable<I, T> {
         let pos = buf.position();
 
         let width = self.width;
         for block in &self.blocks {
-            buf.write_with(|dest| {
-                *dest = *block;
-                width
-            });
+            buf.write_m_with(block, width);
         }
 
         LazyTable::from_position_and_encoded_size(
diff --git a/compiler/rustc_middle/src/hir/map.rs b/compiler/rustc_middle/src/hir/map.rs
diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs
diff --git a/compiler/rustc_session/src/cstore.rs b/compiler/rustc_session/src/cstore.rs
