(feat): cache external (plugin-generated) file content in the defs cache (#10164)

Author: eytan-starkware

crates/cairo-lang-defs/src/cache/mod.rs

@@ -21,7 +21,7 @@ use cairo_lang_syntax::node::ids::{GreenId, SyntaxStablePtrId};
 use cairo_lang_syntax::node::kind::SyntaxKind;
 use cairo_lang_syntax::node::{SyntaxNode, SyntaxNodeId, TypedSyntaxNode, ast};
 use cairo_lang_utils::Intern;
-use cairo_lang_utils::ordered_hash_map::OrderedHashMap;
+use cairo_lang_utils::ordered_hash_map::{Entry, OrderedHashMap};
 use salsa::Database;
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
@@ -41,8 +41,11 @@ use crate::ids::{
 };
 use crate::plugin::{DynGeneratedFileAuxData, PluginDiagnostic};
 
+/// The index of the `external` section in a crate cache blob.
+pub const EXTERNAL_CACHE_SECTION: u8 = 0;
+
 /// The index of the `def` section in a crate cache blob.
-pub const DEF_CACHE_SECTION: u8 = 0;
+pub const DEF_CACHE_SECTION: u8 = EXTERNAL_CACHE_SECTION + 1;
 
 /// Metadata for a cached crate.
 #[derive(Serialize, Deserialize)]
@@ -99,7 +102,7 @@ pub fn load_cached_crate_modules<'db>(
     };
 
     let (metadata, module_data, defs_lookups): DefCache<'_> =
-        CacheBlobReader::read_section(db, content, DEF_CACHE_SECTION, &crate_id);
+        CacheBlobReader::read_section(db, content, DEF_CACHE_SECTION, crate_id);
 
     validate_metadata(crate_id, &metadata, db);
 
@@ -142,10 +145,11 @@ pub fn write_cache_section<T: Serialize>(
 
 /// A cursor over the length-prefixed sections of a crate cache blob (see [`write_cache_section`]).
 ///
-/// A blob is a sequence of sections in the order `[def, semantic, lowering]`; each consumer
-/// advances past the sections preceding the one it needs with [`Self::next_section`].
+/// A blob is a sequence of sections in the order `[external, def, semantic, lowering]`; each
+/// consumer advances past the sections preceding the one it needs with [`Self::next_section`].
 /// Use [`Self::read_section`] to read and parse a specific section only.
-/// Each section should have its index as a const in the cache mod, see [`DEF_CACHE_SECTION`] et al.
+/// Each section should have its index as a const in the cache mod, see [`EXTERNAL_CACHE_SECTION`]
+/// et al.
 #[derive(Debug, Clone, Copy)]
 pub struct CacheBlobReader<'a> {
     content: &'a [u8],
@@ -178,7 +182,7 @@ impl<'a> CacheBlobReader<'a> {
         db: &'db dyn Database,
         content: &'a [u8],
         section: u8,
-        crate_id: &CrateId<'db>,
+        crate_id: CrateId<'db>,
     ) -> T {
         let mut reader = CacheBlobReader::new(content);
         for _ in 0..section {
@@ -400,6 +404,14 @@ impl<'db> DefCacheSavingContext<'db> {
     pub fn new(db: &'db dyn Database, self_crate_id: CrateId<'db>) -> Self {
         Self { db, data: DefCacheSavingData::default(), self_crate_id }
     }
+
+    /// The external-file content table, written as its own blob section (read back by the
+    /// `external_file_contents` query).
+    pub fn external_file_contents(
+        &self,
+    ) -> &OrderedHashMap<ExternalFileKey, ExternalFileContentCached> {
+        &self.data.external_file_contents
+    }
 }
 
 /// Data for saving cache from the database.
@@ -426,6 +438,9 @@ pub struct DefCacheSavingData<'db> {
     syntax_nodes: OrderedHashMap<SyntaxNode<'db>, SyntaxNodeCached>,
     file_ids: OrderedHashMap<FileId<'db>, FileIdCached>,
 
+    /// External (plugin-generated) file content, written as its own blob section in cache.
+    external_file_contents: OrderedHashMap<ExternalFileKey, ExternalFileContentCached>,
+
     pub lookups: DefCacheLookups,
 }
 
@@ -1941,14 +1956,116 @@ impl FileCached {
     }
 }
 
+/// Portable, parse-free identity for an external file — the key into the external-file content
+/// table. Computed during cache load by reading only already-materialized fields, so it never
+/// parses or re-runs plugins (which would cycle back into the cache).
+#[derive(Serialize, Deserialize, Clone, Eq, Hash, PartialEq, Debug)]
+pub enum ExternalFileKey {
+    /// An on-disk file, identified by its path.
+    OnDisk(PathBuf),
+    /// A virtual file: parent (key + span), name, and a content hash (hashed, not stored, so a
+    /// shared ancestor isn't duplicated across keys).
+    Virtual { parent: Option<(Box<ExternalFileKey>, TextSpan)>, name: String, content_hash: u64 },
+    /// A plugin-generated file: the parent file's key, the stable-ptr offset within it, and name.
+    Generated { parent: Box<ExternalFileKey>, offset: u32, name: String },
+}
+
+impl ExternalFileKey {
+    /// Computes the key for `file_id`; see the type docs for why this is parse/plugin/mint-free.
+    pub(crate) fn new<'db>(db: &'db dyn Database, file_id: FileId<'db>) -> Self {
+        match file_id.long(db) {
+            FileLongId::OnDisk(path) => ExternalFileKey::OnDisk(path.clone()),
+            FileLongId::Virtual(vf) => ExternalFileKey::Virtual {
+                parent: vf.parent.map(|parent| {
+                    (Box::new(ExternalFileKey::new(db, parent.file_id)), parent.span)
+                }),
+                name: vf.name.long(db).to_string(),
+                content_hash: {
+                    let mut hasher = xxhash_rust::xxh3::Xxh3::default();
+                    vf.content.long(db).as_str().hash(&mut hasher);
+                    hasher.finish()
+                },
+            },
+            FileLongId::External(external_id) => {
+                let long_id = PluginGeneratedFileId::from_intern_id(*external_id).long(db);
+                ExternalFileKey::Generated {
+                    parent: Box::new(ExternalFileKey::new(db, long_id.stable_ptr.file_id(db))),
+                    offset: long_id.stable_ptr.0.offset(db).as_u32(),
+                    name: long_id.name.clone(),
+                }
+            }
+        }
+    }
+}
+
+/// An external file's content, enough to rebuild its `VirtualFile` on load. `parent` and `name` are
+/// not stored — they come from the runtime `PluginGeneratedFileLongId` at lookup.
+#[derive(Serialize, Deserialize, Clone, PartialEq, Eq, Debug)]
+pub struct ExternalFileContentCached {
+    content: String,
+    code_mappings: Vec<CodeMapping>,
+    kind: FileKind,
+    original_item_removed: bool,
+}
+
+impl ExternalFileContentCached {
+    fn new<'db>(virtual_file: &VirtualFile<'db>, db: &'db dyn Database) -> Self {
+        Self {
+            content: virtual_file.content.to_string(db),
+            code_mappings: virtual_file.code_mappings.to_vec(),
+            kind: virtual_file.kind,
+            original_item_removed: virtual_file.original_item_removed,
+        }
+    }
+
+    /// Rebuilds the `VirtualFile`; `parent` and `name` come from the runtime stable ptr / long id.
+    pub(crate) fn to_virtual_file<'db>(
+        &self,
+        db: &'db dyn Database,
+        name: &str,
+        parent: Option<SpanInFile<'db>>,
+    ) -> VirtualFile<'db> {
+        VirtualFile {
+            parent,
+            name: SmolStrId::from(db, name),
+            content: SmolStrId::from(db, self.content.clone()),
+            code_mappings: self.code_mappings.clone().into(),
+            kind: self.kind,
+            original_item_removed: self.original_item_removed,
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize, Clone, Copy, Eq, Hash, PartialEq, salsa::Update, Debug)]
 pub struct FileIdCached(usize);
 impl FileIdCached {
     fn new<'db>(id: FileId<'db>, ctx: &mut DefCacheSavingContext<'db>) -> Self {
         if let Some(cached_id) = ctx.file_ids.get(&id) {
             return *cached_id;
         }
-        let cached = FileCached::new(id.long(ctx.db), ctx);
+        let long_id = id.long(ctx.db);
+        // For external (plugin-generated) files, also cache their content so `ext_as_virtual` can
+        // serve it from the blob instead of re-running plugins.
+        if let FileLongId::External(external_id) = long_id {
+            // Distinct `FileId`s can map to the same `ExternalFileKey` (the key is coarser), so
+            // dedup against the keys already stored in `external_file_contents`.
+            let key = ExternalFileKey::new(ctx.db, id);
+            let virtual_file = cairo_lang_filesystem::db::ext_as_virtual(ctx.db, *external_id);
+            let content = ExternalFileContentCached::new(virtual_file, ctx.db);
+            match ctx.external_file_contents.entry(key) {
+                Entry::Vacant(entry) => {
+                    entry.insert(content);
+                }
+                // Two distinct external files collapsing to one key must carry identical content,
+                // else the load would serve the wrong bytes. Holds today (generated files have
+                // distinct parent+offset); guard it so a future key change fails loud, not silent.
+                Entry::Occupied(entry) => debug_assert!(
+                    *entry.get() == content,
+                    "external file content key collision with differing content"
+                ),
+            }
+        }
+        let cached = FileCached::new(long_id, ctx);
         let cached_id = FileIdCached(ctx.file_ids_lookup.len());
         ctx.file_ids_lookup.push(cached);
         ctx.file_ids.insert(id, cached_id);

crates/cairo-lang-defs/src/db.rs

@@ -6,7 +6,8 @@ use cairo_lang_diagnostics::{
 };
 use cairo_lang_filesystem::db::{ExtAsVirtual, FilesGroup, files_group_input};
 use cairo_lang_filesystem::ids::{
-    CrateId, CrateInput, Directory, FileId, FileKind, FileLongId, SmolStrId, Tracked, VirtualFile,
+    BlobId, CrateId, CrateInput, Directory, FileId, FileKind, FileLongId, SmolStrId, Tracked,
+    VirtualFile,
 };
 use cairo_lang_parser::db::ParserGroup;
 use cairo_lang_syntax::attribute::consts::{
@@ -25,7 +26,10 @@ use cairo_lang_utils::ordered_hash_set::OrderedHashSet;
 use itertools::{Itertools, chain};
 use salsa::{Database, Setter};
 
-use crate::cache::{DefCacheLoadingData, load_cached_crate_modules};
+use crate::cache::{
+    CacheBlobReader, DefCacheLoadingData, EXTERNAL_CACHE_SECTION, ExternalFileContentCached,
+    ExternalFileKey, load_cached_crate_modules,
+};
 use crate::ids::*;
 use crate::plugin::{DynGeneratedFileAuxData, MacroPlugin, MacroPluginMetadata, PluginDiagnostic};
 use crate::plugin_utils::try_extract_unnamed_arg;
@@ -1110,13 +1114,63 @@ pub fn init_external_files<T: DefsGroup>(db: &mut T) {
     files_group_input(db).set_ext_as_virtual_obj(db).to(Some(ext_as_virtual_impl));
 }
 
+/// The external-file content table from a crate's cache blob, keyed only on the blob — so reading
+/// it from `ext_as_virtual` adds no module-data edge, keeping cache loading cycle-free.
+#[salsa::tracked(returns(ref))]
+fn external_file_contents<'db>(
+    db: &'db dyn Database,
+    blob_id: BlobId<'db>,
+    crate_id: CrateId<'db>,
+) -> OrderedHashMap<ExternalFileKey, ExternalFileContentCached> {
+    let Some(content) = db.blob_content(blob_id) else {
+        return Default::default();
+    };
+
+    CacheBlobReader::read_section(db, content, EXTERNAL_CACHE_SECTION, crate_id)
+}
+
+/// Rebuilds an external file's `VirtualFile` from the cache blob. `None` if the crate has no cache
+/// blob or the blob lacks this file — `ext_as_virtual_impl` then falls back to `module_sub_files`.
+#[salsa::tracked(returns(ref))]
+fn cached_external_virtual_file<'db>(
+    db: &'db dyn Database,
+    external_file: FileId<'db>,
+) -> Option<VirtualFile<'db>> {
+    let FileLongId::External(raw_external_id) = external_file.long(db) else {
+        return None;
+    };
+    let long_id = PluginGeneratedFileId::from_intern_id(*raw_external_id).long(db);
+    let crate_id = long_id.module_id.owning_crate(db);
+    let blob_id = db.crate_config(crate_id)?.cache_file?;
+    // Parse/plugin/mint-free, so it cannot re-enter the cache. See `ExternalFileKey::new`.
+    let key = ExternalFileKey::new(db, external_file);
+    let content = external_file_contents(db, blob_id, crate_id).get(&key)?;
+    Some(content.to_virtual_file(db, &long_id.name, Some(long_id.stable_ptr.span_in_file(db))))
+}
+
 /// Returns the `VirtualFile` matching the given external id.
 pub fn ext_as_virtual_impl<'db>(
     db: &'db dyn Database,
     external_id: salsa::Id,
 ) -> &'db VirtualFile<'db> {
-    let long_id = PluginGeneratedFileId::from_intern_id(external_id).long(db);
     let file_id = FileLongId::External(external_id).intern(db);
+    // Serve from the cache blob first (no module-data edge); see `external_file_contents`.
+    if let Some(virtual_file) = cached_external_virtual_file(db, file_id) {
+        return virtual_file;
+    }
+    let long_id = PluginGeneratedFileId::from_intern_id(external_id).long(db);
+    // A loaded blob collected every external file at save time, so a miss means a stale/corrupt
+    // cache; falling back to `module_sub_files` would re-enter the in-flight load (cycle), so fail
+    // loud. An absent/unreadable blob means the crate isn't cache-backed — the fallback is safe.
+    if let Some(blob_id) =
+        db.crate_config(long_id.module_id.owning_crate(db)).and_then(|config| config.cache_file)
+    {
+        assert!(
+            db.blob_content(blob_id).is_none(),
+            "external file missing from a loaded crate cache: {:?}",
+            file_id.long(db)
+        );
+    }
     let data =
         module_sub_files(db, long_id.module_id, long_id.stable_ptr.file_id(db)).as_ref().unwrap();
     &data.files[&file_id]

crates/cairo-lang-lowering/src/cache/mod.rs

@@ -73,7 +73,7 @@ pub fn load_cached_crate_functions<'db>(
     let semantic_loading_data = db.cached_crate_semantic_data(crate_id)?.loading_data;
 
     let (lookups, lowerings): LookupCache =
-        CacheBlobReader::read_section(db, content, LOWERING_CACHE_SECTION, &crate_id);
+        CacheBlobReader::read_section(db, content, LOWERING_CACHE_SECTION, crate_id);
 
     // TODO(tomer): Fail on version, cfg, and dependencies mismatch.
 
@@ -152,6 +152,11 @@ pub fn generate_crate_cache<'db>(
 
     let mut artifact = Vec::<u8>::new();
 
+    // External (plugin-generated) file content comes first: it's filesystem-layer content (served
+    // by `ext_as_virtual`) that the phase sections below depend on, and putting it first lets the
+    // `external_file_contents` query (in defs) read it without deserializing them.
+    write_cache_section(&mut artifact, ctx.semantic_ctx.defs_ctx.external_file_contents())?;
+
     write_cache_section(
         &mut artifact,
         &(CachedCrateMetadata::new(crate_id, db), def_cache, &ctx.semantic_ctx.defs_ctx.lookups),

crates/cairo-lang-lowering/src/cache/test_data/cache

@@ -265,3 +265,73 @@ blk0 (root):
 Statements:
 End:
   Return()
+
+//! > ==========================================================================
+
+//! > Cache Derive Drop
+
+//! > test_runner_name
+test_cache_check
+
+//! > function_code
+fn foo() -> felt252 {
+    let s = MyStruct { x: 5 };
+    s.x
+}
+
+//! > function_name
+foo
+
+//! > module_code
+#[derive(Drop)]
+struct MyStruct {
+    x: felt252,
+}
+
+//! > semantic_diagnostics
+
+//! > lowering_diagnostics
+
+//! > lowering_flat
+Parameters:
+blk0 (root):
+Statements:
+  (v0: core::felt252) <- 5
+End:
+  Return(v0)
+
+//! > ==========================================================================
+
+//! > Cache Derive in Submodule
+
+//! > test_runner_name
+test_cache_check
+
+//! > function_code
+fn foo() -> felt252 {
+    let s = inner::InnerStruct { x: 5 };
+    s.x
+}
+
+//! > function_name
+foo
+
+//! > module_code
+mod inner {
+    #[derive(Drop)]
+    pub struct InnerStruct {
+        pub x: felt252,
+    }
+}
+
+//! > semantic_diagnostics
+
+//! > lowering_diagnostics
+
+//! > lowering_flat
+Parameters:
+blk0 (root):
+Statements:
+  (v0: core::felt252) <- 5
+End:
+  Return(v0)

crates/cairo-lang-semantic/src/cache/mod.rs

@@ -80,7 +80,7 @@ pub fn load_cached_crate_modules_semantic<'db>(
     };
 
     let (module_data, semantic_lookups): SemanticCache<'_> =
-        CacheBlobReader::read_section(db, content, SEMANTIC_CACHE_SECTION, &crate_id);
+        CacheBlobReader::read_section(db, content, SEMANTIC_CACHE_SECTION, crate_id);
 
     let mut ctx = SemanticCacheLoadingContext::new(db, semantic_lookups, def_loading_data);
     Some(ModuleSemanticDataCacheAndLoadingData {