Rollup merge of #157571 - bjorn3:proc_macro_refactors6, r=petrochenkov

Remove ProcMacro enum from proc macro ABI

Instead encode this information in the crate metadata. This helps shrink the proc-macro ABI to just the parts necessary for transmitting raw bytes to perform RPC on top of.

r? @petrochenkov cc @cyrgani

Author: jhpratt

compiler/rustc_builtin_macros/src/proc_macro_harness.rs

@@ -12,18 +12,16 @@ use rustc_hir::attrs::AttributeKind;
 use rustc_session::Session;
 use rustc_span::hygiene::AstPass;
 use rustc_span::source_map::SourceMap;
-use rustc_span::{DUMMY_SP, Ident, Span, Symbol, kw, sym};
+use rustc_span::{DUMMY_SP, Ident, Span, kw, sym};
 use smallvec::smallvec;
 use thin_vec::{ThinVec, thin_vec};
 
 use crate::diagnostics;
 
 struct ProcMacroDerive {
     id: NodeId,
-    trait_name: Symbol,
     function_ident: Ident,
     span: Span,
-    attrs: ThinVec<Symbol>,
 }
 
 struct ProcMacroDef {
@@ -101,15 +99,12 @@ impl<'a> CollectProcMacros<'a> {
         function_ident: Ident,
         attr: &'a ast::Attribute,
     ) {
-        let Some(rustc_hir::Attribute::Parsed(AttributeKind::ProcMacroDerive {
-            trait_name,
-            helper_attrs,
-            ..
-        })) = AttributeParser::parse_limited(
-            self.session,
-            slice::from_ref(attr),
-            &[sym::proc_macro_derive],
-        )
+        let Some(rustc_hir::Attribute::Parsed(AttributeKind::ProcMacroDerive { .. })) =
+            AttributeParser::parse_limited(
+                self.session,
+                slice::from_ref(attr),
+                &[sym::proc_macro_derive],
+            )
         else {
             return;
         };
@@ -118,9 +113,7 @@ impl<'a> CollectProcMacros<'a> {
             self.macros.push(ProcMacro::Derive(ProcMacroDerive {
                 id: item.id,
                 span: item.span,
-                trait_name,
                 function_ident,
-                attrs: helper_attrs,
             }));
         } else {
             let msg = if !self.in_root {
@@ -291,10 +284,9 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> Box<ast::Item> {
 
     let bridge = Ident::new(sym::bridge, span);
     let client = Ident::new(sym::client, span);
-    let proc_macro_ty = Ident::new(sym::ProcMacro, span);
-    let custom_derive = Ident::new(sym::custom_derive, span);
-    let attr = Ident::new(sym::attr, span);
-    let bang = Ident::new(sym::bang, span);
+    let client_ty = Ident::new(sym::Client, span);
+    let expand1 = Ident::new(sym::expand1, span);
+    let expand2 = Ident::new(sym::expand2, span);
 
     // We add NodeIds to 'resolver.proc_macros' in the order
     // that we generate expressions. The position of each NodeId
@@ -312,7 +304,7 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> Box<ast::Item> {
             let proc_macro_ty_method_path = |cx: &ExtCtxt<'_>, method| {
                 cx.expr_path(cx.path(
                     span.with_ctxt(harness_span.ctxt()),
-                    vec![proc_macro, bridge, client, proc_macro_ty, method],
+                    vec![proc_macro, bridge, client, client_ty, method],
                 ))
             };
             match m {
@@ -322,25 +314,15 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> Box<ast::Item> {
                     // accepts it.
                     cx.expr_call(
                         harness_span,
-                        proc_macro_ty_method_path(cx, custom_derive),
-                        thin_vec![
-                            cx.expr_str(span, cd.trait_name),
-                            cx.expr_array_ref(
-                                span,
-                                cd.attrs
-                                    .iter()
-                                    .map(|&s| cx.expr_str(span, s))
-                                    .collect::<ThinVec<_>>(),
-                            ),
-                            local_path(cx, cd.function_ident),
-                        ],
+                        proc_macro_ty_method_path(cx, expand1),
+                        thin_vec![local_path(cx, cd.function_ident)],
                     )
                 }
                 ProcMacro::Attr(ca) | ProcMacro::Bang(ca) => {
                     cx.resolver.declare_proc_macro(ca.id);
                     let ident = match m {
-                        ProcMacro::Attr(_) => attr,
-                        ProcMacro::Bang(_) => bang,
+                        ProcMacro::Attr(_) => expand2,
+                        ProcMacro::Bang(_) => expand1,
                         ProcMacro::Derive(_) => unreachable!(),
                     };
 
@@ -349,10 +331,7 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> Box<ast::Item> {
                     cx.expr_call(
                         harness_span,
                         proc_macro_ty_method_path(cx, ident),
-                        thin_vec![
-                            cx.expr_str(span, ca.function_ident.name),
-                            local_path(cx, ca.function_ident),
-                        ],
+                        thin_vec![local_path(cx, ca.function_ident)],
                     )
                 }
             }
@@ -367,7 +346,7 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> Box<ast::Item> {
             cx.ty(
                 span,
                 ast::TyKind::Slice(
-                    cx.ty_path(cx.path(span, vec![proc_macro, bridge, client, proc_macro_ty])),
+                    cx.ty_path(cx.path(span, vec![proc_macro, bridge, client, client_ty])),
                 ),
             ),
             None,

compiler/rustc_codegen_ssa/src/back/metadata.rs

@@ -36,7 +36,7 @@ use super::apple;
 /// <dd>The metadata can be found in the `.rustc` section of the shared library.</dd>
 /// </dl>
 #[derive(Debug)]
-pub(crate) struct DefaultMetadataLoader;
+pub struct DefaultMetadataLoader;
 
 static AIX_METADATA_SYMBOL_NAME: &'static str = "__aix_rust_metadata";
 

compiler/rustc_expand/src/proc_macro.rs

@@ -31,7 +31,7 @@ fn record_expand_proc_macro<'a>(
 }
 
 pub struct BangProcMacro {
-    pub client: pm::bridge::client::Client<pm::TokenStream, pm::TokenStream>,
+    pub client: pm::bridge::client::Client,
 }
 
 impl base::BangProcMacro for BangProcMacro {
@@ -46,7 +46,7 @@ impl base::BangProcMacro for BangProcMacro {
         let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace;
         let strategy = exec_strategy(ecx.sess);
         let server = proc_macro_server::Rustc::new(ecx);
-        self.client.run(&strategy, server, input, proc_macro_backtrace).map_err(|e| {
+        self.client.run1(&strategy, server, input, proc_macro_backtrace).map_err(|e| {
             ecx.dcx().emit_err(errors::ProcMacroPanicked {
                 span,
                 message: e.into_string().map(|message| errors::ProcMacroPanickedHelp { message }),
@@ -56,7 +56,7 @@ impl base::BangProcMacro for BangProcMacro {
 }
 
 pub struct AttrProcMacro {
-    pub client: pm::bridge::client::Client<(pm::TokenStream, pm::TokenStream), pm::TokenStream>,
+    pub client: pm::bridge::client::Client,
 }
 
 impl base::AttrProcMacro for AttrProcMacro {
@@ -72,7 +72,7 @@ impl base::AttrProcMacro for AttrProcMacro {
         let proc_macro_backtrace = ecx.ecfg.proc_macro_backtrace;
         let strategy = exec_strategy(ecx.sess);
         let server = proc_macro_server::Rustc::new(ecx);
-        self.client.run(&strategy, server, annotation, annotated, proc_macro_backtrace).map_err(
+        self.client.run2(&strategy, server, annotation, annotated, proc_macro_backtrace).map_err(
             |e| {
                 ecx.dcx().emit_err(errors::CustomAttributePanicked {
                     span,
@@ -176,7 +176,7 @@ pub(super) fn provide_derive_macro_expansion<'tcx>(
     })
 }
 
-type DeriveClient = pm::bridge::client::Client<pm::TokenStream, pm::TokenStream>;
+type DeriveClient = pm::bridge::client::Client;
 
 fn expand_derive_macro(
     invoc_id: LocalExpnId,
@@ -196,7 +196,7 @@ fn expand_derive_macro(
     let strategy = exec_strategy(ecx.sess);
     let server = proc_macro_server::Rustc::new(ecx);
 
-    match client.run(&strategy, server, input, proc_macro_backtrace) {
+    match client.run1(&strategy, server, input, proc_macro_backtrace) {
         Ok(stream) => Ok(stream),
         Err(e) => {
             let invoc_expn_data = invoc_id.expn_data();

compiler/rustc_metadata/src/creader.rs

@@ -23,7 +23,7 @@ use rustc_index::IndexVec;
 use rustc_middle::bug;
 use rustc_middle::ty::data_structures::IndexSet;
 use rustc_middle::ty::{TyCtxt, TyCtxtFeed};
-use rustc_proc_macro::bridge::client::ProcMacro;
+use rustc_proc_macro::bridge::client::Client as ProcMacroClient;
 use rustc_session::config::mitigation_coverage::DeniedPartialMitigationLevel;
 use rustc_session::config::{
     CrateType, ExtendedTargetModifierInfo, ExternLocation, Externs, OptionsTargetModifiers,
@@ -951,12 +951,13 @@ impl CStore {
         sess: &Session,
         path: &Path,
         stable_crate_id: StableCrateId,
-    ) -> Result<&'static [ProcMacro], CrateError> {
+    ) -> Result<&'static [ProcMacroClient], CrateError> {
         let sym_name = sess.generate_proc_macro_decls_symbol(stable_crate_id);
         debug!("trying to dlsym proc_macros {} for symbol `{}`", path.display(), sym_name);
 
         unsafe {
-            let result = load_symbol_from_dylib::<*const &[ProcMacro]>(path, &sym_name);
+            // FIXME(bjorn3) this depends on the unstable slice memory layout
+            let result = load_symbol_from_dylib::<*const &[ProcMacroClient]>(path, &sym_name);
             match result {
                 Ok(result) => {
                     debug!("loaded dlsym proc_macros {} for symbol `{}`", path.display(), sym_name);

compiler/rustc_metadata/src/lib.rs

@@ -31,4 +31,4 @@ pub use native_libs::{
     NativeLibSearchFallback, find_native_static_library, try_find_native_dynamic_library,
     try_find_native_static_library, walk_native_lib_search_dirs,
 };
-pub use rmeta::{EncodedMetadata, METADATA_HEADER, encode_metadata, rendered_const};
+pub use rmeta::{EncodedMetadata, METADATA_HEADER, ProcMacroKind, encode_metadata, rendered_const};

compiler/rustc_metadata/src/locator.rs

@@ -236,7 +236,7 @@ use tracing::{debug, info};
 
 use crate::creader::{Library, MetadataLoader};
 use crate::errors;
-use crate::rmeta::{METADATA_HEADER, MetadataBlob, rustc_version};
+use crate::rmeta::{METADATA_HEADER, MetadataBlob, ProcMacroKind, rustc_version};
 
 #[derive(Clone)]
 pub(crate) struct CrateLocator<'a> {
@@ -821,7 +821,7 @@ fn get_metadata_section<'p>(
     crate_name: Option<Symbol>,
 ) -> Result<MetadataBlob, MetadataError<'p>> {
     if !filename.exists() {
-        return Err(MetadataError::NotPresent(filename));
+        return Err(MetadataError::NotPresent(filename.into()));
     }
     let raw_bytes = match flavor {
         CrateFlavor::Rlib => {
@@ -978,6 +978,20 @@ fn get_flavor_from_path(path: &Path) -> CrateFlavor {
     }
 }
 
+/// A function to get information about all macros inside a proc-macro crate.
+///
+/// Used by rust-analyzer-proc-macro-srv.
+pub fn get_proc_macro_info<'p>(
+    target: &Target,
+    path: &'p Path,
+    metadata_loader: &dyn MetadataLoader,
+    cfg_version: &'static str,
+) -> Result<Vec<ProcMacroKind>, MetadataError<'p>> {
+    let metadata =
+        get_metadata_section(target, CrateFlavor::Dylib, path, metadata_loader, cfg_version, None)?;
+    Ok(metadata.get_proc_macro_info())
+}
+
 // ------------------------------------------ Error reporting -------------------------------------
 
 #[derive(Clone, Debug)]
@@ -1024,9 +1038,10 @@ pub(crate) enum CrateError {
     NotFound(Symbol),
 }
 
-enum MetadataError<'a> {
+#[derive(Debug)]
+pub enum MetadataError<'a> {
     /// The file was missing.
-    NotPresent(&'a Path),
+    NotPresent(Cow<'a, Path>),
     /// The file was present and invalid.
     LoadFailure(String),
     /// The file was present, but compiled with a different rustc version.

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -26,7 +26,7 @@ use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState};
 use rustc_middle::ty::Visibility;
 use rustc_middle::ty::codec::TyDecoder;
 use rustc_middle::{bug, implement_ty_decoder};
-use rustc_proc_macro::bridge::client::ProcMacro;
+use rustc_proc_macro::bridge::client::Client as ProcMacroClient;
 use rustc_serialize::opaque::MemDecoder;
 use rustc_serialize::{Decodable, Decoder};
 use rustc_session::config::TargetModifier;
@@ -104,8 +104,8 @@ pub(crate) struct CrateMetadata {
     /// These can be introduced using either `#![rustc_coherence_is_core]`
     /// or `#[rustc_allow_incoherent_impl]`.
     incoherent_impls: FxIndexMap<SimplifiedType, LazyArray<DefIndex>>,
-    /// Proc macro descriptions for this crate, if it's a proc macro crate.
-    raw_proc_macros: Option<&'static [ProcMacro]>,
+    /// Proc macro function pointers for this crate, if it's a proc macro crate.
+    raw_proc_macros: Option<&'static [ProcMacroClient]>,
     /// Source maps for code from the crate.
     source_map_import_info: Lock<Vec<Option<ImportedSourceFile>>>,
     /// For every definition in this crate, maps its `DefPathHash` to its `DefIndex`.
@@ -930,6 +930,16 @@ impl MetadataBlob {
 
         Ok(())
     }
+
+    pub(crate) fn get_proc_macro_info(&self) -> Vec<ProcMacroKind> {
+        self.get_root()
+            .proc_macro_data
+            .unwrap()
+            .macros
+            .decode(self)
+            .map(|(_id, kind)| kind.decode(self))
+            .collect::<Vec<_>>()
+    }
 }
 
 impl CrateRoot {
@@ -976,19 +986,20 @@ impl CrateMetadata {
         bug!("missing `{descr}` for {:?}", self.local_def_id(id))
     }
 
-    fn raw_proc_macro(&self, tcx: TyCtxt<'_>, id: DefIndex) -> &ProcMacro {
+    fn raw_proc_macro(&self, tcx: TyCtxt<'_>, id: DefIndex) -> (ProcMacroClient, ProcMacroKind) {
         // DefIndex's in root.proc_macro_data have a one-to-one correspondence
         // with items in 'raw_proc_macros'.
-        let pos = self
+        let (pos, (_id, kind)) = self
             .root
             .proc_macro_data
             .as_ref()
             .unwrap()
             .macros
             .decode((self, tcx))
-            .position(|i| i == id)
+            .enumerate()
+            .find(|(_pos, (i, _))| *i == id)
             .unwrap();
-        &self.raw_proc_macros.unwrap()[pos]
+        (self.raw_proc_macros.unwrap()[pos], kind.decode((self, tcx)))
     }
 
     fn opt_item_name(&self, item_index: DefIndex) -> Option<Symbol> {
@@ -1046,20 +1057,20 @@ impl CrateMetadata {
     }
 
     fn load_proc_macro<'tcx>(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> SyntaxExtension {
-        let (name, kind, helper_attrs) = match *self.raw_proc_macro(tcx, id) {
-            ProcMacro::CustomDerive { trait_name, attributes, client } => {
+        let (name, kind, helper_attrs) = match self.raw_proc_macro(tcx, id) {
+            (client, ProcMacroKind::CustomDerive { trait_name, attributes }) => {
                 let helper_attrs =
-                    attributes.iter().cloned().map(Symbol::intern).collect::<Vec<_>>();
+                    attributes.into_iter().map(|attr| Symbol::intern(&attr)).collect();
                 (
                     trait_name,
                     SyntaxExtensionKind::Derive(Arc::new(DeriveProcMacro { client })),
                     helper_attrs,
                 )
             }
-            ProcMacro::Attr { name, client } => {
+            (client, ProcMacroKind::Attr { name }) => {
                 (name, SyntaxExtensionKind::Attr(Arc::new(AttrProcMacro { client })), Vec::new())
             }
-            ProcMacro::Bang { name, client } => {
+            (client, ProcMacroKind::Bang { name }) => {
                 (name, SyntaxExtensionKind::Bang(Arc::new(BangProcMacro { client })), Vec::new())
             }
         };
@@ -1072,7 +1083,7 @@ impl CrateMetadata {
             self.get_span(tcx, id),
             helper_attrs,
             self.root.edition,
-            Symbol::intern(name),
+            Symbol::intern(&name),
             &attrs,
             false,
         )
@@ -1269,7 +1280,7 @@ impl CrateMetadata {
                 // If we are loading as a proc macro, we want to return
                 // the view of this crate as a proc macro crate.
                 if id == CRATE_DEF_INDEX {
-                    for child_index in data.macros.decode((self, tcx)) {
+                    for (child_index, _) in data.macros.decode((self, tcx)) {
                         yield self.get_mod_child(tcx, child_index);
                     }
                 }
@@ -1872,7 +1883,7 @@ impl CrateMetadata {
         tcx: TyCtxt<'_>,
         blob: MetadataBlob,
         root: CrateRoot,
-        raw_proc_macros: Option<&'static [ProcMacro]>,
+        raw_proc_macros: Option<&'static [ProcMacroClient]>,
         cnum: CrateNum,
         cnum_map: CrateNumMap,
         dep_kind: CrateDepKind,
@@ -2022,7 +2033,7 @@ impl CrateMetadata {
     ) -> impl Iterator<Item = DefId> {
         gen move {
             for def_id in self.root.proc_macro_data.as_ref().into_iter().flat_map(move |data| {
-                data.macros.decode((self, tcx)).map(move |index| DefId { index, krate })
+                data.macros.decode((self, tcx)).map(move |(index, _)| DefId { index, krate })
             }) {
                 yield def_id;
             }