Implement support for proc macros on wasm32-wasip2

This is an initial draft implementation. A few problems need to be
addressed before this could be merged:

* Figure out testing strategy -- currently, this hardcodes compiletest
  to build in wasip2, which clearly won't work to land. Most likely
  we'll need an opt-in of some kind, especially given that building the
  standard library for wasip2 currently requires downloading a
  wasi sysroot.
* This duplicates the proc macro ABI with a WIT file for all the
  individual functions. There's not too much complexity inherent to the
  WIT approach -- the duplication doesn't seem to cost us much, since
  the ABI is pretty simple. But it does mean the base data structures
  need replicating, though some of that could be eliminated by always
  using the codegen'd struct/enums in the C bridge as well.
  * There's also some impedance mismatch, e.g., Span is Copy and so we
    end up leaking resources to mitigate the issue there. I think the
    right thing is probably to expose it as a u32 wrapper (similar to
    the old ABI).

But in the meantime this does pass tests (confirmed on aarch64-apple and
x86_64-linux, the two systems I have readily available), so opening up a
draft PR to share the initial state.

Some not necessarily blocking challenges:

* `cc` crate dependency causes us to need an old wasmtime dependency. I
  think we should be able to relax the upstream constraint but haven't
  looked into it yet.

Author: Mark-Simulacrum

Cargo.lock

@@ -31,6 +31,8 @@ rustc_target = { path = "../rustc_target" }
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.15"
 tracing = "0.1"
+wit-bindgen-rust = "0.57.1"
+wit-bindgen-core = "0.57.1"
 # tidy-alphabetical-end
 
 [features]

compiler/rustc_builtin_macros/Cargo.toml

@@ -145,4 +145,9 @@ pub fn register_builtin_macros(resolver: &mut dyn ResolverExpand) {
     register(sym::contracts_requires, requires);
     let ensures = SyntaxExtensionKind::Attr(Arc::new(contracts::ExpandEnsures));
     register(sym::contracts_ensures, ensures);
+
+    register(
+        sym::internal_wit_bindgen,
+        SyntaxExtensionKind::Bang(Arc::new(proc_macro_harness::InternalWitBindgen)),
+    );
 }

compiler/rustc_builtin_macros/src/lib.rs

@@ -89,7 +89,14 @@ pub fn inject(
 impl<'a> CollectProcMacros<'a> {
     fn check_not_pub_in_root(&self, vis: &ast::Visibility, sp: Span) {
         if self.is_proc_macro_crate && self.in_root && vis.kind.is_pub() {
-            self.dcx.emit_err(diagnostics::ProcMacro { span: sp });
+            // On wasm we end up generating other public exports (though the exact specifics are
+            // internal). For now omit this check, we'll need to refine it before stabilizing wasm
+            // proc macros.
+            if !(self.session.opts.unstable_opts.wasm_proc_macros
+                && self.session.target.is_like_wasm)
+            {
+                self.dcx.emit_err(diagnostics::ProcMacro { span: sp });
+            }
         }
     }
 
@@ -270,6 +277,9 @@ impl<'a> Visitor<'a> for CollectProcMacros<'a> {
 //              // ...
 //          ];
 //      }
+//
+// If we're targeting wasm32, we also inject a macro call to generate_export!(DECLS). This produces
+// the WASI component model ABI exports/imports to support the proc macro's execution.
 fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> Box<ast::Item> {
     let expn_id = cx.resolver.expansion_for_ast_pass(
         DUMMY_SP,
@@ -361,9 +371,33 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> Box<ast::Item> {
         cx.attr_nested_word(sym::allow, sym::deprecated, span),
     ]);
 
-    let block = ast::ConstItemRhsKind::new_body(cx.expr_block(
-        cx.block(span, thin_vec![cx.stmt_item(span, krate), cx.stmt_item(span, decls_static)]),
-    ));
+    // For wasm targets, exporting from proc-macros requires that we generate additional symbols
+    // (not just the single #[used] static). We use a macro defined in the proc_macro crate to do
+    // so.
+    let block_contents = if cx.sess.target.is_like_wasm {
+        let mac_call = cx.stmt_semi(cx.expr_macro_call(
+            span,
+            cx.macro_call(
+                span,
+                cx.path(
+                    span,
+                    vec![proc_macro, bridge, client, Ident::new(sym::generate_export, span)],
+                ),
+                rustc_ast::token::Delimiter::Parenthesis,
+                rustc_ast::tokenstream::TokenStream::new(vec![
+                    rustc_ast::tokenstream::TokenTree::Token(
+                        rustc_ast::token::Token::from_ast_ident(Ident::new(sym::_DECLS, span)),
+                        rustc_ast::tokenstream::Spacing::Alone,
+                    ),
+                ]),
+            ),
+        ));
+        thin_vec![cx.stmt_item(span, krate), cx.stmt_item(span, decls_static), mac_call]
+    } else {
+        thin_vec![cx.stmt_item(span, krate), cx.stmt_item(span, decls_static)]
+    };
+
+    let block = ast::ConstItemRhsKind::new_body(cx.expr_block(cx.block(span, block_contents)));
 
     let anon_constant = cx.item_const(
         span,
@@ -376,3 +410,66 @@ fn mk_decls(cx: &mut ExtCtxt<'_>, macros: &[ProcMacro]) -> Box<ast::Item> {
     let items = AstFragment::Items(smallvec![anon_constant]);
     cx.monotonic_expander().fully_expand_fragment(items).make_items().pop().unwrap()
 }
+
+pub(crate) struct InternalWitBindgen;
+
+impl rustc_expand::base::BangProcMacro for InternalWitBindgen {
+    fn expand<'cx>(
+        &self,
+        ecx: &'cx mut ExtCtxt<'_>,
+        span: Span,
+        _ts: rustc_ast::tokenstream::TokenStream,
+    ) -> Result<rustc_ast::tokenstream::TokenStream, rustc_span::ErrorGuaranteed> {
+        let mut options = wit_bindgen_rust::Opts::default();
+        options.pub_export_macro = true;
+
+        let mut resolve = wit_bindgen_core::wit_parser::Resolve::default();
+        let pkg_id = resolve
+            .push_str(
+                "proc-macro-wasm.wit",
+                include_str!("../../../library/proc_macro/wasm-interface.wit"),
+            )
+            .unwrap();
+        let world = resolve.select_world(&[pkg_id], None).unwrap();
+
+        let mut generator = options.build();
+        let mut files = Default::default();
+        wit_bindgen_core::WorldGenerator::generate(&mut generator, &mut resolve, world, &mut files)
+            .expect("generation successful");
+        let (_, src) = files.iter().next().unwrap();
+        let src = std::str::from_utf8(src).unwrap();
+
+        let expn_data = ecx.current_expansion.id.expn_data();
+        let call_site = ecx.with_call_site_ctxt(expn_data.call_site);
+
+        let needle = "macro_rules! __export_rust_lang_rust_custom_derive_cabi";
+        let output = rustc_parse::source_str_to_stream(
+            ecx.psess(),
+            rustc_span::FileName::proc_macro_source_code(&src),
+            src.replace(
+                needle,
+                // #[allow_internal_unstable(...)] is not transitive on macro expansions and so
+                // doesn't apply to this inner macro -- which also references unstable types from
+                // the generated bindings.
+                //
+                // This string replacement is obviously a hack, but it works OK in practice. If you
+                // can remove this and keep wasm proc macro tests passing, go for it.
+                &format!("#[allow_internal_unstable(proc_macro_internals)] {needle}"),
+            ),
+            Some(call_site),
+        );
+
+        match output {
+            Ok(o) => Ok(o),
+            Err(diags) => {
+                for diag in diags {
+                    diag.emit();
+                }
+
+                Err(ecx
+                    .dcx()
+                    .span_delayed_bug(span, "failed to parse wit-bindgen generated source"))
+            }
+        }
+    }
+}

compiler/rustc_builtin_macros/src/proc_macro_harness.rs

@@ -1811,13 +1811,25 @@ pub(crate) fn exported_symbols(
             .collect();
     }
 
-    let mut symbols = if let CrateType::ProcMacro = crate_type {
-        exported_symbols_for_proc_macro_crate(tcx)
-    } else {
-        exported_symbols_for_non_proc_macro(tcx, crate_type)
-    };
+    let mut symbols = Vec::new();
+
+    // We include all symbols in the export list if this is a regular crate, or if the
+    // -Zwasm-proc-macros flag is passed. With wasm proc macros we need to expose more than just the
+    // global static (in fact, that static is not used on wasm targets at all), and for now that
+    // just means letting it use 'normal' crate rules.
+    if crate_type != CrateType::ProcMacro || tcx.sess.opts.unstable_opts.wasm_proc_macros {
+        symbols.extend(exported_symbols_for_non_proc_macro(tcx, crate_type));
+    }
+
+    // If this is a proc macro, then add the proc macro specific symbols too.
+    // See comment above for more details.
+    if let CrateType::ProcMacro = crate_type {
+        symbols.extend(exported_symbols_for_proc_macro_crate(tcx));
+    }
 
-    if crate_type == CrateType::Dylib || crate_type == CrateType::ProcMacro {
+    if crate_type == CrateType::Dylib
+        || (crate_type == CrateType::ProcMacro && !tcx.sess.target.is_like_wasm)
+    {
         let metadata_symbol_name = exported_symbols::metadata_symbol_name(tcx);
         symbols.push((metadata_symbol_name, SymbolExportKind::Data));
     }

compiler/rustc_codegen_ssa/src/back/linker.rs

@@ -32,4 +32,10 @@ scoped-tls = "1.0"
 smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
 thin-vec = "0.2.15"
 tracing = "0.1"
+wasmtime-wasi = "38"
 # tidy-alphabetical-end
+
+[dependencies.wasmtime]
+version = "38"
+default-features = false
+features = ["runtime", "winch", "component-model", "std", "cranelift"]

compiler/rustc_expand/Cargo.toml

@@ -22,6 +22,7 @@ pub mod config;
 pub mod expand;
 pub mod module;
 pub mod proc_macro;
+pub mod wasm_proc_macro;
 
 pub fn provide(providers: &mut rustc_middle::query::Providers) {
     providers.derive_macro_expansion = proc_macro::provide_derive_macro_expansion;