asm.rs

// cSpell:ignoreRegExp [afkspqvwy]reg

use std::borrow::Cow;

use gccjit::{LValue, RValue, ToRValue, Type};
use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_codegen_ssa::mir::operand::OperandValue;
use rustc_codegen_ssa::mir::place::PlaceRef;
use rustc_codegen_ssa::traits::{
    AsmBuilderMethods, AsmCodegenMethods, BaseTypeCodegenMethods, BuilderMethods,
    GlobalAsmOperandRef, InlineAsmOperandRef,
};
use rustc_middle::bug;
use rustc_middle::ty::Instance;
use rustc_span::Span;
use rustc_target::asm::*;

use crate::builder::Builder;
use crate::callee::get_fn;
use crate::context::CodegenCx;
use crate::errors::UnwindingInlineAsm;
use crate::type_of::LayoutGccExt;

// Rust asm! and GCC Extended Asm semantics differ substantially.
//
// 1. Rust asm operands go along as one list of operands. Operands themselves indicate
//    if they're "in" or "out". "In" and "out" operands can interleave. One operand can be
//    both "in" and "out" (`inout(reg)`).
//
//    GCC asm has two different lists for "in" and "out" operands. In terms of gccjit,
//    this means that all "out" operands must go before "in" operands. "In" and "out" operands
//    cannot interleave.
//
// 2. Operand lists in both Rust and GCC are indexed. Index starts from 0. Indexes are important
//    because the asm template refers to operands by index.
//
//    Mapping from Rust to GCC index would be 1-1 if it wasn't for...
//
// 3. Clobbers. GCC has a separate list of clobbers, and clobbers don't have indexes.
//    Contrary, Rust expresses clobbers through "out" operands that aren't tied to
//    a variable (`_`),  and such "clobbers" do have index. Input operands cannot also
//    be clobbered.
//
// 4. Furthermore, GCC Extended Asm does not support explicit register constraints
//    (like `out("eax")`) directly, offering so-called "local register variables"
//    as a workaround. These variables need to be declared and initialized *before*
//    the Extended Asm block but *after* normal local variables
//    (see comment in `codegen_inline_asm` for explanation).
//
// With that in mind, let's see how we translate Rust syntax to GCC
// (from now on, `CC` stands for "constraint code"):
//
// * `out(reg_class) var`   -> translated to output operand: `"=CC"(var)`
// * `inout(reg_class) var` -> translated to output operand: `"+CC"(var)`
// * `in(reg_class) var`    -> translated to input operand: `"CC"(var)`
//
// * `out(reg_class) _` -> translated to one `=r(tmp)`, where "tmp" is a temporary unused variable
//
// * `out("explicit register") _` -> not translated to any operands, register is simply added to clobbers list
//
// * `inout(reg_class) in_var => out_var` -> translated to two operands:
//                              output: `"=CC"(in_var)`
//                              input:  `"num"(out_var)` where num is the GCC index
//                                       of the corresponding output operand
//
// * `inout(reg_class) in_var => _` -> same as `inout(reg_class) in_var => tmp`,
//                                      where "tmp" is a temporary unused variable
//
// * `out/in/inout("explicit register") var` -> translated to one or two operands as described above
//                                              with `"r"(var)` constraint,
//                                              and one register variable assigned to the desired register.

const ATT_SYNTAX_INS: &str = ".att_syntax noprefix\n\t";
const INTEL_SYNTAX_INS: &str = "\n\t.intel_syntax noprefix";

struct AsmOutOperand<'a, 'tcx, 'gcc> {
    rust_idx: usize,
    constraint: &'a str,
    late: bool,
    readwrite: bool,

    tmp_var: LValue<'gcc>,
    out_place: Option<PlaceRef<'tcx, RValue<'gcc>>>,
}

struct AsmInOperand<'a, 'tcx> {
    rust_idx: usize,
    constraint: Cow<'a, str>,
    val: RValue<'tcx>,
}

impl AsmOutOperand<'_, '_, '_> {
    fn to_constraint(&self) -> String {
        let mut res = String::with_capacity(self.constraint.len() + self.late as usize + 1);

        let sign = if self.readwrite { '+' } else { '=' };
        res.push(sign);
        if !self.late {
            res.push('&');
        }

        res.push_str(self.constraint);
        res
    }
}

enum ConstraintOrRegister {
    Constraint(&'static str),
    Register(&'static str),
}

impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
    fn codegen_inline_asm(
        &mut self,
        template: &[InlineAsmTemplatePiece],
        rust_operands: &[InlineAsmOperandRef<'tcx, Self>],
        options: InlineAsmOptions,
        span: &[Span],
        instance: Instance<'_>,
        dest: Option<Self::BasicBlock>,
        _dest_catch_funclet: Option<(Self::BasicBlock, Option<&Self::Funclet>)>,
    ) {
        if options.contains(InlineAsmOptions::MAY_UNWIND) {
            self.sess().dcx().create_err(UnwindingInlineAsm { span: span[0] }).emit();
            return;
        }

        let asm_arch = self.tcx.sess.asm_arch.unwrap();
        let is_x86 = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64);
        let att_dialect = is_x86 && options.contains(InlineAsmOptions::ATT_SYNTAX);

        // GCC index of an output operand equals its position in the array
        let mut outputs = vec![];

        // GCC index of an input operand equals its position in the array
        // added to `outputs.len()`
        let mut inputs = vec![];

        // GCC index of a label equals its position in the array added to
        // `outputs.len() + inputs.len()`.
        let mut labels = vec![];

        // Clobbers collected from `out("explicit register") _` and `inout("explicit_reg") var => _`
        let mut clobbers = vec![];

        // We're trying to preallocate space for the template
        let mut constants_len = 0;

        // There are rules we must adhere to if we want GCC to do the right thing:
        //
        // * Every local variable that the asm block uses as an output must be declared *before*
        //   the asm block.
        // * There must be no instructions whatsoever between the register variables and the asm.
        //
        // Therefore, the backend must generate the instructions strictly in this order:
        //
        // 1. Output variables.
        // 2. Register variables.
        // 3. The asm block.
        //
        // We also must make sure that no input operands are emitted before output operands.
        //
        // This is why we work in passes, first emitting local vars, then local register vars.
        // Also, we don't emit any asm operands immediately; we save them to
        // the one of the buffers to be emitted later.

        let mut input_registers = vec![];

        for op in rust_operands {
            if let InlineAsmOperandRef::In { reg, .. } = *op
                && let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg)
            {
                input_registers.push(reg_name);
            }
        }

        // 1. Normal variables (and saving operands to buffers).
        for (rust_idx, op) in rust_operands.iter().enumerate() {
            match *op {
                InlineAsmOperandRef::Out { reg, late, place } => {
                    use ConstraintOrRegister::*;

                    let (constraint, ty) = match (reg_to_gcc(reg), place) {
                        (Constraint(constraint), Some(place)) => {
                            (constraint, place.layout.gcc_type(self.cx))
                        }
                        // When `reg` is a class and not an explicit register but the out place is not specified,
                        // we need to create an unused output variable to assign the output to. This var
                        // needs to be of a type that's "compatible" with the register class, but specific type
                        // doesn't matter.
                        (Constraint(constraint), None) => {
                            (constraint, dummy_output_type(self.cx, reg.reg_class()))
                        }
                        (Register(_), Some(_)) => {
                            // left for the next pass
                            continue;
                        }
                        (Register(reg_name), None) => {
                            if input_registers.contains(&reg_name) {
                                // the `clobber_abi` operand is converted into a series of
                                // `lateout("reg") _` operands. Of course, a user could also
                                // explicitly define such an output operand.
                                //
                                // GCC does not allow input registers to be clobbered, so if this out register
                                // is also used as an in register, do not add it to the clobbers list.
                                // it will be treated as a lateout register with `out_place: None`
                                if !late {
                                    bug!("input registers can only be used as lateout registers");
                                }
                                ("r", dummy_output_type(self.cx, reg.reg_class()))
                            } else {
                                let is_target_supported = match reg.reg_class() {
                                    // `clobber_abi` clobbers spe_acc on all PowerPC targets. This
                                    // register is unique to the powerpc*spe target, and the target
                                    // is not supported by gcc. Ignore it.
                                    InlineAsmRegClass::PowerPC(
                                        PowerPCInlineAsmRegClass::spe_acc,
                                    ) => false,
                                    // `clobber_abi` can add lots of clobbers that are not supported by the target,
                                    // such as AVX-512 registers, so we just ignore unsupported registers
                                    x => x.supported_types(asm_arch, true).iter().any(
                                        |&(_, feature)| {
                                            if let Some(feature) = feature {
                                                self.tcx
                                                    .asm_target_features(instance.def_id())
                                                    .contains(&feature)
                                            } else {
                                                true // Register class is unconditionally supported
                                            }
                                        },
                                    ),
                                };

                                if is_target_supported && !clobbers.contains(&reg_name) {
                                    clobbers.push(reg_name);
                                }
                                continue;
                            }
                        }
                    };

                    let tmp_var = self.current_func().new_local(None, ty, "output_register");
                    outputs.push(AsmOutOperand {
                        constraint,
                        rust_idx,
                        late,
                        readwrite: false,
                        tmp_var,
                        out_place: place,
                    });
                }

                InlineAsmOperandRef::In { reg, value } => {
                    if let ConstraintOrRegister::Constraint(constraint) = reg_to_gcc(reg) {
                        inputs.push(AsmInOperand {
                            constraint: Cow::Borrowed(constraint),
                            rust_idx,
                            val: value.immediate(),
                        });
                    } else {
                        // left for the next pass
                        continue;
                    }
                }

                InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
                    let ConstraintOrRegister::Constraint(constraint) = reg_to_gcc(reg) else {
                        // left for the next pass
                        continue;
                    };

                    // Rustc frontend guarantees that input and output types are "compatible",
                    // so we can just use input var's type for the output variable.
                    //
                    // This decision is also backed by the fact that LLVM needs in and out
                    // values to be of *exactly the same type*, not just "compatible".
                    // I'm not sure if GCC is so picky too, but better safe than sorry.
                    let ty = in_value.layout.gcc_type(self.cx);
                    let tmp_var = self.current_func().new_local(None, ty, "output_register");

                    // If the out_place is None (i.e `inout(reg) _` syntax was used), we translate
                    // it to one "readwrite (+) output variable", otherwise we translate it to two
                    // "out and tied in" vars as described above.
                    let readwrite = out_place.is_none();
                    outputs.push(AsmOutOperand {
                        constraint,
                        rust_idx,
                        late,
                        readwrite,
                        tmp_var,
                        out_place,
                    });

                    if !readwrite {
                        let out_gcc_idx = outputs.len() - 1;
                        let constraint = Cow::Owned(out_gcc_idx.to_string());

                        inputs.push(AsmInOperand {
                            constraint,
                            rust_idx,
                            val: in_value.immediate(),
                        });
                    }
                }

                InlineAsmOperandRef::Const { ref string } => {
                    constants_len += string.len() + att_dialect as usize;
                }

                InlineAsmOperandRef::SymFn { instance } => {
                    // TODO(@Amanieu): Additional mangling is needed on
                    // some targets to add a leading underscore (Mach-O)
                    // or byte count suffixes (x86 Windows).
                    constants_len += self.tcx.symbol_name(instance).name.len();
                }
                InlineAsmOperandRef::SymStatic { def_id } => {
                    // TODO(@Amanieu): Additional mangling is needed on
                    // some targets to add a leading underscore (Mach-O).
                    constants_len +=
                        self.tcx.symbol_name(Instance::mono(self.tcx, def_id)).name.len();
                }

                InlineAsmOperandRef::Label { label } => {
                    labels.push(label);
                }
            }
        }

        // 2. Register variables.
        for (rust_idx, op) in rust_operands.iter().enumerate() {
            match *op {
                // `out("explicit register") var`
                InlineAsmOperandRef::Out { reg, late, place } => {
                    if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) {
                        let out_place = if let Some(place) = place {
                            place
                        } else {
                            // processed in the previous pass
                            continue;
                        };

                        let ty = out_place.layout.gcc_type(self.cx);
                        let tmp_var = self.current_func().new_local(None, ty, "output_register");
                        tmp_var.set_register_name(reg_name);

                        outputs.push(AsmOutOperand {
                            constraint: "r",
                            rust_idx,
                            late,
                            readwrite: false,
                            tmp_var,
                            out_place: Some(out_place),
                        });
                    }

                    // processed in the previous pass
                }

                // `in("explicit register") var`
                InlineAsmOperandRef::In { reg, value } => {
                    if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) {
                        let ty = value.layout.gcc_type(self.cx);
                        let reg_var = self.current_func().new_local(None, ty, "input_register");
                        reg_var.set_register_name(reg_name);
                        self.llbb().add_assignment(None, reg_var, value.immediate());

                        inputs.push(AsmInOperand {
                            constraint: "r".into(),
                            rust_idx,
                            val: reg_var.to_rvalue(),
                        });
                    }

                    // processed in the previous pass
                }

                // `inout("explicit register") in_var => out_var`
                InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => {
                    if let ConstraintOrRegister::Register(reg_name) = reg_to_gcc(reg) {
                        // See explanation in the first pass.
                        let ty = in_value.layout.gcc_type(self.cx);
                        let tmp_var = self.current_func().new_local(None, ty, "output_register");
                        tmp_var.set_register_name(reg_name);

                        outputs.push(AsmOutOperand {
                            constraint: "r",
                            rust_idx,
                            late,
                            readwrite: false,
                            tmp_var,
                            out_place,
                        });

                        let constraint = Cow::Owned((outputs.len() - 1).to_string());
                        inputs.push(AsmInOperand {
                            constraint,
                            rust_idx,
                            val: in_value.immediate(),
                        });
                    }

                    // processed in the previous pass
                }

                InlineAsmOperandRef::SymFn { instance } => {
                    inputs.push(AsmInOperand {
                        constraint: "X".into(),
                        rust_idx,
                        val: get_fn(self.cx, instance).get_address(None),
                    });
                }

                InlineAsmOperandRef::SymStatic { def_id } => {
                    inputs.push(AsmInOperand {
                        constraint: "X".into(),
                        rust_idx,
                        val: self.cx.get_static(def_id).get_address(None),
                    });
                }

                InlineAsmOperandRef::Const { .. } => {
                    // processed in the previous pass
                }

                InlineAsmOperandRef::Label { .. } => {
                    // processed in the previous pass
                }
            }
        }

        // 3. Build the template string

        let mut template_str =
            String::with_capacity(estimate_template_length(template, constants_len, att_dialect));
        if att_dialect {
            template_str.push_str(ATT_SYNTAX_INS);
        }

        for piece in template {
            match *piece {
                InlineAsmTemplatePiece::String(ref string) => {
                    for char in string.chars() {
                        // TODO(antoyo): might also need to escape | if rustc doesn't do it.
                        let escaped_char = match char {
                            '%' => "%%",
                            '{' => "%{",
                            '}' => "%}",
                            _ => {
                                template_str.push(char);
                                continue;
                            }
                        };
                        template_str.push_str(escaped_char);
                    }
                }
                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => {
                    let mut push_to_template = |modifier, gcc_idx| {
                        use std::fmt::Write;

                        template_str.push('%');
                        if let Some(modifier) = modifier {
                            template_str.push(modifier);
                        }
                        write!(template_str, "{}", gcc_idx).expect("pushing to string failed");
                    };

                    match rust_operands[operand_idx] {
                        InlineAsmOperandRef::Out { reg, .. } => {
                            let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier);
                            let gcc_index = outputs
                                .iter()
                                .position(|op| operand_idx == op.rust_idx)
                                .expect("wrong rust index");
                            push_to_template(modifier, gcc_index);
                        }

                        InlineAsmOperandRef::In { reg, .. } => {
                            let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier);
                            let in_gcc_index = inputs
                                .iter()
                                .position(|op| operand_idx == op.rust_idx)
                                .expect("wrong rust index");
                            let gcc_index = in_gcc_index + outputs.len();
                            push_to_template(modifier, gcc_index);
                        }

                        InlineAsmOperandRef::InOut { reg, .. } => {
                            let modifier = modifier_to_gcc(asm_arch, reg.reg_class(), modifier);

                            // The input register is tied to the output, so we can just use the index of the output register
                            let gcc_index = outputs
                                .iter()
                                .position(|op| operand_idx == op.rust_idx)
                                .expect("wrong rust index");
                            push_to_template(modifier, gcc_index);
                        }

                        InlineAsmOperandRef::SymFn { instance } => {
                            // TODO(@Amanieu): Additional mangling is needed on
                            // some targets to add a leading underscore (Mach-O)
                            // or byte count suffixes (x86 Windows).
                            let name = self.tcx.symbol_name(instance).name;
                            template_str.push_str(name);
                        }

                        InlineAsmOperandRef::SymStatic { def_id } => {
                            // TODO(@Amanieu): Additional mangling is needed on
                            // some targets to add a leading underscore (Mach-O).
                            let instance = Instance::mono(self.tcx, def_id);
                            let name = self.tcx.symbol_name(instance).name;
                            template_str.push_str(name);
                        }

                        InlineAsmOperandRef::Const { ref string } => {
                            template_str.push_str(string);
                        }

                        InlineAsmOperandRef::Label { label } => {
                            let label_gcc_index =
                                labels.iter().position(|&l| l == label).expect("wrong rust index");
                            let gcc_index = label_gcc_index + outputs.len() + inputs.len();
                            push_to_template(Some('l'), gcc_index);
                        }
                    }
                }
            }
        }

        if att_dialect {
            template_str.push_str(INTEL_SYNTAX_INS);
        }

        // 4. Generate Extended Asm block

        let block = self.llbb();
        let extended_asm = if let Some(dest) = dest {
            assert!(!labels.is_empty());
            block.end_with_extended_asm_goto(None, &template_str, &labels, Some(dest))
        } else {
            block.add_extended_asm(None, &template_str)
        };

        for op in &outputs {
            extended_asm.add_output_operand(None, &op.to_constraint(), op.tmp_var);
        }

        for op in &inputs {
            extended_asm.add_input_operand(None, &op.constraint, op.val);
        }

        for clobber in clobbers.iter() {
            extended_asm.add_clobber(clobber);
        }

        if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) {
            match asm_arch {
                InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {
                    // "cc" is cr0 on powerpc.
                }
                // TODO(@Commeownist): I'm not 100% sure this one clobber is sufficient
                // on all architectures. For instance, what about FP stack?
                _ => {
                    extended_asm.add_clobber("cc");
                }
            }
        }
        if !options.contains(InlineAsmOptions::NOMEM) {
            extended_asm.add_clobber("memory");
        }
        if !options.contains(InlineAsmOptions::PURE) {
            extended_asm.set_volatile_flag(true);
        }
        if !options.contains(InlineAsmOptions::NOSTACK) {
            // TODO(@Commeownist): figure out how to align stack
        }
        if dest.is_none() && options.contains(InlineAsmOptions::NORETURN) {
            let builtin_unreachable = self.context.get_builtin_function("__builtin_unreachable");
            let builtin_unreachable: RValue<'gcc> =
                unsafe { std::mem::transmute(builtin_unreachable) };
            self.call(self.type_void(), None, None, builtin_unreachable, &[], None, None);
        }

        // Write results to outputs.
        //
        // We need to do this because:
        //  1. Turning `PlaceRef` into `RValue` is error-prone and has nasty edge cases
        //     (especially with current `rustc_backend_ssa` API).
        //  2. Not every output operand has an `out_place`, and it's required by `add_output_operand`.
        //
        // Instead, we generate a temporary output variable for each output operand, and then this loop,
        // generates `out_place = tmp_var;` assignments if out_place exists.
        for op in &outputs {
            if let Some(place) = op.out_place {
                OperandValue::Immediate(op.tmp_var.to_rvalue()).store(self, place);
            }
        }
    }
}

fn estimate_template_length(
    template: &[InlineAsmTemplatePiece],
    constants_len: usize,
    att_dialect: bool,
) -> usize {
    let len: usize = template
        .iter()
        .map(|piece| {
            match *piece {
                InlineAsmTemplatePiece::String(ref string) => string.len(),
                InlineAsmTemplatePiece::Placeholder { .. } => {
                    // '%' + 1 char modifier + 1 char index
                    3
                }
            }
        })
        .sum();

    // increase it by 5% to account for possible '%' signs that'll be duplicated
    // I pulled the number out of blue, but should be fair enough
    // as the upper bound
    let mut res = (len as f32 * 1.05) as usize + constants_len;

    if att_dialect {
        res += INTEL_SYNTAX_INS.len() + ATT_SYNTAX_INS.len();
    }
    res
}

/// Converts a register class to a GCC constraint code.
fn reg_to_gcc(reg_or_reg_class: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
    match reg_or_reg_class {
        InlineAsmRegOrRegClass::Reg(reg) => {
            ConstraintOrRegister::Register(explicit_reg_to_gcc(reg))
        }
        InlineAsmRegOrRegClass::RegClass(reg_class) => {
            ConstraintOrRegister::Constraint(reg_class_to_gcc(reg_class))
        }
    }
}

fn explicit_reg_to_gcc(reg: InlineAsmReg) -> &'static str {
    // For explicit registers, we have to create a register variable: https://stackoverflow.com/a/31774784/389119
    match reg {
        InlineAsmReg::X86(reg) => {
            // TODO(antoyo): add support for vector register.
            match reg.reg_class() {
                X86InlineAsmRegClass::reg_byte => {
                    // GCC does not support the `b` suffix, so we just strip it
                    // see https://github.com/rust-lang/rustc_codegen_gcc/issues/485
                    reg.name().trim_end_matches('b')
                }
                _ => match reg.name() {
                    // Some of registers' names does not map 1-1 from rust to gcc
                    "st(0)" => "st",

                    name => name,
                },
            }
        }
        InlineAsmReg::Arm(reg) => reg.name(),
        InlineAsmReg::AArch64(reg) => reg.name(),
        _ => unimplemented!(),
    }
}

/// They can be retrieved from https://gcc.gnu.org/onlinedocs/gcc/Machine-Constraints.html
fn reg_class_to_gcc(reg_class: InlineAsmRegClass) -> &'static str {
    match reg_class {
        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w",
        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x",
        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "t",
        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => "d",
        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => "r",
        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w",
        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e",
        InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w",
        InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::preg) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => "f",
        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => "a",
        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => "d",
        InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::freg) => "f",
        InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "d", // more specific than "r"
        InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f",
        InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r",
        // https://github.com/gcc-mirror/gcc/blob/master/gcc/config/nvptx/nvptx.md -> look for
        // "define_constraint".
        InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h",
        InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r",
        InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l",

        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b",
        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f",
        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vreg) => "v",
        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vsreg) => "wa",
        InlineAsmRegClass::PowerPC(
            PowerPCInlineAsmRegClass::cr
            | PowerPCInlineAsmRegClass::ctr
            | PowerPCInlineAsmRegClass::lr
            | PowerPCInlineAsmRegClass::xer
            | PowerPCInlineAsmRegClass::spe_acc,
        ) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg_pair) => "R",
        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f",
        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q",
        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q",
        InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
        | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x",
        InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v",
        InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "Yk",
        InlineAsmRegClass::X86(
            X86InlineAsmRegClass::kreg0
            | X86InlineAsmRegClass::x87_reg
            | X86InlineAsmRegClass::mmx_reg
            | X86InlineAsmRegClass::tmm_reg,
        ) => unreachable!("clobber-only"),
        InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
            bug!("GCC backend does not support SPIR-V")
        }
        InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r",
        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg_addr) => "a",
        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f",
        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::vreg) => "v",
        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::areg) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::Sparc(SparcInlineAsmRegClass::reg) => "r",
        InlineAsmRegClass::Sparc(SparcInlineAsmRegClass::yreg) => unreachable!("clobber-only"),
        InlineAsmRegClass::Err => unreachable!(),
    }
}

/// Type to use for outputs that are discarded. It doesn't really matter what
/// the type is, as long as it is valid for the constraint code.
fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegClass) -> Type<'gcc> {
    match reg {
        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(),
        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
            cx.type_vector(cx.type_i64(), 2)
        }
        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(),
        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(),
        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(),
        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
            cx.type_vector(cx.type_i64(), 2)
        }
        InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(),
        InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::preg) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::reg) => cx.type_i32(),
        InlineAsmRegClass::LoongArch(LoongArchInlineAsmRegClass::freg) => cx.type_f32(),
        InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(),
        InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(),
        InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(),
        InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(),
        InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(),
        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(),
        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(),
        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(),
        InlineAsmRegClass::PowerPC(
            PowerPCInlineAsmRegClass::vreg | PowerPCInlineAsmRegClass::vsreg,
        ) => cx.type_vector(cx.type_i32(), 4),
        InlineAsmRegClass::PowerPC(
            PowerPCInlineAsmRegClass::cr
            | PowerPCInlineAsmRegClass::ctr
            | PowerPCInlineAsmRegClass::lr
            | PowerPCInlineAsmRegClass::xer
            | PowerPCInlineAsmRegClass::spe_acc,
        ) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(),
        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg_pair) => {
            if cx.tcx.sess.asm_arch.unwrap() == InlineAsmArch::RiscV64 {
                cx.type_i128()
            } else {
                cx.type_i64()
            }
        }
        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(),
        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
        | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(),
        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(),
        InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg)
        | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg)
        | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(),
        InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(),
        InlineAsmRegClass::X86(X86InlineAsmRegClass::x87_reg)
        | InlineAsmRegClass::X86(X86InlineAsmRegClass::mmx_reg)
        | InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg0)
        | InlineAsmRegClass::X86(X86InlineAsmRegClass::tmm_reg) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(),
        InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(),
        InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(),
        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(),
        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(),
        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(),
        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(),
        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
        InlineAsmRegClass::S390x(
            S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr,
        ) => cx.type_i32(),
        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i64(), 2),
        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::areg) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::Sparc(SparcInlineAsmRegClass::reg) => cx.type_i32(),
        InlineAsmRegClass::Sparc(SparcInlineAsmRegClass::yreg) => unreachable!("clobber-only"),
        InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),
        InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_data) => cx.type_i32(),
        InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::reg) => cx.type_i32(),
        InlineAsmRegClass::CSKY(CSKYInlineAsmRegClass::freg) => cx.type_f32(),
        InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
            bug!("GCC backend does not support SPIR-V")
        }
        InlineAsmRegClass::Err => unreachable!(),
    }
}

impl<'gcc, 'tcx> AsmCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
    fn codegen_global_asm(
        &mut self,
        template: &[InlineAsmTemplatePiece],
        operands: &[GlobalAsmOperandRef<'tcx>],
        options: InlineAsmOptions,
        _line_spans: &[Span],
    ) {
        let asm_arch = self.tcx.sess.asm_arch.unwrap();

        // Default to Intel syntax on x86
        let att_dialect = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64)
            && options.contains(InlineAsmOptions::ATT_SYNTAX);

        // Build the template string
        let mut template_str = ".pushsection .text\n".to_owned();
        if att_dialect {
            template_str.push_str(".att_syntax\n");
        }
        for piece in template {
            match *piece {
                InlineAsmTemplatePiece::String(ref string) => {
                    let mut index = 0;
                    while index < string.len() {
                        // NOTE: gcc does not allow inline comment, so remove them.
                        let comment_index = string[index..]
                            .find("//")
                            .map(|comment_index| comment_index + index)
                            .unwrap_or(string.len());
                        template_str.push_str(&string[index..comment_index]);
                        index = string[comment_index..]
                            .find('\n')
                            .map(|index| index + comment_index)
                            .unwrap_or(string.len());
                    }
                }
                InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => {
                    match operands[operand_idx] {
                        GlobalAsmOperandRef::Const { ref string } => {
                            // Const operands get injected directly into the
                            // template. Note that we don't need to escape %
                            // here unlike normal inline assembly.
                            template_str.push_str(string);
                        }

                        GlobalAsmOperandRef::SymFn { instance } => {
                            let function = get_fn(self, instance);
                            self.add_used_function(function);
                            // TODO(@Amanieu): Additional mangling is needed on
                            // some targets to add a leading underscore (Mach-O)
                            // or byte count suffixes (x86 Windows).
                            let name = self.tcx.symbol_name(instance).name;
                            template_str.push_str(name);
                        }

                        GlobalAsmOperandRef::SymStatic { def_id } => {
                            // TODO(antoyo): set the global variable as used.
                            // TODO(@Amanieu): Additional mangling is needed on
                            // some targets to add a leading underscore (Mach-O).
                            let instance = Instance::mono(self.tcx, def_id);
                            let name = self.tcx.symbol_name(instance).name;
                            template_str.push_str(name);
                        }
                    }
                }
            }
        }

        if att_dialect {
            template_str.push_str("\n\t.intel_syntax noprefix");
        }
        // NOTE: seems like gcc will put the asm in the wrong section, so set it to .text manually.
        template_str.push_str("\n.popsection");
        self.context.add_top_level_asm(None, &template_str);
    }

    fn mangled_name(&self, instance: Instance<'tcx>) -> String {
        // TODO(@Amanieu): Additional mangling is needed on
        // some targets to add a leading underscore (Mach-O)
        // or byte count suffixes (x86 Windows).
        self.tcx.symbol_name(instance).name.to_string()
    }
}

fn modifier_to_gcc(
    arch: InlineAsmArch,
    reg: InlineAsmRegClass,
    modifier: Option<char>,
) -> Option<char> {
    // The modifiers can be retrieved from
    // https://gcc.gnu.org/onlinedocs/gcc/Modifiers.html#Modifiers
    match reg {
        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier,
        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg)
        | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => {
            if modifier == Some('v') { None } else { modifier }
        }
        InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None,
        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None,
        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'),
        InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8)
        | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => {
            if modifier.is_none() {
                Some('q')
            } else {
                modifier
            }
        }
        InlineAsmRegClass::Hexagon(_) => None,
        InlineAsmRegClass::LoongArch(_) => None,
        InlineAsmRegClass::Mips(_) => None,
        InlineAsmRegClass::Nvptx(_) => None,
        InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::vsreg) => {
            if modifier.is_none() {
                Some('x')
            } else {
                modifier
            }
        }
        InlineAsmRegClass::PowerPC(_) => None,
        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg)
        | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg_pair)
        | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None,
        InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg)
        | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier {
            None => {
                if arch == InlineAsmArch::X86_64 {
                    Some('q')
                } else {
                    Some('k')
                }
            }
            Some('l') => Some('b'),
            Some('h') => Some('h'),
            Some('x') => Some('w'),
            Some('e') => Some('k'),
            Some('r') => Some('q'),
            _ => unreachable!(),
        },
        InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None,
        InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg)
        | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg)
        | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) {
            (X86InlineAsmRegClass::xmm_reg, None) => Some('x'),
            (X86InlineAsmRegClass::ymm_reg, None) => Some('t'),
            (X86InlineAsmRegClass::zmm_reg, None) => Some('g'),
            (_, Some('x')) => Some('x'),
            (_, Some('y')) => Some('t'),
            (_, Some('z')) => Some('g'),
            _ => unreachable!(),
        },
        InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None,
        InlineAsmRegClass::X86(
            X86InlineAsmRegClass::x87_reg
            | X86InlineAsmRegClass::mmx_reg
            | X86InlineAsmRegClass::kreg0
            | X86InlineAsmRegClass::tmm_reg,
        ) => {
            unreachable!("clobber-only")
        }
        InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None,
        InlineAsmRegClass::Bpf(_) => None,
        InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair)
        | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw)
        | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier {
            Some('h') => Some('B'),
            Some('l') => Some('A'),
            _ => None,
        },
        InlineAsmRegClass::Avr(_) => None,
        InlineAsmRegClass::S390x(_) => None,
        InlineAsmRegClass::Sparc(_) => None,
        InlineAsmRegClass::Msp430(_) => None,
        InlineAsmRegClass::M68k(_) => None,
        InlineAsmRegClass::CSKY(_) => None,
        InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => {
            bug!("LLVM backend does not support SPIR-V")
        }
        InlineAsmRegClass::Err => unreachable!(),
    }
}