Ensure demangling recovers calling convention correctly

plafosse · plafosse · commit 6b58d2e6fbed · 2026-05-01T15:34:21.000-04:00
diff --git a/demangle.cpp b/demangle.cpp
@@ -47,8 +47,19 @@ namespace BinaryNinja {
 	bool DemangleMS(Architecture* arch, const std::string& mangledName, Ref<Type>& outType, QualifiedName& outVarName,
 	    BinaryView* view)
 	{
-		const bool simplify = Settings::Instance()->Get<bool>("analysis.types.templateSimplifier", view);
-		return DemangleMS(arch, mangledName, outType, outVarName, simplify);
+		BNType* localType = nullptr;
+		char** localVarName = nullptr;
+		size_t localSize = 0;
+		if (!BNDemangleMSWithOptions(arch->GetObject(), mangledName.c_str(), &localType, &localVarName, &localSize,
+			view ? view->GetObject() : nullptr))
+			return false;
+		outType = localType ? new Type(localType) : nullptr;
+		for (size_t i = 0; i < localSize; i++)
+		{
+			outVarName.push_back(localVarName[i]);
+		}
+		BNFreeDemangledName(&localVarName, localSize);
+		return true;
 	}
 
 	bool DemangleMS(Architecture* arch, const std::string& mangledName, Ref<Type>& outType, QualifiedName& outVarName,
diff --git a/demangler/gnu3/demangled_type_node.cpp b/demangler/gnu3/demangled_type_node.cpp
@@ -84,7 +84,7 @@ DemangledTypeNode::DemangledTypeNode()
 	: m_typeClass(VoidTypeClass), m_ntrClass(UnknownNamedTypeClass),
 	  m_pointerReference(PointerReferenceType), m_nameType(NoNameType),
 	  m_callingConventionName(NoCallingConvention), m_pointerSuffixBits(0),
-	  m_returnTypeConfidence(BN_DEFAULT_CONFIDENCE),
+	  m_returnTypeConfidence(BN_FULL_CONFIDENCE),
 	  m_const(false), m_volatile(false), m_signed(false), m_hasVariableArgs(false),
 	  m_hasTemplateArgs(false), m_width(0),
 	  m_isMemberPointer(false),
@@ -703,8 +703,11 @@ Ref<Type> DemangledTypeNode::Finalize() const
 			Ref<Type> pType = p.type ? p.type->Finalize() : Ref<Type>(Type::VoidType());
 			finalParams.push_back({p.name, pType, true, Variable()});
 		}
+		Confidence<Ref<CallingConvention>> callingConvention;
+		if (m_callingConvention)
+			callingConvention = Confidence<Ref<CallingConvention>>(m_callingConvention, BN_FULL_CONFIDENCE);
 		TypeBuilder tb = TypeBuilder::FunctionType(
-			retType->WithConfidence(static_cast<uint8_t>(m_returnTypeConfidence)), nullptr, finalParams,
+			retType->WithConfidence(static_cast<uint8_t>(m_returnTypeConfidence)), callingConvention, finalParams,
 			Confidence<bool>(m_hasVariableArgs, m_hasVariableArgs ? BN_DEFAULT_CONFIDENCE : 0));
 		tb.SetConst(m_const);
 		tb.SetVolatile(m_volatile);
diff --git a/demangler/gnu3/demangled_type_node.h b/demangler/gnu3/demangled_type_node.h
@@ -111,6 +111,7 @@ class DemangledTypeNode
 	void AddPointerSuffix(BNPointerSuffix ps) { m_pointerSuffixBits |= PointerSuffixBit(ps); }
 	void SetReturnTypeConfidence(int8_t c) { m_returnTypeConfidence = c; }
 	void SetCallingConventionName(BNCallingConventionName cc) { m_callingConventionName = cc; }
+	void SetCallingConvention(BN::Ref<BN::CallingConvention> cc) { m_callingConvention = std::move(cc); }
 	void SetNTRType(BNNamedTypeReferenceClass cls) { m_ntrClass = cls; }
 	void SetImplicitThisParameter(DemangledTypeNode type);
 
@@ -129,6 +130,7 @@ class DemangledTypeNode
 	BNReferenceType m_pointerReference;
 	BNNameType m_nameType;
 	BNCallingConventionName m_callingConventionName;
+	BN::Ref<BN::CallingConvention> m_callingConvention;
 	uint8_t m_pointerSuffixBits;
 	uint8_t m_returnTypeConfidence;
 	bool m_const;
diff --git a/demangler/msvc/demangle_msvc.cpp b/demangler/msvc/demangle_msvc.cpp
@@ -1914,6 +1914,38 @@ BNCallingConventionName Demangle::DemangleCallingConvention()
 }
 
 
+Ref<CallingConvention> Demangle::ResolveCallingConvention(BNCallingConventionName cc) const
+{
+	switch (cc)
+	{
+	case CdeclCallingConvention:
+		if (m_platform && m_platform->GetCdeclCallingConvention())
+			return m_platform->GetCdeclCallingConvention();
+		if (m_arch && m_arch->GetCdeclCallingConvention())
+			return m_arch->GetCdeclCallingConvention();
+		return m_arch ? m_arch->GetCallingConventionByName("cdecl") : nullptr;
+	case STDCallCallingConvention:
+		if (m_platform && m_platform->GetStdcallCallingConvention())
+			return m_platform->GetStdcallCallingConvention();
+		if (m_arch && m_arch->GetStdcallCallingConvention())
+			return m_arch->GetStdcallCallingConvention();
+		return m_arch ? m_arch->GetCallingConventionByName("stdcall") : nullptr;
+	case FastcallCallingConvention:
+		if (m_platform && m_platform->GetFastcallCallingConvention())
+			return m_platform->GetFastcallCallingConvention();
+		if (m_arch && m_arch->GetFastcallCallingConvention())
+			return m_arch->GetFastcallCallingConvention();
+		return m_arch ? m_arch->GetCallingConventionByName("fastcall") : nullptr;
+	case ThisCallCallingConvention:
+		if (m_arch)
+			return m_arch->GetCallingConventionByName("thiscall");
+		return nullptr;
+	default:
+		return nullptr;
+	}
+}
+
+
 void Demangle::ConsumeExtendedModifierPrefix()
 {
 	while (reader.PeekMatch("$A", 2))
@@ -2112,25 +2144,10 @@ DemangledTypeNode Demangle::DemangleFunction(BNNameType classFunctionType, bool
 
 	MSVC_TRACE("\tDemangle Function Parameters %s", reader.GetRaw());
 	vector<DemangledTypeNode::Param> params;
-	bool needsThisPtr = false;
-	if (includeImplicitThis && cc == ThisCallCallingConvention)
-	{
-		needsThisPtr = true;
-	}
-	if (includeImplicitThis && funcClass != NoneFunctionClass)
-	{
-		if ((funcClass & VirtualFunctionClass) == VirtualFunctionClass
-		    || (funcClass & StaticThunkFunctionClass) == StaticThunkFunctionClass
-		    || (funcClass & VirtualThunkFunctionClass) == VirtualThunkFunctionClass)
-		{
-			needsThisPtr = true;
-		}
-		else if ((funcClass & StaticFunctionClass) != StaticFunctionClass
-		         && (funcClass & GlobalFunctionClass) != GlobalFunctionClass)
-		{
-			needsThisPtr = true;
-		}
-	}
+	bool needsThisPtr = includeImplicitThis
+		&& funcClass != NoneFunctionClass
+		&& (funcClass & StaticFunctionClass) != StaticFunctionClass
+		&& (funcClass & GlobalFunctionClass) != GlobalFunctionClass;
 
 	DemangleVariableList(params, nameBackrefList);
 
@@ -2145,12 +2162,14 @@ DemangledTypeNode Demangle::DemangleFunction(BNNameType classFunctionType, bool
 	newType.SetPointerSuffixBits(suffix);
 	newType.SetNameType(classFunctionType);
 	newType.SetCallingConventionName(cc);
+	if (auto callingConvention = ResolveCallingConvention(cc))
+		newType.SetCallingConvention(callingConvention);
 	if (needsThisPtr)
 	{
 		NameList thisName = m_varName;
 		if (classFunctionType != OperatorReturnTypeNameType && !thisName.empty())
 			thisName.pop_back();
-		auto thisNamedType = DemangledTypeNode::NamedType(UnknownNamedTypeClass, std::move(thisName));
+		auto thisNamedType = DemangledTypeNode::NamedType(TypedefNamedTypeClass, std::move(thisName));
 		newType.SetImplicitThisParameter(DemangledTypeNode::PointerType(
 			m_arch, std::move(thisNamedType), false, false, PointerReferenceType));
 	}
@@ -2516,6 +2535,8 @@ bool Demangle::DemangleMS(Architecture* arch, const string& mangledName, Ref<Typ
 	outType = nullptr;
 	if (mangledName.empty() || (mangledName[0] != '?' && mangledName[0] != '.'))
 		return false;
+	if (view)
+		return DemangleMS(mangledName, outType, outVarName, view);
 	return DemangleMS(arch, mangledName, outType, outVarName);
 }
 
@@ -2525,9 +2546,32 @@ bool Demangle::DemangleMS(Architecture* arch, const string& mangledName, Ref<Typ
 	outType = nullptr;
 	if (mangledName.empty() || (mangledName[0] != '?' && mangledName[0] != '.'))
 		return false;
+	if (view)
+		return DemangleMS(mangledName, outType, outVarName, Ref<BinaryView>(view));
 	return DemangleMS(arch, mangledName, outType, outVarName);
 }
 
+bool Demangle::DemangleMS(Platform* platform, const string& mangledName, Ref<Type>& outType,
+                          QualifiedName& outVarName)
+{
+	outType = nullptr;
+	if (!platform || mangledName.empty() || (mangledName[0] != '?' && mangledName[0] != '.'))
+		return false;
+	try
+	{
+		Demangle demangle(Ref<Platform>(platform), mangledName);
+		auto result = demangle.Finalize();
+		outType = std::move(result.first);
+		outVarName = std::move(result.second);
+	}
+	catch (DemangleException &e)
+	{
+		LogDebugForException(e, "Demangling Failed '%s' '%s;", mangledName.c_str(), e.what());
+		return false;
+	}
+	return true;
+}
+
 bool Demangle::DemangleMS(Architecture* arch, const string& mangledName, Ref<Type>& outType,
                           QualifiedName& outVarName)
 {
@@ -2573,6 +2617,15 @@ bool Demangle::DemangleMS(const string& mangledName, Ref<Type>& outType,
 	return true;
 }
 
+bool Demangle::DemangleMS(const string& mangledName, Ref<Type>& outType,
+                          QualifiedName& outVarName, BinaryView* view)
+{
+	outType = nullptr;
+	if (!view)
+		return false;
+	return DemangleMS(mangledName, outType, outVarName, Ref<BinaryView>(view));
+}
+
 
 class MSDemangler: public Demangler
 {
diff --git a/demangler/msvc/demangle_msvc.h b/demangler/msvc/demangle_msvc.h
@@ -234,6 +234,7 @@ class Demangle
 	                  BackrefList& nameBackrefList,
 	                  bool typeNameContext = false);
 	BNCallingConventionName DemangleCallingConvention();
+	BN::Ref<BN::CallingConvention> ResolveCallingConvention(BNCallingConventionName cc) const;
 	void ConsumeExtendedModifierPrefix();
 	DemangledTypeNode DemangleFunction(BNNameType classFunctionType, bool pointerSuffix, BackrefList& varList,
 		int funcClass = NoneFunctionClass, bool includeImplicitThis = true);
@@ -266,6 +267,8 @@ class Demangle
 	                       BN::QualifiedName& outVarName, const BN::Ref<BN::BinaryView>& view);
 	static bool DemangleMS(BN::Architecture* arch, const _STD_STRING& mangledName, BN::Ref<BN::Type>& outType,
 	                       BN::QualifiedName& outVarName, BN::BinaryView* view);
+	static bool DemangleMS(BN::Platform* platform, const _STD_STRING& mangledName, BN::Ref<BN::Type>& outType,
+	                       BN::QualifiedName& outVarName);
 	static bool DemangleMS(BN::Architecture* arch, const _STD_STRING& mangledName, BN::Ref<BN::Type>& outType,
 	                       BN::QualifiedName& outVarName);
 
diff --git a/plugins/pdb-ng/src/symbol_parser.rs b/plugins/pdb-ng/src/symbol_parser.rs
@@ -39,7 +39,7 @@ use crate::PDBParserInstance;
 use binaryninja::architecture::{Architecture, ArchitectureExt, Register, RegisterId};
 use binaryninja::binary_view::BinaryViewBase;
 use binaryninja::confidence::{Conf, MAX_CONFIDENCE, MIN_CONFIDENCE};
-use binaryninja::demangle::demangle_ms;
+use binaryninja::demangle::demangle_ms_with_view;
 use binaryninja::rc::Ref;
 use binaryninja::types::{FunctionParameter, QualifiedName, StructureBuilder, Type, TypeClass};
 use binaryninja::variable::{Variable, VariableSourceType};
@@ -1820,7 +1820,7 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
         raw_name: &String,
         rva: Rva,
     ) -> Result<(Option<Conf<Ref<Type>>>, Option<QualifiedName>)> {
-        let (mut t, mut name) = match demangle_ms(&self.arch, raw_name, true) {
+        let (mut t, mut name) = match demangle_ms_with_view(&self.arch, raw_name, Some(self.bv)) {
             Some((name, Some(t))) => (Some(Conf::new(t, DEMANGLE_CONFIDENCE)), name),
             Some((name, _)) => (None, name),
             _ => (None, QualifiedName::new(vec![raw_name.clone()])),
diff --git a/plugins/rtti/rtti.cpp b/plugins/rtti/rtti.cpp
@@ -3,6 +3,20 @@
 using namespace BinaryNinja;
 using namespace BinaryNinja::RTTI;
 
+namespace
+{
+	std::string NormalizeRTTIClassName(std::string name)
+	{
+		size_t beginFind = name.find_first_of(' ');
+		if (beginFind != std::string::npos)
+			name.erase(0, beginFind + 1);
+		size_t endFind = name.find(" `RTTI Type Descriptor Name'");
+		if (endFind != std::string::npos)
+			name.erase(endFind, name.length());
+		return name;
+	}
+}
+
 
 Ref<Symbol> RTTI::GetRealSymbol(BinaryView *view, uint64_t relocAddr, uint64_t symAddr)
 {
@@ -24,9 +38,9 @@ std::optional<std::string> RTTI::DemangleNameMS(BinaryView* view, bool allowMang
 {
     QualifiedName demangledName = {};
     Ref<Type> outType = {};
-    if (!DemangleMS(view->GetDefaultArchitecture(), mangledName, outType, demangledName, true))
+    if (!DemangleMS(view->GetDefaultArchitecture(), mangledName, outType, demangledName, view))
         return DemangleNameLLVM(allowMangled, mangledName);
-    return demangledName.GetString();
+    return NormalizeRTTIClassName(demangledName.GetString());
 }
 
 
@@ -90,14 +104,7 @@ std::optional<std::string> RTTI::DemangleNameLLVM(bool allowMangled, const std::
     Ref<Type> outType = {};
     if (!DemangleLLVM(mangledName, demangledName, true))
         return allowMangled ? std::optional(mangledName) : std::nullopt;
-    auto demangledNameStr = demangledName.GetString();
-    size_t beginFind = demangledNameStr.find_first_of(' ');
-    if (beginFind != std::string::npos)
-        demangledNameStr.erase(0, beginFind + 1);
-    size_t endFind = demangledNameStr.find(" `RTTI Type Descriptor Name'");
-    if (endFind != std::string::npos)
-        demangledNameStr.erase(endFind, demangledNameStr.length());
-    return demangledNameStr;
+    return NormalizeRTTIClassName(demangledName.GetString());
 }
 
 
diff --git a/rust/src/demangle.rs b/rust/src/demangle.rs
@@ -165,6 +165,46 @@ pub fn demangle_ms(
     }
 }
 
+pub fn demangle_ms_with_view(
+    arch: &CoreArchitecture,
+    mangled_name: &str,
+    view: Option<&BinaryView>,
+) -> Option<(QualifiedName, Option<Ref<Type>>)> {
+    let mangled_name = mangled_name.to_cstr();
+    let mut out_type: *mut BNType = std::ptr::null_mut();
+    let mut out_name: *mut *mut std::os::raw::c_char = std::ptr::null_mut();
+    let mut out_size: usize = 0;
+    let res = unsafe {
+        BNDemangleMSWithOptions(
+            arch.handle,
+            mangled_name.as_ptr(),
+            &mut out_type,
+            &mut out_name,
+            &mut out_size,
+            view.map(|v| v.handle).unwrap_or(std::ptr::null_mut()),
+        )
+    };
+
+    match res {
+        true => {
+            assert!(!out_name.is_null());
+            let names: Vec<_> = unsafe { ArrayGuard::<BnString>::new(out_name, out_size, ()) }
+                .iter()
+                .map(str::to_string)
+                .collect();
+            unsafe { BNFreeDemangledName(&mut out_name, out_size) };
+
+            let out_type = match out_type.is_null() {
+                true => None,
+                false => Some(unsafe { Type::ref_from_raw(out_type) }),
+            };
+
+            Some((names.into(), out_type))
+        }
+        false => None,
+    }
+}
+
 #[derive(PartialEq, Eq, Hash)]
 pub struct Demangler {
     pub(crate) handle: *mut BNDemangler,
diff --git a/view/pe/coffview.cpp b/view/pe/coffview.cpp
@@ -1531,7 +1531,7 @@ void COFFView::AddCOFFSymbol(BNSymbolType type, const string& dll, const string&
 	{
 		QualifiedName demangledName;
 		Ref<Type> demangledType;
-		if (DemangleGeneric(m_arch, rawName, demangledType, demangledName, nullptr, m_simplifyTemplates))
+		if (DemangleGeneric(m_arch, rawName, demangledType, demangledName, this, m_simplifyTemplates))
 		{
 			shortName = demangledName.GetString();
 			fullName = shortName;
diff --git a/view/pe/peview.cpp b/view/pe/peview.cpp
@@ -3562,7 +3562,7 @@ void PEView::AddPESymbol(BNSymbolType type, const string& dll, const string& nam
 			{
 				QualifiedName demangledName;
 				Ref<Type> demangledType;
-				if (DemangleGeneric(m_arch, rawName, demangledType, demangledName, nullptr, m_simplifyTemplates))
+				if (DemangleGeneric(m_arch, rawName, demangledType, demangledName, this, m_simplifyTemplates))
 				{
 					shortName = demangledName.GetString();
 					fullName = shortName;

Original file line number	Diff line number	Diff line change
`@@ -1531,7 +1531,7 @@ void COFFView::AddCOFFSymbol(BNSymbolType type, const string& dll, const string&`
`1531`	`1531`	`{`
`1532`	`1532`	`QualifiedName demangledName;`
`1533`	`1533`	`Ref<Type> demangledType;`
`1534`		`- if (DemangleGeneric(m_arch, rawName, demangledType, demangledName, nullptr, m_simplifyTemplates))`
	`1534`	`+ if (DemangleGeneric(m_arch, rawName, demangledType, demangledName, this, m_simplifyTemplates))`
`1535`	`1535`	`{`
`1536`	`1536`	`shortName = demangledName.GetString();`
`1537`	`1537`	`fullName = shortName;`
Original file line number	Diff line number	Diff line change
`@@ -3562,7 +3562,7 @@ void PEView::AddPESymbol(BNSymbolType type, const string& dll, const string& nam`
`3562`	`3562`	`{`
`3563`	`3563`	`QualifiedName demangledName;`
`3564`	`3564`	`Ref<Type> demangledType;`
`3565`		`- if (DemangleGeneric(m_arch, rawName, demangledType, demangledName, nullptr, m_simplifyTemplates))`
	`3565`	`+ if (DemangleGeneric(m_arch, rawName, demangledType, demangledName, this, m_simplifyTemplates))`
`3566`	`3566`	`{`
`3567`	`3567`	`shortName = demangledName.GetString();`
`3568`	`3568`	`fullName = shortName;`