In demanglers selectively reduce confidence under some conditions

Author: plafosse

demangler/gnu3/demangled_type_node.cpp

@@ -16,6 +16,7 @@
 #ifdef BINARYNINJACORE_LIBRARY
 #include "binaryview.h"
 #endif
+#include <algorithm>
 #include <cinttypes>
 
 #ifdef BINARYNINJACORE_LIBRARY
@@ -957,6 +958,26 @@ string DemangledTypeNode::GetTypeAndName(const StringList& name) const
 }
 
 
+bool DemangledTypeNode::HasUndeterminedTopLevelSize() const
+{
+	switch (m_typeClass)
+	{
+	case NamedTypeReferenceClass:
+		return m_widthKind == FixedWidth && m_width == 0;
+	case ArrayTypeClass:
+		return m_childType && m_childType->HasUndeterminedTopLevelSize();
+	default:
+		return false;
+	}
+}
+
+
+uint8_t DemangledTypeNode::GetValueConfidence() const
+{
+	return HasUndeterminedTopLevelSize() ? BN_DEFAULT_CONFIDENCE : BN_FULL_CONFIDENCE;
+}
+
+
 Ref<Type> DemangledTypeNode::Finalize(Architecture* arch, Platform* platform) const
 {
 	switch (m_typeClass)
@@ -1043,14 +1064,22 @@ Ref<Type> DemangledTypeNode::Finalize(Architecture* arch, Platform* platform) co
 	case FunctionTypeClass:
 	{
 		Ref<Type> retType = m_childType ? m_childType->Finalize(arch, platform) : Ref<Type>(Type::VoidType());
+		uint8_t retTypeConfidence = m_childType ? m_childType->GetValueConfidence() : BN_FULL_CONFIDENCE;
+		retTypeConfidence = std::min(retTypeConfidence, m_returnTypeConfidence);
+
 		vector<FunctionParameter> finalParams;
 		finalParams.reserve(m_params.size() + (m_implicitThisParameterType ? 1 : 0));
 		if (m_implicitThisParameterType)
-			finalParams.push_back({"this", m_implicitThisParameterType->Finalize(arch, platform), true, Variable()});
+		{
+			Ref<Type> thisType = m_implicitThisParameterType->Finalize(arch, platform);
+			finalParams.push_back({"this", thisType->WithConfidence(m_implicitThisParameterType->GetValueConfidence()),
+				true, Variable()});
+		}
 		for (auto& p : m_params)
 		{
 			Ref<Type> pType = p.type ? p.type->Finalize(arch, platform) : Ref<Type>(Type::VoidType());
-			finalParams.push_back({p.name, pType, true, Variable()});
+			uint8_t pTypeConfidence = p.type ? p.type->GetValueConfidence() : BN_FULL_CONFIDENCE;
+			finalParams.push_back({p.name, pType->WithConfidence(pTypeConfidence), true, Variable()});
 		}
 		Confidence<Ref<CallingConvention>> callingConvention;
 		if (m_callingConvention)
@@ -1061,7 +1090,7 @@ Ref<Type> DemangledTypeNode::Finalize(Architecture* arch, Platform* platform) co
 				callingConvention = Confidence<Ref<CallingConvention>>(resolvedCallingConvention, BN_FULL_CONFIDENCE);
 		}
 		TypeBuilder tb = TypeBuilder::FunctionType(
-			retType->WithConfidence(static_cast<uint8_t>(m_returnTypeConfidence)), callingConvention, finalParams,
+			retType->WithConfidence(retTypeConfidence), callingConvention, finalParams,
 			Confidence<bool>(m_hasVariableArgs, m_hasVariableArgs ? BN_DEFAULT_CONFIDENCE : 0));
 		tb.SetConst(m_const);
 		tb.SetVolatile(m_volatile);

demangler/gnu3/demangled_type_node.h

@@ -137,7 +137,7 @@ class DemangledTypeNode
 	void SetTemplateArguments(_STD_VECTOR<Param> args, bool spaceAfterComma = false);
 	void SetPointerSuffixBits(uint8_t bits) { m_pointerSuffixBits = bits; }
 	void AddPointerSuffix(BNPointerSuffix ps) { m_pointerSuffixBits |= PointerSuffixBit(ps); }
-	void SetReturnTypeConfidence(int8_t c) { m_returnTypeConfidence = c; }
+	void SetReturnTypeConfidence(uint8_t c) { m_returnTypeConfidence = c; }
 	void SetCallingConventionName(BNCallingConventionName cc) { m_callingConventionName = cc; }
 	void SetCallingConvention(BN::Ref<BN::CallingConvention> cc) { m_callingConvention = std::move(cc); }
 	void SetNTRType(BNNamedTypeReferenceClass cls) { m_ntrClass = cls; }
@@ -155,6 +155,9 @@ class DemangledTypeNode
 	BN::Ref<BN::Type> Finalize(BN::Architecture* arch = nullptr, BN::Platform* platform = nullptr) const;
 
 private:
+	bool HasUndeterminedTopLevelSize() const;
+	uint8_t GetValueConfidence() const;
+
 	BNTypeClass m_typeClass;
 	BNNamedTypeReferenceClass m_ntrClass;
 	BNReferenceType m_pointerReference;