Fix sysv x86 indirect return stack adjustment, and add APIs and documentation for dereferencing NTRs for GetCallLayout

Author: D0ntPanic

arch/x86/arch_x86.cpp

@@ -3862,12 +3862,24 @@ class X86SystemVCallingConvention: public X86BaseCallingConvention
 			return true;
 		if (type->IsStructure() || type->IsArray())
 			return false;
+
+		// If we have an unresolved NTR, we don't actually know what the type is. But it is more likely to
+		// be a structure than anything else, so use the same logic as an identified structure.
+		if (type->IsNamedTypeRefer())
+			return false;
+
 		if (type->GetWidth() == 0 || type->GetWidth() == 1 || type->GetWidth() == 2 || type->GetWidth() == 4
 			|| type->GetWidth() == 8)
 			return true;
 		return false;
 	}
 
+	bool IsStackAdjustedOnReturn() override
+	{
+		// Only for indirect returns, see GetStackAdjustmentForLocations below
+		return true;
+	}
+
 	int64_t GetStackAdjustmentForLocations(BinaryView*, const std::optional<ValueLocation>& returnValue,
 		const vector<ValueLocation>&, const vector<Ref<Type>>&) override
 	{

binaryninjaapi.h

@@ -5520,6 +5520,8 @@ namespace BinaryNinja {
 	class TypeArchive;
 	class MemoryMap;
 	struct HighLevelILInstruction;
+	struct FunctionParameter;
+	struct ReturnValue;
 
 	class QueryMetadataException : public ExceptionWithStackTrace
 	{
@@ -8253,6 +8255,9 @@ namespace BinaryNinja {
 		void SetUserGlobalPointerValue(const Confidence<RegisterValue>& value);
 
 		std::optional<std::pair<std::string, BNStringType>> StringifyUnicodeData(Architecture* arch, const DataBuffer& buffer, bool nullTerminates = true, bool allowShortStrings = false);
+
+		std::vector<FunctionParameter> DerefParameterNamedTypeRefs(const std::vector<FunctionParameter>& params);
+		ReturnValue DerefReturnValueNamedTypeRefs(const ReturnValue& returnValue);
 	};
 
 	/*! MemoryMap provides access to the system-level memory map describing how a BinaryView is loaded into memory.
@@ -18252,6 +18257,11 @@ namespace BinaryNinja {
 
 			The default implementation calls GetDefaultCallLayout.
 
+			When calling this function to query the layout of a function, the return value and parameters
+			should have their named type references dereferenced before passing them to this function.
+			Calling the functions BinaryView::DerefReturnValueNamedTypeRefs and
+			BinaryView::DerefParameterNamedTypeRefs will perform this dereferencing.
+
 			\param view BinaryView providing type information
 			\param returnValue Return value of the call
 			\param params Parameters of the call

binaryview.cpp

@@ -5847,6 +5847,26 @@ Ref<Relocation> BinaryView::GetNextRelocation(uint64_t addr, uint64_t maxAddr)
 }
 
 
+vector<FunctionParameter> BinaryView::DerefParameterNamedTypeRefs(const vector<FunctionParameter>& params)
+{
+	vector<FunctionParameter> result;
+	result.reserve(params.size());
+	for (auto& i : params)
+		result.emplace_back(i.name, i.type->DerefNamedTypeReference(this)->WithConfidence(i.type.GetConfidence()),
+			i.locationSource, i.location);
+	return result;
+}
+
+
+ReturnValue BinaryView::DerefReturnValueNamedTypeRefs(const ReturnValue& returnValue)
+{
+	ReturnValue result = returnValue;
+	if (result.type.GetValue())
+		result.type.SetValue(result.type->DerefNamedTypeReference(this));
+	return result;
+}
+
+
 Relocation::Relocation(BNRelocation* reloc)
 {
 	m_object = reloc;

python/binaryview.py

@@ -11037,6 +11037,24 @@ def stringify_unicode_data(self, arch: Optional['architecture.Architecture'], bu
 		core.free_string(string)
 		return result, StringType(string_type.value)
 
+	def deref_parameter_named_type_references(self, params: List['_types.FunctionParameter']):
+		result = []
+		for param in params:
+			if param.type is None:
+				ty = None
+			else:
+				ty = param.type.deref_named_type_reference(self).with_confidence(param.type.confidence)
+			result.append(_types.FunctionParameter(ty, param.name, param.location, param.location_source))
+		return result
+
+	def deref_return_value_named_type_references(self, value: '_types.ReturnValue'):
+		if value.type is None:
+			ty = None
+		else:
+			ty = value.type.deref_named_type_reference(self).with_confidence(value.type.confidence)
+		return _types.ReturnValue(ty, value.location)
+
+
 class BinaryReader:
 	"""
 	``class BinaryReader`` is a convenience class for reading binary data.

python/callingconvention.py

@@ -1071,6 +1071,11 @@ def get_call_layout(
 
 		The default implementation calls :py:meth:`get_default_call_layout`.
 
+		When calling this method to query the layout of a function, the return value and parameters
+		should have their named type references dereferenced before passing them to this method.
+		Calling the methods :py:meth:`BinaryView.deref_return_value_named_type_references` and
+		:py:meth:`BinaryView.deref_parameter_named_type_references` will perform this dereferencing.
+
 		:param BinaryView view: binary view providing type information
 		:param return_value: return value of the call
 		:param params: parameters of the call

rust/src/binary_view.rs

@@ -51,8 +51,9 @@ use crate::string::*;
 use crate::symbol::{Symbol, SymbolType};
 use crate::tags::{Tag, TagReference, TagType};
 use crate::types::{
-    NamedTypeReference, QualifiedName, QualifiedNameAndType, QualifiedNameTypeAndId, Type,
-    TypeArchive, TypeArchiveId, TypeContainer, TypeLibrary,
+    FunctionParameter, NamedTypeReference, QualifiedName, QualifiedNameAndType,
+    QualifiedNameTypeAndId, ReturnValue, Type, TypeArchive, TypeArchiveId, TypeContainer,
+    TypeLibrary,
 };
 use crate::variable::DataVariable;
 use crate::workflow::Workflow;
@@ -2964,6 +2965,36 @@ impl BinaryView {
         let path_str = unsafe { BnString::into_string(result) };
         Some(PathBuf::from(path_str))
     }
+
+    pub fn deref_return_value_named_type_references(
+        &self,
+        return_value: &ReturnValue,
+    ) -> ReturnValue {
+        ReturnValue {
+            ty: Conf::new(
+                return_value.ty.contents.deref_named_type_reference(self),
+                return_value.ty.confidence,
+            ),
+            location: return_value.location.clone(),
+        }
+    }
+
+    pub fn deref_parameter_named_type_references(
+        &self,
+        params: &[FunctionParameter],
+    ) -> Vec<FunctionParameter> {
+        params
+            .iter()
+            .map(|param| FunctionParameter {
+                ty: Conf::new(
+                    param.ty.contents.deref_named_type_reference(self),
+                    param.ty.confidence,
+                ),
+                name: param.name.clone(),
+                location: param.location.clone(),
+            })
+            .collect()
+    }
 }
 
 impl BinaryViewBase for BinaryView {

rust/src/calling_convention.rs

@@ -210,6 +210,11 @@ pub trait CallingConvention: 'static + Sync + Sized + AsRef<CoreCallingConventio
     /// It is recommended to only override this method if the calling convention behavior cannot be
     /// modeled with [`CallingConvention::return_value_location`] and/or
     /// [`CallingConvention::parameter_locations`].
+    ///
+    /// When calling this function to query the layout of a function, the return value and
+    /// parameters should have their named type references dereferenced before passing them to
+    /// this function. Calling the functions [`BinaryView::deref_return_value_named_type_references`]
+    /// and [`BinaryView::deref_parameter_named_type_references`] will perform this dereferencing.
     fn call_layout(
         &self,
         view: Option<&BinaryView>,

rust/src/types.rs

@@ -1084,6 +1084,10 @@ impl Type {
         QualifiedName::free_raw(raw_name);
         type_id
     }
+
+    pub fn deref_named_type_reference(&self, view: &BinaryView) -> Ref<Type> {
+        unsafe { Self::ref_from_raw(BNDerefNamedTypeReference(view.handle, self.handle)) }
+    }
 }
 
 impl Display for Type {