Add derived strings and string recognizer API (WIP)

Author: D0ntPanic

binaryninjaapi.h

@@ -4074,6 +4074,7 @@ namespace BinaryNinja {
 	class Segment;
 	class Component;
 	class TypeArchive;
+	struct DerivedString;
 
 	/*!
 
@@ -4107,6 +4108,8 @@ namespace BinaryNinja {
 
 		static void StringFoundCallback(void* ctxt, BNBinaryView* data, BNStringType type, uint64_t offset, size_t len);
 		static void StringRemovedCallback(void* ctxt, BNBinaryView* data, BNStringType type, uint64_t offset, size_t len);
+		static void DerivedStringFoundCallback(void* ctxt, BNBinaryView* data, BNDerivedString* str);
+		static void DerivedStringRemovedCallback(void* ctxt, BNBinaryView* data, BNDerivedString* str);
 		static void TypeDefinedCallback(void* ctxt, BNBinaryView* data, BNQualifiedName* name, BNType* type);
 		static void TypeUndefinedCallback(void* ctxt, BNBinaryView* data, BNQualifiedName* name, BNType* type);
 		static void TypeReferenceChangedCallback(void* ctx, BNBinaryView* data, BNQualifiedName* name, BNType* type);
@@ -4203,6 +4206,8 @@ namespace BinaryNinja {
 			UndoEntryTaken = 1ULL << 50,
 			RedoEntryTaken = 1ULL << 51,
 			Rebased = 1ULL << 52,
+			DerivedStringFound = 1ULL << 53,
+			DerivedStringRemoved = 1ULL << 54,
 
 			BinaryDataUpdates = DataWritten | DataInserted | DataRemoved,
 			FunctionLifetime = FunctionAdded | FunctionRemoved,
@@ -4213,7 +4218,7 @@ namespace BinaryNinja {
 			TagUpdates = TagLifetime | TagUpdated,
 			SymbolLifetime = SymbolAdded | SymbolRemoved,
 			SymbolUpdates = SymbolLifetime | SymbolUpdated,
-			StringUpdates = StringFound | StringRemoved,
+			StringUpdates = StringFound | StringRemoved | DerivedStringFound | DerivedStringRemoved,
 			TypeLifetime = TypeDefined | TypeUndefined,
 			TypeUpdates = TypeLifetime | TypeReferenceChanged | TypeFieldReferenceChanged,
 			SegmentLifetime = SegmentAdded | SegmentRemoved,
@@ -4350,6 +4355,16 @@ namespace BinaryNinja {
 			(void)offset;
 			(void)len;
 		}
+		virtual void OnDerivedStringFound(BinaryView* data, const DerivedString& str)
+		{
+			(void)data;
+			(void)str;
+		}
+		virtual void OnDerivedStringRemoved(BinaryView* data, const DerivedString& str)
+		{
+			(void)data;
+			(void)str;
+		}
 		virtual void OnTypeDefined(BinaryView* data, const QualifiedName& name, Type* type)
 		{
 			(void)data;
@@ -4822,7 +4837,7 @@ namespace BinaryNinja {
 
 		const char* c_str() const;
 		size_t size() const;
-		BNStringRef* GetObject() { return m_ref; }
+		BNStringRef* GetObject() const { return m_ref; }
 
 		bool operator==(const StringRef& other) const { return this->operator std::string_view() == other.operator std::string_view(); }
 		bool operator!=(const StringRef& other) const { return this->operator std::string_view() != other.operator std::string_view(); }
@@ -5356,6 +5371,42 @@ namespace BinaryNinja {
 		std::vector<TypeReferenceSource> typeRefs;
 	};
 
+	class CustomStringType: public StaticCoreRefCountObject<BNCustomStringType>
+	{
+	public:
+		CustomStringType(BNCustomStringType* type);
+		std::string GetName() const;
+	};
+
+	struct DerivedString
+	{
+		StringRef value;
+		std::optional<std::pair<uint64_t, uint64_t>> addrAndLength;
+		Ref<CustomStringType> customType;
+
+		bool operator==(const DerivedString& other) const
+		{
+			if (value != other.value)
+				return false;
+			if (addrAndLength != other.addrAndLength)
+				return false;
+			return customType == other.customType;
+		}
+
+		bool operator<(const DerivedString& other) const
+		{
+			if (value < other.value)
+				return true;
+			if (other.value < value)
+				return false;
+			if (addrAndLength < other.addrAndLength)
+				return true;
+			if (other.addrAndLength < addrAndLength)
+				return false;
+			return customType < other.customType;
+		}
+	};
+
 	struct QualifiedNameAndType;
 	struct PossibleValueSet;
 	class Metadata;
@@ -7131,6 +7182,10 @@ namespace BinaryNinja {
 		*/
 		std::vector<BNStringReference> GetStrings(uint64_t start, uint64_t len);
 
+		std::vector<DerivedString> GetDerivedStrings();
+		std::vector<ReferenceSource> GetDerivedStringCodeReferences(
+			const DerivedString& str, std::optional<size_t> maxItems = std::nullopt);
+
 		/*! Sets up a call back function to be called when analysis has been completed.
 
 			This is helpful when using `UpdateAnalysis` which does not wait for analysis completion before returning.

binaryninjacore.h

@@ -312,6 +312,8 @@ extern "C"
 	typedef struct BNIndirectBranchInfo BNIndirectBranchInfo;
 	typedef struct BNArchitectureAndAddress BNArchitectureAndAddress;
 	typedef struct BNConstantRenderer BNConstantRenderer;
+	typedef struct BNStringRecognizer BNStringRecognizer;
+	typedef struct BNCustomStringType BNCustomStringType;
 
 	typedef bool(*BNProgressFunction)(void*, size_t, size_t);
 
@@ -1631,6 +1633,15 @@ extern "C"
 		size_t nameCount;
 	} BNQualifiedName;
 
+	typedef struct BNDerivedString
+	{
+		BNStringRef* value;
+		bool addrValid;
+		uint64_t addr;
+		uint64_t len;
+		BNCustomStringType* customType;
+	} BNDerivedString;
+
 	typedef struct BNBinaryDataNotification
 	{
 		void* context;
@@ -1656,6 +1667,8 @@ extern "C"
 		void (*symbolUpdated)(void* ctxt, BNBinaryView* view, BNSymbol* sym);
 		void (*stringFound)(void* ctxt, BNBinaryView* view, BNStringType type, uint64_t offset, size_t len);
 		void (*stringRemoved)(void* ctxt, BNBinaryView* view, BNStringType type, uint64_t offset, size_t len);
+		void (*derivedStringFound)(void* ctxt, BNBinaryView* view, BNDerivedString* str);
+		void (*derivedStringRemoved)(void* ctxt, BNBinaryView* view, BNDerivedString* str);
 		void (*typeDefined)(void* ctxt, BNBinaryView* view, BNQualifiedName* name, BNType* type);
 		void (*typeUndefined)(void* ctxt, BNBinaryView* view, BNQualifiedName* name, BNType* type);
 		void (*typeReferenceChanged)(void* ctxt, BNBinaryView* view, BNQualifiedName* name, BNType* type);
@@ -3817,6 +3830,31 @@ extern "C"
 			BNOperatorPrecedence precedence);
 	} BNCustomConstantRenderer;
 
+	typedef struct BNCustomStringRecognizerResult
+	{
+		BNCustomStringType* type;
+		bool addrValid;
+		uint64_t addr;
+		size_t len;
+		BNDataBuffer* value;
+	} BNCustomStringRecognizerResult;
+
+	typedef struct BNCustomStringRecognizer
+	{
+		void* context;
+		void (*freeObject)(void* ctxt);
+		bool (*isValidForType)(void* ctxt, BNHighLevelILFunction* hlil, BNType* type);
+		bool (*recognizeConstantPointer)(void* ctxt, BNHighLevelILFunction* hlil, size_t expr, BNType* type,
+			int64_t val, BNCustomStringRecognizerResult* result);
+	} BNCustomStringRecognizer;
+
+	typedef struct BNCustomStringTypeInfo
+	{
+		char* name;
+		char* stringPrefix;
+		char* stringPostfix;
+	} BNCustomStringTypeInfo;
+
 	BINARYNINJACOREAPI char* BNAllocString(const char* contents);
 	BINARYNINJACOREAPI char* BNAllocStringWithLength(const char* contents, size_t len);
 	BINARYNINJACOREAPI void BNFreeString(char* str);
@@ -5260,6 +5298,11 @@ extern "C"
 	    BNBinaryView* view, uint64_t start, uint64_t len, size_t* count);
 	BINARYNINJACOREAPI void BNFreeStringReferenceList(BNStringReference* strings);
 
+	BINARYNINJACOREAPI BNDerivedString* BNGetDerivedStrings(BNBinaryView* view, size_t* count);
+	BINARYNINJACOREAPI BNReferenceSource* BNGetDerivedStringCodeReferences(
+		BNBinaryView* view, BNDerivedString* str, size_t* count, bool limit, size_t maxItems);
+	BINARYNINJACOREAPI void BNFreeDerivedStringList(BNDerivedString* strings, size_t count);
+
 	BINARYNINJACOREAPI BNVariableNameAndType* BNGetStackLayout(BNFunction* func, size_t* count);
 	BINARYNINJACOREAPI void BNFreeVariableNameAndTypeList(BNVariableNameAndType* vars, size_t count);
 	BINARYNINJACOREAPI void BNCreateAutoStackVariable(
@@ -8691,9 +8734,25 @@ extern "C"
 		const char* name, BNCustomConstantRenderer* renderer);
 	BINARYNINJACOREAPI BNConstantRenderer* BNGetConstantRendererByName(const char* name);
 	BINARYNINJACOREAPI BNConstantRenderer** BNGetConstantRendererList(size_t* count);
-	BINARYNINJACOREAPI void BNFreeConstantRendererList(BNLanguageRepresentationFunctionType** renderers);
+	BINARYNINJACOREAPI void BNFreeConstantRendererList(BNConstantRenderer** renderers);
 	BINARYNINJACOREAPI char* BNGetConstantRendererName(BNConstantRenderer* renderer);
 
+	// String recognizers
+	BINARYNINJACOREAPI BNCustomStringType* BNRegisterCustomStringType(BNCustomStringTypeInfo* info);
+	BINARYNINJACOREAPI BNCustomStringType* BNGetCustomStringTypeByName(const char* name);
+	BINARYNINJACOREAPI BNCustomStringType* BNGetCustomStringTypeByID(uint32_t id);
+	BINARYNINJACOREAPI BNCustomStringType** BNGetCustomStringTypeList(size_t* count);
+	BINARYNINJACOREAPI void BNFreeCustomStringTypeList(BNCustomStringType** types);
+	BINARYNINJACOREAPI char* BNGetCustomStringTypeName(BNCustomStringType* type);
+	BINARYNINJACOREAPI char* BNGetCustomStringTypePrefix(BNCustomStringType* type);
+	BINARYNINJACOREAPI char* BNGetCustomStringTypePostfix(BNCustomStringType* type);
+	BINARYNINJACOREAPI BNStringRecognizer* BNRegisterStringRecognizer(
+		const char* name, BNCustomStringRecognizer* recognizer);
+	BINARYNINJACOREAPI BNStringRecognizer* BNGetStringRecognizerByName(const char* name);
+	BINARYNINJACOREAPI BNStringRecognizer** BNGetStringRecognizerList(size_t* count);
+	BINARYNINJACOREAPI void BNFreeStringRecognizerList(BNStringRecognizer** recognizers);
+	BINARYNINJACOREAPI char* BNGetStringRecognizerName(BNStringRecognizer* recognizer);
+
 #ifdef __cplusplus
 }
 #endif

binaryview.cpp

@@ -222,6 +222,32 @@ void BinaryDataNotification::StringRemovedCallback(
 }
 
 
+void BinaryDataNotification::DerivedStringFoundCallback(void* ctxt, BNBinaryView* data, BNDerivedString* str)
+{
+	BinaryDataNotification* notify = (BinaryDataNotification*)ctxt;
+	Ref<BinaryView> view = new BinaryView(BNNewViewReference(data));
+	DerivedString derivedStr;
+	derivedStr.value = StringRef(BNDuplicateStringRef(str->value));
+	derivedStr.customType = str->customType ? new CustomStringType(str->customType) : nullptr;
+	if (str->addrValid)
+		derivedStr.addrAndLength = {str->addr, str->len};
+	notify->OnDerivedStringFound(view, derivedStr);
+}
+
+
+void BinaryDataNotification::DerivedStringRemovedCallback(void* ctxt, BNBinaryView* data, BNDerivedString* str)
+{
+	BinaryDataNotification* notify = (BinaryDataNotification*)ctxt;
+	Ref<BinaryView> view = new BinaryView(BNNewViewReference(data));
+	DerivedString derivedStr;
+	derivedStr.value = StringRef(BNDuplicateStringRef(str->value));
+	derivedStr.customType = str->customType ? new CustomStringType(str->customType) : nullptr;
+	if (str->addrValid)
+		derivedStr.addrAndLength = {str->addr, str->len};
+	notify->OnDerivedStringRemoved(view, derivedStr);
+}
+
+
 void BinaryDataNotification::TypeDefinedCallback(void* ctxt, BNBinaryView* data, BNQualifiedName* name, BNType* type)
 {
 	BinaryDataNotification* notify = (BinaryDataNotification*)ctxt;
@@ -556,6 +582,8 @@ BinaryDataNotification::BinaryDataNotification()
 	m_callbacks.symbolRemoved = SymbolRemovedCallback;
 	m_callbacks.stringFound = StringFoundCallback;
 	m_callbacks.stringRemoved = StringRemovedCallback;
+	m_callbacks.derivedStringFound = DerivedStringFoundCallback;
+	m_callbacks.derivedStringRemoved = DerivedStringRemovedCallback;
 	m_callbacks.typeDefined = TypeDefinedCallback;
 	m_callbacks.typeUndefined = TypeUndefinedCallback;
 	m_callbacks.typeReferenceChanged = TypeReferenceChangedCallback;
@@ -615,6 +643,8 @@ BinaryDataNotification::BinaryDataNotification(NotificationTypes notifications)
 	m_callbacks.symbolRemoved = (notifications & NotificationType::SymbolRemoved) ? SymbolRemovedCallback : nullptr;
 	m_callbacks.stringFound = (notifications & NotificationType::StringFound) ? StringFoundCallback : nullptr;
 	m_callbacks.stringRemoved = (notifications & NotificationType::StringRemoved) ? StringRemovedCallback : nullptr;
+	m_callbacks.derivedStringFound = (notifications & NotificationType::DerivedStringFound) ? DerivedStringFoundCallback : nullptr;
+	m_callbacks.derivedStringRemoved = (notifications & NotificationType::DerivedStringRemoved) ? DerivedStringRemovedCallback : nullptr;
 	m_callbacks.typeDefined = (notifications & NotificationType::TypeDefined) ? TypeDefinedCallback : nullptr;
 	m_callbacks.typeUndefined = (notifications & NotificationType::TypeUndefined) ? TypeUndefinedCallback : nullptr;
 	m_callbacks.typeReferenceChanged = (notifications & NotificationType::TypeReferenceChanged) ? TypeReferenceChangedCallback : nullptr;
@@ -664,7 +694,7 @@ StringRef::StringRef(BNStringRef* ref)
 
 StringRef::StringRef(const StringRef& other)
 {
-	m_ref = BNDuplicateStringRef(other.m_ref);
+	m_ref = other.m_ref ? BNDuplicateStringRef(other.m_ref) : nullptr;
 }
 
 
@@ -4007,6 +4037,57 @@ vector<BNStringReference> BinaryView::GetStrings(uint64_t start, uint64_t len)
 }
 
 
+vector<DerivedString> BinaryView::GetDerivedStrings()
+{
+	size_t count;
+	BNDerivedString* strings = BNGetDerivedStrings(m_object, &count);
+	vector<DerivedString> result;
+	for (size_t i = 0; i < count; i++)
+	{
+		DerivedString str;
+		str.value = StringRef(BNDuplicateStringRef(strings[i].value));
+		str.customType = strings[i].customType ? new CustomStringType(strings[i].customType) : nullptr;
+		if (strings[i].addrValid)
+			str.addrAndLength = {strings[i].addr, strings[i].len};
+		result.push_back(str);
+	}
+	BNFreeDerivedStringList(strings, count);
+	return result;
+}
+
+
+vector<ReferenceSource> BinaryView::GetDerivedStringCodeReferences(
+	const DerivedString& str, std::optional<size_t> maxItems)
+{
+	BNDerivedString derivedStr;
+	derivedStr.value = str.value.GetObject();
+	derivedStr.customType = str.customType ? str.customType->GetObject() : nullptr;
+	derivedStr.addrValid = str.addrAndLength.has_value();
+	if (str.addrAndLength.has_value())
+	{
+		derivedStr.addr = str.addrAndLength->first;
+		derivedStr.len = str.addrAndLength->second;
+	}
+
+	size_t count;
+	BNReferenceSource* refs = BNGetDerivedStringCodeReferences(
+		m_object, &derivedStr, &count, maxItems.has_value(), maxItems.value_or(0));
+	vector<ReferenceSource> result;
+	result.reserve(count);
+	for (size_t i = 0; i < count; i++)
+	{
+		ReferenceSource src;
+		src.func = new Function(BNNewFunctionReference(refs[i].func));
+		src.arch = new CoreArchitecture(refs[i].arch);
+		src.addr = refs[i].addr;
+		result.push_back(src);
+	}
+
+	BNFreeCodeReferences(refs, count);
+	return result;
+}
+
+
 // The caller of this function must hold a reference to the returned Ref<AnalysisCompletionEvent>.
 // Otherwise, it can be freed before the callback is triggered, leading to a crash.
 Ref<AnalysisCompletionEvent> BinaryView::AddAnalysisCompletionEvent(const function<void()>& callback)
@@ -5943,3 +6024,18 @@ void SymbolQueue::Process()
 {
 	BNProcessSymbolQueue(m_object);
 }
+
+
+CustomStringType::CustomStringType(BNCustomStringType* type)
+{
+	m_object = type;
+}
+
+
+string CustomStringType::GetName() const
+{
+	char* name = BNGetCustomStringTypeName(m_object);
+	string result = name;
+	BNFreeString(name);
+	return result;
+}

python/__init__.py

@@ -83,6 +83,7 @@
 from .lineformatter import *
 from .renderlayer import *
 from .constantrenderer import *
+from .stringrecognizer import *
 # We import each of these by name to prevent conflicts between
 # log.py and the function 'log' which we don't import below
 from .log import (