Implement ASAN: a framework to detect memory errors

Makefile

@@ -28,7 +28,7 @@ libdragon: CXXFLAGS+=$(N64_CXXFLAGS) $(LIBDRAGON_CFLAGS)
 libdragon: ASFLAGS+=$(N64_ASFLAGS) $(LIBDRAGON_CFLAGS)
 libdragon: RSPASFLAGS+=$(N64_RSPASFLAGS) $(LIBDRAGON_CFLAGS)
 libdragon: LDFLAGS+=$(N64_LDFLAGS)
-libdragon: libdragon.a libdragonsys.a gen-version
+libdragon: libdragon.a libdragon_asan.a libdragonsys.a gen-version
 
 libdragonsys.a: $(BUILD_DIR)/system.o
 
@@ -81,6 +81,7 @@ LIBDRAGON_OBJS += \
 	$(BUILD_DIR)/dma.o \
 	$(BUILD_DIR)/timer.o \
 	$(BUILD_DIR)/exception.o \
+	$(BUILD_DIR)/asan_weak.o \
 	$(BUILD_DIR)/do_ctors.o \
 	$(BUILD_DIR)/dlfcn.o \
 	$(BUILD_DIR)/hashtable.o \
@@ -114,6 +115,7 @@ $(SOURCE_DIR)/magma/rsp_magma.h: $(BUILD_DIR)/magma/rsp_magma.o
 $(BUILD_DIR)/magma/magma.o: $(SOURCE_DIR)/magma/rsp_magma.h
 
 libdragon.a: $(LIBDRAGON_OBJS)
+libdragon_asan.a: $(BUILD_DIR)/asan.o
 
 %.a:
 	@echo "    [AR] $@"
@@ -186,6 +188,7 @@ install: install-mk libdragon
 	@echo "    [INSTALL] libdragon"
 	mkdir -p $(INSTALLDIR)/$(N64_TARGET)/lib
 	install -Cv -m 0644 libdragon.a $(INSTALLDIR)/$(N64_TARGET)/lib/libdragon.a
+	install -Cv -m 0644 libdragon_asan.a $(INSTALLDIR)/$(N64_TARGET)/lib/libdragon_asan.a
 	install -Cv -m 0644 n64.ld $(INSTALLDIR)/$(N64_TARGET)/lib/n64.ld
 	install -Cv -m 0644 dso.ld $(INSTALLDIR)/$(N64_TARGET)/lib/dso.ld
 	install -Cv -m 0644 rsp.ld $(INSTALLDIR)/$(N64_TARGET)/lib/rsp.ld

include/asan.h

@@ -0,0 +1,55 @@
+/**
+ * @file asan.h
+ * @author Giovanni Bajo <giovannibajo@gmail.com>
+ * @brief Emulator-assisted heap memory sanitizer (XASAN)
+ * @ingroup asan
+ * @defgroup asan asan
+ *
+ * XASAN is an emulator-assisted memory access sanitizer for the VR4300.
+ * Enable it at link time with `N64_ASAN=1` in n64.mk.
+ *
+ * When disabled (default), the functions resolve to weak no-op stubs with
+ * no runtime overhead beyond a call when actually used.
+ *
+ * @note Global object registration (`.xasan_globals`) is not implemented yet.
+ */
+
+#ifndef LIBDRAGON_ASAN_H
+#define LIBDRAGON_ASAN_H
+
+#include <stddef.h>
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * @brief Mark a memory region as user-poisoned (inaccessible)
+ *
+ * @param ptr   Base address (rounded down to 8 bytes)
+ * @param size  Size in bytes (rounded up to 8 bytes)
+ */
+void asan_poison(void *ptr, size_t size);
+
+/**
+ * @brief Mark a memory region as accessible
+ *
+ * @param ptr   Base address (rounded down to 8 bytes)
+ * @param size  Size in bytes (rounded up to 8 bytes)
+ */
+void asan_unpoison(void *ptr, size_t size);
+
+/**
+ * @brief Return whether XASAN is linked in and active
+ *
+ * When the app is linked with `N64_ASAN=1` and the emulator supports XASAN,
+ * this returns true after startup initialization.
+ */
+bool asan_enabled(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* LIBDRAGON_ASAN_H */

include/emux.h

@@ -17,6 +17,7 @@
 ///@cond
 #ifndef __ASSEMBLER__
 #include <stdint.h>
+#include <stddef.h>
 #define cast64(x) (uint64_t)(x)
 #else
 #define cast64(x) x
@@ -44,6 +45,7 @@
 #define EMUX_XPROF(slot, code)          EMUX_OP(0x28, slot,      0,  code)  ///< Control profiler
 #define EMUX_XPROFREAD(slot, metric)    EMUX_OP(0x29, slot, metric, 0x000)  ///< Read profiler metric
 #define EMUX_XEXCEPTION(mask)           EMUX_OP(0x2A,    0,   mask, 0x000)  ///< Set exception mask
+#define EMUX_XASAN(rd, rt, code)        EMUX_OP(0x2B,   rd,     rt,  code)  ///< XASAN memory sanitizer
 #define EMUX_XIOCTL(code)               EMUX_OP(0x2C,    0,      0,  code)  ///< Modify emulator behavior
 /** @} */
 
@@ -56,8 +58,35 @@
 #define EMUX_FEAT1_HEXDUMP                      (1 << 0x7)    ///< Hexdump support
 #define EMUX_FEAT1_PROFILER                     (1 << 0x8)    ///< Profiling support
 #define EMUX_FEAT1_EXCEPTION                    (1 << 0xA)    ///< Exception support
+#define EMUX_FEAT1_XASAN                        (1 << 0xB)    ///< XASAN memory sanitizer support
 #define EMUX_FEAT1_IOCTL                        (1 << 0xC)    ///< Emulator behavior support
 
+#define EMUX_XASAN_DISABLE                      0x0           ///< Disable XASAN checking (refcount--)
+#define EMUX_XASAN_ENABLE                       0x1           ///< Enable XASAN checking (refcount++)
+#define EMUX_XASAN_POISON                       0x2           ///< Poison a memory region
+#define EMUX_XASAN_UNPOISON                     0x3           ///< Unpoison a memory region
+
+#define EMUX_XASAN_TAG_ACCESSIBLE               0             ///< Accessible region
+#define EMUX_XASAN_TAG_LEFT                     1             ///< Left redzone
+#define EMUX_XASAN_TAG_RIGHT                    2             ///< Right redzone
+#define EMUX_XASAN_TAG_FREED                    3             ///< Freed memory
+#define EMUX_XASAN_TAG_GLOBAL                   4             ///< Global redzone
+#define EMUX_XASAN_TAG_USER                     5             ///< User-poisoned region
+#define EMUX_XASAN_TAG_UNALLOC                  6             ///< Unallocated heap memory
+
+#define EMUX_XASAN_ACCESS_UNKNOWN               0x00
+#define EMUX_XASAN_ACCESS_CPU_READ              0x01
+#define EMUX_XASAN_ACCESS_CPU_WRITE             0x02
+#define EMUX_XASAN_ACCESS_CPU_EXEC              0x03
+#define EMUX_XASAN_ACCESS_STACK                 0x04
+#define EMUX_XASAN_ACCESS_RSP_DMA_READ          0x05
+#define EMUX_XASAN_ACCESS_RSP_DMA_WRITE         0x06
+
+#define EMUX_XASAN_FAULT_UNKNOWN                0x00
+#define EMUX_XASAN_FAULT_PERMISSION             0x01
+#define EMUX_XASAN_FAULT_POISON                 0x02
+#define EMUX_XASAN_FAULT_TAIL                   0x03
+
 #define EMUX_LOG_ASCIIZ                         0x000      ///< Log a zero-terminated string
 #define EMUX_LOG_LENGTH                         0x001      ///< Log a non-zero-terminated string
 
@@ -145,6 +174,7 @@
 #define EMUX_EXCEPTION_CPU_CACHED_ACCESS       (cast64(1) << 1)  ///< CPU cached access to non-RDRAM area
 #define EMUX_EXCEPTION_CPU_64BIT_READ          (cast64(1) << 2)  ///< CPU 64-bit read from non-RDRAM area
 #define EMUX_EXCEPTION_CPU_UNMAPPED_ACCESS     (cast64(1) << 3)  ///< CPU access to RCP unmapped area
+#define EMUX_EXCEPTION_XASAN                   (cast64(1) << 4)  ///< XASAN memory access violation
 
 #ifndef __ASSEMBLER__
 
@@ -418,6 +448,65 @@ inline void emux_exception_set_mask(uint64_t mask)
     __asm__ __volatile__(" .word %1\n" :: "r"(__mask), "i"(EMUX_XEXCEPTION(REG_T0)) : "memory");
 }
 
+/** @brief Run an XASAN EMUX opcode with address and size operands */
+#define EMUX_XASAN_RUN(addr, size, code) do { \
+    const int REG_T0 = 8; \
+    const int REG_T1 = 9; \
+    register const void *__xasan_addr asm("t0") = (const void *)(addr); \
+    register size_t __xasan_size asm("t1") = (size_t)(size); \
+    __asm__ __volatile__(" .word %2\n" \
+        :: "r"(__xasan_addr), "r"(__xasan_size), "i"(EMUX_XASAN(REG_T0, REG_T1, code)) \
+        : "memory"); \
+} while(0)
+
+/** @brief Enable XASAN checking in the emulator */
+inline void emux_xasan_enable(void)
+{
+    __asm__ __volatile__(" .word %0\n" :: "i"(EMUX_XASAN(0, 0, EMUX_XASAN_ENABLE)) : "memory");
+}
+
+/** @brief Disable XASAN checking in the emulator */
+inline void emux_xasan_disable(void)
+{
+    __asm__ __volatile__(" .word %0\n" :: "i"(EMUX_XASAN(0, 0, EMUX_XASAN_DISABLE)) : "memory");
+}
+
+/** @brief Mark a memory region as accessible */
+inline void emux_xasan_unpoison(const void *addr, size_t size)
+{
+    EMUX_XASAN_RUN(addr, size, EMUX_XASAN_UNPOISON);
+}
+
+/** @brief Mark a memory region as user-poisoned */
+inline void emux_xasan_poison_user(const void *addr, size_t size)
+{
+    EMUX_XASAN_RUN(addr, size, EMUX_XASAN_POISON | (EMUX_XASAN_TAG_USER << 4));
+}
+
+/** @brief Mark a memory region as left redzone */
+inline void emux_xasan_poison_left(const void *addr, size_t size)
+{
+    EMUX_XASAN_RUN(addr, size, EMUX_XASAN_POISON | (EMUX_XASAN_TAG_LEFT << 4));
+}
+
+/** @brief Mark a memory region as right redzone */
+inline void emux_xasan_poison_right(const void *addr, size_t size)
+{
+    EMUX_XASAN_RUN(addr, size, EMUX_XASAN_POISON | (EMUX_XASAN_TAG_RIGHT << 4));
+}
+
+/** @brief Mark a memory region as freed */
+inline void emux_xasan_poison_freed(const void *addr, size_t size)
+{
+    EMUX_XASAN_RUN(addr, size, EMUX_XASAN_POISON | (EMUX_XASAN_TAG_FREED << 4));
+}
+
+/** @brief Mark a memory region as unallocated heap memory */
+inline void emux_xasan_poison_unalloc(const void *addr, size_t size)
+{
+    EMUX_XASAN_RUN(addr, size, EMUX_XASAN_POISON | (EMUX_XASAN_TAG_UNALLOC << 4));
+}
+
 #ifdef __cplusplus
 }
 #endif

include/libdragon.h

@@ -30,6 +30,7 @@
 #include "entropy.h"
 #include "console.h"
 #include "emux.h"
+#include "asan.h"
 #include "debug.h"
 #include "fat.h"
 #include "joybus.h"

n64.mk

@@ -82,6 +82,13 @@ N64_CXXFLAGS = $(N64_C_AND_CXX_FLAGS) -std=gnu++17
 N64_ASFLAGS = -mtune=vr4300 -march=vr4300 -mabi=o64 -Wa,--fatal-warnings -I$(N64_INCLUDEDIR)
 N64_RSPASFLAGS = -march=mips1 -mabi=32 -Wa,--fatal-warnings -I$(N64_INCLUDEDIR)
 N64_LDFLAGS = -g -L$(N64_LIBDIR) -ldragon -lm -ldragonsys -Tn64.ld --gc-sections --wrap __do_global_ctors
+
+ifeq ($(N64_ASAN),1)
+N64_C_AND_CXX_FLAGS += -fstack-protector-strong
+N64_LDFLAGS += -ldragon_asan
+N64_LDFLAGS += --wrap=_malloc_r --wrap=_free_r --wrap=_realloc_r --wrap=_calloc_r --wrap=_memalign_r --wrap=sbrk_top --wrap=_mallinfo_r -u __asan_init
+N64_LDFLAGS += --wrap=__stack_chk_fail
+endif
 N64_DSOLDFLAGS = --emit-relocs --unresolved-symbols=ignore-all --nmagic -T$(N64_LIBDIR)/dso.ld
 
 N64_TOOLFLAGS = --toc