i#20: Port dynamorio kernel module to current DR

.github/workflows/ci-drk.yml

@@ -17,6 +17,13 @@ jobs:
           sudo apt install -y doxygen vera++ libelf-dev scons cmake g++-multilib
           echo "MAKE_CMD=make -j$(nproc)" >> $GITHUB_ENV
 
+      - name: Generate DynamoRIO headers via CMake
+        run: |
+          mkdir -p build
+          cd build
+          cmake ..
+          $MAKE_CMD
+
       - name: Download and extract kernel source
         run: |
           VERSION="6.6.130"
@@ -30,12 +37,6 @@ jobs:
           make defconfig
           $MAKE_CMD modules_prepare
 
-      - name: Generate DynamoRIO headers via CMake
-        run: |
-          mkdir -p build
-          cd build
-          cmake ..
-
       - name: Build drk
         # TODO i#10: Convert to cmake.
         run: cd core && $MAKE_CMD -f drk.mk KERNELDIR=$GITHUB_WORKSPACE/$KERNEL_DIR

core/annotations.c

@@ -258,7 +258,7 @@ free_annotation_handler(void *p);
  */
 
 void
-annotation_init()
+annotation_init(void)
 {
     handlers = strhash_hash_create(
         GLOBAL_DCONTEXT, 8, 80, /* favor a small table */
@@ -302,7 +302,7 @@ annotation_init()
 }
 
 void
-annotation_exit()
+annotation_exit(void)
 {
 #    if !(defined(WINDOWS) && defined(X64))
     uint i;

core/annotations.h

@@ -124,10 +124,10 @@ typedef struct _dr_annotation_handler_t {
 } dr_annotation_handler_t;
 
 void
-annotation_init();
+annotation_init(void);
 
 void
-annotation_exit();
+annotation_exit(void);
 
 static inline bool
 is_annotation_label(instr_t *instr)

core/arch/arch.c

@@ -327,6 +327,7 @@ shared_gencode_emit(generated_code_t *gencode _IF_X86_64(bool x86_mode))
     pc = check_size_and_cache_line(isa_mode, gencode, pc);
     gencode->fcache_enter = pc;
     pc = emit_fcache_enter_shared(GLOBAL_DCONTEXT, gencode, pc);
+    gencode->fcache_enter_end = pc;
     pc = check_size_and_cache_line(isa_mode, gencode, pc);
     gencode->fcache_return = pc;
     pc = emit_fcache_return_shared(GLOBAL_DCONTEXT, gencode, pc);
@@ -989,15 +990,40 @@ emit_ibl_routine_and_template(dcontext_t *dcontext, generated_code_t *code, byte
     ibl_code->thread_shared_routine = thread_shared;
     ibl_code->branch_type = branch_type;
     ibl_code->source_fragment_type = source_type;
+#ifdef LINUX_KERNEL
+    ibl_code->num_ibl_found_exits = 0;
+#endif
 
     pc = emit_indirect_branch_lookup(dcontext, code, pc, fcache_return_pc,
                                      target_trace_table, inline_ibl_head, ibl_code);
+#ifdef LINUX_KERNEL
+    ibl_code->indirect_branch_lookup_routine_end = pc;
+#endif
     if (inline_ibl_head) {
         /* create the inlined ibl template */
         pc = check_size_and_cache_line(isa_mode, code, pc);
         pc = emit_inline_ibl_stub(dcontext, pc, ibl_code, target_trace_table);
     }
 
+#ifdef LINUX_KERNEL
+    pc = check_size_and_cache_line(isa_mode, code, pc);
+    ibl_code->found_unlinked = pc;
+    pc = emit_ibl_found_unlinked_code(dcontext, pc, fcache_return_pc, ibl_code, false,
+                                      false);
+    pc = check_size_and_cache_line(isa_mode, code, pc);
+    ibl_code->found_unlinked_prefix = pc;
+    pc = emit_ibl_found_unlinked_code(dcontext, pc, fcache_return_pc, ibl_code, false,
+                                      true);
+    pc = check_size_and_cache_line(isa_mode, code, pc);
+    ibl_code->found_unlinked_eflags = pc;
+    pc = emit_ibl_found_unlinked_code(dcontext, pc, fcache_return_pc, ibl_code, true,
+                                      false);
+    pc = check_size_and_cache_line(isa_mode, code, pc);
+    ibl_code->found_unlinked_eflags_prefix = pc;
+    pc = emit_ibl_found_unlinked_code(dcontext, pc, fcache_return_pc, ibl_code, true,
+                                      true);
+#endif
+
     ibl_code->far_ibl = pc;
     pc = emit_far_ibl(
         dcontext, pc, ibl_code,
@@ -1148,6 +1174,25 @@ emit_syscall_routines(dcontext_t *dcontext, generated_code_t *code, byte *pc,
     code->do_syscall = pc;
     pc = emit_do_syscall(dcontext, code, pc, code->fcache_return, thread_shared, 0,
                          &code->do_syscall_offs);
+#elif defined(LINUX_KERNEL)
+    ASSERT(thread_shared);
+    pc = check_size_and_cache_line(isa_mode, code, pc);
+    code->syscall_entry = pc;
+    pc = emit_syscall_entry(dcontext, code->fcache_return,
+                            os_get_native_syscall_entry(dcontext), pc);
+    pc = check_size_and_cache_line(isa_mode, code, pc);
+    code->common_vector_entry = pc;
+    pc = emit_common_vector_entry(dcontext, os_get_tls_base(dcontext),
+                                  os_get_interrupt_handler(0), pc);
+    pc = check_size_and_cache_line(isa_mode, code, pc);
+    for (int vector = VECTOR_START; vector < VECTOR_END; vector++) {
+        /* Pack as many as we can on a cache line. */
+        if (CROSSES_ALIGNMENT(pc, VECTOR_ENTRY_CODE_SIZE, proc_get_cache_line_size())) {
+            pc = check_size_and_cache_line(isa_mode, code, pc);
+        }
+        code->vector_entry[vector - VECTOR_START] = pc;
+        pc = emit_vector_entry(dcontext, code->common_vector_entry, vector, pc);
+    }
 #else /* UNIX */
     pc = check_size_and_cache_line(isa_mode, code, pc);
     code->do_syscall = pc;
@@ -1180,6 +1225,19 @@ emit_syscall_routines(dcontext_t *dcontext, generated_code_t *code, byte *pc,
     return pc;
 }
 
+#ifdef LINUX_KERNEL
+void
+optimize_syscall_code(dcontext_t *dcontext, fragment_t *f)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    byte *pc = code->syscall_entry;
+    /* Assert pc points to swapgs. */
+    ASSERT(pc[0] == 0x0f && pc[1] == 0x01 && pc[2] == 0xf8);
+    pc += 3;
+    insert_relative_jump(pc, f->start_pc, false);
+}
+#endif
+
 void
 arch_thread_init(dcontext_t *dcontext)
 {
@@ -1285,6 +1343,7 @@ arch_thread_init(dcontext_t *dcontext)
     pc = check_size_and_cache_line(isa_mode, code, pc);
     code->fcache_enter = pc;
     pc = emit_fcache_enter(dcontext, code, pc);
+    code->fcache_enter_end = pc;
     pc = check_size_and_cache_line(isa_mode, code, pc);
     code->fcache_return = pc;
     pc = emit_fcache_return(dcontext, code, pc);
@@ -1783,6 +1842,32 @@ in_generated_routine(dcontext_t *dcontext, cache_pc pc)
     /* XXX: what about inlined IBL stubs */
 }
 
+static bool
+in_fcache_enter_for_gencode(generated_code_t *code, cache_pc pc)
+{
+    return pc != NULL && pc >= code->fcache_enter && pc < code->fcache_enter_end;
+}
+
+bool
+in_fcache_enter(dcontext_t *dcontext, cache_pc pc)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    if (in_fcache_enter_for_gencode(code, pc))
+        return true;
+    if (USE_SHARED_GENCODE()) {
+        if (in_fcache_enter_for_gencode(shared_code, pc))
+            return true;
+#if defined(X86) && defined(X64)
+        if (shared_code_x86 != NULL && in_fcache_enter_for_gencode(shared_code_x86, pc))
+            return true;
+        if (shared_code_x86_to_x64 != NULL &&
+            in_fcache_enter_for_gencode(shared_code_x86_to_x64, pc))
+            return true;
+#endif
+    }
+    return false;
+}
+
 static bool
 in_fcache_return_for_gencode(generated_code_t *code, cache_pc pc)
 {
@@ -1978,6 +2063,52 @@ get_do_vmkuw_syscall_entry(dcontext_t *dcontext)
 #    endif
 #endif
 
+#ifdef LINUX_KERNEL
+cache_pc
+get_syscall_entry(dcontext_t *dcontext)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    return (cache_pc)code->syscall_entry;
+}
+
+cache_pc
+get_vector_entry(dcontext_t *dcontext, interrupt_vector_t vector)
+{
+    generated_code_t *code = THREAD_GENCODE(dcontext);
+    ASSERT(vector >= VECTOR_START && vector < VECTOR_END);
+    return (cache_pc)code->vector_entry[vector - VECTOR_START];
+}
+
+bool
+vector_has_error_code(interrupt_vector_t vector)
+{
+    ASSERT(vector >= VECTOR_START && vector < VECTOR_END);
+    switch (vector) {
+    case VECTOR_DOUBLE_FAULT:
+    case VECTOR_INVALID_TSS:
+    case VECTOR_SEGMENT_NOT_PRESENT:
+    case VECTOR_STACK_FAULT:
+    case VECTOR_GENERAL_PROTECTION:
+    case VECTOR_PAGE_FAULT:
+    case VECTOR_ALIGNMENT_CHECK: return true;
+    default: return false;
+    }
+}
+
+bool
+vector_is_synchronous(interrupt_vector_t vector)
+{
+    ASSERT(vector >= VECTOR_START && vector < VECTOR_END);
+    return vector >= VECTOR_EXCEPTION_START && vector < VECTOR_EXCEPTION_END;
+}
+
+bool
+was_kernel_interrupted(interrupt_stack_frame_t *frame)
+{
+    return (frame->cs & 3) == 0;
+}
+#endif
+
 cache_pc
 fcache_return_routine(dcontext_t *dcontext)
 {
@@ -2540,6 +2671,39 @@ get_ibl_routine_code(dcontext_t *dcontext, ibl_branch_type_t branch_type,
                                       : GENCODE_FROM_DCONTEXT));
 }
 
+#ifdef LINUX_KERNEL
+ibl_code_t *
+get_ibl_code_from_routine_pc(dcontext_t *dcontext, cache_pc pc)
+{
+    ibl_source_fragment_type_t source_fragment_type;
+    ibl_branch_type_t branch_type;
+#    if defined(X86) && defined(X64)
+    gencode_mode_t mode;
+#    endif
+
+    for (source_fragment_type = IBL_SOURCE_TYPE_START;
+         source_fragment_type < IBL_SOURCE_TYPE_END; source_fragment_type++) {
+        for (branch_type = IBL_BRANCH_TYPE_START; branch_type < IBL_BRANCH_TYPE_END;
+             branch_type++) {
+#    if defined(X86) && defined(X64)
+            for (mode = GENCODE_X64; mode <= GENCODE_X86_TO_X64; mode++) {
+#    endif
+                ibl_code_t *code = get_ibl_routine_code_internal(
+                    dcontext, source_fragment_type, branch_type _IF_X86_64(mode));
+                if (code != NULL && code->initialized &&
+                    pc >= code->indirect_branch_lookup_routine &&
+                    pc < code->indirect_branch_lookup_routine_end) {
+                    return code;
+                }
+#    if defined(X86) && defined(X64)
+            }
+#    endif
+        }
+    }
+    return NULL;
+}
+#endif
+
 #ifdef WINDOWS
 /* XXX We support a private and shared fragments simultaneously targeting
  * shared syscall -- -shared_fragment_shared_syscalls must be on and both
@@ -2818,6 +2982,10 @@ get_fcache_target(dcontext_t *dcontext)
 void
 set_fcache_target(dcontext_t *dcontext, cache_pc value)
 {
+#ifdef LINUX_KERNEL
+    /* Save the last app tag. We need this to handle interrupts in fcache_enter. */
+    dcontext->next_app_tag = dcontext->next_tag;
+#endif
     /* we used to use mcontext.pc, but that's in the writable
      * portion of the dcontext, and so for self-protection we use the
      * next_tag slot, which is protected
@@ -2881,7 +3049,7 @@ get_global_do_syscall_entry()
 /* used only by cleanup_and_terminate to avoid the sysenter
  * sygate hack version */
 byte *
-get_cleanup_and_terminate_global_do_syscall_entry()
+get_cleanup_and_terminate_global_do_syscall_entry(void)
 {
     /* see note above: for 32-bit linux apps we use int.
      * xref PR 332427 as well where sysenter causes a crash
@@ -4026,17 +4194,17 @@ unit_test_atomic_ops(void)
     ATOMIC_8BYTE_WRITE(&count3, value64, false);
     EXPECT(count3, -1);
 #    endif
-    EXPECT(atomic_inc_and_test(&count1), true);      /* result is 0 */
-    EXPECT(atomic_inc_and_test(&count1), false);     /* result is 1 */
-    EXPECT(atomic_dec_and_test(&count1), false);     /* init value is 1, result is 0 */
-    EXPECT(atomic_dec_and_test(&count1), true);      /* init value is 0, result is -1 */
-    EXPECT(atomic_dec_becomes_zero(&count1), false); /* result is -2 */
+    EXPECT(d_r_atomic_inc_and_test(&count1), true);  /* result is 0 */
+    EXPECT(d_r_atomic_inc_and_test(&count1), false); /* result is 1 */
+    EXPECT(d_r_atomic_dec_and_test(&count1), false); /* init value is 1, result is 0 */
+    EXPECT(d_r_atomic_dec_and_test(&count1), true);  /* init value is 0, result is -1 */
+    EXPECT(d_r_atomic_dec_becomes_zero(&count1), false);        /* result is -2 */
     EXPECT(atomic_compare_exchange_int(&count1, -3, 1), false); /* no exchange */
     EXPECT(count1, -2);
     EXPECT(atomic_compare_exchange_int(&count1, -2, 1), true); /* exchange */
-    EXPECT(atomic_dec_becomes_zero(&count1), true);            /* result is 0 */
-    EXPECT(atomic_dec_and_test(&count1), true); /* init value is 0, result is -1 */
-    EXPECT(atomic_dec_and_test(&count1), true); /* init value is -1, result is -2 */
+    EXPECT(d_r_atomic_dec_becomes_zero(&count1), true);        /* result is 0 */
+    EXPECT(d_r_atomic_dec_and_test(&count1), true); /* init value is 0, result is -1 */
+    EXPECT(d_r_atomic_dec_and_test(&count1), true); /* init value is -1, result is -2 */
     ATOMIC_4BYTE_WRITE(&count1, 0, false);
     do_parallel_updates();
     EXPECT(count1, MAX_NUM_THREADS);