improve x64 decompiler coverage and cross-platform portability

mylovereturns · mylovereturns · commit cbfd68e88a82 · 2026-05-22T22:13:32.000+03:00
Decompiler: Expanded the x64 lifter to support more instructions including inc/dec, imul/idiv, movsx/movzx, setcc, and various shifts/rotates.
Analysis: Refined Dead Code Elimination (DCE) to safely handle function epilogues and improved type inference with more standard library signatures (string/memory operations).
Portability: Implemented native process enumeration for Linux (/proc) and macOS (libproc) in the debugger.
Emitter: Added explicit integer type casting (e.g., uint32_t) to the generated pseudo-code for better readability.
Build: Updated CMake configuration to include missing macOS frameworks.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -61,7 +61,7 @@ if(WIN32)
 endif()
 
 if(APPLE)
-    target_link_libraries(${PROJECT_NAME} PRIVATE "-framework Cocoa" "-framework IOKit")
+    target_link_libraries(${PROJECT_NAME} PRIVATE "-framework Cocoa" "-framework IOKit" "-framework CoreFoundation")
 endif()
 
 target_compile_definitions(${PROJECT_NAME} PRIVATE
diff --git a/src/core/database/export/ida_export.cpp b/src/core/database/export/ida_export.cpp
@@ -5,7 +5,7 @@
 
 namespace hype {
 
-bool IDAExport::write(const std::filesystem::path& path, const PEImage& img, const AnalysisDB& db) {
+bool IDAExport::write(const std::filesystem::path& path, const PEImage& /*img*/, const AnalysisDB& db) {
     std::ofstream f(path);
     if (!f) { spdlog::error("cannot write: {}", path.string()); return false; }
 
diff --git a/src/core/decompiler/dce.cpp b/src/core/decompiler/dce.cpp
@@ -48,51 +48,32 @@ void DCE::run(PcodeFunc& func) {
     static constexpr int RSP_ID = 4;
     static constexpr int RBP_ID = 5;
 
-    // Pass 1: eliminate ALL rsp/rbp arithmetic (prologue/epilogue)
-    // Keep only: CALL, RETURN, CBRANCH, BRANCH, and ops that produce non-stack results
+    // Pass 1: eliminate ONLY redundant prologue/epilogue register saves/restores
+    // and flag computations not used by branches.
     for (auto& blk : func.blocks) {
         for (auto& op : blk.ops) {
             if (op.op == PcodeOp::NOP || op.op == PcodeOp::CALL ||
                 op.op == PcodeOp::RETURN || op.op == PcodeOp::CBRANCH ||
                 op.op == PcodeOp::BRANCH) continue;
 
-            // kill RSP/RBP modifications (sub rsp, add rsp, mov rbp rsp, etc.)
-            if (op.output.valid() && (op.output.id == RSP_ID || op.output.id == RBP_ID) && op.output.is_reg()) {
-                op.op = PcodeOp::NOP;
-                continue;
-            }
-
-            // kill stores to stack (push operations, spills)
-            if (op.op == PcodeOp::STORE) {
-                bool stack_store = false;
-                if (!op.inputs.empty()) {
-                    auto& addr = op.inputs[0];
-                    if (addr.is_reg() && (addr.id == RSP_ID || addr.id == RBP_ID))
-                        stack_store = true;
-                    if (addr.kind == VarnodeKind::Stack)
-                        stack_store = true;
-                    // temp that was derived from rsp
-                    if (addr.is_temp())
-                        stack_store = true;
-                }
-                if (stack_store) { op.op = PcodeOp::NOP; continue; }
-            }
-
-            // kill loads from stack that restore callee-saved regs
-            if (op.op == PcodeOp::LOAD && op.output.valid() && op.output.is_reg()) {
-                if (is_callee_saved(op.output.id)) {
-                    op.op = PcodeOp::NOP;
-                    continue;
-                }
-            }
-
             // kill flag computations not used by branches
             if (op.output.valid() && is_flag_reg(op.output.id)) {
                 if (!is_used(op.output, func, -1, -1)) {
                     op.op = PcodeOp::NOP;
                     continue;
                 }
             }
+
+            // kill loads from stack that restore callee-saved regs (typical epilogue)
+            if (op.op == PcodeOp::LOAD && op.output.valid() && op.output.is_reg()) {
+                if (is_callee_saved(op.output.id)) {
+                    // Check if this reg is used after this LOAD. If not, it's just an epilogue restore.
+                    if (!is_used(op.output, func, -1, -1)) {
+                        op.op = PcodeOp::NOP;
+                        continue;
+                    }
+                }
+            }
         }
     }
 
diff --git a/src/core/decompiler/decompiler.cpp b/src/core/decompiler/decompiler.cpp
@@ -6,10 +6,11 @@ namespace hype {
 
 std::vector<PseudoLine> Decompiler::decompile(const Function& func, const AnalysisDB& db,
                                                const RTTIParser* rtti) {
+    (void)rtti;
     if (func.blocks.empty())
         return {{0, "// empty function", func.entry}};
 
-    if (func.blocks.size() > 200) {
+    if (func.blocks.size() > 5000) {
         std::vector<PseudoLine> out;
         out.push_back({0, fmt::format("// function too complex ({} blocks)", func.blocks.size()), func.entry});
         out.push_back({0, fmt::format("void {}() {{", func.name), func.entry});
@@ -45,8 +46,16 @@ std::vector<PseudoLine> Decompiler::decompile(const Function& func, const Analys
     else if (db.arch == Arch::X64 || db.arch == Arch::X86)
         pf = lifter_.lift(func, db);
     else {
-        // Fallback for other architectures or unsupported ones
-        return {{0, fmt::format("// architecture not supported in decompiler yet"), func.entry}};
+        std::vector<PseudoLine> out;
+        out.push_back({0, fmt::format("// architecture not supported in decompiler yet"), func.entry});
+        out.push_back({0, fmt::format("void {}() {{", func.name), func.entry});
+        for (auto& [ba, bb] : func.blocks) {
+            for (auto& insn : bb.insns) {
+                out.push_back({1, fmt::format("__asm {{ {} {} }}", insn.mnemonic, insn.op_str), insn.addr});
+            }
+        }
+        out.push_back({0, "}", 0});
+        return out;
     }
     ssa_.build(pf);
     dce_.run(pf);
diff --git a/src/core/decompiler/emitter.cpp b/src/core/decompiler/emitter.cpp
@@ -150,6 +150,13 @@ std::string Emitter::expr_str(const CExpr& e) {
         return fmt::format("~{}", expr_str(e.args[0]));
     }
 
+    if (e.op == PcodeOp::INT_ZEXT && !e.args.empty()) {
+        return fmt::format("(uint{}_t){}", e.vn.size * 8, expr_str(e.args[0]));
+    }
+    if (e.op == PcodeOp::INT_SEXT && !e.args.empty()) {
+        return fmt::format("(int{}_t){}", e.vn.size * 8, expr_str(e.args[0]));
+    }
+
     if (e.args.size() == 2) {
         std::string l = expr_str(e.args[0]);
         std::string r = expr_str(e.args[1]);
diff --git a/src/core/decompiler/lifter.cpp b/src/core/decompiler/lifter.cpp
@@ -257,7 +257,8 @@ void Lifter::lift_insn(const Insn& insn, const AnalysisDB& db, PcodeBlock& out)
     }
     case InsnType::Add: case InsnType::Sub:
     case InsnType::And: case InsnType::Or: case InsnType::Xor:
-    case InsnType::Shl: case InsnType::Shr: {
+    case InsnType::Shl: case InsnType::Shr: case InsnType::Sar:
+    case InsnType::Rol: case InsnType::Ror: {
         auto& dst_op = insn.ops[0];
         Varnode lhs = operand_read(insn, 0, db, out);
         Varnode rhs = operand_read(insn, 1, db, out);
@@ -278,29 +279,60 @@ void Lifter::lift_insn(const Insn& insn, const AnalysisDB& db, PcodeBlock& out)
         case InsnType::Xor: pop = PcodeOp::XOR; break;
         case InsnType::Shl: pop = PcodeOp::SHIFT_LEFT; break;
         case InsnType::Shr: pop = PcodeOp::SHIFT_RIGHT; break;
+        case InsnType::Sar: pop = PcodeOp::SHIFT_RIGHT; break; // simplified
+        case InsnType::Rol: pop = PcodeOp::SHIFT_LEFT; break;  // simplified
+        case InsnType::Ror: pop = PcodeOp::SHIFT_RIGHT; break; // simplified
         default: break;
         }
         Varnode result = alloc_temp(lhs.size);
         emit(out, pop, result, {lhs, rhs});
         operand_write(insn, 0, result, out);
         break;
     }
-    case InsnType::Mul: {
-        auto rax_v = vn_reg(REG_RAX, "rax", 8);
-        Varnode src = operand_read(insn, insn.op_count > 1 ? 1 : 0, db, out);
+    case InsnType::Inc: case InsnType::Dec: {
+        Varnode val = operand_read(insn, 0, db, out);
+        Varnode result = alloc_temp(val.size);
+        emit(out, insn.type == InsnType::Inc ? PcodeOp::ADD : PcodeOp::SUB,
+             result, {val, vn_const(1, val.size)});
+        operand_write(insn, 0, result, out);
+        break;
+    }
+    case InsnType::Mul: case InsnType::Imul: {
         Varnode result = alloc_temp();
-        emit(out, PcodeOp::INT_MULT, result, {rax_v, src});
-        emit(out, PcodeOp::COPY, rax_v, {result});
+        if (insn.op_count == 1) {
+            auto rax_v = vn_reg(REG_RAX, "rax", 8);
+            Varnode src = operand_read(insn, 0, db, out);
+            emit(out, PcodeOp::INT_MULT, result, {rax_v, src});
+            emit(out, PcodeOp::COPY, rax_v, {result});
+        } else if (insn.op_count == 2) {
+            Varnode lhs = operand_read(insn, 0, db, out);
+            Varnode rhs = operand_read(insn, 1, db, out);
+            emit(out, PcodeOp::INT_MULT, result, {lhs, rhs});
+            operand_write(insn, 0, result, out);
+        } else if (insn.op_count == 3) {
+            Varnode lhs = operand_read(insn, 1, db, out);
+            Varnode rhs = operand_read(insn, 2, db, out);
+            emit(out, PcodeOp::INT_MULT, result, {lhs, rhs});
+            operand_write(insn, 0, result, out);
+        }
         break;
     }
-    case InsnType::Div: {
+    case InsnType::Div: case InsnType::Idiv: {
         auto rax_v = vn_reg(REG_RAX, "rax", 8);
-        Varnode src = operand_read(insn, insn.op_count > 1 ? 1 : 0, db, out);
+        Varnode src = operand_read(insn, 0, db, out);
         Varnode result = alloc_temp();
         emit(out, PcodeOp::INT_DIV, result, {rax_v, src});
         emit(out, PcodeOp::COPY, rax_v, {result});
         break;
     }
+    case InsnType::Movsx: case InsnType::Movzx: {
+        Varnode src = operand_read(insn, 1, db, out);
+        Varnode result = alloc_temp(insn.ops[0].size / 8);
+        emit(out, insn.type == InsnType::Movsx ? PcodeOp::INT_SEXT : PcodeOp::INT_ZEXT,
+             result, {src});
+        operand_write(insn, 0, result, out);
+        break;
+    }
     case InsnType::Not: {
         Varnode src = operand_read(insn, 0, db, out);
         Varnode result = alloc_temp(src.size);
@@ -325,16 +357,24 @@ void Lifter::lift_insn(const Insn& insn, const AnalysisDB& db, PcodeBlock& out)
         emit(out, PcodeOp::COPY, rsp, {new_sp});
         break;
     }
-    case InsnType::Cmp: {
+    case InsnType::Cmp: case InsnType::Test: {
         Varnode lhs = operand_read(insn, 0, db, out);
         Varnode rhs = operand_read(insn, 1, db, out);
-        emit_flags(insn, lhs, rhs, out, false);
+        emit_flags(insn, lhs, rhs, out, insn.type == InsnType::Test);
         break;
     }
-    case InsnType::Test: {
-        Varnode lhs = operand_read(insn, 0, db, out);
-        Varnode rhs = operand_read(insn, 1, db, out);
-        emit_flags(insn, lhs, rhs, out, true);
+    case InsnType::Setcc: {
+        int flag_reg; bool negate;
+        jcc_to_flag_op(insn.mnemonic_id, flag_reg, negate);
+        Varnode flag = vn_reg(flag_reg, "flag", 1);
+        Varnode result = flag;
+        if (negate) {
+            result = alloc_temp(1);
+            emit(out, PcodeOp::BOOL_NOT, result, {flag});
+        }
+        Varnode final = alloc_temp(1);
+        emit(out, PcodeOp::INT_ZEXT, final, {result});
+        operand_write(insn, 0, final, out);
         break;
     }
     case InsnType::Jcc: {
diff --git a/src/core/decompiler/type_infer.cpp b/src/core/decompiler/type_infer.cpp
@@ -68,6 +68,10 @@ void TypeInfer::init_known_funcs() {
         {"calloc",    void_ptr,{sizet, sizet}, {"count", "size"}},
         {"realloc",   void_ptr,{void_ptr, sizet}, {"ptr", "size"}},
         {"free",      DecompType::make_void(), {void_ptr}, {"ptr"}},
+        {"_stricmp",  int32,   {const_char_ptr, const_char_ptr}, {"s1", "s2"}},
+        {"_strnicmp", int32,   {const_char_ptr, const_char_ptr, sizet}, {"s1", "s2", "n"}},
+        {"strstr",    char_ptr,{const_char_ptr, const_char_ptr}, {"haystack", "needle"}},
+        {"strchr",    char_ptr,{const_char_ptr, int32}, {"str", "c"}},
         {"printf",    int32,   {const_char_ptr}, {"fmt"}},
         {"sprintf",   int32,   {char_ptr, const_char_ptr}, {"buf", "fmt"}},
         {"puts",      int32,   {const_char_ptr}, {"str"}},
@@ -198,6 +202,10 @@ void TypeInfer::name_variables(const PcodeFunc& func) {
                 names_[op.output.id] = "argv";
             else if (fn_vn.name == "__p___argc")
                 names_[op.output.id] = "argc";
+            else if (fn_vn.name == "strstr" || fn_vn.name == "strchr")
+                names_[op.output.id] = "found";
+            else if (fn_vn.name == "strcmp" || fn_vn.name == "strncmp" || fn_vn.name == "_stricmp" || fn_vn.name == "_strnicmp")
+                names_[op.output.id] = "cmp";
         }
     }
 }
diff --git a/src/core/loader/dotnet_loader.cpp b/src/core/loader/dotnet_loader.cpp
@@ -146,8 +146,8 @@ bool DotNetLoader::parse_tables() {
     // TypeDef = table 2, MethodDef = table 6
     // simplified: just extract names from strings heap
     // parse TypeDef rows
-    u32 num_types = tables_[2].rows;
-    u32 num_methods = tables_[6].rows;
+    // u32 num_types = tables_[2].rows;
+    // u32 num_methods = tables_[6].rows;
 
     // We can't easily compute row sizes without knowing all coded index sizes
     // Simplified: scan the strings heap for type/method names
@@ -319,7 +319,7 @@ std::string DotNetLoader::resolve_token(u32 token) {
     }
 }
 
-void DotNetLoader::populate_db(AnalysisDB& db, const PEImage& img) {
+void DotNetLoader::populate_db(AnalysisDB& db, const PEImage& /*img*/) {
     for (auto& m : dn_.methods) {
         Function f;
         f.entry = m.rva;
diff --git a/src/debugger/debug_engine.cpp b/src/debugger/debug_engine.cpp
@@ -1,13 +1,17 @@
 #include "debug_engine.h"
 #include <spdlog/spdlog.h>
 #include <fmt/format.h>
+
+#ifdef _WIN32
 #include <psapi.h>
 #include <tlhelp32.h>
-
 #pragma comment(lib, "psapi.lib")
+#endif
 
 namespace hype {
 
+#ifdef _WIN32
+
 DebugEngine::~DebugEngine() {
     if (attached_) detach();
 }
@@ -667,4 +671,36 @@ void DebugEngine::log(const std::string& msg) {
     if (log_cb_) log_cb_(msg);
 }
 
+#else
+
+DebugEngine::~DebugEngine() {}
+bool DebugEngine::attach(u32 pid, DebugMode mode) { return false; }
+bool DebugEngine::detach() { return false; }
+void DebugEngine::run() {}
+void DebugEngine::pause() {}
+void DebugEngine::step_into() {}
+void DebugEngine::step_over() {}
+void DebugEngine::step_out() {}
+bool DebugEngine::set_breakpoint(va_t addr) { return false; }
+bool DebugEngine::remove_breakpoint(va_t addr) { return false; }
+bool DebugEngine::set_hw_breakpoint(va_t addr, int slot) { return false; }
+bool DebugEngine::remove_hw_breakpoint(int slot) { return false; }
+bool DebugEngine::read_memory(va_t addr, void* buf, size_t len) { return false; }
+bool DebugEngine::write_memory(va_t addr, const void* buf, size_t len) { return false; }
+DebugEngine::Registers DebugEngine::get_registers(u32 tid) { return {}; }
+bool DebugEngine::set_registers(const Registers& regs, u32 tid) { return false; }
+DebugEngine::DebugEvent DebugEngine::poll_event() { return {}; }
+void DebugEngine::debug_loop() {}
+void DebugEngine::update_modules() {}
+void DebugEngine::update_threads() {}
+void DebugEngine::restore_breakpoints_for_step(va_t addr) {}
+void DebugEngine::emit(DebugEvent ev) {}
+void DebugEngine::log(const std::string& msg) {}
+void DebugEngine::handle_breakpoint_hit(va_t addr, u32 tid) {}
+void DebugEngine::handle_single_step(u32 tid) {}
+void DebugEngine::handle_exception(const DEBUG_EVENT& ev) {}
+HANDLE DebugEngine::thread_handle(u32 tid) { return nullptr; }
+
+#endif
+
 }
diff --git a/src/debugger/debug_engine.h b/src/debugger/debug_engine.h
@@ -9,6 +9,14 @@
 #include <atomic>
 #include <functional>
 
+#ifdef _WIN32
+#include <windows.h>
+#else
+using HANDLE = void*;
+struct DEBUG_EVENT { int dummy; };
+struct LARGE_INTEGER { long long QuadPart; };
+#endif
+
 namespace hype {
 
 enum class DebugMode { Normal, Stealth };
diff --git a/src/debugger/instrumentation_cb.cpp b/src/debugger/instrumentation_cb.cpp
@@ -1,13 +1,18 @@
 #include "instrumentation_cb.h"
 #include <spdlog/spdlog.h>
 #include <fmt/format.h>
+
+#ifdef _WIN32
 #include <winternl.h>
 #include <cstring>
 
 #pragma comment(lib, "ntdll.lib")
+#endif
 
 namespace hype {
 
+#ifdef _WIN32
+
 namespace {
 
 using NtQueryInformationProcess_t = NTSTATUS(NTAPI*)(HANDLE, PROCESSINFOCLASS, PVOID, ULONG, PULONG);
@@ -239,4 +244,18 @@ bool InstrumentationBreakpoints::update_bp_list() {
     return true;
 }
 
+#else
+
+InstrumentationBreakpoints::~InstrumentationBreakpoints() {}
+bool InstrumentationBreakpoints::install(HANDLE process, u32 pid) { return false; }
+void InstrumentationBreakpoints::uninstall() {}
+bool InstrumentationBreakpoints::add_breakpoint(va_t addr) { return false; }
+bool InstrumentationBreakpoints::remove_breakpoint(va_t addr) { return false; }
+va_t InstrumentationBreakpoints::poll_hit() { return 0; }
+bool InstrumentationBreakpoints::alloc_and_write_shellcode() { return false; }
+bool InstrumentationBreakpoints::set_peb_callback() { return false; }
+bool InstrumentationBreakpoints::update_bp_list() { return false; }
+
+#endif
+
 }
diff --git a/src/debugger/process_list.cpp b/src/debugger/process_list.cpp
diff --git a/src/debugger/stealth.cpp b/src/debugger/stealth.cpp
