feat(x86): implemented encoder

Dax89 · Dax89 · commit bcd154bded56 · 2026-07-04T23:33:45.000+02:00
diff --git a/capstonebundle/capstone.c b/capstonebundle/capstone.c
@@ -5,7 +5,7 @@ Capstone* capstone_create(const CapstoneInitData* data, int size) {
     cs_err err = cs_open(data->arch, data->mode, &h);
 
     if(err) {
-        rd_log(RD_LOG_FAIL, "%s", cs_strerror(err));
+        rd_log(RD_LOGLEVEL_FAIL, "%s", cs_strerror(err));
         return NULL;
     }
 
diff --git a/x86/CMakeLists.txt b/x86/CMakeLists.txt
@@ -13,12 +13,19 @@ add_library(${PROJECT_NAME} SHARED)
 
 target_sources(${PROJECT_NAME}
     PRIVATE 
+        encoder/encoder.c
+        encoder/grammar.c
         plugin.c
         lifter.c
         common.c
         registers.c
 )
 
+target_include_directories(${PROJECT_NAME}
+    PRIVATE
+        "${CMAKE_CURRENT_SOURCE_DIR}"
+)
+
 target_link_libraries(${PROJECT_NAME} 
     PRIVATE 
         redasm::redasm
diff --git a/x86/common.h b/x86/common.h
@@ -3,6 +3,19 @@
 #include <Zydis/Zydis.h>
 #include <redasm/redasm.h>
 
+typedef struct X86UserData {
+    ZydisMachineMode mode;
+    ZydisStackWidth width;
+} X86UserData;
+
+typedef struct X86Processor {
+    const X86UserData* userdata;
+    ZydisDecoder decoder;
+    char buffer[ZYDIS_MAX_INSTRUCTION_LENGTH];
+    RDLexer* lex;
+    // const RDCallingConvention** calling_conventions{nullptr};
+} X86Processor;
+
 typedef struct X86Address {
     RDAddress value;
     bool has_value;
diff --git a/x86/encoder/encoder.c b/x86/encoder/encoder.c
@@ -0,0 +1,30 @@
+#include "encoder.h"
+#include "common.h"
+#include "grammar.h"
+
+bool x86_encode(RDContext* ctx, RDAddress address, const char* s,
+                RDScratchBuffer* buf, RDProcessor* proc) {
+    if(!s) s = "nop";
+
+    const X86Processor* self = (X86Processor*)proc;
+
+    X86GrammarData g = {
+        .ctx = ctx,
+        .req = {.machine_mode = self->userdata->mode},
+        .err_buf = buf,
+    };
+
+    if(!x86_encoder_parse(self->lex, s, &g)) return false;
+
+    char buffer[ZYDIS_MAX_INSTRUCTION_LENGTH] = {0};
+    ZyanUSize length = rd_count_of(buffer);
+
+    if(!ZYAN_SUCCESS(ZydisEncoderEncodeInstructionAbsolute(
+           &g.req, buffer, &length, (ZyanU64)address))) {
+        RD_LOG_FAIL("%s", rd_format_to(buf, "failed to encode '%s'", s));
+        return false;
+    }
+
+    rd_scratch_append(buf, buffer, (usize)length);
+    return true;
+}
diff --git a/x86/encoder/encoder.h b/x86/encoder/encoder.h
@@ -0,0 +1,6 @@
+#pragma once
+
+#include <redasm/redasm.h>
+
+bool x86_encode(RDContext* ctx, RDAddress address, const char* s,
+                RDScratchBuffer* scratch, RDProcessor* proc);
diff --git a/x86/encoder/grammar.c b/x86/encoder/grammar.c
@@ -0,0 +1,245 @@
+#include "grammar.h"
+#include <string.h>
+
+static ZydisMnemonic _x86_mnemonic_from_token(const RDToken* tok) {
+    for(int i = 0; i < ZYDIS_MNEMONIC_MAX_VALUE; i++) {
+        const char* s = ZydisMnemonicGetString((ZydisMnemonic)i);
+
+        if(s && strlen(s) == tok->length &&
+           !rd_strnicmp(s, tok->value, (int)tok->length))
+            return (ZydisMnemonic)i;
+    }
+
+    return ZYDIS_MNEMONIC_INVALID;
+}
+
+static ZydisRegister _x86_register_from_token(const RDToken* tok) {
+    for(int i = 0; i < ZYDIS_REGISTER_MAX_VALUE; i++) {
+        const char* s = ZydisRegisterGetString((ZydisRegister)i);
+
+        if(s && strlen(s) == tok->length &&
+           !rd_strnicmp(s, tok->value, (int)tok->length))
+            return (ZydisRegister)i;
+    }
+
+    return ZYDIS_REGISTER_NONE;
+}
+
+static bool _x86_size_keyword_to_bytes(const RDToken* tok, usize* out) {
+    static const struct {
+        const char* name;
+        usize size;
+    } SIZES[] = {
+        {"byte", 1},
+        {"word", 2},
+        {"dword", 4},
+        {"qword", 8},
+    };
+
+    for(usize i = 0; i < rd_count_of(SIZES); i++) {
+        if(strlen(SIZES[i].name) == tok->length &&
+           !rd_strnicmp(SIZES[i].name, tok->value, (int)tok->length)) {
+            *out = SIZES[i].size;
+            return true;
+        }
+    }
+
+    return false;
+}
+
+static bool _reg_operand_rule(RDLexer* lex, void* userdata) {
+    X86GrammarData* data = (X86GrammarData*)userdata;
+
+    RDToken tok;
+    if(!rd_lexer_next_expect(lex, RD_TOK_IDENTIFIER, &tok)) return false;
+
+    ZydisRegister reg = _x86_register_from_token(&tok);
+    if(reg == ZYDIS_REGISTER_NONE) return false;
+
+    ZydisEncoderOperand* zop = &data->req.operands[data->req.operand_count++];
+    zop->type = ZYDIS_OPERAND_TYPE_REGISTER;
+    zop->reg.value = reg;
+    return true;
+}
+
+static bool _imm_operand_rule(RDLexer* lex, void* userdata) {
+    X86GrammarData* data = (X86GrammarData*)userdata;
+
+    RDToken tok;
+    if(!rd_lexer_next(lex, &tok)) return false;
+
+    bool negative = tok.type == RD_TOK_MINUS;
+    if(negative && !rd_lexer_next(lex, &tok)) return false;
+
+    i64 value;
+
+    if(tok.type == RD_TOK_NUMBER) {
+        value = negative ? -(i64)tok.u_value : (i64)tok.u_value;
+    }
+    else if(tok.type == RD_TOK_IDENTIFIER) {
+        const char* name = rd_lexer_token_value(lex, &tok);
+        RDAddress addr;
+        if(!rd_get_address(data->ctx, name, &addr)) return false;
+        value = negative ? -(i64)addr : (i64)addr;
+    }
+    else
+        return false;
+
+    ZydisEncoderOperand* zop = &data->req.operands[data->req.operand_count++];
+    zop->type = ZYDIS_OPERAND_TYPE_IMMEDIATE;
+    zop->imm.s = value;
+    return true;
+}
+
+static bool _displ_operand_rule(RDLexer* lex, void* userdata) {
+    X86GrammarData* data = (X86GrammarData*)userdata;
+    usize size = 0;
+    RDToken tok;
+
+    if(rd_lexer_peek_expect(lex, RD_TOK_IDENTIFIER, &tok) &&
+       _x86_size_keyword_to_bytes(&tok, &size)) {
+        rd_lexer_consume(lex);
+
+        if(!rd_lexer_next_expect(lex, RD_TOK_IDENTIFIER, &tok) ||
+           tok.length != 3 || rd_strnicmp("ptr", tok.value, 3))
+            return false;
+    }
+
+    if(!rd_lexer_next_expect(lex, RD_TOK_OPEN_SQUARE, &tok)) return false;
+
+    ZydisRegister base = ZYDIS_REGISTER_NONE, index = ZYDIS_REGISTER_NONE;
+    u8 scale = 1;
+    i64 disp = 0;
+    bool have_disp = false;
+
+    // optional base register: "[0x1000]" (absolute) is valid with none at all
+    if(rd_lexer_peek_expect(lex, RD_TOK_IDENTIFIER, &tok)) {
+        ZydisRegister reg = _x86_register_from_token(&tok);
+
+        if(reg != ZYDIS_REGISTER_NONE) {
+            rd_lexer_consume(lex);
+            base = reg;
+        }
+    }
+
+    for(;;) {
+        if(!rd_lexer_next(lex, &tok)) return false;
+        if(tok.type == RD_TOK_CLOSE_SQUARE) break;
+
+        bool plus = tok.type == RD_TOK_PLUS, minus = tok.type == RD_TOK_MINUS;
+        if(!plus && !minus) return false;
+        if(!rd_lexer_next(lex, &tok)) return false;
+
+        if(tok.type == RD_TOK_IDENTIFIER) { // index register, optional "*scale"
+            ZydisRegister reg = _x86_register_from_token(&tok);
+            if(reg == ZYDIS_REGISTER_NONE) return false;
+            index = reg;
+
+            if(rd_lexer_peek_expect(lex, RD_TOK_ASTERISK, &tok)) {
+                rd_lexer_consume(lex);
+
+                if(!rd_lexer_next_expect(lex, RD_TOK_NUMBER, &tok))
+                    return false;
+
+                scale = (u8)tok.u_value;
+            }
+        }
+        else if(tok.type == RD_TOK_NUMBER) {
+            disp = minus ? -(i64)tok.u_value : (i64)tok.u_value;
+            have_disp = true;
+        }
+        else
+            return false;
+    }
+
+    return true;
+}
+
+bool x86_encoder_parse(RDLexer* lex, const char* s, X86GrammarData* data) {
+    rd_lexer_reset(lex, s);
+
+    RDToken tok;
+    if(!rd_lexer_next_expect(lex, RD_TOK_IDENTIFIER, &tok)) {
+        RD_LOG_FAIL(
+            "%s", rd_format_to(data->err_buf, "expected mnemonic in '%s'", s));
+        return false;
+    }
+
+    ZydisMnemonic mnem = _x86_mnemonic_from_token(&tok);
+    if(mnem == ZYDIS_MNEMONIC_INVALID) {
+        RD_LOG_FAIL("%s", rd_format_to(data->err_buf,
+                                       "unknown mnemonic '%.*s' in '%s'",
+                                       (int)tok.length, tok.value, s));
+        return false;
+    }
+
+    data->req.mnemonic = mnem;
+
+    static const RDLexerTryFn OPERAND_ENTRIES[] = {
+        _reg_operand_rule,
+        _imm_operand_rule,
+        _displ_operand_rule,
+        NULL,
+    };
+
+    RDToken peek;
+    if(rd_lexer_peek(lex, &peek)) {
+        if(!rd_lexer_try_any(lex, OPERAND_ENTRIES, data)) {
+            RD_LOG_FAIL("%s", rd_format_to(data->err_buf,
+                                           "expected operand in '%s'", s));
+            return false;
+        }
+
+        while(rd_lexer_peek_expect(lex, RD_TOK_COMMA, &tok)) {
+            rd_lexer_consume(lex);
+
+            if(data->req.operand_count >= ZYDIS_ENCODER_MAX_OPERANDS) {
+                RD_LOG_FAIL("%s", rd_format_to(data->err_buf,
+                                               "too many operands in '%s'", s));
+                return false;
+            }
+
+            if(!rd_lexer_try_any(lex, OPERAND_ENTRIES, data)) {
+                RD_LOG_FAIL("%s", rd_format_to(
+                                      data->err_buf,
+                                      "expected operand after ',' in '%s'", s));
+                return false;
+            }
+        }
+    }
+
+    if(!rd_lexer_at_end(lex)) {
+        RD_LOG_FAIL("%s", rd_format_to(data->err_buf,
+                                       "unexpected trailing input in '%s'", s));
+        return false;
+    }
+
+    // clang-format off
+    usize reg_size = 0;
+    for(usize i = 0; i < data->req.operand_count; i++) {
+        if(data->req.operands[i].type == ZYDIS_OPERAND_TYPE_REGISTER) {
+            reg_size = ZydisRegisterGetWidth(data->req.machine_mode,
+                                             data->req.operands[i].reg.value) / 8;
+            break;
+        }
+    }
+    // clang-format on
+
+    for(usize i = 0; i < data->req.operand_count; i++) {
+        ZydisEncoderOperand* op = &data->req.operands[i];
+        if(op->type != ZYDIS_OPERAND_TYPE_MEMORY || op->mem.size != 0) continue;
+
+        if(!reg_size) {
+            RD_LOG_FAIL("%s",
+                        rd_format_to(data->err_buf,
+                                     "ambiguous memory operand size in '%s'"
+                                     "specify byte/word/dword/qword ptr",
+                                     s));
+            return false;
+        }
+
+        op->mem.size = (ZyanU16)reg_size;
+    }
+
+    return true;
+}
diff --git a/x86/encoder/grammar.h b/x86/encoder/grammar.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include <Zydis/Zydis.h>
+#include <redasm/redasm.h>
+
+typedef struct X86GrammarData {
+    RDContext* ctx;
+    ZydisEncoderRequest req;
+    RDScratchBuffer* err_buf;
+} X86GrammarData;
+
+bool x86_encoder_parse(RDLexer* lex, const char* s, X86GrammarData* data);
diff --git a/x86/plugin.c b/x86/plugin.c

Original file line number	Diff line number	Diff line change
`@@ -5,7 +5,7 @@ Capstone* capstone_create(const CapstoneInitData* data, int size) {`
`5`	`5`	`cs_err err = cs_open(data->arch, data->mode, &h);`
`6`	`6`
`7`	`7`	`if(err) {`
`8`		`- rd_log(RD_LOG_FAIL, "%s", cs_strerror(err));`
	`8`	`+ rd_log(RD_LOGLEVEL_FAIL, "%s", cs_strerror(err));`
`9`	`9`	`return NULL;`
`10`	`10`	`}`
`11`	`11`