Add subhook_set_disasm_handler()

README.md

@@ -78,7 +78,8 @@ int main() {
 Please note that subhook has a very simple length disassmebler engine (LDE)
 that works only with most common prologue instructions like push, mov, call,
 etc. When it encounters an unknown instruction subhook_get_trampoline() will
-return NULL.
+return NULL. You can delegate instruction decoding to a custom disassembler
+of your choice via `subhook_set_disasm_handler()`.
 
 ### C++
 
@@ -117,10 +118,9 @@ Known issues
 ------------
 
 * `subhook_get_trampoline()` may return NULL because only a small subset of
-  x86 instructions is supported by the disassembler in this library (only 
+  x86 instructions is supported by the disassembler in this library (just 
   common prologue instructions). As a workaround you can plug in a more
-  advanced disassembler engine in `subhook_disasm()` (currently there is no
+  advanced instruction length decoder using `subhook_set_disasm_handler()`.
-  simple way to do it, you have to modify the code manually).
 
 * If a target function (the function you are hooking) is less than N bytes
   in length, for example if it's a short 2-byte jump to a nearby location

subhook.c

@@ -26,6 +26,8 @@
 #include "subhook.h"
 #include "subhook_private.h"
 
+subhook_disasm_handler_t subhook_disasm_handler = NULL;
+
 SUBHOOK_EXPORT void *SUBHOOK_API subhook_get_src(subhook_t hook) {
   if (hook == NULL) {
     return NULL;
@@ -54,6 +56,11 @@ SUBHOOK_EXPORT int SUBHOOK_API subhook_is_installed(subhook_t hook) {
   return hook->installed;
 }
 
+SUBHOOK_EXPORT void SUBHOOK_API subhook_set_disasm_handler(
+  subhook_disasm_handler_t handler) {
+  subhook_disasm_handler = handler;
+}
+
 #ifndef SUBHOOK_SEPARATE_SOURCE_FILES
 
 #if defined SUBHOOK_WINDOWS

subhook.h

@@ -97,6 +97,10 @@ typedef enum subhook_flags {
 struct subhook_struct;
 typedef struct subhook_struct *subhook_t;
 
+typedef int (SUBHOOK_API *subhook_disasm_handler_t)(
+  void *src,
+  int *reloc_op_offset);
+
 SUBHOOK_EXPORT subhook_t SUBHOOK_API subhook_new(void *src,
                                                  void *dst,
                                                  subhook_flags_t flags);
@@ -112,11 +116,21 @@ SUBHOOK_EXPORT int SUBHOOK_API subhook_remove(subhook_t hook);
 
 /* Reads hook destination address from code.
  *
- * This is useful when you don't know the address or want to check
+ * This is useful when you don't know the address or want to check whether
- * whether src is already hooked.
+ * src is already hooked.
  */
 SUBHOOK_EXPORT void *SUBHOOK_API subhook_read_dst(void *src);
 
+/* Set a custom disassmbler function to use in place of the default one
+ * (subhook_disasm).
+ *
+ * The default function recognized a small st of x86 instructiosn commonly
+ * in prologues. If it fails in your situation you might want to use a more
+ * advanced disassembler library.
+ */
+SUBHOOK_EXPORT void SUBHOOK_API subhook_set_disasm_handler(
+  subhook_disasm_handler_t handler);
+
 #ifdef __cplusplus
 
 namespace subhook {
@@ -136,6 +150,14 @@ inline HookFlags operator&(HookFlags o1, HookFlags o2) {
       static_cast<unsigned int>(o1) & static_cast<unsigned int>(o2));
 }
 
+inline void *ReadHookDst(void *src) {
+  return subhook_read_dst(src);
+}
+
+inline void SetDisasmHandler(subhook_disasm_handler_t handler) {
+  subhook_set_disasm_handler(handler);
+}
+
 class Hook {
  public:
   Hook() : hook_(0) {}
@@ -174,10 +196,6 @@ class Hook {
     return !!subhook_is_installed(hook_);
   }
 
-  static void *ReadDst(void *src) {
-    return subhook_read_dst(src);
-  }
-
  private:
   Hook(const Hook &);
   void operator=(const Hook &);

subhook_x86.c

@@ -85,7 +85,9 @@ struct subhook_jmp64 {
 
 #pragma pack(pop)
 
-static size_t subhook_disasm(void *src, int32_t *reloc_op_offset) {
+extern subhook_disasm_handler_t subhook_disasm_handler;
+
+static int subhook_disasm(void *src, int *reloc_op_offset) {
   enum flags {
     MODRM      = 1,
     PLUS_R     = 1 << 1,
@@ -202,8 +204,8 @@ static size_t subhook_disasm(void *src, int32_t *reloc_op_offset) {
 
   uint8_t *code = src;
   size_t i;
-  size_t len = 0;
+  int len = 0;
-  size_t operand_size = 4;
+  int operand_size = 4;
   uint8_t opcode = 0;
   int found_opcode = false;
 
@@ -255,7 +257,7 @@ static size_t subhook_disasm(void *src, int32_t *reloc_op_offset) {
   }
 
   if (reloc_op_offset != NULL && opcodes[i].flags & RELOC) {
-    *reloc_op_offset = (int32_t)len; /* relative call or jump */
+    *reloc_op_offset = len; /* relative call or jump */
   }
 
   if (opcodes[i].flags & MODRM) {
@@ -378,6 +380,8 @@ static int subhook_make_trampoline(void *trampoline,
   size_t insn_len;
   intptr_t trampoline_addr = (intptr_t)trampoline;
   intptr_t src_addr = (intptr_t)src;
+  subhook_disasm_handler_t disasm_handler =
+    subhook_disasm_handler != NULL ? subhook_disasm_handler : subhook_disasm;
 
   assert(trampoline_len != NULL);
 
@@ -385,10 +389,10 @@ static int subhook_make_trampoline(void *trampoline,
    * to the trampoline.
    */
   while (orig_size < jmp_size) {
-    int32_t reloc_op_offset = 0;
+    int reloc_op_offset = 0;
 
     insn_len =
-      subhook_disasm((void *)(src_addr + orig_size), &reloc_op_offset);
+      disasm_handler((void *)(src_addr + orig_size), &reloc_op_offset);
 
     if (insn_len == 0) {
       return -EINVAL;