Merge remote-tracking branch 'origin/dev' into next

.github/workflows/build-ci.yml

@@ -80,7 +80,7 @@ jobs:
         cd ../qiling
         cd ../examples/rootfs/x86_linux/kernel && unzip -P infected m0hamed_rootkit.ko.zip
         cd ../../../../
-        pip3 install -e .[evm]
+        pip3 install -e .[evm,RE]
 
         if [ ${{ matrix.os }} == 'ubuntu-18.04' ] and [ ${{ matrix.python-version }} == '3.9' ]; then
           docker run -it --rm -v ${GITHUB_WORKSPACE}:/qiling qilingframework/qiling:dev bash -c "cd tests && ./test_onlinux.sh"

CREDITS.md

@@ -23,7 +23,7 @@
 #### CI, Website，Documentations, Logo & Swags
 
 - FOO Kevin (chfl4gs) <chfl4gs_at_qiling_io>
-- SU muchen (miraisuu) <suu_at_iling_io>
+- SU muchen (miraisuu) <suu_at_qiling_io>
 
 
 #### Key Contributors (in no particular order)

README.md

@@ -182,7 +182,7 @@ With qltool, easy execution can be performed:
 With shellcode:
 
 ```
-$ ./qltool shellcode --os linux --arch arm --hex -f examples/shellcodes/linarm32_tcp_reverse_shell.hex
+$ ./qltool code --os linux --arch arm --format hex -f examples/shellcodes/linarm32_tcp_reverse_shell.hex
 ```
 
 With binary file:

examples/crackme_x86_windows.py

@@ -15,7 +15,7 @@ ROOTFS = r"rootfs/x86_windows"
 class Solver:
     def __init__(self, invalid: bytes):
         # create a silent qiling instance
-        self.ql = Qiling([rf"{ROOTFS}/bin/crackme.exe"], ROOTFS, verbose=QL_VERBOSE.OFF)
+        self.ql = Qiling([rf"{ROOTFS}/bin/crackme.exe"], ROOTFS, verbose=QL_VERBOSE.DISABLED)
 
         self.ql.os.stdin = pipe.SimpleInStream(sys.stdin.fileno())  # take over the input to the program using a fake stdin
         self.ql.os.stdout = pipe.NullOutStream(sys.stdout.fileno()) # disregard program output

examples/crackme_x86_windows_setcallback.py

@@ -9,26 +9,36 @@ sys.path.append("..")
 from qiling import Qiling
 
 def force_call_dialog_func(ql: Qiling):
+    # this hook is invoked after returning from DialogBoxParamA, so its
+    # stack frame content is still available to us.
+
     # get DialogFunc address
-    lpDialogFunc = ql.unpack32(ql.mem.read(ql.arch.regs.esp - 0x8, 4))
+    lpDialogFunc = ql.stack_read(-8)
+
     # setup stack for DialogFunc
     ql.stack_push(0)
     ql.stack_push(1001)
     ql.stack_push(273)
     ql.stack_push(0)
     ql.stack_push(0x0401018)
+
     # force EIP to DialogFunc
     ql.arch.regs.eip = lpDialogFunc
 
 def my_sandbox(path, rootfs):
     ql = Qiling(path, rootfs)
 
+    # patch the input validation code: overwrite all its breaking points
+    # denoted with "jne 0x401135", so it would keep going even if there
+    # is an error
     ql.patch(0x004010B5, b'\x90\x90')
     ql.patch(0x004010CD, b'\x90\x90')
     ql.patch(0x0040110B, b'\x90\x90')
     ql.patch(0x00401112, b'\x90\x90')
 
+    # hook the instruction after returning from DialogBoxParamA
     ql.hook_address(force_call_dialog_func, 0x00401016)
+
     ql.run()
 
 if __name__ == "__main__":

examples/crackme_x86_windows_unpatch.py

@@ -9,21 +9,28 @@ sys.path.append("..")
 from qiling import Qiling
 
 def force_call_dialog_func(ql: Qiling):
+    # this hook is invoked after returning from DialogBoxParamA, so its
+    # stack frame content is still available to us.
+
     # get DialogFunc address
-    lpDialogFunc = ql.unpack32(ql.mem.read(ql.arch.regs.esp - 0x8, 4))
+    lpDialogFunc = ql.stack_read(-8)
+
     # setup stack for DialogFunc
     ql.stack_push(0)
     ql.stack_push(1001)
     ql.stack_push(273)
     ql.stack_push(0)
     ql.stack_push(0x0401018)
+
     # force EIP to DialogFunc
     ql.arch.regs.eip = lpDialogFunc
 
 def our_sandbox(path, rootfs):
     ql = Qiling(path, rootfs)
 
+    # hook the instruction after returning from DialogBoxParamA
     ql.hook_address(force_call_dialog_func, 0x00401016)
+
     ql.run()
 
 if __name__ == "__main__":

examples/extensions/r2/hello_r2.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+#
+# Cross Platform and Multi Architecture Advanced Binary Emulation Framework
+#
+
+import sys
+sys.path.append('..')
+
+from qiling import Qiling
+from qiling.const import QL_VERBOSE
+from qiling.extensions.r2 import R2
+
+
+def func(ql: Qiling, *args, **kwargs):
+    ql.os.stdout.write(b"=====hooked main=====!\n")
+    return
+
+def my_sandbox(path, rootfs):
+    ql = Qiling(path, rootfs, verbose=QL_VERBOSE.DISASM)
+    # QL_VERBOSE.DISASM will be monkey-patched when r2 is available
+    r2 = R2(ql)
+
+    # search bytes sequence using ql.mem.search
+    addrs = ql.mem.search(b'llo worl')  # return all matching results
+    print(r2.at(addrs[0]))  # find corresponding flag at the address and the offset to the flag
+    # search string using r2
+    addr = r2.strings['Hello world!'].vaddr  # key must be exactly same
+    print(addrs[0], addr)
+    # print xref to string "Hello world!"
+    print(r2.refto(addr))
+    # write to string using ql.mem.write
+    ql.mem.write(addr, b"No hello, Bye!\x00")
+
+    # get function address and hook it
+    ql.hook_address(func, r2.functions['main'].offset)
+    # enable trace powered by r2 symsmap
+    # r2.enable_trace()
+    ql.run()
+
+if __name__ == "__main__":
+    my_sandbox(["rootfs/x86_windows/bin/x86_hello.exe"], "rootfs/x86_windows")
+
+    # test shellcode mode
+    ARM64_LIN = bytes.fromhex('420002ca210080d2400080d2c81880d2010000d4e60300aa01020010020280d2681980d2010000d4410080d2420002cae00306aa080380d2010000d4210400f165ffff54e0000010420002ca210001caa81b80d2010000d4020004d27f0000012f62696e2f736800')
+    print("\nLinux ARM 64bit Shellcode")
+    ql = Qiling(code=ARM64_LIN, archtype="arm64", ostype="linux", verbose=QL_VERBOSE.DEBUG)
+    r2 = R2(ql)
+    # disassemble 32 instructions
+    print(r2._cmd('pd 32'))
+    ql.run()

examples/hello_arm_qnx_customapi.py

@@ -7,23 +7,26 @@ import sys
 sys.path.append("..")
 
 from qiling import Qiling
-from qiling.const import QL_INTERCEPT, QL_CALL_BLOCK, QL_VERBOSE
+from qiling.const import QL_INTERCEPT, QL_CALL_BLOCK
 from qiling.os.const import STRING
 
 def my_puts_onenter(ql: Qiling):
     params = ql.os.resolve_fcall_params({'s': STRING})
 
     print(f'puts("{params["s"]}")')
+
     return QL_CALL_BLOCK
 
 def my_printf_onenter(ql: Qiling):
     params = ql.os.resolve_fcall_params({'s': STRING})
 
     print(f'printf("{params["s"]}")')
+
     return QL_CALL_BLOCK
 
 def my_puts_onexit(ql: Qiling):
     print(f'after puts')
+
     return QL_CALL_BLOCK
 
 if __name__ == "__main__":

examples/hello_arm_uboot.py

@@ -11,8 +11,12 @@ from qiling.const import QL_VERBOSE
 from qiling.os.const import STRING
 
 def get_kaimendaji_password():
-    def my_getenv(ql, *args, **kwargs):
-        env = {"ID": b"000000000000000", "ethaddr": b"11:22:33:44:55:66"}
+    def my_getenv(ql: Qiling):
+        env = {
+            "ID"      : b"000000000000000",
+            "ethaddr" : b"11:22:33:44:55:66"
+        }
+
         params = ql.os.resolve_fcall_params({'key': STRING})
         value = env.get(params["key"], b"")
 
@@ -22,7 +26,7 @@ def get_kaimendaji_password():
         ql.arch.regs.r0 = value_addr
         ql.arch.regs.arch_pc = ql.arch.regs.lr
 
-    def get_password(ql, *args, **kwargs):
+    def get_password(ql: Qiling):
         password_raw = ql.mem.read(ql.arch.regs.r0, ql.arch.regs.r2)
 
         password = ''
@@ -34,7 +38,7 @@ def get_kaimendaji_password():
 
         print("The password is: %s" % password)
 
-    def partial_run_init(ql):
+    def partial_run_init(ql: Qiling):
         # argv prepare
         ql.arch.regs.arch_sp -= 0x30
         arg0_ptr = ql.arch.regs.arch_sp

examples/hello_linuxx8664_intercept.py

@@ -7,21 +7,24 @@ import sys
 sys.path.append("..")
 
 from qiling import Qiling
-from qiling.const import QL_INTERCEPT, QL_VERBOSE
+from qiling.const import QL_INTERCEPT
 from qiling.os.linux.syscall_nums import SYSCALL_NR
 
-def write_onenter(ql: Qiling, arg1, arg2, arg3, *args):
+def write_onenter(ql: Qiling, fd: int, buf: int, count: int):
     print("enter write syscall!")
-    ql.arch.regs.rsi = arg2 + 1
-    ql.arch.regs.rdx = arg3 - 1
 
-def write_onexit(ql: Qiling, arg1, arg2, arg3, *args):
+    ql.arch.regs.rsi = buf + 1
+    ql.arch.regs.rdx = count - 1
+
+def write_onexit(ql: Qiling, fd: int, buf: int, count: int, retval: int):
     print("exit write syscall!")
-    ql.arch.regs.rax = arg3 + 1
+
+    ql.arch.regs.rax = count + 1
 
 if __name__ == "__main__":
-    ql = Qiling(["rootfs/x8664_linux/bin/x8664_hello"], "rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG)
+    ql = Qiling(["rootfs/x8664_linux/bin/x8664_hello"], "rootfs/x8664_linux")
 
     ql.os.set_syscall(SYSCALL_NR.write, write_onenter, QL_INTERCEPT.ENTER)
     ql.os.set_syscall(SYSCALL_NR.write, write_onexit, QL_INTERCEPT.EXIT)
+
     ql.run()

examples/hello_x8664_linux_part_exec.py

@@ -10,13 +10,20 @@ from qiling import Qiling
 from qiling.const import QL_VERBOSE
 
 if __name__ == "__main__":
-    ql = Qiling(["rootfs/x8664_linux/bin/sleep_hello"], "rootfs/x8664_linux", verbose=QL_VERBOSE.DEFAULT)
+    def dump(ql, *args, **kw):
+        ql.save(reg=False, cpu_context=True, snapshot="/tmp/snapshot.bin")
+        ql.emu_stop()
 
+    ql = Qiling(["rootfs/x8664_linux/bin/sleep_hello"], "rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG)
+    X64BASE = int(ql.profile.get("OS64", "load_address"), 16)
+    ql.hook_address(dump, X64BASE + 0x1094)
+    ql.run()
+
+    ql = Qiling(["rootfs/x8664_linux/bin/sleep_hello"], "rootfs/x8664_linux", verbose=QL_VERBOSE.DISASM)
     # load base address from profile file
     X64BASE = int(ql.profile.get("OS64", "load_address"), 16)
-
+    ql.restore(snapshot="/tmp/snapshot.bin")
     # set execution starting and ending points
     begin_point = X64BASE + 0x109e
     end_point = X64BASE + 0x10bc
-
-    ql.run(begin=begin_point, end=end_point)
+    ql.run(begin = begin_point, end = end_point)

examples/sality.py

@@ -119,16 +119,16 @@ def hook_WriteFile(ql: Qiling, address: int, params):
     if hFile == 0x13371337:
         buffer = ql.mem.read(lpBuffer, nNumberOfBytesToWrite)
         try:
-            r, nNumberOfBytesToWrite = utils.io_Write(ql.amsint32_driver, buffer)
+            nNumberOfBytesToWrite = utils.io_Write(ql.amsint32_driver, buffer)
             ql.mem.write_ptr(lpNumberOfBytesWritten, nNumberOfBytesToWrite, 4)
-        except Exception as e:
+        except Exception:
             ql.log.exception("")
-            print("Exception = %s" % str(e))
-            r = 1
-        if r:
-            return 1
+            r = False
         else:
-            return 0
+            r = True
+
+        return int(r)
+
     else:
         return _WriteFile(ql, address, params)
 

examples/scripts/dllscollector.bat

@@ -57,6 +57,7 @@ CALL :collect_dll32 kdcom.dll
 CALL :collect_dll32 kernel32.dll
 CALL :collect_dll32 KernelBase.dll
 CALL :collect_dll32 mpr.dll
+CALL :collect_dll32 mscoree.dll
 CALL :collect_dll32 msvcp_win.dll
 CALL :collect_dll32 msvcp60.dll
 CALL :collect_dll32 msvcr120_clr0400.dll, msvcr110.dll
@@ -102,6 +103,7 @@ CALL :collect_dll64 advapi32.dll
 CALL :collect_dll64 gdi32.dll
 CALL :collect_dll64 kernel32.dll
 CALL :collect_dll64 KernelBase.dll
+CALL :collect_dll64 mscoree.dll
 CALL :collect_dll64 msvcrt.dll
 CALL :collect_dll64 ntdll.dll
 CALL :collect_dll64 ntoskrnl.exe

examples/uboot_bin.ql

@@ -4,17 +4,5 @@ entry_point = 0x80800000
 heap_size = 0x300000
 
 
-[LOG]
-# log directory output
-# usage: dir = qlog
-dir =
-# split log file, use with multithread
-split = False
-
-
 [MISC]
-# append string into different logs
-# maily for multiple times Ql run with one file
-# usage: append = test1
-append =
 current_path = /

examples/uefi_sanitized_heap.py

@@ -16,12 +16,18 @@ def my_abort(msg):
     os.abort()
 
 def enable_sanitized_heap(ql, fault_rate=0):
-    ql.os.heap = QlSanitizedMemoryHeap(ql, ql.os.heap, fault_rate=fault_rate)
+    heap = QlSanitizedMemoryHeap(ql, ql.os.heap, fault_rate=fault_rate)
 
-    ql.os.heap.oob_handler      = lambda *args: my_abort("Out-of-bounds read detected")
-    ql.os.heap.bo_handler       = lambda *args: my_abort("Buffer overflow/underflow detected")
-    ql.os.heap.bad_free_handler = lambda *args: my_abort("Double free or bad free detected")
-    ql.os.heap.uaf_handler      = lambda *args: my_abort("Use-after-free detected")
+    heap.oob_handler      = lambda *args: my_abort(f'Out-of-bounds read detected')
+    heap.bo_handler       = lambda *args: my_abort(f'Buffer overflow/underflow detected')
+    heap.bad_free_handler = lambda *args: my_abort(f'Double free or bad free detected')
+    heap.uaf_handler      = lambda *args: my_abort(f'Use-after-free detected')
+
+    # make sure future allocated buffers are not too close to UEFI data
+    heap.alloc(0x1000)
+
+    ql.os.heap = heap
+    ql.loader.dxe_context.heap = heap
 
 def sanitized_emulate(path, rootfs, fault_type, verbose=QL_VERBOSE.DEBUG):
     env = {'FaultType': fault_type}
@@ -39,7 +45,7 @@ Usage: ./uefi_santizied_heap.py <fault-type>
 Valid fault types:
 0 - POOL_OVERFLOW_MEMCPY
 1 - POOL_UNDERFLOW_MEMCPY
-2 - POOL_OVERFLOW_USER,
+2 - POOL_OVERFLOW_USER
 3 - POOL_UNDERFLOW_USER
 4 - POOL_OOB_READ_AHEAD
 5 - POOL_OOB_READ_BEHIND

qiling/arch/evm/__init__.py

@@ -1,5 +1,9 @@
 import sys
 
+# python 3.10 has not been supported yet in the latest blake2b-py release
+if sys.version_info >= (3,10):
+    sys.exit('Sorry, Python > 3.10 is not supported for now')
+
 #  Ensure we can reach 1024 frames of recursion
 #
 EVM_RECURSION_LIMIT = 1024 * 12

qiling/arch/utils.py

@@ -73,7 +73,12 @@ class QlArchUtils:
             self._block_hook = None
 
         if verbosity >= QL_VERBOSE.DISASM:
-            self._disasm_hook = self.ql.hook_code(self.disassembler)
+            try:  # monkey patch disassembler
+                from qiling.extensions.r2 import R2
+                r2 = R2(self.ql)
+                self._disasm_hook = self.ql.hook_code(r2.disassembler)
+            except (ImportError, ModuleNotFoundError):
+                self._disasm_hook = self.ql.hook_code(self.disassembler)
 
             if verbosity >= QL_VERBOSE.DUMP:
                 self._block_hook = self.ql.hook_block(ql_hook_block_disasm)

qiling/arch/x86_utils.py

@@ -1,5 +1,6 @@
 
 from abc import abstractmethod
+from typing import Optional
 
 from qiling import Qiling
 from qiling.arch.x86 import QlArchIntel
@@ -32,7 +33,7 @@ class GDTArray:
 
         self.mem.write(self.base + (index * self.entsize), data)
 
-    def get_next_free(self, start: int = None, end: int = None) -> int:
+    def get_next_free(self, start: Optional[int] = None, end: Optional[int] = None) -> int:
         # The Linux kernel determines whether the segment is empty by judging whether the content in the current GDT segment is 0.
         null_entry = b'\x00' * self.entsize
 
@@ -112,7 +113,7 @@ class GDTManager:
     def set_entry(self, index: int, data: bytes) -> None:
         self.array[index] = data
 
-    def get_free_idx(self, start: int = None, end: int = None) -> int:
+    def get_free_idx(self, start: Optional[int] = None, end: Optional[int] = None) -> int:
         return self.array.get_next_free(start, end)
 
 

qiling/core.py

@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, AnyStr, List, Mapping, MutableMapping, Op
 
 # See https://stackoverflow.com/questions/39740632/python-type-hinting-without-cyclic-imports
 if TYPE_CHECKING:
+    from os import PathLike
     from unicorn.unicorn import Uc
     from configparser import ConfigParser
     from logging import Logger
@@ -651,10 +652,13 @@ class Qiling(QlCoreHooks, QlCoreStructs):
 
 
     # Map "ql_path" to any objects which implements QlFsMappedObject.
-    def add_fs_mapper(self, ql_path, real_dest):
+    def add_fs_mapper(self, ql_path: Union["PathLike", str], real_dest):
         self.os.fs_mapper.add_fs_mapping(ql_path, real_dest)
 
-
+    # Remove "ql_path" mapping.
+    def remove_fs_mapper(self, ql_path: Union["PathLike", str]):
+        self.os.fs_mapper.remove_fs_mapping(ql_path)
+    
     # push to stack bottom, and update stack register
     def stack_push(self, data):
         return self.arch.stack_push(data)
@@ -703,6 +707,9 @@ class Qiling(QlCoreHooks, QlCoreStructs):
             count  : max emulation steps (instructions count); unlimited by default
         """
 
+        if self._arch.type in (QL_ARCH.ARM, QL_ARCH.CORTEX_M) and self._arch._init_thumb:
+            begin |= 1
+
         self.uc.emu_start(begin, end, timeout, count)
 
         if self._internal_exception is not None:

qiling/debugger/gdb/gdb.py

@@ -13,9 +13,10 @@
 # gdb remote protocol:
 #   https://sourceware.org/gdb/current/onlinedocs/gdb/Remote-Protocol.html
 
-import os, socket, re
+import os, socket, re, tempfile
 from logging import Logger
-from typing import Iterator, Optional, Union
+from typing import Iterator, Mapping, Optional, Union
+import typing
 
 from unicorn import UcError
 from unicorn.unicorn_const import (
@@ -30,6 +31,8 @@ from qiling.const import QL_ARCH, QL_ENDIAN, QL_OS
 from qiling.debugger import QlDebugger
 from qiling.debugger.gdb.xmlregs import QlGdbFeatures
 from qiling.debugger.gdb.utils import QlGdbUtils
+from qiling.os.linux.procfs import QlProcFS
+from qiling.os.mapper import QlFsMappedCallable, QlFsMappedObject
 
 # gdb logging prompt
 PROMPT = r'gdb>'
@@ -63,7 +66,7 @@ class QlGdb(QlDebugger):
     """A simple gdbserver implementation.
     """
 
-    def __init__(self, ql: Qiling, ip: str = '127.0.01', port: int = 9999):
+    def __init__(self, ql: Qiling, ip: str = '127.0.0.1', port: int = 9999):
         super().__init__(ql)
 
         if type(port) is str:
@@ -101,6 +104,11 @@ class QlGdb(QlDebugger):
         self.features = QlGdbFeatures(self.ql.arch.type, self.ql.os.type)
         self.regsmap = self.features.regsmap
 
+        # https://sourceware.org/bugzilla/show_bug.cgi?id=17760
+        # 42000 is the magic pid to indicate the remote process.
+        self.ql.add_fs_mapper(r'/proc/42000/maps',  QlFsMappedCallable(QlProcFS.self_map, self.ql.mem))
+        self.fake_procfs: Mapping[int, typing.IO] = {}
+
     def run(self):
         server = GdbSerialConn(self.ip, self.port, self.ql.log)
         killed = False
@@ -378,7 +386,7 @@ class QlGdb(QlDebugger):
 
             if query == 'Supported':
                 # list of supported features excluding the multithreading-related ones
-                common = (
+                features = [
                     'BreakpointCommands+',
                     'ConditionalBreakpoints+',
                     'ConditionalTracepoints+',
@@ -408,23 +416,21 @@ class QlGdb(QlDebugger):
                     'multiprocess+',
                     'no-resumed+',
                     'qXfer:auxv:read+',
-                    'qXfer:exec-file:read+',
                     'qXfer:features:read+',
                     # 'qXfer:libraries-svr4:read+',
                     # 'qXfer:osdata:read+',
                     'qXfer:siginfo:read+',
                     'qXfer:siginfo:write+',
                     'qXfer:statictrace:read+',
-                    'qXfer:threads:read+',
                     'qXfer:traceframe-info:read+',
                     'swbreak+',
                     'tracenz+',
                     'vfork-events+'
-                )
+                ]
 
                 # might or might not need for multi thread
                 if self.ql.multithread:
-                    features = (
+                    features += [
                         'PacketSize=47ff',
                         'FastTracepoints+',
                         'QThreadEvents+',
@@ -436,16 +442,26 @@ class QlGdb(QlDebugger):
                         'qXfer:btrace-conf:read+',
                         'qXfer:btrace:read+',
                         'vContSupported+'
-                    )
+                    ]
 
                 else:
-                    features = (
+                    features += [
                         'PacketSize=3fff',
                         'qXfer:spu:read+',
                         'qXfer:spu:write+'
-                    )
+                    ]
 
-                return ';'.join(common + features)
+                # os dependent features
+                if not self.ql.interpreter:
+                    # filesystem dependent features
+                    if hasattr(self.ql.os, 'path'):
+                        features.append('qXfer:exec-file:read+')
+
+                    # process dependent features
+                    if hasattr(self.ql.os, 'pid'):
+                        features.append('qXfer:threads:read+')
+
+                return ';'.join(features)
 
             elif query == 'Xfer':
                 feature, op, annex, params = data
@@ -462,39 +478,27 @@ class QlGdb(QlDebugger):
                     return f'{"l" if len(content) < length else "m"}{content}'
 
                 elif feature == 'threads' and op == 'read':
-                    if not self.ql.baremetal and hasattr(self.ql.os, 'pid'):
-                        content = '\r\n'.join((
-                            '<threads>',
-                            f'<thread id="{self.ql.os.pid}" core="1" name="{self.ql.targetname}"/>',
-                            '</threads>'
-                        ))
-
-                    else:
-                        content = ''
+                    content = '\r\n'.join((
+                        '<threads>',
+                        f'<thread id="{self.ql.os.pid}" core="1" name="{self.ql.targetname}"/>',
+                        '</threads>'
+                    ))
 
                     return f'l{content}'
 
                 elif feature == 'auxv' and op == 'read':
-                    auxv_data = bytearray()
+                    try:
+                        with self.ql.os.fs_mapper.open('/proc/self/auxv', 'rb') as infile:
+                            infile.seek(offset, 0) # SEEK_SET
+                            auxv_data = infile.read(length)
 
-                    if hasattr(self.ql.loader, 'auxv'):
-                        nbytes = self.ql.arch.bits // 8
+                    except FileNotFoundError:
+                        auxv_data = b''
 
-                        auxv_addr = self.ql.loader.auxv + offset
-                        null_entry = bytes(nbytes * 2)
-
-                        # keep reading until AUXV.AT_NULL is reached
-                        while not auxv_data.endswith(null_entry):
-                            auxv_data.extend(self.ql.mem.read(auxv_addr, nbytes))
-                            auxv_addr += nbytes
-
-                            auxv_data.extend(self.ql.mem.read(auxv_addr, nbytes))
-                            auxv_addr += nbytes
-
-                    return b'l' + auxv_data[:length]
+                    return b'l' + auxv_data
 
                 elif feature == 'exec-file' and op == 'read':
-                    return f'l{os.path.abspath(self.ql.path)}'
+                    return f'l{self.ql.os.path.host_to_virtual_path(self.ql.path)}'
 
                 elif feature == 'libraries-svr4' and op == 'read':
                     # TODO: this one requires information of loaded libraries which currently not provided
@@ -584,16 +588,24 @@ class QlGdb(QlDebugger):
                         flags = int(flags, 16)
                         mode = int(mode, 16)
 
-                        # try to guess whether this is an emulated path or real one
-                        if path.startswith(os.path.abspath(self.ql.rootfs)):
-                            host_path = path
+                        virtpath = self.ql.os.path.virtual_abspath(path)
+
+                        if virtpath.startswith(r'/proc') and self.ql.os.fs_mapper.has_mapping(virtpath):
+                            # Mapped object by itself is not backed with a host fd and thus a tempfile can
+                            # 1. Make pread easy to implement and avoid duplicate code like seek, fd etc.
+                            # 2. Avoid fd clash if we assign a generated fd.
+                            tfile = tempfile.TemporaryFile("rb+")
+                            tfile.write(self.ql.os.fs_mapper.open(virtpath, "rb+").read())
+                            tfile.seek(0, os.SEEK_SET)
+                            fd = tfile.fileno()
+                            self.fake_procfs[fd] = tfile
                         else:
                             host_path = self.ql.os.path.virtual_to_host_path(path)
 
-                        self.ql.log.debug(f'{PROMPT} target file: {host_path}')
+                            self.ql.log.debug(f'{PROMPT} target host path: {host_path}')
 
-                        if os.path.exists(host_path) and not path.startswith(r'/proc'):
-                            fd = os.open(host_path, flags, mode)
+                            if os.path.exists(host_path):
+                                fd = os.open(host_path, flags, mode)
 
                     return f'F{fd:x}'
 
@@ -609,6 +621,10 @@ class QlGdb(QlDebugger):
                     fd = int(fd, 16)
 
                     os.close(fd)
+                    
+                    if fd in self.fake_procfs:
+                        del self.fake_procfs[fd]
+                    
                     return 'F0'
 
                 return REPLY_EMPTY
@@ -628,7 +644,7 @@ class QlGdb(QlDebugger):
                     groups = subcmd.split(';')[1:]
 
                     for grp in groups:
-                        cmd, tid = grp.split(':', maxsplit=1)
+                        cmd, *tid = grp.split(':', maxsplit=1)
 
                         if cmd in ('c', f'C{SIGTRAP:02x}'):
                             return handle_c('')

qiling/extensions/pipe.py

@@ -2,52 +2,20 @@
 #
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 #
-
+import io
+import os
 from typing import TextIO
 
 from qiling.os.posix import stat
 
-class SimpleStringBuffer(TextIO):
+class SimpleStringBuffer(io.BytesIO):
     """Simple FIFO pipe.
     """
 
     def __init__(self):
-        self.buff = bytearray()
-
-    def read(self, n: int = -1) -> bytes:
-        if n == -1:
-            ret = self.buff
-            rem = bytearray()
-        else:
-            ret = self.buff[:n]
-            rem = self.buff[n:]
-
-        self.buff = rem
-
-        return bytes(ret)
-
-    def readline(self, limit: int = -1) -> bytes:
-        ret = bytearray()
-
-        while not (ret.endswith(b'\n') or len(ret) == limit):
-            ret.extend(self.read(1))
-
-        return bytes(ret)
-
-    def write(self, s: bytes) -> int:
-        self.buff.extend(s)
-
-        return len(s)
-
-    def flush(self) -> None:
-        pass
-
-    def writable(self) -> bool:
-        return True
-
-    def readable(self) -> bool:
-        return True
-
+        super().__init__()
+    
+    # Compatible with old implementation
     def seek(self, offset: int, origin: int = 0) -> int:
         # Imitate os.lseek
         raise OSError("Illega Seek")
@@ -55,9 +23,22 @@ class SimpleStringBuffer(TextIO):
     def seekable(self) -> bool:
         return False
 
+    def write(self, buf: bytes) -> int:
+        # For the FIFO stream, the write doesn't change pos.
+        pos = super().tell()
+        super().seek(0, os.SEEK_END)
+        ret = super().write(buf)
+        super().seek(pos)
+        return ret
+    
+    # Compatible with previous TextIO
+    @property
+    def name(self):
+        return None
+
 class SimpleStreamBase:
-    def __init__(self, fd: int, *args):
-        super().__init__(*args)
+    def __init__(self, fd: int):
+        super().__init__()
 
         self.__fd = fd
         self.__closed = False
@@ -65,6 +46,7 @@ class SimpleStreamBase:
     def close(self) -> None:
         self.__closed = True
     
+    @property
     def closed(self) -> bool:
         return self.__closed
 
@@ -99,72 +81,9 @@ class NullOutStream(SimpleStreamBase):
     def writable(self) -> bool:
         return True
 
-class SimpleBufferedStream(TextIO):
+class SimpleBufferedStream(io.BytesIO):
     """Simple buffered IO.
     """
 
     def __init__(self):
-        self.buff = bytearray()
-        self.cur = 0
-
-    def seek(self, offset: int, origin: int = 0) -> int:
-        if origin == 0: # SEEK_SET
-            base = 0
-        elif origin == 1: # SEEK_CUR
-            base = self.cur
-        else: # SEEK_END
-            base = len(self.buff)
-
-        if base + offset >= len(self.buff):
-            self.cur = len(self.buff)
-        elif base + offset <= 0:
-            self.cur = 0
-        else:
-            self.cur = base + offset
-
-        return self.cur
-    
-    def tell(self) -> int:
-        return self.cur
-
-    def read(self, n: int = -1) -> bytes:
-        if n == -1:
-            ret = self.buff[self.cur:]
-            self.cur = len(self.buff)
-        else:
-            ret = self.buff[self.cur:self.cur + n]
-
-            if self.cur + n >= len(self.buff):
-                self.cur = len(self.buff)
-            else:
-                self.cur = self.cur + n
-
-        return bytes(ret)
-
-    def readline(self, limit: int = -1) -> bytes:
-        ret = bytearray()
-
-        while not (ret.endswith(b'\n') or len(ret) == limit):
-            ret.extend(self.read(1))
-
-        return bytes(ret)
-
-    def write(self, s: bytes) -> int:
-        self.buff = self.buff[:self.cur]
-        self.buff.extend(s)
-
-        self.cur += len(s)
-
-        return len(s)
-
-    def flush(self) -> None:
-        pass
-
-    def writable(self) -> bool:
-        return True
-
-    def readable(self) -> bool:
-        return True
-
-    def seekable(self) -> bool:
-        return True
+        super.__init__()

qiling/extensions/r2/__init__.py

@@ -0,0 +1 @@
+from .r2 import R2

qiling/extensions/r2/r2.py

@@ -0,0 +1,305 @@
+#!/usr/bin/env python3
+#
+# Cross Platform and Multi Architecture Advanced Binary Emulation Framework
+#
+
+import ctypes
+import json
+import re
+import libr
+from dataclasses import dataclass, field, fields
+from functools import cached_property, wraps
+from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Pattern, Tuple, Union
+from qiling.const import QL_ARCH
+from qiling.extensions import trace
+from unicorn import UC_PROT_NONE, UC_PROT_READ, UC_PROT_WRITE, UC_PROT_EXEC, UC_PROT_ALL
+
+if TYPE_CHECKING:
+    from qiling.core import Qiling
+
+def perm2uc(permstr: str) -> int:
+    '''convert "-rwx" to unicorn const'''
+    perm = UC_PROT_NONE
+    dic = {
+        "r": UC_PROT_READ,
+        "w": UC_PROT_WRITE,
+        "x": UC_PROT_EXEC,
+    }
+    for ch in permstr:
+        perm += dic.get(ch, 0)
+    return perm
+
+
+class R2Data:
+    def __init__(self, **kwargs):
+        names = set([f.name for f in fields(self)])
+        for k, v in kwargs.items():
+            if k in names:
+                setattr(self, k, v)
+
+
+@dataclass(unsafe_hash=True, init=False)
+class Section(R2Data):
+    name: str
+    size: int
+    vsize: int
+    paddr: int
+    vaddr: int
+    perm: int
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.perm = perm2uc(self.perm)
+
+
+@dataclass(unsafe_hash=True, init=False)
+class String(R2Data):
+    string: str
+    vaddr: int
+    paddr: int
+    size: int
+    length: int
+    section: str = None
+
+
+@dataclass(unsafe_hash=True, init=False)
+class Symbol(R2Data):
+    # see https://github.com/rizinorg/rizin/blob/dev/librz/include/rz_bin.h
+    SymbolType = Literal["NOTYPE", "OBJ", "FUNC", "FIELD", "IFACE", "METH", "STATIC", "SECT",
+                         "FILE", "COMMON", "TLS", "NUM", "LOOS", "HIOS", "LOPROC", "HIPROC", "SPCL", "UNK"]
+
+    SymbolBind = Literal["LOCAL", "GLOBAL", "WEAK", "NUM", "LOOS", "HIOS", "LOPROC", "HIPROC", "IMPORT", "UNKNOWN"]
+
+    name: str
+    realname: str
+    bind: SymbolBind
+    size: int
+    type: SymbolType
+    vaddr: int
+    paddr: int
+    is_imported: bool
+
+
+@dataclass(unsafe_hash=True, init=False)
+class Instruction(R2Data):
+    offset: int
+    size: int
+    opcode: str  # raw opcode
+    disasm: str = ''  # flag resolved opcode
+    bytes: bytes
+    type: str
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.bytes = bytes.fromhex(kwargs["bytes"])
+
+
+@dataclass(unsafe_hash=True, init=False)
+class Function(R2Data):
+    name: str
+    offset: int
+    size: int
+    signature: str
+
+
+@dataclass(unsafe_hash=True, init=False)
+class Flag(R2Data):
+    offset: int  # should be addr but r2 calls it offset
+    name: str = ''
+    size: int = 0
+
+    def __lt__(self, other):
+        return self.offset < other.offset
+
+
+@dataclass(unsafe_hash=True, init=False)
+class Xref(R2Data):
+    XrefType = Literal["NULL", "CODE", "CALL", "DATA", "STRN", "UNKN"]
+
+    name: str
+    fromaddr: int  # from is reserved word in Python
+    type: XrefType
+    perm: int
+    addr: int
+    refname: str
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.fromaddr = kwargs["from"]
+        self.perm = perm2uc(self.perm)
+
+    def __lt__(self, other):
+        return self.fromaddr < other.fromaddr
+
+
+class R2:
+    def __init__(self, ql: "Qiling", baseaddr=(1 << 64) - 1, loadaddr=0):
+        super().__init__()
+        self.ql = ql
+        # r2 -B [baddr]   set base address for PIE binaries
+        self.baseaddr = baseaddr
+        self.loadaddr = loadaddr  # r2 -m [addr]    map file at given address
+        self.analyzed = False
+        self._r2c = libr.r_core.r_core_new()
+        if ql.code:
+            self._setup_code(ql.code)
+        else:
+            self._setup_file(ql.path)
+
+    def _qlarch2r(self, archtype: QL_ARCH) -> str:
+        return {
+            QL_ARCH.X86: "x86",
+            QL_ARCH.X8664: "x86",
+            QL_ARCH.ARM: "arm",
+            QL_ARCH.ARM64: "arm",
+            QL_ARCH.A8086: "x86",
+            QL_ARCH.EVM: "evm.cs",
+            QL_ARCH.CORTEX_M: "arm",
+            QL_ARCH.MIPS: "mips",
+            QL_ARCH.RISCV: "riscv",
+            QL_ARCH.RISCV64: "riscv",
+            QL_ARCH.PPC: "ppc",
+        }[archtype]
+
+    def _setup_code(self, code: bytes):
+        path = f'malloc://{len(code)}'.encode()
+        fh = libr.r_core.r_core_file_open(self._r2c, path, UC_PROT_ALL, self.loadaddr)
+        libr.r_core.r_core_bin_load(self._r2c, path, self.baseaddr)
+        self._cmd(f'wx {code.hex()}')
+        # set architecture and bits for r2 asm
+        arch = self._qlarch2r(self.ql.arch.type)
+        self._cmd(f"e,asm.arch={arch},asm.bits={self.ql.arch.bits}")
+
+    def _setup_file(self, path: str):
+        path = path.encode()
+        fh = libr.r_core.r_core_file_open(self._r2c, path, UC_PROT_READ | UC_PROT_EXEC, self.loadaddr)
+        libr.r_core.r_core_bin_load(self._r2c, path, self.baseaddr)
+
+    def _cmd(self, cmd: str) -> str:
+        r = libr.r_core.r_core_cmd_str(
+            self._r2c, ctypes.create_string_buffer(cmd.encode("utf-8")))
+        return ctypes.string_at(r).decode('utf-8')
+
+    def _cmdj(self, cmd: str) -> Union[Dict, List[Dict]]:
+        return json.loads(self._cmd(cmd))
+
+    def aaa(fun):
+        @wraps(fun)
+        def wrapper(self):
+            if self.analyzed is False:
+                self._cmd("aaa")
+                self.analyzed = True
+            return fun(self)
+        return wrapper
+
+    @cached_property
+    def binfo(self) -> Dict[str, str]:
+        return self._cmdj("iIj")
+
+    @cached_property
+    def baddr(self) -> int:
+        return self.binfo["baddr"]
+
+    @cached_property
+    def bintype(self) -> str:
+        return self.binfo["bintype"]
+
+    @cached_property
+    def sections(self) -> Dict[str, Section]:
+        sec_lst = self._cmdj("iSj")
+        return {dic['name']: Section(**dic) for dic in sec_lst}
+
+    @cached_property
+    def strings(self) -> Dict[str, String]:
+        str_lst = self._cmdj("izzj")
+        return {dic['string']: String(**dic) for dic in str_lst}
+
+    @cached_property
+    def symbols(self) -> Dict[str, Symbol]:
+        sym_lst = self._cmdj("isj")
+        return {dic['name']: Symbol(**dic).vaddr for dic in sym_lst}
+
+    @cached_property
+    @aaa
+    def functions(self) -> Dict[str, Function]:
+        fcn_lst = self._cmdj("aflj")
+        return {dic['name']: Function(**dic) for dic in fcn_lst}
+
+    @cached_property
+    @aaa
+    def flags(self) -> List[Flag]:
+        return [Flag(**dic) for dic in self._cmdj("fj")]
+
+    @cached_property
+    @aaa
+    def xrefs(self) -> List[Xref]:
+        return [Xref(**dic) for dic in self._cmdj("axj")]
+
+    def at(self, addr: int, parse=False) -> Union[str, Tuple[str, int]]:
+        '''Given an address, return [name, offset] or "name + offset"'''
+        name = self._cmd(f'fd {addr}').strip()
+        if parse:
+            try:
+                name, offset = name.split(' + ')
+                offset = int(offset)
+            except ValueError:  # split fail when offset=0
+                offset = 0
+            return name, offset
+        return name
+
+    def where(self, name: str, offset: int=0) -> int:
+        '''Given a name (+ offset), return its address (0 when not found)'''
+        if offset != 0:  # name can already have offset, multiple + is allowd
+            name += f' + {offset}'
+        addr = self._cmd(f'?v {name}').strip()  # 0x0 when name is not found
+        return int(addr, 16)
+
+    def refrom(self, addr: int) -> List[Xref]:
+        return [x for x in self.xrefs if x.fromaddr == addr]
+
+    def refto(self, addr: int) -> List[Xref]:
+        return [x for x in self.xrefs if x.addr == addr]
+
+    def read(self, addr: int, size: int) -> bytes:
+        hexstr = self._cmd(f"p8 {size} @ {addr}")
+        return bytes.fromhex(hexstr)
+
+    def dis_nbytes(self, addr: int, size: int) -> List[Instruction]:
+        insts = [Instruction(**dic) for dic in self._cmdj(f"pDj {size} @ {addr}")]
+        return insts
+
+    def disassembler(self, ql: 'Qiling', addr: int, size: int, filt: Pattern[str]=None) -> int:
+        '''A human-friendly monkey patch of QlArchUtils.disassembler powered by r2, can be used for hook_code
+            :param ql: Qiling instance
+            :param addr: start address for disassembly
+            :param size: size in bytes
+            :param filt: regex pattern to filter instructions
+            :return: progress of dissembler, should be equal to size if success
+        '''
+        anibbles = ql.arch.bits // 4
+        progress = 0
+        for inst in self.dis_nbytes(addr, size):
+            if inst.type.lower() == 'invalid':
+                break  # stop disasm
+            name, offset = self.at(inst.offset, parse=True)
+            if filt is None or filt.search(name):
+                ql.log.info(f'{inst.offset:0{anibbles}x} [{name:20s} + {offset:#08x}] {inst.bytes.hex(" "):20s} {inst.disasm}')
+            progress = inst.offset + inst.size - addr
+        if progress < size:
+            ql.arch.utils.disassembler(ql, addr + progress, size - progress)
+        return progress
+
+    def enable_disasm(self, filt_str: str=''):
+        filt = re.compile(filt_str)
+        self.ql.hook_code(self.disassembler, filt)
+
+    def enable_trace(self, mode='full'):
+        # simple map from addr to flag name, cannot resolve addresses in the middle
+        self.ql.loader.symsmap = {flag.offset: flag.name for flag in self.flags}
+        if mode == 'full':
+            trace.enable_full_trace(self.ql)
+        elif mode == 'history':
+            trace.enable_history_trace(self.ql)
+
+    def __del__(self):
+        libr.r_core.r_core_free(self._r2c)

qiling/extensions/sanitizers/heap.py

@@ -6,6 +6,8 @@
 import random
 from enum import Enum
 
+from qiling import Qiling
+
 class CaneryType(Enum):
         underflow = 0
         overflow = 1
@@ -22,7 +24,9 @@ class QlSanitizedMemoryHeap():
     ql.os.heap.uaf_handler = my_uaf_handler
     """
 
-    def __init__(self, ql, heap, fault_rate=0, canary_byte=b'\xCD'):
+    CANARY_SIZE = 4
+
+    def __init__(self, ql: Qiling, heap, fault_rate=0, canary_byte=b'\xCD'):
         self.ql = ql
         self.heap = heap
         self.fault_rate = fault_rate
@@ -77,34 +81,39 @@ class QlSanitizedMemoryHeap():
         """
         pass
 
-    def alloc(self, size):
+    def alloc(self, size: int):
         chance = random.randint(1, 100)
         if chance <= self.fault_rate:
             # Fail the allocation.
             return 0
 
-        # Add 8 bytes to the requested size so as to accomodate the canaries.
-        addr = self.heap.alloc(size + 8)
-        self.ql.mem.write(addr, self.canary_byte * (size + 8))
+        addr = self.heap.alloc(size + self.CANARY_SIZE * 2)
 
-        # Install canary hooks for overflow/underflow detection.
-        underflow_canary = (addr, addr + 3, CaneryType.underflow)
-        self.ql.hook_mem_write(self.bo_handler, begin=underflow_canary[0], end=underflow_canary[1])
-        self.ql.hook_mem_read(self.oob_handler, begin=underflow_canary[0], end=underflow_canary[1])
-        self.canaries.append(underflow_canary)
+        canary_begins = addr
+        canary_ends = canary_begins + self.CANARY_SIZE - 1
 
-        overflow_canary = (addr + 4 + size, addr + 4 + size + 3, CaneryType.overflow)
-        self.ql.hook_mem_write(self.bo_handler, begin=overflow_canary[0], end=overflow_canary[1])
-        self.ql.hook_mem_read(self.oob_handler, begin=overflow_canary[0], end=overflow_canary[1])
-        self.canaries.append(overflow_canary)
+        # install underflow canary and detection hooks
+        self.ql.mem.write(canary_begins, self.canary_byte * self.CANARY_SIZE)
+        self.ql.hook_mem_write(self.bo_handler, begin=canary_begins, end=canary_ends)
+        self.ql.hook_mem_read(self.oob_handler, begin=canary_begins, end=canary_ends)
+        self.canaries.append((canary_begins, canary_ends, CaneryType.underflow))
 
-        return (addr + 4)
+        canary_begins = addr + self.CANARY_SIZE + size
+        canary_ends = canary_begins + self.CANARY_SIZE - 1
 
-    def size(self, addr):
-        return self.heap.size(addr - 4)
+        # install overflow canary and detection hooks
+        self.ql.mem.write(canary_begins, self.canary_byte * self.CANARY_SIZE)
+        self.ql.hook_mem_write(self.bo_handler, begin=canary_begins, end=canary_ends)
+        self.ql.hook_mem_read(self.oob_handler, begin=canary_begins, end=canary_ends)
+        self.canaries.append((canary_begins, canary_ends, CaneryType.overflow))
 
-    def free(self, addr):
-        chunk = self.heap._find(addr - 4)
+        return (addr + self.CANARY_SIZE)
+
+    def size(self, addr: int):
+        return self.heap.size(addr - self.CANARY_SIZE)
+
+    def free(self, addr: int) -> bool:
+        chunk = self.heap._find(addr - self.CANARY_SIZE)
 
         if not chunk:
             self.bad_free_handler(self.ql, addr)
@@ -118,13 +127,15 @@ class QlSanitizedMemoryHeap():
 
         # Make sure the chunk won't be re-used by the underlying heap.
         self.heap.chunks.remove(chunk)
+
         return True
 
-    def validate(self):
-        for (canary_begin, canary_end, canery_type) in self.canaries:
+    def validate(self) -> bool:
+        for canary_begin, canary_end, _ in self.canaries:
             size = canary_end - canary_begin + 1
             canary = self.ql.mem.read(canary_begin, size)
+
             if canary.count(self.canary_byte) != len(canary):
                 return False
+
         return True
-        

qiling/extensions/trace.py

@@ -97,7 +97,7 @@ def __to_trace_line(record: TraceRecord, symsmap: Mapping[int, str] = {}) -> str
 		"""
 
 		if op.type == CS_OP_REG:
-			return insn.reg_name(op.value.reg)
+			return insn.reg_name(op.value.reg) or '?'
 
 		elif op.type == CS_OP_IMM:
 			imm = op.value.imm
@@ -113,6 +113,7 @@ def __to_trace_line(record: TraceRecord, symsmap: Mapping[int, str] = {}) -> str
 			disp  = mem.disp
 
 			ea = base + index * scale + disp
+			seg = f'{insn.reg_name(mem.segment)}:' if mem.segment else ''
 
 			# we construct the string representation for each operand; denote memory
 			# dereferenes with the appropriate 'ptr' prefix. the 'lea' instruction is
@@ -131,7 +132,7 @@ def __to_trace_line(record: TraceRecord, symsmap: Mapping[int, str] = {}) -> str
 
 				qualifier = f'{ptr} ptr '
 
-			return f'{qualifier}[{__resolve(ea) or f"{ea:#x}"}]'
+			return f'{qualifier}{seg}[{__resolve(ea) or f"{ea:#x}"}]'
 
 		# unexpected op type
 		raise RuntimeError

qiling/extensions/tracing/README.md

@@ -0,0 +1 @@
+For latest documentation, please visit https://docs.qiling.io

qiling/extensions/tracing/formats/__init__.py

@@ -0,0 +1,2 @@
+# This code structure is copied and modified from the coverage extension
+__all__ = ["base", "tenet"]

qiling/extensions/tracing/formats/base.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+# 
+# Cross Platform and Multi Architecture Advanced Binary Emulation Framework
+# This code structure is copied and modified from the coverage extension
+
+from abc import ABC, abstractmethod
+
+
+class QlBaseTrace(ABC):
+    """
+    An abstract base class for trace collectors.
+    To add support for a new coverage format, just derive from this class and implement
+    all the methods marked with the @abstractmethod decorator.
+    """
+    
+    def __init__(self):
+        super().__init__()
+
+    @property
+    @staticmethod
+    @abstractmethod
+    def FORMAT_NAME():
+        raise NotImplementedError
+
+    @abstractmethod
+    def activate(self):
+        pass
+
+    @abstractmethod
+    def deactivate(self):
+        pass
+
+    @abstractmethod
+    def dump_trace(self, trace_file):
+        pass

qiling/extensions/tracing/formats/registers.py

@@ -0,0 +1,111 @@
+import unicorn
+from qiling.arch import arm64, arm, x86
+
+
+
+class ArchRegs():
+    """
+    Stores architecture's registers relevant to tracing in a format
+    universally usable by different tracers
+    """
+
+    arm_registers = {
+        '$r0': unicorn.arm_const.UC_ARM_REG_R0,
+        '$r1': unicorn.arm_const.UC_ARM_REG_R1,
+        '$r2': unicorn.arm_const.UC_ARM_REG_R2,
+        '$r3': unicorn.arm_const.UC_ARM_REG_R3,
+        '$r4': unicorn.arm_const.UC_ARM_REG_R4,
+        '$r5': unicorn.arm_const.UC_ARM_REG_R5,
+        '$r6': unicorn.arm_const.UC_ARM_REG_R6,
+        '$r7': unicorn.arm_const.UC_ARM_REG_R7,
+        '$r8': unicorn.arm_const.UC_ARM_REG_R8,
+        '$r9': unicorn.arm_const.UC_ARM_REG_R9,
+        '$r10': unicorn.arm_const.UC_ARM_REG_R10,
+        '$r11': unicorn.arm_const.UC_ARM_REG_R11,
+        '$r12': unicorn.arm_const.UC_ARM_REG_R12,
+        '$sp': unicorn.arm_const.UC_ARM_REG_SP,
+        '$lr': unicorn.arm_const.UC_ARM_REG_LR,
+        '$pc': unicorn.arm_const.UC_ARM_REG_PC,
+        '$cpsr': unicorn.arm_const.UC_ARM_REG_CPSR
+    }
+    arm64_registers = {
+        "$pc": unicorn.arm64_const.UC_ARM64_REG_PC,
+        "$sp": unicorn.arm64_const.UC_ARM64_REG_SP,
+        "$x0": unicorn.arm64_const.UC_ARM64_REG_X0,
+        "$x1": unicorn.arm64_const.UC_ARM64_REG_X1,
+        "$x2": unicorn.arm64_const.UC_ARM64_REG_X2,
+        "$x3": unicorn.arm64_const.UC_ARM64_REG_X3,
+        "$x4": unicorn.arm64_const.UC_ARM64_REG_X4,
+        "$x5": unicorn.arm64_const.UC_ARM64_REG_X5,
+        "$x6": unicorn.arm64_const.UC_ARM64_REG_X6,
+        "$x7": unicorn.arm64_const.UC_ARM64_REG_X7,
+        "$x8": unicorn.arm64_const.UC_ARM64_REG_X8,
+        "$x9": unicorn.arm64_const.UC_ARM64_REG_X9,
+        "$x10": unicorn.arm64_const.UC_ARM64_REG_X10,
+        "$x11": unicorn.arm64_const.UC_ARM64_REG_X11,
+        "$x12": unicorn.arm64_const.UC_ARM64_REG_X12,
+        "$x13": unicorn.arm64_const.UC_ARM64_REG_X13,
+        "$x14": unicorn.arm64_const.UC_ARM64_REG_X14,
+        "$x15": unicorn.arm64_const.UC_ARM64_REG_X15,
+        "$x16": unicorn.arm64_const.UC_ARM64_REG_X16,
+        "$x17": unicorn.arm64_const.UC_ARM64_REG_X17,
+        "$x18": unicorn.arm64_const.UC_ARM64_REG_X18,
+        "$x19": unicorn.arm64_const.UC_ARM64_REG_X19,
+        "$x20": unicorn.arm64_const.UC_ARM64_REG_X20,
+        "$x21": unicorn.arm64_const.UC_ARM64_REG_X21,
+        "$x22": unicorn.arm64_const.UC_ARM64_REG_X22,
+        "$x23": unicorn.arm64_const.UC_ARM64_REG_X23,
+        "$x24": unicorn.arm64_const.UC_ARM64_REG_X24,
+        "$x25": unicorn.arm64_const.UC_ARM64_REG_X25,
+        "$x26": unicorn.arm64_const.UC_ARM64_REG_X26,
+        "$x27": unicorn.arm64_const.UC_ARM64_REG_X27,
+        "$x28": unicorn.arm64_const.UC_ARM64_REG_X28,
+        "$x29": unicorn.arm64_const.UC_ARM64_REG_X29,
+    }
+
+    x86_registers = {
+        '$eax': unicorn.x86_const.UC_X86_REG_EAX,
+        '$ebx': unicorn.x86_const.UC_X86_REG_EBX,
+        '$ecx': unicorn.x86_const.UC_X86_REG_ECX,
+        '$edx': unicorn.x86_const.UC_X86_REG_ECX,
+        '$ebp': unicorn.x86_const.UC_X86_REG_EBP,
+        '$esp': unicorn.x86_const.UC_X86_REG_ESP,
+        '$esi': unicorn.x86_const.UC_X86_REG_ESI,
+        '$edi': unicorn.x86_const.UC_X86_REG_EDI,
+        '$eip': unicorn.x86_const.UC_X86_REG_EIP,
+    }
+    x86_64_registers = {
+        '$rax': unicorn.x86_const.UC_X86_REG_RAX ,
+        '$rbx': unicorn.x86_const.UC_X86_REG_RBX,
+        '$rcx': unicorn.x86_const.UC_X86_REG_RCX,
+        '$rdx': unicorn.x86_const.UC_X86_REG_RCX,
+        '$rbp': unicorn.x86_const.UC_X86_REG_RBP,
+        '$rsp': unicorn.x86_const.UC_X86_REG_RSP,
+        '$rsi': unicorn.x86_const.UC_X86_REG_RSI,
+        '$rdi': unicorn.x86_const.UC_X86_REG_RDI,
+        '$rip': unicorn.x86_const.UC_X86_REG_RIP,
+        '$r8': unicorn.x86_const.UC_X86_REG_R8,
+        '$r9': unicorn.x86_const.UC_X86_REG_R9,
+        '$r10': unicorn.x86_const.UC_X86_REG_R10,
+        '$r11': unicorn.x86_const.UC_X86_REG_R11,
+        '$r12': unicorn.x86_const.UC_X86_REG_R12,
+        '$r13': unicorn.x86_const.UC_X86_REG_R13,
+        '$r14': unicorn.x86_const.UC_X86_REG_R14,
+        '$r15': unicorn.x86_const.UC_X86_REG_R15,
+    }
+
+    def __init__(self, arch):
+        if isinstance(arch, arm.QlArchARM):
+            self.registers = self.arm_registers
+            self.pc_key = "$pc"
+        elif isinstance(arch,arm64.QlArchARM64):
+            self.registers = self.arm64_registers
+            self.pc_key = "$pc"
+        elif isinstance(arch,x86.QlArchX86):
+            self.registers = self.x86_registers
+            self.pc_key = "$eip"
+        elif isinstance(arch,x86.QlArchX8664):
+            self.registers = self.x86_64_registers
+            self.pc_key = "$rip"
+        else:
+            raise("Unsupported arch")

qiling/extensions/tracing/formats/tenet.py

@@ -0,0 +1,103 @@
+# This code structure is copied and modified from the coverage extension
+
+import qiling
+from qiling.const import QL_ENDIAN
+from .base import QlBaseTrace
+from .registers import ArchRegs
+from unicorn.unicorn_const import UC_MEM_READ, UC_MEM_WRITE
+
+class QlDrTrace(QlBaseTrace):
+    """
+    Traces emulation and puts it into a format viewable in Tenet
+    IDAPro plugin Tenet: https://github.com/gaasedelen/tenet
+    """
+
+    FORMAT_NAME = "tenet"
+
+    def __init__(self, ql: qiling.Qiling):
+        super().__init__()
+        self.ql             = ql
+        self.deltas         = []
+        self.current_delta  = []
+        self.current_pc     = 0x0
+
+        self.arch_regs = ArchRegs(ql.arch)
+        self.register_values= dict()
+
+        # Initialize with ridiculous value so first delta isn't missed
+        for register in self.arch_regs.registers:
+            self.register_values[register] = 0xFEEDBABE
+
+
+    def _add_delta(self):
+        # Cover glitch cases where nothing changed
+        if self.current_delta != []:
+            # Join all delta fragments into delta line and append 
+            self.deltas.append(",".join(self.current_delta))
+            self.current_delta = []
+        return
+
+
+    @staticmethod
+    def mem_access_callback(ql, access, address, size, value, self):
+        access_type = None
+        # Set delta based on access type
+        if access == UC_MEM_READ:
+            # Since we are reading memory, we just read it ourselves
+            access_type = "mr"
+            value = ql.mem.read(address, size)
+            delta = f"{access_type}={hex(address)}:{value.hex()}"
+        elif access == UC_MEM_WRITE:
+            # Hook is before it's written, so we have to use the "value"
+            access_type = "mw"
+            if ql.arch.endian == QL_ENDIAN.EL:
+                endian = 'little'
+            else:
+                endian = 'big'
+            if value < 0:
+                sign = True
+            else:
+                sign = False
+            value = int.to_bytes(value, size, endian, signed=sign)
+            delta = f"{access_type}={hex(address)}:{value.hex()}"
+        else:
+            print("Invalid access type")
+            return
+        # <ACCESS_TYPE>=<ACCESS_ADDRESS>:<HEX_BYTE_0><HEX_BYTE_1><HEX_BYTE_2>... 
+        self.current_delta.append(delta)
+        return
+
+    @staticmethod
+    def code_callback(ql, address, size, self):
+        # Check if PC changed for next delta
+        pc = ql.arch.regs.read(self.arch_regs.registers[self.arch_regs.pc_key])
+        if pc != self.current_pc:
+            self._add_delta()
+            self.current_pc = pc
+        # Go through each register and see if it changed
+        for register in self.arch_regs.registers:
+            value = ql.arch.regs.read(self.arch_regs.registers[register])
+            if value != self.register_values[register]:
+                # <REG_NAME>=<REG_VALUE_AS_BASE_16>
+                delta = f"{register[1::]}={hex(value)}"
+                self.current_delta.append(delta)
+                # Update value
+                self.register_values[register] = value
+        
+        return
+        
+    def activate(self):
+        self.code_callback = self.ql.hook_code(self.code_callback, user_data=self)
+        self.mem_write_callback = self.ql.hook_mem_read(self.mem_access_callback, user_data=self)
+        self.mem_read_callback = self.ql.hook_mem_write(self.mem_access_callback, user_data=self)
+
+    def deactivate(self):
+        self.ql.hook_del(self.code_callback)
+        self.ql.hook_del(self.mem_write_callback)
+        self.ql.hook_del(self.mem_read_callback)
+    
+    def dump_trace(self, trace_file: str):
+        with open(trace_file, "w") as trace:
+            # Write out each delta on a separate line
+            for delta in self.deltas:
+                trace.write(delta + "\n")

qiling/extensions/tracing/utils.py

@@ -0,0 +1,46 @@
+# This code structure is copied and modified from the coverage extension
+
+from contextlib import contextmanager
+from .formats import *
+
+
+# Returns subclasses recursively.
+def get_all_subclasses(cls):
+    all_subclasses = []
+
+    for subclass in cls.__subclasses__():
+        all_subclasses.append(subclass)
+        all_subclasses.extend(get_all_subclasses(subclass))
+
+    return all_subclasses
+
+class TraceFactory():
+    def __init__(self):
+        self.trace_collectors = {subcls.FORMAT_NAME:subcls for subcls in get_all_subclasses(base.QlBaseTrace)}
+
+    @property
+    def formats(self):
+        return self.trace_collectors.keys()
+
+    def get_trace_collector(self, ql, name):
+        return self.trace_collectors[name](ql)
+
+factory = TraceFactory()
+
+@contextmanager
+def collect_trace(ql, name: str, trace_file: str):
+    """
+    Context manager for emulating a given piece of code with tracing.
+    Example:
+    with collect_trace(ql, 'tenet', 'trace.0.log'):
+        ql.run(...)
+    """
+
+    trace = factory.get_trace_collector(ql, name)
+    trace.activate()
+    try:
+        yield
+    finally:
+        trace.deactivate()
+        if trace_file:
+            trace.dump_trace(trace_file)

qiling/loader/elf.py

@@ -15,7 +15,6 @@ from elftools.elf.elffile import ELFFile
 from elftools.elf.relocation import RelocationHandler
 from elftools.elf.sections import Symbol, SymbolTableSection
 from elftools.elf.descriptions import describe_reloc_type
-from elftools.elf.segments import InterpSegment
 from unicorn.unicorn_const import UC_PROT_NONE, UC_PROT_READ, UC_PROT_WRITE, UC_PROT_EXEC
 
 from qiling import Qiling
@@ -100,7 +99,7 @@ class QlLoaderELF(QlLoader):
 
         # is it a driver?
         if elftype == 'ET_REL':
-            self.load_driver(elffile, stack_address + stack_size)
+            self.load_driver(elffile, stack_address + stack_size, loadbase=0x8000000)
             self.ql.hook_code(hook_kernel_api)
 
         # is it an executable?
@@ -271,7 +270,7 @@ class QlLoaderELF(QlLoader):
             Top of stack remains aligned to pointer size
             """
 
-            data = (s if isinstance(s, bytes) else s.encode("utf-8")) + b'\x00'
+            data = s.encode('utf-8') + b'\x00'
             top = self.ql.mem.align(top - len(data), self.ql.arch.pointersize)
             self.ql.mem.write(top, data)
 
@@ -365,28 +364,31 @@ class QlLoaderELF(QlLoader):
 
         # map vsyscall section for some specific needs
         if self.ql.arch.type == QL_ARCH.X8664 and self.ql.os.type == QL_OS.LINUX:
-            _vsyscall_addr = int(self.profile.get('vsyscall_address'), 0)
-            _vsyscall_size = int(self.profile.get('vsyscall_size'), 0)
+            vsyscall_addr = self.profile.getint('vsyscall_address')
 
-            if self.ql.mem.is_available(_vsyscall_addr, _vsyscall_size):
+            vsyscall_ids = (
+                SYSCALL_NR.gettimeofday,
+                SYSCALL_NR.time,
+                SYSCALL_NR.getcpu
+            )
+
+            # each syscall should be 1KiB away
+            entry_size = 1024
+            vsyscall_size = self.ql.mem.align_up(len(vsyscall_ids) * entry_size)
+
+            if self.ql.mem.is_available(vsyscall_addr, vsyscall_size):
                 # initialize with int3 instructions then insert syscall entry
-                # each syscall should be 1KiB away
-                self.ql.mem.map(_vsyscall_addr, _vsyscall_size, info="[vsyscall]")
-                self.ql.mem.write(_vsyscall_addr, _vsyscall_size * b'\xcc')
+                self.ql.mem.map(vsyscall_addr, vsyscall_size, info="[vsyscall]")
                 assembler = self.ql.arch.assembler
 
                 def __assemble(asm: str) -> bytes:
                     bs, _ = assembler.asm(asm)
                     return bytes(bs)
 
-                _vsyscall_ids = (
-                    SYSCALL_NR.gettimeofday,
-                    SYSCALL_NR.time,
-                    SYSCALL_NR.getcpu
-                )
+                for i, scid in enumerate(vsyscall_ids):
+                    entry = __assemble(f'mov rax, {scid:#x}; syscall; ret')
 
-                for i, scid in enumerate(_vsyscall_ids):
-                    self.ql.mem.write(_vsyscall_addr + i * 1024, __assemble(f'mov rax, {scid:#x}; syscall; ret'))
+                    self.ql.mem.write(vsyscall_addr + i * entry_size, entry.ljust(entry_size, b'\xcc'))
 
     def lkm_get_init(self, elffile: ELFFile) -> int:
         """Get file offset of the init_module function.
@@ -400,7 +402,7 @@ class QlLoaderELF(QlLoader):
             if syms:
                 sym = syms[0]
                 addr = sym['st_value'] + elffile.get_section(sym['st_shndx'])['sh_offset']
-                self.ql.log.info(f'init_module = {addr:#x}')
+
                 return addr
 
         raise QlErrorELFFormat('invalid module: symbol init_module not found')
@@ -479,6 +481,10 @@ class QlLoaderELF(QlLoader):
                                 rev_reloc_symbols[symbol_name] = self.ql.os.hook_addr
                                 sym_offset = self.ql.os.hook_addr - mem_start
                                 self.ql.os.hook_addr += self.ql.arch.pointersize
+
+                            elif _symbol['st_shndx'] == 'SHN_ABS':
+                                rev_reloc_symbols[symbol_name] = _symbol['st_value']
+
                             else:
                                 # local symbol
                                 _section = elffile.get_section(_symbol['st_shndx'])
@@ -521,12 +527,12 @@ class QlLoaderELF(QlLoader):
 
                     elif desc in ('R_386_PC32', 'R_386_PLT32'):
                         val = ql.mem.read_ptr(loc, 4)
-                        val = rev_reloc_symbols[symbol_name] + val - loc
+                        val += rev_reloc_symbols[symbol_name] - loc
                         ql.mem.write_ptr(loc, (val & 0xFFFFFFFF), 4)
 
                     elif desc in ('R_386_32', 'R_MIPS_32'):
                         val = ql.mem.read_ptr(loc, 4)
-                        val = rev_reloc_symbols[symbol_name] + val
+                        val += rev_reloc_symbols[symbol_name]
                         ql.mem.write_ptr(loc, (val & 0xFFFFFFFF), 4)
 
                     elif desc == 'R_MIPS_HI16':
@@ -551,23 +557,9 @@ class QlLoaderELF(QlLoader):
     def load_driver(self, elffile: ELFFile, stack_addr: int, loadbase: int = 0) -> None:
         elfdata_mapping = self.get_elfdata_mapping(elffile)
 
-        # Determine the range of memory space opened up
-        # mem_start = -1
-        # mem_end = -1
-        #
-        # for i in super().parse_program_header(ql):
-        #     if i['p_type'] == PT_LOAD:
-        #         if mem_start > i['p_vaddr'] or mem_start == -1:
-        #             mem_start = i['p_vaddr']
-        #         if mem_end < i['p_vaddr'] + i['p_memsz'] or mem_end == -1:
-        #             mem_end = i['p_vaddr'] + i['p_memsz']
-        #
-        # mem_start = int(mem_start // 0x1000) * 0x1000
-        # mem_end = int(mem_end // 0x1000 + 1) * 0x1000
-
-        # FIXME
-        mem_start = 0x1000
-        mem_end = mem_start + (len(elfdata_mapping) // 0x1000 + 1) * 0x1000
+        # FIXME: determine true memory boundaries, taking relocation into account (if requested)
+        mem_start = 0
+        mem_end = mem_start + self.ql.mem.align_up(len(elfdata_mapping), 0x1000)
 
         # map some memory to intercept external functions of Linux kernel
         self.ql.mem.map(API_HOOK_MEM, 0x1000, info="[api_mem]")
@@ -579,15 +571,17 @@ class QlLoaderELF(QlLoader):
         self.ql.mem.map(loadbase + mem_start, mem_end - mem_start, info=self.ql.path)
         self.ql.mem.write(loadbase + mem_start, elfdata_mapping)
 
-        entry_point = self.lkm_get_init(elffile) + loadbase + mem_start
+        init_module = self.lkm_get_init(elffile) + loadbase + mem_start
+        self.ql.log.debug(f'init_module : {init_module:#x}')
+
         self.brk_address = mem_end + loadbase
 
         # Set MMAP addr
-        mmap_address = int(self.profile.get('mmap_address'), 0)
+        mmap_address = self.profile.getint('mmap_address')
         self.ql.log.debug(f'mmap_address is : {mmap_address:#x}')
 
         # self.ql.os.elf_entry = self.elf_entry = loadbase + elfhead['e_entry']
-        self.ql.os.entry_point = self.entry_point = entry_point
+        self.ql.os.entry_point = self.entry_point = init_module
         self.elf_entry = self.ql.os.elf_entry = self.ql.os.entry_point
 
         self.stack_address = self.ql.mem.align(stack_addr, self.ql.arch.pointersize)
@@ -625,6 +619,25 @@ class QlLoaderELF(QlLoader):
         self.import_symbols[self.ql.os.hook_addr + 2 * self.ql.arch.pointersize] = hook_sys_open
 
     def get_elfdata_mapping(self, elffile: ELFFile) -> bytes:
+        # from io import BytesIO
+        #
+        # rh = RelocationHandler(elffile)
+        #
+        # for sec in elffile.iter_sections():
+        #     rs = rh.find_relocations_for_section(sec)
+        #
+        #     if rs is not None:
+        #         ss = BytesIO(sec.data())
+        #         rh.apply_section_relocations(ss, rs)
+        #
+        #         # apply changes to stream
+        #         elffile.stream.seek(sec['sh_offset'])
+        #         elffile.stream.write(ss.getbuffer())
+        #
+        # TODO: need to patch hooked symbols with their hook targets
+        # (e.g. replace calls to 'printk' with the hooked address that
+        # was allocate for it)
+
         elfdata_mapping = bytearray()
 
         # pick up elf header
@@ -634,7 +647,15 @@ class QlLoaderELF(QlLoader):
 
         elfdata_mapping.extend(elf_header)
 
-        # pick up loadable sections and relocate them if needed
+        # FIXME: normally the address of a section would be determined by its 'sh_addr' value.
+        # in case of a relocatable object all its sections' sh_addr will be set to zero, so
+        # the value in 'sh_offset' should be used to determine the final address.
+        # see: https://refspecs.linuxbase.org/elf/gabi4+/ch4.sheader.html
+        #
+        # here we presume this a relocatable object and don't do any relocation (that is, it
+        # is relocated to 0)
+
+        # pick up loadable sections
         for sec in elffile.iter_sections():
             if sec['sh_flags'] & SH_FLAGS.SHF_ALLOC:
                 # pad aggregated elf data to the offset of the current section 

qiling/loader/pe.py

@@ -248,7 +248,12 @@ class Process:
             # in case of a dll loaded from a hooked API call, failures would not be
             # recoverable and we have to give up its DllMain.
             if not self.ql.os.PE_RUN:
+
+                # temporarily set PE_RUN to allow proper fcall unwinding during
+                # execution of DllMain
+                self.ql.os.PE_RUN = True
                 self.call_dll_entrypoint(dll, dll_base, dll_len, dll_name)
+                self.ql.os.PE_RUN = False
 
         self.ql.log.info(f'Done loading {dll_name}')
 
@@ -330,68 +335,66 @@ class Process:
         return {"name": name, "address": address}
 
     def init_teb(self):
-        teb_obj = make_teb(self.ql.arch.bits)
+        teb_struct = make_teb(self.ql.arch.bits)
 
         teb_addr = self.structure_last_addr
-        peb_addr = self.ql.mem.align_up(teb_addr + ctypes.sizeof(teb_obj), 0x10)
+        peb_addr = self.ql.mem.align_up(teb_addr + teb_struct.sizeof(), 0x10)
 
+        teb_obj = teb_struct.volatile_ref(self.ql.mem, teb_addr)
         teb_obj.StackBase  = self.stack_address + self.stack_size
         teb_obj.StackLimit = self.stack_address
         teb_obj.TebAddress = teb_addr
         teb_obj.PebAddress = peb_addr
 
         self.ql.log.info(f'TEB is at {teb_addr:#x}')
-        self.ql.mem.write(teb_addr, bytes(teb_obj))
 
         self.structure_last_addr = peb_addr
         self.TEB = teb_obj
 
     def init_peb(self):
-        peb_obj = make_peb(self.ql.arch.bits)
+        peb_struct = make_peb(self.ql.arch.bits)
 
         peb_addr = self.structure_last_addr
-        ldr_addr = self.ql.mem.align_up(peb_addr + ctypes.sizeof(peb_obj), 0x10)
+        ldr_addr = self.ql.mem.align_up(peb_addr + peb_struct.sizeof(), 0x10)
 
-        # we must set an heap, will try to retrieve this value. Is ok to be all \x00
+        # we must set a heap, will try to retrieve this value. Is ok to be all \x00
+        peb_obj = peb_struct.volatile_ref(self.ql.mem, peb_addr)
+        peb_obj.ImageBaseAddress   = self.pe_image_address
         peb_obj.LdrAddress         = ldr_addr
         peb_obj.ProcessParameters  = self.ql.os.heap.alloc(0x100)
         peb_obj.ProcessHeap        = self.ql.os.heap.alloc(0x100)
         peb_obj.NumberOfProcessors = self.ql.os.profile.getint('HARDWARE', 'number_processors')
 
         self.ql.log.info(f'PEB is at {peb_addr:#x}')
-        self.ql.mem.write(peb_addr, bytes(peb_obj))
 
         self.structure_last_addr = ldr_addr
         self.PEB = peb_obj
 
     def init_ldr_data(self):
-        ldr_obj = make_ldr_data(self.ql.arch.bits)
-        ldr_cls = ldr_obj.__class__
+        ldr_struct = make_ldr_data(self.ql.arch.bits)
 
         ldr_addr = self.structure_last_addr
-        nobj_addr = self.ql.mem.align_up(ldr_addr + ctypes.sizeof(ldr_obj), 0x10)
+        nobj_addr = self.ql.mem.align_up(ldr_addr + ldr_struct.sizeof(), 0x10)
 
-        ldr_obj.InLoadOrderModuleList.Flink = ldr_addr + ldr_cls.InLoadOrderModuleList.offset
-        ldr_obj.InLoadOrderModuleList.Blink = ldr_addr + ldr_cls.InLoadOrderModuleList.offset
+        ldr_obj = ldr_struct.volatile_ref(self.ql.mem, ldr_addr)
+        ldr_obj.InLoadOrderModuleList.Flink = ldr_addr + ldr_struct.InLoadOrderModuleList.offset
+        ldr_obj.InLoadOrderModuleList.Blink = ldr_addr + ldr_struct.InLoadOrderModuleList.offset
 
-        ldr_obj.InMemoryOrderModuleList.Flink = ldr_addr + ldr_cls.InMemoryOrderModuleList.offset
-        ldr_obj.InMemoryOrderModuleList.Blink = ldr_addr + ldr_cls.InMemoryOrderModuleList.offset
+        ldr_obj.InMemoryOrderModuleList.Flink = ldr_addr + ldr_struct.InMemoryOrderModuleList.offset
+        ldr_obj.InMemoryOrderModuleList.Blink = ldr_addr + ldr_struct.InMemoryOrderModuleList.offset
 
-        ldr_obj.InInitializationOrderModuleList.Flink = ldr_addr + ldr_cls.InInitializationOrderModuleList.offset
-        ldr_obj.InInitializationOrderModuleList.Blink = ldr_addr + ldr_cls.InInitializationOrderModuleList.offset
+        ldr_obj.InInitializationOrderModuleList.Flink = ldr_addr + ldr_struct.InInitializationOrderModuleList.offset
+        ldr_obj.InInitializationOrderModuleList.Blink = ldr_addr + ldr_struct.InInitializationOrderModuleList.offset
 
         self.ql.log.info(f'LDR is at {ldr_addr:#x}')
-        self.ql.mem.write(ldr_addr, bytes(ldr_obj))
 
         self.structure_last_addr = nobj_addr
         self.LDR = ldr_obj
 
     def add_ldr_data_table_entry(self, dll_name: str):
-        entry_obj = make_ldr_data_table_entry(self.ql.arch.bits)
-        entry_cls = entry_obj.__class__
+        entry_struct = make_ldr_data_table_entry(self.ql.arch.bits)
 
-        entry_size = ctypes.sizeof(entry_obj)
-        entry_addr = self.ql.os.heap.alloc(entry_size)
+        entry_addr = self.ql.os.heap.alloc(entry_struct.sizeof())
 
         def populate_unistr(obj, s: str) -> None:
             encoded = s.encode('utf-16le')
@@ -407,50 +410,47 @@ class Process:
         image = self.get_image_by_name(dll_name, casefold=True)
         assert image, 'image should have been added to loader.images first'
 
-        entry_obj.DllBase = image.base
-        populate_unistr(entry_obj.FullDllName, ntpath.join(self.ql.os.winsys, dll_name))
-        populate_unistr(entry_obj.BaseDllName, dll_name)
+        with entry_struct.ref(self.ql.mem, entry_addr) as entry_obj:
+            entry_obj.DllBase = image.base
+            populate_unistr(entry_obj.FullDllName, ntpath.join(self.ql.os.winsys, dll_name))
+            populate_unistr(entry_obj.BaseDllName, dll_name)
 
-        # Flink
-        if self.ldr_list:
-            flink_base, flink = self.ldr_list[-1]
+            # Flink
+            if self.ldr_list:
+                with entry_struct.ref(self.ql.mem, self.ldr_list[-1]) as flink:
+                    entry_obj.InLoadOrderLinks.Flink = flink.InLoadOrderLinks.Flink
+                    entry_obj.InMemoryOrderLinks.Flink = flink.InMemoryOrderLinks.Flink
+                    entry_obj.InInitializationOrderLinks.Flink = flink.InInitializationOrderLinks.Flink
 
-            entry_obj.InLoadOrderLinks.Flink = flink.InLoadOrderLinks.Flink
-            entry_obj.InMemoryOrderLinks.Flink = flink.InMemoryOrderLinks.Flink
-            entry_obj.InInitializationOrderLinks.Flink = flink.InInitializationOrderLinks.Flink
+                    flink.InLoadOrderLinks.Flink = entry_addr + entry_struct.InLoadOrderLinks.offset
+                    flink.InMemoryOrderLinks.Flink = entry_addr + entry_struct.InMemoryOrderLinks.offset
+                    flink.InInitializationOrderLinks.Flink = entry_addr + entry_struct.InInitializationOrderLinks.offset
 
-            flink.InLoadOrderLinks.Flink = entry_addr + entry_cls.InLoadOrderLinks.offset
-            flink.InMemoryOrderLinks.Flink = entry_addr + entry_cls.InMemoryOrderLinks.offset
-            flink.InInitializationOrderLinks.Flink = entry_addr + entry_cls.InInitializationOrderLinks.offset
+            else:
+                # a volatile ref to self.PEB.LdrAddress
+                flink = self.LDR
 
-        else:
-            flink_base, flink = (self.PEB.LdrAddress, self.LDR)
+                entry_obj.InLoadOrderLinks.Flink = flink.InLoadOrderModuleList.Flink
+                entry_obj.InMemoryOrderLinks.Flink = flink.InMemoryOrderModuleList.Flink
+                entry_obj.InInitializationOrderLinks.Flink = flink.InInitializationOrderModuleList.Flink
 
-            entry_obj.InLoadOrderLinks.Flink = flink.InLoadOrderModuleList.Flink
-            entry_obj.InMemoryOrderLinks.Flink = flink.InMemoryOrderModuleList.Flink
-            entry_obj.InInitializationOrderLinks.Flink = flink.InInitializationOrderModuleList.Flink
+                flink.InLoadOrderModuleList.Flink = entry_addr + entry_struct.InLoadOrderLinks.offset
+                flink.InMemoryOrderModuleList.Flink = entry_addr + entry_struct.InMemoryOrderLinks.offset
+                flink.InInitializationOrderModuleList.Flink = entry_addr + entry_struct.InInitializationOrderLinks.offset
 
-            flink.InLoadOrderModuleList.Flink = entry_addr + entry_cls.InLoadOrderLinks.offset
-            flink.InMemoryOrderModuleList.Flink = entry_addr + entry_cls.InMemoryOrderLinks.offset
-            flink.InInitializationOrderModuleList.Flink = entry_addr + entry_cls.InInitializationOrderLinks.offset
+            # Blink
+            blink = self.LDR
 
-        # Blink
-        blink_base = self.PEB.LdrAddress
-        blink = self.LDR
+            entry_obj.InLoadOrderLinks.Blink = blink.InLoadOrderModuleList.Blink
+            entry_obj.InMemoryOrderLinks.Blink = blink.InMemoryOrderModuleList.Blink
+            entry_obj.InInitializationOrderLinks.Blink = blink.InInitializationOrderModuleList.Blink
 
-        entry_obj.InLoadOrderLinks.Blink = blink.InLoadOrderModuleList.Blink
-        entry_obj.InMemoryOrderLinks.Blink = blink.InMemoryOrderModuleList.Blink
-        entry_obj.InInitializationOrderLinks.Blink = blink.InInitializationOrderModuleList.Blink
+            blink.InLoadOrderModuleList.Blink = entry_addr + entry_struct.InLoadOrderLinks.offset
+            blink.InMemoryOrderModuleList.Blink = entry_addr + entry_struct.InMemoryOrderLinks.offset
+            blink.InInitializationOrderModuleList.Blink = entry_addr + entry_struct.InInitializationOrderLinks.offset
 
-        blink.InLoadOrderModuleList.Blink = entry_addr + entry_cls.InLoadOrderLinks.offset
-        blink.InMemoryOrderModuleList.Blink = entry_addr + entry_cls.InMemoryOrderLinks.offset
-        blink.InInitializationOrderModuleList.Blink = entry_addr + entry_cls.InInitializationOrderLinks.offset
 
-        self.ql.mem.write(flink_base, bytes(flink))
-        self.ql.mem.write(blink_base, bytes(blink))
-        self.ql.mem.write(entry_addr, bytes(entry_obj))
-
-        self.ldr_list.append((entry_addr, entry_obj))
+        self.ldr_list.append(entry_addr)
 
     @staticmethod
     def directory_exists(pe: pefile.PE, entry: str) -> bool:
@@ -577,53 +577,68 @@ class Process:
         drv_addr = self.structure_last_addr
 
         # PDRIVER_OBJECT DriverObject
-        drv_obj = make_driver_object(self.ql, drv_addr, self.ql.arch.bits)
-        nobj_addr = self.ql.mem.align_up(drv_addr + ctypes.sizeof(drv_obj), 0x10)
+        drvobj_cls = make_driver_object(self.ql.arch.bits)
+        nobj_addr = self.ql.mem.align_up(drv_addr + drvobj_cls.sizeof(), 0x10)
 
         self.ql.log.info(f'DriverObject is at {drv_addr:#x}')
-        self.ql.mem.write(drv_addr, bytes(drv_obj))
+        # note: driver object is volatile; no need to flush its contents to mem
 
         self.structure_last_addr = nobj_addr
         self.driver_object_address = drv_addr
-        self.driver_object = drv_obj
+        self.driver_object = drvobj_cls.volatile_ref(self.ql.mem, drv_addr)
 
     def init_registry_path(self):
         regpath_addr = self.structure_last_addr
 
         # PUNICODE_STRING RegistryPath
-        regpath_obj = make_unicode_string(self.ql.arch.bits,
+        ucstrtype = make_unicode_string(self.ql.arch.bits)
+
+        regpath_obj = ucstrtype(
                 Length=0,
                 MaximumLength=0,
-                Buffer=regpath_addr
+                Buffer=regpath_addr     # FIXME: pointing to self? this does not seem right
         )
 
-        nobj_addr = self.ql.mem.align_up(regpath_addr + ctypes.sizeof(regpath_obj), 0x10)
+        nobj_addr = self.ql.mem.align_up(regpath_addr + ucstrtype.sizeof(), 0x10)
 
         self.ql.log.info(f'RegistryPath is at {regpath_addr:#x}')
-        self.ql.mem.write(regpath_addr, bytes(regpath_obj))
+        regpath_obj.save_to(self.ql.mem, regpath_addr)
 
         self.structure_last_addr = nobj_addr
         self.regitry_path_address = regpath_addr
 
     def init_eprocess(self):
-        eproc_obj = make_eprocess(self.ql.arch.bits)
-
         eproc_addr = self.structure_last_addr
-        nobj_addr = self.ql.mem.align_up(eproc_addr + ctypes.sizeof(eproc_obj), 0x10)
 
-        self.ql.mem.write(eproc_addr, bytes(eproc_obj))
+        eproc_struct = make_eprocess(self.ql.arch.bits)
+        nobj_addr = self.ql.mem.align_up(eproc_addr + eproc_struct.sizeof(), 0x10)
+
+        with eproc_struct.ref(self.ql.mem, eproc_addr) as eproc_obj:
+            eproc_obj.dummy = b''
 
         self.structure_last_addr = nobj_addr
         self.eprocess_address = eproc_addr
 
     def init_ki_user_shared_data(self):
-        addr = self.ql.os.profile.getint(f'OS{self.ql.arch.bits}', 'KI_USER_SHARED_DATA')
+        sysconf = self.ql.os.profile['SYSTEM']
+        osconf  = self.ql.os.profile[f'OS{self.ql.arch.bits}']
 
-        user_shared_data_obj = KUSER_SHARED_DATA()
-        user_shared_data_size = ctypes.sizeof(KUSER_SHARED_DATA)
+        kusd_addr = osconf.getint('KI_USER_SHARED_DATA')
+        kust_struct = KUSER_SHARED_DATA
+        self.ql.mem.map(kusd_addr, self.ql.mem.align_up(kust_struct.sizeof()), info='[kuser shared data]')
 
-        self.ql.mem.map(addr, self.ql.mem.align_up(user_shared_data_size))
-        self.ql.mem.write(addr, bytes(user_shared_data_obj))
+        # initialize an instance with a few key fields
+        kusd_obj = kust_struct.volatile_ref(self.ql.mem, kusd_addr)
+        kusd_obj.ImageNumberLow = 0x014c    # IMAGE_FILE_MACHINE_I386
+        kusd_obj.ImageNumberHigh = 0x8664   # IMAGE_FILE_MACHINE_AMD64
+        kusd_obj.NtSystemRoot = self.ql.os.windir
+        kusd_obj.NtProductType = sysconf.getint('productType')
+        kusd_obj.NtMajorVersion = sysconf.getint('majorVersion')
+        kusd_obj.NtMinorVersion = sysconf.getint('minorVersion')
+        kusd_obj.KdDebuggerEnabled = 0
+        kusd_obj.NXSupportPolicy = 0        # NX_SUPPORT_POLICY_ALWAYSOFF
+
+        self.ql.os.KUSER_SHARED_DATA = kusd_obj
 
     def init_security_cookie(self, pe: pefile.PE, image_base: int):
         if not Process.directory_exists(pe, 'IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG'):
@@ -657,6 +672,7 @@ class QlLoaderPE(QlLoader, Process):
         self.sys_dlls = (
             'ntdll.dll',
             'kernel32.dll',
+            'mscoree.dll',
             'ucrtbase.dll'
         )
 
@@ -695,7 +711,6 @@ class QlLoaderPE(QlLoader, Process):
         cmdline = ntpath.join(self.ql.os.userprofile, 'Desktop', self.ql.targetname)
         cmdargs = ' '.join(f'"{arg}"' if ' ' in arg else arg for arg in self.argv[1:])
 
-        self.filepath = bytes(f'{cmdline}\x00', "utf-8")
         self.cmdline = bytes(f'{cmdline} {cmdargs}\x00', "utf-8")
 
         self.load(pe)
@@ -722,14 +737,13 @@ class QlLoaderPE(QlLoader, Process):
             self.ql.log.info(f'Loading {self.path} to {image_base:#x}')
             self.ql.log.info(f'PE entry point at {self.entry_point:#x}')
 
-            self.ql.mem.map(image_base, image_size, info=f'[{image_name}]')
+            self.ql.mem.map(image_base, image_size, info=f'{image_name}')
             self.images.append(Image(image_base, image_base + pe.NT_HEADERS.OPTIONAL_HEADER.SizeOfImage, os.path.abspath(self.path)))
 
             if self.is_driver:
                 self.init_driver_object()
                 self.init_registry_path()
                 self.init_eprocess()
-                self.init_ki_user_shared_data()
 
                 # set IRQ Level in CR8 to PASSIVE_LEVEL
                 self.ql.arch.regs.write(UC_X86_REG_CR8, 0)
@@ -748,6 +762,8 @@ class QlLoaderPE(QlLoader, Process):
                 # add image to ldr table
                 self.add_ldr_data_table_entry(image_name)
 
+            self.init_ki_user_shared_data()
+
             pe.parse_data_directories()
 
             # done manipulating pe file; write its contents into memory
@@ -817,8 +833,6 @@ class QlLoaderPE(QlLoader, Process):
                 self.ql.os.fcall.call_native(self.entry_point, args, None)
 
         elif pe is None:
-            self.filepath = b""
-
             self.ql.mem.map(self.entry_point, self.ql.os.code_ram_size, info="[shellcode]")
 
             self.init_teb()

qiling/os/fcall.py

@@ -2,7 +2,7 @@
 # 
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 
-from typing import Any, Callable, Iterable, MutableMapping, Optional, Mapping, Tuple, Sequence
+from typing import Any, Callable, Iterable, Iterator, MutableMapping, Optional, Mapping, Tuple, Sequence
 
 from qiling import Qiling
 from qiling.cc import QlCC
@@ -50,6 +50,21 @@ class QlFunctionCall:
 		# let the user override default accessors or add custom ones
 		self.accessors.update(accessors)
 
+	def readEllipsis(self, ptypes: Sequence[Any]) -> Iterator[int]:
+		"""
+		"""
+
+		default = self.accessors[PARAM_INTN]
+
+		# count skipped slots
+		si = sum(self.accessors.get(typ, default)[2] for typ in ptypes)
+
+		while True:
+			read, _, nslots = default
+
+			yield read(si)
+			si += nslots
+
 	def readParams(self, ptypes: Sequence[Any]) -> Sequence[int]:
 		"""Walk the function parameters list and get their values.
 

qiling/os/linux/kernel_api/kernel_api.py

@@ -31,12 +31,10 @@ def hook_printk(ql: Qiling, address: int, params):
         return 0
 
     level = int(format[1]) if format[1].isdigit() else 4
-    nargs = format.count("%")
-    ptypes = (POINTER, ) + (PARAM_INTN, ) * nargs
-    args = ql.os.fcall.readParams(ptypes)[1:]
+    args = ql.os.fcall.readEllipsis(params.values())
 
-    count = ql.os.utils.printf(f'{PRINTK_LEVEL[level]} {format[2:]}', args, wstring=False)
-    ql.os.utils.update_ellipsis(params, args)
+    count, upd_args = ql.os.utils.printf(f'{PRINTK_LEVEL[level]} {format[2:]}', args, wstring=False)
+    upd_args(params)
 
     return count
 

qiling/os/linux/linux.py

@@ -14,6 +14,8 @@ from qiling.cc import QlCC, intel, arm, mips, riscv, ppc
 from qiling.const import QL_ARCH, QL_OS
 from qiling.os.fcall import QlFunctionCall
 from qiling.os.const import *
+from qiling.os.linux.procfs import QlProcFS
+from qiling.os.mapper import QlFsMappedCallable
 from qiling.os.posix.posix import QlOsPosix
 
 from . import futex
@@ -118,6 +120,14 @@ class QlOsLinux(QlOsPosix):
             if getattr(self.fd[i], 'close_on_exec', 0):
                 self.fd[i] = None
 
+
+    def setup_procfs(self):
+        self.fs_mapper.add_fs_mapping(r'/proc/self/auxv',    QlFsMappedCallable(QlProcFS.self_auxv, self))
+        self.fs_mapper.add_fs_mapping(r'/proc/self/cmdline', QlFsMappedCallable(QlProcFS.self_cmdline, self))
+        self.fs_mapper.add_fs_mapping(r'/proc/self/environ', QlFsMappedCallable(QlProcFS.self_environ, self))
+        self.fs_mapper.add_fs_mapping(r'/proc/self/exe',     QlFsMappedCallable(QlProcFS.self_exe, self))
+        self.fs_mapper.add_fs_mapping(r'/proc/self/maps',    QlFsMappedCallable(QlProcFS.self_map, self.ql.mem))
+
     def hook_syscall(self, ql, intno = None):
         return self.load_syscall()
 
@@ -133,6 +143,10 @@ class QlOsLinux(QlOsPosix):
 
 
     def run(self):
+        # do not set-up procfs for drivers and shellcode
+        if not self.ql.code and not self.ql.loader.is_driver:
+            self.setup_procfs()
+
         if self.ql.exit_point is not None:
             self.exit_point = self.ql.exit_point
 

qiling/os/linux/procfs.py

@@ -0,0 +1,71 @@
+
+import io
+from typing import TYPE_CHECKING, AnyStr, Optional, Sized
+
+from qiling.os.mapper import QlFsMappedObject
+
+if TYPE_CHECKING:
+    from qiling.os.linux.linux import QlOsLinux
+    from qiling.os.memory import QlMemoryManager
+
+
+class QlProcFS:
+
+    @staticmethod
+    def self_auxv(os: 'QlOsLinux') -> QlFsMappedObject:
+        nbytes = os.ql.arch.bits // 8
+
+        auxv_addr = os.ql.loader.auxv
+        null_entry = bytes(nbytes * 2)
+
+        auxv_data = bytearray()
+
+        # keep reading until AUXV.AT_NULL is reached
+        while not auxv_data.endswith(null_entry):
+            auxv_data.extend(os.ql.mem.read(auxv_addr, nbytes))
+            auxv_addr += nbytes
+
+            auxv_data.extend(os.ql.mem.read(auxv_addr, nbytes))
+            auxv_addr += nbytes
+    
+        return io.BytesIO(bytes(auxv_data))
+
+
+    @staticmethod
+    def self_cmdline(os: 'QlOsLinux') -> QlFsMappedObject:
+        entries = (arg.encode('utf-8') for arg in os.ql.argv)
+        cmdline = b'\x00'.join(entries) + b'\x00'
+
+        return io.BytesIO(cmdline)
+
+
+    @staticmethod
+    def self_environ(os: 'QlOsLinux') -> QlFsMappedObject:
+        def __to_bytes(s: AnyStr) -> bytes:
+            if isinstance(s, str):
+                return s.encode('utf-8')
+
+            return s
+
+        entries = (b'='.join((__to_bytes(k), __to_bytes(v))) for k, v in os.ql.env.items())
+        environ = b'\x00'.join(entries) + b'\x00'
+
+        return io.BytesIO(environ)
+
+
+    @staticmethod
+    def self_exe(os: 'QlOsLinux') -> QlFsMappedObject:
+        with open(os.ql.path, 'rb') as exefile:
+            content = exefile.read()
+
+        return io.BytesIO(content)
+    
+    @staticmethod
+    def self_map(mem: 'QlMemoryManager') -> QlFsMappedObject:
+        content = b""
+        mapinfo = mem.get_mapinfo()
+
+        for lbound, ubound, perms, label, container in mapinfo:
+            content += f"{lbound:x}-{ubound:x}\t{perms}p\t0\t00:00\t0\t{container if container else label}\n".encode("utf-8")
+
+        return io.BytesIO(content)

qiling/os/linux/thread.py

@@ -11,6 +11,7 @@ from abc import abstractmethod
 from unicorn.unicorn import UcError
 
 from qiling import Qiling
+from qiling.const import QL_ARCH
 from qiling.os.thread import *
 from qiling.arch.x86_const import *
 from qiling.exception import QlErrorExecutionStop
@@ -593,6 +594,8 @@ class QlLinuxThreadManagement:
     def stop(self):
         self.ql.log.debug("[Thread Manager] Stop the world.")
         self.ql.emu_stop()
+        global LINUX_THREAD_ID
+        LINUX_THREAD_ID = 2000
         while len(self.threads) != 0:
             t = self.threads.pop()
             self.ql.log.debug(f"[Thread Manager] Thread IDs: { {t.id for t in self.threads} }")

qiling/os/mapper.py

@@ -57,6 +57,19 @@ class QlFsMappedObject:
     def name(self):
         raise NotImplementedError("QlFsMappedObject property not implemented: name")
 
+# This is a helper class to allow users to pass any class to add_fs_mapper
+#
+# Everytime open is called on the mapped path, cls(*args, **kwargs) will be called.
+class QlFsMappedCallable:
+    
+    def __init__(self, cls, *args, **kwargs) -> None:
+        self._args = args
+        self._kwargs = kwargs
+        self._cls = cls
+    
+    def __call__(self) -> QlFsMappedObject:
+        return self._cls(*self._args, **self._kwargs)
+    
 class QlFsMapper:
 
     def __init__(self, path: QlOsPath):
@@ -69,7 +82,7 @@ class QlFsMapper:
         if isinstance(real_dest, str):
             obj = ql_file.open(real_dest, openflags, openmode)
 
-        elif inspect.isclass(real_dest):
+        elif callable(real_dest):
             obj = real_dest()
 
         else:
@@ -83,7 +96,7 @@ class QlFsMapper:
         if isinstance(real_dest, str):
             obj = open(real_dest, openmode)
 
-        elif inspect.isclass(real_dest):
+        elif callable(real_dest):
             obj = real_dest()
 
         else:
@@ -101,15 +114,23 @@ class QlFsMapper:
         if self.has_mapping(path):
             return self._open_mapping_ql_file(path, openflags, openmode)
 
-        real_path = self.path.transform_to_real_path(path)
-        return ql_file.open(real_path, openflags, openmode)
+        host_path = self.path.virtual_to_host_path(path)
+
+        if not self.path.is_safe_host_path(host_path):
+            raise PermissionError(f'unsafe path: {host_path}')
+
+        return ql_file.open(host_path, openflags, openmode)
 
     def open(self, path: str, openmode: str):
         if self.has_mapping(path):
             return self._open_mapping(path, openmode)
 
-        real_path = self.path.transform_to_real_path(path)
-        return open(real_path, openmode)
+        host_path = self.path.virtual_to_host_path(path)
+
+        if not self.path.is_safe_host_path(host_path):
+            raise PermissionError(f'unsafe path: {host_path}')
+
+        return open(host_path, openmode)
 
     def _parse_path(self, p: Union[os.PathLike, str]) -> str:
         fspath = getattr(p, '__fspath__', None)
@@ -123,18 +144,26 @@ class QlFsMapper:
 
         return p
 
-    def add_fs_mapping(self, ql_path: Union[os.PathLike, str], real_dest: Any) -> None:
+    def add_fs_mapping(self, ql_path: Union[os.PathLike, str], real_dest: Union[str, QlFsMappedObject, QlFsMappedCallable]) -> None:
         """Map an object to Qiling emulated file system.
 
         Args:
             ql_path: Emulated path which should be convertable to a string or a hashable object. e.g. pathlib.Path
-            real_dest: Mapped object, can be a string, an object or a class.
+            real_dest: Mapped object, can be a string, an object or a callable(class).
                 string: mapped path in the host machine, e.g. '/dev/urandom' -> '/dev/urandom'
                 object: mapped object, will be returned each time the emulated path has been opened
-                class:  mapped class, will be used to create a new instance each time the emulated path has been opened
+                class:  mapped callable, will be used to create a new instance each time the emulated path has been opened
         """
 
         ql_path = self._parse_path(ql_path)
         real_dest = self._parse_path(real_dest)
 
         self._mapping[ql_path] = real_dest
+        
+    def remove_fs_mapping(self, ql_path: Union[os.PathLike, str]):
+        """Remove a mapping from the fs mapper.
+
+        Args:
+            ql_path (Union[os.PathLike, str]): The mapped path.
+        """
+        del self._mapping[self._parse_path(ql_path)]

qiling/os/mcu/__init__.py

@@ -0,0 +1 @@
+

qiling/os/path.py

@@ -42,9 +42,9 @@ class QlOsPath:
         self.cwd = cwd
 
         # <TEMPORARY>
+        # temporary aliases for backward compatibility
         self.transform_to_relative_path = self.virtual_abspath
         self.transform_to_real_path = self.virtual_to_host_path
-        self.transform_to_link_path = self.virtual_to_host_path
         # </TEMPORARY>
 
     @staticmethod
@@ -258,6 +258,23 @@ class QlOsPath:
         else:
             return True
 
+    def host_to_virtual_path(self, hostpath: str) -> str:
+        """Convert a host path to its corresponding virtual path relative to rootfs.
+
+        Args:
+            hostpath : host path
+
+        Returns: the corresponding virtual path
+
+        Raises: `ValueError` in case the host path does not map to a rootfs location
+        """
+
+        resolved = Path(hostpath).resolve(strict=False)
+        virtpath = self._cwd_anchor / resolved.relative_to(self._rootfs_path)
+
+        return str(virtpath)
+
+
     def virtual_abspath(self, virtpath: str) -> str:
         """Convert a relative virtual path to an absolute virtual path based
         on the current working directory.

qiling/os/posix/filestruct.py

@@ -3,133 +3,136 @@
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 #
 import os
-
-from qiling.exception import *
+from socket import socket, AddressFamily, SocketKind
+from typing import Union
 
 try:
     import fcntl
 except ImportError:
     pass
-import socket
 
 class ql_socket:
-    def __init__(self, socket):
+    def __init__(self, socket: socket):
         self.__fd = socket.fileno()
         self.__socket = socket
 
     def __getstate__(self, *args, **kwargs):
-
         _state = self.__dict__.copy()
 
-        _state["_ql_socket__socket"] = {
-                "family": self.__dict__["_ql_socket__socket"].family,
-                "type": self.__dict__["_ql_socket__socket"].type,
-                "proto": self.__dict__["_ql_socket__socket"].proto,
-                "laddr": self.__dict__["_ql_socket__socket"].getsockname(),
-                }
+        fname = f'_{self.__class__.__name__}__socket'
+        sock = self.__dict__[fname]
+
+        _state[fname] = {
+            "family" : sock.family,
+            "type"   : sock.type,
+            "proto"  : sock.proto,
+            "laddr"  : sock.getsockname(),
+        }
 
         return _state
 
     def __setstate__(self, state):
         self.__dict__ = state
-    
-    @classmethod
-    def open(self, socket_domain, socket_type, socket_protocol, opts=None):
-        s = socket.socket(socket_domain, socket_type, socket_protocol)
-        if opts:
-            s.setsockopt(*opts)
-        return self(s)
 
-    def read(self, read_len):
-        return os.read(self.__fd, read_len)
-    
-    def write(self, write_buf):
-        return os.write(self.__fd, write_buf)
-    
-    def fileno(self):
+    @classmethod
+    def open(cls, domain: Union[AddressFamily, int], socktype: Union[SocketKind, int], protocol: int):
+        s = socket(domain, socktype, protocol)
+
+        return cls(s)
+
+    def read(self, length: int) -> bytes:
+        return os.read(self.__fd, length)
+
+    def write(self, data: bytes) -> int:
+        return os.write(self.__fd, data)
+
+    def fileno(self) -> int:
         return self.__fd
-    
-    def close(self):
-        return os.close(self.__fd)
-    
-    def fcntl(self, fcntl_cmd, fcntl_arg):
+
+    def close(self) -> None:
+        os.close(self.__fd)
+
+    def fcntl(self, cmd, arg):
         try:
-            return fcntl.fcntl(self.__fd, fcntl_cmd, fcntl_arg)
+            return fcntl.fcntl(self.__fd, cmd, arg)
         except Exception:
             pass
 
-    def ioctl(self, ioctl_cmd, ioctl_arg):
+    def ioctl(self, cmd, arg):
         try:
-            return fcntl.ioctl(self.__fd, ioctl_cmd, ioctl_arg)
+            return fcntl.ioctl(self.__fd, cmd, arg)
         except Exception:
-            pass    
-    
-    def dup(self):
-        new_s = self.__socket.dup()
-        new_ql_socket = ql_socket(new_s)
-        return new_ql_socket
+            pass
 
-    def connect(self, connect_addr):
-        return self.__socket.connect(connect_addr)
-    
-    def shutdown(self, shutdown_how):
-        return self.__socket.shutdown(shutdown_how)
-    
-    def bind(self, bind_addr):
-        return self.__socket.bind(bind_addr)
-    
-    def listen(self, listen_num):
-        return self.__socket.listen(listen_num)
+    def dup(self) -> 'ql_socket':
+        new_s = self.__socket.dup()
+
+        return ql_socket(new_s)
+
+    def connect(self, address) -> None:
+        self.__socket.connect(address)
+
+    def shutdown(self, how: int) -> None:
+        return self.__socket.shutdown(how)
+
+    def bind(self, address) -> None:
+        return self.__socket.bind(address)
+
+    def listen(self, backlog: int) -> None:
+        return self.__socket.listen(backlog)
 
     def getsockname(self):
         return self.__socket.getsockname()
-        
+
     def getpeername(self):
         return self.__socket.getpeername()
 
-    def getsockopt(self, level, optname, optlen):
-        return self.__socket.getsockopt(level, optname, optlen)
+    def getsockopt(self, level: int, optname: int, buflen: int):
+        return self.__socket.getsockopt(level, optname, buflen)
 
-    def setsockopt(self, level, optname, optval, optlen):
-        if optval is not None:
-            return self.__socket.setsockopt(level, optname, optval)
+    def setsockopt(self, level: int, optname: int, value: Union[int, bytes, None], optlen: int = 0) -> None:
+        if value is None:
+            self.__socket.setsockopt(level, optname, None, optlen)
         else:
-            return self.__socket.setsockopt(level, optname, None, optval)
-    
+            self.__socket.setsockopt(level, optname, value)
+
     def accept(self):
         try:
             con, addr = self.__socket.accept()
-            new_ql_socket = ql_socket(con)
         except BlockingIOError:
             # For support non-blocking sockets
-            return None, None
-        return new_ql_socket, addr
-    
-    def recv(self, recv_len, recv_flags):
-        return self.__socket.recv(recv_len, recv_flags)
-    
-    def send(self, send_buf, send_flags):
-        return self.__socket.send(send_buf, send_flags)
+            addr = None
+            new_ql_socket = None
+        else:
+            new_ql_socket = ql_socket(con)
 
-    def recvmsg(self, bufsize, ancbufsize, flags):
+        return new_ql_socket, addr
+
+    def recv(self, bufsize: int, flags: int) -> bytes:
+        return self.__socket.recv(bufsize, flags)
+
+    def send(self, data, flags: int) -> int:
+        return self.__socket.send(data, flags)
+
+    def recvmsg(self, bufsize: int, ancbufsize: int, flags: int):
         return self.__socket.recvmsg(bufsize, ancbufsize, flags)
 
-    def recvfrom(self, recvfrom_len, recvfrom_flags):
-        return self.__socket.recvfrom(recvfrom_len, recvfrom_flags)
+    def recvfrom(self, bufsize: int, flags: int):
+        return self.__socket.recvfrom(bufsize, flags)
 
     def sendto(self, sendto_buf, sendto_flags, sendto_addr):
         return self.__socket.sendto(sendto_buf, sendto_flags, sendto_addr)
 
     @property
-    def family(self):
+    def family(self) -> AddressFamily:
         return self.__socket.family
 
     @property
-    def socktype(self):
+    def socktype(self) -> SocketKind:
         return self.__socket.type
 
     @property
-    def socket(self):
+    def socket(self) -> socket:
         return self.__socket
 
     # def __getattr__(self,name):  
@@ -139,41 +142,40 @@ class ql_socket:
     #         raise AttributeError("A instance has no attribute '%s'" % name)
 
 class ql_pipe:
-    def __init__(self, fd):
+    def __init__(self, fd: int):
         self.__fd = fd
 
     @classmethod
-    def open(self):
+    def open(cls):
         r, w = os.pipe()
-        return (self(r), self(w))
 
-    def read(self, read_len):
-        return os.read(self.__fd, read_len)
-    
-    def write(self, write_buf):
-        return os.write(self.__fd, write_buf)
-    
-    def fileno(self):
+        return (cls(r), cls(w))
+
+    def read(self, length: int) -> bytes:
+        return os.read(self.__fd, length)
+
+    def write(self, data: bytes) -> int:
+        return os.write(self.__fd, data)
+
+    def fileno(self) -> int:
         return self.__fd
 
-    def close(self):
-        return os.close(self.__fd)
-    
-    def fcntl(self, fcntl_cmd, fcntl_arg):
+    def close(self) -> None:
+        os.close(self.__fd)
+
+    def fcntl(self, cmd, arg):
         try:
-            return fcntl.fcntl(self.__fd, fcntl_cmd, fcntl_arg)
+            return fcntl.fcntl(self.__fd, cmd, arg)
         except Exception:
             pass
 
-    def ioctl(self, ioctl_cmd, ioctl_arg):
+    def ioctl(self, cmd, arg):
         try:
-            return fcntl.ioctl(self.__fd, ioctl_cmd, ioctl_arg)
+            return fcntl.ioctl(self.__fd, cmd, arg)
         except Exception:
-            pass    
-    
-    def dup(self):
+            pass
+
+    def dup(self) -> 'ql_pipe':
         new_fd = os.dup(self.__fd)
-        new_ql_pipe = ql_pipe(new_fd)
-        return new_ql_pipe
-
 
+        return ql_pipe(new_fd)

qiling/os/posix/syscall/socket.py

@@ -67,40 +67,57 @@ def ql_bin_to_ip(ip):
     return ipaddress.ip_address(ip).compressed
 
 
+def ql_unix_socket_path(ql: Qiling, sun_path: bytearray) -> str:
+    if sun_path[0] == 0:
+        # Abstract Unix namespace
+        # TODO: isolate from host namespace
+        # TODO: Windows
+        ql.log.warning(f'Beware! Usage of hosts abstract socket namespace {bytes(sun_path)}')
+        return sun_path.decode()
+    sun_path = sun_path.split(b'\0')[0].decode()
+    return ql.os.path.transform_to_real_path(sun_path)
+
+
 def ql_syscall_socket(ql: Qiling, socket_domain, socket_type, socket_protocol):
     idx = next((i for i in range(NR_OPEN) if ql.os.fd[i] is None), -1)
-    regreturn = idx
 
     if idx != -1:
+        emu_socket_value = socket_type
+
         # ql_socket.open should use host platform based socket_type.
         try:
-            emu_socket_value = socket_type
             emu_socket_type = socket_type_mapping(socket_type, ql.arch.type, ql.os.type)
-            socket_type = getattr(socket, emu_socket_type)
-            ql.log.debug(f'Convert emu_socket_type {emu_socket_type}:{emu_socket_value} to host platform based socket_type {emu_socket_type}:{socket_type}')
-
-        except AttributeError:
-            ql.log.error(f'Cannot convert emu_socket_type {emu_socket_type}:{emu_socket_value} to host platform based socket_type')
-            raise
-
-        except Exception:
+        except KeyError:
             ql.log.error(f'Cannot convert emu_socket_type {emu_socket_value} to host platform based socket_type')
             raise
 
         try:
-            if ql.verbose >= QL_VERBOSE.DEBUG:  # set REUSEADDR options under debug mode
-                ql.os.fd[idx] = ql_socket.open(socket_domain, socket_type, socket_protocol, (socket.SOL_SOCKET, socket.SO_REUSEADDR, 1))
-            else:
-                ql.os.fd[idx] = ql_socket.open(socket_domain, socket_type, socket_protocol)
-        except OSError as e:  # May raise error: Protocol not supported
+            socket_type = getattr(socket, emu_socket_type)
+        except AttributeError:
+            ql.log.error(f'Cannot convert emu_socket_type {emu_socket_type}:{emu_socket_value} to host platform based socket_type')
+            raise
+
+        ql.log.debug(f'Convert emu_socket_type {emu_socket_type}:{emu_socket_value} to host platform based socket_type {emu_socket_type}:{socket_type}')
+
+        try:
+            sock = ql_socket.open(socket_domain, socket_type, socket_protocol)
+
+            # set REUSEADDR options under debug mode
+            if ql.verbose >= QL_VERBOSE.DEBUG:
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+            ql.os.fd[idx] = sock
+
+        # May raise error: Protocol not supported
+        except OSError as e:
             ql.log.debug(f'{e}: {socket_domain=}, {socket_type=}, {socket_protocol=}')
-            regreturn = -1
+            idx = -1
 
     socket_type = socket_type_mapping(socket_type, ql.arch.type, ql.os.type)
     socket_domain = socket_domain_mapping(socket_domain, ql.arch.type, ql.os.type)
-    ql.log.debug("socket(%s, %s, %s) = %d" % (socket_domain, socket_type, socket_protocol, regreturn))
+    ql.log.debug("socket(%s, %s, %s) = %d" % (socket_domain, socket_type, socket_protocol, idx))
 
-    return regreturn
+    return idx
 
 
 def ql_syscall_connect(ql: Qiling, connect_sockfd, connect_addr, connect_addrlen):
@@ -115,8 +132,7 @@ def ql_syscall_connect(ql: Qiling, connect_sockfd, connect_addr, connect_addrlen
     try:
         if s.family == family:
             if s.family == AF_UNIX:
-                sun_path = sock_addr[2 : ].split(b"\x00")[0]
-                sun_path = ql.os.path.transform_to_real_path(sun_path.decode())
+                sun_path = ql_unix_socket_path(ql, sock_addr[2:])
                 s.connect(sun_path)
                 regreturn = 0
             elif s.family == AF_INET:
@@ -146,29 +162,31 @@ def ql_syscall_getsockopt(ql: Qiling, sockfd, level, optname, optval_addr, optle
 
     try:
         optlen = min(ql.unpack32s(ql.mem.read(optlen_addr, 4)), 1024)
+
         if optlen < 0:
             return -EINVAL
 
+        emu_level = level
+
         try:
-            emu_level = level
             emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.os.type)
+        except KeyError:
+            ql.log.error(f"Can't convert emu_level {emu_level} to host platform based level")
+            raise
+
+        try:
             level = getattr(socket, emu_level_name)
-            ql.log.debug("Convert emu_level {}:{} to host platform based level {}:{}".format(
-                emu_level_name, emu_level, emu_level_name, level))
-
         except AttributeError:
-            ql.log.error("Can't convert emu_level {}:{} to host platform based emu_level".format(
-                emu_level_name, emu_level))
+            ql.log.error(f"Can't convert emu_level {emu_level_name}:{emu_level} to host platform based emu_level")
             raise
 
-        except Exception:
-            ql.log.error("Can't convert emu_level {} to host platform based level".format(emu_level))
-            raise
+        ql.log.debug(f"Convert emu_level {emu_level_name}:{emu_level} to host platform based level {emu_level_name}:{level}")
+
+        emu_opt = optname
 
         try:
-            emu_opt = optname
-
             emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.os.type)
+
             # emu_opt_name is based on level
             if emu_level_name == "IPPROTO_IP":
                 emu_opt_name = socket_ip_option_mapping(emu_opt, ql.arch.type, ql.os.type)
@@ -180,18 +198,17 @@ def ql_syscall_getsockopt(ql: Qiling, sockfd, level, optname, optval_addr, optle
                 if emu_opt_name.endswith("_NEW") or emu_opt_name.endswith("_OLD"):
                     emu_opt_name = emu_opt_name[:-4]
 
+        except KeyError:
+            ql.log.error(f"Can't convert emu_optname {emu_opt} to host platform based optname")
+            raise
+
+        try:
             optname = getattr(socket, emu_opt_name)
-            ql.log.debug("Convert emu_optname {}:{} to host platform based optname {}:{}".format(
-                emu_opt_name, emu_opt, emu_opt_name, optname))
-
         except AttributeError:
-            ql.log.error("Can't convert emu_optname {}:{} to host platform based emu_optname".format(
-                emu_opt_name, emu_opt))
+            ql.log.error(f"Can't convert emu_optname {emu_opt_name}:{emu_opt} to host platform based emu_optname")
             raise
 
-        except Exception:
-            ql.log.error("Can't convert emu_optname {} to host platform based optname".format(emu_opt))
-            raise
+        ql.log.debug(f"Convert emu_optname {emu_opt_name}:{emu_opt} to host platform based optname {emu_opt_name}:{optname}")
 
         optval = ql.os.fd[sockfd].getsockopt(level, optname, optlen)
         ql.mem.write(optval_addr, optval)
@@ -210,26 +227,27 @@ def ql_syscall_setsockopt(ql: Qiling, sockfd, level, optname, optval_addr, optle
         ql.os.fd[sockfd].setsockopt(level, optname, None, optlen)
     else:
         try:
+            emu_level = level
+
             try:
-                emu_level = level
                 emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.os.type)
+            except KeyError:
+                ql.log.error(f"Can't convert emu_level {emu_level} to host platform based level")
+                raise
+
+            try:
                 level = getattr(socket, emu_level_name)
-                ql.log.debug("Convert emu_level {}:{} to host platform based level {}:{}".format(
-                    emu_level_name, emu_level, emu_level_name, level))
-
             except AttributeError:
-                ql.log.error("Can't convert emu_level {}:{} to host platform based emu_level".format(
-                    emu_level_name, emu_level))
+                ql.log.error(f"Can't convert emu_level {emu_level_name}:{emu_level} to host platform based emu_level")
                 raise
 
-            except Exception:
-                ql.log.error("Can't convert emu_level {} to host platform based level".format(emu_level))
-                raise
+            ql.log.debug(f"Convert emu_level {emu_level_name}:{emu_level} to host platform based level {emu_level_name}:{level}")
+
+            emu_opt = optname
 
             try:
-                emu_opt = optname
-
                 emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.os.type)
+
                 # emu_opt_name is based on level
                 if emu_level_name == "IPPROTO_IP":
                     emu_opt_name = socket_ip_option_mapping(emu_opt, ql.arch.type, ql.os.type)
@@ -241,18 +259,17 @@ def ql_syscall_setsockopt(ql: Qiling, sockfd, level, optname, optval_addr, optle
                     if emu_opt_name.endswith("_NEW") or emu_opt_name.endswith("_OLD"):
                         emu_opt_name = emu_opt_name[:-4]
 
+            except KeyError:
+                ql.log.error(f"Can't convert emu_optname {emu_opt} to host platform based optname")
+                raise
+
+            try:
                 optname = getattr(socket, emu_opt_name)
-                ql.log.debug("Convert emu_optname {}:{} to host platform based optname {}:{}".format(
-                    emu_opt_name, emu_opt, emu_opt_name, optname))
-
             except AttributeError:
-                ql.log.error("Can't convert emu_optname {}:{} to host platform based emu_optname".format(
-                    emu_opt_name, emu_opt))
+                ql.log.error(f"Can't convert emu_optname {emu_opt_name}:{emu_opt} to host platform based emu_optname")
                 raise
 
-            except Exception:
-                ql.log.error("Can't convert emu_optname {} to host platform based optname".format(emu_opt))
-                raise
+            ql.log.debug(f"Convert emu_optname {emu_opt_name}:{emu_opt} to host platform based optname {emu_opt_name}:{optname}")
 
             optval = ql.mem.read(optval_addr, optlen)
             ql.os.fd[sockfd].setsockopt(level, optname, optval, None)
@@ -297,20 +314,19 @@ def ql_syscall_bind(ql: Qiling, bind_fd, bind_addr, bind_addrlen):
         port = port + 8000
 
     if sin_family == 1:
-        path = data[2 : ].split(b'\x00')[0]
-        path = ql.os.path.transform_to_real_path(path.decode())
+        path = ql_unix_socket_path(ql, data[2:])
         ql.log.info(path)
         ql.os.fd[bind_fd].bind(path)
 
     # need a proper fix, for now ipv4 comes first
     elif sin_family == 2 and ql.os.bindtolocalhost == True:
-        ql.os.fd[bind_fd].bind(('127.0.0.1', port))
         host = "127.0.0.1"
+        ql.os.fd[bind_fd].bind((host, port))
 
     # IPv4 should comes first
     elif ql.os.ipv6 == True and sin_family == 10 and ql.os.bindtolocalhost == True:
-        ql.os.fd[bind_fd].bind(('::1', port))
         host = "::1"
+        ql.os.fd[bind_fd].bind((host, port))
 
     elif ql.os.bindtolocalhost == False:
         ql.os.fd[bind_fd].bind((host, port))
@@ -545,19 +561,22 @@ def ql_syscall_recvfrom(ql: Qiling, sockfd: int, buf: int, length: int, flags: i
         ql.log.debug("%s" % tmp_buf)
 
     sin_family = int(sock.family)
-    data = struct.pack("<h", sin_family)
+    sockaddr_out = struct.pack("<h", sin_family)
 
     if sin_family == 1:
-        ql.log.debug("recvfrom() path is " + tmp_addr)
-        data += tmp_addr.encode()
+        # Abstract Unix socket path is not filled in recvfrom
+        ql.log.debug(f"recvfrom() path is '{tmp_addr or 'UNIX ABSTRACT NAMESPACE'}'")
+        if tmp_addr:
+            sockaddr_out += tmp_addr.encode()
     else:
         ql.log.debug("recvfrom() addr is %s:%d" % (tmp_addr[0], tmp_addr[1]))
-        data += struct.pack(">H", tmp_addr[1])
-        data += ipaddress.ip_address(tmp_addr[0]).packed
+        sockaddr_out += struct.pack(">H", tmp_addr[1])
+        sockaddr_out += ipaddress.ip_address(tmp_addr[0]).packed
         addrlen = ql.mem.read_ptr(addrlen)
-        data = data[:addrlen]
+        sockaddr_out = sockaddr_out[:addrlen]
 
-    ql.mem.write(addr, data)
+    if addr:
+        ql.mem.write(addr, sockaddr_out)
     ql.mem.write(buf, tmp_buf)
 
     return len(tmp_buf)
@@ -600,8 +619,7 @@ def ql_syscall_sendto(ql: Qiling, sockfd: int, sendto_buf, sendto_len, sendto_fl
         ql.log.debug("sendto() len is " + str(sendto_len))
 
         if sin_family == 1:
-            path = data[2 : ].split(b'\x00')[0]
-            path = ql.os.path.transform_to_real_path(path.decode())
+            path = ql_unix_socket_path(ql, data[2:])
 
             ql.log.debug("sendto() path is " + str(path))
             regreturn = sock.sendto(bytes(tmp_buf), sendto_flags, path)

qiling/os/posix/syscall/uio.py

@@ -17,7 +17,7 @@ def ql_syscall_writev(ql: Qiling, fd: int, vec: int, vlen: int):
         regreturn += l
 
         buf = ql.mem.read(addr, l)
-        ql.log.debug(f'{buf.decode()!r}')
+        ql.log.debug(f'{bytes(buf)}')
 
         if hasattr(ql.os.fd[fd], 'write'):
             ql.os.fd[fd].write(buf)

qiling/os/posix/syscall/unistd.py

@@ -143,12 +143,8 @@ def ql_syscall_faccessat(ql: Qiling, dfd: int, filename: int, mode: int):
 
     if not os.path.exists(real_path):
         regreturn = -1
-
-    elif stat.S_ISFIFO(Stat(real_path).st_mode):
-        regreturn = 0
-
     else:
-        regreturn = -1
+        regreturn = 0
 
     if regreturn == -1:
         ql.log.debug(f'File not found or skipped: {access_path}')
@@ -338,23 +334,25 @@ def ql_syscall_write(ql: Qiling, fd: int, buf: int, count: int):
 def ql_syscall_readlink(ql: Qiling, path_name: int, path_buff: int, path_buffsize: int):
     pathname = ql.os.utils.read_cstring(path_name)
     # pathname = str(pathname, 'utf-8', errors="ignore")
-    real_path = ql.os.path.transform_to_link_path(pathname)
-    relative_path = ql.os.path.transform_to_relative_path(pathname)
+    host_path = ql.os.path.virtual_to_host_path(pathname)
+    virt_path = ql.os.path.virtual_abspath(pathname)
 
-    if not os.path.exists(real_path):
-        regreturn = -1
+    # cover procfs psaudo files first
+    # TODO: /proc/self/root, /proc/self/cwd
+    if virt_path == r'/proc/self/exe':
+        p = ql.os.path.host_to_virtual_path(ql.path)
+        p = p.encode('utf-8')
 
-    elif relative_path == r'/proc/self/exe':
-        localpath = os.path.abspath(ql.path)
-        localpath = bytes(localpath, 'utf-8') + b'\x00'
+        ql.mem.write(path_buff, p + b'\x00')
+        regreturn = len(p)
 
-        ql.mem.write(path_buff, localpath)
-        regreturn = len(localpath) - 1
-
-    else:
+    elif os.path.exists(host_path):
         regreturn = 0
 
-    ql.log.debug("readlink(%s, 0x%x, 0x%x) = %d" % (relative_path, path_buff, path_buffsize, regreturn))
+    else:
+        regreturn = -1
+
+    ql.log.debug('readlink("%s", 0x%x, 0x%x) = %d' % (virt_path, path_buff, path_buffsize, regreturn))
 
     return regreturn
 
@@ -376,16 +374,17 @@ def ql_syscall_getcwd(ql: Qiling, path_buff: int, path_buffsize: int):
 
 def ql_syscall_chdir(ql: Qiling, path_name: int):
     pathname = ql.os.utils.read_cstring(path_name)
-    real_path = ql.os.path.transform_to_real_path(pathname)
-    relative_path = ql.os.path.transform_to_relative_path(pathname)
+    host_path = ql.os.path.virtual_to_host_path(pathname)
+    virt_path = ql.os.path.virtual_abspath(pathname)
+
+    if os.path.exists(host_path) and os.path.isdir(host_path):
+        ql.os.path.cwd = virt_path
 
-    if os.path.exists(real_path) and os.path.isdir(real_path):
-        ql.os.path.cwd = relative_path
         regreturn = 0
-        ql.log.debug("chdir(%s) = %d"% (relative_path, regreturn))
+        ql.log.debug("chdir(%s) = %d"% (virt_path, regreturn))
     else:
         regreturn = -1
-        ql.log.warning("chdir(%s) = %d : not found" % (relative_path, regreturn))
+        ql.log.warning("chdir(%s) = %d : not found" % (virt_path, regreturn))
 
     return regreturn
 
@@ -393,20 +392,25 @@ def ql_syscall_chdir(ql: Qiling, path_name: int):
 def ql_syscall_readlinkat(ql: Qiling, dfd: int, path: int, buf: int, bufsize: int):
     pathname = ql.os.utils.read_cstring(path)
     # pathname = str(pathname, 'utf-8', errors="ignore")
-    real_path = ql.os.path.transform_to_link_path(pathname)
-    relative_path = ql.os.path.transform_to_relative_path(pathname)
+    host_path = ql.os.path.virtual_to_host_path(pathname)
+    virt_path = ql.os.path.virtual_abspath(pathname)
 
-    if not os.path.exists(real_path):
+    # cover procfs psaudo files first
+    # TODO: /proc/self/root, /proc/self/cwd
+    if virt_path == r'/proc/self/exe':
+        p = ql.os.path.host_to_virtual_path(ql.path)
+        p = p.encode('utf-8')
+
+        ql.mem.write(buf, p + b'\x00')
+        regreturn = len(p)
+
+    elif os.path.exists(host_path):
+        regreturn = 0
+
+    else:
         regreturn = -1
 
-    elif relative_path == r'/proc/self/exe':
-        localpath = os.path.abspath(ql.path)
-        localpath = bytes(localpath, 'utf-8') + b'\x00'
-
-        ql.mem.write(buf, localpath)
-        regreturn = len(localpath) -1
-    else:
-        regreturn = 0
+    ql.log.debug('readlinkat(%d, "%s", 0x%x, 0x%x) = %d' % (dfd, virt_path, buf, bufsize, regreturn))
 
     return regreturn
 

qiling/os/struct.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+import ctypes
+
+from contextlib import contextmanager
+from functools import lru_cache
+from typing import TYPE_CHECKING, Any, Iterator, Type, TypeVar, Optional
+
+if TYPE_CHECKING:
+    from qiling.os.memory import QlMemoryManager
+
+
+class BaseStruct(ctypes.LittleEndianStructure):
+    """An abstract class for C structures.
+    """
+
+    T = TypeVar('T', bound='BaseStruct')
+
+    def save_to(self, mem: QlMemoryManager, address: int) -> None:
+        """Store structure contents to a specified memory address.
+
+        Args:
+            mem: memory manager instance
+            address: destination address
+        """
+
+        data = bytes(self)
+
+        mem.write(address, data)
+
+    @classmethod
+    def load_from(cls: Type[T], mem: QlMemoryManager, address: int) -> T:
+        """Construct and populate a structure from saved contents.
+
+        Args:
+            mem: memory manager instance
+            address: source address
+
+        Returns: populated structure instance
+        """
+
+        data = mem.read(address, cls.sizeof())
+
+        return cls.from_buffer(data)
+
+    @classmethod
+    def volatile_ref(cls: Type[T], mem: QlMemoryManager, address: int) -> T:
+        """Refer to a memory location as a volatile structure variable.
+
+        Args:
+            mem : memory manager instance
+            address : bind address
+
+        Example:
+            >>> class Point(BaseStruct):
+            ...     _fields_ = [
+            ...         ('x', ctypes.c_uint32),
+            ...         ('y', ctypes.c_uint32)
+            ...     ]
+
+            >>> # bind a volatile Point structure to address `ptr`
+            >>> p = Point.volatile_ref(ql.mem, ptr)
+            ... if p.x > 10:    # x value is read directly from memory
+            ...     p.x = 10    # x value is written directly to memory
+            ... # y value in memory remains unchanged
+            >>> 
+        """
+
+        # map all structure field names to their types
+        _fields = dict((fname, ftype) for fname, ftype, *_ in cls._fields_)
+
+        class VolatileStructRef(cls):
+            """Turn a BaseStruct subclass into a volatile structure.
+
+            Field values are never cached: when retrieving a field's value, its value
+            is read from memory and when setting a field's value, its value is flushed
+            to memory.
+
+            This is useful to make sure a structure's fields are alway synced with memory.
+            """
+
+            def __getattribute__(self, name: str) -> Any:
+                # accessing a structure field?
+                if name in _fields:
+                    field = cls.__dict__[name]
+                    ftype = _fields[name]
+
+                    if issubclass(ftype, BaseStruct):
+                        fvalue = ftype.volatile_ref(mem, address + field.offset)
+
+                    else:
+                        # load field's bytes from memory and tranform them into a value
+                        data = mem.read(address + field.offset, field.size)
+                        fvalue = ftype.from_buffer(data)
+
+                        if hasattr(fvalue, 'value'):
+                            fvalue = fvalue.value
+
+                    # set the value to the structure in order to maintain consistency with ctypes.Structure
+                    super().__setattr__(name, fvalue)
+                    return fvalue
+
+                # return attribute value
+                return super().__getattribute__(name)
+
+            def __setattr__(self, name: str, value: Any) -> None:
+                # accessing a structure field?
+                if name in _fields:
+                    field = cls.__dict__[name]
+                    ftype = _fields[name]
+
+                    # transform value into field bytes and write them to memory
+                    fvalue = ftype(*value) if hasattr(ftype, '_length_') else ftype(value)
+                    data = bytes(fvalue)
+
+                    mem.write(address + field.offset, data)
+
+                    # proceed to set the value to the structure in order to maintain consistency with ctypes.Structure
+
+                # set attribute value
+                super().__setattr__(name, value)
+
+
+        return VolatileStructRef()
+
+    @classmethod
+    @contextmanager
+    def ref(cls: Type[T], mem: QlMemoryManager, address: int) -> Iterator[T]:
+        """A structure context manager to facilitate updating structure contents.
+
+        On context enter, a structure is created and populated from the specified memory
+        address. All changes to structure content are written back to memory on context
+        exit. If the structure content has not changed, no memory writes occur.
+
+        Args:
+            mem : memory manager instance
+            address : bind address
+
+        Example:
+            >>> class Point(BaseStruct):
+            ...     _fields_ = [
+            ...         ('x', ctypes.c_uint32),
+            ...         ('y', ctypes.c_uint32)
+            ...     ]
+
+            >>> # bind a Point structure to address `ptr`
+            >>> with Point.ref(ql.mem, ptr) as p:
+            ...     p.x = 10
+            ...     p.y = 20
+            >>> # p data has changed and will be written back to `ptr`
+
+            >>> # bind a Point structure to address `ptr`
+            >>> with Point.ref(ql.mem, ptr) as p:
+            ...     print(f'saved coordinates: {p.x}, {p.y}')
+            >>> # p data has not changed and nothing will be written back
+        """
+
+        instance = cls.load_from(mem, address)
+        orig_data = hash(bytes(instance))
+
+        try:
+            yield instance
+        finally:
+            curr_data = hash(bytes(instance))
+
+            if curr_data != orig_data:
+                instance.save_to(mem, address)
+
+    @classmethod
+    def sizeof(cls) -> int:
+        """Get structure size in bytes.
+        """
+
+        return ctypes.sizeof(cls)
+
+    @classmethod
+    def offsetof(cls, fname: str) -> int:
+        """Get field offset within the structure.
+
+        Args:
+            fname: field name
+
+        Returns: field offset in bytes
+        Raises: `AttributeError` if the specified field does not exist
+        """
+
+        return getattr(cls, fname).offset
+
+    @classmethod
+    def memberat(cls, offset: int) -> Optional[str]:
+        """Get the member name at a given offset.
+
+        Args:
+            offset: field offset within the structure
+
+        Returns: field name, or None if no field starts at the specified offset
+        """
+
+        return next((fname for fname, *_ in cls._fields_ if cls.offsetof(fname) == offset), None)
+
+
+# TODO: replace the lru_cache decorator with functools.cache when moving to Python 3.9
+@lru_cache(maxsize=2)
+def get_aligned_struct(archbits: int) -> Type[BaseStruct]:
+    """Provide an aligned version of BaseStruct based on the underlying
+    architecture properties.
+
+    Args:
+        archbits: required alignment in bits
+    """
+
+    class AlignedStruct(BaseStruct):
+        _pack_ = archbits // 8
+
+    return AlignedStruct
+
+def get_aligned_union(archbits: int):
+    """Provide an aligned union class based on the underlying architecture
+    properties. This class does not inherit the special BaseStruct methods.
+
+    FIXME: ctypes.Union endianess cannot be set arbitrarily, rather it depends
+    on the hosting system. ctypes.LittleEndianUnion and ctypes.BigEndianUnion
+    are available only starting from Python 3.11
+
+    Args:
+        archbits: required alignment in bits
+    """
+
+    class AlignedUnion(ctypes.Union):
+        _pack_ = archbits // 8
+
+    return AlignedUnion
+
+def get_native_type(archbits: int) -> Type[ctypes._SimpleCData]:
+    """Select a ctypes integer type whose size matches the emulated
+    architecture native size.
+    """
+
+    __type = {
+        32 : ctypes.c_uint32,
+        64 : ctypes.c_uint64
+    }
+
+    return __type[archbits]

qiling/os/thread.py

@@ -3,20 +3,20 @@
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 #
 
-from abc import ABCMeta, abstractmethod
+from abc import abstractmethod
 from gevent import Greenlet
-from qiling.const import *
-from .const import *
+
+from qiling import Qiling
 
 class QlThread(Greenlet):
-    __metaclass__=ABCMeta
 
-    def __init__(self, ql):
-        super(QlThread, self).__init__()
+    def __init__(self, ql: Qiling):
+        super().__init__()
+
         self.ql = ql
         self.log_file_fd = None
 
-    def __str__(self):
+    def __str__(self) -> str:
         return f"QlThread {self.get_id()}"
 
 # the common functions which are used in qiling core.

qiling/os/utils.py

@@ -7,12 +7,15 @@
 This module is intended for general purpose functions that are only used in qiling.os
 """
 
-from typing import MutableMapping, Union, Sequence, MutableSequence, Tuple
+from typing import Callable, Iterable, Iterator, List, MutableMapping, Sequence, Tuple, TypeVar, Union
 from uuid import UUID
 
 from qiling import Qiling
 from qiling.const import QL_VERBOSE
 
+# TODO: separate windows-specific implementation
+from qiling.os.windows.structs import make_unicode_string
+
 class QlOsUtils:
 
     ELLIPSIS_PREF = r'__qlva_'
@@ -101,39 +104,118 @@ class QlOsUtils:
         else:
             self.ql.log.info(log)
 
-    def __common_printf(self, format: str, args: MutableSequence, wstring: bool):
-        fmtstr = format.split("%")[1:]
+    def __common_printf(self, format: str, va_args: Iterator[int], wstring: bool) -> Tuple[str, Sequence[int]]:
+        import re
+
+        # https://docs.microsoft.com/en-us/cpp/c-runtime-library/format-specification-syntax-printf-and-wprintf-functions
+        # %[flags][width][.precision][size]type
+        fmtstr = re.compile(r'''%
+            (?P<follows>%|
+                (?P<flags>[-+0 #]+)?
+                (?P<width>[*]|[0-9]+)?
+                (?:.(?P<precision>[*]|[0-9]+))?
+                (?P<size>hh|ll|I32|I64|[hjltwzIL])?
+                (?P<type>[diopuaAcCeEfFgGsSxXZ])
+            )
+        ''', re.VERBOSE)
+
+        T = TypeVar('T')
+
+        def __dup(iterator: Iterator[T], out: List[T]) -> Iterator[T]:
+            """A wrapper iterator to record iterator elements as they are being yielded.
+            """
+
+            for elem in iterator:
+                out.append(elem)
+                yield elem
+
+        repl_args = []  # processed arguments
+        orig_args = []  # original arguments
+
+        va_list = __dup(va_args, orig_args)
+
         read_string = self.read_wstring if wstring else self.read_cstring
 
-        for i, f in enumerate(fmtstr):
-            if f.startswith("s"):
-                args[i] = read_string(args[i])
+        def __repl(m: re.Match) -> str:
+            """Convert printf format string tokens into Python's.
+            """
 
-        out = format.replace(r'%llx', r'%x')
-        out = out.replace(r'%p', r'%#x')
+            if m['follows'] == '%':
+                return '%%'
 
-        return out % tuple(args)
+            else:
+                flags = m['flags'] or ''
 
-    def va_list(self, format: str, ptr: int) -> MutableSequence[int]:
-        count = format.count("%")
+                fill  = ' ' if ' ' in flags else ''
+                align = '<' if '-' in flags else ''
+                sign  = '+' if '+' in flags else ''
+                pound = '#' if '#' in flags else ''
+                zeros = '0' if '0' in flags else ''
 
-        return [self.ql.mem.read_ptr(ptr + i * self.ql.arch.pointersize) for i in range(count)]
+                width = m['width'] or ''
 
-    def sprintf(self, buff: int, format: str, args: MutableSequence, wstring: bool = False) -> int:
-        out = self.__common_printf(format, args, wstring)
+                if width == '*':
+                    width = f'{next(va_list)}'
+
+                prec = m['precision'] or ''
+
+                if prec == '*':
+                    prec = f'{next(va_list)}'
+
+                if prec:
+                    prec = f'.{prec}'
+
+                typ = m['type']
+                arg = next(va_list)
+
+                if typ in 'sS':
+                    typ = 's'
+                    arg = read_string(arg)
+
+                elif typ == 'Z':
+                    # note: ANSI_STRING and UNICODE_STRING have identical layout
+                    ucstr_struct = make_unicode_string(self.ql.arch.bits)
+
+                    with ucstr_struct.ref(self.ql.mem, arg) as ucstr_obj:
+                        typ = 's'
+                        arg = read_string(ucstr_obj.Buffer)
+
+                elif typ == 'p':
+                    pound = '#'
+                    typ = 'x'
+
+                repl_args.append(arg)
+
+                return f'%{fill}{align}{sign}{pound}{zeros}{width}{prec}{typ}'
+
+        out = fmtstr.sub(__repl, format)
+
+        return out % tuple(repl_args), orig_args
+
+    def va_list(self, ptr: int) -> Iterator[int]:
+        while True:
+            yield self.ql.mem.read_ptr(ptr)
+
+            ptr += self.ql.arch.pointersize
+
+    def sprintf(self, buff: int, format: str, va_args: Iterator[int], wstring: bool = False) -> Tuple[int, Callable]:
+        out, args = self.__common_printf(format, va_args, wstring)
         enc = 'utf-16le' if wstring else 'utf-8'
 
         self.ql.mem.write(buff, (out + '\x00').encode(enc))
 
-        return len(out)
+        return len(out), self.__update_ellipsis(args)
 
-    def printf(self, format: str, args: MutableSequence, wstring: bool = False) -> int:
-        out = self.__common_printf(format, args, wstring)
+    def printf(self, format: str, va_args: Iterator[int], wstring: bool = False) -> Tuple[int, Callable]:
+        out, args = self.__common_printf(format, va_args, wstring)
         enc = 'utf-8'
 
         self.ql.os.stdout.write(out.encode(enc))
 
-        return len(out)
+        return len(out), self.__update_ellipsis(args)
 
-    def update_ellipsis(self, params: MutableMapping, args: Sequence) -> None:
-        params.update((f'{QlOsUtils.ELLIPSIS_PREF}{i}', a) for i, a in enumerate(args))
+    def __update_ellipsis(self, args: Iterable[int]) -> Callable[[MutableMapping], None]:
+        def __do_update(params: MutableMapping) -> None:
+            params.update((f'{QlOsUtils.ELLIPSIS_PREF}{i}', a) for i, a in enumerate(args))
+
+        return __do_update

qiling/os/windows/api.py

@@ -100,6 +100,7 @@ LPINTERNET_BUFFERSA         = POINTER
 LPMESSAGEFILTER             = POINTER
 LPMODULEINFO                = POINTER
 LPNLSVERSIONINFO            = POINTER
+LPOSVERSIONINFOEXA          = POINTER
 LPOSVERSIONINFOEXW          = POINTER
 LPOVERLAPPED                = POINTER
 LPPOINT                     = POINTER
@@ -111,6 +112,7 @@ LPSTR                       = POINTER
 LPSYSTEMTIME                = POINTER
 LPSYSTEM_INFO               = POINTER
 LPTHREAD_START_ROUTINE      = POINTER
+LPTIME_ZONE_INFORMATION     = POINTER
 LPTOP_LEVEL_EXCEPTION_FILTER= POINTER
 LPUNKNOWN                   = POINTER
 LPVOID                      = POINTER

qiling/os/windows/clipboard.py

@@ -1,16 +1,17 @@
 #!/usr/bin/env python3
-# 
+#
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 #
 # A Simple Windows Clipboard Simulation
 
 from typing import TYPE_CHECKING, Optional
 
+from qiling.os.windows.const import *
+
 if TYPE_CHECKING:
     from qiling.os.windows.windows import QlOsWindows
 
 NOT_LOCKED = -1
-ERROR_CLIPBOARD_NOT_OPEN = 0x58a
 
 class Clipboard:
 
@@ -19,9 +20,12 @@ class Clipboard:
         self.data = b"Default Clipboard Data"
         self.os = os
 
-        # Valid formats taken from https://doxygen.reactos.org/d8/dd6/base_
-        # 2applications_2mstsc_2constants_8h_source.html
-        self.formats = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 128, 129, 130, 131, 142, 512, 767, 768, 1023]
+        self.formats = [
+            CF_TEXT, CF_BITMAP, CF_METAFILEPICT, CF_SYLK, CF_DIF, CF_TIFF, CF_OEMTEXT, CF_DIB,
+            CF_PALETTE, CF_PENDATA, CF_RIFF, CF_WAVE, CF_UNICODETEXT, CF_ENHMETAFILE, CF_HDROP,
+            CF_LOCALE, CF_MAX, CF_OWNERDISPLAY, CF_DSPTEXT, CF_DSPBITMAP, CF_DSPMETAFILEPICT,
+            CF_DSPENHMETAFILE, CF_PRIVATEFIRST, CF_PRIVATELAST, CF_GDIOBJFIRST, CF_GDIOBJLAST
+        ]
 
     def open(self, h_wnd: int) -> bool:
         """Lock clipboard to hWnd if not already locked.
@@ -50,23 +54,35 @@ class Clipboard:
     def format_available(self, fmt: int) -> bool:
         return fmt in self.formats
 
-    def set_data(self, fmt: int, data: bytes) -> int:
-        if fmt not in self.formats:
-            return 0
-
+    def set_data(self, format: int, address: int) -> int:
         hWnd = self.os.thread_manager.cur_thread.id
 
         if self.locked_by != hWnd:
             self.os.last_error = ERROR_CLIPBOARD_NOT_OPEN
             return 0
 
-        self.data = data
+        def __handle_text(a: int) -> bytes:
+            return self.os.utils.read_cstring(a).encode()
 
-        # BUG: this should be the handle of the clipboard object
-        return 1
+        def __handle_uctext(a: int) -> bytes:
+            return self.os.utils.read_wstring(a).encode()
 
-    def get_data(self, fmt: int) -> Optional[bytes]:
-        if fmt not in self.formats:
+        # TODO: support more clipboard formats
+        format_handlers = {
+            CF_TEXT        : __handle_text,
+            CF_UNICODETEXT : __handle_uctext
+        }
+
+        if format not in format_handlers:
+            return 0
+
+        self.data = format_handlers[format](address)
+
+        # TODO: should create a handle for the clipboard object?
+        return address
+
+    def get_data(self, format: int) -> Optional[bytes]:
+        if format not in self.formats:
             return None
 
         hWnd = self.os.thread_manager.cur_thread.id

qiling/os/windows/const.py

@@ -12,14 +12,20 @@ ERROR_FILE_NOT_FOUND = 0x2
 ERROR_PATH_NOT_FOUND = 0x3
 ERROR_ACCESS_DENIED = 0x5
 ERROR_INVALID_HANDLE = 0x6
+ERROR_GEN_FAILURE = 0x1f
+ERROR_FILE_EXISTS = 0x50
 ERROR_INVALID_PARAMETER = 0x57
 ERROR_BUFFER_OVERFLOW = 0x6F
 ERROR_INSUFFICIENT_BUFFER = 0x7A
+ERROR_BAD_PATHNAME = 0xa1
 ERROR_ALREADY_EXISTS = 0xB7
 ERROR_MORE_DATA = 0xEA
 ERROR_NO_MORE_ITEMS = 0x103
+ERROR_DIRECTORY = 0x10b
 ERROR_NOT_OWNER = 0x120
-ERROR_OLD_WIN_VERSION = 0X47E
+ERROR_NO_UNICODE_TRANSLATION = 0x459
+ERROR_OLD_WIN_VERSION = 0x47E
+ERROR_CLIPBOARD_NOT_OPEN = 0x58a
 # ...
 
 # https://docs.microsoft.com/en-us/openspecs/windows_protocols/ms-erref/596a1078-e883-4972-9bbc-49e60bebca55
@@ -35,25 +41,12 @@ STATUS_NO_YIELD_PERFORMED = 0x40000024
 
 INVALID_HANDLE_VALUE = -1
 
-STD_INPUT_HANDLE = 0xfffffff6
-STD_OUTPUT_HANDLE = 0xfffffff5
-STD_ERROR_HANDLE = 0xfffffff4
+STD_INPUT_HANDLE  = 0xfffffff6  # -10
+STD_OUTPUT_HANDLE = 0xfffffff5  # -11
+STD_ERROR_HANDLE  = 0xfffffff4  # -12
 
 # Registry Type
 
-# Predefined Keys
-REG_KEYS = {
-    0x80000000: "HKEY_CLASSES_ROOT",
-    0x80000005: "HKEY_CURRENT_CONFIG",
-    0x80000001: "HKEY_CURRENT_USER",
-    0x80000007: "HKEY_CURRENT_USER_LOCAL_SETTINGS",
-    0x80000002: "HKEY_LOCAL_MACHINE",
-    0x80000004: "HKEY_PERFORMANCE_DATA",
-    0x80000060: "HKEY_PERFORMANCE_NLSTEXT",
-    0x80000050: "HKEY_PERFORMANCE_TEXT",
-    0x80000003: "HKEY_USERS"
-}
-
 REG_TYPES = {
     "REG_NONE": Registry.RegNone,
     "REG_SZ": Registry.RegSZ,
@@ -611,17 +604,23 @@ PF_XSAVE_ENABLED = 0x17
 # ...
 
 
+# documented PROCESSINFOCLASS values
 # https://docs.microsoft.com/en-us/windows/win32/procthread/zwqueryinformationprocess
-# https://www.pinvoke.net/default.aspx/ntdll/PROCESSINFOCLASS.html
 ProcessBasicInformation = 0
 ProcessDebugPort = 7
-ProcessExecuteFlags = 0x22
 ProcessWow64Information = 26
 ProcessImageFileName = 27
 ProcessBreakOnTermination = 29
 ProcessProtectionInformation = 61
-ProcessDebugObjectHandle = 0x1E
-ProcessDebugFlags = 0x1F
+
+# more PROCESSINFOCLASS values
+# https://www.pinvoke.net/default.aspx/ntdll/PROCESSINFOCLASS.html
+ProcessDebugObjectHandle = 30
+ProcessDebugFlags = 31
+ProcessExecuteFlags = 34
+ProcessImageInformation = 37
+ProcessMitigationPolicy = 52
+ProcessFaultInformation = 63
 
 # https://www.geoffchappell.com/studies/windows/km/ntoskrnl/api/ps/psquery/class.htm
 ThreadBasicInformation = 0x0
@@ -632,26 +631,79 @@ ThreadHideFromDebugger = 0x11
 # ...
 
 
+# message box types:
 # https://docs.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-messageboxw
 
-MB_ABORTRETRYIGNORE = 0x000000020
-MB_CANCELTRYCONTINUE = 0x000000060
-MB_HELP = 0x000040000
-MB_OK = 0x000000000
-MB_OKCANCEL = 0x000000010
-MB_RETRYCANCEL = 0x000000050
-MB_YESNO = 0x000000040
-MB_YESNOCANCEL = 0x000000030
+MB_OK                = 0x00000000
+MB_OKCANCEL          = 0x00000001
+MB_ABORTRETRYIGNORE  = 0x00000002
+MB_YESNOCANCEL       = 0x00000003
+MB_YESNO             = 0x00000004
+MB_RETRYCANCEL       = 0x00000005
+MB_CANCELTRYCONTINUE = 0x00000006
 
-IDABORT = 3
-IDCANCEL = 2
-IDCONTINUE = 11
-IDIGNORE = 5
-IDNO = 7
-IDOK = 1
-IDRETRY = 4
+MB_ICONERROR       = 0x00000010
+MB_ICONQUESTION    = 0x00000020
+MB_ICONWARNING     = 0x00000030
+MB_ICONINFORMATION = 0x00000040
+
+MB_DEFBUTTON1 = 0x00000000
+MB_DEFBUTTON2 = 0x00000100
+MB_DEFBUTTON3 = 0x00000200
+MB_DEFBUTTON4 = 0x00000300
+
+MB_APPLMODAL   = 0x00000000
+MB_SYSTEMMODAL = 0x00001000
+MB_TASKMODAL   = 0x00002000
+MB_HELP        = 0x00004000
+
+MB_SETFOREGROUND        = 0x00010000
+MB_DEFAULT_DESKTOP_ONLY = 0x00020000
+MB_TOPMOST              = 0x00040000
+MB_RIGHT                = 0x00080000
+MB_RTLREADING           = 0x00100000
+MB_SERVICE_NOTIFICATION = 0x00200000
+
+# message box responses:
+IDOK       = 1
+IDCANCEL   = 2
+IDABORT    = 3
+IDRETRY    = 4
+IDIGNORE   = 5
+IDYES      = 6
+IDNO       = 7
 IDTRYAGAIN = 10
-IDYES = 6
+IDCONTINUE = 11
+
+# clipboard data types
+# https://doxygen.reactos.org/d8/dd6/base_2applications_2mstsc_2constants_8h_source.html
+
+CF_TEXT            = 1
+CF_BITMAP          = 2
+CF_METAFILEPICT    = 3
+CF_SYLK            = 4
+CF_DIF             = 5
+CF_TIFF            = 6
+CF_OEMTEXT         = 7
+CF_DIB             = 8
+CF_PALETTE         = 9
+CF_PENDATA         = 10
+CF_RIFF            = 11
+CF_WAVE            = 12
+CF_UNICODETEXT     = 13
+CF_ENHMETAFILE     = 14
+CF_HDROP           = 15
+CF_LOCALE          = 16
+CF_MAX             = 17
+CF_OWNERDISPLAY    = 128
+CF_DSPTEXT         = 129
+CF_DSPBITMAP       = 130
+CF_DSPMETAFILEPICT = 131
+CF_DSPENHMETAFILE  = 142
+CF_PRIVATEFIRST    = 512
+CF_PRIVATELAST     = 767
+CF_GDIOBJFIRST     = 768
+CF_GDIOBJLAST      = 1023
 
 # https://docs.microsoft.com/en-us/windows/win32/api/libloaderapi/nf-libloaderapi-setdefaultdlldirectories
 
@@ -709,3 +761,26 @@ OF_SHARE_EXCLUSIVE  = 0x00000010
 OF_SHARE_DENY_WRITE = 0x00000020
 OF_SHARE_DENY_READ  = 0x00000030
 OF_SHARE_DENY_NONE  = 0x00000040
+
+# code pages identifiers
+CP_ACP = 0
+CP_OEMCP = 1
+CP_THREAD_ACP = 3
+CP_UTF16 = 1200
+CP_UTF16BE = 1201
+CP_ASCII = 20127
+CP_UTF7 = 65000
+CP_UTF8 = 65001
+
+# conversion types
+WC_DISCARDNS         = 0x0010
+WC_SEPCHARS          = 0x0020
+WC_DEFAULTCHAR       = 0x0040
+WC_ERR_INVALID_CHARS = 0x0080
+WC_COMPOSITECHECK    = 0x0200
+WC_NO_BEST_FIT_CHARS = 0x0400
+
+TIME_ZONE_ID_INVALID  = -1
+TIME_ZONE_ID_UNKNOWN  = 0
+TIME_ZONE_ID_STANDARD = 1
+TIME_ZONE_ID_DAYLIGHT = 2

qiling/os/windows/dlls/advapi32.py

@@ -3,6 +3,8 @@
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 #
 
+from typing import Callable, Tuple
+
 from qiling import Qiling
 from qiling.os.windows.api import *
 from qiling.os.windows.const import *
@@ -14,27 +16,35 @@ def __RegOpenKey(ql: Qiling, address: int, params):
     lpSubKey = params["lpSubKey"]
     phkResult = params["phkResult"]
 
-    if hKey not in REG_KEYS:
+    handle = ql.os.handle_manager.get(hKey)
+
+    if handle is None or not handle.name.startswith('HKEY'):
         return ERROR_FILE_NOT_FOUND
 
-    s_hKey = REG_KEYS[hKey]
-    key = s_hKey + "\\" + lpSubKey
+    params["hKey"] = handle.name
 
-    # Keys in the profile are saved as KEY\PARAM = VALUE, so i just want to check that the key is the same
-    keys_profile = [key.rsplit("\\", 1)[0] for key in ql.os.profile["REGISTRY"].keys()]
-    if key.lower() in keys_profile:
-        ql.log.debug("Using profile for key of  %s" % key)
-        ql.os.registry_manager.access(key)
-    else:
-        if not ql.os.registry_manager.exists(key):
-            ql.log.debug("Value key %s not present" % key)
-            return ERROR_FILE_NOT_FOUND
+    if lpSubKey:
+        key = f'{handle.name}\\{lpSubKey}'
+
+        # Keys in the profile are saved as KEY\PARAM = VALUE, so i just want to check that the key is the same
+        keys_profile = [entry.casefold() for entry in ql.os.profile["REGISTRY"].keys()]
+
+        if key.casefold() in keys_profile:
+            ql.log.debug("Using profile for key of  %s" % key)
+            ql.os.registry_manager.access(key)
+
+        else:
+            if not ql.os.registry_manager.exists(key):
+                ql.log.debug("Value key %s not present" % key)
+                return ERROR_FILE_NOT_FOUND
+
+        # new handle
+        handle = Handle(obj=key)
+        ql.os.handle_manager.append(handle)
+
+    if phkResult:
+        ql.mem.write_ptr(phkResult, handle.id)
 
-    # new handle
-    new_handle = Handle(obj=key)
-    ql.os.handle_manager.append(new_handle)
-    if phkResult != 0:
-        ql.mem.write_ptr(phkResult, new_handle.id)
     return ERROR_SUCCESS
 
 def __RegQueryValue(ql: Qiling, address: int, params, wstring: bool):
@@ -84,35 +94,34 @@ def __RegQueryValue(ql: Qiling, address: int, params, wstring: bool):
     return ret
 
 def __RegCreateKey(ql: Qiling, address: int, params):
-    ret = ERROR_SUCCESS
-
     hKey = params["hKey"]
     lpSubKey = params["lpSubKey"]
     phkResult = params["phkResult"]
 
-    if hKey not in REG_KEYS:
+    handle = ql.os.handle_manager.get(hKey)
+
+    if handle is None or not handle.name.startswith('HKEY'):
         return ERROR_FILE_NOT_FOUND
 
-    s_hKey = REG_KEYS[hKey]
-    params["hKey"] = s_hKey
+    params["hKey"] = handle.name
 
-    keyname = f'{s_hKey}\\{lpSubKey}'
+    if lpSubKey:
+        keyname = f'{handle.name}\\{lpSubKey}'
 
-    if not ql.os.registry_manager.exists(keyname):
-        ql.os.registry_manager.create(keyname)
-        ret = ERROR_SUCCESS
+        if not ql.os.registry_manager.exists(keyname):
+            ql.os.registry_manager.create(keyname)
 
-    # new handle
-    if ret == ERROR_SUCCESS:
-        new_handle = Handle(obj=keyname)
-        ql.os.handle_manager.append(new_handle)
-        if phkResult != 0:
-            ql.mem.write_ptr(phkResult, new_handle.id)
-    else:
-        # elicn: is this even reachable?
-        new_handle = 0
+        handle = ql.os.handle_manager.search_by_obj(keyname)
 
-    return ret
+        # make sure we have a handle for this keyname
+        if handle is None:
+            handle = Handle(obj=keyname)
+            ql.os.handle_manager.append(handle)
+
+    if phkResult:
+        ql.mem.write_ptr(phkResult, handle.id)
+
+    return ERROR_SUCCESS
 
 def __RegSetValue(ql: Qiling, address: int, params, wstring: bool):
     hKey = params["hKey"]
@@ -458,11 +467,17 @@ def hook_GetTokenInformation(ql: Qiling, address: int, params):
     TokenInformationLength = params["TokenInformationLength"]
     ReturnLength = params["ReturnLength"]
 
-    token = ql.os.handle_manager.get(TokenHandle).obj
+    handle = ql.os.handle_manager.get(TokenHandle)
+
+    if handle is None:
+        ql.os.last_error = ERROR_INVALID_HANDLE
+        return 0
+
+    token = handle.obj
     information_value = token.get(TokenInformationClass)
 
-    ql.mem.write_ptr(ReturnLength, len(information_value), 4)
-    return_size = ql.mem.read_ptr(ReturnLength, 4)
+    return_size = len(information_value)
+    ql.mem.write_ptr(ReturnLength, return_size, 4)
 
     ql.log.debug("The target is checking for its permissions")
 
@@ -483,10 +498,12 @@ def hook_GetTokenInformation(ql: Qiling, address: int, params):
     'pSid' : PSID
 })
 def hook_GetSidSubAuthorityCount(ql: Qiling, address: int, params):
-    sid = ql.os.handle_manager.get(params["pSid"]).obj
-    addr_authority_count = sid.addr + 1  # +1 because the first byte is revision
+    pSid = params['pSid']
 
-    return addr_authority_count
+    # SID address is used as its handle id
+    sid = ql.os.handle_manager.get(pSid).obj
+
+    return pSid + sid.offsetof('SubAuthorityCount')
 
 # PDWORD GetSidSubAuthority(
 #   PSID  pSid,
@@ -497,11 +514,13 @@ def hook_GetSidSubAuthorityCount(ql: Qiling, address: int, params):
     'nSubAuthority' : DWORD
 })
 def hook_GetSidSubAuthority(ql: Qiling, address: int, params):
-    num = params["nSubAuthority"]
-    sid = ql.os.handle_manager.get(params["pSid"]).obj
-    addr_authority = sid.addr + 8 + (ql.arch.pointersize * num)
+    pSid = params['pSid']
+    nSubAuthority = params['nSubAuthority']
 
-    return addr_authority
+    # SID address is used as its handle id
+    sid = ql.os.handle_manager.get(pSid).obj
+
+    return pSid + sid.offsetof('SubAuthority') + (4 * nSubAuthority)
 
 # LSTATUS RegEnumValueA(
 #   HKEY    hKey,
@@ -664,69 +683,91 @@ def hook_StartServiceA(ql: Qiling, address: int, params):
 })
 def hook_AllocateAndInitializeSid(ql: Qiling, address: int, params):
     count = params["nSubAuthorityCount"]
-    subs = b''.join(ql.pack32(params[f'nSubAuthority{i}']) for i in range(count))
+    subauths = tuple(params[f'nSubAuthority{i}'] for i in range(count))
 
-    sid = Sid(ql, revision=1, subs_count=count, identifier=5, subs=subs)
-    sid_addr = ql.os.heap.alloc(sid.size)
-    sid.write(sid_addr)
+    sid_struct = make_sid(auth_count=len(subauths))
+    sid_addr = ql.os.heap.alloc(sid_struct.sizeof())
 
-    handle = Handle(obj=sid, id=sid_addr)
+    sid_obj = sid_struct(
+        Revision = 1,
+        SubAuthorityCount = len(subauths),
+        IdentifierAuthority = (5,),
+        SubAuthority = subauths
+    )
+
+    sid_obj.save_to(ql.mem, sid_addr)
+
+    handle = Handle(obj=sid_obj, id=sid_addr)
     ql.os.handle_manager.append(handle)
+
     dest = params["pSid"]
     ql.mem.write_ptr(dest, sid_addr)
 
     return 1
 
-# Some default Sids:
-__adminsid = None # Administrators (S-1-5-32-544)
-__userssid = None # All Users (S-1-5-32-545)
-__guestssid = None # All Users (S-1-5-32-546)
-__poweruserssid = None # Power Users (S-1-5-32-547)
+def __create_default_sid(ql: Qiling, subauths: Tuple[int, ...]):
+    sid_struct = make_sid(auth_count=len(subauths))
 
+    sid_obj = sid_struct(
+        Revision = 1,
+        SubAuthorityCount = len(subauths),
+        IdentifierAuthority = (5,),
+        SubAuthority = tuple(subauths)
+    )
 
-def get_adminsid(ql):
-    global __adminsid
+    return sid_obj
 
-    if __adminsid is None:
-        # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20]
-        # nSubAuthority1 = DOMAIN_ALIAS_RID_ADMINS[0x220]
-        subs = b"\x20\x00\x00\x00\x20\x02\x00\x00"
-        __adminsid = Sid(ql, revision=1, identifier=5, subs=subs, subs_count=2)
+def singleton(func: Callable):
+    """A decorator for functions that produce singleton objects.
 
-    return __adminsid
+    When a decorated function is called for the first time, its
+    singleton object will be created. The same object will be returned
+    on all consequent calls regardless of the passed arguments (if any).
+    """
 
-def get_userssid(ql):
-    global __userssid
+    __singleton = None
 
-    if __userssid is None:
-        # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20]
-        # nSubAuthority1 = DOMAIN_ALIAS_RID_USERS[0x221]
-        subs = b"\x20\x00\x00\x00\x21\x02\x00\x00"
-        __userssid = Sid(ql, revision=1, identifier=5, subs=subs, subs_count=2)
+    def wrapper(*args, **kwargs):
+        nonlocal __singleton
 
-    return __userssid
+        if __singleton is None:
+            __singleton = func(*args, **kwargs)
 
-def get_guestssid(ql):
-    global __guestssid
+        return __singleton
 
-    if __guestssid is None:
-        # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20]
-        # nSubAuthority1 = DOMAIN_ALIAS_RID_GUESTS[0x222]
-        subs = b"\x20\x00\x00\x00\x22\x02\x00\x00"
-        __guestssid = Sid(ql, revision=1, identifier=5, subs=subs, subs_count=2)
+    return wrapper
 
-    return __guestssid
+# Administrators (S-1-5-32-544)
+@singleton
+def __admin_sid(ql: Qiling):
+    # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20]
+    # nSubAuthority1 = DOMAIN_ALIAS_RID_ADMINS[0x220]
 
-def get_poweruserssid(ql):
-    global __poweruserssid
+    return __create_default_sid(ql, (0x20, 0x220))
 
-    if __poweruserssid is None:
-        # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20]
-        # nSubAuthority1 = DOMAIN_ALIAS_RID_POWER_USERS[0x223]
-        subs = b"\x20\x00\x00\x00\x23\x02\x00\x00"
-        __poweruserssid = Sid(ql, revision=1, identifier=5, subs=subs, subs_count=2)
+# All Users (S-1-5-32-545)
+@singleton
+def __users_sid(ql: Qiling):
+    # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20]
+    # nSubAuthority1 = DOMAIN_ALIAS_RID_USERS[0x221]
 
-    return __poweruserssid
+    return __create_default_sid(ql, (0x20, 0x221))
+
+# All Users (S-1-5-32-546)
+@singleton
+def __guests_sid(ql: Qiling):
+    # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20]
+    # nSubAuthority1 = DOMAIN_ALIAS_RID_GUESTS[0x222]
+
+    return __create_default_sid(ql, (0x20, 0x222))
+
+# Power Users (S-1-5-32-547)
+@singleton
+def __powerusers_sid(ql: Qiling):
+    # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20]
+    # nSubAuthority1 = DOMAIN_ALIAS_RID_POWER_USERS[0x223]
+
+    return __create_default_sid(ql, (0x20, 0x223))
 
 
 # BOOL WINAPI CheckTokenMembership(
@@ -740,24 +781,32 @@ def get_poweruserssid(ql):
     'IsMember'    : PBOOL
 })
 def hook_CheckTokenMembership(ql: Qiling, address: int, params):
-    token_handle = params["TokenHandle"]
-    sid = ql.os.handle_manager.get(params["SidToCheck"]).obj
+    TokenHandle = params['TokenHandle']
+    SidToCheck = params['SidToCheck']
+
+    sid = ql.os.handle_manager.get(SidToCheck).obj
+    IsMember = False
+
     # If TokenHandle is NULL, CheckTokenMembership uses the impersonation token of the calling thread.
-    IsMember = 0
-    if token_handle == 0:
+    if not TokenHandle:
         # For now, treat power users as admins
-        if get_adminsid(ql) == sid or get_poweruserssid(ql) == sid:
-            IsMember = 1 if ql.os.profile["SYSTEM"]["permission"] == "root" else 0
-        elif get_userssid(ql) == sid:
+        if __admin_sid(ql) == sid or __powerusers_sid(ql) == sid:
+            IsMember = ql.os.profile["SYSTEM"]["permission"] == "root"
+
+        elif __users_sid(ql) == sid:
             # FIXME: is this true for all tokens? probably not...
-            IsMember = 1
-        elif get_guestssid(ql) == sid:
-            IsMember = 0
+            IsMember = True
+
+        elif __guests_sid(ql) == sid:
+            IsMember = False
+
         else:
-            assert False, 'unimplemented'
+            raise NotImplementedError
     else:
-        assert False, 'unimplemented'
-    ql.mem.write_ptr(params['IsMember'], IsMember)
+        raise NotImplementedError
+
+    ql.mem.write_ptr(params['IsMember'], int(IsMember))
+
     return 1
 
 
@@ -768,6 +817,7 @@ def hook_CheckTokenMembership(ql: Qiling, address: int, params):
     'pSid' : PSID
 })
 def hook_FreeSid(ql: Qiling, address: int, params):
+    # TODO: should also remove from ql.os.handle_manager ?
     ql.os.heap.free(params["pSid"])
 
     return 0

qiling/os/windows/dlls/kernel32/__init__.py

@@ -23,3 +23,4 @@ from .consoleapi2 import *
 from .consoleapi3 import *
 from .debugapi import *
 from .psapi import *
+from .timezoneapi import *

qiling/os/windows/dlls/kernel32/debugapi.py

@@ -8,10 +8,17 @@ from qiling.os.windows.api import *
 from qiling.os.windows.const import *
 from qiling.os.windows.fncc import *
 
+def __is_debugger_present(ql: Qiling) -> int:
+    """Read PEB.BeingDebugger fied to determine whether debugger
+    is present or not.
+    """
+
+    return ql.loader.PEB.BeingDebugged
+
 # BOOL IsDebuggerPresent();
 @winsdkapi(cc=STDCALL, params={})
 def hook_IsDebuggerPresent(ql: Qiling, address: int, params):
-    return 0
+    return __is_debugger_present(ql)
 
 # BOOL CheckRemoteDebuggerPresent(
 #   HANDLE hProcess,
@@ -22,9 +29,10 @@ def hook_IsDebuggerPresent(ql: Qiling, address: int, params):
     'pbDebuggerPresent' : PBOOL
 })
 def hook_CheckRemoteDebuggerPresent(ql: Qiling, address: int, params):
-    pointer = params["pbDebuggerPresent"]
+    pbDebuggerPresent = params['pbDebuggerPresent']
 
-    ql.mem.write(pointer, b'\x00')
+    res = __is_debugger_present(ql)
+    ql.mem.write(pbDebuggerPresent, ql.pack8(res))
 
     return 1
 

qiling/os/windows/dlls/kernel32/errhandlingapi.py

@@ -22,10 +22,13 @@ def hook_SetUnhandledExceptionFilter(ql: Qiling, address: int, params):
     if handle is None:
         handle = Handle(name="TopLevelExceptionHandler", obj=addr)
         ql.os.handle_manager.append(handle)
+        prev_filter = 0
+
     else:
+        prev_filter = handle.obj
         handle.obj = addr
 
-    return 0
+    return prev_filter
 
 # _Post_equals_last_error_ DWORD GetLastError();
 @winsdkapi(cc=STDCALL, params={})
@@ -73,11 +76,19 @@ def hook_SetErrorMode(ql: Qiling, address: int, params):
     'lpArguments'        : POINTER
 })
 def hook_RaiseException(ql: Qiling, address: int, params):
-    func_addr = ql.os.handle_manager.search("TopLevelExceptionHandler").obj
+    nNumberOfArguments = params['nNumberOfArguments']
+    lpArguments = params['lpArguments']
 
-    ql.os.fcall.call_native(func_addr, [], None)
+    handle = ql.os.handle_manager.search("TopLevelExceptionHandler")
 
-    return 0
+    if handle is None:
+        ql.log.warning(f'RaiseException: top level exception handler not found')
+        return
+
+    exception_handler = handle.obj
+    args = [(PARAM_INTN, ql.mem.read_ptr(lpArguments + i * ql.arch.pointersize)) for i in range(nNumberOfArguments)] if lpArguments else []
+
+    ql.os.fcall.call_native(exception_handler, args, None)
 
 # PVOID AddVectoredExceptionHandler(
 #   ULONG                       First,

qiling/os/windows/dlls/kernel32/fileapi.py

@@ -15,7 +15,7 @@ from qiling.os.windows.api import *
 from qiling.os.windows.const import *
 from qiling.os.windows.fncc import *
 from qiling.os.windows.handle import Handle
-from qiling.os.windows.structs import Win32FindData
+from qiling.os.windows.structs import FILETIME, make_win32_find_data
 
 # DWORD GetFileType(
 #   HANDLE hFile
@@ -46,50 +46,67 @@ def hook_FindFirstFileA(ql: Qiling, address: int, params):
     filename = params['lpFileName']
     pointer = params['lpFindFileData']
 
-    if filename == 0:
+    if not filename:
         return INVALID_HANDLE_VALUE
-    elif len(filename) >= MAX_PATH:
+
+    if len(filename) >= MAX_PATH:
         return ERROR_INVALID_PARAMETER
 
-    target_dir = os.path.join(ql.rootfs, filename.replace("\\", os.sep))
-    ql.log.info('TARGET_DIR = %s' % target_dir)
-    real_path = ql.os.path.transform_to_real_path(filename)
+    host_path = ql.os.path.virtual_to_host_path(filename)
 
     # Verify the directory is in ql.rootfs to ensure no path traversal has taken place
-    if not os.path.exists(real_path):
+    if not ql.os.path.is_safe_host_path(host_path):
         ql.os.last_error = ERROR_FILE_NOT_FOUND
+
         return INVALID_HANDLE_VALUE
 
     # Check if path exists
     filesize = 0
+
     try:
-        f = ql.os.fs_mapper.open(real_path, "r")
-        filesize = os.path.getsize(real_path)
+        f = ql.os.fs_mapper.open(host_path, "r")
+
+        filesize = os.path.getsize(host_path)
     except FileNotFoundError:
         ql.os.last_error = ERROR_FILE_NOT_FOUND
-        return INVALID_HANDLE_VALUE
 
-    # Get size of the file
-    file_size_low = filesize & 0xffffff
-    file_size_high = filesize >> 32
+        return INVALID_HANDLE_VALUE
 
     # Create a handle for the path
     new_handle = Handle(obj=f)
     ql.os.handle_manager.append(new_handle)
 
-    # Spoof filetime values
-    filetime = ql.pack64(datetime.now().microsecond)
+    # calculate file time
+    epoch = datetime(1601, 1, 1)
+    elapsed = datetime.now() - epoch
 
-    find_data = Win32FindData(
-                ql,
-                FILE_ATTRIBUTE_NORMAL,
-                filetime, filetime, filetime,
-                file_size_high, file_size_low,
-                0, 0,
-                filename,
-                0, 0, 0, 0,)
+    # number of 100-nanosecond intervals since Jan 1, 1601 utc
+    # where: (10 ** 9) / 100 -> (10 ** 7)
+    hnano = int(elapsed.total_seconds() * (10 ** 7))
 
-    find_data.write(pointer)
+    mask = (1 << 32) - 1
+
+    ftime = FILETIME(
+        (hnano >>  0) & mask,
+        (hnano >> 32) & mask
+    )
+
+    fdata_struct = make_win32_find_data(ql.arch.bits, wide=False)
+
+    with fdata_struct.ref(ql.mem, pointer) as fdata_obj:
+        fdata_obj.dwFileAttributes   = FILE_ATTRIBUTE_NORMAL
+        fdata_obj.ftCreationTime     = ftime
+        fdata_obj.ftLastAccessTime   = ftime
+        fdata_obj.ftLastWriteTime    = ftime
+        fdata_obj.nFileSizeHigh      = (filesize >> 32) & mask
+        fdata_obj.nFileSizeLow       = (filesize >>  0) & mask
+        fdata_obj.dwReserved0        = 0
+        fdata_obj.dwReserved1        = 0
+        fdata_obj.cFileName          = filename
+        fdata_obj.cAlternateFileName = 0
+        fdata_obj.dwFileType         = 0
+        fdata_obj.dwCreatorType      = 0
+        fdata_obj.wFinderFlags       = 0
 
     return new_handle.id
 
@@ -190,13 +207,12 @@ def hook_WriteFile(ql: Qiling, address: int, params):
         ql.os.last_error = ERROR_INVALID_HANDLE
         return 0
 
-    fobj = handle.obj
     data = ql.mem.read(lpBuffer, nNumberOfBytesToWrite)
 
-    if hFile == STD_OUTPUT_HANDLE:
+    if hFile in (STD_OUTPUT_HANDLE, STD_ERROR_HANDLE):
         ql.os.stats.log_string(data.decode())
 
-    written = fobj.write(bytes(data))
+    written = handle.obj.write(bytes(data))
     ql.mem.write_ptr(lpNumberOfBytesWritten, written, 4)
 
     return 1
@@ -211,11 +227,10 @@ def _CreateFile(ql: Qiling, address: int, params):
     # hTemplateFile = params["hTemplateFile"]
 
     # access mask DesiredAccess
-    mode = ""
     if dwDesiredAccess & GENERIC_WRITE:
-        mode += "wb"
+        mode = "wb"
     else:
-        mode += "rb"
+        mode = "rb"
 
     try:
         f = ql.os.fs_mapper.open(s_lpFileName, mode)
@@ -270,11 +285,13 @@ def hook_CreateFileA(ql: Qiling, address: int, params):
 def hook_CreateFileW(ql: Qiling, address: int, params):
     return  _CreateFile(ql, address, params)
 
-def _GetTempPath(ql: Qiling, address: int, params, wide: bool):
-    temp_path = ntpath.join(ql.rootfs, 'Windows', 'Temp')
+def _GetTempPath(ql: Qiling, address: int, params, *, wide: bool):
+    vtmpdir = ntpath.join(ql.os.windir, 'Temp')
+    htmpdir = ql.os.path.virtual_to_host_path(vtmpdir)
 
-    if not os.path.exists(temp_path):
-        os.makedirs(temp_path, 0o755)
+    if ql.os.path.is_safe_host_path(htmpdir):
+        if not os.path.exists(htmpdir):
+            os.makedirs(htmpdir, 0o755)
 
     nBufferLength = params['nBufferLength']
     lpBuffer = params['lpBuffer']
@@ -282,7 +299,7 @@ def _GetTempPath(ql: Qiling, address: int, params, wide: bool):
     enc = 'utf-16le' if wide else 'utf-8'
 
     # temp dir path has to end with a path separator
-    tmpdir = f'{ntpath.join(ql.os.windir, "Temp")}{ntpath.sep}'.encode(enc)
+    tmpdir = f'{vtmpdir}{ntpath.sep}'.encode(enc)
     cstr = tmpdir + '\x00'.encode(enc)
 
     if nBufferLength >= len(cstr):
@@ -324,29 +341,26 @@ def hook_GetTempPathA(ql: Qiling, address: int, params):
     'cchBuffer'     : DWORD
 })
 def hook_GetShortPathNameW(ql: Qiling, address: int, params):
-    paths = params["lpszLongPath"].split("\\")
-    dst = params["lpszShortPath"]
-    max_size = params["cchBuffer"]
-    res = paths[0]
+    lpszLongPath = params['lpszLongPath']
+    lpszShortPath = params['lpszShortPath']
+    cchBuffer = params['cchBuffer']
 
-    for path in paths[1:]:
-        nameAndExt = path.split(".")
-        name = nameAndExt[0]
-        ext = "" if len(nameAndExt) == 1 else "." + nameAndExt[1]
+    def __shorten(p: str) -> str:
+        name, ext = ntpath.splitext(p)
 
-        if len(name) > 8:
-            name = name[:6] + "~1"
+        return f'{(name[:6] + "~1") if len(name) > 8 else name}{ext}'
 
-        res += "\\" + name + ext
+    shortpath = ntpath.join(*(__shorten(elem) for elem in lpszLongPath.split(ntpath.sep)))
+    encoded = f'{shortpath}\x00'.encode('utf-16le')
 
-    res += "\x00"
-    res = res.encode("utf-16le")
+    if len(shortpath) > cchBuffer:
+        return len(shortpath) + 1
 
-    if max_size < len(res):
-        return len(res)
+    if lpszShortPath:
+        ql.mem.write(lpszShortPath, encoded)
 
-    ql.mem.write(dst, res)
-    return len(res) - 1
+    # on succes, return chars count excluding null-term
+    return len(shortpath)
 
 
 # BOOL GetVolumeInformationW(
@@ -471,18 +485,20 @@ def hook_GetDiskFreeSpaceW(ql: Qiling, address: int, params):
     'lpSecurityAttributes' : LPSECURITY_ATTRIBUTES
 })
 def hook_CreateDirectoryA(ql: Qiling, address: int, params):
-    path_name = params["lpPathName"]
-    target_dir = os.path.join(ql.rootfs, path_name.replace("\\", os.sep))
-    ql.log.info('TARGET_DIR = %s' % target_dir)
+    lpPathName = params['lpPathName']
 
-    # Verify the directory is in ql.rootfs to ensure no path traversal has taken place
-    real_path = ql.os.path.transform_to_real_path(path_name)
+    dst = ql.os.path.virtual_to_host_path(lpPathName)
 
-    if os.path.exists(real_path):
+    if not ql.os.path.is_safe_host_path(dst):
+        ql.os.last_error = ERROR_GEN_FAILURE
+        return 0
+
+    if os.path.exists(dst):
         ql.os.last_error = ERROR_ALREADY_EXISTS
         return 0
 
-    os.mkdir(real_path)
+    os.mkdir(dst, 0o755)
+
     return 1
 
 # DWORD GetFileSize(
@@ -494,13 +510,20 @@ def hook_CreateDirectoryA(ql: Qiling, address: int, params):
     'lpFileSizeHigh' : LPDWORD
 })
 def hook_GetFileSize(ql: Qiling, address: int, params):
+    hFile = params["hFile"]
+
+    handle = ql.os.handle_manager.get(hFile)
+
+    if handle is None:
+        ql.os.last_error = ERROR_INVALID_HANDLE
+        return -1 # INVALID_FILE_SIZE
+
     try:
-        handle = ql.os.handle_manager.get(params['hFile'])
 
         return os.path.getsize(handle.obj.name)
     except:
         ql.os.last_error = ERROR_INVALID_HANDLE
-        return 0xFFFFFFFF #INVALID_FILE_SIZE
+        return -1 # INVALID_FILE_SIZE
 
 def _CreateFileMapping(ql: Qiling, address: int, params):
     hFile = params['hFile']
@@ -619,14 +642,23 @@ def hook_UnmapViewOfFile(ql: Qiling, address: int, params):
     'bFailIfExists'      : BOOL
 })
 def hook_CopyFileA(ql: Qiling, address: int, params):
-    lpExistingFileName = ql.os.path.transform_to_real_path(params["lpExistingFileName"])
-    lpNewFileName = ql.os.path.transform_to_real_path(params["lpNewFileName"])
-    bFailIfExists = params["bFailIfExists"]
+    lpExistingFileName = params['lpExistingFileName']
+    lpNewFileName = params['lpNewFileName']
+    bFailIfExists = params['bFailIfExists']
 
-    if bFailIfExists and os.path.exists(lpNewFileName):
+    src = ql.os.path.virtual_to_host_path(lpExistingFileName)
+    dst = ql.os.path.virtual_to_host_path(lpNewFileName)
+
+    if not ql.os.path.is_safe_host_path(src) or not ql.os.path.is_safe_host_path(dst):
+        ql.os.last_error = ERROR_GEN_FAILURE
         return 0
 
-    copyfile(lpExistingFileName, lpNewFileName)
+    if bFailIfExists and os.path.exists(dst):
+        ql.os.last_error = ERROR_FILE_EXISTS
+        return 0
+
+    copyfile(src, dst)
+
     return 1
 
 # BOOL SetFileAttributesA(
@@ -663,6 +695,22 @@ def hook_SetFileApisToANSI(ql: Qiling, address: int, params):
 def hook_SetFileApisToOEM(ql: Qiling, address: int, params):
     pass
 
+def _DeleteFile(ql: Qiling, address: int, params):
+    lpFileName = params["lpFileName"]
+
+    dst = ql.os.path.virtual_to_host_path(lpFileName)
+
+    if not ql.os.path.is_safe_host_path(dst):
+        ql.os.last_error = ERROR_GEN_FAILURE
+        return 0
+
+    try:
+        os.remove(dst)
+    except OSError:
+        return 0
+
+    return 1
+
 # BOOL DeleteFileA(
 #   LPCSTR lpFileName
 # );
@@ -670,12 +718,7 @@ def hook_SetFileApisToOEM(ql: Qiling, address: int, params):
     'lpFileName' : LPCSTR
 })
 def hook_DeleteFileA(ql: Qiling, address: int, params):
-    lpFileName = ql.os.path.transform_to_real_path(params["lpFileName"])
-    try:
-        os.remove(lpFileName)
-        return 1
-    except:
-        return 0
+    return _DeleteFile(ql, address, params)
 
 # BOOL DeleteFileW(
 #   LPCWSTR lpFileName
@@ -684,9 +727,4 @@ def hook_DeleteFileA(ql: Qiling, address: int, params):
     'lpFileName' : LPCWSTR
 })
 def hook_DeleteFileW(ql: Qiling, address: int, params):
-    lpFileName = ql.os.path.transform_to_real_path(params["lpFileName"])
-    try:
-        os.remove(lpFileName)
-        return 1
-    except:
-        return 0
+    return _DeleteFile(ql, address, params)

qiling/os/windows/dlls/kernel32/handleapi.py

@@ -48,12 +48,12 @@ def hook_CloseHandle(ql: Qiling, address: int, params):
     if handle is None:
         ql.os.last_error = ERROR_INVALID_HANDLE
         return 0
-    else:
-        if handle.permissions is not None and handle.permissions & HANDLE_FLAG_PROTECT_FROM_CLOSE >= 1:
-            # FIXME: add error
-            return 0
-        else:
-            ql.os.handle_manager.delete(value)
+
+    if handle.permissions is not None and handle.permissions & HANDLE_FLAG_PROTECT_FROM_CLOSE:
+        # FIXME: add error
+        return 0
+
+    ql.os.handle_manager.delete(value)
 
     return 1
 

qiling/os/windows/dlls/kernel32/libloaderapi.py

@@ -18,12 +18,10 @@ def _GetModuleHandle(ql: Qiling, address: int, params):
     if lpModuleName == 0:
         ret = ql.loader.pe_image_address
     else:
-        lpModuleName = lpModuleName.lower()
-
         if not has_lib_ext(lpModuleName):
             lpModuleName = f'{lpModuleName}.dll'
 
-        image = ql.loader.get_image_by_name(lpModuleName)
+        image = ql.loader.get_image_by_name(lpModuleName, casefold=True)
 
         if image:
             ret = image.base
@@ -69,6 +67,35 @@ def hook_GetModuleHandleExW(ql: Qiling, address: int, params):
 
     return res
 
+def __GetModuleFileName(ql: Qiling, address: int, params, *, wide: bool):
+    hModule = params["hModule"]
+    lpFilename = params["lpFilename"]
+    nSize = params["nSize"]
+
+    if not hModule:
+        if ql.code:
+            raise QlErrorNotImplemented('cannot retrieve module file name in shellcode mode')
+
+        hModule = ql.loader.pe_image_address
+
+    hpath = next((image.path for image in ql.loader.images if image.base == hModule), None)
+
+    if hpath is None:
+        ql.os.last_error = ERROR_INVALID_HANDLE
+        return 0
+
+    encname = 'utf-16le' if wide else 'latin'
+    vpath = ql.os.path.host_to_virtual_path(hpath)
+    truncated = vpath[:nSize - 1] + '\x00'
+    encoded = truncated.encode(encname)
+
+    if len(vpath) + 1 > nSize:
+        ql.os.last_error = ERROR_INSUFFICIENT_BUFFER
+
+    ql.mem.write(lpFilename, encoded)
+
+    return min(len(vpath), nSize)
+
 # DWORD GetModuleFileNameA(
 #   HMODULE hModule,
 #   LPSTR   lpFilename,
@@ -80,30 +107,7 @@ def hook_GetModuleHandleExW(ql: Qiling, address: int, params):
     'nSize'      : DWORD
 })
 def hook_GetModuleFileNameA(ql: Qiling, address: int, params):
-    hModule = params["hModule"]
-    lpFilename = params["lpFilename"]
-    nSize = params["nSize"]
-    ret = 0
-
-    # GetModuleHandle can return pe_image_address as handle, and GetModuleFileName will try to retrieve it.
-    # Pretty much 0 and pe_image_address value should do the same operations
-    if not ql.code and (hModule == 0 or hModule == ql.loader.pe_image_address):
-        filename = ql.loader.filepath
-        filename_len = len(filename)
-
-        if filename_len > nSize - 1:
-            filename = ql.loader.filepath[:nSize - 1]
-            ret = nSize
-        else:
-            ret = filename_len
-
-        ql.mem.write(lpFilename, filename + b"\x00")
-
-    else:
-        ql.log.debug("hModule %x" % hModule)
-        raise QlErrorNotImplemented("API not implemented")
-
-    return ret
+    return __GetModuleFileName(ql, address, params, wide=False)
 
 # DWORD GetModuleFileNameW(
 #   HMODULE hModule,
@@ -116,30 +120,7 @@ def hook_GetModuleFileNameA(ql: Qiling, address: int, params):
     'nSize'      : DWORD
 })
 def hook_GetModuleFileNameW(ql: Qiling, address: int, params):
-    hModule = params["hModule"]
-    lpFilename = params["lpFilename"]
-    nSize = params["nSize"]
-    ret = 0
-
-    # GetModuleHandle can return pe_image_address as handle, and GetModuleFileName will try to retrieve it.
-    # Pretty much 0 and pe_image_address value should do the same operations
-    if not ql.code and (hModule == 0 or hModule == ql.loader.pe_image_address):
-        filename = ql.loader.filepath.decode('ascii').encode('utf-16le')
-        filename_len = len(filename)
-
-        if filename_len > nSize - 1:
-            filename = ql.loader.filepath[:nSize - 1]
-            ret = nSize
-        else:
-            ret = filename_len
-
-        ql.mem.write(lpFilename, filename + b"\x00")
-
-    else:
-        ql.log.debug("hModule %x" % hModule)
-        raise QlErrorNotImplemented("API not implemented")
-
-    return ret
+    return __GetModuleFileName(ql, address, params, wide=True)
 
 # FARPROC GetProcAddress(
 #   HMODULE hModule,
@@ -188,9 +169,11 @@ def hook_GetProcAddress(ql: Qiling, address: int, params):
 def _LoadLibrary(ql: Qiling, address: int, params):
     lpLibFileName = params["lpLibFileName"]
 
-    if not ql.code and lpLibFileName == ql.loader.filepath.decode():
-        # Loading self
-        return ql.loader.pe_image_address
+    # TODO: this searches only by basename; do we need to search by full path as well?
+    dll = ql.loader.get_image_by_name(lpLibFileName, casefold=True)
+
+    if dll is not None:
+        return dll.base
 
     return ql.loader.load_dll(lpLibFileName)
 

qiling/os/windows/dlls/kernel32/processthreadsapi.py

@@ -8,9 +8,9 @@ from qiling.os.windows.api import *
 from qiling.os.windows.const import *
 from qiling.os.windows.fncc import *
 
-from qiling.os.windows.thread import QlWindowsThread
+from qiling.os.windows.thread import QlWindowsThread, THREAD_STATUS
 from qiling.os.windows.handle import Handle
-from qiling.os.windows.structs import Token, StartupInfo
+from qiling.os.windows.structs import Token, make_startup_info
 
 # void ExitProcess(
 #   UINT uExitCode
@@ -22,14 +22,34 @@ def hook_ExitProcess(ql: Qiling, address: int, params):
     ql.emu_stop()
     ql.os.PE_RUN = False
 
-def _GetStartupInfo(ql: Qiling, address: int, params):
-    startup_info = StartupInfo(ql, 0xc3c930, 0, 0, 0, 0x64, 0x64, 0x84, 0x80, 0xff, 0x40, 0x1, STD_INPUT_HANDLE,
-                               STD_OUTPUT_HANDLE, STD_ERROR_HANDLE)
+def _GetStartupInfo(ql: Qiling, address: int, params, *, wide: bool):
+    lpStartupInfo = params['lpStartupInfo']
+    sui_struct = make_startup_info(ql.arch.bits)
 
-    pointer = params["lpStartupInfo"]
-    startup_info.write(pointer)
+    enc = 'utf-16le' if wide else 'latin1'
+    desktop_title = f'QilingDesktop\x00'.encode(enc)
 
-    return 0
+    # TODO: fill out with real / configurable values rather than bogus / fixed ones
+    with sui_struct.ref(ql.mem, lpStartupInfo) as sui_obj:
+        sui_obj.cb              = sui_struct.sizeof()
+        sui_obj.lpDesktop       = ql.os.heap.alloc(len(desktop_title))
+        sui_obj.lpTitle         = 0
+        sui_obj.dwX             = 0
+        sui_obj.dwY             = 0
+        sui_obj.dwXSize         = 100
+        sui_obj.dwYSize         = 100
+        sui_obj.dwXCountChars   = 132
+        sui_obj.dwYCountChars   = 128
+        sui_obj.dwFillAttribute = 0xff
+        sui_obj.dwFlags         = 0x40
+        sui_obj.wShowWindow     = 1
+        sui_obj.cbReserved2     = 0
+        sui_obj.lpReserved2     = 0
+        sui_obj.hStdInput       = STD_INPUT_HANDLE
+        sui_obj.hStdOutput      = STD_OUTPUT_HANDLE
+        sui_obj.hStdError       = STD_ERROR_HANDLE
+
+    return STATUS_SUCCESS
 
 # VOID WINAPI GetStartupInfoA(
 #   _Out_ LPSTARTUPINFO lpStartupInfo
@@ -38,7 +58,7 @@ def _GetStartupInfo(ql: Qiling, address: int, params):
     'lpStartupInfo' : LPSTARTUPINFOA
 })
 def hook_GetStartupInfoA(ql: Qiling, address: int, params):
-    return _GetStartupInfo(ql, address, params)
+    return _GetStartupInfo(ql, address, params, wide=False)
 
 # VOID WINAPI GetStartupInfoW(
 #   _Out_ LPSTARTUPINFO lpStartupInfo
@@ -47,7 +67,7 @@ def hook_GetStartupInfoA(ql: Qiling, address: int, params):
     'lpStartupInfo' : LPSTARTUPINFOW
 })
 def hook_GetStartupInfoW(ql: Qiling, address: int, params):
-    return _GetStartupInfo(ql, address, params)
+    return _GetStartupInfo(ql, address, params, wide=True)
 
 # DWORD TlsAlloc();
 @winsdkapi(cc=STDCALL, params={})
@@ -164,20 +184,13 @@ def hook_CreateThread(ql: Qiling, address: int, params):
     dwCreationFlags = params["dwCreationFlags"]
     lpThreadId = params["lpThreadId"]
 
-    # new thread obj
-    new_thread = QlWindowsThread(ql)
-
-    if dwCreationFlags & CREATE_SUSPENDED == CREATE_SUSPENDED:
-        thread_status = QlWindowsThread.READY
+    if dwCreationFlags & CREATE_SUSPENDED:
+        thread_status = THREAD_STATUS.READY
     else:
-        thread_status = QlWindowsThread.RUNNING
+        thread_status = THREAD_STATUS.RUNNING
 
     # create new thread
-    thread_id = new_thread.create(
-        lpStartAddress,
-        lpParameter,
-        thread_status
-    )
+    new_thread = QlWindowsThread.create(ql, dwStackSize, lpStartAddress, lpParameter, thread_status)
 
     # append the new thread to ThreadManager
     ql.os.thread_manager.append(new_thread)

qiling/os/windows/dlls/kernel32/stringapiset.py

@@ -7,6 +7,7 @@ from qiling import Qiling
 from qiling.os.windows.api import *
 from qiling.os.windows.fncc import *
 from qiling.os.windows.utils import cmp
+from qiling.os.windows.const import *
 
 # BOOL GetStringTypeW(
 #   DWORD                         dwInfoType,
@@ -43,6 +44,42 @@ def hook_GetStringTypeExA(ql: Qiling, address: int, params):
     # TODO: implement
     return 1
 
+def __encoding_name(codepage: int) -> str:
+    '''Get python encoding name from codepage value.
+
+    @see: https://docs.python.org/3.8/library/codecs.html#standard-encodings
+    '''
+
+    # mapping of special codepage values to encodings
+    encodings = {
+        # available only on windows hosts
+        CP_ACP        : 'mbcs',
+        CP_OEMCP      : 'oem',
+        CP_THREAD_ACP : 'mbcs',
+
+        # avaiable everywhere
+        CP_UTF16      : 'utf-16',
+        CP_UTF16BE    : 'utf-16be',
+        CP_ASCII      : 'ascii',
+        CP_UTF7       : 'utf-7',
+        CP_UTF8       : 'utf-8'
+    }
+
+    if codepage in encodings:
+        encname = encodings[codepage]
+
+    else:
+        encname = f'cp{codepage}'
+
+    # encoding might break on the hosting system; test it and
+    # fallback to windows-1252 ('western') if it fails
+    try:
+        _ = '\x00'.encode(encname)
+    except LookupError:
+        encname = 'cp1252'
+
+    return encname
+
 # int WideCharToMultiByte(
 #   UINT                               CodePage,
 #   DWORD                              dwFlags,
@@ -56,7 +93,7 @@ def hook_GetStringTypeExA(ql: Qiling, address: int, params):
 @winsdkapi(cc=STDCALL, params={
     'CodePage'          : UINT,
     'dwFlags'           : DWORD,
-    'lpWideCharStr'     : WSTRING, # LPCWCH
+    'lpWideCharStr'     : POINTER, # WSTRING
     'cchWideChar'       : INT,
     'lpMultiByteStr'    : LPSTR,
     'cbMultiByte'       : INT,
@@ -64,16 +101,54 @@ def hook_GetStringTypeExA(ql: Qiling, address: int, params):
     'lpUsedDefaultChar' : LPBOOL
 })
 def hook_WideCharToMultiByte(ql: Qiling, address: int, params):
+    CodePage = params['CodePage']
+    dwFlags = params['dwFlags']
+    lpWideCharStr = params['lpWideCharStr']
+    cchWideChar = params['cchWideChar']
+    lpMultiByteStr = params['lpMultiByteStr']
     cbMultiByte = params["cbMultiByte"]
-    s_lpWideCharStr = params["lpWideCharStr"]
-    lpMultiByteStr = params["lpMultiByteStr"]
 
-    s = (s_lpWideCharStr + "\x00").encode("utf-16le")
+    if not cchWideChar:
+        # TODO: set last error
+        return 0
 
-    if cbMultiByte != 0 and lpMultiByteStr != 0:
-        ql.mem.write(lpMultiByteStr, s)
+    # -1 indicates the string is null-terminated. the string is
+    # read along with its null-terminator
+    elif cchWideChar == 0xffffffff:
+        wcstr = bytearray()
+        ch = 1
 
-    return len(s)
+        while ch != b'\x00\x00':
+            ch = ql.mem.read(lpWideCharStr, 2)
+            wcstr.extend(ch)
+
+            lpWideCharStr += 2
+
+    # read exactly cbMultiByte bytes. that may or may not include
+    # a null-terminator
+    else:
+        wcstr = ql.mem.read(lpWideCharStr, cchWideChar * 2)
+
+    decoded = wcstr.decode('utf-16le')
+
+    encname = __encoding_name(CodePage)
+    errors = 'strict' if dwFlags & WC_ERR_INVALID_CHARS else 'replace'
+
+    try:
+        encoded = decoded.encode(encname, errors)
+    except UnicodeEncodeError:
+        ql.os.last_error = ERROR_NO_UNICODE_TRANSLATION
+        return 0
+
+    if not cbMultiByte:
+        return len(encoded)
+
+    if not lpMultiByteStr:
+        return 0
+
+    ql.mem.write(lpMultiByteStr, encoded[:cbMultiByte])
+
+    return min(len(encoded), cbMultiByte)
 
 # int MultiByteToWideChar(
 #  UINT                              CodePage,
@@ -86,18 +161,53 @@ def hook_WideCharToMultiByte(ql: Qiling, address: int, params):
 @winsdkapi(cc=STDCALL, params={
     'CodePage'       : UINT,
     'dwFlags'        : DWORD,
-    'lpMultiByteStr' : WSTRING, # LPCCH
+    'lpMultiByteStr' : POINTER, # STRING,
     'cbMultiByte'    : INT,
     'lpWideCharStr'  : LPWSTR,
     'cchWideChar'    : INT
 })
 def hook_MultiByteToWideChar(ql: Qiling, address: int, params):
-    wide_str = (params['lpMultiByteStr'] + "\x00").encode('utf-16le')
+    CodePage = params['CodePage']
+    lpMultiByteStr = params['lpMultiByteStr']
+    cbMultiByte = params['cbMultiByte']
+    lpWideCharStr = params['lpWideCharStr']
+    cchWideChar = params['cchWideChar']
 
-    if params['cchWideChar'] != 0:
-        ql.mem.write(params['lpWideCharStr'], wide_str)
+    if not cbMultiByte:
+        # TODO: set last error
+        return 0
 
-    return len(wide_str)
+    # -1 indicates the string is null-terminated. the string is
+    # read along with its null-terminator
+    elif cbMultiByte == 0xffffffff:
+        mbstr = bytearray()
+        ch = 1
+
+        while ch != b'\x00':
+            ch = ql.mem.read(lpMultiByteStr, 1)
+            mbstr.extend(ch)
+
+            lpMultiByteStr += 1
+
+    # read exactly cbMultiByte bytes. that may or may not include
+    # a null-terminator
+    else:
+        mbstr = ql.mem.read(lpMultiByteStr, cbMultiByte)
+
+    # use specified code page to translate bytes into string
+    encname = __encoding_name(CodePage)
+    decoded = mbstr.decode(encname)
+
+    # this is a dry-run; just return the amount of chars
+    if not cchWideChar:
+        return len(decoded)
+
+    if not lpWideCharStr:
+        return 0
+
+    ql.mem.write(lpWideCharStr, decoded[:cchWideChar].encode('utf-16le'))
+
+    return min(len(decoded), cchWideChar)
 
 def __CompareString(ql: Qiling, address: int, params) -> int:
     lpString1 = params["lpString1"]

qiling/os/windows/dlls/kernel32/synchapi.py

@@ -145,15 +145,17 @@ def hook_WaitForMultipleObjects(ql: Qiling, address: int, params):
     return 0
 
 def __OpenMutex(ql: Qiling, address: int, params):
+    lpName = params["lpName"]
+
+    if not lpName:
+        return 0
+
     # The name can have a "Global" or "Local" prefix to explicitly open an object in the global or session namespace.
     # It can also have no prefix
-    try:
-        _type, name = params["lpName"].split("\\")
-    except ValueError:
-        name = params["lpName"]
-        _type = ""
+    _type, name = lpName.split("\\") if '\\' in lpName else ('', lpName)
 
     handle = ql.os.handle_manager.search(name)
+
     if _type == "Global":
         # if is global is a Windows lock. We always return a valid handle because we have no way to emulate them
         # example sample: Gandcrab e42431d37561cc695de03b85e8e99c9e31321742
@@ -175,11 +177,13 @@ def __OpenMutex(ql: Qiling, address: int, params):
         raise QlErrorNotImplemented("API not implemented")
 
 def __CreateMutex(ql: Qiling, address: int, params):
-    try:
-        _type, name = params["lpName"].split("\\")
-    except ValueError:
-        name = params["lpName"]
-        _type = ""
+    lpName = params["lpName"]
+    owning = params["bInitialOwner"]
+
+    if not lpName:
+        return 0
+
+    _type, name = lpName.split("\\") if '\\' in lpName else ('', lpName)
 
     handle = ql.os.handle_manager.search(name)
 
@@ -187,7 +191,6 @@ def __CreateMutex(ql: Qiling, address: int, params):
         # ql.os.last_error = ERROR_ALREADY_EXISTS
         return 0
 
-    owning = params["bInitialOwner"]
     mutex = Mutex(name, _type)
 
     if owning:
@@ -279,30 +282,30 @@ def hook_ReleaseMutex(ql: Qiling, address: int, params):
 
 def __CreateEvent(ql: Qiling, address: int, params):
     # Implementation seems similar enough to Mutex to just use it
+    lpName = params["lpName"]
 
-    try:
-        namespace, name = params["lpName"].split("\\")
-    except ValueError:
-        name = params["lpName"]
-        namespace = ""
+    if lpName:
+        namespace, name = lpName.split('\\') if '\\' in lpName else ('', lpName)
+        handle = ql.os.handle_manager.search(name)
 
-    handle = ql.os.handle_manager.search(name)
+        if handle is not None:
+            ql.os.last_error = ERROR_ALREADY_EXISTS
+            return handle.id
 
-    if handle is not None:
-        ql.os.last_error = ERROR_ALREADY_EXISTS
-        # FIXME: fail with a nullptr?
-        # return 0
     else:
-        mutex = Mutex(name, namespace)
+        # TODO: should be None instead of empty strings?
+        namespace = ''
+        name = ''
 
-        if params['bInitialState']:
-            mutex.lock()
+    mutex = Mutex(name, namespace)
 
-        handle = Handle(obj=mutex, name=name)
-        ql.os.handle_manager.append(handle)
+    if params['bInitialState']:
+        mutex.lock()
 
-    # FIXME: shouldn't it be 'id' instead of 'ID'?
-    return handle.ID
+    handle = Handle(obj=mutex, name=name)
+    ql.os.handle_manager.append(handle)
+
+    return handle.id
 
 # HANDLE CreateEventA(
 #  LPSECURITY_ATTRIBUTES lpEventAttributes,

qiling/os/windows/dlls/kernel32/sysinfoapi.py

@@ -10,7 +10,7 @@ from qiling import Qiling
 from qiling.os.windows.api import *
 from qiling.os.windows.const import *
 from qiling.os.windows.fncc import *
-from qiling.os.windows.structs import SystemInfo, SystemTime
+from qiling.os.windows.structs import FILETIME, SYSTEMTIME, make_system_info
 
 # NOT_BUILD_WINDOWS_DEPRECATE DWORD GetVersion(
 # );
@@ -40,13 +40,24 @@ def hook_GetVersionExW(ql: Qiling, address: int, params):
     return __GetVersionEx(ql, address, params)
 
 def __GetSystemInfo(ql: Qiling, address: int, params):
-    pointer = params["lpSystemInfo"]
+    lpSystemInfo = params['lpSystemInfo']
 
-    # FIXME: dll_size no longer reflects the upper bound of used memory; should find a better way to specify max_address
-    system_info = SystemInfo(ql, 0, ql.mem.pagesize, ql.loader.pe_image_address,
-                             ql.loader.dll_address + ql.loader.dll_size, 0x3, 0x4, 0x24a, ql.mem.pagesize * 10,
-                             0x6, 0x4601)
-    system_info.write(pointer)
+    sysinfo_struct = make_system_info(ql.arch.bits)
+
+    # FIXME:
+    #  - load configurable values rather than fixed / bogus ones
+    #  - loader.dll_size no longer reflects the upper bound of used memory; should find a better way to specify max_address
+    with sysinfo_struct.ref(ql.mem, lpSystemInfo) as si:
+        si.dwOemId = 0
+        si.dwPageSize = ql.mem.pagesize
+        si.lpMinimumApplicationAddress = ql.loader.pe_image_address
+        si.lpMaximumApplicationAddress = ql.loader.dll_address + ql.loader.dll_size
+        si.dwActiveProcessorMask = 0x3
+        si.dwNumberOfProcessors = 0x4
+        si.dwProcessorType = 0x24a
+        si.dwAllocationGranularity = ql.mem.pagesize * 10
+        si.wProcessorLevel = 0x6
+        si.wProcessorRevision = 0x4601
 
     return 0
 
@@ -66,11 +77,18 @@ def hook_GetSystemInfo(ql: Qiling, address: int, params):
     'lpSystemTime' : LPSYSTEMTIME
 })
 def hook_GetLocalTime(ql: Qiling, address: int, params):
-    ptr = params['lpSystemTime']
-    d = datetime.now()
+    lpSystemTime = params['lpSystemTime']
+    now = datetime.now()
 
-    system_time = SystemTime(ql, d.year, d.month, d.isoweekday(), d.day, d.hour, d.minute, d.second, d.microsecond // 1000)
-    system_time.write(ptr)
+    with SYSTEMTIME.ref(ql.mem, lpSystemTime) as st:
+        st.wYear = now.year
+        st.wMonth = now.month
+        st.wDayOfWeek = now.isoweekday()
+        st.wDay = now.day
+        st.wHour = now.hour
+        st.wMinute = now.minute
+        st.wSecond = now.second
+        st.wMilliseconds = now.microsecond // 1000
 
     return 0
 
@@ -81,8 +99,23 @@ def hook_GetLocalTime(ql: Qiling, address: int, params):
     'lpSystemTimeAsFileTime' : LPFILETIME
 })
 def hook_GetSystemTimeAsFileTime(ql: Qiling, address: int, params):
-    # TODO
-    pass
+    ptr = params['lpSystemTimeAsFileTime']
+
+    epoch = datetime(1601, 1, 1)
+    elapsed = datetime.now() - epoch
+
+    # number of 100-nanosecond intervals since Jan 1, 1601 utc
+    # where: (10 ** 9) / 100 -> (10 ** 7)
+    hnano = int(elapsed.total_seconds() * (10 ** 7))
+
+    mask = (1 << 32) - 1
+
+    ftime = FILETIME(
+        (hnano >>  0) & mask,
+        (hnano >> 32) & mask
+    )
+
+    ftime.save_to(ql.mem, ptr)
 
 # DWORD GetTickCount(
 # );

qiling/os/windows/dlls/kernel32/timezoneapi.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+#
+# Cross Platform and Multi Architecture Advanced Binary Emulation Framework
+#
+
+from qiling import Qiling
+from qiling.os.windows.api import *
+from qiling.os.windows.const import *
+from qiling.os.windows.fncc import *
+
+# DWORD GetTimeZoneInformation(
+#   [out] LPTIME_ZONE_INFORMATION lpTimeZoneInformation
+# );
+@winsdkapi(cc=STDCALL, params={
+	'lpTimeZoneInformation' : LPTIME_ZONE_INFORMATION
+})
+def hook_GetTimeZoneInformation(ql: Qiling, address: int, params):
+    # TODO: implement this later. fail for now
+	return TIME_ZONE_ID_INVALID

qiling/os/windows/dlls/kernel32/tlhelp32.py

@@ -48,5 +48,12 @@ def hook_Process32FirstW(ql: Qiling, address: int, params):
     'lppe'      : LPPROCESSENTRY32W
 })
 def hook_Process32NextW(ql: Qiling, address: int, params):
-    # Return True if more process, 0 else
-    return int(ql.os.syscall_count["Process32NextW"] < 3)  # I don' know how many process the sample want's to cycle
+    global __call_count
+
+    __call_count += 1
+
+    # FIXME: this is an undocumented workaround, probably to satisfy one of
+    # the samples. better implement that as an ad-hoc hook there
+    return int(__call_count < 3)
+
+__call_count = 0

qiling/os/windows/dlls/kernel32/winbase.py

@@ -11,7 +11,7 @@ from qiling.exception import QlErrorNotImplemented
 from qiling.os.windows.api import *
 from qiling.os.windows.const import *
 from qiling.os.windows.fncc import *
-from qiling.os.windows.structs import OsVersionInfoExA
+from qiling.os.windows.structs import make_os_version_info_ex
 from qiling.os.windows.utils import cmp
 
 # HFILE _lclose(
@@ -544,20 +544,6 @@ def hook_IsBadWritePtr(ql: Qiling, address: int, params):
     # Check write permission for size of memory
     return 0 # ACCESS_TRUE
 
-def compare(p1: int, operator: int, p2: int) -> bool:
-    op = {
-        VER_EQUAL         : lambda a, b: a == b,
-        VER_GREATER       : lambda a, b: a > b,
-        VER_GREATER_EQUAL : lambda a, b: a >= b,
-        VER_LESS          : lambda a, b: a < b,
-        VER_LESS_EQUAL    : lambda a, b: a <= b
-    }.get(operator)
-
-    if not op:
-        raise QlErrorNotImplemented('')
-
-    return op(p1, p2)
-
 # BOOL VerifyVersionInfoW(
 #   LPOSVERSIONINFOEXW lpVersionInformation,
 #   DWORD              dwTypeMask,
@@ -569,64 +555,98 @@ def compare(p1: int, operator: int, p2: int) -> bool:
     'dwlConditionMask'     : DWORDLONG
 })
 def hook_VerifyVersionInfoW(ql: Qiling, address: int, params):
-    # https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-verifyversioninfow2
-    pointer = params["lpVersionInformation"]
+    return __VerifyVersionInfo(ql, address, params, wide=True)
 
-    os_asked = OsVersionInfoExA(ql)
-    os_asked.read(pointer)
+# BOOL VerifyVersionInfoA(
+#   LPOSVERSIONINFOEXA lpVersionInformation,
+#   DWORD              dwTypeMask,
+#   DWORDLONG          dwlConditionMask
+# );
+@winsdkapi(cc=STDCALL, params={
+    'lpVersionInformation' : LPOSVERSIONINFOEXA,
+    'dwTypeMask'           : DWORD,
+    'dwlConditionMask'     : DWORDLONG
+})
+def hook_VerifyVersionInfoA(ql: Qiling, address: int, params):
+    return __VerifyVersionInfo(ql, address, params, wide=False)
+
+def __VerifyVersionInfo(ql: Qiling, address: int, params, *, wide: bool):
+    # see: https://docs.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-verifyversioninfow
+
+    lpVersionInformation = params['lpVersionInformation']
+    dwTypeMask = params['dwTypeMask']
+    dwlConditionMask = params['dwlConditionMask']
+
+    oviex_struct = make_os_version_info_ex(ql.arch.bits, wide=wide)
+
+    askedOsVersionInfo = oviex_struct.load_from(ql.mem, lpVersionInformation)
+
+    # reading emulated os version info from profile
+    # FIXME: read the necessary information from KUSER_SHARED_DATA instead
+    osconfig = ql.os.profile['SYSTEM']
+
+    emulOsVersionInfo = oviex_struct(
+        dwMajorVersion      = osconfig.getint('majorVersion'),
+        dwMinorVersion      = osconfig.getint('minorVersion'),
+        dwBuildNumber       = 0,
+        dwPlatformId        = 0,
+        wServicePackMajor   = osconfig.getint('VER_SERVICEPACKMAJOR'),
+        wServicePackMinor   = 0,
+        wSuiteMask          = 0,
+        wProductType        = osconfig.getint('productType')
+    )
+
+    # check criteria by the order they should be evaluated. the online microsoft
+    # documentation only specify the first five, so not sure about the other three.
+    #
+    # each criteria is associated with the OSVERSIONINFOEX[A|W] it corresponds to.
+    checks = (
+        (1, 'dwMajorVersion'),      # VER_MAJORVERSION
+        (0, 'dwMinorVersion'),      # VER_MINORVERSION
+        (2, 'dwBuildNumber'),       # VER_BUILDNUMBER
+        (5, 'wServicePackMajor'),   # VER_SERVICEPACKMAJOR
+        (4, 'wServicePackMinor'),   # VER_SERVICEPACKMINOR
+        (3, 'dwPlatformId'),        # VER_PLATFORMID
+        (6, 'wSuiteMask'),          # VER_SUITENAME
+        (7, 'wProductType')         # VER_PRODUCT_TYPE
+    )
 
-    ConditionMask: dict = ql.os.hooks_variables["ConditionMask"]
     res = True
 
-    opstr = {
-        VER_EQUAL         : '==',
-        VER_GREATER       : '>',
-        VER_GREATER_EQUAL : '>=',
-        VER_LESS          : '<',
-        VER_LESS_EQUAL    : '<='
-    }
+    for bit, field in checks:
+        if dwTypeMask & (1 << bit):
+            asked = getattr(askedOsVersionInfo, field)
+            emuld = getattr(emulOsVersionInfo, field)
 
-    for key, value in ConditionMask.items():
-        if value not in opstr:
-            raise QlErrorNotImplemented(f'API not implemented with operator {value}')
+            # extract the condition code for the required field
+            cond = (dwlConditionMask >> (bit * VER_NUM_BITS_PER_CONDITION_MASK)) & VER_CONDITION_MASK
 
-        # Versions should be compared together
-        if key in (VER_MAJORVERSION, VER_MINORVERSION, VER_PRODUCT_TYPE):
-            concat = f'{os_asked.major[0]}{os_asked.minor[0]}{os_asked.product[0]}'
+            # special case for VER_SUITENAME
+            if bit == 6:
+                cond_op = {
+                    VER_AND : lambda a, b: (a & b) == b,  # all members of b must be present
+                    VER_OR  : lambda a, b: (a & b) != 0   # at least one member of b must be present
+                }[cond]
 
-            # Just a print for analysts, will remove it from here in the future
-            if key == VER_MAJORVERSION:
-                ql.log.debug("The Target is checking the windows Version!")
-                version_asked = SYSTEMS_VERSION.get(concat, None)
+                res &= cond_op(emuld, asked)
 
-                if version_asked is None:
-                    raise QlErrorNotImplemented(f'API not implemented for version {concat}')
+            else:
+                # cond operates as a bitmask, so multiple 'if' statements are appropriately
+                # used here. do not turn this into an 'elif' construct.
 
-                ql.log.debug(f'The target asks for version {opstr[value]} {version_asked}')
+                if (cond & VER_GREATER) and (emuld > asked):
+                    return 1
 
-            qiling_os = \
-                f'{ql.os.profile.get("SYSTEM", "majorVersion")}' + \
-                f'{ql.os.profile.get("SYSTEM", "minorVersion")}' + \
-                f'{ql.os.profile.get("SYSTEM", "productType")}'
+                if (cond & VER_LESS) and (emuld < asked):
+                    return 1
 
-            # We can finally compare
-            res = compare(int(qiling_os), value, int(concat))
+                if (cond & VER_EQUAL):
+                    res &= (emuld == asked)
 
-        elif key == VER_SERVICEPACKMAJOR:
-            res = compare(ql.os.profile.getint("SYSTEM", "VER_SERVICEPACKMAJOR"), value, os_asked.service_major[0])
+    if not res:
+        ql.os.last_error = ERROR_OLD_WIN_VERSION
 
-        else:
-            raise QlErrorNotImplemented("API not implemented for key %s" % key)
-
-        # The result is a AND between every value, so if we find a False we just exit from the loop
-        if not res:
-            ql.os.last_error = ERROR_OLD_WIN_VERSION
-            return 0
-
-    # reset mask
-    ql.os.hooks_variables["ConditionMask"] = {}
-
-    return res
+    return int(res)
 
 def __GetUserName(ql: Qiling, address: int, params, wstring: bool):
     lpBuffer = params["lpBuffer"]

qiling/os/windows/dlls/kernel32/winnt.py

@@ -66,52 +66,17 @@ def hook_InterlockedDecrement(ql: Qiling, address: int, params):
     'Condition'     : BYTE
 })
 def hook_VerSetConditionMask(ql: Qiling, address: int, params):
-    # ConditionMask = params["ConditionMask"]
-    TypeMask = params["TypeMask"]
-    Condition = params["Condition"]
+    # see: https://docs.microsoft.com/en-us/windows/win32/sysinfo/verifying-the-system-version
 
-    ConditionMask = ql.os.hooks_variables.get("ConditionMask", {})
+    ConditionMask = params['ConditionMask']
+    TypeMask = params['TypeMask']
+    Condition = params['Condition']
 
-    if TypeMask == 0:
-        ret = ConditionMask
-    else:
-        Condition &= VER_CONDITION_MASK
+    Condition &= VER_CONDITION_MASK
 
-        if Condition == 0:
-            ret = ConditionMask
-        else:
-            if TypeMask & VER_PRODUCT_TYPE:
-                # ConditionMask |= ullCondMask << (7 * VER_NUM_BITS_PER_CONDITION_MASK)
-                ConditionMask[VER_PRODUCT_TYPE] = Condition
-            elif TypeMask & VER_SUITENAME:
-                # ConditionMask |= ullCondMask << (6 * VER_NUM_BITS_PER_CONDITION_MASK)
-                ConditionMask[VER_SUITENAME] = Condition
-            elif TypeMask & VER_SERVICEPACKMAJOR:
-                # ConditionMask |= ullCondMask << (5 * VER_NUM_BITS_PER_CONDITION_MASK)
-                ConditionMask[VER_SERVICEPACKMAJOR] = Condition
-            elif TypeMask & VER_SERVICEPACKMINOR:
-                # ConditionMask |= ullCondMask << (4 * VER_NUM_BITS_PER_CONDITION_MASK)
-                ConditionMask[VER_SERVICEPACKMINOR] = Condition
-            elif TypeMask & VER_PLATFORMID:
-                # ConditionMask |= ullCondMask << (3 * VER_NUM_BITS_PER_CONDITION_MASK)
-                ConditionMask[VER_PLATFORMID] = Condition
-            elif TypeMask & VER_BUILDNUMBER:
-                # ConditionMask |= ullCondMask << (2 * VER_NUM_BITS_PER_CONDITION_MASK)
-                ConditionMask[VER_BUILDNUMBER] = Condition
-            elif TypeMask & VER_MAJORVERSION:
-                # ConditionMask |= ullCondMask << (1 * VER_NUM_BITS_PER_CONDITION_MASK)
-                ConditionMask[VER_MAJORVERSION] = Condition
-            elif TypeMask & VER_MINORVERSION:
-                # ConditionMask |= ullCondMask << (0 * VER_NUM_BITS_PER_CONDITION_MASK)
-                ConditionMask[VER_MINORVERSION] = Condition
+    if Condition:
+        for i in range(8):
+            if TypeMask & (1 << i):
+                ConditionMask |= Condition << (i * VER_NUM_BITS_PER_CONDITION_MASK)
 
-            ret = 1
-
-    # ConditionMask should be updated locally
-    # https://docs.microsoft.com/it-it/windows/win32/sysinfo/verifying-the-system-version
-    # But since we don't have the pointer to the variable, an hack is to use the environment.
-    # Feel free to push a better solution
-    # Since I can't work with bits, and since we had to work with the environment anyway, let's use a dict
-    ql.os.hooks_variables["ConditionMask"] = ConditionMask
-
-    return ret
+    return ConditionMask

qiling/os/windows/dlls/msvcrt.py

@@ -4,6 +4,7 @@
 #
 
 import time
+from typing import Sequence
 
 from qiling import Qiling
 from qiling.exception import QlErrorNotImplemented
@@ -21,6 +22,49 @@ from qiling.os.windows.handle import Handle
 def hook___set_app_type(ql: Qiling, address: int, params):
     pass
 
+def __alloc_strings_array(ql: Qiling, items: Sequence[str], *, wide: bool) -> int:
+    '''Allocate and populate an array of strings and return its address.
+    '''
+
+    enc = 'utf-16le' if wide else 'utf-8'
+
+    nitems = len(items)
+
+    # allocate room for pointers to items and a trailing null pointer
+    p_array = ql.os.heap.alloc((nitems + 1) * ql.arch.pointersize)
+
+    # encode all arguments into bytes
+    items_bytes = [f'{item}\x00'.encode(enc) for item in items]
+
+    # allocate room for the items
+    p_items_bytes = ql.os.heap.alloc(sum(len(a) for a in items_bytes))
+
+    for i, item in enumerate(items_bytes):
+        # write argument data
+        ql.mem.write(p_items_bytes, item)
+
+        # write pointer to argument data into the argv array
+        ql.mem.write_ptr(p_array + (i * ql.arch.pointersize), p_items_bytes)
+
+        p_items_bytes += len(item)
+
+    # write trailing null pointer
+    ql.mem.write_ptr(p_array + (nitems * ql.arch.pointersize), 0)
+
+    return p_array
+
+def __getmainargs(ql: Qiling, params, wide: bool) -> int:
+    argc = len(ql.argv)
+    argv = __alloc_strings_array(ql, ql.argv, wide=wide)
+    env = __alloc_strings_array(ql, [f'{k}={v}' for k, v in ql.env], wide=wide)
+
+    # write out paramters
+    ql.mem.write_ptr(params['_Argc'], argc, 4)
+    ql.mem.write_ptr(params['_Argv'], argv)
+    ql.mem.write_ptr(params['_Env'], env)
+
+    return 0
+
 # int __getmainargs(
 #    int * _Argc,
 #    char *** _Argv,
@@ -36,7 +80,17 @@ def hook___set_app_type(ql: Qiling, address: int, params):
     '_StartInfo'  : POINTER
 })
 def hook___getmainargs(ql: Qiling, address: int, params):
-    return 0
+    return __getmainargs(ql, params, wide=False)
+
+@winsdkapi(cc=CDECL, params={
+    '_Argc'       : POINTER,
+    '_Argv'       : POINTER,
+    '_Env'        : POINTER,
+    '_DoWildCard' : INT,
+    '_StartInfo'  : POINTER
+})
+def hook___wgetmainargs(ql: Qiling, address: int, params):
+    return __getmainargs(ql, params, wide=True)
 
 # int* __p__fmode();
 @winsdkapi(cc=CDECL, params={})
@@ -53,12 +107,14 @@ def hook___p__commode(ql: Qiling, address: int, params):
 # char** __p__acmdln();
 @winsdkapi(cc=CDECL, params={})
 def hook___p__acmdln(ql: Qiling, address: int, params):
+    # TODO: use values calculated at __getmainargs
     addr = ql.loader.import_address_table['msvcrt.dll'][b'_acmdln']
     return addr
 
 # wchar_t ** __p__wcmdln();
 @winsdkapi(cc=CDECL, params={})
 def hook___p__wcmdln(ql: Qiling, address: int, params):
+    # TODO: use values calculated at __getmainargs
     addr = ql.loader.import_address_table['msvcrt.dll'][b'_wcmdln']
     return addr
 
@@ -127,7 +183,7 @@ def hook__cexit(ql: Qiling, address: int, params):
     'pfend'   : POINTER
 })
 def hook__initterm(ql: Qiling, address: int, params):
-    pass
+    return 0
 
 # void exit(
 #    int const status
@@ -207,9 +263,10 @@ def hook_sprintf(ql: Qiling, address: int, params):
     if format == 0:
         format = "(null)"
 
-    args = ql.os.utils.va_list(format, arglist)
-    count = ql.os.utils.sprintf(buff, format, args, wstring=False)
-    ql.os.utils.update_ellipsis(params, args)
+    args = ql.os.utils.va_list(arglist)
+
+    count, upd_args = ql.os.utils.sprintf(buff, format, args, wstring=False)
+    upd_args(params)
 
     return count
 
@@ -223,12 +280,10 @@ def hook_printf(ql: Qiling, address: int, params):
     if format == 0:
         format = "(null)"
 
-    nargs = format.count("%")
-    ptypes = (POINTER, ) + (PARAM_INTN, ) * nargs
-    args = ql.os.fcall.readParams(ptypes)[1:]
+    args = ql.os.fcall.readEllipsis(params.values())
 
-    count = ql.os.utils.printf(format, args, wstring=False)
-    ql.os.utils.update_ellipsis(params, args)
+    count, upd_args = ql.os.utils.printf(format, args, wstring=False)
+    upd_args(params)
 
     return count
 
@@ -242,12 +297,10 @@ def hook_wprintf(ql: Qiling, address: int, params):
     if format == 0:
         format = "(null)"
 
-    nargs = format.count("%")
-    ptypes = (POINTER, ) + (PARAM_INTN, ) * nargs
-    args = ql.os.fcall.readParams(ptypes)[1:]
+    args = ql.os.fcall.readEllipsis(params.values())
 
-    count = ql.os.utils.printf(format, args, wstring=True)
-    ql.os.utils.update_ellipsis(params, args)
+    count, upd_args = ql.os.utils.printf(format, args, wstring=True)
+    upd_args(params)
 
     return count
 
@@ -264,9 +317,10 @@ def __stdio_common_vfprintf(ql: Qiling, address: int, params, wstring: bool):
 
     # TODO: take _Stream into account
 
-    args = ql.os.utils.va_list(format, arglist)
-    count = ql.os.utils.printf(format, args, wstring)
-    ql.os.utils.update_ellipsis(params, args)
+    args = ql.os.utils.va_list(arglist)
+
+    count, upd_args = ql.os.utils.printf(format, args, wstring)
+    upd_args(params)
 
     return count
 
@@ -297,9 +351,10 @@ def __stdio_common_vsprintf(ql: Qiling, address: int, params, wstring: bool):
 
     # TODO: take _BufferCount into account
 
-    args = ql.os.utils.va_list(format, arglist)
-    count = ql.os.utils.sprintf(buff, format, args, wstring)
-    ql.os.utils.update_ellipsis(params, args)
+    args = ql.os.utils.va_list(arglist)
+
+    count, upd_args = ql.os.utils.sprintf(buff, format, args, wstring)
+    upd_args(params)
 
     return count
 
@@ -373,6 +428,19 @@ def hook___stdio_common_vswprintf_s(ql: Qiling, address: int, params):
 def hook___lconv_init(ql: Qiling, address: int, params):
     return 0
 
+@winsdkapi(cc=CDECL, params={
+    'str'     : POINTER,
+    'maxsize' : SIZE_T
+})
+def hook___strncnt(ql: Qiling, address: int, params):
+    s = params["str"]
+    maxsize = params["maxsize"]
+
+    data = ql.mem.read(s, maxsize)
+
+    # a simple hack to make sure a null terminator is found at most at 'maxsize'
+    return (data + b'\x00').find(b'\00')
+
 # size_t strlen(
 #    const char *str
 # );

qiling/os/windows/dlls/ntdll.py

@@ -31,50 +31,54 @@ def hook_memcpy(ql: Qiling, address: int, params):
     src = params['src']
     count = params['count']
 
-    try:
-        data = bytes(ql.mem.read(src, count))
-        ql.mem.write(dest, data)
-    except Exception:
-        ql.log.exception("")
+    data = bytes(ql.mem.read(src, count))
+    ql.mem.write(dest, data)
 
     return dest
 
 def _QueryInformationProcess(ql: Qiling, address: int, params):
     flag = params["ProcessInformationClass"]
-    dst = params["ProcessInformation"]
-    pt_res = params["ReturnLength"]
+    obuf_ptr = params["ProcessInformation"]
+    obuf_len = params['ProcessInformationLength']
+    res_size_ptr = params["ReturnLength"]
 
     if flag == ProcessDebugFlags:
-        value = b"\x01" * 0x4
+        res_data = ql.pack32(1)
 
     elif flag == ProcessDebugPort:
-        value = b"\x00" * 0x4
+        res_data = ql.pack32(0)
 
     elif flag == ProcessDebugObjectHandle:
         return STATUS_PORT_NOT_SET
 
     elif flag == ProcessBasicInformation:
-        pbi = structs.ProcessBasicInformation(ql,
-            exitStatus=0,
-            pebBaseAddress=ql.loader.TEB.PebAddress,
-            affinityMask=0,
-            basePriority=0,
-            uniqueId=ql.os.profile.getint("KERNEL", "pid"),
-            parentPid=ql.os.profile.getint("KERNEL", "parent_pid")
+        kconf = ql.os.profile['KERNEL']
+        pbi_struct = structs.make_process_basic_info(ql.arch.bits)
+
+        pci_obj = pbi_struct(
+            ExitStatus=0,
+            PebBaseAddress=ql.loader.TEB.PebAddress,
+            AffinityMask=0,
+            BasePriority=0,
+            UniqueProcessId=kconf.getint('pid'),
+            InheritedFromUniqueProcessId=kconf.getint('parent_pid')
         )
 
-        addr = ql.os.heap.alloc(pbi.size)
-        pbi.write(addr)
-        value = ql.pack(addr)
+        res_data = bytes(pci_obj)
+
     else:
-        ql.log.debug(str(flag))
-        raise QlErrorNotImplemented("API not implemented")
+        # TODO: support more info class ("flag") values
+        ql.log.info(f'SetInformationProcess: no implementation for info class {flag:#04x}')
 
-    ql.log.debug("The target is checking the debugger via QueryInformationProcess ")
-    ql.mem.write(dst, value)
+        return STATUS_UNSUCCESSFUL
 
-    if pt_res:
-        ql.mem.write_ptr(pt_res, 8, 1)
+    res_size = len(res_data)
+
+    if obuf_len >= res_size:
+        ql.mem.write(obuf_ptr, res_data)
+
+    if res_size_ptr:
+        ql.mem.write_ptr(res_size_ptr, res_size)
 
     return STATUS_SUCCESS
 
@@ -85,7 +89,7 @@ def _QueryInformationProcess(ql: Qiling, address: int, params):
 #   _In_      ULONG            ProcessInformationLength,
 #   _Out_opt_ PULONG           ReturnLength
 # );
-@winsdkapi(cc=CDECL, params={
+@winsdkapi(cc=STDCALL, params={
     'ProcessHandle'            : HANDLE,
     'ProcessInformationClass'  : PROCESSINFOCLASS,
     'ProcessInformation'       : PVOID,
@@ -117,46 +121,43 @@ def hook_NtQueryInformationProcess(ql: Qiling, address: int, params):
     return _QueryInformationProcess(ql, address, params)
 
 def _QuerySystemInformation(ql: Qiling, address: int, params):
-    siClass = params["SystemInformationClass"]
-    pt_res = params["ReturnLength"]
-    dst = params["SystemInformation"]
-
-    if (siClass == SystemBasicInformation):
-        bufferLength = params["SystemInformationLength"]
+    SystemInformationClass = params['SystemInformationClass']
+    SystemInformation = params['SystemInformation']
+    SystemInformationLength = params['SystemInformationLength']
+    ReturnLength = params['ReturnLength']
 
+    if SystemInformationClass == SystemBasicInformation:
         max_uaddr = {
             QL_ARCH.X86  : 0x7FFEFFFF,
             QL_ARCH.X8664: 0x7FFFFFFEFFFF
         }[ql.arch.type]
 
-        sbi = structs.SystemBasicInforation(
-            ql,
-            Reserved=0,
-            TimerResolution=156250,
-            PageSize=ql.mem.pagesize,
-            NumberOfPhysicalPages=0x003FC38A,
-            LowestPhysicalPageNumber=1,
-            HighestPhysicalPageNumber=0x0046DFFF,
-            AllocationGranularity=1,
-            MinimumUserModeAddress=0x10000,
-            MaximumUserModeAddress=max_uaddr,
-            ActiveProcessorsAffinityMask=0x3F,
-            NumberOfProcessors=0x6
+        sbi_struct = structs.make_system_basic_info(ql.arch.bits)
+
+        # FIXME: retrieve the necessary info from KUSER_SHARED_DATA
+        sbi_obj = sbi_struct(
+            TimerResolution              = 156250,
+            PageSize                     = ql.mem.pagesize,
+            NumberOfPhysicalPages        = 0x003FC38A,
+            LowestPhysicalPageNumber     = 1,
+            HighestPhysicalPageNumber    = 0x0046DFFF,
+            AllocationGranularity        = 1,
+            MinimumUserModeAddress       = 0x10000,
+            MaximumUserModeAddress       = max_uaddr,
+            ActiveProcessorsAffinityMask = 0x3F,
+            NumberOfProcessors           = 6
         )
 
-        if (bufferLength==sbi.size):
-            sbi.write(dst)
-
-            if pt_res:
-                ql.mem.write_ptr(pt_res, sbi.size, 1)
-        else:
-            if pt_res:
-                ql.mem.write_ptr(pt_res, sbi.size, 1)
+        if ReturnLength:
+            ql.mem.write_ptr(ReturnLength, sbi_struct.sizeof(), 4)
 
+        if SystemInformationLength < sbi_struct.sizeof():
             return STATUS_INFO_LENGTH_MISMATCH
+
+        sbi_obj.save_to(ql.mem, SystemInformation)
+
     else:
-        ql.log.debug(str(siClass))
-        raise QlErrorNotImplemented("API not implemented")
+        raise QlErrorNotImplemented(f'not implemented for {SystemInformationClass=}')
 
     return STATUS_SUCCESS
 
@@ -229,35 +230,64 @@ def hook_ZwCreateDebugObject(ql: Qiling, address: int, params):
     'ReturnLength'            : PULONG
 })
 def hook_ZwQueryObject(ql: Qiling, address: int, params):
-    infoClass = params["ObjectInformationClass"]
-    dest = params["ObjectInformation"]
-    size_dest = params["ReturnLength"]
-    string = "DebugObject".encode("utf-16le")
+    handle = params['Handle']
+    ObjectInformationClass = params['ObjectInformationClass']
+    ObjectInformation = params['ObjectInformation']
+    ObjectInformationLength = params['ObjectInformationLength']
+    ReturnLength = params['ReturnLength']
 
-    string_addr = ql.os.heap.alloc(len(string))
-    ql.log.debug(str(string_addr))
-    ql.log.debug(str(string))
-    ql.mem.write(string_addr, string)
-    us = structs.UnicodeString(ql, len(string), len(string), string_addr)
+    s = 'DebugObject'.encode('utf-16le')
+    addr = ql.os.heap.alloc(len(s))
+    ql.mem.write(addr, s)
 
-    if infoClass == ObjectTypeInformation:
-        res = structs.ObjectTypeInformation(ql, us, 1, 1)
+    unistr_struct = structs.make_unicode_string(ql.arch.bits)
 
-    elif infoClass == ObjectAllTypesInformation:
-        # FIXME: there is an error in how these structs are read by al-khaser. Have no idea on where, so we are
-        #  bypassing it
-        # oti = structs.ObjectTypeInformation(ql, us, 1, 1)
-        # res = structs.ObjectAllTypesInformation(ql, 2, oti)
-        return 1
+    unistr_obj = unistr_struct(
+        Length        = len(s),
+        MaximumLength = len(s),
+        Buffer        = addr
+    )
+
+    oti_struct = structs.make_object_type_info(ql.arch.bits)
+
+    oti_obj = oti_struct(
+        TypeName             = unistr_obj,
+        TotalNumberOfObjects = 1,
+        TotalNumberOfHandles = 1
+    )
+
+    oati_struct = structs.make_object_all_types_info(ql.arch.bits, 1)
+
+    if ObjectInformationClass == ObjectTypeInformation:
+        out = oti_obj
+
+    elif ObjectInformationClass == ObjectAllTypesInformation:
+        # FIXME: al-khaser refers the object named 'DebugObject' twice: the first time it creates a handle
+        # for it (so number of handles is expected to be higher than 0) and then closes it. the next time
+        # it accesses it (here), it expects the number of handles to be 0.
+        #
+        # ideally we would track the handles for each object, but since we do not - this is a hack to let
+        # it pass.
+        oti_obj.TotalNumberOfHandles = 0
+
+        oati_obj = oati_struct(
+            NumberOfObjectTypes   = 1,
+            ObjectTypeInformation = (oti_obj,)
+        )
+
+        out = oati_obj
 
     else:
-        raise QlErrorNotImplemented("API not implemented")
+        raise QlErrorNotImplemented(f'API not implemented ({ObjectInformationClass=})')
 
-    if dest and params["Handle"]:
-        res.write(dest)
+    if ReturnLength:
+        ql.mem.write_ptr(ReturnLength, out.sizeof(), 4)
 
-    if size_dest:
-        ql.mem.write_ptr(size_dest, res.size, 4)
+    if ObjectInformationLength < out.sizeof():
+        return STATUS_INFO_LENGTH_MISMATCH
+
+    if ObjectInformation and handle:
+        out.save_to(ql.mem, ObjectInformation)
 
     return STATUS_SUCCESS
 
@@ -289,7 +319,7 @@ def _SetInformationProcess(ql: Qiling, address: int, params):
     process = params["ProcessHandle"]
     flag = params["ProcessInformationClass"]
     dst = params["ProcessInformation"]
-    pt_res = params["ReturnLength"]
+    dst_size = params["ProcessInformationLength"]
 
     if flag == ProcessDebugFlags:
         value = b"\x01" * 0x4
@@ -310,23 +340,26 @@ def _SetInformationProcess(ql: Qiling, address: int, params):
             ql.mem.write_ptr(dst, 0, 1)
 
     elif flag == ProcessBasicInformation:
-        pbi = structs.ProcessBasicInformation(
-            ql,
-            exitStatus=0,
-            pebBaseAddress=ql.loader.TEB.PebAddress,
-            affinityMask=0,
-            basePriority=0,
-            uniqueId=ql.os.profile.getint("KERNEL", "pid"),
-            parentPid=ql.os.profile.getint("KERNEL", "parent_pid")
+        kconf = ql.os.profile['KERNEL']
+        pbi_struct = structs.make_process_basic_info(ql.arch.bits)
+
+        pci_obj = pbi_struct(
+            ExitStatus=0,
+            PebBaseAddress=ql.loader.TEB.PebAddress,
+            AffinityMask=0,
+            BasePriority=0,
+            UniqueProcessId=kconf.getint('pid'),
+            InheritedFromUniqueProcessId=kconf.getint('parent_pid')
         )
 
         ql.log.debug("The target may be attempting to modify the PEB debug flag")
-        addr = ql.os.heap.alloc(pbi.size)
-        pbi.write(addr)
-        value = ql.pack(addr)
+        value = bytes(pbi_obj)
+
     else:
-        ql.log.debug(str(flag))
-        raise QlErrorNotImplemented("API not implemented")
+        # TODO: support more info class ("flag") values
+        ql.log.info(f'SetInformationProcess: no implementation for info class {flag:#04x}')
+
+        return STATUS_UNSUCCESSFUL
 
     # TODO: value is never used after assignment
 

qiling/os/windows/dlls/ntoskrnl.py

@@ -14,15 +14,6 @@ from qiling.os.windows.structs import *
 from qiling.os.windows.wdk_const import DO_DEVICE_INITIALIZING, DO_EXCLUSIVE
 from qiling.utils import verify_ret
 
-# typedef struct _OSVERSIONINFOW {
-#   ULONG dwOSVersionInfoSize;
-#   ULONG dwMajorVersion;
-#   ULONG dwMinorVersion;
-#   ULONG dwBuildNumber;
-#   ULONG dwPlatformId;
-#   WCHAR szCSDVersion[128];
-# }
-
 # NTSYSAPI NTSTATUS RtlGetVersion(
 #   PRTL_OSVERSIONINFOW lpVersionInformation
 # );
@@ -30,13 +21,16 @@ from qiling.utils import verify_ret
     'lpVersionInformation' : PRTL_OSVERSIONINFOW
 })
 def hook_RtlGetVersion(ql: Qiling, address: int, params):
-    pointer = params["lpVersionInformation"]
+    pointer = params['lpVersionInformation']
 
-    os = OsVersionInfoW(ql)
-    os.read(pointer)
-    os.major[0] = ql.os.profile.getint("SYSTEM", "majorVersion")
-    os.minor[0] = ql.os.profile.getint("SYSTEM", "minorVersion")
-    os.write(pointer)
+    osverinfo_struct = make_os_version_info(ql.arch.bits, wide=True)
+    with osverinfo_struct.ref(ql.mem, pointer) as osverinfo_obj:
+        # read the necessary information from KUSER_SHARED_DATA
+        kusd_obj = ql.os.KUSER_SHARED_DATA
+
+        osverinfo_obj.dwOSVersionInfoSize = osverinfo_struct.sizeof()
+        osverinfo_obj.dwMajorVersion = kusd_obj.NtMajorVersion
+        osverinfo_obj.dwMinorVersion = kusd_obj.NtMinorVersion
 
     ql.log.debug("The target is checking the windows Version!")
 
@@ -77,6 +71,16 @@ def hook_ZwSetInformationThread(ql: Qiling, address: int, params):
 
     return STATUS_SUCCESS
 
+def __Close(ql: Qiling, address: int, params):
+    value = params["Handle"]
+
+    handle = ql.os.handle_manager.get(value)
+
+    if handle is None:
+        return STATUS_INVALID_HANDLE
+
+    return STATUS_SUCCESS
+
 # NTSYSAPI NTSTATUS ZwClose(
 #   HANDLE Handle
 # );
@@ -84,27 +88,13 @@ def hook_ZwSetInformationThread(ql: Qiling, address: int, params):
     'Handle' : HANDLE
 })
 def hook_ZwClose(ql: Qiling, address: int, params):
-    value = params["Handle"]
-
-    handle = ql.os.handle_manager.get(value)
-
-    if handle is None:
-        return STATUS_INVALID_HANDLE
-
-    return STATUS_SUCCESS
+    return __Close(ql, address, params)
 
 @winsdkapi(cc=STDCALL, params={
     'Handle' : HANDLE
 })
 def hook_NtClose(ql: Qiling, address: int, params):
-    value = params["Handle"]
-
-    handle = ql.os.handle_manager.get(value)
-
-    if handle is None:
-        return STATUS_INVALID_HANDLE
-
-    return STATUS_SUCCESS
+    return __Close(ql, address, params)
 
 # NTSYSAPI ULONG DbgPrintEx(
 #   ULONG ComponentId,
@@ -124,12 +114,10 @@ def hook_DbgPrintEx(ql: Qiling, address: int, params):
     if Format == 0:
         Format = "(null)"
 
-    nargs = Format.count("%")
-    ptypes = (ULONG, ULONG, POINTER) + (PARAM_INTN, ) * nargs
-    args = ql.os.fcall.readParams(ptypes)[3:]
+    args = ql.os.fcall.readEllipsis(params.values())
 
-    count = ql.os.utils.printf(Format, args, wstring=False)
-    ql.os.utils.update_ellipsis(params, args)
+    count, upd_args = ql.os.utils.printf(Format, args, wstring=False)
+    upd_args(params)
 
     return count
 
@@ -147,12 +135,10 @@ def hook_DbgPrint(ql: Qiling, address: int, params):
     if Format == 0:
         Format = "(null)"
 
-    nargs = Format.count("%")
-    ptypes = (POINTER, ) + (PARAM_INTN, ) * nargs
-    args = ql.os.fcall.readParams(ptypes)[1:]
+    args = ql.os.fcall.readEllipsis(params.values())
 
-    count = ql.os.utils.printf(Format, args, wstring=False)
-    ql.os.utils.update_ellipsis(params, args)
+    count, upd_args = ql.os.utils.printf(Format, args, wstring=False)
+    upd_args(params)
 
     return count
 
@@ -162,30 +148,27 @@ def __IoCreateDevice(ql: Qiling, address: int, params):
     DeviceCharacteristics = params['DeviceCharacteristics']
     DeviceObject = params['DeviceObject']
 
-    device_object = make_device_object(ql.arch.bits)
-    device_object.Type = 3 # FILE_DEVICE_CD_ROM_FILE_SYSTEM ?
-    device_object.DeviceExtension = ql.os.heap.alloc(DeviceExtensionSize)
-    device_object.Size = ctypes.sizeof(device_object) + DeviceExtensionSize
-    device_object.ReferenceCount = 1
-    device_object.DriverObject.value = DriverObject
-    device_object.NextDevice.value = 0
-    device_object.AttachedDevice.value = 0
-    device_object.CurrentIrp.value = 0
-    device_object.Timer.value = 0
-    device_object.Flags = DO_DEVICE_INITIALIZING
+    devobj_struct = make_device_object(ql.arch.bits)
+    devobj_addr = ql.os.heap.alloc(devobj_struct.sizeof())
 
-    if params.get('Exclusive'):
-        device_object.Flags |= DO_EXCLUSIVE
+    with devobj_struct.ref(ql.mem, devobj_addr) as devobj_obj:
+        devobj_obj.Type = 3 # FILE_DEVICE_CD_ROM_FILE_SYSTEM ?
+        devobj_obj.DeviceExtension = ql.os.heap.alloc(DeviceExtensionSize)
+        devobj_obj.Size = devobj_struct.sizeof() + DeviceExtensionSize
+        devobj_obj.ReferenceCount = 1
+        devobj_obj.DriverObject = DriverObject
+        devobj_obj.NextDevice = 0
+        devobj_obj.AttachedDevice = 0
+        devobj_obj.CurrentIrp = 0
+        devobj_obj.Timer = 0
+        devobj_obj.Flags = DO_DEVICE_INITIALIZING | (DO_EXCLUSIVE if params.get('Exclusive') else 0)
+        devobj_obj.Characteristics = DeviceCharacteristics
 
-    device_object.Characteristics = DeviceCharacteristics
-
-    addr = ql.os.heap.alloc(ctypes.sizeof(device_object))
-
-    ql.mem.write(addr, bytes(device_object))
-    ql.mem.write_ptr(DeviceObject, addr)
+    # update out param
+    ql.mem.write_ptr(DeviceObject, devobj_addr)
 
     # update DriverObject.DeviceObject
-    ql.loader.driver_object.DeviceObject = addr
+    ql.loader.driver_object.DeviceObject = devobj_addr
 
     return STATUS_SUCCESS
 
@@ -638,12 +621,12 @@ def hook_KeLeaveCriticalRegion(ql: Qiling, address: int, params):
 def hook_MmMapLockedPagesSpecifyCache(ql: Qiling, address: int, params):
     MemoryDescriptorList = params['MemoryDescriptorList']
 
-    mdl_class = make_mdl(ql.arch.bits).__class__
+    mdl_struct = make_mdl(ql.arch.bits)
 
-    mdl_buffer = ql.mem.read(MemoryDescriptorList, ctypes.sizeof(mdl_class))
-    mdl = mdl_class.from_buffer(mdl_buffer)
+    with mdl_struct.ref(ql.mem, MemoryDescriptorList) as mdl_obj:
+        address = mdl_obj.MappedSystemVa
 
-    return mdl.MappedSystemVa.value
+    return address
 
 # void ProbeForRead(
 # const volatile VOID *Address,
@@ -747,43 +730,51 @@ def hook_RtlMultiByteToUnicodeN(ql: Qiling, address: int, params):
 #   OUT PULONG                  ReturnLength
 # );
 def _NtQuerySystemInformation(ql: Qiling, address: int, params):
-    if params["SystemInformationClass"] == 0xb:  # SystemModuleInformation
-        # if SystemInformationLength = 0, we return the total size in ReturnLength
-        NumberOfModules = 1
+    # see: https://www.geoffchappell.com/studies/windows/km/ntoskrnl/api/ex/sysinfo/query.htm
 
-        rpmi_class = make_rtl_process_module_info(ql.arch.bits).__class__
+    SystemInformationClass = params['SystemInformationClass']
+    ReturnLength = params['ReturnLength']
+    SystemInformationLength = params['SystemInformationLength']
+    SystemInformation = params['SystemInformation']
 
+    if SystemInformationClass == 0xb:  # SystemModuleInformation
         # only 1 module for ntoskrnl.exe
         # FIXME: let users customize this?
-        size = 4 + ctypes.sizeof(rpmi_class) * NumberOfModules
+        num_modules = 1
 
-        if params["ReturnLength"] != 0:
-            ql.mem.write_ptr(params["ReturnLength"], size)
+        rpm_struct = make_rtl_process_modules(ql.arch.bits, num_modules)
 
-        if params["SystemInformationLength"] < size:
+        if ReturnLength:
+            ql.mem.write_ptr(ReturnLength, rpm_struct.sizeof())
+
+        # if SystemInformationLength = 0, we return the total size in ReturnLength
+        if SystemInformationLength < rpm_struct.sizeof():
             return STATUS_INFO_LENGTH_MISMATCH
 
-        else:  # return all the loaded modules
-            module = make_rtl_process_module_info(ql.arch.bits)
-            module.Section = 0
-            module.MappedBase = 0
+        with rpm_struct.ref(ql.mem, SystemInformation) as rpm_obj:
+            rpm_obj.NumberOfModules = num_modules
 
-            if ql.loader.is_driver:
-                image = ql.loader.get_image_by_name("ntoskrnl.exe")
-                assert image, 'image is a driver, but ntoskrnl.exe was not loaded'
+            # cycle through all the loaded modules
+            for i in range(num_modules):
+                rpmi_obj = rpm_obj.Modules[i]
 
-                module.ImageBase = image.base
+                # FIXME: load real values instead of bogus ones
+                rpmi_obj.Section = 0
+                rpmi_obj.MappedBase = 0
 
-            module.ImageSize = 0xab000
-            module.Flags = 0x8804000
-            module.LoadOrderIndex = 0  # order of this module
-            module.InitOrderIndex = 0
-            module.LoadCount = 1
-            module.OffsetToFileName = len(b"\\SystemRoot\\system32\\")
-            module.FullPathName = b"\\SystemRoot\\system32\\ntoskrnl.exe"
+                if ql.loader.is_driver:
+                    image = ql.loader.get_image_by_name("ntoskrnl.exe")
+                    assert image, 'image is a driver, but ntoskrnl.exe was not loaded'
 
-            process_modules = ql.pack32(NumberOfModules) + bytes(module)
-            ql.mem.write(params["SystemInformation"], process_modules)
+                    rpmi_obj.ImageBase = image.base
+
+                rpmi_obj.ImageSize = 0xab000
+                rpmi_obj.Flags = 0x8804000
+                rpmi_obj.LoadOrderIndex = 0  # order of this module
+                rpmi_obj.InitOrderIndex = 0
+                rpmi_obj.LoadCount = 1
+                rpmi_obj.OffsetToFileName = len(b"\\SystemRoot\\system32\\")
+                rpmi_obj.FullPathName = b"\\SystemRoot\\system32\\ntoskrnl.exe"
 
     return STATUS_SUCCESS
 
@@ -871,8 +862,6 @@ def hook_PsGetCurrentProcess(ql: Qiling, address: int, params):
 # HANDLE PsGetCurrentProcessId();
 @winsdkapi(cc=STDCALL, params={})
 def hook_PsGetCurrentProcessId(ql: Qiling, address: int, params):
-    # current process ID is 101
-    # TODO: let user customize this?
     return ql.os.pid
 
 # NTSTATUS

qiling/os/windows/dlls/shell32.py

@@ -5,7 +5,6 @@
 
 import ntpath
 import os
-from typing import Sequence
 
 from qiling import Qiling
 from qiling.os.windows.api import *
@@ -13,9 +12,9 @@ from qiling.os.windows.const import *
 from qiling.os.windows.fncc import *
 
 from qiling.os.windows.handle import Handle
-from qiling.os.windows.thread import QlWindowsThread
+from qiling.os.windows.thread import QlWindowsThread, THREAD_STATUS
 from qiling.exception import QlErrorNotImplemented
-from qiling.os.windows.structs import ShellExecuteInfoA
+from qiling.os.windows.structs import make_shellex_info
 
 def _SHGetFileInfo(ql: Qiling, address: int, params) -> int:
     uFlags = params["uFlags"]
@@ -61,20 +60,22 @@ def hook_SHGetFileInfoA(ql: Qiling, address: int, params):
 def hook_SHGetFileInfoW(ql: Qiling, address: int, params):
     return _SHGetFileInfo(ql, address, params)
 
-def _ShellExecute(ql: Qiling, obj: ShellExecuteInfoA):
-    def __wstr(shellex: Sequence):
-        return ql.os.utils.read_wstring(shellex[0]) if shellex[0] else ''
+def _ShellExecute(ql: Qiling, shellex_obj, *, wide: bool):
+    read_str = ql.os.utils.read_wstring if wide else ql.os.utils.read_cstring
+
+    def __read_str(ptr: int):
+        return read_str(ptr) if ptr else ''
 
     ql.log.debug(f'Target executed a shell command!')
-    ql.log.debug(f' | Operation  : "{__wstr(obj.verb)}"')
-    ql.log.debug(f' | Parameters : "{__wstr(obj.params)}"')
-    ql.log.debug(f' | File       : "{__wstr(obj.file)}"')
-    ql.log.debug(f' | Directory  : "{__wstr(obj.dir)}"')
+    ql.log.debug(f' | Operation  : "{__read_str(shellex_obj.lpVerb)}"')
+    ql.log.debug(f' | File       : "{__read_str(shellex_obj.lpFile)}"')
+    ql.log.debug(f' | Parameters : "{__read_str(shellex_obj.lpParameters)}"')
+    ql.log.debug(f' | Directory  : "{__read_str(shellex_obj.lpDirectory)}"')
 
-    if obj.show[0] == SW_HIDE:
-        ql.log.debug(" | With an hidden window")
+    if shellex_obj.nShow == SW_HIDE:
+        ql.log.debug(' | With an hidden window')
 
-    process = QlWindowsThread(ql, status=0, isFake=True)
+    process = QlWindowsThread(ql, status=THREAD_STATUS.READY)
     handle = Handle(obj=process)
     ql.os.handle_manager.append(handle)
 
@@ -87,17 +88,16 @@ def _ShellExecute(ql: Qiling, obj: ShellExecuteInfoA):
     'pExecInfo' : POINTER
 })
 def hook_ShellExecuteExW(ql: Qiling, address: int, params):
-    pointer = params["pExecInfo"]
+    pExecInfo = params['pExecInfo']
 
-    shellInfo = ShellExecuteInfoA(ql)
-    shellInfo.read(pointer)
+    shellex_struct = make_shellex_info(ql.arch.bits)
 
-    handle = _ShellExecute(ql, shellInfo)
+    with shellex_struct.ref(ql.mem, pExecInfo) as shellex_obj:
+        handle = _ShellExecute(ql, shellex_obj, wide=True)
 
-    # Write results
-    shellInfo.instApp[0] = 0x21
-    shellInfo.process[0] = handle.id
-    shellInfo.write(pointer)
+        # Write results
+        shellex_obj.hInstApp = 33
+        shellex_obj.hProcess = handle.id
 
     return 1
 
@@ -118,15 +118,18 @@ def hook_ShellExecuteExW(ql: Qiling, address: int, params):
     'nShowCmd'     : INT
 })
 def hook_ShellExecuteW(ql: Qiling, address: int, params):
-    _ShellExecute(ql, ShellExecuteInfoA(
-        ql,
-        hwnd=params["hwnd"],
-        lpVerb=params["lpOperation"],
-        lpFile=params["lpFile"],
-        lpParams=params["lpParameters"],
-        lpDir=params["lpDirectory"],
-        show=params["nShowCmd"]
-    ))
+    shellex_struct = make_shellex_info(ql.arch.bits)
+
+    shellex_obj = shellex_struct(
+        hwnd         = params['hwnd'],
+        lpVerb       = params['lpOperation'],
+        lpFile       = params['lpFile'],
+        lpParameters = params['lpParameters'],
+        lpDirectory  = params['lpDirectory'],
+        nShow        = params['nShowCmd']
+    )
+
+    _ShellExecute(ql, shellex_obj, wide=True)
 
     return 33
 

qiling/os/windows/dlls/user32.py

@@ -156,13 +156,15 @@ def hook_CloseClipboard(ql: Qiling, address: int, params):
     'hMem'    : HANDLE
 })
 def hook_SetClipboardData(ql: Qiling, address: int, params):
-    try:
-        data = bytes(params['hMem'], 'ascii', 'ignore')
-    except (UnicodeEncodeError, TypeError):
-        data = b""
-        ql.log.debug('Failed to set clipboard data')
+    uFormat = params['uFormat']
+    hMem = params['hMem']
 
-    return ql.os.clipboard.set_data(params['uFormat'], data)
+    handle = ql.os.clipboard.set_data(uFormat, hMem)
+
+    if not handle:
+        ql.log.debug(f'Failed to set clipboard data (format = {uFormat})')
+
+    return handle
 
 
 # HANDLE GetClipboardData(
@@ -172,14 +174,17 @@ def hook_SetClipboardData(ql: Qiling, address: int, params):
     'uFormat' : UINT
 })
 def hook_GetClipboardData(ql: Qiling, address: int, params):
-    data = ql.os.clipboard.get_data(params['uFormat'])
+    uFormat = params['uFormat']
+
+    data = ql.os.clipboard.get_data(uFormat)
 
     if data:
         addr = ql.os.heap.alloc(len(data))
         ql.mem.write(addr, data)
+
     else:
+        ql.log.debug(f'Failed to get clipboard data (format = {uFormat})')
         addr = 0
-        ql.log.debug('Failed to get clipboard data')
 
     return addr
 
@@ -690,12 +695,10 @@ def hook_wsprintfW(ql: Qiling, address: int, params):
     if Format == 0:
         Format = "(null)"
 
-    nargs = Format.count("%")
-    ptypes = (POINTER, POINTER) + (PARAM_INTN, ) * nargs
-    args = ql.os.fcall.readParams(ptypes)[2:]
+    args = ql.os.fcall.readEllipsis(params.values())
 
-    count = ql.os.utils.sprintf(Buffer, Format, args, wstring=True)
-    ql.os.utils.update_ellipsis(params, args)
+    count, upd_args = ql.os.utils.sprintf(Buffer, Format, args, wstring=True)
+    upd_args(params)
 
     return count
 
@@ -771,12 +774,10 @@ def hook_wsprintfA(ql: Qiling, address: int, params):
     if Format == 0:
         Format = "(null)"
 
-    nargs = Format.count("%")
-    ptypes = (POINTER, POINTER) + (PARAM_INTN, ) * nargs
-    args = ql.os.fcall.readParams(ptypes)[2:]
+    args = ql.os.fcall.readEllipsis(params.values())
 
-    count = ql.os.utils.sprintf(Buffer, Format, args, wstring=False)
-    ql.os.utils.update_ellipsis(params, args)
+    count, upd_args = ql.os.utils.sprintf(Buffer, Format, args, wstring=False)
+    upd_args(params)
 
     return count
 
@@ -793,17 +794,27 @@ def hook_wsprintfA(ql: Qiling, address: int, params):
     'uType'     : UINT
 })
 def hook_MessageBoxW(ql: Qiling, address: int, params):
-    # We always return a positive result
-    type_box = params["uType"]
+    uType = params["uType"]
 
-    if type_box in (MB_OK, MB_OKCANCEL):
-        return IDOK
+    buttons = uType & 0x0000000f
+    # icon    = uType & 0x000000f0
+    # default = uType & 0x00000f00
+    # modal   = uType & 0x0000f000
+    # order   = uType & 0x00ff0000
 
-    if type_box in (MB_YESNO, MB_YESNOCANCEL):
-        return IDYES
+    # we strive to return a positive result when possible.
+    # if there is an "ok", "yes" or "continue" button, press it
+    press = {
+        MB_OK                : IDOK,
+        MB_OKCANCEL          : IDOK,
+        MB_ABORTRETRYIGNORE  : IDABORT,
+        MB_YESNOCANCEL       : IDYES,
+        MB_YESNO             : IDYES,
+        MB_RETRYCANCEL       : IDCANCEL,
+        MB_CANCELTRYCONTINUE : IDCONTINUE
+    }
 
-    ql.log.debug(type_box)
-    raise QlErrorNotImplemented("API not implemented")
+    return press[buttons]
 
 # int MessageBoxA(
 #   HWND    hWnd,
@@ -827,11 +838,11 @@ def hook_MessageBoxA(ql: Qiling, address: int, params):
     'lpPoint' : LPPOINT
 })
 def hook_GetCursorPos(ql: Qiling, address: int, params):
-    dest = params["lpPoint"]
+    lpPoint = params['lpPoint']
 
     # TODO: maybe we can add it to the profile too
-    p = Point(ql, 50, 50)
-    p.write(dest)
+    p = Point(50, 50)
+    p.save_to(ql.mem, lpPoint)
 
     return 0
 

qiling/os/windows/dlls/wsock32.py

@@ -51,22 +51,41 @@ def hook_WSASocketA(ql: Qiling, address: int, params):
     'namelen' : INT
 })
 def hook_connect(ql: Qiling, address: int, params):
-    sin_family = ql.mem.read(params["name"], 1)[0]
-    sin_port = int.from_bytes(ql.mem.read(params["name"] + 2, 2), byteorder="big")
+    name = params["name"]
+    namelen = params['namelen']
+
+    # sockaddr structure type needs to be defined based on the family field value.
+    # both types (for ipv4 and ipv6) have it on the same offset, so we assume ipv4
+    # by default for convinience.
+    sockaddr_struct = make_sockaddr_in()
+
+    # sockaddr is not based on BaseStruct, so we have to read it manually
+    sockaddr_data = ql.mem.read(name, namelen)
+    sockaddr_obj = sockaddr_struct.from_buffer_copy(sockaddr_data[:ctypes.sizeof(sockaddr_struct)])
+
+    sin_family = sockaddr_obj.sin_family
+    sin_port = sockaddr_obj.sin_port
 
     if sin_family == 0x17:  # IPv6
-        segments = list(map("{:02x}".format, ql.mem.read(params["name"] + 8, 16)))
-        sin_addr = ":".join(["".join(x) for x in zip(segments[0::2], segments[1::2])])
+        sockaddr_struct = make_sockaddr_in6()
+        sockaddr_obj = sockaddr_struct.from_buffer_copy(sockaddr_data[:ctypes.sizeof(sockaddr_struct)])
+
+        # read address bytes and show them as pairs
+        segments = [f'{b:02x}' for b in sockaddr_obj.sin6_addr.Byte]
+        sin_addr = ':'.join(''.join(x) for x in zip(segments[0::2], segments[1::2]))
 
     elif sin_family == 0x2:  # IPv4
-        sin_addr = ".".join((str(octet) for octet in ql.mem.read(params["name"] + 4, 4)))
+        a = sockaddr_obj.sin_addr
+        sin_addr = '.'.join((a.s_b1, a.s_b2, a.s_b3, a.s_b4))
+
     else:
         ql.log.debug("sockaddr sin_family unhandled variant")
         return 0
 
-    ql.log.debug(f"{params['name']:#010x}: sockaddr_in{6 if sin_family == 0x17 else ''}",
-              f"{{sin_family={sin_family:#04x}, sin_port={sin_port}, sin_addr={sin_addr}}}",
-              sep="")
+    ql.log.debug(f'{sockaddr_struct.__name__} @ {name:#010x} : {sin_family=:#04x}, {sin_port=}, {sin_addr=}')
+
+    # FIXME: wait.. we just printed stuff out, and did not connect anywhere..
+
     return 0
 
 # hostent * gethostbyname(
@@ -76,14 +95,39 @@ def hook_connect(ql: Qiling, address: int, params):
     'name' : POINTER
 })
 def hook_gethostbyname(ql: Qiling, address: int, params):
-    ip_str = ql.os.profile.get("NETWORK", "dns_response_ip")
+    name_ptr = params['name']
+
+    # set string value back to arg to let it show on log
+    params['name'] = ql.os.utils.read_cstring(name_ptr)
+
+    # prepare the ip address data bytes
+    ip_str = ql.os.profile.get('NETWORK', 'dns_response_ip')
     ip = bytes((int(octet) for octet in reversed(ip_str.split('.'))))
 
-    name_ptr = params["name"]
-    # params["name"] = ql.os.utils.read_cstring(name_ptr)
+    # allocate room for the ip address data bytes
+    list_item = ql.os.heap.alloc(len(ip))
+    ql.mem.write(list_item, ip)
 
-    hostnet = Hostent(ql, name_ptr, 0, 2, 4, ip)
-    hostnet_addr = ql.os.heap.alloc(hostnet.size)
-    hostnet.write(hostnet_addr)
+    # allocate room for a list with one item, followed by a nullptr sentinel
+    addr_list = ql.os.heap.alloc(2 * ql.arch.pointersize)
+
+    ql.mem.write_ptr(addr_list + 0 * ql.arch.pointersize, list_item)
+    ql.mem.write_ptr(addr_list + 1 * ql.arch.pointersize, 0)
+
+    # we need a pointer to a nullptr; reuse the sentinel address for that
+    d_nullptr = addr_list + 1 * ql.arch.pointersize
+
+    hostent_struct = make_hostent(ql.arch.bits)
+    hostnet_addr = ql.os.heap.alloc(hostent_struct.sizeof())
+
+    hostent_obj = hostent_struct(
+        h_name      = name_ptr,
+        h_aliases   = d_nullptr,
+        h_addrtype  = 2,  # AF_INET
+        h_length    = len(ip),
+        h_addr_list = addr_list
+    )
+
+    hostent_obj.save_to(ql.mem, hostnet_addr)
 
     return hostnet_addr

qiling/os/windows/handle.py

@@ -32,24 +32,25 @@ class HandleManager:
     STDOUT = Handle(id=STD_OUTPUT_HANDLE)
     STDERR = Handle(id=STD_ERROR_HANDLE)
 
-    # Register
-    HKEY_CLASSES_ROOT        = Handle(id=0x80000000)
-    HKEY_CURRENT_CONFIG      = Handle(id=0x80000005)
-    HKEY_CURRENT_USER        = Handle(id=0x80000001)
-    HKEY_CURRENT_USER_LOCAL_SETTINGS = Handle(id=0x80000007)
-    HKEY_LOCAL_MACHINE       = Handle(id=0x80000002)
-    HKEY_PERFORMANCE_DATA    = Handle(id=0x80000004)
-    HKEY_PERFORMANCE_NLSTEXT = Handle(id=0x80000060)
-    HKEY_PERFORMANCE_TEXT    = Handle(id=0x80000050)
-    HKEY_USERS               = Handle(id=0x80000003)
-
     def __init__(self):
         self.handles: MutableMapping[int, Handle] = {}
 
+        # standard io streams
         self.append(HandleManager.STDIN)
         self.append(HandleManager.STDOUT)
         self.append(HandleManager.STDERR)
 
+        # registry hives
+        self.append(Handle(id=0x80000000, name='HKEY_CLASSES_ROOT'))
+        self.append(Handle(id=0x80000001, name='HKEY_CURRENT_USER'))
+        self.append(Handle(id=0x80000002, name='HKEY_LOCAL_MACHINE'))
+        self.append(Handle(id=0x80000003, name='HKEY_USERS'))
+        self.append(Handle(id=0x80000004, name='HKEY_PERFORMANCE_DATA'))
+        self.append(Handle(id=0x80000005, name='HKEY_CURRENT_CONFIG'))
+        self.append(Handle(id=0x80000007, name='HKEY_CURRENT_USER_LOCAL_SETTINGS'))
+        self.append(Handle(id=0x80000060, name='HKEY_PERFORMANCE_NLSTEXT'))
+        self.append(Handle(id=0x80000050, name='HKEY_PERFORMANCE_TEXT'))
+
     def append(self, handle: Handle) -> None:
         self.handles[handle.id] = handle
 

qiling/os/windows/registry.py

@@ -74,7 +74,7 @@ class RegConf:
     def save(self, fname: str):
         if self.conf:
             with open(fname, 'wb') as ofile:
-                data = json.dumps(self.conf)
+                data = json.dumps(self.conf, indent=4)
 
                 ofile.write(data.encode('utf-8'))
 

qiling/os/windows/thread.py

@@ -3,23 +3,26 @@
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 #
 
+from enum import Enum
 from typing import TYPE_CHECKING, cast
 
 from qiling import Qiling
-from qiling.const import QL_ARCH
+from qiling.const import QL_ARCH, QL_HOOK_BLOCK
 from qiling.os.thread import QlThread
 
 if TYPE_CHECKING:
     from qiling.os.windows.windows import QlOsWindows
 
-class QlWindowsThread(QlThread):
-    # static var
-    ID = 0
+class THREAD_STATUS(Enum):
     READY = 0
     RUNNING = 1
     TERMINATED = 2
 
-    def __init__(self, ql: Qiling, status: int = 1, isFake: bool = False):
+class QlWindowsThread(QlThread):
+    # static var
+    ID = 0
+
+    def __init__(self, ql: Qiling, status: THREAD_STATUS = THREAD_STATUS.RUNNING):
         super().__init__(ql)
 
         self.ql = ql
@@ -29,56 +32,56 @@ class QlWindowsThread(QlThread):
         self.waitforthreads = []
         self.tls = {}
         self.tls_index = 0
-        self.fake = isFake
 
-    # create new thread
-    def create(self, func_addr: int, func_params: int, status: int) -> int:
-        os = cast('QlOsWindows', self.ql.os)
+    # create a new thread with context
+    @classmethod
+    def create(cls, ql: Qiling, stack_size: int, func_addr: int, func_params: int, status: THREAD_STATUS) -> 'QlWindowsThread':
+        os = cast('QlOsWindows', ql.os)
+
+        thread = cls(ql, status)
 
         # create new stack
-        stack_size = 1024
         new_stack = os.heap.alloc(stack_size) + stack_size
 
-        asize = self.ql.arch.pointersize
-        context = self.ql.arch.regs.save()
+        asize = ql.arch.pointersize
+        context = ql.arch.regs.save()
 
         # set return address
-        self.ql.mem.write_ptr(new_stack - asize, os.thread_manager.thread_ret_addr)
+        ql.mem.write_ptr(new_stack - asize, os.thread_manager.thread_ret_addr)
 
         # set parameters
-        if self.ql.arch.type == QL_ARCH.X86:
-            self.ql.mem.write_ptr(new_stack, func_params)
-        elif self.ql.arch.type == QL_ARCH.X8664:
+        if ql.arch.type == QL_ARCH.X86:
+            ql.mem.write_ptr(new_stack, func_params)
+        elif ql.arch.type == QL_ARCH.X8664:
             context["rcx"] = func_params
 
         # set eip/rip, ebp/rbp, esp/rsp
-        if self.ql.arch.type == QL_ARCH.X86:
+        if ql.arch.type == QL_ARCH.X86:
             context["eip"] = func_addr
             context["ebp"] = new_stack - asize
             context["esp"] = new_stack - asize
 
-        elif self.ql.arch.type == QL_ARCH.X8664:
+        elif ql.arch.type == QL_ARCH.X8664:
             context["rip"] = func_addr
             context["rbp"] = new_stack - asize
             context["rsp"] = new_stack - asize
 
-        self.saved_context = context
-        self.status = status
+        thread.saved_context = context
 
-        return self.id
+        return thread
 
     def suspend(self) -> None:
         self.saved_context = self.ql.arch.regs.save()
 
     def resume(self) -> None:
         self.ql.arch.regs.restore(self.saved_context)
-        self.status = QlWindowsThread.RUNNING
+        self.status = THREAD_STATUS.RUNNING
 
     def stop(self) -> None:
-        self.status = QlWindowsThread.TERMINATED
+        self.status = THREAD_STATUS.TERMINATED
 
     def is_stop(self) -> bool:
-        return self.status == QlWindowsThread.TERMINATED
+        return self.status == THREAD_STATUS.TERMINATED
 
     def waitfor(self, thread: 'QlWindowsThread') -> None:
         self.waitforthreads.append(thread)
@@ -88,13 +91,12 @@ class QlWindowsThread(QlThread):
 
 
 # Simple Thread Manager
-class QlWindowsThreadManagement(QlThread):
+class QlWindowsThreadManagement:
     TIME_SLICE = 10
 
     def __init__(self, ql: Qiling, os: 'QlOsWindows', cur_thread: QlWindowsThread):
-        super().__init__(ql)
-
         self.ql = ql
+
         # main thread
         self.cur_thread = cur_thread
         self.threads = [self.cur_thread]
@@ -110,29 +112,37 @@ class QlWindowsThreadManagement(QlThread):
             else:
                 self.icount += 1
 
-            self.do_schedule()
+            switched = self.do_schedule()
+
+            # in case another thread was resumed, all remaining hooks should be skipped to prevent them
+            # from running with the new thread's context.
+
+            return QL_HOOK_BLOCK if switched else 0
 
         ql.hook_code(__thread_scheduler)
 
     def append(self, thread: QlWindowsThread):
         self.threads.append(thread)
 
-    def need_schedule(self):
-        return self.cur_thread.is_stop() or (self.icount % QlWindowsThreadManagement.TIME_SLICE) == 0
+    def do_schedule(self) -> bool:
+        need_schedule = self.cur_thread.is_stop() or (self.icount % QlWindowsThreadManagement.TIME_SLICE) == 0
+        switched = False
 
-    def do_schedule(self) -> None:
-        if self.need_schedule():
+        if need_schedule:
             # if there is less than one thread, this loop won't run
             for i in range(1, len(self.threads)):
                 next_id = (self.cur_thread.id + i) % len(self.threads)
                 next_thread = self.threads[next_id]
 
                 # find next thread
-                if next_thread.status == QlWindowsThread.RUNNING and not next_thread.has_waitfor():
+                if next_thread.status == THREAD_STATUS.RUNNING and not next_thread.has_waitfor():
                     if not self.cur_thread.is_stop():
                         self.cur_thread.suspend()
 
                     next_thread.resume()
                     self.cur_thread = next_thread
 
+                    switched = True
                     break
+
+        return switched

qiling/os/windows/utils.py

@@ -3,13 +3,12 @@
 # Cross Platform and Multi Architecture Advanced Binary Emulation Framework
 #
 
-import ctypes
 from typing import Iterable, Tuple, TypeVar
 
 from unicorn import UcError
 
 from qiling import Qiling
-from qiling.const import QL_OS
+from qiling.exception import QlErrorSyscallError
 from qiling.os.const import POINTER
 from qiling.os.windows.fncc import STDCALL
 from qiling.os.windows.wdk_const import *
@@ -29,76 +28,76 @@ def has_lib_ext(name: str) -> bool:
     return ext in ("dll", "exe", "sys", "drv")
 
 
-def io_Write(ql: Qiling, in_buffer: bytes):
-    heap = ql.os.heap
+def io_Write(ql: Qiling, in_buffer: bytes) -> int:
+    major_func = ql.loader.driver_object.MajorFunction[IRP_MJ_WRITE]
 
-    if ql.loader.driver_object.MajorFunction[IRP_MJ_WRITE] == 0:
-        # raise error?
-        return (False, None)
+    if not major_func:
+        raise QlErrorSyscallError('null MajorFunction field')
 
-    driver_object_cls = ql.loader.driver_object.__class__
-    buf = ql.mem.read(ql.loader.driver_object.DeviceObject, ctypes.sizeof(driver_object_cls))
-    device_object = driver_object_cls.from_buffer(buf)
+    # keep track of all heap allocation within this scope to be able
+    # to free them when done
+    allocations = []
 
-    alloc_addr = []
-    def build_mdl(buffer_size: int, data=None):
-        mdl = make_mdl(ql.arch.bits)
+    def __heap_alloc(size: int) -> int:
+        address = ql.os.heap.alloc(size)
+        allocations.append(address)
 
-        mapped_address = heap.alloc(buffer_size)
-        alloc_addr.append(mapped_address)
-        mdl.MappedSystemVa.value = mapped_address
-        mdl.StartVa.value = mapped_address
-        mdl.ByteOffset = 0
-        mdl.ByteCount = buffer_size
-        if data:
-            written = data if len(data) <= buffer_size else data[:buffer_size]
-            ql.mem.write(mapped_address, written)
+        return address
 
-        return mdl
+    def __free_all(allocations: Iterable[int]) -> None:
+        for address in allocations:
+            ql.os.heap.free(address)
 
-    # allocate memory regions for IRP and IO_STACK_LOCATION
-    irp = make_irp(ql.arch.bits)
-    irpstack_class = irp.irpstack._type_
+    # allocate memory for IRP
+    irp_struct = make_irp(ql.arch.bits)
+    irp_addr = __heap_alloc(irp_struct.sizeof())
+    ql.log.info(f'IRP is at {irp_addr:#x}')
 
-    irp_addr = heap.alloc(ctypes.sizeof(irp))
-    alloc_addr.append(irp_addr)
+    # populate the structure
+    with irp_struct.ref(ql.mem, irp_addr) as irp_obj:
 
-    irpstack_addr = heap.alloc(ctypes.sizeof(irpstack_class))
-    alloc_addr.append(irpstack_addr)
+        # allocate memory for IO_STACK_LOCATION
+        irpstack_struct = make_io_stack_location(ql.arch.bits)
+        irpstack_addr = __heap_alloc(irpstack_struct.sizeof())
+        ql.log.info(f'IO_STACK_LOCATION is at {irpstack_addr:#x}')
 
-    # setup irp stack parameters
-    irpstack = irpstack_class()
-    # setup IRP structure
-    irp.irpstack = ctypes.cast(irpstack_addr, ctypes.POINTER(irpstack_class))
+        # populate the structure
+        with irpstack_struct.ref(ql.mem, irpstack_addr) as irpstack_obj:
+            irpstack_obj.MajorFunction = IRP_MJ_WRITE
+            irpstack_obj.Parameters.Write.Length = len(in_buffer)
 
-    irpstack.MajorFunction = IRP_MJ_WRITE
-    irpstack.Parameters.Write.Length = len(in_buffer)
-    ql.mem.write(irpstack_addr, bytes(irpstack))
+        # load DeviceObject from memory
+        drvobj_struct = ql.loader.driver_object.__class__
+        devobj_obj = drvobj_struct.load_from(ql.loader.driver_object.DeviceObject)
 
-    if device_object.Flags & DO_BUFFERED_IO:
         # BUFFERED_IO
-        system_buffer_addr = heap.alloc(len(in_buffer))
-        alloc_addr.append(system_buffer_addr)
-        ql.mem.write(system_buffer_addr, bytes(in_buffer))
-        irp.AssociatedIrp.SystemBuffer.value = system_buffer_addr
-    elif device_object.Flags & DO_DIRECT_IO:
+        if devobj_obj.Flags & DO_BUFFERED_IO:
+            system_buffer_addr = __heap_alloc(len(in_buffer))
+
+            ql.mem.write(system_buffer_addr, bytes(in_buffer))
+            irp_obj.AssociatedIrp.SystemBuffer = system_buffer_addr
+
         # DIRECT_IO
-        mdl = build_mdl(len(in_buffer))
-        mdl_addr = heap.alloc(ctypes.sizeof(mdl))
-        alloc_addr.append(mdl_addr)
+        elif devobj_obj.Flags & DO_DIRECT_IO:
+            mdl_struct = make_mdl(ql.arch.bits)
+            mdl_addr = __heap_alloc(mdl_struct.sizeof())
+
+            with mdl_struct.ref(ql.mem, mdl_addr) as mdl_obj:
+                mapped_address = __heap_alloc(len(in_buffer))
+
+                mdl_obj.MappedSystemVa = mapped_address
+                mdl_obj.StartVa = mapped_address
+                mdl_obj.ByteOffset = 0
+                mdl_obj.ByteCount = len(in_buffer)
+
+            irp_obj.MdlAddress = mdl_addr
 
-        ql.mem.write(mdl_addr, bytes(mdl))
-        irp.MdlAddress.value = mdl_addr
-    else:
         # NEITHER_IO
-        input_buffer_size = len(in_buffer)
-        input_buffer_addr = heap.alloc(input_buffer_size)
-        alloc_addr.append(input_buffer_addr)
-        ql.mem.write(input_buffer_addr, bytes(in_buffer))
-        irp.UserBuffer.value = input_buffer_addr
+        else:
+            input_buffer_addr = __heap_alloc(len(in_buffer))
 
-    # everything is done! Write IRP to memory
-    ql.mem.write(irp_addr, bytes(irp))
+            ql.mem.write(input_buffer_addr, bytes(in_buffer))
+            irp_obj.UserBuffer = input_buffer_addr
 
     # set function args
     # TODO: make sure this is indeed STDCALL
@@ -108,23 +107,22 @@ def io_Write(ql: Qiling, in_buffer: bytes):
         (POINTER, irp_addr)
     ))
 
+    ql.log.info(f'Executing from {major_func:#x}')
+
     try:
         # now emulate 
-        ql.run(ql.loader.driver_object.MajorFunction[IRP_MJ_WRITE])
+        ql.run(major_func)
     except UcError as err:
         verify_ret(ql, err)
 
-    # read current IRP state
-    irp_buffer = ql.mem.read(irp_addr, ctypes.sizeof(irp))
-    irp = irp.from_buffer(irp_buffer)
+    # read updated IRP state before releasing resources
+    with irp_struct.ref(ql.mem, irp_addr) as irp_obj:
+        info = irp_obj.IoStatus.Information
 
-    io_status = irp.IoStatus
-    # now free all alloc memory
-    for addr in alloc_addr:
-        # print("freeing heap memory at 0x%x" %addr) # FIXME: the output is not deterministic??
-        heap.free(addr)
+    # free all allocated memory
+    __free_all(allocations)
 
-    return True, io_status.Information.value
+    return info
 
 # Emulate DeviceIoControl() of Windows
 # BOOL DeviceIoControl(
@@ -136,8 +134,14 @@ def io_Write(ql: Qiling, in_buffer: bytes):
 #      DWORD        nOutBufferSize,
 #      LPDWORD      lpBytesReturned,
 #      LPOVERLAPPED lpOverlapped);
-def ioctl(ql: Qiling, params: Tuple[Tuple, int, bytes]) -> Tuple:
+def ioctl(ql: Qiling, params: Tuple[Tuple, int, bytes]) -> Tuple[int, int, bytes]:
+    major_func = ql.loader.driver_object.MajorFunction[IRP_MJ_DEVICE_CONTROL]
 
+    if not major_func:
+        raise QlErrorSyscallError('null MajorFunction field')
+
+    # keep track of all heap allocation within this scope to be able
+    # to free them when done
     allocations = []
 
     def __heap_alloc(size: int) -> int:
@@ -153,27 +157,9 @@ def ioctl(ql: Qiling, params: Tuple[Tuple, int, bytes]) -> Tuple:
     def ioctl_code(DeviceType: int, Function: int, Method: int, Access: int) -> int:
         return (DeviceType << 16) | (Access << 14) | (Function << 2) | Method
 
-    def build_mdl(buffer_size, data=None):
-        mdl = make_mdl(ql.arch.bits)
-
-        mapped_address = __heap_alloc(buffer_size)
-        mdl.MappedSystemVa.value = mapped_address
-        mdl.StartVa.value = mapped_address
-        mdl.ByteOffset = 0
-        mdl.ByteCount = buffer_size
-
-        if data:
-            written = data if len(data) <= buffer_size else data[:buffer_size]
-            ql.mem.write(mapped_address, written)
-
-        return mdl
-
-    if ql.loader.driver_object.MajorFunction[IRP_MJ_DEVICE_CONTROL] == 0:
-        # raise error?
-        return (None, None, None)
-
     # create new memory region to store input data
     _ioctl_code, output_buffer_size, in_buffer = params
+
     # extract data transfer method
     devicetype, function, ctl_method, access = _ioctl_code
 
@@ -184,52 +170,57 @@ def ioctl(ql: Qiling, params: Tuple[Tuple, int, bytes]) -> Tuple:
     # create new memory region to store out data
     output_buffer_addr = __heap_alloc(output_buffer_size)
 
-    # allocate memory regions for IRP and IO_STACK_LOCATION
-    irp = make_irp(ql.arch.bits)
-    irpstack_class = irp.irpstack._type_
-
-    irp_addr = __heap_alloc(ctypes.sizeof(irp))
-    irpstack_addr = __heap_alloc(ctypes.sizeof(irpstack_class))
-
-    # setup irp stack parameters
-    irpstack = irpstack_class()
-    # setup IRP structure
-    irp.irpstack = ctypes.cast(irpstack_addr, ctypes.POINTER(irpstack_class))
-
-    ql.log.info("IRP is at 0x%x, IO_STACK_LOCATION is at 0x%x" %(irp_addr, irpstack_addr))
-
-    irpstack.Parameters.DeviceIoControl.IoControlCode = ioctl_code(devicetype, function, ctl_method, access)
-    irpstack.Parameters.DeviceIoControl.OutputBufferLength = output_buffer_size
-    irpstack.Parameters.DeviceIoControl.InputBufferLength = input_buffer_size
-    irpstack.Parameters.DeviceIoControl.Type3InputBuffer.value = input_buffer_addr # used by IOCTL_METHOD_NEITHER
-    ql.mem.write(irpstack_addr, bytes(irpstack))
-
-    if ctl_method == METHOD_NEITHER:
-        irp.UserBuffer.value = output_buffer_addr  # used by IOCTL_METHOD_NEITHER
-
     # allocate memory for AssociatedIrp.SystemBuffer
     # used by IOCTL_METHOD_IN_DIRECT, IOCTL_METHOD_OUT_DIRECT and IOCTL_METHOD_BUFFERED
     system_buffer_size = max(input_buffer_size, output_buffer_size)
     system_buffer_addr = __heap_alloc(system_buffer_size)
-
-    # init data from input buffer
     ql.mem.write(system_buffer_addr, bytes(in_buffer))
-    irp.AssociatedIrp.SystemBuffer.value = system_buffer_addr
 
-    if ctl_method in (METHOD_IN_DIRECT, METHOD_OUT_DIRECT):
-        # Create MDL structure for output data
-        # used by both IOCTL_METHOD_IN_DIRECT and IOCTL_METHOD_OUT_DIRECT
-        mdl = build_mdl(output_buffer_size)
-        mdl_addr = __heap_alloc(ctypes.sizeof(mdl))
+    # allocate memory for IRP
+    irp_struct = make_irp(ql.arch.bits)
+    irp_addr = __heap_alloc(irp_struct.sizeof())
+    ql.log.info(f'IRP is at {irp_addr:#x}')
 
-        ql.mem.write(mdl_addr, bytes(mdl))
-        irp.MdlAddress.value = mdl_addr
+    # populate the structure
+    with irp_struct.ref(ql.mem, irp_addr) as irp_obj:
 
-    # everything is done! Write IRP to memory
-    ql.mem.write(irp_addr, bytes(irp))
+        # allocate memory for IO_STACK_LOCATION
+        irpstack_struct = make_io_stack_location(ql.arch.bits)
+        irpstack_addr = __heap_alloc(irpstack_struct.sizeof())
+        ql.log.info(f'IO_STACK_LOCATION is at {irpstack_addr:#x}')
+
+        # populate the structure
+        with irpstack_struct.ref(ql.mem, irpstack_addr) as irpstack_obj:
+            irpstack_obj.Parameters.DeviceIoControl.IoControlCode = ioctl_code(devicetype, function, ctl_method, access)
+            irpstack_obj.Parameters.DeviceIoControl.OutputBufferLength = output_buffer_size
+            irpstack_obj.Parameters.DeviceIoControl.InputBufferLength = input_buffer_size
+            irpstack_obj.Parameters.DeviceIoControl.Type3InputBuffer = input_buffer_addr # used by IOCTL_METHOD_NEITHER
+
+        irp_obj.irpstack = irpstack_addr
+
+        if ctl_method in (METHOD_IN_DIRECT, METHOD_OUT_DIRECT):
+            mdl_struct = make_mdl(ql.arch.bits)
+            mdl_addr = __heap_alloc(mdl_struct.sizeof())
+
+            # Create MDL structure for output data
+            with mdl_struct.ref(ql.mem, mdl_addr) as mdl_obj:
+                mapped_address = __heap_alloc(output_buffer_size)
+
+                mdl_obj.MappedSystemVa = mapped_address
+                mdl_obj.StartVa = mapped_address
+                mdl_obj.ByteOffset = 0
+                mdl_obj.ByteCount = output_buffer_size
+
+            # used by both IOCTL_METHOD_IN_DIRECT and IOCTL_METHOD_OUT_DIRECT
+            irp_obj.MdlAddress = mdl_addr
+
+        elif ctl_method == METHOD_NEITHER:
+            # used by IOCTL_METHOD_NEITHER
+            irp_obj.UserBuffer = output_buffer_addr
+
+        irp_obj.AssociatedIrp.SystemBuffer = system_buffer_addr
 
     # set function args
-    ql.log.info("Executing IOCTL with DeviceObject = 0x%x, IRP = 0x%x" %(ql.loader.driver_object.DeviceObject, irp_addr))
     # TODO: make sure this is indeed STDCALL
     ql.os.fcall = ql.os.fcall_select(STDCALL)
     ql.os.fcall.writeParams((
@@ -237,30 +228,39 @@ def ioctl(ql: Qiling, params: Tuple[Tuple, int, bytes]) -> Tuple:
         (POINTER, irp_addr)
     ))
 
+    ql.log.info(f'Executing from {major_func:#x}')
+
     try:
-        ql.log.info(f"Executing from: {ql.loader.driver_object.MajorFunction[IRP_MJ_DEVICE_CONTROL]:#x}")
         # now emulate IOCTL's DeviceControl
-        ql.run(ql.loader.driver_object.MajorFunction[IRP_MJ_DEVICE_CONTROL])
+        ql.run(major_func)
     except UcError as err:
         verify_ret(ql, err)
 
-    # read current IRP state
-    irp_buffer = ql.mem.read(irp_addr, ctypes.sizeof(irp))
-    irp = irp.__class__.from_buffer(irp_buffer)
+    # read updated IRP state before releasing resources
+    with irp_struct.ref(ql.mem, irp_addr) as irp_obj:
+        io_status = irp_obj.IoStatus
+        mdl_addr = irp_obj.MdlAddress
 
-    io_status = irp.IoStatus
+        info = io_status.Information
+        status = io_status.Status.Status
 
     # read output data
     output_data = b''
-    if io_status.Status.Status >= 0:
+
+    if status >= 0:
         if ctl_method == METHOD_BUFFERED:
-            output_data = ql.mem.read(system_buffer_addr, io_status.Information.value)
-        if ctl_method in (METHOD_IN_DIRECT, METHOD_OUT_DIRECT):
-            output_data = ql.mem.read(mdl.MappedSystemVa.value, io_status.Information.value)
-        if ctl_method == METHOD_NEITHER:
-            output_data = ql.mem.read(output_buffer_addr, io_status.Information.value)
+            output_data = ql.mem.read(system_buffer_addr, info)
+
+        elif ctl_method in (METHOD_IN_DIRECT, METHOD_OUT_DIRECT):
+            with mdl_struct.ref(ql.mem, mdl_addr) as mdl_obj:
+                mapped_va = mdl_obj.MappedSystemVa
+
+            output_data = ql.mem.read(mapped_va, info)
+
+        elif ctl_method == METHOD_NEITHER:
+            output_data = ql.mem.read(output_buffer_addr, info)
 
     # now free all alloc memory
     __free_all(allocations)
 
-    return io_status.Status.Status, io_status.Information.value, output_data
+    return status, info, output_data

qiling/os/windows/windows.py

@@ -65,9 +65,9 @@ class QlOsWindows(QlOs):
 
         self.stats = QlWinStats()
 
-        ossection = f'OS{self.ql.arch.bits}'
-        heap_base = self.profile.getint(ossection, 'heap_address')
-        heap_size = self.profile.getint(ossection, 'heap_size')
+        ossection = self.profile[f'OS{self.ql.arch.bits}']
+        heap_base = ossection.getint('heap_address')
+        heap_size = ossection.getint('heap_size')
 
         self.heap = QlMemoryHeap(self.ql, heap_base, heap_base + heap_size)
 
@@ -82,9 +82,7 @@ class QlOsWindows(QlOs):
 
         self.PE_RUN = False
         self.last_error = 0
-        # variables used inside hooks
-        self.hooks_variables = {}
-        self.syscall_count = {}
+
         self.argv = self.ql.argv
         self.env = self.ql.env
         self.pid = self.profile.getint('KERNEL', 'pid')
@@ -191,9 +189,6 @@ class QlOsWindows(QlOs):
                 api_func = getattr(api, f'hook_{api_name}', None)
 
             if api_func:
-                self.syscall_count.setdefault(api_name, 0)
-                self.syscall_count[api_name] += 1
-
                 try:
                     api_func(ql, address, api_name)
                 except Exception as ex:

qiling/profiles/dos.ql

@@ -1,10 +1,3 @@
-[LOG]
-# log directory output
-# usage: dir = qlog
-dir =
-# split log file, use with multithread
-split = False
-
 [KERNEL]
 version = 7
 ticks_per_second = 18.206
@@ -17,9 +10,5 @@ stack_size = 0x4000
 base_address = 0x7000 
 
 [MISC]
-# append string into different logs
-# maily for multiple times Ql run with one file
-# usage: append = test1
-append =
 automatize_input = False
 current_path = A:\

qiling/profiles/freebsd.ql

@@ -17,19 +17,7 @@ gid = 1000
 pid = 1996
 
 
-[LOG]
-# log directory output
-# usage: dir = qlog
-dir =
-# split log file, use with multithread
-split = False
-
-
 [MISC]
-# append string into different logs
-# maily for multiple times Ql run with one file
-# usage: append = test1
-append =
 current_path = /
 
 

qiling/profiles/linux.ql

@@ -11,7 +11,6 @@ load_address = 0x555555554000
 interp_address = 0x7ffff7dd5000
 mmap_address = 0x7fffb7dd6000
 vsyscall_address = 0xffffffffff600000
-vsyscall_size = 0x1000
 
 
 [OS32]
@@ -28,19 +27,7 @@ gid = 1000
 pid = 1996
 
 
-[LOG]
-# log directory output
-# usage: dir = qlog
-dir =
-# split log file, use with multithread
-split = False
-
-
 [MISC]
-# append string into different logs
-# maily for multiple times Ql run with one file
-# usage: append = test1
-append =
 current_path = /
 
 

qiling/profiles/macos.ql

@@ -24,19 +24,7 @@ gid = 1000
 pid = 1996
 
 
-[LOG]
-# log directory output
-# usage: dir = qlog
-dir =
-# split log file, use with multithread
-split = False
-
-
 [MISC]
-# append string into different logs
-# maily for multiple times Ql run with one file
-# usage: append = test1
-append =
 current_path = /
 
 

qiling/profiles/qnx.ql

@@ -15,27 +15,17 @@ cpupage_address = 0xfc4048d8
 cpupage_tls_address = 0xfc405000
 tls_data_address = 0xfc406000
 
+
 [KERNEL]
 uid = 1000
 gid = 1000
 pid = 1996
 
 
-[LOG]
-# log directory output
-# usage: dir = qlog
-dir =
-# split log file, use with multithread
-split = False
-
-
 [MISC]
-# append string into different logs
-# maily for multiple times Ql run with one file
-# usage: append = test1
-append =
 current_path = /
 
+
 [NETWORK]
 # To use IPv6 or not, to avoid binary double bind. ipv6 and ipv4 bind the same port at the same time
 bindtolocalhost = True

qiling/profiles/windows.ql

@@ -6,7 +6,8 @@ stack_size = 0x40000
 image_address = 0x400000
 dll_address = 0x7ffff0000000
 entry_point = 0x140000000
-KI_USER_SHARED_DATA = 0xfffff78000000000
+# KI_USER_SHARED_DATA = 0xfffff78000000000
+KI_USER_SHARED_DATA = 0x7ffe0000
 
 [OS32]
 heap_address = 0x5000000
@@ -16,7 +17,8 @@ stack_size = 0x21000
 image_address = 0x400000
 dll_address = 0x10000000
 entry_point = 0x40000
-KI_USER_SHARED_DATA = 0xffdf0000
+# KI_USER_SHARED_DATA = 0xffdf0000
+KI_USER_SHARED_DATA = 0x7ffe0000
 
 [CODE]
 # ram_size 0xa00000 is 10MB
@@ -28,18 +30,7 @@ pid = 1996
 parent_pid = 0
 shell_pid = 10
 
-[LOG]
-# log directory output
-# usage: dir = qlog
-dir =
-# split log file, use with multithread
-split = False
-
 [MISC]
-# append string into different logs
-# maily for multiple times Ql run with one file
-# usage: append = test1
-append =
 current_path = C:\
 
 [SYSTEM]

qiling/utils.py

@@ -433,10 +433,15 @@ def profile_setup(ostype: QL_OS, filename: Optional[str]):
     return config
 
 # verify if emulator returns properly
-def verify_ret(ql, err):
+def verify_ret(ql: 'Qiling', err):
+    # init_sp location is not consistent; this is here to work around that
+    if not hasattr(ql.os, 'init_sp'):
+        ql.os.init_sp = ql.loader.init_sp
+
     ql.log.debug("Got exception %u: init SP = %x, current SP = %x, PC = %x" %(err.errno, ql.os.init_sp, ql.arch.regs.arch_sp, ql.arch.regs.arch_pc))
 
-    ql.os.RUN = False
+    if hasattr(ql.os, 'RUN'):
+        ql.os.RUN = False
 
     # timeout is acceptable in this case
     if err.errno in (UC_ERR_READ_UNMAPPED, UC_ERR_FETCH_UNMAPPED):

qltool

@@ -90,6 +90,7 @@ def handle_code(options: argparse.Namespace):
         profile=options.profile,
         filter=options.filter,
         endian=archendian,
+        thumb=options.thumb,
     )
 
     return ql
@@ -184,7 +185,7 @@ if __name__ == '__main__':
     code_parser.add_argument('-f', '--filename', metavar="FILE", help="filename")
     code_parser.add_argument('-i', '--input', metavar="INPUT", dest="input", help='input hex value')
     code_parser.add_argument('--arch', required=True, choices=arch_map)
-    code_parser.add_argument('--thumb', action='store_true', help='specify thumb mode for ARM')
+    code_parser.add_argument('--thumb', action='store_true', default=False, help='specify thumb mode for ARM')
     code_parser.add_argument('--endian', choices=('little', 'big'), default='little', help='specify endianess for bi-endian archs')
     code_parser.add_argument('--os', required=True, choices=os_map)
     code_parser.add_argument('--rootfs', default='.', help='emulated root filesystem, that is where all libraries reside')

setup.py

@@ -10,7 +10,7 @@ VERSION = "1.4.4" + "-dev"
 
 requirements = [
     "capstone>=4.0.1",
-    "unicorn>=2.0.0-rc7",
+    "unicorn>=2.0.0",
     "pefile>=2022.5.30",
     "python-registry>=1.3.1",
     "keystone-engine>=0.9.2",
@@ -18,7 +18,7 @@ requirements = [
     "gevent>=20.9.0",
     "multiprocess>=0.70.12.2",
     "windows-curses>=2.1.0;platform_system=='Windows'",
-    "pyyaml>=6.0"
+    "pyyaml>=6.0",
 ]
 
 extras = {
@@ -40,8 +40,11 @@ extras = {
         "cmd2"
     ],
     "fuzz" : [
-        "unicornafl>=2.0.0;platform_system=='Windows'",
+        "unicornafl>=2.0.0;platform_system!='Windows'",
         "fuzzercorn>=0.0.1;platform_system=='Linux'"
+    ],
+    "RE": [
+       "r2libr>=5.7.4",
     ]
 }
 

tests/profiles/append_test.ql

@@ -1,5 +0,0 @@
-[LOG]
-dir = log_test
-
-[MISC]
-append = append_test

tests/profiles/uboot_bin.ql

@@ -4,17 +4,5 @@ entry_point = 0x80800000
 heap_size = 0x300000
 
 
-[LOG]
-# log directory output
-# usage: dir = qlog
-dir =
-# split log file, use with multithread
-split = False
-
-
 [MISC]
-# append string into different logs
-# maily for multiple times Ql run with one file
-# usage: append = test1
-append =
 current_path = /

tests/test_android.py

@@ -44,17 +44,17 @@ class TestAndroid(unittest.TestCase):
         del ql
 
 
-    #@unittest.skipUnless(platform.system() == 'Linux', 'run only on Linux')
-    #def test_android_arm(self):
-    #    test_binary = "../examples/rootfs/arm64_android6.0/bin/arm_android_jniart"
-    #    rootfs = "../examples/rootfs/arm64_android6.0"
-    #    env = {"ANDROID_DATA":"/data", "ANDROID_ROOT":"/system"}
+    @unittest.skipUnless(platform.system() == 'Linux', 'run only on Linux')
+    def test_android_arm(self):
+        test_binary = "../examples/rootfs/arm64_android6.0/bin/arm_android_jniart"
+        rootfs = "../examples/rootfs/arm64_android6.0"
+        env = {"ANDROID_DATA":"/data", "ANDROID_ROOT":"/system"}
 
-    #    ql = Qiling([test_binary], rootfs, env, multithread=True)
-    #    ql.os.set_syscall("close", my_syscall_close)
-    #    ql.add_fs_mapper("/proc/self/task/2000/maps", Fake_maps(ql))
-    #    ql.run()
-    #    del ql
+        ql = Qiling([test_binary], rootfs, env, multithread=True)
+        ql.os.set_syscall("close", my_syscall_close)
+        ql.add_fs_mapper("/proc/self/task/2000/maps", Fake_maps(ql))
+        ql.run()
+        del ql
 
 
 if __name__ == "__main__":

tests/test_elf.py

@@ -209,8 +209,12 @@ class ELFTest(unittest.TestCase):
 
 
     def test_elf_linux_x86(self):
-        ql = Qiling(["../examples/rootfs/x86_linux/bin/x86_hello"], "../examples/rootfs/x86_linux", verbose=QL_VERBOSE.DEBUG, log_file="test.qlog")     
+        filename = 'test.qlog'
+
+        ql = Qiling(["../examples/rootfs/x86_linux/bin/x86_hello"], "../examples/rootfs/x86_linux", verbose=QL_VERBOSE.DEBUG, log_file=filename)
         ql.run()
+
+        os.remove(filename)
         del ql
 
 
@@ -352,7 +356,7 @@ class ELFTest(unittest.TestCase):
             all_mem = ql.mem.save()
             ql.mem.restore(all_mem)
 
-        ql = Qiling(["../examples/rootfs/arm_linux/bin/arm_hello"], "../examples/rootfs/arm_linux", verbose=QL_VERBOSE.DEBUG, profile='profiles/append_test.ql')
+        ql = Qiling(["../examples/rootfs/arm_linux/bin/arm_hello"], "../examples/rootfs/arm_linux", verbose=QL_VERBOSE.DEBUG)
         ql.os.set_api('puts', my_puts)
         ql.run()
         del ql
@@ -1073,7 +1077,7 @@ class ELFTest(unittest.TestCase):
         del ql
 
     def test_elf_linux_armeb(self):     
-        ql = Qiling(["../examples/rootfs/armeb_linux/bin/armeb_hello"], "../examples/rootfs/armeb_linux", verbose=QL_VERBOSE.DEBUG, profile='profiles/append_test.ql')
+        ql = Qiling(["../examples/rootfs/armeb_linux/bin/armeb_hello"], "../examples/rootfs/armeb_linux", verbose=QL_VERBOSE.DEBUG)
         ql.run()
         del ql
 

tests/test_elf_ko.py

@@ -5,8 +5,6 @@
 
 import os, sys, unittest
 
-from unicorn import UcError
-
 sys.path.append("..")
 from qiling import Qiling
 from qiling.const import QL_INTERCEPT, QL_VERBOSE
@@ -19,61 +17,64 @@ class ELF_KO_Test(unittest.TestCase):
 
     @unittest.skipIf(IS_FAST_TEST, 'fast test')
     def test_demigod_m0hamed_x86(self):
-        checklist = {}
+        checklist = []
 
         @linux_kernel_api(params={
             "format": STRING
         })
-        def my_printk(ql: Qiling, address: int, params):
-            ql.log.info(f'oncall printk: params = {params}')
+        def __my_printk(ql: Qiling, address: int, params):
+            ql.log.info(f'my printk: {params=}')
 
-            checklist['oncall'] = params['format']
+            checklist.append(params['format'])
 
             return 0
 
         ql = Qiling(["../examples/rootfs/x86_linux/kernel/m0hamed_rootkit.ko"],  "../examples/rootfs/x86_linux", verbose=QL_VERBOSE.DEBUG)
-        ql.os.set_api("printk", my_printk)
+        ql.os.set_api("printk", __my_printk)
 
         ba = ql.loader.load_address
+        ql.run(ba + 0x01e0, ba + 0x01fa)
 
-        try:
-            ql.run(ba + 0x11e0, ba + 0x11fa)
-        except UcError as e:
-            self.fail(e)
-        else:
-            self.assertEqual("DONT YOU EVER TRY TO READ THIS FILE OR I AM GOING TO DESTROY YOUR MOST SECRET DREAMS", checklist['oncall'])
+        self.assertEqual("DONT YOU EVER TRY TO READ THIS FILE OR I AM GOING TO DESTROY YOUR MOST SECRET DREAMS", checklist.pop(0))
+        self.assertEqual(len(checklist), 0)
 
     def test_demigod_hello_x8664(self):
-        checklist = {}
+        checklist = []
 
-        def my_onenter(ql: Qiling, address: int, params):
-            ql.log.info(f'onenter printk: params = {params}')
+        def __onenter_printk(ql: Qiling, address: int, params):
+            ql.log.info(f'about to enter printk: {params=}')
 
-            checklist['onenter'] = params['format']
+            checklist.append(params['format'])
 
         ql = Qiling(["../examples/rootfs/x8664_linux/kernel/hello.ko"],  "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG)
-        ql.os.set_api("printk", my_onenter, QL_INTERCEPT.ENTER)
+        ql.os.set_api("printk", __onenter_printk, QL_INTERCEPT.ENTER)
 
         ba = ql.loader.load_address
-        ql.run(ba + 0x1064, ba + 0x107e)
+        ql.run(ba + 0x64, ba + 0x7e)    # run lkm_example_init
+        ql.run(ba + 0x7f, ba + 0x90)    # run lkm_example_exit
 
-        self.assertEqual("\x016Hello, World: %p!\n", checklist['onenter'])
+        self.assertIn('Hello', checklist.pop(0))
+        self.assertIn('Goodbye', checklist.pop(0))
+        self.assertEqual(len(checklist), 0)
 
     def test_demigod_hello_mips32(self):
-        checklist = {}
+        checklist = []
 
-        def my_onexit(ql: Qiling, address: int, params, retval: int):
-            ql.log.info(f'onexit printk: params = {params}')
+        def __onexit_printk(ql: Qiling, address: int, params, retval: int):
+            ql.log.info(f'done with printk: {params=}')
 
-            checklist['onexit'] = params['format']
+            checklist.append(params['format'])
 
         ql = Qiling(["../examples/rootfs/mips32_linux/kernel/hello.ko"],  "../examples/rootfs/mips32_linux", verbose=QL_VERBOSE.DEBUG)
-        ql.os.set_api("printk", my_onexit, QL_INTERCEPT.EXIT)
+        ql.os.set_api("printk", __onexit_printk, QL_INTERCEPT.EXIT)
 
         ba = ql.loader.load_address
-        ql.run(ba + 0x1060, ba + 0x1084)
+        ql.run(ba + 0x60, ba + 0x84)    # run hello
+        ql.run(ba + 0x88, ba + 0x98)    # run goodbye
+
+        self.assertIn('Hello', checklist.pop(0))
+        self.assertEqual(len(checklist), 0)
 
-        self.assertEqual("\x016Hello, World!\n", checklist['onexit'])
 
 if __name__ == "__main__":
     unittest.main()

tests/test_elf_multithread.py

@@ -34,12 +34,20 @@ class ELFTest(unittest.TestCase):
                     verbose=QL_VERBOSE.DEBUG,
                     multithread=True)
 
-        err = ql_file.open('output.txt', os.O_RDWR | os.O_CREAT, 0o777)
+        filename = 'output.txt'
+        err = ql_file.open(filename, os.O_RDWR | os.O_CREAT, 0o777)
+
         ql.os.stderr = err
         ql.run()
-        os.close(err.fileno())
-        with open('output.txt', 'rb') as f:
-            self.assertTrue(b'fail' in f.read())            
+        err.close()
+
+        with open(filename, 'rb') as f:
+            content = f.read()
+
+        # cleanup
+        os.remove(filename)
+
+        self.assertIn(b'fail', content)
 
         del ql
 
@@ -92,7 +100,7 @@ class ELFTest(unittest.TestCase):
             except:
                 pass
         buf_out = None
-        ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_multithreading"], "../examples/rootfs/x8664_linux", multithread=True, profile= "profiles/append_test.ql")
+        ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_multithreading"], "../examples/rootfs/x8664_linux", multithread=True)
         ql.os.set_syscall("write", check_write, QL_INTERCEPT.ENTER)
         ql.run()
 

tests/test_pe.py

@@ -68,8 +68,7 @@ class PETest(unittest.TestCase):
 
     def test_pe_win_x8664_hello(self):
         def _t():
-            ql = Qiling(["../examples/rootfs/x8664_windows/bin/x8664_hello.exe"], "../examples/rootfs/x8664_windows",
-                        verbose=QL_VERBOSE.DEFAULT)
+            ql = Qiling(["../examples/rootfs/x8664_windows/bin/x8664_hello.exe"], "../examples/rootfs/x8664_windows")
             ql.run()
             del ql
             return True
@@ -79,8 +78,7 @@ class PETest(unittest.TestCase):
 
     def test_pe_win_x86_hello(self):
         def _t():
-            ql = Qiling(["../examples/rootfs/x86_windows/bin/x86_hello.exe"], "../examples/rootfs/x86_windows",
-                        verbose=QL_VERBOSE.DEFAULT, profile="profiles/append_test.ql")
+            ql = Qiling(["../examples/rootfs/x86_windows/bin/x86_hello.exe"], "../examples/rootfs/x86_windows")
             ql.run()
             del ql
             return True
@@ -90,8 +88,7 @@ class PETest(unittest.TestCase):
 
     def test_pe_win_x8664_file_upx(self):
         def _t():
-            ql = Qiling(["../examples/rootfs/x8664_windows/bin/x8664_file_upx.exe"], "../examples/rootfs/x8664_windows",
-                        verbose=QL_VERBOSE.DEFAULT)
+            ql = Qiling(["../examples/rootfs/x8664_windows/bin/x8664_file_upx.exe"], "../examples/rootfs/x8664_windows")
             ql.run()
             del ql
             return True
@@ -101,8 +98,7 @@ class PETest(unittest.TestCase):
 
     def test_pe_win_x86_file_upx(self):
         def _t():
-            ql = Qiling(["../examples/rootfs/x86_windows/bin/x86_file_upx.exe"], "../examples/rootfs/x86_windows",
-                        verbose=QL_VERBOSE.DEFAULT)
+            ql = Qiling(["../examples/rootfs/x86_windows/bin/x86_file_upx.exe"], "../examples/rootfs/x86_windows")
             ql.run()
             del ql
             return True
@@ -218,22 +214,21 @@ class PETest(unittest.TestCase):
 
     def test_pe_win_x86_multithread(self):
         def _t():
-            thread_id = None
-            def ThreadId_onEnter(ql, address, params):
+            thread_id = -1
+
+            def ThreadId_onEnter(ql: Qiling, address: int, params):
                 nonlocal thread_id
+
                 thread_id = ql.os.thread_manager.cur_thread.id
-                return address, params
 
             ql = Qiling(["../examples/rootfs/x86_windows/bin/MultiThread.exe"], "../examples/rootfs/x86_windows")
             ql.os.set_api("GetCurrentThreadId", ThreadId_onEnter, QL_INTERCEPT.ENTER)
             ql.run()
-            
-            if not ( 1<= thread_id < 255):
-                return False
-            
+
             del ql
-            return True
-        
+
+            return (1 <= thread_id < 255)
+
         self.assertTrue(QLWinSingleTest(_t).run())
 
 
@@ -360,30 +355,12 @@ class PETest(unittest.TestCase):
     @unittest.skipIf(IS_FAST_TEST, 'fast test')
     def test_pe_win_al_khaser(self):
         def _t():
-            ql = Qiling(["../examples/rootfs/x86_windows/bin/al-khaser.bin"], "../examples/rootfs/x86_windows")
+            ql = Qiling(["../examples/rootfs/x86_windows/bin/al-khaser.bin"], "../examples/rootfs/x86_windows", verbose=QL_VERBOSE.OFF)
 
-            # The hooks are to remove the prints to file. It crashes. will debug why in the future
-            # def results(ql):
-            #
-            #     if ql.arch.regs.ebx == 1:
-            #         print("BAD")
-            #     else:
-            #         print("GOOD ")
-            #     ql.arch.regs.eip = 0x402ee4
-            #
-            #ql.hook_address(results, 0x00402e66)
+            # ole32 functions are not implemented yet; stop before the binary
+            # starts using them
+            ql.run(end=0x004016ae)
 
-            # the program alloc 4 bytes and then tries to write 0x2cc bytes.
-            # I have no idea of why this code should work without this patch
-            ql.patch(0x00401984, b'\xb8\x04\x00\x00\x00')
-
-            def end(ql):
-                print("We are finally done")
-                ql.emu_stop()
-
-            ql.hook_address(end, 0x004016ae)
-
-            ql.run()
             del ql
             return True
 

tests/test_pe_sys.py

@@ -11,7 +11,8 @@ from unicorn import UcError
 sys.path.append("..")
 from qiling import Qiling
 from qiling.const import QL_STOP, QL_VERBOSE
-from qiling.os.const import POINTER, DWORD, STRING, HANDLE
+from qiling.os.const import POINTER, DWORD, HANDLE
+from qiling.exception import QlErrorSyscallError
 from qiling.os.windows import utils
 from qiling.os.windows.wdk_const import *
 from qiling.os.windows.api import *
@@ -98,8 +99,7 @@ class PETest(unittest.TestCase):
             buffer = ql.mem.read(lpBuffer, nNumberOfBytesToWrite)
 
             if hFile == 0x13371337:
-                success, nNumberOfBytesToWrite = utils.io_Write(ql.amsint32_driver, buffer)
-                r = int(success)
+                nNumberOfBytesToWrite = utils.io_Write(ql.amsint32_driver, buffer)
 
             elif hFile == 0xfffffff5:
                 s = buffer.decode()
@@ -203,7 +203,12 @@ class PETest(unittest.TestCase):
         # asmint32 driver should have been initialized by now. otherwise we get an exception
         amsint32: Qiling = getattr(ql, 'amsint32_driver')
 
-        utils.io_Write(amsint32, ql.pack32(0xdeadbeef))
+        # asmint32 driver init doesn't get to run far enough to initialize necessary data
+        # structures. it is expected to fail.
+        try:
+            utils.io_Write(amsint32, ql.pack32(0xdeadbeef))
+        except QlErrorSyscallError:
+            pass
 
         # TODO: not sure whether this one is really STDCALL
         fcall = amsint32.os.fcall_select(STDCALL)
@@ -264,6 +269,6 @@ class PETest(unittest.TestCase):
         # - Call DriverUnload
 
         del ql
-        
+
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()

tests/test_r2.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+import sys, unittest
+
+sys.path.append("..")
+from qiling import Qiling
+from qiling.const import QL_VERBOSE
+from qiling.extensions.r2.r2 import R2
+
+
+EVM_CODE = bytes.fromhex("6060604052341561000f57600080fd5b60405160208061031c833981016040528080519060200190919050508060018190556000803373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000208190555050610299806100836000396000f300606060405260043610610057576000357c0100000000000000000000000000000000000000000000000000000000900463ffffffff16806318160ddd1461005c57806370a0823114610085578063a9059cbb146100d2575b600080fd5b341561006757600080fd5b61006f61012c565b6040518082815260200191505060405180910390f35b341561009057600080fd5b6100bc600480803573ffffffffffffffffffffffffffffffffffffffff16906020019091905050610132565b6040518082815260200191505060405180910390f35b34156100dd57600080fd5b610112600480803573ffffffffffffffffffffffffffffffffffffffff1690602001909190803590602001909190505061017a565b604051808215151515815260200191505060405180910390f35b60015481565b60008060008373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001908152602001600020549050919050565b600080826000803373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000205403101515156101cb57600080fd5b816000803373ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200190815260200160002060008282540392505081905550816000808573ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff1681526020019081526020016000206000828254019250508190555060019050929150505600a165627a7a7230582098f1551a391a3e65b3ce45cfa2b3fa5f91eea9a3e7181a81454e025ea0d7151c0029")
+
+
+class R2Test(unittest.TestCase):
+    def test_shellcode_disasm(self):
+        ql = Qiling(code=EVM_CODE, archtype="evm", verbose=QL_VERBOSE.DISABLED)
+        r2 = R2(ql)
+        pd = r2._cmd("pd 32")
+        self.assertTrue('callvalue' in pd)
+
+    def test_addr_flag(self):
+        ql = Qiling(["../examples/rootfs/x86_windows/bin/x86_hello.exe"], "../examples/rootfs/x86_windows",
+                    verbose=QL_VERBOSE.DISABLED)  # x8864_hello does not have 'main'
+        r2 = R2(ql)
+        print(r2.where('main'))
+        self.assertEqual(r2.at(r2.where('main')), 'main')
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_shellcode.py

@@ -18,6 +18,7 @@ MIPS32EL_LIN = unhexlify('ffff0628ffffd004ffff05280110e4270ff08424ab0f02240c0101
 X86_WIN = unhexlify('fce8820000006089e531c0648b50308b520c8b52148b72280fb74a2631ffac3c617c022c20c1cf0d01c7e2f252578b52108b4a3c8b4c1178e34801d1518b592001d38b4918e33a498b348b01d631ffacc1cf0d01c738e075f6037df83b7d2475e4588b582401d3668b0c4b8b581c01d38b048b01d0894424245b5b61595a51ffe05f5f5a8b12eb8d5d6a01eb2668318b6f87ffd5bbf0b5a25668a695bd9dffd53c067c0a80fbe07505bb4713726f6a0053ffd5e8d5ffffff63616c6300')
 X8664_WIN = unhexlify('fc4881e4f0ffffffe8d0000000415141505251564831d265488b52603e488b52183e488b52203e488b72503e480fb74a4a4d31c94831c0ac3c617c022c2041c1c90d4101c1e2ed5241513e488b52203e8b423c4801d03e8b80880000004885c0746f4801d0503e8b48183e448b40204901d0e35c48ffc93e418b34884801d64d31c94831c0ac41c1c90d4101c138e075f13e4c034c24084539d175d6583e448b40244901d0663e418b0c483e448b401c4901d03e418b04884801d0415841585e595a41584159415a4883ec204152ffe05841595a3e488b12e949ffffff5d49c7c1000000003e488d95fe0000003e4c8d850f0100004831c941ba45835607ffd54831c941baf0b5a256ffd548656c6c6f2c2066726f6d204d534621004d657373616765426f7800') 
 ARM_LIN = unhexlify('01308fe213ff2fe178460e300190491a921a0827c251033701df2f62696e2f2f7368')
+ARM_THUMB = unhexlify('401c01464fea011200bf')
 ARM64_LIN = unhexlify('420002ca210080d2400080d2c81880d2010000d4e60300aa01020010020280d2681980d2010000d4410080d2420002cae00306aa080380d2010000d4210400f165ffff54e0000010420002ca210001caa81b80d2010000d4020004d27f0000012f62696e2f736800')
 X8664_FBSD = unhexlify('6a61586a025f6a015e990f054897baff02aaaa80f2ff524889e699046680c2100f05046a0f05041e4831f6990f0548976a035852488d7424f080c2100f0548b8523243427730637257488d3e48af74084831c048ffc00f055f4889d04889fe48ffceb05a0f0575f799043b48bb2f62696e2f2f73685253545f5257545e0f05')
 X8664_macos = unhexlify('4831f65648bf2f2f62696e2f7368574889e74831d24831c0b00248c1c828b03b0f05')
@@ -44,6 +45,12 @@ class TestShellcode(unittest.TestCase):
        ql = Qiling(code = ARM_LIN, archtype = "arm", ostype = "linux", verbose=QL_VERBOSE.OFF)
        ql.run()
 
+
+    def test_linux_arm_thumb(self):
+        print("Linux ARM Thumb Shllcode")
+        ql = Qiling(code = ARM_THUMB, archtype = "arm", ostype = "linux", verbose=QL_VERBOSE.OFF, thumb = True)
+        ql.run()
+
     
     def test_linux_arm64(self):
         print("Linux ARM 64bit Shellcode")

tests/test_struct.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+#
+# Cross Platform and Multi Architecture Advanced Binary Emulation Framework
+#
+
+import sys, unittest
+from typing import Optional
+
+sys.path.append("..")
+
+import ctypes
+
+from qiling import Qiling
+from qiling.const import QL_ARCH, QL_OS
+from qiling.os.struct import BaseStruct
+
+class DummyInternalStruct(BaseStruct):
+    _fields_ = [
+        ('X', ctypes.c_uint32)
+    ]
+
+    # this is defined to let 'assertEqual' work as expected
+    def __eq__(self, other) -> bool:
+        return isinstance(other, DummyInternalStruct) and self.X == other.X
+
+
+class DummyStruct(BaseStruct):
+    _fields_ = [
+        ('A', ctypes.c_uint32),
+        ('B', ctypes.c_uint64),
+        ('C', DummyInternalStruct),
+        ('D', ctypes.c_char * 16)
+    ]
+
+
+# we only need context and not going to run anything anyway, so just use whatever
+NOPSLED = b'\x90' * 8
+ROOTFS = r'../examples/rootfs/x8664_linux'
+
+class StructTest(unittest.TestCase):
+
+    def setUp(self) -> None:
+        ql = Qiling(code=NOPSLED, rootfs=ROOTFS, archtype=QL_ARCH.X8664, ostype=QL_OS.LINUX)
+
+        self.ptr = 0x100000
+        self.mem = ql.mem
+
+        self.expected = {
+            'A' : 0xdeadface,
+            'B' : 0x1020304050607080,
+            'C' : DummyInternalStruct(0x11213141),
+            'D' : b'Hello World!',
+        }
+
+        # create a dummy structure with expected values
+        dummy = DummyStruct(**self.expected)
+
+        # emit dummy structure to memory
+        ql.mem.map(self.ptr, ql.mem.align_up(dummy.sizeof()))
+        ql.mem.write(self.ptr, bytes(dummy))
+
+
+    def __read_data(self, offset: int = 0, size: Optional[int] = None) -> bytearray:
+        return self.mem.read(self.ptr + offset, size or DummyStruct.sizeof())
+
+
+    def __write_data(self, offset: int, data: bytes) -> None:
+        self.mem.write(self.ptr + offset, data)
+
+
+    @staticmethod
+    def __to_uint(data: bytearray) -> int:
+        return int.from_bytes(data, 'little', signed=False)
+
+
+    def test_load_from(self):
+        dummy = DummyStruct.load_from(self.mem, self.ptr)
+
+        self.assertEqual(self.expected['A'], dummy.A)
+        self.assertEqual(self.expected['B'], dummy.B)
+        self.assertEqual(self.expected['C'], dummy.C)
+        self.assertEqual(self.expected['D'], dummy.D)
+
+
+    def test_save_to(self):
+        dummy = DummyStruct(
+            A=0x0c0a0f0e,
+            B=0x1828384858687888,
+            C=DummyInternalStruct(0x19293949),
+            D=b'Goodbye World!'
+        )
+
+        dummy.save_to(self.mem, self.ptr)
+
+        obj_data = bytes(dummy)
+        mem_data = self.__read_data()
+
+        self.assertEqual(obj_data, mem_data)
+
+    def test_ref_discard(self):
+        data_before = self.__read_data()
+
+        unused = []
+        with DummyStruct.ref(self.mem, self.ptr) as dummy:
+            print(f'B = {dummy.B:#x}')
+            print(f'C = {dummy.C}')
+
+            unused.append(dummy.A + 1337)
+
+        data_after = self.__read_data()
+
+        self.assertEqual(data_before, data_after)
+
+    def test_ref_save(self):
+        expected = 0x10303070
+
+        with DummyStruct.ref(self.mem, self.ptr) as dummy:
+            print(f'B = {dummy.B:#x}')
+            print(f'C = {dummy.C}')
+
+            dummy.A = expected
+
+        data = self.__read_data(DummyStruct.offsetof('A'), 4)
+        self.assertEqual(expected, StructTest.__to_uint(data))
+
+    def test_ref_save_internal(self):
+        expected = 0x16363676
+
+        with DummyStruct.ref(self.mem, self.ptr) as dummy:
+            dummy.C.X = expected
+
+        data = self.__read_data(DummyStruct.offsetof('C') + DummyInternalStruct.offsetof('X'), 4)
+        self.assertEqual(expected, StructTest.__to_uint(data))
+
+    def test_volatile_ref(self):
+        dummy = DummyStruct.volatile_ref(self.mem, self.ptr)
+
+        expected = 0x01030307
+        dummy.A = expected
+        data = self.__read_data(DummyStruct.offsetof('A'), 4)
+        self.assertEqual(expected, StructTest.__to_uint(data))
+
+        self.assertEqual(self.expected['B'], dummy.B)
+        self.assertEqual(self.expected['C'], dummy.C)
+
+        expected = b'Volatility Test!'
+        self.__write_data(DummyStruct.offsetof('D'), expected)
+        self.assertEqual(expected, dummy.D)
+
+    def test_volatile_ref_internal(self):
+        dummy = DummyStruct.volatile_ref(self.mem, self.ptr)
+
+        expected = 0x51535357
+        dummy.C.X = expected
+        data = self.__read_data(DummyStruct.offsetof('C') + DummyInternalStruct.offsetof('X'), 4)
+        self.assertEqual(expected, StructTest.__to_uint(data))
+
+
+if __name__ == "__main__":
+    unittest.main()