diff --git a/.github/workflows/giteesync.yml b/.github/workflows/giteesync.yml index b770d37c..dbf79749 100644 --- a/.github/workflows/giteesync.yml +++ b/.github/workflows/giteesync.yml @@ -5,12 +5,14 @@ jobs: deploy: runs-on: ubuntu-latest + if: github.repository_owner == 'qilingframework' steps: - uses: actions/checkout@v2 with: fetch-depth: 0 - uses: xwings/sync-repo-action@master with: + run: git config --global --add safe.directory * ssh_private_key: ${{ secrets.GITEE_KEY }} target_repo: ssh://git@gitee.com/qilingframework/qiling.git diff --git a/CREDITS.md b/CREDITS.md index b1cba944..9ea9f0ed 100644 --- a/CREDITS.md +++ b/CREDITS.md @@ -53,6 +53,8 @@ - madprogrammer - danielmoos - sigeryang +- bet4it +- nullableVoidPtr #### Legacy Core Developers diff --git a/ChangeLog b/ChangeLog index 7ef9840d..b178267b 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,7 +1,14 @@ This file details the changelog of Qiling Framework. ------------------------------------ -[Version 1.4.3]: April 7th, 2022 +[Version 1.4.4]: July XX, 2022 + + +------------------------------------ +[Version 1.4.3]: June 1st, 2022 + +New features: +- Introduce PowerPC architecture support (#1140) Improvements: - Fix fuzzing for tendaac15 (#1096) @@ -10,11 +17,22 @@ Improvements: - Minor PE Loader fix (#1104) - Minor quality changes (#1106) - Fix cacheflush syscall typo (#1115) +- Improvements and fixes for Windows and PE (#1118) - Add vm_context to EVM hooks (#1119) - Load interpreter segments with correct perms and vaddr (#1120) - Fix mistakes in fuzz_x8664_linux binary (#1121) - Add EVM ABI helpers, fix EVM DBG stack view (#1123) - Fix regression caused by missing exception handling when opening socket (#1124) +- CI improvement (#1128 #1134) +- Add macho load command 'LC_LOAD_WEAK_DYLIB' support (#1133) +- Fix breakage of non-Windows binary emulation on Windows host (#1143) +- Remove misused region bound check of unmap_all (#1144) +- Change deprecated interfaces of IDA (#1145) +- Use importlib to retrieve package version (#1146) +- New and improved gdbserver (#1148) +- Rewrite package data reading (#1150) +- Misc improvements (#1154) +- Fix memory exhaustion problem caused by the logger (#1161) Contributors: - wtdcode @@ -22,9 +40,18 @@ Contributors: - elicn - xwings - cq674350529 +- elicn - TheZ3ro - bet4it - chinggg +- kabeor +- chfl4gs +- profiles +- OlfillasOdikno +- nmantan +- machinewu +- nullableVoidPtr +- Phat3 ------------------------------------ diff --git a/Dockerfile b/Dockerfile index 2a1360b4..99a0a9bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ WORKDIR /qiling RUN apt-get update \ && apt-get install -y --no-install-recommends unzip apt-utils \ && rm -rf /var/lib/apt/lists/* \ - && pip3 install wheels/*.whl \ + && pip3 install --no-deps wheels/*.whl \ && rm -rf wheels ENV HOME /qiling diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index ead0863c..00000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -recursive-include qiling/debugger/gdb/xml * -recursive-include qiling/extensions/windows_sdk/defs * -recursive-include qiling/profiles * -include qiling/os/uefi/guids.csv -include qiling/os/qnx/syspage.bin diff --git a/README.md b/README.md index 48af44c7..8abbe0ef 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ Qiling is an advanced binary emulation framework, with the following features: -- Emulate multi-platforms: Windows, MacOS, Linux, BSD, UEFI, DOS, MBR, Ethereum Virtual Machine -- Emulate multi-architectures: X86, X86_64, Arm, Arm64, MIPS, 8086 +- Emulate multi-platforms: Windows, MacOS, Linux, Android, BSD, UEFI, DOS, MBR, Ethereum Virtual Machine +- Emulate multi-architectures: 8086, X86, X86_64, ARM, ARM64, MIPS, RISCV, PowerPC - Support multiple file formats: PE, MachO, ELF, COM, MBR - Support Windows Driver (.sys), Linux Kernel Module (.ko) & MacOS Kernel (.kext) via [Demigod](https://groundx.io/demigod/) - Emulates & sandbox code in an isolated environment diff --git a/examples/crackme_x86_linux.py b/examples/crackme_x86_linux.py index 7c1d1647..9373d133 100644 --- a/examples/crackme_x86_linux.py +++ b/examples/crackme_x86_linux.py @@ -89,21 +89,26 @@ class Solver: return False +def progress(msg: str) -> None: + print(msg, end='\r', file=sys.stderr, flush=True) + def main(): - idx_list = (1, 4, 2, 0, 3) - flag = [0] * len(idx_list) + flag = bytearray(b'*****') + indices = (1, 4, 2, 0, 3) - solver = Solver(bytes(flag)) + # all possible flag characters (may be reduced to uppercase and digits to save time) + charset = string.printable - for idx in idx_list: + progress('Initializing...') + solver = Solver(flag) - # bruteforce all possible flag characters - for ch in string.printable: - flag[idx] = ord(ch) + for i in indices: + for ch in charset: + flag[i] = ord(ch) - print(f'Guessing... [{"".join(chr(ch) if ch else "_" for ch in flag)}]', end='\r', file=sys.stderr, flush=True) + progress(f'Guessing... {flag.decode()}') - if solver.replay(bytes(flag)): + if solver.replay(flag): break else: @@ -113,3 +118,5 @@ def main(): if __name__ == "__main__": main() + +# expected flag: L1NUX diff --git a/examples/crackme_x86_windows.py b/examples/crackme_x86_windows.py index d60f40c8..c47da07e 100644 --- a/examples/crackme_x86_windows.py +++ b/examples/crackme_x86_windows.py @@ -10,47 +10,111 @@ from qiling import Qiling from qiling.const import QL_VERBOSE from qiling.extensions import pipe -def instruction_count(ql: Qiling, address: int, size: int, user_data): - user_data[0] += 1 +ROOTFS = r"rootfs/x86_windows" -def get_count(flag): - ql = Qiling(["rootfs/x86_windows/bin/crackme.exe"], "rootfs/x86_windows", verbose=QL_VERBOSE.OFF) +class Solver: + def __init__(self, invalid: bytes): + # create a silent qiling instance + self.ql = Qiling([rf"{ROOTFS}/bin/crackme.exe"], ROOTFS, verbose=QL_VERBOSE.OFF) - ql.os.stdin = pipe.SimpleStringBuffer() - ql.os.stdout = pipe.SimpleStringBuffer() + self.ql.os.stdin = pipe.SimpleInStream(sys.stdin.fileno()) # take over the input to the program using a fake stdin + self.ql.os.stdout = pipe.NullOutStream(sys.stdout.fileno()) # disregard program output - ql.os.stdin.write(bytes("".join(flag) + "\n", 'utf-8')) - count = [0] + # execute program until it reaches the part that asks for input + self.ql.run(end=0x401030) - ql.hook_code(instruction_count, count) - ql.run() + # record replay starting and ending points. + # + # since the emulation halted upon entering a function, its return address is there on + # the stack. we use it to limit the emulation till function returns + self.replay_starts = self.ql.arch.regs.arch_pc + self.replay_ends = self.ql.stack_read(0) - print(ql.os.stdout.read().decode('utf-8'), end='') - print(" ============ count: %d ============ " % count[0]) + # instead of restarting the whole program every time a new flag character is guessed, + # we will restore its state to the latest point possible, fast-forwarding a good + # amount of start-up code that is not affected by the input. + # + # here we save the state just when the read input function is about to be called so we + # could use it to jumpstart the initialization part and get to the read input immediately + self.jumpstart = self.ql.save() or {} - return count[0] + # calibrate the replay instruction count by running the code with an invalid input + # first. the instruction count returned from the calibration process will be then + # used as a baseline for consequent replays + self.best_icount = self.__run(invalid) -def solve(): - # BJWXB_CTF{C5307D46-E70E-4038-B6F9-8C3F698B7C53} - prefix = list("BJWXB_CTF{C5307D46-E70E-4038-B6F9-8C3F698B7C") - flag = list("\x00" * 100) - base = get_count(prefix + flag) - i = 0 + def __run(self, input: bytes) -> int: + icount = [0] - try: - for i in range(len(flag)): - for j in "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-{}": - flag[i] = j - data = get_count(prefix + flag) - if data > base: - base = data - print("\n\n\n>>> FLAG: " + "".join(prefix + flag) + "\n\n\n") - break - if flag[i] == "}": + def __count_instructions(ql: Qiling, address: int, size: int): + icount[0] += 1 + + # set a hook to fire up every time an instruction is about to execute + hobj = self.ql.hook_code(__count_instructions) + + # feed stdin with input + self.ql.os.stdin.write(input + b'\n') + + # resume emulation till function returns + self.ql.run(begin=self.replay_starts, end=self.replay_ends) + + hobj.remove() + + return icount[0] + + def replay(self, input: bytes) -> bool: + """Restore state and replay with a new input. + + Returns an indication to execution progress: `True` if a progress + was made, `False` otherwise + """ + + # restore program's state back to the starting point + self.ql.restore(self.jumpstart) + + # resume emulation and count emulated instructions + curr_icount = self.__run(input) + + # the larger part of the input is correct, the more instructions are expected to be executed. this is true + # for traditional loop-based validations like strcmp or memcmp which bails as soon as a mismatch is found: + # more correct characters mean more loop iterations - thus more executed instructions. + # + # if we got a higher instruction count, it means we made a progress in the right direction + if curr_icount > self.best_icount: + self.best_icount = curr_icount + + return True + + return False + +def progress(msg: str) -> None: + print(msg, end='\r', file=sys.stderr, flush=True) + +def main(): + flag = bytearray(b'BJWXB_CTF{********-****-****-****-************}') + indices = (i for i, ch in enumerate(flag) if ch == ord('*')) + + # uppercase hex digits + charset = '0123456789ABCDEF' + + progress('Initializing...') + solver = Solver(flag) + + for i in indices: + for ch in charset: + flag[i] = ord(ch) + + progress(f'Guessing... {flag.decode()}') + + if solver.replay(flag): break - print("SOLVED!!!") - except KeyboardInterrupt: - print("STOP: KeyboardInterrupt") + + else: + raise RuntimeError('no match found') + + print(f'\nFlag found!') if __name__ == "__main__": - solve() + main() + +# expected flag: BJWXB_CTF{C5307D46-E70E-4038-B6F9-8C3F698B7C53} diff --git a/examples/hello_arm_linux_custom_syscall.py b/examples/hello_arm_linux_custom_syscall.py index ec21c687..6ae06db2 100644 --- a/examples/hello_arm_linux_custom_syscall.py +++ b/examples/hello_arm_linux_custom_syscall.py @@ -9,30 +9,37 @@ sys.path.append("..") from qiling import Qiling from qiling.const import QL_VERBOSE -def my_syscall_write(ql: Qiling, write_fd, write_buf, write_count, *args, **kw): - regreturn = 0 - +# customized system calls always use the same arguments list as the original ones, +# but with a Qiling instance as first argument +def my_syscall_write(ql: Qiling, fd: int, buf: int, count: int): try: - buf = ql.mem.read(write_buf, write_count) - ql.log.info("\n+++++++++\nmy write(%d,%x,%i) = %d\n+++++++++" % (write_fd, write_buf, write_count, regreturn)) - ql.os.fd[write_fd].write(buf) - regreturn = write_count + # read data from emulated memory + data = ql.mem.read(buf, count) + + # select the emulated file object that corresponds to the requested + # file descriptor + fobj = ql.os.fd[fd] + + # write the data into the file object, if it supports write operations + if hasattr(fobj, 'write'): + fobj.write(data) except: - regreturn = -1 - ql.log.info("\n+++++++++\nmy write(%d,%x,%i) = %d\n+++++++++" % (write_fd, write_buf, write_count, regreturn)) + ret = -1 + else: + ret = count - return regreturn + ql.log.info(f'my_syscall_write({fd}, {buf:#x}, {count}) = {ret}') + return ret if __name__ == "__main__": - ql = Qiling(["rootfs/arm_linux/bin/arm_hello"], "rootfs/arm_linux", verbose=QL_VERBOSE.DEBUG) - # Custom syscall handler by syscall name or syscall number. - # Known issue: If the syscall func is not be implemented in qiling, qiling does - # not know which func should be replaced. - # In that case, you must specify syscall by its number. + ql = Qiling([r'rootfs/arm_linux/bin/arm_hello'], r'rootfs/arm_linux', verbose=QL_VERBOSE.DEBUG) + + # replacing a system call with a custom implementation. + # note that system calls may be referred to either by their name or number. ql.os.set_syscall(0x04, my_syscall_write) - # set syscall by syscall name - #ql.os.set_syscall("write", my_syscall_write) + # an possible alternative: refer to a syscall by its name + #ql.os.set_syscall('write', my_syscall_write) ql.run() diff --git a/examples/hello_arm_linux_debug.py b/examples/hello_arm_linux_debug.py index a77b639f..4c7e502c 100644 --- a/examples/hello_arm_linux_debug.py +++ b/examples/hello_arm_linux_debug.py @@ -9,12 +9,13 @@ sys.path.append("..") from qiling import Qiling from qiling.const import QL_VERBOSE -def run_sandbox(path, rootfs, verbose): - ql = Qiling(path, rootfs, verbose = verbose) +if __name__ == "__main__": + ql = Qiling([r'rootfs/arm_linux/bin/arm_hello'], r'rootfs/arm_linux', verbose=QL_VERBOSE.DEBUG) + ql.debugger = "qdb" # enable qdb without options + + # other possible alternatives: # ql.debugger = "qdb::rr" # switch on record and replay with rr # ql.debugger = "qdb:0x1030c" # enable qdb and setup breakpoin at 0x1030c - ql.run() -if __name__ == "__main__": - run_sandbox(["rootfs/arm_linux/bin/arm_hello"], "rootfs/arm_linux", QL_VERBOSE.DEBUG) + ql.run() diff --git a/examples/rootfs b/examples/rootfs index 00760d17..15c8d842 160000 --- a/examples/rootfs +++ b/examples/rootfs @@ -1 +1 @@ -Subproject commit 00760d170e46095fcff97f79421262ef0e75d901 +Subproject commit 15c8d842876d709f013e4e42643d5d605cddbac9 diff --git a/examples/sality.py b/examples/sality.py index 6cc54812..b07c4273 100644 --- a/examples/sality.py +++ b/examples/sality.py @@ -151,8 +151,12 @@ def hook_StartServiceA(ql: Qiling, address: int, params): if service_handle.name in ql.os.services: service_path = ql.os.services[service_handle.name] service_path = ql.os.path.transform_to_real_path(service_path) + ql.amsint32_driver = Qiling([service_path], ql.rootfs, verbose=QL_VERBOSE.DEBUG) - init_unseen_symbols(ql.amsint32_driver, ql.amsint32_driver.loader.dlls["ntoskrnl.exe"]+0xb7695, b"NtTerminateProcess", 0, "ntoskrnl.exe") + ntoskrnl = ql.amsint32_driver.loader.get_image_by_name("ntoskrnl.exe") + assert ntoskrnl, 'ntoskernl.exe was not loaded' + + init_unseen_symbols(ql.amsint32_driver, ntoskrnl.base+0xb7695, b"NtTerminateProcess", 0, "ntoskrnl.exe") #ql.amsint32_driver.debugger= ":9999" try: ql.amsint32_driver.load() diff --git a/examples/scripts/dllscollector.bat b/examples/scripts/dllscollector.bat index 40c9e307..d45a55fa 100644 --- a/examples/scripts/dllscollector.bat +++ b/examples/scripts/dllscollector.bat @@ -1,90 +1,157 @@ -@echo off -REM -REM Example batch script to copy windows required DLLs and registry -REM - -mkdir examples\rootfs\x86_windows\Windows\System32 -mkdir examples\rootfs\x86_windows\Windows\System32\drivers -mkdir examples\rootfs\x86_windows\Windows\registry -mkdir examples\rootfs\x8664_windows\Windows\System32 -mkdir examples\rootfs\x8664_windows\Windows\SysWOW64 -mkdir examples\rootfs\x8664_windows\Windows\registry - -REM -REM Registry -REM -echo f | xcopy /f /y C:\Users\Default\NTUSER.DAT examples\rootfs\x8664_windows\Windows\registry\NTUSER.DAT -reg save hklm\system examples\rootfs\x8664_windows\Windows\registry\SYSTEM -reg save hklm\security examples\rootfs\x8664_windows\Windows\registry\SECURITY -reg save hklm\software examples\rootfs\x8664_windows\Windows\registry\SOFTWARE -reg save hklm\hardware examples\rootfs\x8664_windows\Windows\registry\HARDWARE -reg save hklm\SAM examples\rootfs\x8664_windows\Windows\registry\SAM -xcopy /d /y examples\rootfs\x8664_windows\Windows\registry\* examples\rootfs\x86_windows\Windows\registry\ - -REM -REM Dlls -REM -if exist %WINDIR%\SysWOW64\advapi32.dll xcopy /f /y %WINDIR%\SysWOW64\advapi32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\rpcrt4.dll xcopy /f /y %WINDIR%\SysWOW64\rpcrt4.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\crypt32.dll xcopy /f /y %WINDIR%\SysWOW64\crypt32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\iphlpapi.dll xcopy /f /y %WINDIR%\SysWOW64\iphlpapi.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\kernel32.dll xcopy /f /y %WINDIR%\SysWOW64\kernel32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\KernelBase.dll xcopy /f /y %WINDIR%\SysWOW64\KernelBase.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\mpr.dll xcopy /f /y %WINDIR%\SysWOW64\mpr.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\msvcp60.dll xcopy /f /y %WINDIR%\SysWOW64\msvcp60.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\msvcrt.dll xcopy /f /y %WINDIR%\SysWOW64\msvcrt.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\netapi32.dll xcopy /f /y %WINDIR%\SysWOW64\netapi32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\ntdll.dll xcopy /f /y %WINDIR%\SysWOW64\ntdll.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\ole32.dll xcopy /f /y %WINDIR%\SysWOW64\ole32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\urlmon.dll xcopy /f /y %WINDIR%\SysWOW64\urlmon.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\user32.dll xcopy /f /y %WINDIR%\SysWOW64\user32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\wsock32.dll xcopy /f /y %WINDIR%\SysWOW64\wsock32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\version.dll xcopy /f /y %WINDIR%\SysWOW64\version.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\winmm.dll xcopy /f /y %WINDIR%\SysWOW64\winmm.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\comctl32.dll xcopy /f /y %WINDIR%\SysWOW64\comctl32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\wininet.dll xcopy /f /y %WINDIR%\SysWOW64\wininet.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\psapi.dll xcopy /f /y %WINDIR%\SysWOW64\psapi.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\userenv.dll xcopy /f /y %WINDIR%\SysWOW64\userenv.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\uxtheme.dll xcopy /f /y %WINDIR%\SysWOW64\uxtheme.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\gdi32.dll xcopy /f /y %WINDIR%\SysWOW64\gdi32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\comdlg32.dll xcopy /f /y %WINDIR%\SysWOW64\comdlg32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\shell32.dll xcopy /f /y %WINDIR%\SysWOW64\shell32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\oleaut32.dll xcopy /f /y %WINDIR%\SysWOW64\oleaut32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\vcruntime140.dll xcopy /f /y %WINDIR%\SysWOW64\vcruntime140.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\winhttp.dll xcopy /f /y %WINDIR%\SysWOW64\winhttp.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\wininet.dll xcopy /f /y %WINDIR%\SysWOW64\wininet.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\ws2_32.dll xcopy /f /y %WINDIR%\SysWOW64\ws2_32.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\msvcr120_clr0400.dll echo f | xcopy /f /y %WINDIR%\SysWOW64\msvcr120_clr0400.dll "examples\rootfs\x86_windows\Windows\System32\msvcr110.dll" -if exist %WINDIR%\SysWOW64\downlevel\api-ms-win-crt-stdio-l1-1-0.dll xcopy /f /y %WINDIR%\SysWOW64\downlevel\api-ms-win-crt-stdio-l1-1-0.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\downlevel\api-ms-win-crt-runtime-l1-1-0.dll xcopy /f /y %WINDIR%\SysWOW64\downlevel\api-ms-win-crt-runtime-l1-1-0.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\downlevel\api-ms-win-crt-math-l1-1-0.dll xcopy /f /y %WINDIR%\SysWOW64\downlevel\api-ms-win-crt-math-l1-1-0.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\downlevel\api-ms-win-crt-locale-l1-1-0.dll xcopy /f /y %WINDIR%\SysWOW64\downlevel\api-ms-win-crt-locale-l1-1-0.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\downlevel\api-ms-win-crt-heap-l1-1-0.dll xcopy /f /y %WINDIR%\SysWOW64\downlevel\api-ms-win-crt-heap-l1-1-0.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\downlevel\api-ms-win-core-synch-l1-2-0.dll xcopy /f /y %WINDIR%\SysWOW64\downlevel\api-ms-win-core-synch-l1-2-0.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\downlevel\api-ms-win-core-fibers-l1-1-1.dll xcopy /f /y %WINDIR%\SysWOW64\downlevel\api-ms-win-core-fibers-l1-1-1.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\downlevel\api-ms-win-core-localization-l1-2-1.dll xcopy /f /y %WINDIR%\SysWOW64\downlevel\api-ms-win-core-localization-l1-2-1.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\downlevel\api-ms-win-core-sysinfo-l1-2-1.dll xcopy /f /y %WINDIR%\SysWOW64\downlevel\api-ms-win-core-sysinfo-l1-2-1.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\shlwapi.dll xcopy /f /y %WINDIR%\SysWOW64\shlwapi.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\SysWOW64\setupapi.dll xcopy /f /y %WINDIR%\SysWOW64\setupapi.dll "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\System32\ntoskrnl.exe xcopy /f /y %WINDIR%\System32\ntoskrnl.exe "examples\rootfs\x86_windows\Windows\System32\" -if exist %WINDIR%\winsxs\amd64_microsoft-windows-printing-xpsprint_31bf3856ad364e35_10.0.17763.194_none_20349c5a971eb293\XpsPrint.dll xcopy /f /y %WINDIR%\winsxs\amd64_microsoft-windows-printing-xpsprint_31bf3856ad364e35_10.0.17763.194_none_20349c5a971eb293\XpsPrint.dll "examples\rootfs\x86_windows\Windows\System32\" - -if exist %WINDIR%\System32\ntoskrnl.exe xcopy /f /y %WINDIR%\System32\ntoskrnl.exe "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\advapi32.dll xcopy /f /y %WINDIR%\System32\advapi32.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\kernel32.dll xcopy /f /y %WINDIR%\System32\kernel32.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\KernelBase.dll xcopy /f /y %WINDIR%\System32\KernelBase.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\msvcrt.dll xcopy /f /y %WINDIR%\System32\msvcrt.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\ntdll.dll xcopy /f /y %WINDIR%\System32\ntdll.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\urlmon.dll xcopy /f /y %WINDIR%\System32\urlmon.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\user32.dll xcopy /f /y %WINDIR%\System32\user32.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\ws2_32.dll xcopy /f /y %WINDIR%\System32\ws2_32.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\vcruntime140.dll xcopy /f /y %WINDIR%\System32\vcruntime140.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\downlevel\api-ms-win-crt-stdio-l1-1-0.dll xcopy /f /y %WINDIR%\System32\downlevel\api-ms-win-crt-stdio-l1-1-0.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\downlevel\api-ms-win-crt-runtime-l1-1-0.dll xcopy /f /y %WINDIR%\System32\downlevel\api-ms-win-crt-runtime-l1-1-0.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\downlevel\api-ms-win-crt-math-l1-1-0.dll xcopy /f /y %WINDIR%\System32\downlevel\api-ms-win-crt-math-l1-1-0.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\downlevel\api-ms-win-crt-locale-l1-1-0.dll xcopy /f /y %WINDIR%\System32\downlevel\api-ms-win-crt-locale-l1-1-0.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\downlevel\api-ms-win-crt-heap-l1-1-0.dll xcopy /f /y %WINDIR%\System32\downlevel\api-ms-win-crt-heap-l1-1-0.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\vcruntime140d.dll xcopy /f /y %WINDIR%\System32\vcruntime140d.dll "examples\rootfs\x8664_windows\Windows\System32\" -if exist %WINDIR%\System32\ucrtbased.dll xcopy /f /y %WINDIR%\System32\ucrtbased.dll "examples\rootfs\x8664_windows\Windows\System32\" - -exit /b +@ECHO OFF + +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:: Create the emulated Windows directory structure and registry :: +:::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: + +:: Host system directories +SET SYSDIR32="%WINDIR%\SysWOW64" +SET SYSDIR64="%WINDIR%\System32" + +:: Qiling rootfs directories +SET QL_WINDIR32="examples\rootfs\x86_windows\Windows" +SET QL_WINDIR64="examples\rootfs\x8664_windows\Windows" + +SET QL_SYSDIR32="%QL_WINDIR32%\System32" +SET QL_SYSDIR64="%QL_WINDIR64%\System32" + +SET QL_REGDIR32="%QL_WINDIR32%\registry" +SET QL_REGDIR64="%QL_WINDIR64%\registry" + +:: Create emulated Windows directory structure +MKDIR %QL_REGDIR32% +MKDIR %QL_SYSDIR32% +MKDIR "%QL_SYSDIR32%\drivers" +MKDIR "%QL_WINDIR32%\Temp" + +MKDIR %QL_REGDIR64% +MKDIR %QL_SYSDIR64% +MKDIR "%QL_SYSDIR64%\drivers" +MKDIR "%QL_WINDIR64%\Temp" + +:: Generate emulated Windows registry (requires Administrator privileges) +REG SAVE HKLM\SYSTEM %QL_REGDIR64%\SYSTEM /Y +REG SAVE HKLM\SECURITY %QL_REGDIR64%\SECURITY /Y +REG SAVE HKLM\SOFTWARE %QL_REGDIR64%\SOFTWARE /Y +REG SAVE HKLM\HARDWARE %QL_REGDIR64%\HARDWARE /Y +REG SAVE HKLM\SAM %QL_REGDIR64%\SAM /Y +COPY /B /Y C:\Users\Default\NTUSER.DAT "%QL_REGDIR64%\NTUSER.DAT" + +:: Duplicate generated registry +XCOPY /F /D /Y %QL_REGDIR64%\* %QL_REGDIR32% + +:: Collect 32-bit DLL files +CALL :collect_dll32 advapi32.dll +CALL :collect_dll32 bcrypt.dll +CALL :collect_dll32 cfgmgr32.dll +CALL :collect_dll32 ci.dll +CALL :collect_dll32 combase.dll +CALL :collect_dll32 comctl32.dll +CALL :collect_dll32 comdlg32.dll +CALL :collect_dll32 crypt32.dll +CALL :collect_dll32 cryptbase.dll +CALL :collect_dll32 gdi32.dll +CALL :collect_dll32 hal.dll +CALL :collect_dll32 iphlpapi.dll +CALL :collect_dll32 kdcom.dll +CALL :collect_dll32 kernel32.dll +CALL :collect_dll32 KernelBase.dll +CALL :collect_dll32 mpr.dll +CALL :collect_dll32 msvcp_win.dll +CALL :collect_dll32 msvcp60.dll +CALL :collect_dll32 msvcr120_clr0400.dll, msvcr110.dll +CALL :collect_dll32 msvcrt.dll +CALL :collect_dll32 netapi32.dll +CALL :collect_dll32 ntdll.dll +CALL :collect_dll32 ole32.dll +CALL :collect_dll32 oleaut32.dll +CALL :collect_dll32 psapi.dll +CALL :collect_dll32 rpcrt4.dll +CALL :collect_dll32 sechost.dll +CALL :collect_dll32 setupapi.dll +CALL :collect_dll32 shell32.dll +CALL :collect_dll32 shlwapi.dll +CALL :collect_dll32 sspicli.dll +CALL :collect_dll32 ucrtbase.dll +CALL :collect_dll32 ucrtbased.dll +CALL :collect_dll32 urlmon.dll +CALL :collect_dll32 user32.dll +CALL :collect_dll32 userenv.dll +CALL :collect_dll32 uxtheme.dll +CALL :collect_dll32 vcruntime140.dll +CALL :collect_dll32 version.dll +CALL :collect_dll32 win32u.dll +CALL :collect_dll32 winhttp.dll +CALL :collect_dll32 wininet.dll +CALL :collect_dll32 winmm.dll +CALL :collect_dll32 ws2_32.dll +CALL :collect_dll32 wsock32.dll + +CALL :collect_dll32 downlevel\api-ms-win-core-fibers-l1-1-1.dll +CALL :collect_dll32 downlevel\api-ms-win-core-localization-l1-2-1.dll +CALL :collect_dll32 downlevel\api-ms-win-core-synch-l1-2-0.dll +CALL :collect_dll32 downlevel\api-ms-win-core-sysinfo-l1-2-1.dll +CALL :collect_dll32 downlevel\api-ms-win-crt-heap-l1-1-0.dll +CALL :collect_dll32 downlevel\api-ms-win-crt-locale-l1-1-0.dll +CALL :collect_dll32 downlevel\api-ms-win-crt-math-l1-1-0.dll +CALL :collect_dll32 downlevel\api-ms-win-crt-runtime-l1-1-0.dll +CALL :collect_dll32 downlevel\api-ms-win-crt-stdio-l1-1-0.dll + +:: Collect 64-bit DLL files +CALL :collect_dll64 advapi32.dll +CALL :collect_dll64 gdi32.dll +CALL :collect_dll64 kernel32.dll +CALL :collect_dll64 KernelBase.dll +CALL :collect_dll64 msvcrt.dll +CALL :collect_dll64 ntdll.dll +CALL :collect_dll64 ntoskrnl.exe +CALL :collect_dll64 msvcp_win.dll +CALL :collect_dll64 ucrtbase.dll +CALL :collect_dll64 ucrtbased.dll +CALL :collect_dll64 urlmon.dll +CALL :collect_dll64 rpcrt4.dll +CALL :collect_dll64 sechost.dll +CALL :collect_dll64 shell32.dll +CALL :collect_dll64 shlwapi.dll +CALL :collect_dll64 user32.dll +CALL :collect_dll64 vcruntime140.dll +CALL :collect_dll64 vcruntime140d.dll +CALL :collect_dll64 win32u.dll +CALL :collect_dll64 winhttp.dll +CALL :collect_dll64 wininet.dll +CALL :collect_dll64 ws2_32.dll + +CALL :collect_dll64 downlevel\api-ms-win-crt-heap-l1-1-0.dll +CALL :collect_dll64 downlevel\api-ms-win-crt-locale-l1-1-0.dll +CALL :collect_dll64 downlevel\api-ms-win-crt-math-l1-1-0.dll +CALL :collect_dll64 downlevel\api-ms-win-crt-runtime-l1-1-0.dll +CALL :collect_dll64 downlevel\api-ms-win-crt-stdio-l1-1-0.dll + +:: Collect extras +CALL :collect %SYSDIR64%, ntoskrnl.exe, %QL_SYSDIR32% + +:: All done! +EXIT /B 0 + +:: Functions definitions +:normpath +SET %1=%~dpfn2 +EXIT /B + +:collect +CALL :normpath SRC, %~1\%~2 +CALL :normpath DST, %~3\%~4 + +IF EXIST %SRC% ( + ECHO %SRC% -^> %DST% + COPY /B /Y "%SRC%" "%DST%" >NUL +) +EXIT /B + +:collect_dll64 +CALL :collect %SYSDIR64%, %~1, %QL_SYSDIR64%, %~2 +EXIT /B + +:collect_dll32 +CALL :collect %SYSDIR32%, %~1, %QL_SYSDIR32%, %~2 +EXIT /B diff --git a/examples/windows_trace.py b/examples/windows_trace.py index e218b833..8394e3f9 100644 --- a/examples/windows_trace.py +++ b/examples/windows_trace.py @@ -115,7 +115,6 @@ if __name__ == "__main__": parser.add_argument("-t", "--trace", help="Enable full trace", action='store_true', default=False) parser.add_argument("-R", "--root", help="rootfs", default=None) parser.add_argument("-d", "--dump", help="Directory to dump memory regions to", default="dump") - #parser.add_argument("-a", "--automatize_input", help="Automatize writes on standard input", default=False) parser.add_argument("-p ", "--profile", help="customized profile", default="qiling/profiles/windows.ql") parser.add_argument('input', nargs='*') diff --git a/qiling/__init__.py b/qiling/__init__.py index 5a5ddbac..96f11337 100644 --- a/qiling/__init__.py +++ b/qiling/__init__.py @@ -1,2 +1,9 @@ +import importlib.metadata from .core import Qiling -from .__version__ import __version__ \ No newline at end of file + +try: + __version__ = importlib.metadata.version(__package__ or __name__) +except importlib.metadata.PackageNotFoundError: + __version__ = "0.0.0" + +__all__ = ['Qiling'] diff --git a/qiling/__version__.py b/qiling/__version__.py deleted file mode 100644 index 94f2af16..00000000 --- a/qiling/__version__.py +++ /dev/null @@ -1,3 +0,0 @@ -# NOTE: use "-dev" for dev branch -#__version__ = "1.4.3" -__version__ = "1.4.3" + "-dev" diff --git a/qiling/arch/arm.py b/qiling/arch/arm.py index c03d0d8c..394258f7 100644 --- a/qiling/arch/arm.py +++ b/qiling/arch/arm.py @@ -41,7 +41,11 @@ class QlArchARM(QlArch): @cached_property def regs(self) -> QlRegisterManager: - regs_map = arm_const.reg_map + regs_map = dict( + **arm_const.reg_map, + **arm_const.reg_vfp + ) + pc_reg = 'pc' sp_reg = 'sp' @@ -53,17 +57,7 @@ class QlArchARM(QlArch): @property def endian(self) -> QL_ENDIAN: - # FIXME: ARM is a bi-endian architecture which allows flipping core endianess - # while running. endianess is tested in runtime through CPSR[9], however unicorn - # doesn't reflect the endianess correctly through that bit. - # @see: https://github.com/unicorn-engine/unicorn/issues/1542 - # - # we work around this by using the initial endianess configuration, even though - # it might have been changed since. - # - # return QL_ENDIAN.EB if self.regs.cpsr & (1 << 9) else QL_ENDIAN.EL - - return self._init_endian + return QL_ENDIAN.EB if self.regs.cpsr & (1 << 9) else QL_ENDIAN.EL @property def effective_pc(self) -> int: diff --git a/qiling/arch/arm64.py b/qiling/arch/arm64.py index 0367a818..65949669 100644 --- a/qiling/arch/arm64.py +++ b/qiling/arch/arm64.py @@ -26,7 +26,13 @@ class QlArchARM64(QlArch): def regs(self) -> QlRegisterManager: regs_map = dict( **arm64_const.reg_map, - **arm64_const.reg_map_w + **arm64_const.reg_map_b, + **arm64_const.reg_map_d, + **arm64_const.reg_map_h, + **arm64_const.reg_map_q, + **arm64_const.reg_map_s, + **arm64_const.reg_map_w, + **arm64_const.reg_map_v ) pc_reg = 'pc' diff --git a/qiling/arch/arm64_const.py b/qiling/arch/arm64_const.py index c706d10c..ee2b5516 100644 --- a/qiling/arch/arm64_const.py +++ b/qiling/arch/arm64_const.py @@ -1,79 +1,289 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # from unicorn.arm64_const import * reg_map = { - "x0": UC_ARM64_REG_X0, - "x1": UC_ARM64_REG_X1, - "x2": UC_ARM64_REG_X2, - "x3": UC_ARM64_REG_X3, - "x4": UC_ARM64_REG_X4, - "x5": UC_ARM64_REG_X5, - "x6": UC_ARM64_REG_X6, - "x7": UC_ARM64_REG_X7, - "x8": UC_ARM64_REG_X8, - "x9": UC_ARM64_REG_X9, - "x10": UC_ARM64_REG_X10, - "x11": UC_ARM64_REG_X11, - "x12": UC_ARM64_REG_X12, - "x13": UC_ARM64_REG_X13, - "x14": UC_ARM64_REG_X14, - "x15": UC_ARM64_REG_X15, - "x16": UC_ARM64_REG_X16, - "x17": UC_ARM64_REG_X17, - "x18": UC_ARM64_REG_X18, - "x19": UC_ARM64_REG_X19, - "x20": UC_ARM64_REG_X20, - "x21": UC_ARM64_REG_X21, - "x22": UC_ARM64_REG_X22, - "x23": UC_ARM64_REG_X23, - "x24": UC_ARM64_REG_X24, - "x25": UC_ARM64_REG_X25, - "x26": UC_ARM64_REG_X26, - "x27": UC_ARM64_REG_X27, - "x28": UC_ARM64_REG_X28, - "x29": UC_ARM64_REG_X29, - "x30": UC_ARM64_REG_X30, - "sp": UC_ARM64_REG_SP, - "pc": UC_ARM64_REG_PC, - "lr": UC_ARM64_REG_LR, - "cpacr_el1": UC_ARM64_REG_CPACR_EL1, - "tpidr_el0": UC_ARM64_REG_TPIDR_EL0, + "x0": UC_ARM64_REG_X0, + "x1": UC_ARM64_REG_X1, + "x2": UC_ARM64_REG_X2, + "x3": UC_ARM64_REG_X3, + "x4": UC_ARM64_REG_X4, + "x5": UC_ARM64_REG_X5, + "x6": UC_ARM64_REG_X6, + "x7": UC_ARM64_REG_X7, + "x8": UC_ARM64_REG_X8, + "x9": UC_ARM64_REG_X9, + "x10": UC_ARM64_REG_X10, + "x11": UC_ARM64_REG_X11, + "x12": UC_ARM64_REG_X12, + "x13": UC_ARM64_REG_X13, + "x14": UC_ARM64_REG_X14, + "x15": UC_ARM64_REG_X15, + "x16": UC_ARM64_REG_X16, + "x17": UC_ARM64_REG_X17, + "x18": UC_ARM64_REG_X18, + "x19": UC_ARM64_REG_X19, + "x20": UC_ARM64_REG_X20, + "x21": UC_ARM64_REG_X21, + "x22": UC_ARM64_REG_X22, + "x23": UC_ARM64_REG_X23, + "x24": UC_ARM64_REG_X24, + "x25": UC_ARM64_REG_X25, + "x26": UC_ARM64_REG_X26, + "x27": UC_ARM64_REG_X27, + "x28": UC_ARM64_REG_X28, + "x29": UC_ARM64_REG_X29, + "x30": UC_ARM64_REG_X30, + "sp": UC_ARM64_REG_SP, + "pc": UC_ARM64_REG_PC, + "lr": UC_ARM64_REG_LR, + "cpacr_el1": UC_ARM64_REG_CPACR_EL1, + "tpidr_el0": UC_ARM64_REG_TPIDR_EL0 +} + +reg_map_b = { + "b0" : UC_ARM64_REG_B0, + "b1" : UC_ARM64_REG_B1, + "b2" : UC_ARM64_REG_B2, + "b3" : UC_ARM64_REG_B3, + "b4" : UC_ARM64_REG_B4, + "b5" : UC_ARM64_REG_B5, + "b6" : UC_ARM64_REG_B6, + "b7" : UC_ARM64_REG_B7, + "b8" : UC_ARM64_REG_B8, + "b9" : UC_ARM64_REG_B9, + "b10" : UC_ARM64_REG_B10, + "b11" : UC_ARM64_REG_B11, + "b12" : UC_ARM64_REG_B12, + "b13" : UC_ARM64_REG_B13, + "b14" : UC_ARM64_REG_B14, + "b15" : UC_ARM64_REG_B15, + "b16" : UC_ARM64_REG_B16, + "b17" : UC_ARM64_REG_B17, + "b18" : UC_ARM64_REG_B18, + "b19" : UC_ARM64_REG_B19, + "b20" : UC_ARM64_REG_B20, + "b21" : UC_ARM64_REG_B21, + "b22" : UC_ARM64_REG_B22, + "b23" : UC_ARM64_REG_B23, + "b24" : UC_ARM64_REG_B24, + "b25" : UC_ARM64_REG_B25, + "b26" : UC_ARM64_REG_B26, + "b27" : UC_ARM64_REG_B27, + "b28" : UC_ARM64_REG_B28, + "b29" : UC_ARM64_REG_B29, + "b30" : UC_ARM64_REG_B30, + "b31" : UC_ARM64_REG_B31 +} + +reg_map_d = { + "d0" : UC_ARM64_REG_D0, + "d1" : UC_ARM64_REG_D1, + "d2" : UC_ARM64_REG_D2, + "d3" : UC_ARM64_REG_D3, + "d4" : UC_ARM64_REG_D4, + "d5" : UC_ARM64_REG_D5, + "d6" : UC_ARM64_REG_D6, + "d7" : UC_ARM64_REG_D7, + "d8" : UC_ARM64_REG_D8, + "d9" : UC_ARM64_REG_D9, + "d10" : UC_ARM64_REG_D10, + "d11" : UC_ARM64_REG_D11, + "d12" : UC_ARM64_REG_D12, + "d13" : UC_ARM64_REG_D13, + "d14" : UC_ARM64_REG_D14, + "d15" : UC_ARM64_REG_D15, + "d16" : UC_ARM64_REG_D16, + "d17" : UC_ARM64_REG_D17, + "d18" : UC_ARM64_REG_D18, + "d19" : UC_ARM64_REG_D19, + "d20" : UC_ARM64_REG_D20, + "d21" : UC_ARM64_REG_D21, + "d22" : UC_ARM64_REG_D22, + "d23" : UC_ARM64_REG_D23, + "d24" : UC_ARM64_REG_D24, + "d25" : UC_ARM64_REG_D25, + "d26" : UC_ARM64_REG_D26, + "d27" : UC_ARM64_REG_D27, + "d28" : UC_ARM64_REG_D28, + "d29" : UC_ARM64_REG_D29, + "d30" : UC_ARM64_REG_D30, + "d31" : UC_ARM64_REG_D31 +} + +reg_map_h = { + "h0" : UC_ARM64_REG_H0, + "h1" : UC_ARM64_REG_H1, + "h2" : UC_ARM64_REG_H2, + "h3" : UC_ARM64_REG_H3, + "h4" : UC_ARM64_REG_H4, + "h5" : UC_ARM64_REG_H5, + "h6" : UC_ARM64_REG_H6, + "h7" : UC_ARM64_REG_H7, + "h8" : UC_ARM64_REG_H8, + "h9" : UC_ARM64_REG_H9, + "h10" : UC_ARM64_REG_H10, + "h11" : UC_ARM64_REG_H11, + "h12" : UC_ARM64_REG_H12, + "h13" : UC_ARM64_REG_H13, + "h14" : UC_ARM64_REG_H14, + "h15" : UC_ARM64_REG_H15, + "h16" : UC_ARM64_REG_H16, + "h17" : UC_ARM64_REG_H17, + "h18" : UC_ARM64_REG_H18, + "h19" : UC_ARM64_REG_H19, + "h20" : UC_ARM64_REG_H20, + "h21" : UC_ARM64_REG_H21, + "h22" : UC_ARM64_REG_H22, + "h23" : UC_ARM64_REG_H23, + "h24" : UC_ARM64_REG_H24, + "h25" : UC_ARM64_REG_H25, + "h26" : UC_ARM64_REG_H26, + "h27" : UC_ARM64_REG_H27, + "h28" : UC_ARM64_REG_H28, + "h29" : UC_ARM64_REG_H29, + "h30" : UC_ARM64_REG_H30, + "h31" : UC_ARM64_REG_H31 +} + +reg_map_q = { + "q0" : UC_ARM64_REG_Q0, + "q1" : UC_ARM64_REG_Q1, + "q2" : UC_ARM64_REG_Q2, + "q3" : UC_ARM64_REG_Q3, + "q4" : UC_ARM64_REG_Q4, + "q5" : UC_ARM64_REG_Q5, + "q6" : UC_ARM64_REG_Q6, + "q7" : UC_ARM64_REG_Q7, + "q8" : UC_ARM64_REG_Q8, + "q9" : UC_ARM64_REG_Q9, + "q10" : UC_ARM64_REG_Q10, + "q11" : UC_ARM64_REG_Q11, + "q12" : UC_ARM64_REG_Q12, + "q13" : UC_ARM64_REG_Q13, + "q14" : UC_ARM64_REG_Q14, + "q15" : UC_ARM64_REG_Q15, + "q16" : UC_ARM64_REG_Q16, + "q17" : UC_ARM64_REG_Q17, + "q18" : UC_ARM64_REG_Q18, + "q19" : UC_ARM64_REG_Q19, + "q20" : UC_ARM64_REG_Q20, + "q21" : UC_ARM64_REG_Q21, + "q22" : UC_ARM64_REG_Q22, + "q23" : UC_ARM64_REG_Q23, + "q24" : UC_ARM64_REG_Q24, + "q25" : UC_ARM64_REG_Q25, + "q26" : UC_ARM64_REG_Q26, + "q27" : UC_ARM64_REG_Q27, + "q28" : UC_ARM64_REG_Q28, + "q29" : UC_ARM64_REG_Q29, + "q30" : UC_ARM64_REG_Q30, + "q31" : UC_ARM64_REG_Q31 +} + +reg_map_s = { + "s0" : UC_ARM64_REG_S0, + "s1" : UC_ARM64_REG_S1, + "s2" : UC_ARM64_REG_S2, + "s3" : UC_ARM64_REG_S3, + "s4" : UC_ARM64_REG_S4, + "s5" : UC_ARM64_REG_S5, + "s6" : UC_ARM64_REG_S6, + "s7" : UC_ARM64_REG_S7, + "s8" : UC_ARM64_REG_S8, + "s9" : UC_ARM64_REG_S9, + "s10" : UC_ARM64_REG_S10, + "s11" : UC_ARM64_REG_S11, + "s12" : UC_ARM64_REG_S12, + "s13" : UC_ARM64_REG_S13, + "s14" : UC_ARM64_REG_S14, + "s15" : UC_ARM64_REG_S15, + "s16" : UC_ARM64_REG_S16, + "s17" : UC_ARM64_REG_S17, + "s18" : UC_ARM64_REG_S18, + "s19" : UC_ARM64_REG_S19, + "s20" : UC_ARM64_REG_S20, + "s21" : UC_ARM64_REG_S21, + "s22" : UC_ARM64_REG_S22, + "s23" : UC_ARM64_REG_S23, + "s24" : UC_ARM64_REG_S24, + "s25" : UC_ARM64_REG_S25, + "s26" : UC_ARM64_REG_S26, + "s27" : UC_ARM64_REG_S27, + "s28" : UC_ARM64_REG_S28, + "s29" : UC_ARM64_REG_S29, + "s30" : UC_ARM64_REG_S30, + "s31" : UC_ARM64_REG_S31 } reg_map_w = { - "w0" : UC_ARM64_REG_W0, - "w1" : UC_ARM64_REG_W1, - "w2" : UC_ARM64_REG_W2, - "w3" : UC_ARM64_REG_W3, - "w4" : UC_ARM64_REG_W4, - "w5" : UC_ARM64_REG_W5, - "w6" : UC_ARM64_REG_W6, - "w7" : UC_ARM64_REG_W7, - "w8" : UC_ARM64_REG_W8, - "w9" : UC_ARM64_REG_W9, - "w10" : UC_ARM64_REG_W10, - "w11" : UC_ARM64_REG_W11, - "w12" : UC_ARM64_REG_W12, - "w13" : UC_ARM64_REG_W13, - "w14" : UC_ARM64_REG_W14, - "w15" : UC_ARM64_REG_W15, - "w16" : UC_ARM64_REG_W16, - "w17" : UC_ARM64_REG_W17, - "w18" : UC_ARM64_REG_W18, - "w19" : UC_ARM64_REG_W19, - "w20" : UC_ARM64_REG_W20, - "w21" : UC_ARM64_REG_W21, - "w22" : UC_ARM64_REG_W22, - "w23" : UC_ARM64_REG_W23, - "w24" : UC_ARM64_REG_W24, - "w25" : UC_ARM64_REG_W25, - "w26" : UC_ARM64_REG_W26, - "w27" : UC_ARM64_REG_W27, - "w28" : UC_ARM64_REG_W28, - "w29" : UC_ARM64_REG_W29, - "w30" : UC_ARM64_REG_W30, -} \ No newline at end of file + "w0" : UC_ARM64_REG_W0, + "w1" : UC_ARM64_REG_W1, + "w2" : UC_ARM64_REG_W2, + "w3" : UC_ARM64_REG_W3, + "w4" : UC_ARM64_REG_W4, + "w5" : UC_ARM64_REG_W5, + "w6" : UC_ARM64_REG_W6, + "w7" : UC_ARM64_REG_W7, + "w8" : UC_ARM64_REG_W8, + "w9" : UC_ARM64_REG_W9, + "w10" : UC_ARM64_REG_W10, + "w11" : UC_ARM64_REG_W11, + "w12" : UC_ARM64_REG_W12, + "w13" : UC_ARM64_REG_W13, + "w14" : UC_ARM64_REG_W14, + "w15" : UC_ARM64_REG_W15, + "w16" : UC_ARM64_REG_W16, + "w17" : UC_ARM64_REG_W17, + "w18" : UC_ARM64_REG_W18, + "w19" : UC_ARM64_REG_W19, + "w20" : UC_ARM64_REG_W20, + "w21" : UC_ARM64_REG_W21, + "w22" : UC_ARM64_REG_W22, + "w23" : UC_ARM64_REG_W23, + "w24" : UC_ARM64_REG_W24, + "w25" : UC_ARM64_REG_W25, + "w26" : UC_ARM64_REG_W26, + "w27" : UC_ARM64_REG_W27, + "w28" : UC_ARM64_REG_W28, + "w29" : UC_ARM64_REG_W29, + "w30" : UC_ARM64_REG_W30 +} + +reg_map_v = { + "v0" : UC_ARM64_REG_V0, + "v1" : UC_ARM64_REG_V1, + "v2" : UC_ARM64_REG_V2, + "v3" : UC_ARM64_REG_V3, + "v4" : UC_ARM64_REG_V4, + "v5" : UC_ARM64_REG_V5, + "v6" : UC_ARM64_REG_V6, + "v7" : UC_ARM64_REG_V7, + "v8" : UC_ARM64_REG_V8, + "v9" : UC_ARM64_REG_V9, + "v10" : UC_ARM64_REG_V10, + "v11" : UC_ARM64_REG_V11, + "v12" : UC_ARM64_REG_V12, + "v13" : UC_ARM64_REG_V13, + "v14" : UC_ARM64_REG_V14, + "v15" : UC_ARM64_REG_V15, + "v16" : UC_ARM64_REG_V16, + "v17" : UC_ARM64_REG_V17, + "v18" : UC_ARM64_REG_V18, + "v19" : UC_ARM64_REG_V19, + "v20" : UC_ARM64_REG_V20, + "v21" : UC_ARM64_REG_V21, + "v22" : UC_ARM64_REG_V22, + "v23" : UC_ARM64_REG_V23, + "v24" : UC_ARM64_REG_V24, + "v25" : UC_ARM64_REG_V25, + "v26" : UC_ARM64_REG_V26, + "v27" : UC_ARM64_REG_V27, + "v28" : UC_ARM64_REG_V28, + "v29" : UC_ARM64_REG_V29, + "v30" : UC_ARM64_REG_V30, + "v31" : UC_ARM64_REG_V31 +} diff --git a/qiling/arch/arm_const.py b/qiling/arch/arm_const.py index e9339366..2a268cdf 100644 --- a/qiling/arch/arm_const.py +++ b/qiling/arch/arm_const.py @@ -4,39 +4,63 @@ # from unicorn.arm_const import * -from enum import IntEnum reg_map = { - "r0": UC_ARM_REG_R0, - "r1": UC_ARM_REG_R1, - "r2": UC_ARM_REG_R2, - "r3": UC_ARM_REG_R3, - "r4": UC_ARM_REG_R4, - "r5": UC_ARM_REG_R5, - "r6": UC_ARM_REG_R6, - "r7": UC_ARM_REG_R7, - "r8": UC_ARM_REG_R8, - "r9": UC_ARM_REG_R9, - "r10": UC_ARM_REG_R10, - "r11": UC_ARM_REG_R11, - "r12": UC_ARM_REG_R12, - "sp": UC_ARM_REG_SP, - "lr": UC_ARM_REG_LR, - "pc": UC_ARM_REG_PC, - - # CPSR needs to be at offset 25 for GDB, see https://sourceware.org/git/?p=binutils-gdb.git;a=blob;f=gdb/arch/arm.h;h=fa589fd0582c0add627a068e6f4947a909c45e86;hb=HEAD#l34 - # The fp registers inbetween have become obsolete - "f0": UC_ARM_REG_INVALID, - "f1": UC_ARM_REG_INVALID, - "f2": UC_ARM_REG_INVALID, - "f3": UC_ARM_REG_INVALID, - "f4": UC_ARM_REG_INVALID, - "f5": UC_ARM_REG_INVALID, - "f6": UC_ARM_REG_INVALID, - "f7": UC_ARM_REG_INVALID, - "fps": UC_ARM_REG_INVALID, - "cpsr": UC_ARM_REG_CPSR, - "c1_c0_2": UC_ARM_REG_C1_C0_2, - "c13_c0_3": UC_ARM_REG_C13_C0_3, - "fpexc": UC_ARM_REG_FPEXC, + "r0": UC_ARM_REG_R0, + "r1": UC_ARM_REG_R1, + "r2": UC_ARM_REG_R2, + "r3": UC_ARM_REG_R3, + "r4": UC_ARM_REG_R4, + "r5": UC_ARM_REG_R5, + "r6": UC_ARM_REG_R6, + "r7": UC_ARM_REG_R7, + "r8": UC_ARM_REG_R8, + "r9": UC_ARM_REG_R9, + "r10": UC_ARM_REG_R10, + "r11": UC_ARM_REG_R11, + "r12": UC_ARM_REG_R12, + "sp": UC_ARM_REG_SP, + "lr": UC_ARM_REG_LR, + "pc": UC_ARM_REG_PC, + + "cpsr": UC_ARM_REG_CPSR, + "c1_c0_2": UC_ARM_REG_C1_C0_2, + "c13_c0_3": UC_ARM_REG_C13_C0_3, + "fpexc": UC_ARM_REG_FPEXC +} + +reg_vfp = { + "d0" : UC_ARM_REG_D0, + "d1" : UC_ARM_REG_D1, + "d2" : UC_ARM_REG_D2, + "d3" : UC_ARM_REG_D3, + "d4" : UC_ARM_REG_D4, + "d5" : UC_ARM_REG_D5, + "d6" : UC_ARM_REG_D6, + "d7" : UC_ARM_REG_D7, + "d8" : UC_ARM_REG_D8, + "d9" : UC_ARM_REG_D9, + "d10" : UC_ARM_REG_D10, + "d11" : UC_ARM_REG_D11, + "d12" : UC_ARM_REG_D12, + "d13" : UC_ARM_REG_D13, + "d14" : UC_ARM_REG_D14, + "d15" : UC_ARM_REG_D15, + "d16" : UC_ARM_REG_D16, + "d17" : UC_ARM_REG_D17, + "d18" : UC_ARM_REG_D18, + "d19" : UC_ARM_REG_D19, + "d20" : UC_ARM_REG_D20, + "d21" : UC_ARM_REG_D21, + "d22" : UC_ARM_REG_D22, + "d23" : UC_ARM_REG_D23, + "d24" : UC_ARM_REG_D24, + "d25" : UC_ARM_REG_D25, + "d26" : UC_ARM_REG_D26, + "d27" : UC_ARM_REG_D27, + "d28" : UC_ARM_REG_D28, + "d29" : UC_ARM_REG_D29, + "d30" : UC_ARM_REG_D30, + "d31" : UC_ARM_REG_D31, + "fpscr" : UC_ARM_REG_FPSCR } diff --git a/qiling/arch/evm/analysis/signatures.py b/qiling/arch/evm/analysis/signatures.py index aebe55f0..e7eed509 100644 --- a/qiling/arch/evm/analysis/signatures.py +++ b/qiling/arch/evm/analysis/signatures.py @@ -1,4 +1,5 @@ -import os +import inspect +from pathlib import Path import re import logging import json @@ -92,8 +93,8 @@ def analysis_func_sign(insns:list, engine_num=1): class signatures_engine_1: @staticmethod def find_signature(sign): - path = os.path.split(os.path.realpath(__file__))[0] + '/signatures.json' - with open(path) as data_file: + path = Path(inspect.getfile(inspect.getframe())).parent / 'signatures.json' + with path.open('r') as data_file: data = json.load(data_file) list_name = [name for name, hexa in data.items() if hexa == sign] diff --git a/qiling/arch/evm/evm.py b/qiling/arch/evm/evm.py index b7479cd5..81ce642c 100644 --- a/qiling/arch/evm/evm.py +++ b/qiling/arch/evm/evm.py @@ -3,10 +3,13 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework -from qiling.const import * -from ..arch import QlArch -from .vm.evm import QlArchEVMEmulator +import types +from qiling.arch.arch import QlArch +from qiling.arch.evm.hooks import monkeypatch_core_hooks +from qiling.arch.evm.vm.evm import QlArchEVMEmulator +from qiling.arch.evm.vm.message import Message +from qiling.const import * class QlArchEVM(QlArch): type = QL_ARCH.EVM @@ -16,6 +19,9 @@ class QlArchEVM(QlArch): super(QlArchEVM, self).__init__(ql) self.evm = QlArchEVMEmulator(self.ql) + monkeypatch_core_hooks(self.ql) + monkeypatch_core_methods(self.ql) + def run(self, msg): return self.evm.vm.execute_message(msg) @@ -38,3 +44,13 @@ class QlArchEVM(QlArch): @property def endian(self) -> QL_ENDIAN: return QL_ENDIAN.EL + + +def __evm_run(self, code: Message): + return self.arch.run(code) + +def monkeypatch_core_methods(ql): + """Monkeypatch core methods for evm + """ + + ql.run = types.MethodType(__evm_run, ql) diff --git a/qiling/arch/evm/hooks.py b/qiling/arch/evm/hooks.py index f796929c..c2354c71 100644 --- a/qiling/arch/evm/hooks.py +++ b/qiling/arch/evm/hooks.py @@ -2,70 +2,72 @@ # # Cross Platform and Multi Architecture Advanced Binary Emulation Framework +import types +from enum import IntEnum +from typing import MutableMapping, MutableSequence from qiling.core_hooks_types import Hook, HookAddr, HookIntr, HookRet +class EVM_HOOK(IntEnum): + CODE = (1 << 0) + ADDR = (1 << 1) + INSN = (1 << 2) class QlArchEVMHooks: def __init__(self) -> None: - super().__init__() - self.hook_code_list = [] - self.hook_insn_list = [] - self.hook_addr_dict = {} + self.hook_code_list: MutableSequence[Hook] = [] + self.hook_insn_list: MutableSequence[HookIntr] = [] + self.hook_addr_dict: MutableMapping[int, MutableSequence[HookAddr]] = {} evm_hooks_info = QlArchEVMHooks() - -def _ql_evm_hook(ql, hook_type, h, *args): - base_type = [ - "HOOK_CODE", - "HOOK_INSN", - "HOOK_ADDR" - ] - - if hook_type in base_type: - if hook_type in ["HOOK_CODE"]: - evm_hooks_info.hook_code_list.append(h) - elif hook_type in ["HOOK_INSN"]: - evm_hooks_info.hook_insn_list.append(h) - elif hook_type in ["HOOK_ADDR"]: - address = args[0] - - if address not in evm_hooks_info.hook_addr_dict.keys(): - evm_hooks_info.hook_addr_dict[address] = [] - - evm_hooks_info.hook_addr_dict[address].append(h) - -def ql_evm_hooks(ql, hook_type, callback, user_data=None, begin=1, end=0, *args): +def __evm_hook_code(ql, callback, user_data=None, begin=1, end=0): h = Hook(callback, user_data, begin, end) - _ql_evm_hook(ql, hook_type, h, *args) - return HookRet(ql, hook_type, h) + evm_hooks_info.hook_code_list.append(h) -def evm_hook_insn(ql, hook_type, callback, intno, user_data=None, begin=1, end=0): + return HookRet(ql, EVM_HOOK.CODE, h) + +def __evm_hook_insn(ql, callback, intno, user_data=None, begin=1, end=0): h = HookIntr(callback, intno, user_data) - _ql_evm_hook(ql, hook_type, h) - return HookRet(ql, hook_type, h) + evm_hooks_info.hook_insn_list.append(h) -def evm_hook_address(ql, hook_type, h, address): - _ql_evm_hook(ql, hook_type, h, address) - return HookRet(ql, hook_type, h) + return HookRet(ql, EVM_HOOK.INSN, h) -def evm_hook_del(hook_type, h): - base_type = [ - "HOOK_CODE", - "HOOK_INSN", - "HOOK_ADDR" - ] +def __evm_hook_address(ql, callback, address, user_data=None): + h = HookAddr(callback, address, user_data) - if isinstance(h, HookAddr): - if h.addr in evm_hooks_info.hook_addr_dict.keys(): - if h in evm_hooks_info.hook_addr_dict[h.addr]: - evm_hooks_info.hook_addr_dict[h.addr].remove(h) - if len(evm_hooks_info.hook_addr_dict[h.addr]) == 0: - del evm_hooks_info.hook_addr_dict[h.addr] + if address not in evm_hooks_info.hook_addr_dict: + evm_hooks_info.hook_addr_dict[address] = [] - if hook_type in base_type: - if hook_type in ["HOOK_CODE"]: - evm_hooks_info.hook_code_list.remove(h) - elif hook_type in ["HOOK_INSN"]: - evm_hooks_info.hook_insn_list.remove(h) \ No newline at end of file + evm_hooks_info.hook_addr_dict[address].append(h) + + return HookRet(ql, EVM_HOOK.ADDR, h) + +def __evm_hook_del(ql, hret): + h = hret.obj + hook_type = hret.type + + if hook_type == EVM_HOOK.CODE: + evm_hooks_info.hook_code_list.remove(h) + + elif hook_type == EVM_HOOK.INSN: + evm_hooks_info.hook_insn_list.remove(h) + + elif hook_type == EVM_HOOK.ADDR: + if h.addr in evm_hooks_info.hook_addr_dict: + hooks_list = evm_hooks_info.hook_addr_dict[h.addr] + + if h in hooks_list: + hooks_list.remove(h) + + if not hooks_list: + del evm_hooks_info.hook_addr_dict[h.addr] + +def monkeypatch_core_hooks(ql): + """Monkeypatch core hooks for evm + """ + + ql.hook_code = types.MethodType(__evm_hook_code, ql) + ql.hook_address = types.MethodType(__evm_hook_address, ql) + ql.hook_insn = types.MethodType(__evm_hook_insn, ql) + ql.hook_del = types.MethodType(__evm_hook_del, ql) diff --git a/qiling/arch/evm/vm/debug.py b/qiling/arch/evm/vm/debug.py index bf209ba7..319aba92 100644 --- a/qiling/arch/evm/vm/debug.py +++ b/qiling/arch/evm/vm/debug.py @@ -6,7 +6,7 @@ import os, time import cmd2 from rich import print as rprint -from ..hooks import evm_hook_insn +from ..hooks import evm_hooks_info from .disassembler import EVMDisasm from ..analysis.signatures import analysis_func_sign from .utils import analysis_bytecode, bytecode_to_bytes @@ -38,7 +38,7 @@ class EVMDebugger(cmd2.Cmd): load_bytecode, runtime_code, aux_data, constructor_args = analysis_bytecode(self.executor.vm_context.msg.code) - insns = EVMDisasm().disasm(bytecode_to_bytes(runtime_code), evm_hook_insn) + insns = EVMDisasm().disasm(bytecode_to_bytes(runtime_code), evm_hooks_info) self.func_sign = analysis_func_sign(insns, engine_num=2) self.cli_output() diff --git a/qiling/arch/mips.py b/qiling/arch/mips.py index 8391becc..9368634f 100644 --- a/qiling/arch/mips.py +++ b/qiling/arch/mips.py @@ -37,7 +37,8 @@ class QlArchMIPS(QlArch): def regs(self) -> QlRegisterManager: regs_map = dict( **mips_const.reg_map, - **mips_const.reg_map_afpr128 + **mips_const.reg_map_afpr128, + **mips_const.reg_map_fpu ) pc_reg = 'pc' diff --git a/qiling/arch/mips_const.py b/qiling/arch/mips_const.py index 8dd8d48a..8e3141f4 100644 --- a/qiling/arch/mips_const.py +++ b/qiling/arch/mips_const.py @@ -6,6 +6,39 @@ from unicorn.mips_const import * reg_map = { + "r0" : UC_MIPS_REG_0, + "r1" : UC_MIPS_REG_1, + "r2" : UC_MIPS_REG_2, + "r3" : UC_MIPS_REG_3, + "r4" : UC_MIPS_REG_4, + "r5" : UC_MIPS_REG_5, + "r6" : UC_MIPS_REG_6, + "r7" : UC_MIPS_REG_7, + "r8" : UC_MIPS_REG_8, + "r9" : UC_MIPS_REG_9, + "r10" : UC_MIPS_REG_10, + "r11" : UC_MIPS_REG_11, + "r12" : UC_MIPS_REG_12, + "r13" : UC_MIPS_REG_13, + "r14" : UC_MIPS_REG_14, + "r15" : UC_MIPS_REG_15, + "r16" : UC_MIPS_REG_16, + "r17" : UC_MIPS_REG_17, + "r18" : UC_MIPS_REG_18, + "r19" : UC_MIPS_REG_19, + "r20" : UC_MIPS_REG_20, + "r21" : UC_MIPS_REG_21, + "r22" : UC_MIPS_REG_22, + "r23" : UC_MIPS_REG_23, + "r24" : UC_MIPS_REG_24, + "r25" : UC_MIPS_REG_25, + "r26" : UC_MIPS_REG_26, + "r27" : UC_MIPS_REG_27, + "r28" : UC_MIPS_REG_28, + "r29" : UC_MIPS_REG_29, + "r30" : UC_MIPS_REG_30, + "r31" : UC_MIPS_REG_31, + "zero": UC_MIPS_REG_ZERO, "at": UC_MIPS_REG_AT, "v0": UC_MIPS_REG_V0, @@ -49,4 +82,39 @@ reg_map = { reg_map_afpr128 = { "cp0_config3" : UC_MIPS_REG_CP0_CONFIG3, "cp0_userlocal": UC_MIPS_REG_CP0_USERLOCAL, -} \ No newline at end of file +} + +reg_map_fpu = { + "f0" : UC_MIPS_REG_F0, + "f1" : UC_MIPS_REG_F1, + "f2" : UC_MIPS_REG_F2, + "f3" : UC_MIPS_REG_F3, + "f4" : UC_MIPS_REG_F4, + "f5" : UC_MIPS_REG_F5, + "f6" : UC_MIPS_REG_F6, + "f7" : UC_MIPS_REG_F7, + "f8" : UC_MIPS_REG_F8, + "f9" : UC_MIPS_REG_F9, + "f10" : UC_MIPS_REG_F10, + "f11" : UC_MIPS_REG_F11, + "f12" : UC_MIPS_REG_F12, + "f13" : UC_MIPS_REG_F13, + "f14" : UC_MIPS_REG_F14, + "f15" : UC_MIPS_REG_F15, + "f16" : UC_MIPS_REG_F16, + "f17" : UC_MIPS_REG_F17, + "f18" : UC_MIPS_REG_F18, + "f19" : UC_MIPS_REG_F19, + "f20" : UC_MIPS_REG_F20, + "f21" : UC_MIPS_REG_F21, + "f22" : UC_MIPS_REG_F22, + "f23" : UC_MIPS_REG_F23, + "f24" : UC_MIPS_REG_F24, + "f25" : UC_MIPS_REG_F25, + "f26" : UC_MIPS_REG_F26, + "f27" : UC_MIPS_REG_F27, + "f28" : UC_MIPS_REG_F28, + "f29" : UC_MIPS_REG_F29, + "f30" : UC_MIPS_REG_F30, + "f31" : UC_MIPS_REG_F31 +} diff --git a/qiling/arch/ppc.py b/qiling/arch/ppc.py new file mode 100644 index 00000000..4e6d099d --- /dev/null +++ b/qiling/arch/ppc.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +from functools import cached_property + +from unicorn import Uc, UC_ARCH_PPC, UC_MODE_PPC32, UC_MODE_BIG_ENDIAN +from capstone import Cs, CS_ARCH_PPC, CS_MODE_32, CS_MODE_BIG_ENDIAN +from keystone import Ks, KS_ARCH_PPC, KS_MODE_PPC32, KS_MODE_BIG_ENDIAN + +from qiling import Qiling +from qiling.arch.arch import QlArch +from qiling.arch import ppc_const +from qiling.arch.register import QlRegisterManager +from qiling.const import QL_ARCH, QL_ENDIAN + +class QlArchPPC(QlArch): + type = QL_ARCH.PPC + bits = 32 + + @cached_property + def uc(self) -> Uc: + return Uc(UC_ARCH_PPC, UC_MODE_PPC32 + UC_MODE_BIG_ENDIAN) + + @cached_property + def regs(self) -> QlRegisterManager: + regs_map = dict( + **ppc_const.reg_map, + **ppc_const.reg_float_map + ) + + pc_reg = 'pc' + sp_reg = 'r1' + + return QlRegisterManager(self.uc, regs_map, pc_reg, sp_reg) + + @cached_property + def disassembler(self) -> Cs: + return Cs(CS_ARCH_PPC, CS_MODE_32 + CS_MODE_BIG_ENDIAN) + + @cached_property + def assembler(self) -> Ks: + return Ks(KS_ARCH_PPC, KS_MODE_PPC32 + KS_MODE_BIG_ENDIAN) + + @property + def endian(self) -> QL_ENDIAN: + return QL_ENDIAN.EB + + def enable_float(self): + self.regs.msr = self.regs.msr | ppc_const.MSR.FP diff --git a/qiling/arch/ppc_const.py b/qiling/arch/ppc_const.py new file mode 100644 index 00000000..732cba1c --- /dev/null +++ b/qiling/arch/ppc_const.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +from unicorn.ppc_const import * +from enum import IntEnum + +reg_map = { + "r0": UC_PPC_REG_0, + "r1": UC_PPC_REG_1, + "r2": UC_PPC_REG_2, + "r3": UC_PPC_REG_3, + "r4": UC_PPC_REG_4, + "r5": UC_PPC_REG_5, + "r6": UC_PPC_REG_6, + "r7": UC_PPC_REG_7, + "r8": UC_PPC_REG_8, + "r9": UC_PPC_REG_9, + "r10": UC_PPC_REG_10, + "r11": UC_PPC_REG_11, + "r12": UC_PPC_REG_12, + "r13": UC_PPC_REG_13, + "r14": UC_PPC_REG_14, + "r15": UC_PPC_REG_15, + "r16": UC_PPC_REG_16, + "r17": UC_PPC_REG_17, + "r18": UC_PPC_REG_18, + "r19": UC_PPC_REG_19, + "r20": UC_PPC_REG_20, + "r21": UC_PPC_REG_21, + "r22": UC_PPC_REG_22, + "r23": UC_PPC_REG_23, + "r24": UC_PPC_REG_24, + "r25": UC_PPC_REG_25, + "r26": UC_PPC_REG_26, + "r27": UC_PPC_REG_27, + "r28": UC_PPC_REG_28, + "r29": UC_PPC_REG_29, + "r30": UC_PPC_REG_30, + "r31": UC_PPC_REG_31, + "pc": UC_PPC_REG_PC, + "msr": UC_PPC_REG_MSR, + "cr": UC_PPC_REG_CR0, + "lr": UC_PPC_REG_LR, + "ctr": UC_PPC_REG_CTR, + "xer": UC_PPC_REG_XER, +} + +reg_float_map = { + "f0": UC_PPC_REG_FPR0, + "f1": UC_PPC_REG_FPR1, + "f2": UC_PPC_REG_FPR2, + "f3": UC_PPC_REG_FPR3, + "f4": UC_PPC_REG_FPR4, + "f5": UC_PPC_REG_FPR5, + "f6": UC_PPC_REG_FPR6, + "f7": UC_PPC_REG_FPR7, + "f8": UC_PPC_REG_FPR8, + "f9": UC_PPC_REG_FPR9, + "f10": UC_PPC_REG_FPR10, + "f11": UC_PPC_REG_FPR11, + "f12": UC_PPC_REG_FPR12, + "f13": UC_PPC_REG_FPR13, + "f14": UC_PPC_REG_FPR14, + "f15": UC_PPC_REG_FPR15, + "f16": UC_PPC_REG_FPR16, + "f17": UC_PPC_REG_FPR17, + "f18": UC_PPC_REG_FPR18, + "f19": UC_PPC_REG_FPR19, + "f20": UC_PPC_REG_FPR20, + "f21": UC_PPC_REG_FPR21, + "f22": UC_PPC_REG_FPR22, + "f23": UC_PPC_REG_FPR23, + "f24": UC_PPC_REG_FPR24, + "f25": UC_PPC_REG_FPR25, + "f26": UC_PPC_REG_FPR26, + "f27": UC_PPC_REG_FPR27, + "f28": UC_PPC_REG_FPR28, + "f29": UC_PPC_REG_FPR29, + "f30": UC_PPC_REG_FPR30, + "f31": UC_PPC_REG_FPR31, +} + +class MSR(IntEnum): + SF = 1 << 63 + TAG = 1 << 62 + ISF = 1 << 61 + HV = 1 << 60 + TS0 = 1 << 34 + TS1 = 1 << 33 + TM = 1 << 32 + CM = 1 << 31 + ICM = 1 << 30 + GS = 1 << 28 + UCLE = 1 << 26 + VR = 1 << 25 + SPE = 1 << 25 + AP = 1 << 23 + VSX = 1 << 23 + SA = 1 << 22 + KEY = 1 << 19 + POW = 1 << 18 + TGPR = 1 << 17 + CE = 1 << 17 + ILE = 1 << 16 + EE = 1 << 15 + PR = 1 << 14 + FP = 1 << 13 + ME = 1 << 12 + FE0 = 1 << 11 + SE = 1 << 10 + DWE = 1 << 10 + UBLE = 1 << 10 + BE = 1 << 9 + DE = 1 << 9 + FE1 = 1 << 8 + AL = 1 << 7 + EP = 1 << 6 + IR = 1 << 5 + DR = 1 << 4 + IS = 1 << 5 + DS = 1 << 4 + PE = 1 << 3 + PX = 1 << 2 + PMM = 1 << 2 + RI = 1 << 1 + LE = 1 << 0 diff --git a/qiling/arch/register.py b/qiling/arch/register.py index fab8f3af..368864f6 100644 --- a/qiling/arch/register.py +++ b/qiling/arch/register.py @@ -102,12 +102,6 @@ class QlRegisterManager: return self.uc.reg_write(self.uc_pc, value) - @property - def arch_pc_name(self) -> str: - """Get the architectural program counter register name. - """ - - return next(k for k, v in self.register_mapping.items() if v == self.uc_pc) @property def arch_sp(self) -> int: diff --git a/qiling/arch/utils.py b/qiling/arch/utils.py index 2764bd72..ab9583fb 100644 --- a/qiling/arch/utils.py +++ b/qiling/arch/utils.py @@ -11,8 +11,8 @@ from typing import Tuple from os.path import basename from functools import lru_cache -from keystone import (Ks, KS_ARCH_ARM, KS_ARCH_ARM64, KS_ARCH_MIPS, KS_ARCH_X86, - KS_MODE_ARM, KS_MODE_THUMB, KS_MODE_MIPS32, KS_MODE_16, KS_MODE_32, KS_MODE_64, +from keystone import (Ks, KS_ARCH_ARM, KS_ARCH_ARM64, KS_ARCH_MIPS, KS_ARCH_X86, KS_ARCH_PPC, + KS_MODE_ARM, KS_MODE_THUMB, KS_MODE_MIPS32, KS_MODE_PPC32, KS_MODE_16, KS_MODE_32, KS_MODE_64, KS_MODE_LITTLE_ENDIAN, KS_MODE_BIG_ENDIAN) from qiling import Qiling @@ -104,7 +104,8 @@ def assembler(arch: QL_ARCH, endianess: QL_ENDIAN, is_thumb: bool) -> Ks: QL_ARCH.MIPS : (KS_ARCH_MIPS, KS_MODE_MIPS32 + endian), QL_ARCH.A8086 : (KS_ARCH_X86, KS_MODE_16), QL_ARCH.X86 : (KS_ARCH_X86, KS_MODE_32), - QL_ARCH.X8664 : (KS_ARCH_X86, KS_MODE_64) + QL_ARCH.X8664 : (KS_ARCH_X86, KS_MODE_64), + QL_ARCH.PPC : (KS_ARCH_PPC, KS_MODE_PPC32 + KS_MODE_BIG_ENDIAN) } if arch in asm_map: diff --git a/qiling/arch/x86.py b/qiling/arch/x86.py index b634849f..0a787e8b 100644 --- a/qiling/arch/x86.py +++ b/qiling/arch/x86.py @@ -4,14 +4,11 @@ # from functools import cached_property -from typing import Union from unicorn import Uc, UC_ARCH_X86, UC_MODE_16, UC_MODE_32, UC_MODE_64 from capstone import Cs, CS_ARCH_X86, CS_MODE_16, CS_MODE_32, CS_MODE_64 from keystone import Ks, KS_ARCH_X86, KS_MODE_16, KS_MODE_32, KS_MODE_64 -from unicorn.x86_const import UC_X86_REG_EFLAGS - from qiling.arch.arch import QlArch from qiling.arch.msr import QlMsrManager from qiling.arch.register import QlRegisterManager @@ -112,7 +109,10 @@ class QlArchX8664(QlArchIntel): **x86_const.reg_map_64_b, **x86_const.reg_map_64_w, **x86_const.reg_map_64_d, - **x86_const.reg_map_seg_base + **x86_const.reg_map_seg_base, + **x86_const.reg_map_xmm, + **x86_const.reg_map_ymm, + **x86_const.reg_map_zmm ) pc_reg = 'rip' @@ -126,33 +126,3 @@ class QlArchX8664(QlArchIntel): @cached_property def assembler(self) -> Ks: return Ks(KS_ARCH_X86, KS_MODE_64) - - # TODO: generalize this - def __reg_bits(self, register: int) -> int: - # all regs in reg_map_misc are 16 bits except of eflags - if register == UC_X86_REG_EFLAGS: - return 32 - - regmaps = ( - (x86_const.reg_map_8, 8), - (x86_const.reg_map_16, 16), - (x86_const.reg_map_32, 32), - (x86_const.reg_map_64, 64), - (x86_const.reg_map_misc, 16), - (x86_const.reg_map_cr, 64), # 32 bits for x86 - (x86_const.reg_map_st, 32), - (x86_const.reg_map_seg_base, 64), # 32 bits for x86 - ) - - return next((rsize for rmap, rsize in regmaps if register in rmap.values()), 0) - - # note: this method was not generalized for all archs since it requires a bookkeeping - # of all registers, while it is used only by gdb and only for x86-64 - def reg_bits(self, reg: Union[str, int]) -> int: - """Get register size in bits. - """ - - if type(reg) is str: - reg = self.regs.register_mapping[reg] - - return self.__reg_bits(reg) diff --git a/qiling/arch/x86_const.py b/qiling/arch/x86_const.py index 798549ee..1e60479f 100644 --- a/qiling/arch/x86_const.py +++ b/qiling/arch/x86_const.py @@ -177,7 +177,7 @@ reg_map_st = { } reg_map_misc = { - "ef": UC_X86_REG_EFLAGS, + "eflags": UC_X86_REG_EFLAGS, "cs": UC_X86_REG_CS, "ss": UC_X86_REG_SS, "ds": UC_X86_REG_DS, diff --git a/qiling/arch/x86_utils.py b/qiling/arch/x86_utils.py index 39573c12..de6e24f2 100644 --- a/qiling/arch/x86_utils.py +++ b/qiling/arch/x86_utils.py @@ -4,7 +4,7 @@ from abc import abstractmethod from qiling import Qiling from qiling.arch.x86 import QlArchIntel from qiling.arch.x86_const import * -from qiling.exception import QlGDTError +from qiling.exception import QlGDTError, QlMemoryMappedError from qiling.os.memory import QlMemoryManager class GDTArray: @@ -50,8 +50,10 @@ class GDTManager: def __init__(self, ql: Qiling, base = QL_X86_GDT_ADDR, limit = QL_X86_GDT_LIMIT, num_entries = 16): ql.log.debug(f'Mapping GDT at {base:#x} with limit {limit:#x}') - if not ql.mem.is_mapped(base, limit): - ql.mem.map(base, limit, info="[GDT]") + if not ql.mem.is_available(base, limit): + raise QlMemoryMappedError('cannot map GDT, memory location is taken') + + ql.mem.map(base, limit, info="[GDT]") # setup GDT by writing to GDTR ql.arch.regs.write(UC_X86_REG_GDTR, (0, base, limit, 0x0)) diff --git a/qiling/cc/__init__.py b/qiling/cc/__init__.py index 351072c7..5f7216c3 100644 --- a/qiling/cc/__init__.py +++ b/qiling/cc/__init__.py @@ -2,7 +2,7 @@ # # Cross Platform and Multi Architecture Advanced Binary Emulation Framework -from typing import Callable, Tuple +from typing import Callable, Sequence, Tuple from qiling.arch.arch import QlArch @@ -26,7 +26,7 @@ class QlCC: raise NotImplementedError - def getRawParam(self, slot: int, argbits: int = None) -> int: + def getRawParam(self, slot: int, argbits: int = 0) -> int: """Read a value of native size from the specified argument slot. Note that argument slots and argument indexes are not the same. Though they often correlate @@ -41,7 +41,7 @@ class QlCC: raise NotImplementedError - def setRawParam(self, slot: int, value: int, argbits: int = None) -> None: + def setRawParam(self, slot: int, value: int, argbits: int = 0) -> None: """Replace the value in the specified argument slot. Note that argument slots and argument indexes are not the same. Though they often correlate @@ -107,19 +107,17 @@ class QlCommonBaseCC(QlCC): of the QlCC interface. """ - _argregs = () + _retreg: int + _argregs: Sequence _shadow = 0 _retaddr_on_stack = True - def __init__(self, arch: QlArch, retreg: int): + def __init__(self, arch: QlArch): super().__init__(arch) # native address size in bytes self._asize = self.arch.pointersize - # return value register - self._retreg = retreg - def __access_param(self, index: int, stack_access: Callable, reg_access: Callable) -> Tuple[Callable, int]: """[private] Generic accessor to function call parameters by their index. @@ -150,17 +148,17 @@ class QlCommonBaseCC(QlCC): else: return reg_access, reg - def getRawParam(self, index: int, argbits: int = None) -> int: + def getRawParam(self, index: int, argbits: int = 0) -> int: read, loc = self.__access_param(index, self.arch.stack_read, self.arch.regs.read) - mask = (0 if argbits is None else (1 << argbits)) - 1 + mask = (argbits and (1 << argbits)) - 1 return read(loc) & mask - def setRawParam(self, index: int, value: int, argbits: int = None) -> None: + def setRawParam(self, index: int, value: int, argbits: int = 0) -> None: write, loc = self.__access_param(index, self.arch.stack_write, self.arch.regs.write) - mask = (0 if argbits is None else (1 << argbits)) - 1 + mask = (argbits and (1 << argbits)) - 1 write(loc, value & mask) diff --git a/qiling/cc/arm.py b/qiling/cc/arm.py index 6517ee26..2457a537 100644 --- a/qiling/cc/arm.py +++ b/qiling/cc/arm.py @@ -8,7 +8,6 @@ from unicorn.arm64_const import ( UC_ARM64_REG_X4, UC_ARM64_REG_X5, UC_ARM64_REG_X6, UC_ARM64_REG_X7 ) -from qiling.arch.arch import QlArch from qiling.cc import QlCommonBaseCC class QlArmBaseCC(QlCommonBaseCC): @@ -29,13 +28,9 @@ class QlArmBaseCC(QlCommonBaseCC): return self.arch.stack_pop() class aarch64(QlArmBaseCC): + _retreg = UC_ARM64_REG_X0 _argregs = (UC_ARM64_REG_X0, UC_ARM64_REG_X1, UC_ARM64_REG_X2, UC_ARM64_REG_X3, UC_ARM64_REG_X4, UC_ARM64_REG_X5, UC_ARM64_REG_X6, UC_ARM64_REG_X7) + (None, ) * 8 - def __init__(self, arch: QlArch) -> None: - super().__init__(arch, UC_ARM64_REG_X0) - class aarch32(QlArmBaseCC): + _retreg = UC_ARM_REG_R0 _argregs = (UC_ARM_REG_R0, UC_ARM_REG_R1, UC_ARM_REG_R2, UC_ARM_REG_R3) + (None, ) * 12 - - def __init__(self, arch: QlArch) -> None: - super().__init__(arch, UC_ARM_REG_R0) diff --git a/qiling/cc/intel.py b/qiling/cc/intel.py index 4ed9f8fe..9c9aded4 100644 --- a/qiling/cc/intel.py +++ b/qiling/cc/intel.py @@ -3,12 +3,11 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework from unicorn.x86_const import ( - UC_X86_REG_AX, UC_X86_REG_EAX, UC_X86_REG_RAX, UC_X86_REG_RCX, - UC_X86_REG_RDI, UC_X86_REG_RDX, UC_X86_REG_RSI, UC_X86_REG_R8, - UC_X86_REG_R9, UC_X86_REG_R10 + UC_X86_REG_EAX, UC_X86_REG_RAX, UC_X86_REG_RCX, UC_X86_REG_RDI, + UC_X86_REG_RDX, UC_X86_REG_RSI, UC_X86_REG_R8, UC_X86_REG_R9, + UC_X86_REG_R10 ) -from qiling.arch.arch import QlArch from qiling.cc import QlCommonBaseCC class QlIntelBaseCC(QlCommonBaseCC): @@ -16,15 +15,6 @@ class QlIntelBaseCC(QlCommonBaseCC): Supports arguments passing over registers and stack. """ - def __init__(self, arch: QlArch): - retreg = { - 16: UC_X86_REG_AX, - 32: UC_X86_REG_EAX, - 64: UC_X86_REG_RAX - }[arch.bits] - - super().__init__(arch, retreg) - def setReturnAddress(self, addr: int) -> None: self.arch.stack_push(addr) @@ -36,6 +26,8 @@ class QlIntel64(QlIntelBaseCC): """Calling convention base class for Intel-based 64-bit systems. """ + _retreg = UC_X86_REG_RAX + @staticmethod def getNumSlots(argbits: int) -> int: return max(argbits, 64) // 64 @@ -44,11 +36,13 @@ class QlIntel32(QlIntelBaseCC): """Calling convention base class for Intel-based 32-bit systems. """ + _retreg = UC_X86_REG_EAX + @staticmethod def getNumSlots(argbits: int) -> int: return max(argbits, 32) // 32 - def getRawParam(self, slot: int, nbits: int = None) -> int: + def getRawParam(self, slot: int, nbits: int = 0) -> int: __super_getparam = super().getRawParam if nbits == 64: diff --git a/qiling/cc/mips.py b/qiling/cc/mips.py index 9eafc4d0..c1b3a089 100644 --- a/qiling/cc/mips.py +++ b/qiling/cc/mips.py @@ -4,17 +4,14 @@ from unicorn.mips_const import UC_MIPS_REG_V0, UC_MIPS_REG_A0, UC_MIPS_REG_A1, UC_MIPS_REG_A2, UC_MIPS_REG_A3 -from qiling.arch.arch import QlArch from qiling.cc import QlCommonBaseCC class mipso32(QlCommonBaseCC): + _retreg = UC_MIPS_REG_V0 _argregs = (UC_MIPS_REG_A0, UC_MIPS_REG_A1, UC_MIPS_REG_A2, UC_MIPS_REG_A3) + (None, ) * 12 _shadow = 4 _retaddr_on_stack = False - def __init__(self, arch: QlArch): - super().__init__(arch, UC_MIPS_REG_V0) - @staticmethod def getNumSlots(argbits: int): return 1 diff --git a/qiling/cc/ppc.py b/qiling/cc/ppc.py new file mode 100644 index 00000000..9f19e881 --- /dev/null +++ b/qiling/cc/ppc.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework + +from qiling.cc import QlCommonBaseCC + +from unicorn.ppc_const import ( + UC_PPC_REG_3, UC_PPC_REG_4, UC_PPC_REG_5, + UC_PPC_REG_6, UC_PPC_REG_7, UC_PPC_REG_8, +) + + +class ppc(QlCommonBaseCC): + """Default calling convention for PPC + First 6 arguments are passed in regs, the rest are passed on the stack. + """ + + _retreg = UC_PPC_REG_3 + _argregs = (UC_PPC_REG_3, UC_PPC_REG_4, UC_PPC_REG_5, UC_PPC_REG_6, UC_PPC_REG_7, UC_PPC_REG_8) + (None, ) * 10 + + @staticmethod + def getNumSlots(argbits: int): + return 1 diff --git a/qiling/cc/riscv.py b/qiling/cc/riscv.py index 7a834ecd..60bcf21a 100644 --- a/qiling/cc/riscv.py +++ b/qiling/cc/riscv.py @@ -2,25 +2,21 @@ # # Cross Platform and Multi Architecture Advanced Binary Emulation Framework -from qiling.arch.arch import QlArch from qiling.cc import QlCommonBaseCC from unicorn.riscv_const import ( - UC_RISCV_REG_A0, UC_RISCV_REG_A1, UC_RISCV_REG_A2, - UC_RISCV_REG_A3, UC_RISCV_REG_A4, UC_RISCV_REG_A5, + UC_RISCV_REG_A0, UC_RISCV_REG_A1, UC_RISCV_REG_A2, + UC_RISCV_REG_A3, UC_RISCV_REG_A4, UC_RISCV_REG_A5 ) - class riscv(QlCommonBaseCC): """Default calling convention for RISCV - First 6 arguments are passed in regs, the rest are passed on the stack. + First 6 arguments are passed in regs, the rest are passed on the stack. """ + _retreg = UC_RISCV_REG_A0 _argregs = (UC_RISCV_REG_A0, UC_RISCV_REG_A1, UC_RISCV_REG_A2, UC_RISCV_REG_A3, UC_RISCV_REG_A4, UC_RISCV_REG_A5) + (None, ) * 10 - def __init__(self, arch: QlArch): - super().__init__(arch, UC_RISCV_REG_A0) - @staticmethod def getNumSlots(argbits: int): return 1 \ No newline at end of file diff --git a/qiling/const.py b/qiling/const.py index 7e21ae25..d7ac9f8d 100644 --- a/qiling/const.py +++ b/qiling/const.py @@ -4,7 +4,7 @@ # from enum import Enum, Flag, IntEnum -from typing import Any, Mapping, Type +from typing import Mapping, Type, TypeVar class QL_ENDIAN(IntEnum): EL = 1 @@ -21,6 +21,7 @@ class QL_ARCH(IntEnum): CORTEX_M = 109 RISCV = 110 RISCV64 = 111 + PPC = 112 class QL_OS(IntEnum): LINUX = 201 @@ -57,39 +58,32 @@ class QL_STOP(Flag): STACK_POINTER = (1 << 0) EXIT_TRAP = (1 << 1) -QL_OS_NONPID = (QL_OS.DOS, QL_OS.UEFI) -QL_OS_POSIX = (QL_OS.LINUX, QL_OS.FREEBSD, QL_OS.MACOS, QL_OS.QNX) -QL_OS_BAREMETAL = (QL_OS.MCU,) -QL_OS_INTERPRETER = (QL_OS.EVM,) +QL_ARCH_INTERPRETER = (QL_ARCH.EVM,) + +QL_OS_POSIX = (QL_OS.LINUX, QL_OS.FREEBSD, QL_OS.MACOS, QL_OS.QNX) +QL_OS_BAREMETAL = (QL_OS.MCU,) QL_HOOK_BLOCK = 0b0001 QL_CALL_BLOCK = 0b0010 -def __reverse_enum(e: Type[Enum]) -> Mapping[str, Any]: - '''Create a reverse mapping for an enum. +T = TypeVar('T', bound=Enum) +def __casefold_enum(e: Type[T]) -> Mapping[str, T]: + '''Create a casefolded mapping of an enum to allow case-insensitive lookup. ''' - return dict((v.name.lower(), v) for v in e.__members__.values()) + return dict((k.casefold(), v) for k, v in e._member_map_.items()) -debugger_map: Mapping[str, QL_DEBUGGER] = __reverse_enum(QL_DEBUGGER) -arch_map : Mapping[str, QL_ARCH] = __reverse_enum(QL_ARCH) -os_map : Mapping[str, QL_OS] = __reverse_enum(QL_OS) -verbose_map : Mapping[str, QL_VERBOSE] = __reverse_enum(QL_VERBOSE) - -loader_map = { - QL_OS.LINUX : "ELF", - QL_OS.FREEBSD : "ELF", - QL_OS.QNX : "ELF", - QL_OS.MACOS : "MACHO", - QL_OS.WINDOWS : "PE", - QL_OS.UEFI : "PE_UEFI", - QL_OS.DOS : "DOS", - QL_OS.EVM : "EVM", - QL_OS.MCU : "MCU", - QL_OS.BLOB : "BLOB" -} +debugger_map = __casefold_enum(QL_DEBUGGER) +arch_map = __casefold_enum(QL_ARCH) +os_map = __casefold_enum(QL_OS) +verbose_map = __casefold_enum(QL_VERBOSE) arch_os_map = { QL_ARCH.EVM : QL_OS.EVM, QL_ARCH.CORTEX_M : QL_OS.MCU } + +__all__ = [ + 'QL_ENDIAN', 'QL_ARCH', 'QL_OS', 'QL_VERBOSE', 'QL_DEBUGGER', 'QL_INTERCEPT', 'QL_STOP', + 'QL_ARCH_INTERPRETER', 'QL_OS_POSIX', 'QL_OS_BAREMETAL', 'QL_HOOK_BLOCK', 'QL_CALL_BLOCK' +] diff --git a/qiling/core.py b/qiling/core.py index dd3a51c4..e5c1c9ac 100644 --- a/qiling/core.py +++ b/qiling/core.py @@ -3,51 +3,49 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from configparser import ConfigParser import os, pickle +from typing import TYPE_CHECKING, Any, AnyStr, List, Mapping, MutableMapping, Optional, Sequence, Tuple, Union # See https://stackoverflow.com/questions/39740632/python-type-hinting-without-cyclic-imports -from typing import AnyStr, List, Mapping, MutableMapping, Sequence, Union -from typing import TYPE_CHECKING - -from unicorn.unicorn import Uc - if TYPE_CHECKING: + from unicorn.unicorn import Uc + from configparser import ConfigParser + from logging import Logger from .arch.arch import QlArch from .os.os import QlOs from .os.memory import QlMemoryManager from .hw.hw import QlHwManager from .loader.loader import QlLoader -from .const import QL_ARCH, QL_ENDIAN, QL_OS, QL_STOP, QL_VERBOSE, QL_OS_INTERPRETER, QL_OS_BAREMETAL +from .const import QL_ARCH, QL_ENDIAN, QL_OS, QL_STOP, QL_VERBOSE, QL_ARCH_INTERPRETER, QL_OS_BAREMETAL from .exception import QlErrorFileNotFound, QlErrorArch, QlErrorOsType from .host import QlHost +from .log import * from .utils import * from .core_struct import QlCoreStructs from .core_hooks import QlCoreHooks -from .__version__ import __version__ # Mixin Pattern class Qiling(QlCoreHooks, QlCoreStructs): def __init__( self, argv: Sequence[str] = None, - rootfs: str = None, + rootfs: str = r'.', env: MutableMapping[AnyStr, AnyStr] = {}, code: bytes = None, ostype: Union[str, QL_OS] = None, archtype: Union[str, QL_ARCH] = None, verbose: QL_VERBOSE = QL_VERBOSE.DEFAULT, profile: str = None, - console=True, + console: bool = True, log_file=None, log_override=None, - log_plain=False, - multithread = False, + log_plain: bool = False, + multithread: bool = False, filter = None, stop: QL_STOP = QL_STOP.NONE, *, - endian: QL_ENDIAN = None, + endian: Optional[QL_ENDIAN] = None, thumb: bool = False, libcache: bool = False ): @@ -62,11 +60,8 @@ class Qiling(QlCoreHooks, QlCoreStructs): self._env = env self._code = code self._multithread = multithread - self._log_file_fd = None self._log_filter = None - self._filter = filter self._internal_exception = None - self._uc = None self._stop_options = stop ################################## @@ -105,10 +100,7 @@ class Qiling(QlCoreHooks, QlCoreStructs): ################ # rootfs setup # ################ - if rootfs is None: - rootfs = '.' - - elif not os.path.exists(rootfs): + if not os.path.exists(rootfs): raise QlErrorFileNotFound(f'Target rootfs not found: "{rootfs}"') self._rootfs = rootfs @@ -122,7 +114,7 @@ class Qiling(QlCoreHooks, QlCoreStructs): archtype = arch_convert(archtype) if type(ostype) is str: - ostype = ostype_convert(ostype) + ostype = os_convert(ostype) # if provided arch was invalid or not provided, guess arch and os if archtype is None: @@ -141,61 +133,58 @@ class Qiling(QlCoreHooks, QlCoreStructs): ostype = arch_os_convert(archtype) # arch should have been determined by now; fail if not - if archtype is None or not ql_is_valid_arch(archtype): - raise QlErrorArch(f'Uknown or unsupported architecture: "{archtype}"') + if type(archtype) is not QL_ARCH: + raise QlErrorArch(f'Unknown or unsupported architecture: "{archtype}"') # os should have been determined by now; fail if not - if ostype is None or not ql_is_valid_ostype(ostype): + if type(ostype) is not QL_OS: raise QlErrorOsType(f'Unknown or unsupported operating system: "{ostype}"') # if endianess is still undetermined, set it to little-endian. - # this setting is ignored for architectures with predfined endianess + # this setting is ignored for architectures with predefined endianess if endian is None: endian = QL_ENDIAN.EL - # make sure args were canonicalized successfully - assert type(archtype) is QL_ARCH - assert type(ostype) is QL_OS - assert type(endian) is QL_ENDIAN - - self._arch = arch_setup(archtype, endian, thumb, self) - self._ostype = ostype - - self.uc = self.arch.uc + self._arch = select_arch(archtype, endian, thumb)(self) # Once we finish setting up arch, we can init QlCoreStructs and QlCoreHooks - QlCoreStructs.__init__(self, self.arch.endian, self.arch.bits) - if not self.interpreter: + QlCoreStructs.__init__(self, self.arch.endian, self.arch.bits) QlCoreHooks.__init__(self, self.uc) ########## # Logger # ########## - self._log_file_fd, self._log_filter = ql_setup_logger(self, log_file, console, self._filter, log_override, log_plain) + self._log_file_fd = setup_logger(self, log_file, console, log_override, log_plain) + self.filter = filter self.verbose = verbose ########### # Profile # ########### - self._profile = profile_setup(self, self.ostype, profile) + self.log.debug(f'Profile: {profile or "default"}') + self._profile = profile_setup(ostype, profile) ########## # Loader # ########## - self._loader = loader_setup(self, self.ostype, libcache) + self._loader = select_loader(ostype, libcache)(self) ############## # Components # ############## if not self.interpreter: - self._mem = component_setup("os", "memory", self) - self._os = os_setup(self.ostype, self) + self._mem = select_component('os', 'memory')(self) + self._os = select_os(ostype)(self) + + if self.baremetal: + self._hw = select_component('hw', 'hw')(self) # Run the loader self.loader.run() - self._init_stop_guard() + + self._init_stop_guard() ##################### # Qiling Components # @@ -243,7 +232,7 @@ class Qiling(QlCoreHooks, QlCoreStructs): return self._os @property - def log(self) -> logging.Logger: + def log(self) -> "Logger": """ Returns the logger this Qiling instance uses. You can override this log by passing `log_override=your_log` to Qiling.__init__ @@ -271,7 +260,7 @@ class Qiling(QlCoreHooks, QlCoreStructs): return self._multithread @property - def profile(self) -> ConfigParser: + def profile(self) -> "ConfigParser": """ Program profile. See qiling/profiles/*.ql for details. Note: Please pass None or the path string to Qiling.__init__. @@ -307,25 +296,6 @@ class Qiling(QlCoreHooks, QlCoreStructs): """ return self._env - @property - def ostype(self) -> QL_OS: - """ The emulated os type. - - Note: Please pass None or one of the strings below to Qiling.__init__. - If you use shellcode, you must specify ostype and archtype manually. - - Type: int. - Values: - - "macos" : macOS. - - "darwin" : an alias to "macos". - - "freebsd" : FreeBSD - - "windows" : Windows - - "uefi" : UEFI - - "dos" : DOS - Example: Qiling(code=b"\x90", ostype="macos", archtype="x8664") - """ - return self._ostype - @property def code(self) -> bytes: """ The shellcode to execute. @@ -361,7 +331,7 @@ class Qiling(QlCoreHooks, QlCoreStructs): Type: bool """ - return self.ostype in QL_OS_INTERPRETER + return self.arch.type in QL_ARCH_INTERPRETER @property def baremetal(self) -> bool: @@ -370,7 +340,12 @@ class Qiling(QlCoreHooks, QlCoreStructs): Type: bool """ - return self.ostype in QL_OS_BAREMETAL + + # os is not initialized for interpreter archs + if self.interpreter: + return False + + return self.os.type in QL_OS_BAREMETAL @property def host(self) -> QlHost: @@ -394,8 +369,8 @@ class Qiling(QlCoreHooks, QlCoreStructs): Values: `QL_VERBOSE.DISABLED`: turn off logging `QL_VERBOSE.OFF` : mask off anything below warnings, errors and critical severity - `QL_VERBOSE.DEFAULT` : info logging level: default verbosity - `QL_VERBOSE.DEBUG` : debug logging level: higher verbosity + `QL_VERBOSE.DEFAULT` : info logging level; default verbosity + `QL_VERBOSE.DEBUG` : debug logging level; higher verbosity `QL_VERBOSE.DISASM` : debug verbosity along with disassembly trace (slow!) `QL_VERBOSE.DUMP` : disassembly trace along with cpu context dump """ @@ -405,7 +380,7 @@ class Qiling(QlCoreHooks, QlCoreStructs): def verbose(self, v: QL_VERBOSE): self._verbose = v - self.log.setLevel(ql_resolve_logger_level(v)) + self.log.setLevel(resolve_logger_level(v)) self.arch.utils.setup_output(v) @property @@ -436,8 +411,8 @@ class Qiling(QlCoreHooks, QlCoreStructs): return self._debug_stop @debug_stop.setter - def debug_stop(self, ds): - self._debug_stop = ds + def debug_stop(self, enabled: bool): + self._debug_stop = enabled @property def debugger(self) -> bool: @@ -466,28 +441,32 @@ class Qiling(QlCoreHooks, QlCoreStructs): Example: - Qiling(filter=r'^exit') - ql.filter = r'^open' """ - return self._filter + + lf = self._log_filter + + return '' if lf is None else lf._filter.pattern @filter.setter - def filter(self, ft): - self._filter = ft - if self._log_filter is None: - self._log_filter = RegexFilter(ft) - self.log.addFilter(self._log_filter) + def filter(self, regex: Optional[str]): + if regex is None: + if self._log_filter is not None: + self.log.removeFilter(self._log_filter) + else: - self._log_filter.update_filter(ft) + if self._log_filter is None: + self._log_filter = RegexFilter(regex) + + self.log.addFilter(self._log_filter) + + self._log_filter.update_filter(regex) @property - def uc(self) -> Uc: + def uc(self) -> 'Uc': """ Raw uc instance. - Type: Ucgit + Type: Uc """ - return self._uc - - @uc.setter - def uc(self, u): - self._uc = u + return self.arch.uc @property def stop_options(self) -> QL_STOP: @@ -558,20 +537,27 @@ class Qiling(QlCoreHooks, QlCoreStructs): # Qiling APIS # ############### - # Emulate the binary from begin until @end, with timeout in @timeout and - # number of emulated instructions in @count - def run(self, begin=None, end=None, timeout=0, count=0, code=None): + def run(self, begin: Optional[int] = None, end: Optional[int] = None, timeout: int = 0, count: int = 0): + """Start binary emulation. + + Args: + begin : emulation starting address + end : emulation ending address + timeout : limit emulation to a specific amount of time (microseconds); unlimited by default + count : limit emulation to a specific amount of instructions; unlimited by default + """ + # replace the original entry point, exit point, timeout and count self.entry_point = begin self.exit_point = end self.timeout = timeout - self.count = count - - if self.interpreter: - return self.arch.run(code) + self.count = count # init debugger (if set) - debugger = debugger_setup(self._debugger, self) + debugger = select_debugger(self._debugger) + + if debugger: + debugger = debugger(self) # patch binary self.do_bin_patch() diff --git a/qiling/core_hooks.py b/qiling/core_hooks.py index 7b5f85ac..0a8e4cb8 100644 --- a/qiling/core_hooks.py +++ b/qiling/core_hooks.py @@ -8,7 +8,8 @@ # handling hooks # ############################################## -from typing import Callable, MutableMapping, MutableSequence +from typing import Any, Callable, MutableMapping, MutableSequence, Protocol +from typing import TYPE_CHECKING from unicorn import Uc from unicorn.unicorn_const import * @@ -18,6 +19,73 @@ from .utils import catch_KeyboardInterrupt from .const import QL_HOOK_BLOCK from .exception import QlErrorCoreHook +if TYPE_CHECKING: + from qiling import Qiling + +class MemHookCallback(Protocol): + def __call__(self, __ql: 'Qiling', __access: int, __address: int, __size: int, __value: int, *__context: Any) -> Any: + """Memory access hook callback. + + Args: + __ql : the associated qiling instance + __access : the intercepted memory access type, one of UC_HOOK_MEM_* constants + __addr : the target memory location + __size : size of intercepted memory access + __value : the value to write, for write operations, 0 for others + __context : additional context passed on hook creation. if no context was passed, this argument should be omitted + + Returns: + an integer with `QL_HOOK_BLOCK` mask set to block execution of remaining hooks + (if any) or `None` + """ + pass + +class TraceHookCalback(Protocol): + def __call__(self, __ql: 'Qiling', __address: int, __size: int, *__context: Any) -> Any: + """Execution hook callback. + + Args: + __ql : the associated qiling instance + __address : address of the instruction to be executed + __size : instruction size + __context : additional context passed on hook creation. if no context was passed, this argument should be omitted + + Returns: + an integer with `QL_HOOK_BLOCK` mask set to block execution of remaining hooks + (if any) or `None` + """ + pass + +class AddressHookCallback(Protocol): + def __call__(self, __ql: 'Qiling', *__context: Any) -> Any: + """Address hook callback. + + Args: + __ql : the associated qiling instance + __context : additional context passed on hook creation. if no context was passed, this argument should be omitted + + Returns: + an integer with `QL_HOOK_BLOCK` mask set to block execution of remaining hooks + (if any) or `None` + """ + pass + +class InterruptHookCallback(Protocol): + def __call__(self, __ql: 'Qiling', intno: int, *__context: Any) -> Any: + """Interrupt hook callback. + + Args: + __ql : the associated qiling instance + __intno : the intercepted interrupt number + __context : additional context passed on hook creation. if no context was passed, this argument should be omitted + + Returns: + an integer with `QL_HOOK_BLOCK` mask set to block execution of remaining hooks + (if any) or `None` + """ + pass + + # Don't assume self is Qiling. class QlCoreHooks: def __init__(self, uc: Uc): @@ -37,6 +105,9 @@ class QlCoreHooks: # Callback definitions # ######################## def _hook_intr_cb(self, uc: Uc, intno: int, pack_data) -> None: + """Interrupt hooks dispatcher. + """ + ql, hook_type = pack_data handled = False @@ -60,15 +131,18 @@ class QlCoreHooks: def _hook_insn_cb(self, uc: Uc, *args): - ql, hook_type = args[-1] + """Instruction hooks dispatcher. + """ + + *hook_args, (ql, insn_type) = args retval = None - if hook_type in self._insn_hook: - hooks_list = self._insn_hook[hook_type] + if insn_type in self._insn_hook: + hooks_list = self._insn_hook[insn_type] for hook in hooks_list: if hook.bound_check(ql.arch.regs.arch_pc): - ret = hook.call(ql, *args[:-1]) + ret = hook.call(ql, *hook_args) if type(ret) is tuple: ret, retval = ret @@ -81,13 +155,16 @@ class QlCoreHooks: def _hook_trace_cb(self, uc: Uc, addr: int, size: int, pack_data) -> None: + """Code and block hooks dispatcher. + """ + ql, hook_type = pack_data if hook_type in self._hook: hooks_list = self._hook[hook_type] for hook in hooks_list: - if hook.bound_check(ql.arch.regs.arch_pc): + if hook.bound_check(addr, size): ret = hook.call(ql, addr, size) if type(ret) is int and ret & QL_HOOK_BLOCK: @@ -95,6 +172,9 @@ class QlCoreHooks: def _hook_mem_cb(self, uc: Uc, access: int, addr: int, size: int, value: int, pack_data): + """Memory access hooks dispatcher. + """ + ql, hook_type = pack_data handled = False @@ -116,6 +196,9 @@ class QlCoreHooks: def _hook_insn_invalid_cb(self, uc: Uc, pack_data) -> None: + """Invalid instruction hooks dispatcher. + """ + ql, hook_type = pack_data handled = False @@ -134,6 +217,9 @@ class QlCoreHooks: def _hook_addr_cb(self, uc: Uc, addr: int, size: int, pack_data): + """Address hooks dispatcher. + """ + ql = pack_data if addr in self._addr_hook: @@ -148,15 +234,15 @@ class QlCoreHooks: ############### # Class Hooks # ############### - def _ql_hook_internal(self, hook_type, callback, user_data=None, *args) -> int: - _callback = (catch_KeyboardInterrupt(self))(callback) - # pack user_data & callback for wrapper _callback - return self._h_uc.hook_add(hook_type, _callback, (self, user_data), 1, 0, *args) + def _ql_hook_internal(self, hook_type: int, callback: Callable, context: Any, *args) -> int: + _callback = catch_KeyboardInterrupt(self, callback) + + return self._h_uc.hook_add(hook_type, _callback, (self, context), 1, 0, *args) def _ql_hook_addr_internal(self, callback: Callable, address: int) -> int: - _callback = (catch_KeyboardInterrupt(self))(callback) - # pack user_data & callback for wrapper _callback + _callback = catch_KeyboardInterrupt(self, callback) + return self._h_uc.hook_add(UC_HOOK_CODE, _callback, self, address, address) @@ -175,7 +261,7 @@ class QlCoreHooks: ins_t = args[0] if ins_t not in self._insn_hook_fuc: - self._insn_hook_fuc[ins_t] = self._ql_hook_internal(t, self._hook_insn_cb, ins_t, *args) + self._insn_hook_fuc[ins_t] = self._ql_hook_internal(t, self._hook_insn_cb, ins_t, ins_t) if ins_t not in self._insn_hook: self._insn_hook[ins_t] = [] @@ -232,60 +318,203 @@ class QlCoreHooks: handler(t) - def ql_hook(self, hook_type: int, callback: Callable, user_data=None, begin=1, end=0, *args) -> HookRet: + def ql_hook(self, hook_type: int, callback: Callable, user_data: Any = None, begin: int = 1, end: int = 0, *args) -> HookRet: + """Intercept certain emulation events within a specified range. + + Args: + hook_type : event type to intercept; this argument is used as a bitmap and may encode multiple + events to hook with the same calback. see UC_HOOK_* constants for available events + callback : a method to call upon interception; callback signature may vary + depending on the hooked event type + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + hook = Hook(callback, user_data, begin, end) self._ql_hook(hook_type, hook, *args) return HookRet(self, hook_type, hook) - def hook_code(self, callback, user_data=None, begin=1, end=0): - if self.interpreter: - from .arch.evm.hooks import ql_evm_hooks - return ql_evm_hooks(self, 'HOOK_CODE', callback, user_data, begin, end) + def hook_code(self, callback: TraceHookCalback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept assembly instructions before they get executed. + + Args: + callback : a method to call upon interception + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ return self.ql_hook(UC_HOOK_CODE, callback, user_data, begin, end) + # TODO: remove; this is a special case of hook_intno(-1) def hook_intr(self, callback, user_data=None, begin=1, end=0): return self.ql_hook(UC_HOOK_INTR, callback, user_data, begin, end) - def hook_block(self, callback, user_data=None, begin=1, end=0): + def hook_block(self, callback: TraceHookCalback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept landings in new basic blocks in a specified range. + + Args: + callback : a method to call upon interception + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + return self.ql_hook(UC_HOOK_BLOCK, callback, user_data, begin, end) - def hook_mem_unmapped(self, callback, user_data=None, begin=1, end=0): + def hook_mem_unmapped(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept illegal accesses to unmapped memory in a specified range. + + Args: + callback : a method to call upon interception + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + return self.ql_hook(UC_HOOK_MEM_UNMAPPED, callback, user_data, begin, end) - def hook_mem_read_invalid(self, callback, user_data=None, begin=1, end=0): + def hook_mem_read_invalid(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept illegal reading attempts from a specified range. + + Args: + callback : a method to call upon interception + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + return self.ql_hook(UC_HOOK_MEM_READ_INVALID, callback, user_data, begin, end) - def hook_mem_write_invalid(self, callback, user_data=None, begin=1, end=0): + def hook_mem_write_invalid(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept illegal writing attempts to a specified range. + + Args: + callback : a method to call upon interception + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + return self.ql_hook(UC_HOOK_MEM_WRITE_INVALID, callback, user_data, begin, end) - def hook_mem_fetch_invalid(self, callback, user_data=None, begin=1, end=0): + def hook_mem_fetch_invalid(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept illegal code fetching attempts from a specified range. + + Args: + callback : a method to call upon interception + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + return self.ql_hook(UC_HOOK_MEM_FETCH_INVALID, callback, user_data, begin, end) - def hook_mem_valid(self, callback, user_data=None, begin=1, end=0): + def hook_mem_valid(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept benign memory accesses within a specified range. + This is equivalent to hooking memory reads, writes and fetches. + + Args: + callback : a method to call upon interception + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + return self.ql_hook(UC_HOOK_MEM_VALID, callback, user_data, begin, end) - def hook_mem_invalid(self, callback, user_data=None, begin=1, end=0): + def hook_mem_invalid(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept invalid memory accesses within a specified range. + This is equivalent to hooking invalid memory reads, writes and fetches. + + Args: + callback : a method to call upon interception + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + return self.ql_hook(UC_HOOK_MEM_INVALID, callback, user_data, begin, end) - # a convenient API to set callback for a single address - def hook_address(self, callback, address, user_data=None): - hook = HookAddr(callback, address, user_data) + def hook_address(self, callback: AddressHookCallback, address: int, user_data: Any = None) -> HookRet: + """Intercept execution from a certain memory address. - if self.interpreter: - from .arch.evm.hooks import evm_hook_address - return evm_hook_address(self, 'HOOK_ADDR', hook, address) + Args: + callback : a method to call upon interception + address : memory location to watch + user_data : an additional context to pass to callback (default: `None`) + + Returns: + Hook handle + """ + + hook = HookAddr(callback, address, user_data) if address not in self._addr_hook_fuc: self._addr_hook_fuc[address] = self._ql_hook_addr_internal(self._hook_addr_cb, address) @@ -295,80 +524,132 @@ class QlCoreHooks: self._addr_hook[address].append(hook) - return HookRet(self, None, hook) + # note: assuming 0 is not a valid hook type + return HookRet(self, 0, hook) - def get_hook_address(self, address): - return self._addr_hook.get(address, []) + def hook_intno(self, callback: InterruptHookCallback, intno: int, user_data: Any = None) -> HookRet: + """Intercept interrupts. + Args: + callback : a method to call upon interception + intono : interrupt vector number to intercept, or -1 for any + user_data : an additional context to pass to callback (default: `None`) + + Returns: + Hook handle + """ - def hook_intno(self, callback, intno, user_data=None): hook = HookIntr(callback, intno, user_data) self._ql_hook(UC_HOOK_INTR, hook) return HookRet(self, UC_HOOK_INTR, hook) - def hook_mem_read(self, callback, user_data=None, begin=1, end=0): + def hook_mem_read(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept benign memory reads from a specified range. + + Args: + callback : a method to call upon interception + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + return self.ql_hook(UC_HOOK_MEM_READ, callback, user_data, begin, end) - def hook_mem_write(self, callback, user_data=None, begin=1, end=0): + def hook_mem_write(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept benign memory writes to a specified range. + + Args: + callback : a method to call upon interception + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + return self.ql_hook(UC_HOOK_MEM_WRITE, callback, user_data, begin, end) - def hook_mem_fetch(self, callback, user_data=None, begin=1, end=0): + def hook_mem_fetch(self, callback: MemHookCallback, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept benign code fetches from a specified range. + + Args: + callback : a method to call upon interception + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + return self.ql_hook(UC_HOOK_MEM_FETCH, callback, user_data, begin, end) - def hook_insn(self, callback, arg1, user_data=None, begin=1, end=0): - if self.interpreter: - from .arch.evm.hooks import evm_hook_insn - return evm_hook_insn(self, 'HOOK_INSN', callback, arg1, user_data, begin, end) + def hook_insn(self, callback, insn_type: int, user_data: Any = None, begin: int = 1, end: int = 0) -> HookRet: + """Intercept execution of a certain instruction type within a specified range. - return self.ql_hook(UC_HOOK_INSN, callback, user_data, begin, end, arg1) + Args: + callback : a method to call upon interception; the callback arguments list differs + based on the instruction type + insn_type : instruction type to intercept + user_data : an additional context to pass to callback (default: `None`) + begin : start of memory range to watch + end : end of memory range to watch + + Notes: + - The set of supported instruction types is very limited and defined by unicorn. + - If `begin` and `end` are not specified, the entire memory space will be watched. + + Returns: + Hook handle + """ + + return self.ql_hook(UC_HOOK_INSN, callback, user_data, begin, end, insn_type) - def hook_del(self, *args): - if len(args) != 1 and len(args) != 2: - return + def hook_del(self, hret: HookRet) -> None: + """Unregister an existing hook and release its resources. - if isinstance(args[0], HookRet): - args[0].remove() - return + Args: + hret : hook handle + """ - hook_type, h = args + h = hret.obj + hook_type = hret.type - if self.interpreter: - from .arch.evm.hooks import evm_hook_del - return evm_hook_del(hook_type, h) + def __remove(hooks_map, handles_map, key: int) -> None: + if key in hooks_map: + hooks_list = hooks_map[key] - def __handle_common(t: int) -> None: - if t in self._hook: - if h in self._hook[t]: - del self._hook[t][self._hook[t].index(h)] + if h in hooks_list: + hooks_list.remove(h) - if len(self._hook[t]) == 0: - self._h_uc.hook_del(self._hook_fuc[t]) - del self._hook_fuc[t] + if not hooks_list: + uc_handle = handles_map.pop(key) - def __handle_insn(t: int) -> None: - if t in self._insn_hook: - if h in self._insn_hook[t]: - del self._insn_hook[t][self._insn_hook[t].index(h)] + self._h_uc.hook_del(uc_handle) - if len(self._insn_hook[t]) == 0: - self._h_uc.hook_del(self._insn_hook_fuc[t]) - del self._insn_hook_fuc[t] - - def __handle_addr(t: int) -> None: - if t in self._addr_hook: - if h in self._addr_hook[t]: - del self._addr_hook[t][self._addr_hook[t].index(h)] - - if len(self._addr_hook[t]) == 0: - self._h_uc.hook_del(self._addr_hook_fuc[t]) - del self._addr_hook_fuc[t] + __handle_common = lambda k: __remove(self._hook, self._hook_fuc, k) + __handle_insn = lambda i: __remove(self._insn_hook, self._insn_hook_fuc, i) + __handle_addr = lambda a: __remove(self._addr_hook, self._addr_hook_fuc, a) type_handlers = ( (UC_HOOK_INTR, __handle_common), diff --git a/qiling/core_hooks_types.py b/qiling/core_hooks_types.py index cd11c931..896d70a0 100644 --- a/qiling/core_hooks_types.py +++ b/qiling/core_hooks_types.py @@ -46,11 +46,11 @@ class HookIntr(Hook): class HookRet: - def __init__(self, ql, t, h): - self._ql = ql - self._t = t - self._h = h + def __init__(self, ql, hook_type: int, hook_obj: Hook): + self.type = hook_type + self.obj = hook_obj + self.__remove = ql.hook_del - def remove(self): - self._ql.hook_del(self._t, self._h) \ No newline at end of file + def remove(self) -> None: + self.__remove(self) diff --git a/qiling/core_struct.py b/qiling/core_struct.py index 47fd0cee..08c325a0 100644 --- a/qiling/core_struct.py +++ b/qiling/core_struct.py @@ -10,10 +10,13 @@ ############################################## import struct +from typing import Union from .const import QL_ENDIAN from .exception import QlErrorStructConversion +ReadableBuffer = Union[bytes, bytearray, memoryview] + # Don't assume self is Qiling. class QlCoreStructs: def __init__(self, endian: QL_ENDIAN, bit: int): @@ -30,12 +33,11 @@ class QlCoreStructs: self._fmt32s = f'{modifier}i' self._fmt64 = f'{modifier}Q' self._fmt64s = f'{modifier}q' - + handlers = { 64 : (self.pack64, self.pack64s, self.unpack64, self.unpack64s), 32 : (self.pack32, self.pack32s, self.unpack32, self.unpack32s), 16 : (self.pack16, self.pack16s, self.unpack16, self.unpack16s), - 1 : ( None, None, None, None) } if bit not in handlers: @@ -48,50 +50,50 @@ class QlCoreStructs: self.unpack = up self.unpacks = ups - def pack64(self, x): + def pack64(self, x: int, /) -> bytes: return struct.pack(self._fmt64, x) - def pack64s(self, x): + def pack64s(self, x: int, /) -> bytes: return struct.pack(self._fmt64s, x) - def unpack64(self, x): + def unpack64(self, x: ReadableBuffer, /) -> int: return struct.unpack(self._fmt64, x)[0] - def unpack64s(self, x): + def unpack64s(self, x: ReadableBuffer, /) -> int: return struct.unpack(self._fmt64s, x)[0] - def pack32(self, x): + def pack32(self, x: int, /) -> bytes: return struct.pack(self._fmt32, x) - def pack32s(self, x): + def pack32s(self, x: int, /) -> bytes: return struct.pack(self._fmt32s, x) - def unpack32(self, x): + def unpack32(self, x: ReadableBuffer, /) -> int: return struct.unpack(self._fmt32, x)[0] - def unpack32s(self, x): + def unpack32s(self, x: ReadableBuffer, /) -> int: return struct.unpack(self._fmt32s, x)[0] - def pack16(self, x): + def pack16(self, x: int, /) -> bytes: return struct.pack(self._fmt16, x) - def pack16s(self, x): + def pack16s(self, x: int, /) -> bytes: return struct.pack(self._fmt16s, x) - def unpack16(self, x): + def unpack16(self, x: ReadableBuffer, /) -> int: return struct.unpack(self._fmt16, x)[0] - def unpack16s(self, x): + def unpack16s(self, x: ReadableBuffer, /) -> int: return struct.unpack(self._fmt16s, x)[0] - def pack8(self, x): + def pack8(self, x: int, /) -> bytes: return struct.pack(self._fmt8, x) - def pack8s(self, x): + def pack8s(self, x: int, /) -> bytes: return struct.pack(self._fmt8s, x) - def unpack8(self, x): + def unpack8(self, x: ReadableBuffer, /) -> int: return struct.unpack(self._fmt8, x)[0] - def unpack8s(self, x): + def unpack8s(self, x: ReadableBuffer, /) -> int: return struct.unpack(self._fmt8s, x)[0] diff --git a/qiling/debugger/disassember.py b/qiling/debugger/disassember.py index ffcbd501..fea90563 100644 --- a/qiling/debugger/disassember.py +++ b/qiling/debugger/disassember.py @@ -17,7 +17,7 @@ class QlDisassember(): def disasm_all_lines(self): disasm_result = [] - if self.ql.ostype == QL_OS.LINUX: + if self.ql.os.type == QL_OS.LINUX: disasm_result = self.disasm_elf() return disasm_result diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index d3d5d4da..f559a2a8 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -3,47 +3,68 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -# gdbserver --remote-debug 0.0.0.0:9999 /path/to binary -# documentation: according to https://sourceware.org/gdb/current/onlinedocs/gdb/Remote-Protocol.html#Remote-Protocol +# for watching actual protocol messages: +# server: gdbserver --remote-debug 127.0.0.1:9999 /path/to/exec +# client: gdb -q -ex "target remote 127.0.0.1:9999" +# +# also, run this command on the gdb client: +# (gdb) set debug remote 1 +# +# gdb remote protocol: +# https://sourceware.org/gdb/current/onlinedocs/gdb/Remote-Protocol.html -import struct, os, socket -from binascii import unhexlify -from typing import Iterator, Literal +import os, socket, re +from logging import Logger +from typing import Iterator, Optional, Union + +from unicorn import UcError +from unicorn.unicorn_const import ( + UC_ERR_READ_UNMAPPED, UC_ERR_WRITE_UNMAPPED, UC_ERR_FETCH_UNMAPPED, + UC_ERR_READ_PROT, UC_ERR_WRITE_PROT, UC_ERR_FETCH_PROT, + UC_ERR_READ_UNALIGNED, UC_ERR_WRITE_UNALIGNED, UC_ERR_FETCH_UNALIGNED, + UC_ERR_INSN_INVALID +) from qiling import Qiling -from qiling.const import * -from qiling.utils import * +from qiling.const import QL_ARCH, QL_ENDIAN, QL_OS from qiling.debugger import QlDebugger -from qiling.arch.x86_const import reg_map_16 as x86_reg_map_16 -from qiling.arch.x86_const import reg_map_32 as x86_reg_map_32 -from qiling.arch.x86_const import reg_map_64 as x86_reg_map_64 -from qiling.arch.x86_const import reg_map_misc as x86_reg_map_misc -from qiling.arch.x86_const import reg_map_st as x86_reg_map_st -from qiling.arch.arm_const import reg_map as arm_reg_map -from qiling.arch.arm64_const import reg_map as arm64_reg_map -from qiling.arch.mips_const import reg_map as mips_reg_map -from qiling.loader.elf import AUX +from qiling.debugger.gdb.xmlregs import QlGdbFeatures +from qiling.debugger.gdb.utils import QlGdbUtils -from .utils import QlGdbUtils +# gdb logging prompt +PROMPT = r'gdb>' -GDB_SIGNAL_INT = 2 -GDB_SIGNAL_SEGV = 11 -GDB_SIGNAL_GILL = 4 -GDB_SIGNAL_STOP = 17 -GDB_SIGNAL_TRAP = 5 -GDB_SIGNAL_BUS = 10 +# default string encoding +ENCODING = 'latin' +# define a few handy linux signals +SIGINT = 2 +SIGILL = 4 +SIGTRAP = 5 +SIGABRT = 6 +SIGBUS = 7 +SIGKILL = 9 +SIGSEGV = 11 +SIGALRM = 14 +SIGTERM = 15 +SIGCHLD = 16 +SIGCONT = 17 +SIGSTOP = 18 + +# common replies +REPLY_ACK = b'+' +REPLY_EMPTY = b'' +REPLY_OK = b'OK' + +# reply type +Reply = Union[bytes, str] + +class QlGdb(QlDebugger): + """A simple gdbserver implementation. + """ -class QlGdb(QlDebugger, object): - """docstring for Debugsession""" def __init__(self, ql: Qiling, ip: str = '127.0.01', port: int = 9999): - super(QlGdb, self).__init__(ql) - - self.ql = ql - self.last_pkt = None - self.exe_abspath = os.path.abspath(self.ql.argv[0]) - self.rootfs_abspath = os.path.abspath(self.ql.rootfs) - self.gdb = QlGdbUtils() + super().__init__(ql) if type(port) is str: port = int(port, 0) @@ -51,763 +72,895 @@ class QlGdb(QlDebugger, object): self.ip = ip self.port = port - if self.ql.baremetal: - load_address = self.ql.loader.load_address + if ql.baremetal: + load_address = ql.loader.load_address exit_point = load_address + os.path.getsize(ql.path) - elif self.ql.code: - load_address = self.ql.os.entry_point + elif ql.code: + load_address = ql.os.entry_point exit_point = load_address + len(ql.code) else: load_address = ql.loader.load_address exit_point = load_address + os.path.getsize(ql.path) - if self.ql.baremetal: - self.entry_point = self.ql.loader.entry_point - elif self.ql.ostype in (QL_OS.LINUX, QL_OS.FREEBSD) and not self.ql.code: - self.entry_point = self.ql.os.elf_entry + if ql.baremetal: + entry_point = ql.loader.entry_point + elif ql.os.type in (QL_OS.LINUX, QL_OS.FREEBSD) and not ql.code: + entry_point = ql.os.elf_entry else: - self.entry_point = self.ql.os.entry_point + entry_point = ql.os.entry_point # Only part of the binary file will be debugged. - if self.ql.entry_point is not None and self.ql.exit_point is not None: - self.entry_point = self.ql.entry_point - exit_point = self.ql.exit_point + if ql.entry_point is not None: + entry_point = ql.entry_point - self.gdb.initialize(self.ql, self.entry_point, exit_point=exit_point, mappings=[(hex(load_address))]) + if ql.exit_point is not None: + exit_point = ql.exit_point - #Setup register tables, order of tables is important - self.tables = { - QL_ARCH.A8086 : list({**x86_reg_map_16, **x86_reg_map_misc}.keys()), - QL_ARCH.X86 : list({**x86_reg_map_32, **x86_reg_map_misc, **x86_reg_map_st}.keys()), - QL_ARCH.X8664 : list({**x86_reg_map_64, **x86_reg_map_misc, **x86_reg_map_st}.keys()), - QL_ARCH.ARM : list({**arm_reg_map}.keys()), - QL_ARCH.CORTEX_M : list({**arm_reg_map}.keys()), - QL_ARCH.ARM64 : list({**arm64_reg_map}.keys()), - QL_ARCH.MIPS : list({**mips_reg_map}.keys()), + self.gdb = QlGdbUtils(ql, entry_point, exit_point) + + self.features = QlGdbFeatures(self.ql.arch.type, self.ql.os.type) + self.regsmap = self.features.regsmap + + def run(self): + server = GdbSerialConn(self.ip, self.port, self.ql.log) + killed = False + + def __hexstr(value: int, nibbles: int = 0) -> str: + """Encode a value into a hex string. + """ + + length = (nibbles or self.ql.arch.bits // 4) // 2 + byteorder = 'little' if self.ql.arch.endian == QL_ENDIAN.EL else 'big' + + return value.to_bytes(length, byteorder).hex() + + def __unkown_reg_value(nibbles: int) -> str: + """Encode the hex string for unknown regsiter value. + """ + + return 'x' * nibbles + + def __get_reg_value(reg: Optional[int], pos: int, nibbles: int) -> str: + # reg is either None or uc reg invalid + if reg: + value = self.ql.arch.regs.read(reg) + assert type(value) is int + + hexstr = __hexstr(value, nibbles) + else: + hexstr = __unkown_reg_value(nibbles) + + return hexstr + + def __set_reg_value(reg: Optional[int], pos: int, nibbles: int, hexval: str) -> None: + # reg is neither None nor uc reg invalid + if reg and hexval != __unkown_reg_value(nibbles): + assert len(hexval) == nibbles + + val = int(hexval, 16) + + if self.ql.arch.endian == QL_ENDIAN.EL: + val = __swap_endianess(val) + + self.ql.arch.regs.write(reg, val) + + def __swap_endianess(value: int) -> int: + length = (value.bit_length() + 7) // 8 + raw = value.to_bytes(length, 'little') + + return int.from_bytes(raw, 'big') + + + def handle_exclaim(subcmd: str) -> Reply: + return REPLY_OK + + + def handle_qmark(subcmd: str) -> Reply: + """Request status. + + @see: https://sourceware.org/gdb/current/onlinedocs/gdb/Stop-Reply-Packets.html + """ + + from unicorn.x86_const import UC_X86_REG_EBP + from unicorn.x86_const import UC_X86_REG_RBP + from unicorn.arm_const import UC_ARM_REG_R11 + from unicorn.arm64_const import UC_ARM64_REG_X29 + from unicorn.mips_const import UC_MIPS_REG_INVALID + + arch_uc_bp = { + QL_ARCH.X86 : UC_X86_REG_EBP, + QL_ARCH.X8664 : UC_X86_REG_RBP, + QL_ARCH.ARM : UC_ARM_REG_R11, + QL_ARCH.ARM64 : UC_ARM64_REG_X29, + QL_ARCH.MIPS : UC_MIPS_REG_INVALID, # skipped + QL_ARCH.A8086 : UC_X86_REG_EBP, + QL_ARCH.CORTEX_M : UC_ARM_REG_R11 + }[self.ql.arch.type] + + def __get_reg_idx(ucreg: int) -> int: + """Get the index of a uc reg whithin the regsmap array. + + Returns: array index where this reg's info is stored, or -1 if not found + """ + + return next((i for i, (regnum, _, _) in enumerate(self.regsmap) if regnum == ucreg), -1) + + def __get_reg_info(ucreg: int) -> str: + """Retrieve register info and pack it as a pair. + """ + + regnum = __get_reg_idx(ucreg) + hexval = __get_reg_value(*self.regsmap[regnum]) + + return f'{regnum:02x}:{hexval};' + + # mips targets skip this reg info pair + bp_info = '' if self.ql.arch.type == QL_ARCH.MIPS else __get_reg_info(arch_uc_bp) + + # FIXME: a8086 should use 'esp' and 'eip' here instead of 'sp' and 'ip' set by its arch instance + sp_info = __get_reg_info(self.ql.arch.regs.uc_sp) + pc_info = __get_reg_info(self.ql.arch.regs.uc_pc) + + return f'T{SIGTRAP:02x}{bp_info}{sp_info}{pc_info}' + + + def handle_c(subcmd: str) -> Reply: + try: + self.gdb.resume_emu() + except UcError as err: + sigmap = { + UC_ERR_READ_UNMAPPED : SIGSEGV, + UC_ERR_WRITE_UNMAPPED : SIGSEGV, + UC_ERR_FETCH_UNMAPPED : SIGSEGV, + UC_ERR_WRITE_PROT : SIGSEGV, + UC_ERR_READ_PROT : SIGSEGV, + UC_ERR_FETCH_PROT : SIGSEGV, + UC_ERR_READ_UNALIGNED : SIGBUS, + UC_ERR_WRITE_UNALIGNED : SIGBUS, + UC_ERR_FETCH_UNALIGNED : SIGBUS, + UC_ERR_INSN_INVALID : SIGILL + } + + # determine signal from uc error; default to SIGTERM + reply = f'S{sigmap.get(err.errno, SIGTERM):02x}' + + except KeyboardInterrupt: + # emulation was interrupted with ctrl+c + reply = f'S{SIGINT:02x}' + + else: + if self.ql.arch.regs.arch_pc == self.gdb.last_bp: + # emulation stopped because it hit a breakpoint + reply = f'S{SIGTRAP:02x}' + else: + # emulation has completed successfully + reply = f'W{self.ql.os.exit_code:02x}' + + return reply + + + def handle_g(subcmd: str) -> Reply: + # NOTE: in the past the 'g' reply packet for arm included the f0-f7 and fps registers between pc + # and cpsr, which placed cpsr at index (regnum) 25. as the f-registers became obsolete the cpsr + # index decreased. in order to maintain backward compatibility with older gdb versions, the gap + # between pc and cpsr that used to represent the f-registers (96 bits each + 32 bits for fps) is + # filled with unknown reg values. + # + # gdb clients that follow the xml definitions no longer need these placeholders, as registers + # indices are flexible and may be defined arbitrarily though xml. + # + # see: ./xml/arm/arm-fpa.xml + + return ''.join(__get_reg_value(*entry) for entry in self.regsmap) + + + def handle_G(subcmd: str) -> Reply: + data = subcmd + + for reg, pos, nibbles in self.regsmap: + hexval = data[pos : pos + nibbles] + + __set_reg_value(reg, pos, nibbles, hexval) + + return REPLY_OK + + + def handle_H(subcmd: str) -> Reply: + op = subcmd[0] + + if op in ('c', 'g'): + return REPLY_OK + + return REPLY_EMPTY + + + def handle_k(subcmd: str) -> Reply: + global killed + + killed = True + return REPLY_OK + + + def handle_m(subcmd: str) -> Reply: + """Read target memory. + """ + + addr, size = (int(p, 16) for p in subcmd.split(',')) + + try: + data = self.ql.mem.read(addr, size).hex() + except UcError: + return 'E14' + else: + return data + + + def handle_M(subcmd: str) -> Reply: + """Write target memory. + """ + + addr, data = subcmd.split(',') + size, data = data.split(':') + + addr = int(addr, 16) + data = bytes.fromhex(data) + + assert len(data) == size + + try: + self.ql.mem.write(addr, data) + except UcError: + return 'E01' + else: + return REPLY_OK + + + def handle_p(subcmd: str) -> Reply: + """Read register value by index. + """ + + idx = int(subcmd, 16) + + return __get_reg_value(*self.regsmap[idx]) + + + def handle_P(subcmd: str) -> Reply: + """Write register value by index. + """ + + idx, data = subcmd.split('=') + idx = int(idx, 16) + + if idx < len(self.regsmap): + __set_reg_value(*self.regsmap[idx], hexval=data) + + return REPLY_OK + + return 'E00' + + + def handle_Q(subcmd: str) -> Reply: + """General queries. + + @see: https://sourceware.org/gdb/onlinedocs/gdb/General-Query-Packets.html + """ + + feature, *data = subcmd.split(':', maxsplit=1) + + supported = ( + 'DisableRandomization', + 'NonStop', + 'PassSignals', + 'ProgramSignals', + 'StartNoAckMode' + ) + + if feature == 'StartNoAckMode': + server.ack_mode = False + server.log.debug('[noack mode enabled]') + + return REPLY_OK if feature in supported else REPLY_EMPTY + + + def handle_D(subcmd: str) -> Reply: + """Detach. + """ + + return REPLY_OK + + + def handle_q(subcmd: str) -> Reply: + query, *data = subcmd.split(':') + + # qSupported command + # + # @see: https://sourceware.org/gdb/onlinedocs/gdb/General-Query-Packets.html#qSupported + + if query == 'Supported': + # list of supported features excluding the multithreading-related ones + common = ( + 'BreakpointCommands+', + 'ConditionalBreakpoints+', + 'ConditionalTracepoints+', + 'DisconnectedTracing+', + 'EnableDisableTracepoints+', + 'InstallInTrace+', + 'QAgent+', + 'QCatchSyscalls+', + 'QDisableRandomization+', + 'QEnvironmentHexEncoded+', + 'QEnvironmentReset+', + 'QEnvironmentUnset+', + 'QNonStop+', + 'QPassSignals+', + 'QProgramSignals+', + 'QSetWorkingDir+', + 'QStartNoAckMode+', + 'QStartupWithShell+', + 'QTBuffer:size+', + 'StaticTracepoints+', + 'TraceStateVariables+', + 'TracepointSource+', + # 'augmented-libraries-svr4-read+', + 'exec-events+', + 'fork-events+', + 'hwbreak+', + 'multiprocess+', + 'no-resumed+', + 'qXfer:auxv:read+', + 'qXfer:exec-file:read+', + 'qXfer:features:read+', + # 'qXfer:libraries-svr4:read+', + # 'qXfer:osdata:read+', + 'qXfer:siginfo:read+', + 'qXfer:siginfo:write+', + 'qXfer:statictrace:read+', + 'qXfer:threads:read+', + 'qXfer:traceframe-info:read+', + 'swbreak+', + 'tracenz+', + 'vfork-events+' + ) + + # might or might not need for multi thread + if self.ql.multithread: + features = ( + 'PacketSize=47ff', + 'FastTracepoints+', + 'QThreadEvents+', + 'Qbtrace-conf:bts:size+', + 'Qbtrace-conf:pt:size+', + 'Qbtrace:bts+', + 'Qbtrace:off+', + 'Qbtrace:pt+', + 'qXfer:btrace-conf:read+', + 'qXfer:btrace:read+', + 'vContSupported+' + ) + + else: + features = ( + 'PacketSize=3fff', + 'qXfer:spu:read+', + 'qXfer:spu:write+' + ) + + return ';'.join(common + features) + + elif query == 'Xfer': + feature, op, annex, params = data + offset, length = (int(p, 16) for p in params.split(',')) + + if feature == 'features' and op == 'read': + if annex == r'target.xml': + content = self.features.tostring()[offset:offset + length] + + else: + self.ql.log.info(f'{PROMPT} did not expect "{annex}" here') + content = '' + + return f'{"l" if len(content) < length else "m"}{content}' + + elif feature == 'threads' and op == 'read': + if not self.ql.baremetal and hasattr(self.ql.os, 'pid'): + content = '\r\n'.join(( + '', + f'', + '' + )) + + else: + content = '' + + return f'l{content}' + + elif feature == 'auxv' and op == 'read': + auxv_data = bytearray() + + if hasattr(self.ql.loader, 'auxv'): + nbytes = self.ql.arch.bits // 8 + + auxv_addr = self.ql.loader.auxv + offset + null_entry = bytes(nbytes * 2) + + # keep reading until AUXV.AT_NULL is reached + while not auxv_data.endswith(null_entry): + auxv_data.extend(self.ql.mem.read(auxv_addr, nbytes)) + auxv_addr += nbytes + + auxv_data.extend(self.ql.mem.read(auxv_addr, nbytes)) + auxv_addr += nbytes + + return b'l' + auxv_data[:length] + + elif feature == 'exec-file' and op == 'read': + return f'l{os.path.abspath(self.ql.path)}' + + elif feature == 'libraries-svr4' and op == 'read': + # TODO: this one requires information of loaded libraries which currently not provided + # by the ELF loader. until we gather that information, we cannot fulfill this request + # + # see: https://sourceware.org/gdb/current/onlinedocs/gdb/Library-List-Format-for-SVR4-Targets.html + return REPLY_EMPTY + + # if self.ql.os.type in (QL_OS.LINUX, QL_OS.FREEBSD): + # tag = 'library-list-svr4' + # xml_lib_entries = (f'' for lbnd, ubnd, _, _, path in self.ql.mem.get_mapinfo() if path) + # + # xml = '\r\n'.join((f'<{tag} version="1.0">', *xml_lib_entries, f'')) + # + # return f'l{xml}' + # else: + # return f'' + + elif feature == 'btrace-conf' and op == 'read': + return 'E.Btrace not enabled.' + + elif query == 'Attached': + return REPLY_EMPTY + + elif query == 'C': + return REPLY_EMPTY + + elif query == 'L': + return 'M001' + + elif query == 'fThreadInfo': + return 'm0' + + elif query == 'sThreadInfo': + return 'l' + + elif query == 'TStatus': + tsize = __hexstr(0x500000) + + fields = ( + 'T0', + 'tnotrun:0', + 'tframes:0', + 'tcreated:0', + f'tfree:{tsize}', + f'tsize:{tsize}', + 'circular:0', + 'disconn:0', + 'starttime:0', + 'stoptime:0', + 'username:', + 'notes::' + ) + + return ';'.join(fields) + + elif query in ('TfV', 'TsV', 'TfP', 'TsP'): + return 'l' + + elif query == 'Symbol': + return REPLY_OK + + elif query == 'Offsets': + fields = ('Text=0', 'Data=0', 'Bss=0') + + return ';'.join(fields) + + return REPLY_EMPTY + + + def handle_v(subcmd: str) -> Reply: + if subcmd == 'MustReplyEmpty': + return REPLY_EMPTY + + elif subcmd.startswith('File'): + _, op, data = subcmd.split(':', maxsplit=2) + params = data.split(',') + + if op == 'open': + fd = -1 + + # files can be opened only where there is an os that supports filesystem + if not self.ql.interpreter and hasattr(self.ql.os, 'path'): + path, flags, mode = params + + path = bytes.fromhex(path).decode(encoding='utf-8') + flags = int(flags, 16) + mode = int(mode, 16) + + # try to guess whether this is an emulated path or real one + if path.startswith(os.path.abspath(self.ql.rootfs)): + host_path = path + else: + host_path = self.ql.os.path.virtual_to_host_path(path) + + self.ql.log.debug(f'{PROMPT} target file: {host_path}') + + if os.path.exists(host_path) and not path.startswith(r'/proc'): + fd = os.open(host_path, flags, mode) + + return f'F{fd:x}' + + elif op == 'pread': + fd, count, offset = (int(p, 16) for p in params) + + data = os.pread(fd, count, offset) + + return f'F{len(data):x};'.encode() + data + + elif op == 'close': + fd, *_ = params + fd = int(fd, 16) + + os.close(fd) + return 'F0' + + return REPLY_EMPTY + + elif subcmd.startswith('Kill'): + return handle_k('') + + elif subcmd.startswith('Cont'): + # remove 'Cont' prefix + data = subcmd[len('Cont'):] + + if data == '?': + # note 't' and 'r' are currently not supported + return ';'.join(('vCont', 'c', 'C', 's', 'S')) + + elif data.startswith(';'): + groups = subcmd.split(';')[1:] + + for grp in groups: + cmd, tid = grp.split(':', maxsplit=1) + + if cmd in ('c', f'C{SIGTRAP:02x}'): + return handle_c('') + + elif cmd in ('s', f'S{SIGTRAP:02x}'): + return handle_s('') + + # FIXME: not sure how to handle multiple command + # groups, so handling just the first one + break + + return REPLY_EMPTY + + + def handle_s(subcmd: str) -> Reply: + """Perform a single step. + """ + + # BUG: a known unicorn caching issue causes it to emulate more + # steps than requestes. until that issue is fixed, single stepping + # is essentially broken. + # + # @see: https://github.com/unicorn-engine/unicorn/issues/1606 + + self.gdb.resume_emu(steps=1) + + return f'S{SIGTRAP:02x}' + + + def handle_X(subcmd: str) -> Reply: + """Write data to memory. + """ + + params, data = subcmd.split(':', maxsplit=1) + addr, length = (int(p, 16) for p in params.split(',')) + + if length != len(data): + return 'E00' + + try: + if data: + self.ql.mem.write(addr, data.encode(ENCODING)) + except UcError: + return 'E01' + else: + return REPLY_OK + + + def handle_Z(subcmd: str) -> Reply: + """Insert breakpoints or watchpoints. + """ + + params, *conds = subcmd.split(';') + type, addr, kind = (int(p, 16) for p in params.split(',')) + + # type values: + # 0 = sw breakpoint + # 1 = hw breakpoint + # 2 = write watchpoint + # 3 = read watchpoint + # 4 = access watchpoint + + if type == 0: + self.gdb.bp_insert(addr) + return REPLY_OK + + return REPLY_EMPTY + + + def handle_z(subcmd: str) -> Reply: + """Remove breakpoints or watchpoints. + """ + + type, addr, kind = (int(p, 16) for p in subcmd.split(',')) + + if type == 0: + try: + self.gdb.bp_remove(addr) + except ValueError: + return 'E22' + else: + return REPLY_OK + + return REPLY_EMPTY + + + handlers = { + '!': handle_exclaim, + '?': handle_qmark, + 'c': handle_c, + 'C': handle_c, # this is intentional; not a typo + 'D': handle_D, + 'g': handle_g, + 'G': handle_G, + 'H': handle_H, + 'k': handle_k, + 'm': handle_m, + 'M': handle_M, + 'p': handle_p, + 'P': handle_P, + 'q': handle_q, + 'Q': handle_Q, + 's': handle_s, + 'v': handle_v, + 'X': handle_X, + 'Z': handle_Z, + 'z': handle_z } - def addr_to_str(self, addr: int, short: bool = False, endian: Literal['little', 'big'] = 'big') -> str: - # a hacky way to divide archbits by 2 if short, and leave it unchanged if not - nbits = self.ql.arch.bits // (int(short) + 1) + # main server loop + for packet in server.readpackets(): + if server.ack_mode: + server.send(REPLY_ACK, raw=True) + server.log.debug('[sent ack]') - if nbits == 64: - s = f'{int.from_bytes(self.ql.pack64(addr), byteorder=endian):016x}' + cmd, subcmd = packet[0], packet[1:] + handler = handlers.get(f'{cmd:c}') - elif nbits == 32: - s = f'{int.from_bytes(self.ql.pack32(addr), byteorder=endian):08x}' + if handler: + reply = handler(subcmd.decode(ENCODING)) + server.send(reply) - elif nbits == 16: - s = f'{int.from_bytes(self.ql.pack16(addr), byteorder=endian):04x}' - - else: - raise RuntimeError - - return s - - def bin_to_escstr(self, rawbin): - rawbin_escape = "" - - def incomplete_hex_check(hexchar): - if len(hexchar) == 1: - hexchar = "0" + hexchar - return hexchar - - for a in rawbin: - - # The binary data representation uses 7d (ASCII ‘}’) as an escape character. - # Any escaped byte is transmitted as the escape character followed by the original character XORed with 0x20. - # For example, the byte 0x7d would be transmitted as the two bytes 0x7d 0x5d. The bytes 0x23 (ASCII ‘#’), 0x24 (ASCII ‘$’), and 0x7d (ASCII ‘}’) - # must always be escaped. Responses sent by the stub must also escape 0x2a (ASCII ‘*’), - # so that it is not interpreted as the start of a run-length encoded sequence (described next). - - if a in (42,35,36, 125): - a = a ^ 0x20 - a = (str(hex(a)[2:])) - a = incomplete_hex_check(a) - a = str("7d%s" % a) + if killed: + break else: - a = (str(hex(a)[2:])) - a = incomplete_hex_check(a) + self.ql.log.info(f'{PROMPT} command not supported') + server.send(REPLY_EMPTY) - rawbin_escape += a + server.close() - return unhexlify(rawbin_escape) - def setup_server(self): - self.ql.log.info("gdb> Listening on %s:%u" % (self.ip, self.port)) +class GdbSerialConn: + """Serial connection handler. + """ + + # default recieve buffer size + BUFSIZE = 4096 + + def __init__(self, ipaddr: str, port: int, logger: Logger) -> None: + """Create a new gdb serial connection handler. + + Args: + ipaddr : ip address to bind the socket to + port : port number to listen on + logger : logger instance to use + """ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.bind((self.ip, self.port)) - sock.listen(1) - clientsocket, addr = sock.accept() + sock.bind((ipaddr, port)) + sock.listen() - self.sock = sock - self.clientsocket = clientsocket - self.netin = clientsocket.makefile('r') - self.netout = clientsocket.makefile('w') + self.log = logger + self.log.info(f'{PROMPT} listening on {ipaddr}:{port:d}') + + client, _ = sock.accept() + + self.sock = sock + self.client = client + + # ack mode should be turend on by default + self.ack_mode = True def close(self): - self.netin.close() - self.netout.close() - self.clientsocket.close() + """Close the gdb serial connection handler and release its resources. + """ + + self.client.close() self.sock.close() - def run(self): - self.setup_server() - - while self.receive() == 'Good': - pkt = self.last_pkt - self.send_raw('+') - - def handle_qmark(subcmd): - def gdbqmark_converter(arch): - """ - MIPS32_EL : gdbserver response ("$T051d:00e7ff7f;25:40ccfc77;#65") - MIPS32_EB : gdbserver response ("$T051d:7fff6dc0;25:77fc4880;thread:28fa;core:0;"); - ARM64: gdbserver response "$T051d:0*,;1f:80f6f*"ff0* ;20:c02cfdb7f* 0* ;thread:p1f9.1f9;core:0;#56"); - ARM: gdbserver $T050b:0*"00;0d:e0f6ffbe;0f:8079fdb6;#ae" - """ - adapter = { - QL_ARCH.A8086 : [ 0x05, 0x04, 0x08 ], - QL_ARCH.X86 : [ 0x05, 0x04, 0x08 ], - QL_ARCH.X8664 : [ 0x06, 0x07, 0x10 ], - QL_ARCH.MIPS : [ 0x1d, 0x00, 0x25 ], - QL_ARCH.ARM : [ 0x0b, 0x0d, 0x0f ], - QL_ARCH.CORTEX_M : [ 0x0b, 0x0d, 0x0f ], - QL_ARCH.ARM64 : [ 0x1d, 0xf1, 0x20 ] - } - return adapter.get(arch) - - idhex, spid, pcid = gdbqmark_converter(self.ql.arch.type) - sp = self.addr_to_str(self.ql.arch.regs.arch_sp) - pc = self.addr_to_str(self.ql.arch.regs.arch_pc) - nullfill = "0" * int(self.ql.arch.bits / 4) - - if self.ql.arch.type == QL_ARCH.MIPS: - self.send('T%.2x%.2x:%s;%.2x:%s;' %(GDB_SIGNAL_TRAP, idhex, sp, pcid, pc)) - else: - self.send('T%.2x%.2x:%s;%.2x:%s;%.2x:%s;' %(GDB_SIGNAL_TRAP, idhex, nullfill, spid, sp, pcid, pc)) - - - def handle_c(subcmd): - self.gdb.resume_emu(self.ql.arch.regs.arch_pc) - - if self.gdb.bp_list == [self.entry_point]: - self.send("W00") - else: - self.send(('S%.2x' % GDB_SIGNAL_TRAP)) - - - handle_C = handle_c - - - def handle_g(subcmd): - s = '' - - if self.ql.arch.type == QL_ARCH.A8086: - for reg in self.tables[QL_ARCH.A8086][:16]: - r = self.ql.arch.regs.read(reg) - tmp = self.addr_to_str(r) - s += tmp - - elif self.ql.arch.type == QL_ARCH.X86: - for reg in self.tables[QL_ARCH.X86][:16]: - r = self.ql.arch.regs.read(reg) - tmp = self.addr_to_str(r) - s += tmp - - elif self.ql.arch.type == QL_ARCH.X8664: - for reg in self.tables[QL_ARCH.X8664][:24]: - r = self.ql.arch.regs.read(reg) - if self.ql.arch.reg_bits(reg) == 64: - tmp = self.addr_to_str(r) - elif self.ql.arch.reg_bits(reg) == 32: - tmp = self.addr_to_str(r, short = True) - s += tmp - - elif self.ql.arch.type == QL_ARCH.ARM: - - - # r0-r12,sp,lr,pc,cpsr ,see https://sourceware.org/git/?p=binutils-gdb.git;a=blob;f=gdb/arch/arm.h;h=fa589fd0582c0add627a068e6f4947a909c45e86;hb=HEAD#l127 - for reg in self.tables[QL_ARCH.ARM][:16] + [self.tables[QL_ARCH.ARM][25]]: - # if reg is pc, make sure to take thumb mode into account - r = self.ql.arch.effective_pc if reg == "pc" else self.ql.arch.regs.read(reg) - - tmp = self.addr_to_str(r) - s += tmp - - elif self.ql.arch.type == QL_ARCH.ARM64: - for reg in self.tables[QL_ARCH.ARM64][:33]: - r = self.ql.arch.regs.read(reg) - tmp = self.addr_to_str(r) - s += tmp - - elif self.ql.arch.type == QL_ARCH.MIPS: - for reg in self.tables[QL_ARCH.MIPS][:38]: - r = self.ql.arch.regs.read(reg) - tmp = self.addr_to_str(r) - s += tmp - - self.send(s) - - - def handle_G(subcmd): - count = 0 - - if self.ql.arch.type == QL_ARCH.A8086: - for i in range(0, len(subcmd), 8): - reg_data = subcmd[i:i+7] - reg_data = int(reg_data, 16) - self.ql.arch.regs.write(self.tables[QL_ARCH.A8086][count], reg_data) - count += 1 - - elif self.ql.arch.type == QL_ARCH.X86: - for i in range(0, len(subcmd), 8): - reg_data = subcmd[i:i+7] - reg_data = int(reg_data, 16) - self.ql.arch.regs.write(self.tables[QL_ARCH.X86][count], reg_data) - count += 1 - - - elif self.ql.arch.type == QL_ARCH.X8664: - for i in range(0, 17*16, 16): - reg_data = subcmd[i:i+15] - reg_data = int(reg_data, 16) - self.ql.arch.regs.write(self.tables[QL_ARCH.X8664][count], reg_data) - count += 1 - for j in range(17*16, 17*16+15*8, 8): - reg_data = subcmd[j:j+7] - reg_data = int(reg_data, 16) - self.ql.arch.regs.write(self.tables[QL_ARCH.X8664][count], reg_data) - count += 1 - - elif self.ql.arch.type == QL_ARCH.ARM: - for i in range(0, len(subcmd), 8): - reg_data = subcmd[i:i + 7] - reg_data = int(reg_data, 16) - self.ql.arch.regs.write(self.tables[QL_ARCH.ARM][count], reg_data) - count += 1 - - elif self.ql.arch.type == QL_ARCH.ARM64: - for i in range(0, len(subcmd), 16): - reg_data = subcmd[i:i+15] - reg_data = int(reg_data, 16) - self.ql.arch.regs.write(self.tables[QL_ARCH.ARM64][count], reg_data) - count += 1 - - elif self.ql.arch.type == QL_ARCH.MIPS: - for i in range(0, len(subcmd), 8): - reg_data = subcmd[i:i+7] - reg_data = int(reg_data, 16) - self.ql.arch.regs.write(self.tables[QL_ARCH.MIPS][count], reg_data) - count += 1 - - self.send('OK') - - - def handle_H(subcmd): - if subcmd.startswith('g'): - self.send('OK') - if subcmd.startswith('c'): - self.send('OK') - - - def handle_m(subcmd): - addr, size = subcmd.split(',') - addr = int(addr, 16) - size = int(size, 16) - - try: - tmp = '' - for s in range(size): - mem = self.ql.mem.read(addr + s, 1) - mem = "".join( - [str("{:02x}".format(ord(c))) for c in mem.decode('latin1')]) - tmp += mem - self.send(tmp) - - except: - self.send('E14') - - - def handle_M(subcmd): - addr, data = subcmd.split(',') - size, data = data.split(':') - addr = int(addr, 16) - data = bytes.fromhex(data) - try: - self.ql.mem.write(addr, data) - self.send('OK') - except: - self.send('E01') - - - def handle_p(subcmd): - reg_index = int(subcmd, 16) - reg_value = None - try: - if self.ql.arch.type == QL_ARCH.A8086: - if reg_index <= 9: - reg_value = self.ql.arch.regs.read(self.tables[QL_ARCH.A8086][reg_index-1]) - else: - reg_value = 0 - reg_value = self.addr_to_str(reg_value) - - elif self.ql.arch.type == QL_ARCH.X86: - if reg_index <= 24: - reg_value = self.ql.arch.regs.read(self.tables[QL_ARCH.X86][reg_index-1]) - else: - reg_value = 0 - reg_value = self.addr_to_str(reg_value) - - elif self.ql.arch.type == QL_ARCH.X8664: - if reg_index <= 32: - reg_value = self.ql.arch.regs.read(self.tables[QL_ARCH.X8664][reg_index-1]) - else: - reg_value = 0 - if reg_index <= 17: - reg_value = self.addr_to_str(reg_value) - elif 17 < reg_index: - reg_value = self.addr_to_str(reg_value, short = True) - - elif self.ql.arch.type == QL_ARCH.ARM: - if reg_index < 26: - reg_value = self.ql.arch.regs.read(self.tables[QL_ARCH.ARM][reg_index - 1]) - else: - reg_value = 0 - reg_value = self.addr_to_str(reg_value) - - elif self.ql.arch.type == QL_ARCH.ARM64: - if reg_index <= 32: - reg_value = self.ql.arch.regs.read(self.tables[QL_ARCH.ARM64][reg_index - 1]) - else: - reg_value = 0 - reg_value = self.addr_to_str(reg_value) - - elif self.ql.arch.type == QL_ARCH.MIPS: - if reg_index <= 37: - reg_value = self.ql.arch.regs.read(self.tables[QL_ARCH.MIPS][reg_index - 1]) - else: - reg_value = 0 - reg_value = self.addr_to_str(reg_value) - - if type(reg_value) is not str: - reg_value = self.addr_to_str(reg_value) - - self.send(reg_value) - except: - self.close() - raise - - - def handle_P(subcmd): - reg_index, reg_data = subcmd.split('=') - reg_index = int(reg_index, 16) - reg_name = self.tables[self.ql.arch.type][reg_index] - - if self.ql.arch.type == QL_ARCH.A8086: - reg_data = int(reg_data, 16) - reg_data = int.from_bytes(struct.pack(' Write to register %s with %x\n" % (self.tables[self.ql.arch.type][reg_index], reg_data)) - self.send('OK') - - - def handle_Q(subcmd): - if subcmd.startswith( 'StartNoAckMode'): - self.send('OK') - - elif subcmd.startswith( 'DisableRandomization'): - self.send('OK') - - elif subcmd.startswith( 'ProgramSignals'): - self.send('OK') - - elif subcmd.startswith( 'NonStop'): - self.send('OK') - - elif subcmd.startswith('PassSignals'): - self.send('OK') - - elif subcmd.startswith('qemu'): - self.send('') - - def handle_D(subcmd): - self.send('OK') - - def handle_q(subcmd): - if subcmd.startswith('Supported:'): - # might or might not need for multi thread - if self.ql.multithread == False: - self.send("PacketSize=3fff;QPassSignals+;QProgramSignals+;QStartupWithShell+;QEnvironmentHexEncoded+;QEnvironmentReset+;QEnvironmentUnset+;QSetWorkingDir+;QCatchSyscalls+;qXfer:libraries-svr4:read+;augmented-libraries-svr4-read+;qXfer:auxv:read+;qXfer:spu:read+;qXfer:spu:write+;qXfer:siginfo:read+;qXfer:siginfo:write+;qXfer:features:read+;QStartNoAckMode+;qXfer:osdata:read+;multiprocess+;fork-events+;vfork-events+;exec-events+;QNonStop+;QDisableRandomization+;qXfer:threads:read+;ConditionalTracepoints+;TraceStateVariables+;TracepointSource+;DisconnectedTracing+;StaticTracepoints+;InstallInTrace+;qXfer:statictrace:read+;qXfer:traceframe-info:read+;EnableDisableTracepoints+;QTBuffer:size+;tracenz+;ConditionalBreakpoints+;BreakpointCommands+;QAgent+;swbreak+;hwbreak+;qXfer:exec-file:read+;no-resumed+") - else: - self.send("PacketSize=47ff;QPassSignals+;QProgramSignals+;QStartupWithShell+;QEnvironmentHexEncoded+;QEnvironmentReset+;QEnvironmentUnset+;QSetWorkingDir+;QCatchSyscalls+;qXfer:libraries-svr4:read+;augmented-libraries-svr4-read+;qXfer:auxv:read+;qXfer:siginfo:read+;qXfer:siginfo:write+;qXfer:features:read+;QStartNoAckMode+;qXfer:osdata:read+;multiprocess+;fork-events+;vfork-events+;exec-events+;QNonStop+;QDisableRandomization+;qXfer:threads:read+;ConditionalTracepoints+;TraceStateVariables+;TracepointSource+;DisconnectedTracing+;FastTracepoints+;StaticTracepoints+;InstallInTrace+;qXfer:statictrace:read+;qXfer:traceframe-info:read+;EnableDisableTracepoints+;QTBuffer:size+;tracenz+;ConditionalBreakpoints+;BreakpointCommands+;QAgent+;Qbtrace:bts+;Qbtrace-conf:bts:size+;Qbtrace:pt+;Qbtrace-conf:pt:size+;Qbtrace:off+;qXfer:btrace:read+;qXfer:btrace-conf:read+;swbreak+;hwbreak+;qXfer:exec-file:read+;vContSupported+;QThreadEvents+;no-resumed+") - elif subcmd.startswith('Xfer:features:read'): - xfercmd_file = subcmd.split(':')[3] - xfercmd_abspath = os.path.dirname(os.path.abspath(__file__)) - xml_folder = arch_convert_str(self.ql.arch.type).lower() - xfercmd_file = os.path.join(xfercmd_abspath,"xml",xml_folder, xfercmd_file) - - if os.path.exists(xfercmd_file) and self.ql.ostype is not QL_OS.WINDOWS: - with open(xfercmd_file, 'r') as f: - file_contents = f.read() - self.send("l%s" % file_contents) - else: - self.ql.log.info("gdb> Platform is not supported by xml or xml file not found: %s\n" % (xfercmd_file)) - self.send("l") - - - elif subcmd.startswith('Xfer:threads:read::0,'): - if self.ql.ostype in QL_OS_NONPID or self.ql.baremetal: - self.send("l") - else: - file_contents = ("\r\n\r\n") - self.send("l" + file_contents) - - elif subcmd.startswith('Xfer:auxv:read::'): - if self.ql.code: - return - - if self.ql.ostype in (QL_OS.LINUX, QL_OS.FREEBSD): - def __read_auxv() -> Iterator[int]: - auxv_entries = ( - AUX.AT_HWCAP, - AUX.AT_PAGESZ, - AUX.AT_CLKTCK, - AUX.AT_PHDR, - AUX.AT_PHENT, - AUX.AT_PHNUM, - AUX.AT_BASE, - AUX.AT_FLAGS, - AUX.AT_ENTRY, - AUX.AT_UID, - AUX.AT_EUID, - AUX.AT_GID, - AUX.AT_EGID, - AUX.AT_SECURE, - AUX.AT_RANDOM, - AUX.AT_HWCAP2, - AUX.AT_EXECFN, - AUX.AT_PLATFORM, - AUX.AT_NULL - ) - - for e in auxv_entries: - yield e.value - yield self.ql.loader.aux_vec[e] - - annex = self.addr_to_str(0)[:-2] - sysinfo_ehdr = self.addr_to_str(0) - - auxvdata_c = unhexlify(''.join([annex, sysinfo_ehdr] + [self.addr_to_str(val) for val in __read_auxv()])) - auxvdata = self.bin_to_escstr(auxvdata_c) - else: - auxvdata = b"" - - self.send(b'l!%s' % auxvdata) - - elif subcmd.startswith('Xfer:exec-file:read:'): - self.send("l%s" % str(self.exe_abspath)) - - - elif subcmd.startswith('Xfer:libraries-svr4:read:'): - if self.ql.ostype in (QL_OS.LINUX, QL_OS.FREEBSD): - xml_addr_mapping=("") - """ - FIXME: need to find out when do we need this - """ - #for s, e, info in self.ql.map_info: - # addr_mapping += ("" %(info, e, s)) - xml_addr_mapping += ("") - self.send("l%s" % xml_addr_mapping) - else: - self.send("l") - - elif subcmd.startswith("Xfer:btrace-conf:read:"): - self.send("E.Btrace not enabled.") - - elif subcmd == "Attached": - self.send("") - - elif subcmd.startswith("C"): - self.send("") - - elif subcmd.startswith("L:"): - self.send("M001") - - elif subcmd == "fThreadInfo": - self.send("m0") - - elif subcmd == "sThreadInfo": - self.send("l") - - elif subcmd == ("TStatus"): - self.send("T0;tnotrun:0;tframes:0;tcreated:0;tfree:50*!;tsize:50*!;circular:0;disconn:0;starttime:0;stoptime:0;username:;notes::") - - elif subcmd == ("TfV"): - self.send("l") - - elif subcmd == ("TsV"): - self.send("l") - - elif subcmd == ("TfP"): - self.send("l") - - elif subcmd == ("TsP"): - self.send("l") - - - elif subcmd.startswith("Symbol"): - self.send("") - - elif subcmd.startswith("Attached"): - self.send("") - - elif subcmd == "Offsets": - self.send("Text=0;Data=0;Bss=0") - - - def handle_v(subcmd): - - if subcmd == 'MustReplyEmpty': - self.send("") - - elif subcmd.startswith('File:open'): - if self.ql.ostype == QL_OS.UEFI or self.ql.baremetal: - self.send("F-1") - return - - (file_path, flags, mode) = subcmd.split(':')[-1].split(',') - file_path = unhexlify(file_path).decode(encoding='UTF-8') - flags = int(flags, base=16) - mode = int(mode, base=16) - if file_path.startswith(self.rootfs_abspath): - file_abspath = file_path - else: - file_abspath = self.ql.os.path.transform_to_real_path(file_path) - - self.ql.log.debug("gdb> target file: %s" % (file_abspath)) - if os.path.exists(file_abspath) and not (file_path).startswith("/proc"): - fd = os.open(file_abspath, flags, mode) - self.send("F%x" % fd) - else: - self.send("F-1") - return - - elif subcmd.startswith('File:pread:'): - (fd, count, offset) = subcmd.split(':')[-1].split(',') - - fd = int(fd, base=16) - offset = int(offset, base=16) - count = int(count, base=16) - - data = os.pread(fd, count, offset) - size = len(data) - data = self.bin_to_escstr(data) - - if data: - self.send(("F%x;" % size).encode() + (data)) - else: - self.send("F0;") - - elif subcmd.startswith('File:close'): - fd = subcmd.split(':')[-1] - fd = int(fd, base=16) - os.close(fd) - self.send("F0") - - elif subcmd.startswith('Kill'): - self.send('OK') - - elif subcmd.startswith('Cont'): - self.ql.log.debug("gdb> Cont command received: %s" % subcmd) - if subcmd == 'Cont?': - self.send('vCont;c;C;t;s;S;r') - elif subcmd.startswith ("Cont;"): - subcmd = subcmd.split(';') - subcmd = subcmd[1].split(':') - if subcmd[0] in ('c', 'C05'): - handle_c(subcmd) - elif subcmd[0] in ('S', 's', 'S05'): - handle_s(subcmd) - else: - self.send("") - - - def handle_s(subcmd): - current_address = self.gdb.current_address - if current_address is None: - entry_point = self.gdb.entry_point - if entry_point is not None: - self.gdb.soft_bp = True - self.gdb.resume_emu(entry_point) - else: - self.gdb.soft_bp = True - self.gdb.resume_emu() - self.send('S%.2x' % GDB_SIGNAL_TRAP) - - - def handle_X(subcmd): - self.send('') - - - def handle_Z(subcmd): - data = subcmd - ztype = data[data.find('Z') + 1:data.find(',')] - if ztype == '0': - ztype, address, value = data.split(',') - address = int(address, 16) - try: - self.gdb.bp_insert(address) - self.send('OK') - except: - self.send('E22') - else: - self.send('E22') - - - def handle_z(subcmd): - data = subcmd.split(',') - if len(data) != 3: - self.send('E22') - try: - type = data[0] - addr = int(data[1], 16) - length = data[2] - self.gdb.bp_remove(addr, type, length) - self.send('OK') - except: - self.send('E22') - - - def handle_exclaim(subcmd): - self.send('OK') - - commands = { - '!': handle_exclaim, - '?': handle_qmark, - 'c': handle_c, - 'C': handle_C, - 'D': handle_D, - 'g': handle_g, - 'G': handle_G, - 'H': handle_H, - 'm': handle_m, - 'M': handle_M, - 'p': handle_p, - 'P': handle_P, - 'q': handle_q, - 'Q': handle_Q, - 's': handle_s, - 'v': handle_v, - 'X': handle_X, - 'Z': handle_Z, - 'z': handle_z - } - - cmd, subcmd = pkt[0], pkt[1:] - if cmd == 'k': + def readpackets(self) -> Iterator[bytes]: + """Iterate through incoming packets in an active connection until + it is terminated. + """ + + pattern = re.compile(br'^\$(?P[^#]*)#(?P[0-9a-fA-F]{2})') + buffer = bytearray() + + while True: + try: + incoming = self.client.recv(self.BUFSIZE) + except ConnectionError: break - if cmd not in commands: - self.send('') - self.ql.log.info("gdb> Command not supported: %s\n" %(cmd)) + # remote connection closed + if not incoming: + break + + buffer += incoming + + # discard incoming acks + if buffer[0:1] == REPLY_ACK: + del buffer[0] + + packet = pattern.match(buffer) + + # if there is no match, the rest of the packet might be missing + if not packet: continue - self.ql.log.debug("gdb> received: %s%s" % (cmd, subcmd)) - commands[cmd](subcmd) - self.close() + data = packet['data'] + read_csum = int(packet['checksum'], 16) + calc_csum = GdbSerialConn.checksum(data) + if read_csum != calc_csum: + raise IOError(f'checksum error: expected {calc_csum:02x} but got {read_csum:02x}') - def receive(self): - '''Receive a packet from a GDB client''' - csum = 0 - state = 'Finding SOP' - packet = '' - try: - while True: - c = self.netin.read(1) - if c == '\x03': - return 'Error: CTRL+C' + # follow gdbserver debug output format + self.log.debug(f'getpkt ("{GdbSerialConn.__printable_prefix(data).decode(ENCODING)}");') - if len(c) != 1: - return 'Error: EOF' + data = GdbSerialConn.rle_decode(data) + data = GdbSerialConn.unescape(data) - if state == 'Finding SOP': - if c == '$': - state = 'Finding EOP' - elif state == 'Finding EOP': - if c == '#': - if csum != int(self.netin.read(2), 16): - raise Exception('invalid checksum') - self.last_pkt = packet - return 'Good' - else: - packet += c - csum = (csum + ord(c)) & 0xff - else: - raise Exception('should not be here') - except: - self.close() - raise + del buffer[:packet.endpos] + yield data - def checksum(self, data): - checksum = 0 - for c in data: - if type(c) == str: - checksum += (ord(c)) - else: - checksum += c - return checksum & 0xff + def send(self, data: Reply, raw: bool = False) -> None: + """Send out a packet. - def send(self, msg): - """Send a packet to the GDB client""" - if type(msg) == str: - self.send_raw('$%s#%.2x' % (msg, self.checksum(msg))) + Args: + data : data to send out + raw : whether to encapsulate the data with standard header and + checksum or leave it raw + """ + + if type(data) is str: + data = data.encode(ENCODING) + + assert type(data) is bytes + + if raw: + packet = data else: - self.clientsocket.send(b'$%s#%.2x' % (msg, self.checksum(msg))) - self.netout.flush() + data = GdbSerialConn.escape(data) + data = GdbSerialConn.rle_encode(data) - self.ql.log.debug("gdb> send: $%s#%.2x" % (msg, self.checksum(msg))) + packet = b'$' + data + b'#' + f'{GdbSerialConn.checksum(data):02x}'.encode() - def send_raw(self, r): - self.netout.write(r) - self.netout.flush() + # follow gdbserver debug output format + self.log.debug(f'putpkt ("{GdbSerialConn.__printable_prefix(data).decode(ENCODING)}");') + + self.client.sendall(packet) + + @staticmethod + def __printable_prefix(data: bytes) -> bytes: + """Follow the gnu gdbserver debug message format which emits only the + printable prefix of a packet (either incoming or outgoing). Note that + despite of its name, it includes non-printable characters as well. + + Args: + data : packet data to scan + + Returns: a prefix of the specified data buffer + """ + + def __isascii(ch: int) -> bool: + return 0 < ch < 0x80 + + if data.isascii(): + return data + + return data[:next((i for i, ch in enumerate(data) if not __isascii(ch)), len(data))] + + @staticmethod + def escape(data: bytes) -> bytes: + """Escape data according to gdb protocol escaping rules. + """ + + def __repl(m: 're.Match[bytes]') -> bytes: + ch, *_ = m[0] + + return bytes([ord('}'), ch ^ 0x20]) + + return re.sub(br'[*#$}]', __repl, data, flags=re.DOTALL) + + @staticmethod + def unescape(data: bytes) -> bytes: + """Unescape data according to gdb protocol escaping rules. + """ + + def __repl(m: 're.Match[bytes]') -> bytes: + _, ch = m[0] + + return bytes([ch ^ 0x20]) + + return re.sub(br'}.', __repl, data, flags=re.DOTALL) + + @staticmethod + def rle_encode(data: bytes) -> bytes: + """Compact data using run-length encoding. + """ + + def __simple_rep(b: bytes, times: int) -> bytes: + return b * times + + def __runlen_rep(b: bytes, times: int) -> bytes: + return b + b'*' + bytes([times - 1 + 29]) + + def __encode_rep(b: bytes, times: int) -> bytes: + assert times > 0, 'time should be a positive value' + + if 0 < times < 4: + return __simple_rep(b, times) + + elif times == 6+1 or times == 7+1: + return __runlen_rep(b, 6) + __encode_rep(b, times - 6) + + else: + return __runlen_rep(b, times) + + def __repl(m: 're.Match[bytes]') -> bytes: + repetition = m[0] + + ch = repetition[0:1] + times = len(repetition) + + return __encode_rep(ch, times) + + return re.sub(br'(.)\1{3,96}', __repl, data, flags=re.DOTALL) + + @staticmethod + def rle_decode(data: bytes) -> bytes: + """Expand run-length encoded data. + """ + + def __repl(m: 're.Match[bytes]') -> bytes: + ch, _, times = m[0] + + return bytes([ch] * (1 + times - 29)) + + return re.sub(br'.\*.', __repl, data, flags=re.DOTALL) + + @staticmethod + def checksum(data: bytes) -> int: + return sum(data) & 0xff diff --git a/qiling/debugger/gdb/utils.py b/qiling/debugger/gdb/utils.py index 4c8a5247..30bba47e 100644 --- a/qiling/debugger/gdb/utils.py +++ b/qiling/debugger/gdb/utils.py @@ -3,117 +3,76 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from typing import Optional + from qiling import Qiling from qiling.const import QL_ARCH +# this code is partially based on uDbg +# @see: https://github.com/iGio90/uDdbg + PROMPT = r'gdb>' class QlGdbUtils: - def __init__(self): - self.ql: Qiling - - self.current_address = 0x0 - self.current_address_size = 0x0 - self.last_bp = 0x0 - self.entry_point = None - self.exit_point = None - self.soft_bp = False - self.has_soft_bp = False - self.bp_list = [] - # self.mapping = [] - self.breakpoint_count = 0x0 - self.skip_bp_count = 0x0 - - - def initialize(self, ql: Qiling, hook_address: int, exit_point: int, mappings=[]): + def __init__(self, ql: Qiling, entry_point: int, exit_point: int): self.ql = ql - if ql.baremetal: - self.current_address = self.entry_point - else: - self.current_address = self.entry_point = ql.os.entry_point - self.exit_point = exit_point - # self.mapping = mappings + self.bp_list = [] + self.last_bp = None def __entry_point_hook(ql: Qiling): ql.hook_del(ep_hret) ql.hook_code(self.dbg_hook) + + ql.log.info(f'{PROMPT} stopped at entry point: {ql.arch.regs.arch_pc:#x}') ql.stop() - ql.log.info(f'{PROMPT} Stop at entry point: {ql.arch.regs.arch_pc:#x}') - - ep_hret = ql.hook_address(__entry_point_hook, hook_address) + # set a one-time hook to be dispatched upon reaching program entry point. + # that hook will be used to set up the breakpoint handling hook + ep_hret = ql.hook_address(__entry_point_hook, entry_point) def dbg_hook(self, ql: Qiling, address: int, size: int): - """Modified this function for qiling.gdbserver by kabeor from https://github.com/iGio90/uDdbg - """ + if ql.arch.type == QL_ARCH.ARM and ql.arch.is_thumb: + address += 1 - try: - if ql.arch.type == QL_ARCH.ARM: - if ql.arch.is_thumb: - address += 1 + # resuming emulation after hitting a breakpoint will re-enter this hook. + # avoid an endless hooking loop by detecting and skipping this case + if address == self.last_bp: + self.last_bp = None - # self.mapping.append([(hex(address))]) - self.current_address = address - hit_soft_bp = False + elif address in self.bp_list: + self.last_bp = address - if self.soft_bp == True: - self.soft_bp = False - hit_soft_bp = True - - # Breakpoints are always added without the LSB, even in Thumb, so they should be checked like this as well - if ((address & ~1) in self.bp_list and (address & ~1) != self.last_bp) or self.has_soft_bp == True: - if self.skip_bp_count > 0: - self.skip_bp_count -= 1 - else: - self.breakpoint_count += 1 - ql.stop() - self.last_bp = address - ql.log.info(f'{PROMPT} Breakpoint found, stop at address: {address:#x}') - - elif address == self.last_bp: - self.last_bp = 0x0 - - self.has_soft_bp = hit_soft_bp - - if self.current_address + size == self.exit_point: - ql.log.debug(f'{PROMPT} emulation entrypoint at {self.entry_point:#x}') - ql.log.debug(f'{PROMPT} emulation exitpoint at {self.exit_point:#x}') - - except KeyboardInterrupt: - ql.log.info(f'{PROMPT} Paused at {address:#x}, instruction size = {size:d}') + ql.log.info(f'{PROMPT} breakpoint hit, stopped at {address:#x}') ql.stop() - except: - raise + # # TODO: not sure what this is about + # if address + size == self.exit_point: + # ql.log.debug(f'{PROMPT} emulation entrypoint at {self.entry_point:#x}') + # ql.log.debug(f'{PROMPT} emulation exitpoint at {self.exit_point:#x}') def bp_insert(self, addr: int): if addr not in self.bp_list: self.bp_list.append(addr) - self.ql.log.info(f'{PROMPT} Breakpoint added at: {addr:#x}') + self.ql.log.info(f'{PROMPT} breakpoint added at {addr:#x}') - def bp_remove(self, addr: int, type = None, len = None): + def bp_remove(self, addr: int): self.bp_list.remove(addr) - self.ql.log.info(f'{PROMPT} Breakpoint removed at: {addr:#x}') + self.ql.log.info(f'{PROMPT} breakpoint removed from {addr:#x}') - def resume_emu(self, address: int = None, skip_bp: int = 0): - """Modified this function for qiling.gdbserver by kabeor from https://github.com/iGio90/uDdbg - """ + def resume_emu(self, address: Optional[int] = None, steps: int = 0): + if address is None: + address = self.ql.arch.regs.arch_pc - if address is not None: - if self.ql.arch.type == QL_ARCH.ARM: - if self.ql.arch.is_thumb: - address += 1 + if self.ql.arch.type == QL_ARCH.ARM and self.ql.arch.is_thumb: + address += 1 - self.current_address = address + op = f'stepping {steps} instructions' if steps else 'resuming' + self.ql.log.info(f'{PROMPT} {op} from {address:#x}') - self.skip_bp_count = skip_bp - - if self.exit_point is not None: - self.ql.log.info(f'{PROMPT} Resume at: {self.current_address:#x}') - self.ql.emu_start(self.current_address, self.exit_point) + self.ql.emu_start(address, self.exit_point, count=steps) diff --git a/qiling/debugger/gdb/xml/a8086/target.xml b/qiling/debugger/gdb/xml/a8086/target.xml index 71daae8c..43bc64d7 100644 --- a/qiling/debugger/gdb/xml/a8086/target.xml +++ b/qiling/debugger/gdb/xml/a8086/target.xml @@ -1 +1,6 @@ -i8086 + + + + i8086 + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/arm/arm-core.xml b/qiling/debugger/gdb/xml/arm/arm-core.xml index a2c0a65b..5032c503 100644 --- a/qiling/debugger/gdb/xml/arm/arm-core.xml +++ b/qiling/debugger/gdb/xml/arm/arm-core.xml @@ -1,31 +1,31 @@ - + Copying and distribution of this file, with or without modification, + are permitted in any medium without royalty provided the copyright + notice and this notice are preserved. --> - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + - - + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/arm/arm-m-profile-with-fpa.xml b/qiling/debugger/gdb/xml/arm/arm-m-profile-with-fpa.xml deleted file mode 100644 index dd30abe8..00000000 --- a/qiling/debugger/gdb/xml/arm/arm-m-profile-with-fpa.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/qiling/debugger/gdb/xml/arm/arm-m-profile.xml b/qiling/debugger/gdb/xml/arm/arm-m-profile.xml index 5319d764..f0584a20 100644 --- a/qiling/debugger/gdb/xml/arm/arm-m-profile.xml +++ b/qiling/debugger/gdb/xml/arm/arm-m-profile.xml @@ -7,7 +7,7 @@ - + @@ -23,5 +23,6 @@ + diff --git a/qiling/debugger/gdb/xml/arm/target.xml b/qiling/debugger/gdb/xml/arm/target.xml index 579e84b0..89826d47 100644 --- a/qiling/debugger/gdb/xml/arm/target.xml +++ b/qiling/debugger/gdb/xml/arm/target.xml @@ -6,9 +6,9 @@ *!notice and this notice are preserved. --> - - arm - - - - + + arm + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/arm64/aarch64-core.xml b/qiling/debugger/gdb/xml/arm64/aarch64-core.xml index 5b2706e0..599d92b3 100644 --- a/qiling/debugger/gdb/xml/arm64/aarch64-core.xml +++ b/qiling/debugger/gdb/xml/arm64/aarch64-core.xml @@ -1,67 +1,67 @@ + Copying and distribution of this file, with or without modification, + are permitted in any medium without royalty provided the copyright + notice and this notice are preserved. --> - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - - * - * - * - * - * - * - * - * - * + + + + + + + + + + - * - * + + - * - * - * - * - - + + + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/arm64/aarch64-fpu.xml b/qiling/debugger/gdb/xml/arm64/aarch64-fpu.xml index 0b37e15f..0cdf73e5 100644 --- a/qiling/debugger/gdb/xml/arm64/aarch64-fpu.xml +++ b/qiling/debugger/gdb/xml/arm64/aarch64-fpu.xml @@ -1,86 +1,93 @@ + Copying and distribution of this file, with or without modification, + are permitted in any medium without royalty provided the copyright + notice and this notice are preserved. --> - - - - - - - - - - - - - - * - * - * - - - * - * - * - - - * - * - - - * - * - - - * - * - - - * - * - * - * - * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/arm64/target.xml b/qiling/debugger/gdb/xml/arm64/target.xml index d4dbac66..c741efa6 100644 --- a/qiling/debugger/gdb/xml/arm64/target.xml +++ b/qiling/debugger/gdb/xml/arm64/target.xml @@ -1,14 +1,15 @@ + Copying and distribution of this file, with or without modification, + are permitted in any medium without royalty provided the copyright + notice and this notice are preserved. --> - - aarch64 - - + + aarch64 + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/mips/target.xml b/qiling/debugger/gdb/xml/mips/target.xml index a585dffd..0a6a1b62 100644 --- a/qiling/debugger/gdb/xml/mips/target.xml +++ b/qiling/debugger/gdb/xml/mips/target.xml @@ -6,14 +6,15 @@ *!notice and this notice are preserved. --> - - mips - GNU/Linux - - - + + mips + GNU/Linux - - * - + + + + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/x86/32bit-core.xml b/qiling/debugger/gdb/xml/x86/32bit-core.xml index 0958032a..702ae3b0 100644 --- a/qiling/debugger/gdb/xml/x86/32bit-core.xml +++ b/qiling/debugger/gdb/xml/x86/32bit-core.xml @@ -7,59 +7,59 @@ - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - + + + + + + + + - - - - - - - - - + + + + + + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/x86/32bit-mpx.xml b/qiling/debugger/gdb/xml/x86/32bit-mpx.xml index 7772954d..1798dbd8 100644 --- a/qiling/debugger/gdb/xml/x86/32bit-mpx.xml +++ b/qiling/debugger/gdb/xml/x86/32bit-mpx.xml @@ -7,39 +7,41 @@ - - - - + + + + - - - - + + + + - - - - + + + + - - - - - - - + + + + + + + - - - - + + + + - - - - - - - + + + + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/x86/32bit-pkeys.xml b/qiling/debugger/gdb/xml/x86/32bit-pkeys.xml index 6f3c65a8..71aa7dc1 100644 --- a/qiling/debugger/gdb/xml/x86/32bit-pkeys.xml +++ b/qiling/debugger/gdb/xml/x86/32bit-pkeys.xml @@ -7,7 +7,5 @@ - - - - + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/x86/32bit-sse.xml b/qiling/debugger/gdb/xml/x86/32bit-sse.xml index c57a2d84..e28536e5 100644 --- a/qiling/debugger/gdb/xml/x86/32bit-sse.xml +++ b/qiling/debugger/gdb/xml/x86/32bit-sse.xml @@ -7,46 +7,48 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + - - - - - - - - + + + + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/x86/target.xml b/qiling/debugger/gdb/xml/x86/target.xml index a664320d..50270323 100644 --- a/qiling/debugger/gdb/xml/x86/target.xml +++ b/qiling/debugger/gdb/xml/x86/target.xml @@ -1,11 +1,11 @@ -i386 - GNU/Linux - - - - - - - + +i386 + GNU/Linux + + + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/x8664/target.xml b/qiling/debugger/gdb/xml/x8664/target.xml index d58e543e..d68705f6 100644 --- a/qiling/debugger/gdb/xml/x8664/target.xml +++ b/qiling/debugger/gdb/xml/x8664/target.xml @@ -1 +1,13 @@ -i386:x86-64GNU/Linux + + + + i386:x86-64 + GNU/Linux + + + + + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xmlregs.py b/qiling/debugger/gdb/xmlregs.py new file mode 100644 index 00000000..7385a303 --- /dev/null +++ b/qiling/debugger/gdb/xmlregs.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +from typing import Iterator, Mapping, Optional, Sequence, Tuple +from pathlib import PurePath +from xml.etree import ElementTree, ElementInclude + +from qiling.arch.arm_const import reg_map as arm_regs +from qiling.arch.arm_const import reg_vfp as arm_regs_vfp +from qiling.arch.arm64_const import reg_map as arm64_regs +from qiling.arch.arm64_const import reg_map_v as arm64_regs_v +from qiling.arch.mips_const import reg_map as mips_regs_gpr +from qiling.arch.mips_const import reg_map_fpu as mips_regs_fpu +from qiling.arch.x86_const import reg_map_32 as x86_regs_32 +from qiling.arch.x86_const import reg_map_64 as x86_regs_64 +from qiling.arch.x86_const import reg_map_misc as x86_regs_misc +from qiling.arch.x86_const import reg_map_cr as x86_regs_cr +from qiling.arch.x86_const import reg_map_st as x86_regs_st +from qiling.arch.x86_const import reg_map_xmm as x86_regs_xmm +from qiling.arch.x86_const import reg_map_ymm as x86_regs_ymm + +from qiling.const import QL_ARCH, QL_OS + +RegEntry = Tuple[Optional[int], int, int] + +class QlGdbFeatures: + def __init__(self, archtype: QL_ARCH, ostype: QL_OS): + xmltree = QlGdbFeatures.__load_target_xml(archtype, ostype) + regsmap = QlGdbFeatures.__load_regsmap(archtype, xmltree) + + self.xmltree = xmltree + self.regsmap = regsmap + + def tostring(self) -> str: + root = self.xmltree.getroot() + + return ElementTree.tostring(root, encoding='unicode', xml_declaration=True) + + @staticmethod + def __get_xml_path(archtype: QL_ARCH) -> Tuple[str, PurePath]: + import inspect + + p = PurePath(inspect.getfile(QlGdbFeatures)) + basedir = p.parent / 'xml' / archtype.name.lower() + filename = basedir / 'target.xml' + + return str(filename), basedir + + @staticmethod + def __load_target_xml(archtype: QL_ARCH, ostype: QL_OS) -> ElementTree.ElementTree: + filename, base_url = QlGdbFeatures.__get_xml_path(archtype) + + tree = ElementTree.parse(filename) + + # NOTE: this is needed to load xinclude hrefs relative to the main xml file. starting + # from python 3.9 ElementInclude.include has an argument for that called 'base_url'. + # this is a workaround for earlier python versions such as 3.8 + + # + def my_loader(base: PurePath): + def __wrapped(href: str, parse, encoding=None): + abshref = base / href + + return ElementInclude.default_loader(str(abshref), parse, encoding) + + return __wrapped + # + + # inline all xi:include elements + ElementInclude.include(tree.getroot(), loader=my_loader(base_url)) + + # patch xml osabi element with the appropriate abi tag + osabi = tree.find('osabi') + + if osabi is not None: + # NOTE: the 'Windows' abi tag is supported starting from gdb 10. + # earlier gdb versions use 'Cygwin' instead + + abitag = { + QL_OS.LINUX : 'GNU/Linux', + QL_OS.FREEBSD : 'FreeBSD', + QL_OS.MACOS : 'Darwin', + QL_OS.WINDOWS : 'Windows', + QL_OS.UEFI : 'Windows', + QL_OS.DOS : 'Windows', + QL_OS.QNX : 'QNX-Neutrino' + }.get(ostype, 'unknown') + + osabi.text = abitag + + return tree + + @staticmethod + def __walk_xml_regs(xmltree: ElementTree.ElementTree) -> Iterator[Tuple[int, str, int]]: + regnum = -1 + + for reg in xmltree.iter('reg'): + # if regnum is not specified, assume it follows the previous one + regnum = int(reg.get('regnum', regnum + 1)) + + name = reg.attrib['name'] + bitsize = reg.attrib['bitsize'] + + yield regnum, name, int(bitsize) + + @staticmethod + def __load_regsmap(archtype: QL_ARCH, xmltree: ElementTree.ElementTree) -> Sequence[RegEntry]: + """Initialize registers map using available target XML files. + + Args: + archtype: target architecture type + + Returns: a list representing registers data + """ + + # retreive the relevant set of registers; their order of appearance is not + # important as it is determined by the info read from the xml files + ucregs: Mapping[str, int] = { + QL_ARCH.A8086 : dict(**x86_regs_32, **x86_regs_misc, **x86_regs_cr, **x86_regs_st), + QL_ARCH.X86 : dict(**x86_regs_32, **x86_regs_misc, **x86_regs_cr, **x86_regs_st, **x86_regs_xmm), + QL_ARCH.X8664 : dict(**x86_regs_64, **x86_regs_misc, **x86_regs_cr, **x86_regs_st, **x86_regs_xmm, **x86_regs_ymm), + QL_ARCH.ARM : dict(**arm_regs, **arm_regs_vfp), + QL_ARCH.CORTEX_M : arm_regs, + QL_ARCH.ARM64 : dict(**arm64_regs, **arm64_regs_v), + QL_ARCH.MIPS : dict(**mips_regs_gpr, **mips_regs_fpu) + }[archtype] + + regsinfo = sorted(QlGdbFeatures.__walk_xml_regs(xmltree)) + + # pre-allocate regmap and occupy it with null entries + last_regnum = regsinfo[-1][0] + regmap: Sequence[RegEntry] = [(None, 0, 0)] * (last_regnum + 1) + + pos = 0 + + for regnum, name, bitsize in sorted(regsinfo): + # reg value size in nibbles + nibbles = bitsize // 4 + + regmap[regnum] = (ucregs.get(name), pos, nibbles) + + # value position of next reg + pos += nibbles + + return regmap + + +__all__ = ['RegEntry', 'QlGdbFeatures'] diff --git a/qiling/debugger/qdb/arch/arch_x86.py b/qiling/debugger/qdb/arch/arch_x86.py index 4aae910c..10617cbd 100644 --- a/qiling/debugger/qdb/arch/arch_x86.py +++ b/qiling/debugger/qdb/arch/arch_x86.py @@ -21,7 +21,7 @@ class ArchX86(Arch): "eax", "ebx", "ecx", "edx", "esp", "ebp", "esi", "edi", "eip", "ss", "cs", "ds", "es", - "fs", "gs", "ef", + "fs", "gs", "eflags", ) def read_insn(self, address: int) -> bytes: @@ -34,7 +34,7 @@ class ArchX86(Arch): @staticmethod def get_flags(bits: int) -> Mapping[str, bool]: """ - get flags from ql.reg.ef + get flags from ql.reg.eflags """ return { diff --git a/qiling/debugger/qdb/branch_predictor/branch_predictor_x86.py b/qiling/debugger/qdb/branch_predictor/branch_predictor_x86.py index ce92dd9f..656c64da 100644 --- a/qiling/debugger/qdb/branch_predictor/branch_predictor_x86.py +++ b/qiling/debugger/qdb/branch_predictor/branch_predictor_x86.py @@ -91,7 +91,7 @@ class BranchPredictorX86(BranchPredictor, ArchX86): } if line.mnemonic in jump_table: - eflags = self.get_flags(self.ql.arch.regs.ef).values() + eflags = self.get_flags(self.ql.arch.regs.eflags).values() prophecy.going = jump_table.get(line.mnemonic)(*eflags) elif line.mnemonic in jump_reg_table: diff --git a/qiling/debugger/qdb/frontend.py b/qiling/debugger/qdb/frontend.py index d1f53f86..ed243446 100644 --- a/qiling/debugger/qdb/frontend.py +++ b/qiling/debugger/qdb/frontend.py @@ -389,7 +389,7 @@ class CtxManager_X86(CtxManager): "eax", "ebx", "ecx", "edx", "esp", "ebp", "esi", "edi", "eip", "ss", "cs", "ds", "es", - "fs", "gs", "ef", + "fs", "gs", "eflags", ) @context_printer("[ REGISTERS ]") def context_reg(self, saved_reg_dump): @@ -416,7 +416,7 @@ class CtxManager_X86(CtxManager): lines += line print(lines.format(*cur_regs.values())) - print(color.GREEN, "EFLAGS: [CF: {flags[CF]}, PF: {flags[PF]}, AF: {flags[AF]}, ZF: {flags[ZF]}, SF: {flags[SF]}, OF: {flags[OF]}]".format(flags=get_x86_eflags(self.ql.arch.regs.ef)), color.END, sep="") + print(color.GREEN, "EFLAGS: [CF: {flags[CF]}, PF: {flags[PF]}, AF: {flags[AF]}, ZF: {flags[ZF]}, SF: {flags[SF]}, OF: {flags[OF]}]".format(flags=get_x86_eflags(self.ql.arch.regs.eflags)), color.END, sep="") @context_printer("[ DISASM ]", footer=True) def context_asm(self): diff --git a/qiling/debugger/qdb/qdb.py b/qiling/debugger/qdb/qdb.py index ef9fc075..d8810ff2 100644 --- a/qiling/debugger/qdb/qdb.py +++ b/qiling/debugger/qdb/qdb.py @@ -353,7 +353,9 @@ class QlQdb(cmd.Cmd, QlDebugger): show some runtime information """ - self.ql.mem.show_mapinfo() + for info_line in self.ql.mem.get_formatted_mapinfo(): + self.ql.log.info(info_line) + qdb_print(QDB_MSG.INFO, f"Breakpoints: {[hex(addr) for addr in self.bp_list.keys()]}") if self.rr: qdb_print(QDB_MSG.INFO, f"Snapshots: {len([st for st in self.rr.layers if isinstance(st, self.rr.DiffedState)])}") diff --git a/qiling/debugger/qdb/render/render.py b/qiling/debugger/qdb/render/render.py index b7a8f05f..b8d7eb08 100644 --- a/qiling/debugger/qdb/render/render.py +++ b/qiling/debugger/qdb/render/render.py @@ -34,7 +34,11 @@ class Render: def decorator(context_dumper): def wrapper(*args, **kwargs): - width, height = os.get_terminal_size() + try: + width, _ = os.get_terminal_size() + except OSError: + width = 130 + bar = (width - len(field_name)) // 2 - 1 print(ruler * bar, field_name, ruler * bar) context_dumper(*args, **kwargs) diff --git a/qiling/debugger/qdb/render/render_x86.py b/qiling/debugger/qdb/render/render_x86.py index 3597181e..5c43d0e5 100644 --- a/qiling/debugger/qdb/render/render_x86.py +++ b/qiling/debugger/qdb/render/render_x86.py @@ -22,7 +22,7 @@ class ContextRenderX86(ContextRender, ArchX86): cur_regs = self.dump_regs() diff_reg = self.reg_diff(cur_regs, saved_reg_dump) self.render_regs_dump(cur_regs, diff_reg=diff_reg) - print(color.GREEN, "EFLAGS: [CF: {flags[CF]}, PF: {flags[PF]}, AF: {flags[AF]}, ZF: {flags[ZF]}, SF: {flags[SF]}, OF: {flags[OF]}]".format(flags=self.get_flags(self.ql.arch.regs.ef)), color.END, sep="") + print(color.GREEN, "EFLAGS: [CF: {flags[CF]}, PF: {flags[PF]}, AF: {flags[AF]}, ZF: {flags[ZF]}, SF: {flags[SF]}, OF: {flags[OF]}]".format(flags=self.get_flags(self.ql.arch.regs.eflags)), color.END, sep="") @Render.divider_printer("[ DISASM ]") def context_asm(self): diff --git a/qiling/extensions/idaplugin/qilingida.py b/qiling/extensions/idaplugin/qilingida.py index 954b9948..8c5a3d3b 100644 --- a/qiling/extensions/idaplugin/qilingida.py +++ b/qiling/extensions/idaplugin/qilingida.py @@ -333,7 +333,7 @@ class IDA: @staticmethod def get_ql_arch_string(): info = IDA.get_info_structure() - proc = info.get_procName().lower() + proc = info.procname.lower() result = None if proc == "metapc": result = "x86" @@ -805,7 +805,7 @@ class QlEmuMisc: self.action_type = action def activate(self, ctx): - if ctx.form_type == BWN_DISASM: + if ctx.widget_type == BWN_DISASM: self.action_handler.ql_handle_menu_action(self.action_type) return 1 @@ -878,7 +878,7 @@ class QlEmuQiling: def __init__(self): self.path = None self.rootfs = None - self.ql: Qiling + self.ql: Qiling = None self.status = None self.exit_addr = None self.baseaddr = None @@ -893,7 +893,7 @@ class QlEmuQiling: self.ql.os.stderr = QlEmuMisc.QLStdIO('stderr', sys.__stderr__.fileno()) self.exit_addr = self.ql.os.exit_point - if self.ql.ostype == QL_OS.LINUX: + if self.ql.os.type == QL_OS.LINUX: f = open(self.ql.path, 'rb') elffile = ELFFile(f) elf_header = elffile.header diff --git a/qiling/extensions/report/report.py b/qiling/extensions/report/report.py index 59343165..1189e698 100644 --- a/qiling/extensions/report/report.py +++ b/qiling/extensions/report/report.py @@ -12,11 +12,11 @@ class Report: def __init__(self, ql): self.filename = ql.argv self.rootfs = ql.rootfs - self.arch = list(arch_map.keys())[list(arch_map.values()).index(ql.arch.type)] - self.os = list(os_map.keys())[list(os_map.values()).index(ql.ostype)] + self.arch = ql.arch.type.name + self.os = ql.os.type.name self.env = ql.env self.strings = set() - for string in ql.os.stats.appeared_strings: + for string in ql.os.stats.strings: strings = string.split(" ") self.strings |= set(strings) self.profile = {} @@ -31,7 +31,7 @@ class Report: class WindowsReport(Report): def __init__(self, ql): super().__init__(ql) - self.dlls = ql.loader.dlls + self.teb_address = ql.loader.TEB.base self.peb_address = ql.loader.PEB.base self.ldr_address = ql.loader.LDR.base @@ -54,7 +54,7 @@ class WindowsReport(Report): def generate_report(ql, pretty_print=False) -> dict: - if ql.ostype == QL_OS.WINDOWS: + if ql.os.type == QL_OS.WINDOWS: report = WindowsReport(ql) else: report = Report(ql) diff --git a/qiling/host.py b/qiling/host.py index 5acf5063..89ffd2f7 100644 --- a/qiling/host.py +++ b/qiling/host.py @@ -21,7 +21,7 @@ class QlHost: system = platform.system() - return utils.ostype_convert(system.lower()) + return utils.os_convert(system) @cached_property def arch(self) -> Optional[QL_ARCH]: @@ -30,4 +30,4 @@ class QlHost: machine = platform.machine() - return utils.arch_convert(machine.lower()) + return utils.arch_convert(machine) diff --git a/qiling/loader/elf.py b/qiling/loader/elf.py index 33e7b922..ada00229 100644 --- a/qiling/loader/elf.py +++ b/qiling/loader/elf.py @@ -29,7 +29,7 @@ from qiling.os.linux.kernel_api.kernel_api import hook_sys_open, hook_sys_read, # auxiliary vector types # see: https://man7.org/linux/man-pages/man3/getauxval.3.html -class AUX(IntEnum): +class AUXV(IntEnum): AT_NULL = 0 AT_IGNORE = 1 AT_EXECFD = 2 @@ -123,7 +123,7 @@ class QlLoaderELF(QlLoader): self.ql.arch.regs.arch_sp = self.stack_address # No idea why. - if self.ql.ostype == QL_OS.FREEBSD: + if self.ql.os.type == QL_OS.FREEBSD: # self.ql.arch.regs.rbp = self.stack_address + 0x40 self.ql.arch.regs.rdi = self.stack_address self.ql.arch.regs.r14 = self.stack_address @@ -317,36 +317,39 @@ class QlLoaderELF(QlLoader): elf_hwcap = 0xd7b81f # setup aux vector - aux_entries = ( - (AUX.AT_PHDR, elf_phdr), - (AUX.AT_PHENT, elf_phent), - (AUX.AT_PHNUM, elf_phnum), - (AUX.AT_PAGESZ, self.ql.mem.pagesize), - (AUX.AT_BASE, interp_address), - (AUX.AT_FLAGS, 0), - (AUX.AT_ENTRY, self.elf_entry), - (AUX.AT_UID, self.ql.os.uid), - (AUX.AT_EUID, self.ql.os.euid), - (AUX.AT_GID, self.ql.os.gid), - (AUX.AT_EGID, self.ql.os.egid), - (AUX.AT_HWCAP, elf_hwcap), - (AUX.AT_CLKTCK, 100), - (AUX.AT_RANDOM, randstraddr), - (AUX.AT_HWCAP2, 0), - (AUX.AT_EXECFN, execfn), - (AUX.AT_PLATFORM, cpustraddr), - (AUX.AT_SECURE, 0), - (AUX.AT_NULL, 0) + auxv_entries = ( + (AUXV.AT_HWCAP, elf_hwcap), + (AUXV.AT_PAGESZ, self.ql.mem.pagesize), + (AUXV.AT_CLKTCK, 100), + (AUXV.AT_PHDR, elf_phdr), + (AUXV.AT_PHENT, elf_phent), + (AUXV.AT_PHNUM, elf_phnum), + (AUXV.AT_BASE, interp_address), + (AUXV.AT_FLAGS, 0), + (AUXV.AT_ENTRY, self.elf_entry), + (AUXV.AT_UID, self.ql.os.uid), + (AUXV.AT_EUID, self.ql.os.euid), + (AUXV.AT_GID, self.ql.os.gid), + (AUXV.AT_EGID, self.ql.os.egid), + (AUXV.AT_SECURE, 0), + (AUXV.AT_RANDOM, randstraddr), + (AUXV.AT_HWCAP2, 0), + (AUXV.AT_EXECFN, execfn), + (AUXV.AT_PLATFORM, cpustraddr), + (AUXV.AT_NULL, 0) ) - # add all aux entries - for key, val in aux_entries: - elf_table.extend(self.ql.pack(key) + self.ql.pack(val)) + bytes_before_auxv = len(elf_table) + + # add all auxv entries + for key, val in auxv_entries: + elf_table.extend(self.ql.pack(key)) + elf_table.extend(self.ql.pack(val)) new_stack = self.ql.mem.align(new_stack - len(elf_table), 0x10) self.ql.mem.write(new_stack, bytes(elf_table)) - self.aux_vec = dict(aux_entries) + self.auxv = new_stack + bytes_before_auxv self.stack_address = new_stack self.load_address = load_address @@ -361,11 +364,11 @@ class QlLoaderELF(QlLoader): self.skip_exit_check = (self.elf_entry != self.entry_point) # map vsyscall section for some specific needs - if self.ql.arch.type == QL_ARCH.X8664 and self.ql.ostype == QL_OS.LINUX: + if self.ql.arch.type == QL_ARCH.X8664 and self.ql.os.type == QL_OS.LINUX: _vsyscall_addr = int(self.profile.get('vsyscall_address'), 0) _vsyscall_size = int(self.profile.get('vsyscall_size'), 0) - if not self.ql.mem.is_mapped(_vsyscall_addr, _vsyscall_size): + if self.ql.mem.is_available(_vsyscall_addr, _vsyscall_size): # initialize with int3 instructions then insert syscall entry # each syscall should be 1KiB away self.ql.mem.map(_vsyscall_addr, _vsyscall_size, info="[vsyscall]") diff --git a/qiling/loader/loader.py b/qiling/loader/loader.py index 472881cb..6be0ccf1 100644 --- a/qiling/loader/loader.py +++ b/qiling/loader/loader.py @@ -3,11 +3,15 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +import os from typing import Any, Mapping, MutableSequence, NamedTuple, Optional from qiling import Qiling -Image = NamedTuple('Image', (('base', int), ('end', int), ('path', str))) +class Image(NamedTuple): + base: int + end: int + path: str class QlLoader: def __init__(self, ql: Qiling): @@ -25,6 +29,20 @@ class QlLoader: return next((image for image in self.images if image.base <= address < image.end), None) + def get_image_by_name(self, name: str, *, casefold: bool = False) -> Optional[Image]: + """Retrieve an image by its basename. + + Args: + name : image base name + casefold : whether name matching should be case-insensitive (default is case-sensitive) + + Returns: image object whose basename match to the one given, or `None` if not found + """ + + cf = str.casefold if casefold else lambda s: s + + return next((image for image in self.images if cf(os.path.basename(image.path)) == cf(name)), None) + def save(self) -> Mapping[str, Any]: saved_state = { 'images': [tuple(img) for img in self.images] diff --git a/qiling/loader/macho_parser/const.py b/qiling/loader/macho_parser/const.py index edbd3ded..0726f374 100644 --- a/qiling/loader/macho_parser/const.py +++ b/qiling/loader/macho_parser/const.py @@ -45,6 +45,7 @@ LC_DYLD_INFO_ONLY = 0x80000022 LC_LOAD_DYLINKER = 0x0000000E LC_MAIN = 0x80000028 LC_LOAD_DYLIB = 0x0000000C +LC_LOAD_WEAK_DYLIB = 0x80000018 LC_ENCRYPTION_INFO_64 = 0x0000002C LC_BUILD_VERSION = 0x00000032 LC_DYLD_EXPORTS_TRIE = 0x80000033 diff --git a/qiling/loader/macho_parser/loadcommand.py b/qiling/loader/macho_parser/loadcommand.py index e13832bd..1eb18d36 100644 --- a/qiling/loader/macho_parser/loadcommand.py +++ b/qiling/loader/macho_parser/loadcommand.py @@ -39,6 +39,7 @@ class LoadCommand: LC_LOAD_DYLINKER : LoadDylinker, LC_MAIN : LoadMain, LC_LOAD_DYLIB : LoadDyLib, + LC_LOAD_WEAK_DYLIB : LoadWeakDyLib, LC_ENCRYPTION_INFO_64 : LoadEncryptionInfo64, LC_DYLD_EXPORTS_TRIE : LoadDyldExportTrie, LC_DYLD_CHAINED_FIXUPS : LoadDyldChainedFixups, @@ -255,6 +256,11 @@ class LoadDyLib(LoadCommand): pass +class LoadWeakDyLib(LoadDyLib): + def __init__(self, data): + super().__init__(data) + + class LoadUnixThread(LoadCommand): def __init__(self, data): diff --git a/qiling/loader/mcu.py b/qiling/loader/mcu.py index 054e073d..8a91d633 100644 --- a/qiling/loader/mcu.py +++ b/qiling/loader/mcu.py @@ -9,7 +9,6 @@ from elftools.elf.elffile import ELFFile from qiling.const import * from qiling.core import Qiling -from qiling.utils import component_setup from .loader import QlLoader @@ -63,8 +62,6 @@ class QlLoaderMCU(QlLoader): self.load_address = 0 self.filetype = self.guess_filetype() - self.ql._hw = component_setup("hw", "hw", self.ql) - if self.filetype == 'elf': with open(self.ql.path, 'rb') as infile: self.elf = ELFFile(io.BytesIO(infile.read())) diff --git a/qiling/loader/pe.py b/qiling/loader/pe.py index 2a27fd63..1b05e6d1 100644 --- a/qiling/loader/pe.py +++ b/qiling/loader/pe.py @@ -3,95 +3,136 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import os, pefile, pickle, secrets, traceback -from typing import Any, MutableMapping, Optional, Mapping, Sequence +import os, pefile, pickle, secrets, ntpath +from typing import Any, Dict, MutableMapping, NamedTuple, Optional, Mapping, Sequence, Tuple, Union + +from unicorn import UcError +from unicorn.x86_const import UC_X86_REG_CR4, UC_X86_REG_CR8 from qiling import Qiling -from qiling.arch.x86_const import * -from qiling.const import * +from qiling.arch.x86_const import FS_SEGMENT_ADDR, GS_SEGMENT_ADDR +from qiling.const import QL_ARCH +from qiling.exception import QlErrorArch from qiling.os.const import POINTER -from qiling.os.memory import QlMemoryHeap +from qiling.os.windows.api import HINSTANCE, DWORD, LPVOID from qiling.os.windows.fncc import CDECL -from qiling.os.windows.utils import * +from qiling.os.windows.utils import has_lib_ext from qiling.os.windows.structs import * from .loader import QlLoader, Image -class QlPeCacheEntry: - def __init__(self, ba: int, data: bytearray, cmdlines: Sequence, import_symbols: Mapping, import_table: Mapping): - self. ba = ba - self.data = data - self.cmdlines = cmdlines - self.import_symbols = import_symbols - self.import_table = import_table +class QlPeCacheEntry(NamedTuple): + ba: int + data: bytearray + cmdlines: Sequence + import_symbols: MutableMapping[int, dict] + import_table: MutableMapping[Union[str, int], int] + -# A default simple cache implementation class QlPeCache: - def create_filename(self, path: str) -> str: + @staticmethod + def cache_filename(path: str) -> str: + dirname, basename = os.path.split(path) + + # canonicalize basename while preserving the path + path = os.path.join(dirname, basename.casefold()) + return f'{path}.cache2' def restore(self, path: str) -> Optional[QlPeCacheEntry]: - fcache = self.create_filename(path) + fcache = QlPeCache.cache_filename(path) - # pickle file cannot be outdated + # check whether cache file exists and it is not older than the cached file itself if os.path.exists(fcache) and os.stat(fcache).st_mtime > os.stat(path).st_mtime: with open(fcache, "rb") as fcache_file: return QlPeCacheEntry(*pickle.load(fcache_file)) return None - def save(self, path: str, entry: QlPeCacheEntry): - fcache = self.create_filename(path) + def save(self, path: str, entry: QlPeCacheEntry) -> None: + fcache = QlPeCache.cache_filename(path) - data = (entry.ba, entry.data, entry.cmdlines, entry.import_symbols, entry.import_table) # cache this dll file with open(fcache, "wb") as fcache_file: - pickle.dump(data, fcache_file) + pickle.dump(entry, fcache_file) -class Process(): + +class Process: # let linter recognize mixin members + cmdline: bytes + pe_image_address: int + stack_address: int + stack_size: int + dlls: MutableMapping[str, int] - import_address_table: MutableMapping[str, Any] - import_symbols: MutableMapping[int, Any] - export_symbols: MutableMapping[int, Any] + import_address_table: MutableMapping[str, Mapping] + import_symbols: MutableMapping[int, Dict[str, Any]] + export_symbols: MutableMapping[int, Dict[str, Any]] libcache: Optional[QlPeCache] def __init__(self, ql: Qiling): self.ql = ql - def align(self, size: int, unit: int) -> int: - return (size // unit + (1 if size % unit else 0)) * unit + def __get_path_elements(self, name: str) -> Tuple[str, str]: + """Translate DLL virtual path into host path. - def load_dll(self, name: bytes, driver: bool = False) -> int: - dll_name = name.decode() + Args: + name: dll virtual path; either absolute or relative - self.ql.dlls = os.path.join("Windows", "System32") + Returns: dll path on the host and dll basename in a canonicalized form + """ - if dll_name.upper().startswith('C:\\'): - path = self.ql.os.path.transform_to_real_path(dll_name) - dll_name = path_leaf(dll_name) - else: - dll_name = dll_name.lower() + dirname, basename = ntpath.split(name) - if not is_file_library(dll_name): - dll_name = dll_name + ".dll" + if not has_lib_ext(basename): + basename = f'{basename}.dll' - path = os.path.join(self.ql.rootfs, self.ql.dlls, dll_name) + # if only filename was specified assume it is located at the + # system32 folder to prevent potential dll hijacking + if not dirname: + dirname = self.ql.os.winsys - if not os.path.exists(path): - self.ql.log.error("Cannot find dll in %s" % path) + # reconstruct the dll virtual path + vpath = ntpath.join(dirname, basename) + + return self.ql.os.path.virtual_to_host_path(vpath), basename.casefold() + + def load_dll(self, name: str, is_driver: bool = False) -> int: + dll_path, dll_name = self.__get_path_elements(name) + + if dll_name.startswith('api-ms-win-'): + # Usually we should not reach this point and instead imports from such DLLs should be redirected earlier + self.ql.log.debug(f'Refusing to load virtual DLL {dll_name}') return 0 - # If the dll is already loaded - if dll_name in self.dlls: - return self.dlls[dll_name] + # see if this dll was already loaded + image = self.get_image_by_name(dll_name, casefold=True) - self.ql.log.info(f'Loading {path} ...') + if image is not None: + return image.base + + if not os.path.exists(dll_path): + # posix hosts may not find the requested filename if it was saved under a different case. + # For example, 'KernelBase.dll' may not be found because it is stored as 'kernelbase.dll'. + # + # try to locate the requested file while ignoring the case of its path elements. + dll_casefold_path = self.ql.os.path.host_casefold_path(dll_path) + + if dll_casefold_path is None: + self.ql.log.error(f'Could not find DLL file: {dll_path}') + return 0 + + dll_path = dll_casefold_path + + self.ql.log.info(f'Loading {dll_name} ...') + + import_symbols = {} + import_table = {} cached = None loaded = False if self.libcache: - cached = self.libcache.restore(path) + cached = self.libcache.restore(dll_path) if cached: data = cached.data @@ -109,72 +150,72 @@ class Process(): for entry in cached.cmdlines: self.set_cmdline(entry['name'], entry['address'], data) - self.ql.log.info(f'Loaded {path} from cache') + self.ql.log.info(f'Loaded {dll_name} from cache') loaded = True # either file was not cached, or could not be loaded to the same location in memory if not cached or not loaded: - dll = pefile.PE(path, fast_load=True) + dll = pefile.PE(dll_path, fast_load=True) dll.parse_data_directories() warnings = dll.get_warnings() if warnings: - self.ql.log.warning(f'Warnings while loading {path}:') + self.ql.log.debug(f'Warnings while loading {dll_name}:') for warning in warnings: - self.ql.log.warning(f' - {warning}') - - data = bytearray(dll.get_memory_mapped_image()) + self.ql.log.debug(f' - {warning}') image_base = dll.OPTIONAL_HEADER.ImageBase or self.dll_last_address - image_size = self.ql.mem.align_up(len(data)) + image_size = self.ql.mem.align_up(dll.OPTIONAL_HEADER.SizeOfImage) + relocate = False self.ql.log.debug(f'DLL preferred base address: {image_base:#x}') if (image_base + image_size) > self.ql.mem.max_mem_addr: image_base = self.dll_last_address self.ql.log.debug(f'DLL preferred base address exceeds memory upper bound, loading to: {image_base:#x}') + relocate = True if not self.ql.mem.is_available(image_base, image_size): image_base = self.ql.mem.find_free_space(image_size, minaddr=image_base, align=0x10000) self.ql.log.debug(f'DLL preferred base address is taken, loading to: {image_base:#x}') + relocate = True + + if relocate: + with ShowProgress(0.1337): + dll.relocate_image(image_base) + + data = bytearray(dll.get_memory_mapped_image()) + assert image_size >= len(data) cmdlines = [] - import_symbols = {} - import_table = {} - dll_symbols = getattr(getattr(dll, 'DIRECTORY_ENTRY_EXPORT', None), 'symbols', []) - for entry in dll_symbols: - import_symbols[image_base + entry.address] = { - "name": entry.name, - "ordinal": entry.ordinal, - "dll": dll_name.split('.')[0] + for sym in dll.DIRECTORY_ENTRY_EXPORT.symbols: + ea = image_base + sym.address + + import_symbols[ea] = { + 'name' : sym.name, + 'ordinal' : sym.ordinal, + 'dll' : dll_name.split('.')[0] } - if entry.name: - import_table[entry.name] = image_base + entry.address + if sym.name: + import_table[sym.name] = ea - import_table[entry.ordinal] = image_base + entry.address - cmdline_entry = self.set_cmdline(entry.name, entry.address, data) + import_table[sym.ordinal] = ea + cmdline_entry = self.set_cmdline(sym.name, sym.address, data) if cmdline_entry: cmdlines.append(cmdline_entry) if self.libcache: cached = QlPeCacheEntry(image_base, data, cmdlines, import_symbols, import_table) - self.libcache.save(path, cached) - self.ql.log.info("Cached %s" % path) + self.libcache.save(dll_path, cached) + self.ql.log.info(f'Cached {dll_name}') # Add dll to IAT - try: - self.import_address_table[dll_name] = import_table - except Exception as ex: - self.ql.log.exception(f'Unable to add {dll_name} to IAT') - - try: - self.import_symbols.update(import_symbols) - except Exception as ex: - self.ql.log.exception(f'Unable to add {dll_name} import symbols') + self.import_address_table[dll_name] = import_table + self.import_symbols.update(import_symbols) dll_base = image_base dll_len = image_size @@ -183,500 +224,671 @@ class Process(): self.ql.mem.map(dll_base, dll_len, info=dll_name) self.ql.mem.write(dll_base, bytes(data)) - self.dlls[dll_name] = dll_base - if dll_base == self.dll_last_address: - self.dll_last_address += dll_len - - # if this is NOT a driver, add dll to ldr data - if not driver: - self.add_ldr_data_table_entry(dll_name) + self.dll_last_address = self.ql.mem.align_up(self.dll_last_address + dll_len, 0x10000) # add DLL to coverage images - self.images.append(Image(dll_base, dll_base + dll_len, path)) + self.images.append(Image(dll_base, dll_base + dll_len, dll_path)) - self.ql.log.info(f'Done with loading {path}') + # if this is NOT a driver, add dll to ldr data + if not is_driver: + self.add_ldr_data_table_entry(dll_name) + + if not cached or not loaded: + # parse directory entry import + self.ql.log.debug(f'Init imports for {dll_name}') + self.init_imports(dll, is_driver) + + # calling DllMain is essential for dlls to initialize properly. however + # DllMain of system libraries may fail due to incomplete or inaccurate + # mock implementation. due to unicorn limitations, recovering from such + # errors may be possible only if the function was not invoked from within + # a hook. + # + # in case of a dll loaded from a hooked API call, failures would not be + # recoverable and we have to give up its DllMain. + if not self.ql.os.PE_RUN: + self.call_dll_entrypoint(dll, dll_base, dll_len, dll_name) + + self.ql.log.info(f'Done loading {dll_name}') return dll_base + def call_dll_entrypoint(self, dll: pefile.PE, dll_base: int, dll_len: int, dll_name: str): + entry_address = dll.OPTIONAL_HEADER.AddressOfEntryPoint - def _alloc_cmdline(self, wide): - addr = self.ql.os.heap.alloc(len(self.cmdline) * (2 if wide else 1)) - packed_addr = self.ql.pack(addr) - return addr, packed_addr + if dll.get_section_by_rva(entry_address) is None: + return - def set_cmdline(self, name, address, memory): - cmdline_entry = None - if name == b"_acmdln": - addr, packed_addr = self._alloc_cmdline(wide=False) - cmdline_entry = {"name": name, "address": address} - memory[address:address + self.ql.arch.pointersize] = packed_addr - self.ql.mem.write(addr, self.cmdline) - elif name == b"_wcmdln": - addr, packed_addr = self._alloc_cmdline(wide=True) - cmdline_entry = {"name": name, "address": address} - memory[address:address + self.ql.arch.pointersize] = packed_addr - encoded = self.cmdline.decode('ascii').encode('UTF-16LE') - self.ql.mem.write(addr, encoded) + if dll_name in ('kernelbase.dll', 'kernel32.dll'): + self.ql.log.debug(f'Ignoring {dll_name} entry point') + return - return cmdline_entry + # DllMain functions often call many APIs that may crash the program if they + # are not implemented correctly (if at all). here we blacklist the problematic + # DLLs whose DllMain functions are known to be crashing. + # + # the blacklist may be revisited from time to time to see if any of the file + # can be safely unlisted. + blacklist = { + 32 : ('gdi32.dll',), + 64 : ('gdi32.dll',) + }[self.ql.arch.bits] - def init_tib(self): - if self.ql.arch.type == QL_ARCH.X86: - teb_addr = self.structure_last_addr + if dll_name in blacklist: + self.ql.log.debug(f'Ignoring {dll_name} entry point (blacklisted)') + return + + entry_point = dll_base + entry_address + exit_point = dll_base + dll_len - 16 + + args = ( + (HINSTANCE, dll_base), # hinstDLL = base address of DLL + (DWORD, 1), # fdwReason = DLL_PROCESS_ATTACH + (LPVOID, 0) # lpReserved = 0 + ) + + self.ql.log.info(f'Calling {dll_name} DllMain at {entry_point:#x}') + + regs_state = self.ql.arch.regs.save() + + fcall = self.ql.os.fcall_select(CDECL) + fcall.call_native(entry_point, args, exit_point) + + # Execute the call to the entry point + try: + self.ql.emu_start(entry_point, exit_point) + except UcError: + self.ql.log.error(f'Error encountered while running {dll_name} DllMain, bailing') + + self.ql.arch.regs.restore(regs_state) else: - gs = self.structure_last_addr - self.structure_last_addr += 0x30 - teb_addr = self.structure_last_addr + fcall.cc.unwind(len(args)) - self.ql.log.info("TEB addr is 0x%x" %teb_addr) + self.ql.log.info(f'Returned from {dll_name} DllMain') - teb_size = len(TEB(self.ql).bytes()) - teb_data = TEB( - self.ql, - base=teb_addr, - peb_address=teb_addr + teb_size, - stack_base=self.stack_address + self.stack_size, - stack_limit=self.stack_size, - Self=teb_addr) + def set_cmdline(self, name: bytes, address: int, memory: bytearray): + cmdln = { + b'_acmdln' : 1, + b'_wcmdln' : 2 + } - self.ql.mem.write(teb_addr, teb_data.bytes()) + clen = cmdln.get(name, None) - self.structure_last_addr += teb_size - if self.ql.arch.type == QL_ARCH.X8664: - # TEB - self.ql.mem.write_ptr(gs + 0x30, teb_addr) - # PEB - self.ql.mem.write_ptr(gs + 0x60, teb_addr + teb_size) + if clen is None: + return None - self.TEB = self.ql.TEB = teb_data + addr = self.ql.os.heap.alloc(len(self.cmdline) * clen) + memory[address:address + self.ql.arch.pointersize] = self.ql.pack(addr) + data = self.cmdline + + if clen == 2: + data = data.decode('ascii').encode('UTF-16LE') + + self.ql.mem.write(addr, data) + + return {"name": name, "address": address} + + def init_teb(self): + teb_obj = make_teb(self.ql.arch.bits) + + teb_addr = self.structure_last_addr + peb_addr = self.ql.mem.align_up(teb_addr + ctypes.sizeof(teb_obj), 0x10) + + teb_obj.StackBase = self.stack_address + self.stack_size + teb_obj.StackLimit = self.stack_address + teb_obj.TebAddress = teb_addr + teb_obj.PebAddress = peb_addr + + self.ql.log.info(f'TEB is at {teb_addr:#x}') + self.ql.mem.write(teb_addr, bytes(teb_obj)) + + self.structure_last_addr = peb_addr + self.TEB = teb_obj def init_peb(self): - peb_addr = self.structure_last_addr + peb_obj = make_peb(self.ql.arch.bits) - self.ql.log.info("PEB addr is 0x%x" % peb_addr) + peb_addr = self.structure_last_addr + ldr_addr = self.ql.mem.align_up(peb_addr + ctypes.sizeof(peb_obj), 0x10) # we must set an heap, will try to retrieve this value. Is ok to be all \x00 - process_heap = self.ql.os.heap.alloc(0x100) - process_parameters = self.ql.os.heap.alloc(0x100) - peb_data = PEB( - self.ql, - base=peb_addr, - process_heap=process_heap, - process_parameters=process_parameters, - number_processors=self.ql.os.profile.getint("HARDWARE", "number_processors")) - peb_data.LdrAddress = peb_addr + peb_data.size - peb_data.write(peb_addr) - self.structure_last_addr += peb_data.size - self.PEB = self.ql.PEB = peb_data + peb_obj.LdrAddress = ldr_addr + peb_obj.ProcessParameters = self.ql.os.heap.alloc(0x100) + peb_obj.ProcessHeap = self.ql.os.heap.alloc(0x100) + peb_obj.NumberOfProcessors = self.ql.os.profile.getint('HARDWARE', 'number_processors') + + self.ql.log.info(f'PEB is at {peb_addr:#x}') + self.ql.mem.write(peb_addr, bytes(peb_obj)) + + self.structure_last_addr = ldr_addr + self.PEB = peb_obj def init_ldr_data(self): - ldr_addr = self.structure_last_addr - ldr_size = len(LdrData(self.ql).bytes()) - ldr_data = LdrData( - self.ql, - base=ldr_addr, - in_load_order_module_list={ - 'Flink': ldr_addr + 2 * self.ql.arch.pointersize, - 'Blink': ldr_addr + 2 * self.ql.arch.pointersize - }, - in_memory_order_module_list={ - 'Flink': ldr_addr + 4 * self.ql.arch.pointersize, - 'Blink': ldr_addr + 4 * self.ql.arch.pointersize - }, - in_initialization_order_module_list={ - 'Flink': ldr_addr + 6 * self.ql.arch.pointersize, - 'Blink': ldr_addr + 6 * self.ql.arch.pointersize - } - ) - self.ql.mem.write(ldr_addr, ldr_data.bytes()) - self.structure_last_addr += ldr_size - self.LDR = self.ql.LDR = ldr_data + ldr_obj = make_ldr_data(self.ql.arch.bits) + ldr_cls = ldr_obj.__class__ - def add_ldr_data_table_entry(self, dll_name): - dll_base = self.dlls[dll_name] - path = "C:\\Windows\\System32\\" + dll_name - ldr_table_entry_size = len(LdrDataTableEntry(self.ql).bytes()) - base = self.ql.os.heap.alloc(ldr_table_entry_size) - ldr_table_entry = LdrDataTableEntry(self.ql, - base=base, - in_load_order_links={'Flink': 0, 'Blink': 0}, - in_memory_order_links={'Flink': 0, 'Blink': 0}, - in_initialization_order_links={'Flink': 0, 'Blink': 0}, - dll_base=dll_base, - entry_point=0, - full_dll_name=path, - base_dll_name=dll_name) + ldr_addr = self.structure_last_addr + nobj_addr = self.ql.mem.align_up(ldr_addr + ctypes.sizeof(ldr_obj), 0x10) + + ldr_obj.InLoadOrderModuleList.Flink = ldr_addr + ldr_cls.InLoadOrderModuleList.offset + ldr_obj.InLoadOrderModuleList.Blink = ldr_addr + ldr_cls.InLoadOrderModuleList.offset + + ldr_obj.InMemoryOrderModuleList.Flink = ldr_addr + ldr_cls.InMemoryOrderModuleList.offset + ldr_obj.InMemoryOrderModuleList.Blink = ldr_addr + ldr_cls.InMemoryOrderModuleList.offset + + ldr_obj.InInitializationOrderModuleList.Flink = ldr_addr + ldr_cls.InInitializationOrderModuleList.offset + ldr_obj.InInitializationOrderModuleList.Blink = ldr_addr + ldr_cls.InInitializationOrderModuleList.offset + + self.ql.log.info(f'LDR is at {ldr_addr:#x}') + self.ql.mem.write(ldr_addr, bytes(ldr_obj)) + + self.structure_last_addr = nobj_addr + self.LDR = ldr_obj + + def add_ldr_data_table_entry(self, dll_name: str): + entry_obj = make_ldr_data_table_entry(self.ql.arch.bits) + entry_cls = entry_obj.__class__ + + entry_size = ctypes.sizeof(entry_obj) + entry_addr = self.ql.os.heap.alloc(entry_size) + + def populate_unistr(obj, s: str) -> None: + encoded = s.encode('utf-16le') + ucslen = len(encoded) + ucsbuf = self.ql.os.heap.alloc(ucslen + 2) + + self.ql.mem.write(ucsbuf, encoded + b'\x00\x00') + + obj.Length = ucslen + obj.MaximumLength = ucslen + 2 + obj.Buffer = ucsbuf + + image = self.get_image_by_name(dll_name, casefold=True) + assert image, 'image should have been added to loader.images first' + + entry_obj.DllBase = image.base + populate_unistr(entry_obj.FullDllName, ntpath.join(self.ql.os.winsys, dll_name)) + populate_unistr(entry_obj.BaseDllName, dll_name) # Flink - if len(self.ldr_list) == 0: - flink = self.LDR - ldr_table_entry.InLoadOrderLinks['Flink'] = flink.InLoadOrderModuleList['Flink'] - ldr_table_entry.InMemoryOrderLinks['Flink'] = flink.InMemoryOrderModuleList['Flink'] - ldr_table_entry.InInitializationOrderLinks['Flink'] = flink.InInitializationOrderModuleList['Flink'] + if self.ldr_list: + flink_base, flink = self.ldr_list[-1] - flink.InLoadOrderModuleList['Flink'] = ldr_table_entry.base - flink.InMemoryOrderModuleList['Flink'] = ldr_table_entry.base + 2 * self.ql.arch.pointersize - flink.InInitializationOrderModuleList['Flink'] = ldr_table_entry.base + 4 * self.ql.arch.pointersize + entry_obj.InLoadOrderLinks.Flink = flink.InLoadOrderLinks.Flink + entry_obj.InMemoryOrderLinks.Flink = flink.InMemoryOrderLinks.Flink + entry_obj.InInitializationOrderLinks.Flink = flink.InInitializationOrderLinks.Flink + + flink.InLoadOrderLinks.Flink = entry_addr + entry_cls.InLoadOrderLinks.offset + flink.InMemoryOrderLinks.Flink = entry_addr + entry_cls.InMemoryOrderLinks.offset + flink.InInitializationOrderLinks.Flink = entry_addr + entry_cls.InInitializationOrderLinks.offset else: - flink = self.ldr_list[-1] - ldr_table_entry.InLoadOrderLinks['Flink'] = flink.InLoadOrderLinks['Flink'] - ldr_table_entry.InMemoryOrderLinks['Flink'] = flink.InMemoryOrderLinks['Flink'] - ldr_table_entry.InInitializationOrderLinks['Flink'] = flink.InInitializationOrderLinks['Flink'] + flink_base, flink = (self.PEB.LdrAddress, self.LDR) - flink.InLoadOrderLinks['Flink'] = ldr_table_entry.base - flink.InMemoryOrderLinks['Flink'] = ldr_table_entry.base + 2 * self.ql.arch.pointersize - flink.InInitializationOrderLinks['Flink'] = ldr_table_entry.base + 4 * self.ql.arch.pointersize + entry_obj.InLoadOrderLinks.Flink = flink.InLoadOrderModuleList.Flink + entry_obj.InMemoryOrderLinks.Flink = flink.InMemoryOrderModuleList.Flink + entry_obj.InInitializationOrderLinks.Flink = flink.InInitializationOrderModuleList.Flink + + flink.InLoadOrderModuleList.Flink = entry_addr + entry_cls.InLoadOrderLinks.offset + flink.InMemoryOrderModuleList.Flink = entry_addr + entry_cls.InMemoryOrderLinks.offset + flink.InInitializationOrderModuleList.Flink = entry_addr + entry_cls.InInitializationOrderLinks.offset # Blink + blink_base = self.PEB.LdrAddress blink = self.LDR - ldr_table_entry.InLoadOrderLinks['Blink'] = blink.InLoadOrderModuleList['Blink'] - ldr_table_entry.InMemoryOrderLinks['Blink'] = blink.InMemoryOrderModuleList['Blink'] - ldr_table_entry.InInitializationOrderLinks['Blink'] = blink.InInitializationOrderModuleList['Blink'] - blink.InLoadOrderModuleList['Blink'] = ldr_table_entry.base - blink.InMemoryOrderModuleList['Blink'] = ldr_table_entry.base + 2 * self.ql.arch.pointersize - blink.InInitializationOrderModuleList['Blink'] = ldr_table_entry.base + 4 * self.ql.arch.pointersize + entry_obj.InLoadOrderLinks.Blink = blink.InLoadOrderModuleList.Blink + entry_obj.InMemoryOrderLinks.Blink = blink.InMemoryOrderModuleList.Blink + entry_obj.InInitializationOrderLinks.Blink = blink.InInitializationOrderModuleList.Blink - self.ql.mem.write(flink.base, flink.bytes()) - self.ql.mem.write(blink.base, blink.bytes()) - self.ql.mem.write(ldr_table_entry.base, ldr_table_entry.bytes()) + blink.InLoadOrderModuleList.Blink = entry_addr + entry_cls.InLoadOrderLinks.offset + blink.InMemoryOrderModuleList.Blink = entry_addr + entry_cls.InMemoryOrderLinks.offset + blink.InInitializationOrderModuleList.Blink = entry_addr + entry_cls.InInitializationOrderLinks.offset - self.ldr_list.append(ldr_table_entry) + self.ql.mem.write(flink_base, bytes(flink)) + self.ql.mem.write(blink_base, bytes(blink)) + self.ql.mem.write(entry_addr, bytes(entry_obj)) - def init_exports(self): - if self.ql.code: - return - if self.pe.OPTIONAL_HEADER.DATA_DIRECTORY[pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_EXPORT']].VirtualAddress != 0: - # Do a full load if IMAGE_DIRECTORY_ENTRY_EXPORT is present so we can load the exports - self.pe.full_load() - else: + self.ldr_list.append((entry_addr, entry_obj)) + + @staticmethod + def directory_exists(pe: pefile.PE, entry: str) -> bool: + ent = pefile.DIRECTORY_ENTRY[entry] + + return pe.OPTIONAL_HEADER.DATA_DIRECTORY[ent].VirtualAddress != 0 + + def init_imports(self, pe: pefile.PE, is_driver: bool): + if not Process.directory_exists(pe, 'IMAGE_DIRECTORY_ENTRY_IMPORT'): return - try: - # parse directory entry export - dll_name = os.path.basename(self.path) - self.import_address_table[dll_name] = {} - for entry in self.pe.DIRECTORY_ENTRY_EXPORT.symbols: - self.export_symbols[self.pe_image_address + entry.address] = {'name': entry.name, 'ordinal': entry.ordinal} - self.import_address_table[dll_name][entry.name] = self.pe_image_address + entry.address - self.import_address_table[dll_name][entry.ordinal] = self.pe_image_address + entry.address - except: - self.ql.log.info('Failed to load exports for %s:\n%s' % (self.ql.argv, traceback.format_exc())) + pe.full_load() + + for entry in pe.DIRECTORY_ENTRY_IMPORT: + dll_name = entry.dll.decode().casefold() + self.ql.log.debug(f'Requesting imports from {dll_name}') + + orig_dll_name = dll_name + redirected = False + + if dll_name.startswith('api-ms-win-'): + # DLLs starting with this prefix contain no actual code. Instead, the windows loader loads the actual + # code from one of the main windows dlls. + # see https://github.com/lucasg/Dependencies for correct replacement dlls + # + # The correct way to find the dll that replaces all symbols from this dll involves using the hashmap + # inside of apisetschema.dll (see https://lucasg.github.io/2017/10/15/Api-set-resolution/ ). + # + # Currently, we use a simpler, more hacky approach, that seems to work in a lot of cases: we just scan + # through some key dlls and hope that we find the requested symbols there. some symbols may appear on + # more than one dll though; in that case we proceed to the next symbol to see which key dll includes it. + # + # Note: You might be tempted to load the actual dll (dll_name), because they also contain a reference to + # the replacement dll. However, chances are, that these dlls do not exist in the rootfs and maybe they + # don't even exist on windows. Therefore this approach is a bad idea. + + # DLLs that seem to contain most of the requested symbols + key_dlls = ( + 'ntdll.dll', + 'kernelbase.dll', + 'ucrtbase.dll' + ) + + imports = iter(entry.imports) + failed = False + fallback = None + + while not redirected and not failed: + # find all possible redirection options by scanning key dlls for the current imported symbol + imp = next(imports, None) + redirection_options = [fallback] if imp is None else [filename for filename in key_dlls if filename in self.import_address_table and imp.name in self.import_address_table[filename]] + + # no redirection options: failed to redirect dll + if not redirection_options: + failed = True + + # exactly one redirection options: use it + elif len(redirection_options) == 1: + key_dll = redirection_options[0] + redirected = True + + # more than one redirection options: remember one of them and proceed to next symbol + else: + fallback = redirection_options[-1] + + if not redirected: + self.ql.log.warning(f'Failed to resolve {dll_name}') + continue + + self.ql.log.debug(f'Redirecting {dll_name} to {key_dll}') + dll_name = key_dll + + unbound_imports = [imp for imp in entry.imports if not imp.bound] + + if unbound_imports: + # Only load dll if encountered unbound symbol + if not redirected: + dll_base = self.load_dll(entry.dll.decode(), is_driver) + + if not dll_base: + continue + + for imp in unbound_imports: + iat = self.import_address_table[dll_name] + + if imp.name: + if imp.name not in iat: + self.ql.log.debug(f'Error in loading function {imp.name.decode()} ({orig_dll_name}){", probably misdirected" if redirected else ""}') + continue + + addr = iat[imp.name] + else: + addr = iat[imp.ordinal] + + self.ql.mem.write_ptr(imp.address, addr) + + def init_exports(self, pe: pefile.PE): + if not Process.directory_exists(pe, 'IMAGE_DIRECTORY_ENTRY_EXPORT'): + return + + # Do a full load if IMAGE_DIRECTORY_ENTRY_EXPORT is present so we can load the exports + pe.full_load() + + iat = {} + + # parse directory entry export + for entry in pe.DIRECTORY_ENTRY_EXPORT.symbols: + ea = self.pe_image_address + entry.address + + self.export_symbols[ea] = { + 'name' : entry.name, + 'ordinal' : entry.ordinal + } + + if entry.name: + iat[entry.name] = ea + + iat[entry.ordinal] = ea + + dll_name = os.path.basename(self.path) + self.import_address_table[dll_name.casefold()] = iat def init_driver_object(self): + drv_addr = self.structure_last_addr + # PDRIVER_OBJECT DriverObject - driver_object_addr = self.structure_last_addr - self.ql.log.info("Driver object addr is 0x%x" %driver_object_addr) + drv_obj = make_driver_object(self.ql, drv_addr, self.ql.arch.bits) + nobj_addr = self.ql.mem.align_up(drv_addr + ctypes.sizeof(drv_obj), 0x10) - if self.ql.arch.type == QL_ARCH.X86: - self.driver_object = DRIVER_OBJECT32(self.ql, driver_object_addr) - elif self.ql.arch.type == QL_ARCH.X8664: - self.driver_object = DRIVER_OBJECT64(self.ql, driver_object_addr) - - driver_object_size = ctypes.sizeof(self.driver_object) - self.ql.mem.write(driver_object_addr, bytes(self.driver_object)) - self.structure_last_addr += driver_object_size - self.driver_object_address = driver_object_addr + self.ql.log.info(f'DriverObject is at {drv_addr:#x}') + self.ql.mem.write(drv_addr, bytes(drv_obj)) + self.structure_last_addr = nobj_addr + self.driver_object_address = drv_addr + self.driver_object = drv_obj def init_registry_path(self): + regpath_addr = self.structure_last_addr + # PUNICODE_STRING RegistryPath - regitry_path_addr = self.structure_last_addr - self.ql.log.info("Registry path addr is 0x%x" %regitry_path_addr) + regpath_obj = make_unicode_string(self.ql.arch.bits, + Length=0, + MaximumLength=0, + Buffer=regpath_addr + ) - if self.ql.arch.type == QL_ARCH.X86: - regitry_path_data = UNICODE_STRING32(0, 0, regitry_path_addr) - elif self.ql.arch.type == QL_ARCH.X8664: - regitry_path_data = UNICODE_STRING64(0, 0, regitry_path_addr) + nobj_addr = self.ql.mem.align_up(regpath_addr + ctypes.sizeof(regpath_obj), 0x10) - regitry_path_size = ctypes.sizeof(regitry_path_data) - self.ql.mem.write(regitry_path_addr, bytes(regitry_path_data)) - self.structure_last_addr += regitry_path_size - self.regitry_path_address = regitry_path_addr + self.ql.log.info(f'RegistryPath is at {regpath_addr:#x}') + self.ql.mem.write(regpath_addr, bytes(regpath_obj)) + self.structure_last_addr = nobj_addr + self.regitry_path_address = regpath_addr def init_eprocess(self): - addr = self.structure_last_addr - self.ql.log.info("EPROCESS is is 0x%x" %addr) + eproc_obj = make_eprocess(self.ql.arch.bits) + eproc_addr = self.structure_last_addr + nobj_addr = self.ql.mem.align_up(eproc_addr + ctypes.sizeof(eproc_obj), 0x10) - if self.ql.arch.type == QL_ARCH.X86: - self.eprocess_object = EPROCESS32(self.ql, addr) - elif self.ql.arch.type == QL_ARCH.X8664: - self.eprocess_object = EPROCESS64(self.ql, addr) - - size = ctypes.sizeof(self.eprocess_object) - self.ql.mem.write(addr, bytes(self.driver_object)) - self.structure_last_addr += size - self.ql.eprocess_address = addr + self.ql.mem.write(eproc_addr, bytes(eproc_obj)) + self.structure_last_addr = nobj_addr + self.eprocess_address = eproc_addr def init_ki_user_shared_data(self): - ''' - https://www.geoffchappell.com/studies/windows/km/ntoskrnl/structs/kuser_shared_data/index.htm + addr = self.ql.os.profile.getint(f'OS{self.ql.arch.bits}', 'KI_USER_SHARED_DATA') - struct information: - https://doxygen.reactos.org/d8/dae/modules_2rostests_2winetests_2ntdll_2time_8c_source.html - ''' - if self.ql.arch.type == QL_ARCH.X86: - KI_USER_SHARED_DATA = 0xFFDF0000 - elif self.ql.arch.type == QL_ARCH.X8664: - KI_USER_SHARED_DATA = 0xFFFFF78000000000 + user_shared_data_obj = KUSER_SHARED_DATA() + user_shared_data_size = ctypes.sizeof(KUSER_SHARED_DATA) - self.ql.log.info("KI_USER_SHARED_DATA is 0x%x" %KI_USER_SHARED_DATA) + self.ql.mem.map(addr, self.ql.mem.align_up(user_shared_data_size)) + self.ql.mem.write(addr, bytes(user_shared_data_obj)) - shared_user_data = KUSER_SHARED_DATA() + def init_security_cookie(self, pe: pefile.PE, image_base: int): + if not Process.directory_exists(pe, 'IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG'): + return - shared_user_data_len = self.align(ctypes.sizeof(KUSER_SHARED_DATA), 0x1000) - self.ql.mem.map(KI_USER_SHARED_DATA, shared_user_data_len) - self.ql.mem.write(KI_USER_SHARED_DATA, bytes(shared_user_data)) + cookie_rva = pe.DIRECTORY_ENTRY_LOAD_CONFIG.struct.SecurityCookie - pe.OPTIONAL_HEADER.ImageBase + # get a random cookie value but keep the two most significant bytes zeroes + # + # rol rcx, 10h ; rcx = cookie + # test cx, 0FFFFh + cookie = secrets.randbits(self.ql.arch.bits - 16) + + self.ql.mem.write_ptr(cookie_rva + image_base, cookie) class QlLoaderPE(QlLoader, Process): def __init__(self, ql: Qiling, libcache: bool): super().__init__(ql) - self.ql = ql - self.path = self.ql.path - self.is_driver = False - self.libcache = QlPeCache() if libcache else None + self.ql = ql + self.path = self.ql.path + self.libcache = QlPeCache() if libcache else None def run(self): - self.init_dlls = [b"ntdll.dll", b"kernel32.dll", b"user32.dll"] - self.sys_dlls = [b"ntdll.dll", b"kernel32.dll"] - self.pe_entry_point = 0 - self.sizeOfStackReserve = 0 + self.init_dlls = ( + 'ntdll.dll', + 'kernel32.dll', + 'user32.dll' + ) - if not self.ql.code: - self.pe = pefile.PE(self.path, fast_load=True) - self.is_driver = self.pe.is_driver() - if self.is_driver == True: - self.init_dlls.append(b"ntoskrnl.exe") - self.sys_dlls.append(b"ntoskrnl.exe") - + self.sys_dlls = ( + 'ntdll.dll', + 'kernel32.dll', + 'ucrtbase.dll' + ) - if self.ql.arch.type == QL_ARCH.X86: - WINOSARCH = "OS32" - self.structure_last_addr = FS_SEGMENT_ADDR - elif self.ql.arch.type == QL_ARCH.X8664: - WINOSARCH = "OS64" - self.structure_last_addr = GS_SEGMENT_ADDR + if self.ql.code: + pe = None + self.is_driver = False + else: + pe = pefile.PE(self.path, fast_load=True) + self.is_driver = pe.is_driver() + + ossection = f'OS{self.ql.arch.bits}' + + self.stack_address = self.ql.os.profile.getint(ossection, 'stack_address') + self.stack_size = self.ql.os.profile.getint(ossection, 'stack_size') + self.image_address = self.ql.os.profile.getint(ossection, 'image_address') + self.dll_address = self.ql.os.profile.getint(ossection, 'dll_address') + self.entry_point = self.ql.os.profile.getint(ossection, 'entry_point') + + self.structure_last_addr = { + 32 : FS_SEGMENT_ADDR, + 64 : GS_SEGMENT_ADDR + }[self.ql.arch.bits] - self.stack_address = int(self.ql.os.profile.get(WINOSARCH, "stack_address"), 16) - self.stack_size = int(self.ql.os.profile.get(WINOSARCH, "stack_size"), 16) - self.image_address = int(self.ql.os.profile.get(WINOSARCH, "image_address"), 16) - self.dll_address = int(self.ql.os.profile.get(WINOSARCH, "dll_address"), 16) - self.entry_point = int(self.ql.os.profile.get(WINOSARCH, "entry_point"), 16) - self.ql.os.heap_base_address = int(self.ql.os.profile.get(WINOSARCH, "heap_address"), 16) - self.ql.os.heap_base_size = int(self.ql.os.profile.get(WINOSARCH, "heap_size"), 16) - self.dlls = {} self.import_symbols = {} self.export_symbols = {} self.import_address_table = {} self.ldr_list = [] self.pe_image_address = 0 - self.pe_image_address_size = 0 + self.pe_image_size = 0 self.dll_size = 0 self.dll_last_address = self.dll_address - # compatible with ql.__enable_bin_patch() + + # not used, but here to remain compatible with ql.do_bin_patch self.load_address = 0 - self.ql.os.heap = QlMemoryHeap(self.ql, self.ql.os.heap_base_address, self.ql.os.heap_base_address + self.ql.os.heap_base_size) - self.ql.os.setupComponents() - self.ql.os.entry_point = self.entry_point - cmdline = (str(self.ql.os.userprofile)) + "Desktop\\" + self.ql.targetname - self.filepath = bytes(cmdline + "\x00", "utf-8") - for arg in self.argv[1:]: - if ' ' in arg: - cmdline += f' "{arg}"' - else: - cmdline += f' {arg}' - cmdline += "\x00" - self.cmdline = bytes(cmdline, "utf-8") - self.load() + cmdline = ntpath.join(self.ql.os.userprofile, 'Desktop', self.ql.targetname) + cmdargs = ' '.join(f'"{arg}"' if ' ' in arg else arg for arg in self.argv[1:]) - def init_thread_information_block(self): - super().init_tib() - super().init_peb() - super().init_ldr_data() - super().init_exports() + self.filepath = bytes(f'{cmdline}\x00', "utf-8") + self.cmdline = bytes(f'{cmdline} {cmdargs}\x00', "utf-8") - def load(self): + self.load(pe) + + def load(self, pe: Optional[pefile.PE]): # set stack pointer self.ql.log.info("Initiate stack address at 0x%x " % self.stack_address) self.ql.mem.map(self.stack_address, self.stack_size, info="[stack]") - if self.path and not self.ql.code: - # for simplicity, no image base relocation - self.pe_image_address = self.pe.OPTIONAL_HEADER.ImageBase - self.pe_image_address_size = self.ql.mem.align_up(self.pe.OPTIONAL_HEADER.SizeOfImage) + if pe is not None: + image_name = os.path.basename(self.path) + image_base = pe.OPTIONAL_HEADER.ImageBase + image_size = self.ql.mem.align_up(pe.OPTIONAL_HEADER.SizeOfImage) - if self.pe_image_address + self.pe_image_address_size > self.ql.os.heap_base_address: - # pe reloc - self.pe_image_address = self.image_address - self.pe.relocate_image(self.image_address) + # if default base address is taken, use the one specified in profile + if not self.ql.mem.is_available(image_base, image_size): + image_base = self.image_address + pe.relocate_image(image_base) - self.entry_point = self.pe_entry_point = self.pe_image_address + self.pe.OPTIONAL_HEADER.AddressOfEntryPoint - self.sizeOfStackReserve = self.pe.OPTIONAL_HEADER.SizeOfStackReserve - self.ql.log.info("Loading %s to 0x%x" % (self.path, self.pe_image_address)) - self.ql.log.info("PE entry point at 0x%x" % self.entry_point) - self.images.append(Image(self.pe_image_address, self.pe_image_address + self.pe.NT_HEADERS.OPTIONAL_HEADER.SizeOfImage, self.path)) + self.entry_point = image_base + pe.OPTIONAL_HEADER.AddressOfEntryPoint + self.pe_image_address = image_base + self.pe_image_size = image_size - # Stack should not init at the very bottom. Will cause errors with Dlls - sp = self.stack_address + self.stack_size - 0x1000 + self.ql.log.info(f'Loading {self.path} to {image_base:#x}') + self.ql.log.info(f'PE entry point at {self.entry_point:#x}') - if self.ql.arch.type == QL_ARCH.X86: - self.ql.arch.regs.esp = sp - self.ql.arch.regs.ebp = sp + self.ql.mem.map(image_base, image_size, info=f'[{image_name}]') + self.images.append(Image(image_base, image_base + pe.NT_HEADERS.OPTIONAL_HEADER.SizeOfImage, os.path.abspath(self.path))) - if self.pe.is_dll(): - self.ql.log.debug('Setting up DllMain args') - load_addr_bytes = self.pe_image_address.to_bytes(length=4, byteorder='little') + if self.is_driver: + self.init_driver_object() + self.init_registry_path() + self.init_eprocess() + self.init_ki_user_shared_data() - self.ql.log.debug('Writing 0x%08X (IMAGE_BASE) to [ESP+4](0x%08X)' % (self.pe_image_address, sp + 0x4)) - self.ql.mem.write(sp + 0x4, load_addr_bytes) - - self.ql.log.debug('Writing 0x01 (DLL_PROCESS_ATTACH) to [ESP+8](0x%08X)' % (sp + 0x8)) - self.ql.mem.write(sp + 0x8, int(1).to_bytes(length=4, byteorder='little')) - - elif self.ql.arch.type == QL_ARCH.X8664: - self.ql.arch.regs.rsp = sp - self.ql.arch.regs.rbp = sp - - if self.pe.is_dll(): - self.ql.log.debug('Setting up DllMain args') - - self.ql.log.debug('Setting RCX (arg1) to %16X (IMAGE_BASE)' % (self.pe_image_address)) - self.ql.arch.regs.rcx = self.pe_image_address - - self.ql.log.debug('Setting RDX (arg2) to 1 (DLL_PROCESS_ATTACH)') - self.ql.arch.regs.rdx = 1 - else: - raise QlErrorArch("Unknown ql.arch") - - # if this is NOT a driver, init tib/peb/ldr - if not self.is_driver: # userland program - self.init_thread_information_block() - else: # Windows kernel driver - super().init_driver_object() - super().init_registry_path() - super().init_eprocess() - super().init_ki_user_shared_data() - - # setup IRQ Level in CR8 to PASSIVE_LEVEL (0) + # set IRQ Level in CR8 to PASSIVE_LEVEL self.ql.arch.regs.write(UC_X86_REG_CR8, 0) - # setup CR4, some drivers may check this at initialized time - self.ql.arch.regs.write(UC_X86_REG_CR4, 0x6f8) + # setup CR4, enabling: DE, PSE, PAE, MCE, PGE, OSFXSR and OSXMMEXCPT. + # some drivers may check this at initialized + self.ql.arch.regs.write(UC_X86_REG_CR4, 0b0000011011111000) - self.ql.log.debug('Setting up DriverEntry args') - self.ql.stop_execution_pattern = 0xDEADC0DE + else: + # initialize thread information block + self.init_teb() + self.init_peb() + self.init_ldr_data() + self.init_exports(pe) - if self.ql.arch.type == QL_ARCH.X86: # Win32 - if not self.ql.stop_options: - # We know that a driver will return, - # so if the user did not configure stop options, write a sentinel return value - self.ql.mem.write(sp, self.ql.stop_execution_pattern.to_bytes(length=4, byteorder='little')) + # add image to ldr table + self.add_ldr_data_table_entry(image_name) - self.ql.log.debug('Writing 0x%08X (PDRIVER_OBJECT) to [ESP+4](0x%08X)' % (self.ql.loader.driver_object_address, sp+0x4)) - self.ql.log.debug('Writing 0x%08X (RegistryPath) to [ESP+8](0x%08X)' % (self.ql.loader.regitry_path_address, sp+0x8)) - elif self.ql.arch.type == QL_ARCH.X8664: # Win64 - if not self.ql.stop_options: - # We know that a driver will return, - # so if the user did not configure stop options, write a sentinel return value - self.ql.mem.write(sp, self.ql.stop_execution_pattern.to_bytes(length=8, byteorder='little')) + pe.parse_data_directories() - self.ql.log.debug('Setting RCX (arg1) to %16X (PDRIVER_OBJECT)' % (self.ql.loader.driver_object_address)) - self.ql.log.debug('Setting RDX (arg2) to %16X (PUNICODE_STRING)' % (self.ql.loader.regitry_path_address)) + # done manipulating pe file; write its contents into memory + self.ql.mem.write(image_base, bytes(pe.get_memory_mapped_image())) - # setup args for DriverEntry() - self.ql.os.fcall = self.ql.os.fcall_select(CDECL) - self.ql.os.fcall.writeParams(((POINTER, self.ql.loader.driver_object_address), (POINTER, self.ql.loader.regitry_path_address))) - - # mmap PE file into memory - self.ql.mem.map(self.pe_image_address, self.ql.mem.align_up(self.pe_image_address_size), info="[PE]") - self.pe.parse_data_directories() - data = bytearray(self.pe.get_memory_mapped_image()) - self.ql.mem.write(self.pe_image_address, bytes(data)) - if self.is_driver: - # setup IMAGE_LOAD_CONFIG_DIRECTORY - if self.pe.OPTIONAL_HEADER.DATA_DIRECTORY[pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_LOAD_CONFIG']].VirtualAddress != 0: - SecurityCookie_rva = self.pe.DIRECTORY_ENTRY_LOAD_CONFIG.struct.SecurityCookie - self.pe.OPTIONAL_HEADER.ImageBase - SecurityCookie_value = default_security_cookie_value = self.ql.mem.read(self.pe_image_address+SecurityCookie_rva, self.ql.arch.pointersize) - while SecurityCookie_value == default_security_cookie_value: - SecurityCookie_value = secrets.token_bytes(self.ql.arch.pointersize) - # rol rcx, 10h (rcx: cookie) - # test cx, 0FFFFh - SecurityCookie_value_array = bytearray(SecurityCookie_value) - # Sanity question: We are always little endian, right? - SecurityCookie_value_array[-2:] = b'\x00\x00' - SecurityCookie_value = bytes(SecurityCookie_value_array) - self.ql.mem.write(self.pe_image_address+SecurityCookie_rva, SecurityCookie_value) + # security cookie can be written only after image has been loaded to memory + self.init_security_cookie(pe, image_base) - # Add main PE to ldr_data_table - mod_name = os.path.basename(self.path) - self.dlls[mod_name] = self.pe_image_address - # only userland code need LDR table - if not self.is_driver: - super().add_ldr_data_table_entry(mod_name) + # Stack should not init at the very bottom. Will cause errors with Dlls + top_of_stack = self.stack_address + self.stack_size - 0x1000 + + if self.ql.arch.type == QL_ARCH.X86: + bp_reg = 'ebp' + sp_reg = 'esp' + elif self.ql.arch.type == QL_ARCH.X8664: + bp_reg = 'rbp' + sp_reg = 'rsp' + else: + raise QlErrorArch(f'unexpected arch type: {self.ql.arch.type}') + + # we are about to load some dlls and call their DllMain functions. + # the stack should be set first + self.ql.arch.regs.write(bp_reg, top_of_stack) + self.ql.arch.regs.write(sp_reg, top_of_stack) # load system dlls - sys_dlls = self.sys_dlls - for each in sys_dlls: + for each in self.sys_dlls: super().load_dll(each, self.is_driver) + # parse directory entry import - if self.pe.OPTIONAL_HEADER.DATA_DIRECTORY[pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_IMPORT']].VirtualAddress != 0: - for entry in self.pe.DIRECTORY_ENTRY_IMPORT: - dll_name = str(entry.dll.lower(), 'utf-8', 'ignore') - super().load_dll(entry.dll, self.is_driver) - for imp in entry.imports: - # fix IAT - # ql.log.info(imp.name) - # ql.log.info(self.import_address_table[imp.name]) - if imp.name: - try: - addr = self.import_address_table[dll_name][imp.name] - except KeyError: - self.ql.log.debug("Error in loading function %s" % imp.name.decode()) - continue - else: - addr = self.import_address_table[dll_name][imp.ordinal] + self.ql.log.debug(f'Init imports for {self.path}') + super().init_imports(pe, self.is_driver) - if self.ql.arch.type == QL_ARCH.X86: - address = self.ql.pack32(addr) - else: - address = self.ql.pack64(addr) - self.ql.mem.write(imp.address, address) + self.ql.log.debug(f'Done loading {self.path}') - self.ql.log.debug("Done with loading %s" % self.path) - self.ql.os.entry_point = self.entry_point - self.ql.os.pid = 101 + if pe.is_driver(): + args = ( + (POINTER, self.driver_object_address), + (POINTER, self.regitry_path_address) + ) - elif self.ql.code: + self.ql.log.debug('Setting up call frame for DriverEntry:') + self.ql.log.debug(f' PDRIVER_OBJECT DriverObject : {args[0][1]:#010x}') + self.ql.log.debug(f' PUNICODE_STRING RegistryPath : {args[1][1]:#010x}') + + # We know that a driver will return, so if the user did not configure stop + # options, write a sentinel return value + ret = None if self.ql.stop_options else self.ql.stack_write(0, 0xdeadc0de) + + # set up call frame for DriverEntry + self.ql.os.fcall.call_native(self.entry_point, args, ret) + + elif pe.is_dll(): + args = ( + (POINTER, image_base), + (DWORD, 1), # DLL_PROCESS_ATTACH + (POINTER, 0) + ) + + self.ql.log.debug('Setting up call frame for DllMain:') + self.ql.log.debug(f' HINSTANCE hinstDLL : {args[0][1]:#010x}') + self.ql.log.debug(f' DWORD fdwReason : {args[1][1]:#010x}') + self.ql.log.debug(f' LPVOID lpReserved : {args[2][1]:#010x}') + + # set up call frame for DllMain + self.ql.os.fcall.call_native(self.entry_point, args, None) + + elif pe is None: self.filepath = b"" - if self.ql.arch.type == QL_ARCH.X86: - self.ql.arch.regs.esp = self.stack_address + 0x3000 - self.ql.arch.regs.ebp = self.ql.arch.regs.esp - elif self.ql.arch.type == QL_ARCH.X8664: - self.ql.arch.regs.rsp = self.stack_address + 0x3000 - self.ql.arch.regs.rbp = self.ql.arch.regs.rsp - # load shellcode in - self.ql.mem.map(self.entry_point, self.ql.os.code_ram_size, info="[shellcode_base]") - # rewrite entrypoint for windows shellcode - self.ql.os.entry_point = self.entry_point - self.ql.os.pid = 101 + self.ql.mem.map(self.entry_point, self.ql.os.code_ram_size, info="[shellcode]") + self.init_teb() + self.init_peb() + self.init_ldr_data() + + # write shellcode to memory self.ql.mem.write(self.entry_point, self.ql.code) - - self.init_thread_information_block() + + top_of_stack = self.stack_address + self.stack_size + + if self.ql.arch.type == QL_ARCH.X86: + bp_reg = 'ebp' + sp_reg = 'esp' + elif self.ql.arch.type == QL_ARCH.X8664: + bp_reg = 'rbp' + sp_reg = 'rsp' + else: + raise QlErrorArch(f'unexpected arch type: {self.ql.arch.type}') + + self.ql.arch.regs.write(bp_reg, top_of_stack) + self.ql.arch.regs.write(sp_reg, top_of_stack) + # load dlls for each in self.init_dlls: - super().load_dll(each) + super().load_dll(each, self.is_driver) # move entry_point to ql.os self.ql.os.entry_point = self.entry_point self.init_sp = self.ql.arch.regs.arch_sp + +class ShowProgress: + """Display a progress animation while performing a time + consuming task. + + Example: + >>> with ShowProgress(0.1): + ... do_some_time_consuming_task() + """ + + def __init__(self, interval: float) -> None: + import sys + from threading import Thread, Event + + # animation frames; any sequence of chars or strings may be used, as long + # as they are of the same length. e.g. ['> ', '>> ', ' >> ', ' >>', ' >', ' '] + frames = r'/-\|' + stream = sys.stderr + + def show_animation(): + i = 0 + + while not self.stopped.wait(interval): + # TODO: find a proper way to use the logger for that + print(f'[{frames[i % len(frames)]}]', end='\r', flush=True, file=stream) + i += 1 + + def show_nothing(): + pass + + # avoid flooding log files with animation frames + action = show_animation if stream.isatty() else show_nothing + + self.stopped = Event() + self.thread = Thread(target=action) + + def __enter__(self): + self.thread.start() + + return self + + def __exit__(self, *args) -> None: + self.stopped.set() diff --git a/qiling/log.py b/qiling/log.py new file mode 100644 index 00000000..d085266e --- /dev/null +++ b/qiling/log.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +import copy +import logging +import os +import re +import weakref + +from typing import Optional, TextIO + +from qiling.const import QL_VERBOSE + +QL_INSTANCE_ID = 114514 + +FMT_STR = '%(levelname)s\t%(message)s' + +class COLOR: + WHITE = '\033[37m' + CRIMSON = '\033[31m' + RED = '\033[91m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + BLUE = '\033[94m' + MAGENTA = '\033[95m' + CYAN = '\033[96m' + ENDC = '\033[0m' + +class QlBaseFormatter(logging.Formatter): + __level_tag = { + 'WARNING' : '[!]', + 'INFO' : '[=]', + 'DEBUG' : '[+]', + 'CRITICAL' : '[x]', + 'ERROR' : '[x]' + } + + def __init__(self, ql, *args, **kwargs): + super().__init__(*args, **kwargs) + self.ql = weakref.proxy(ql) + + def get_level_tag(self, level: str) -> str: + return self.__level_tag[level] + + def get_thread_tag(self, thread: str) -> str: + return thread + + def format(self, record: logging.LogRecord): + # In case we have multiple formatters, we have to keep a copy of the record. + record = copy.copy(record) + + # early logging may access ql.os when it is not yet set + try: + cur_thread = self.ql.os.thread_management.cur_thread + except AttributeError: + tid = f'' + else: + tid = self.get_thread_tag(str(cur_thread)) + + level = self.get_level_tag(record.levelname) + record.levelname = f'{level} {tid}' + + return super().format(record) + +class QlColoredFormatter(QlBaseFormatter): + __level_color = { + 'WARNING' : COLOR.YELLOW, + 'INFO' : COLOR.BLUE, + 'DEBUG' : COLOR.MAGENTA, + 'CRITICAL' : COLOR.CRIMSON, + 'ERROR' : COLOR.RED + } + + def get_level_tag(self, level: str) -> str: + s = super().get_level_tag(level) + + return f'{self.__level_color[level]}{s}{COLOR.ENDC}' + + def get_thread_tag(self, tid: str) -> str: + s = super().get_thread_tag(tid) + + return f'{COLOR.GREEN}{s}{COLOR.ENDC}' + +class RegexFilter(logging.Filter): + def update_filter(self, regexp: str): + self._filter = re.compile(regexp) + + def filter(self, record: logging.LogRecord): + msg = record.getMessage() + + return self._filter.match(msg) is not None + +def resolve_logger_level(verbose: QL_VERBOSE) -> int: + return { + QL_VERBOSE.DISABLED : logging.CRITICAL, + QL_VERBOSE.OFF : logging.WARNING, + QL_VERBOSE.DEFAULT : logging.INFO, + QL_VERBOSE.DEBUG : logging.DEBUG, + QL_VERBOSE.DISASM : logging.DEBUG, + QL_VERBOSE.DUMP : logging.DEBUG + }[verbose] + +def __is_color_terminal(stream: TextIO) -> bool: + """Determine whether standard output is attached to a color terminal. + + see: https://stackoverflow.com/questions/53574442/how-to-reliably-test-color-capability-of-an-output-terminal-in-python3 + """ + + def __handle_nt(fd: int) -> bool: + import ctypes + import msvcrt + + ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 + + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + hstdout = msvcrt.get_osfhandle(fd) + mode = ctypes.c_ulong() + + return kernel32.GetConsoleMode(hstdout, ctypes.byref(mode)) and (mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING != 0) + + def __handle_posix(fd: int) -> bool: + import curses + + try: + curses.setupterm(fd=fd) + except curses.error: + return True + else: + return curses.tigetnum('colors') > 0 + + def __default(_: int) -> bool: + return True + + handlers = { + 'nt' : __handle_nt, + 'posix' : __handle_posix + } + + handler = handlers.get(os.name, __default) + + return handler(stream.fileno()) + +def setup_logger(ql, log_file: Optional[str], console: bool, log_override: Optional[logging.Logger], log_plain: bool): + global QL_INSTANCE_ID + + # If there is an override for our logger, then use it. + if log_override is not None: + log = log_override + else: + # We should leave the root logger untouched. + log = logging.getLogger(f'qiling{QL_INSTANCE_ID}') + QL_INSTANCE_ID += 1 + + # Disable propagation to avoid duplicate output. + log.propagate = False + # Clear all handlers and filters. + log.handlers = [] + log.filters = [] + + # Do we have console output? + if console: + handler = logging.StreamHandler() + + if log_plain or not __is_color_terminal(handler.stream): + formatter = QlBaseFormatter(ql, FMT_STR) + else: + formatter = QlColoredFormatter(ql, FMT_STR) + + handler.setFormatter(formatter) + log.addHandler(handler) + else: + handler = logging.NullHandler() + log.addHandler(handler) + + # Do we have to write log to a file? + if log_file is not None: + handler = logging.FileHandler(log_file) + formatter = QlBaseFormatter(ql, FMT_STR) + handler.setFormatter(formatter) + log.addHandler(handler) + + log.setLevel(logging.INFO) + + return log + +__all__ = ['RegexFilter', 'setup_logger', 'resolve_logger_level'] diff --git a/qiling/os/blob/__init__.py b/qiling/os/blob/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/qiling/os/blob/blob.py b/qiling/os/blob/blob.py index 09be61e9..02e6f94d 100644 --- a/qiling/os/blob/blob.py +++ b/qiling/os/blob/blob.py @@ -4,8 +4,8 @@ # from qiling import Qiling -from qiling.cc import QlCC, intel, arm, mips -from qiling.const import QL_ARCH +from qiling.cc import QlCC, intel, arm, mips, riscv, ppc +from qiling.const import QL_ARCH, QL_OS from qiling.os.fcall import QlFunctionCall from qiling.os.os import QlOs @@ -17,17 +17,23 @@ class QlOsBlob(QlOs): resolve_fcall_params(), heap or add_fs_mapper() are based on os. To keep the consistence of api usage, QlOsBlob is introduced and placed at its loader temporarily. """ + + type = QL_OS.BLOB + def __init__(self, ql: Qiling): super(QlOsBlob, self).__init__(ql) self.ql = ql cc: QlCC = { - QL_ARCH.X86 : intel.cdecl, - QL_ARCH.X8664 : intel.amd64, - QL_ARCH.ARM : arm.aarch32, - QL_ARCH.ARM64 : arm.aarch64, - QL_ARCH.MIPS : mips.mipso32 + QL_ARCH.X86 : intel.cdecl, + QL_ARCH.X8664 : intel.amd64, + QL_ARCH.ARM : arm.aarch32, + QL_ARCH.ARM64 : arm.aarch64, + QL_ARCH.MIPS : mips.mipso32, + QL_ARCH.RISCV : riscv.riscv, + QL_ARCH.RISCV64 : riscv.riscv, + QL_ARCH.PPC : ppc.ppc, }[ql.arch.type](ql.arch) self.fcall = QlFunctionCall(ql, cc) diff --git a/qiling/os/dos/dos.py b/qiling/os/dos/dos.py index 40cf5da1..fb7f55c1 100644 --- a/qiling/os/dos/dos.py +++ b/qiling/os/dos/dos.py @@ -10,7 +10,7 @@ from datetime import datetime from unicorn import UcError from qiling import Qiling -from qiling.const import QL_INTERCEPT +from qiling.const import QL_OS, QL_INTERCEPT from qiling.os.os import QlOs from .interrupts import handlers @@ -29,6 +29,8 @@ class Flags(IntEnum): IOPL = (3 << 12) # io privilege class QlOsDos(QlOs): + type = QL_OS.DOS + def __init__(self, ql: Qiling): super(QlOsDos, self).__init__(ql) @@ -56,10 +58,10 @@ class QlOsDos(QlOs): curses.endwin() def set_flag_value(self, fl: Flags, val: int) -> None: - self.ql.arch.regs.ef = self.ql.arch.regs.ef & (~fl) | (fl * val) + self.ql.arch.regs.eflags = self.ql.arch.regs.eflags & (~fl) | (fl * val) def test_flags(self, fl): - return self.ql.arch.regs.ef & fl == fl + return self.ql.arch.regs.eflags & fl == fl def set_cf(self): self.set_flag_value(Flags.CF, 0b1) diff --git a/qiling/os/fcall.py b/qiling/os/fcall.py index 4484e8de..c9bd95d2 100644 --- a/qiling/os/fcall.py +++ b/qiling/os/fcall.py @@ -32,8 +32,8 @@ class QlFunctionCall: self.cc = cc def __make_accessor(nbits: int) -> Accessor: - reader = lambda si: cc.getRawParam(si, nbits or None) - writer = lambda si, val: cc.setRawParam(si, val, nbits or None) + reader = lambda si: cc.getRawParam(si, nbits) + writer = lambda si, val: cc.setRawParam(si, val, nbits) nslots = cc.getNumSlots(nbits) return (reader, writer, nslots) diff --git a/qiling/os/freebsd/freebsd.py b/qiling/os/freebsd/freebsd.py index 6b368466..e0218d4d 100644 --- a/qiling/os/freebsd/freebsd.py +++ b/qiling/os/freebsd/freebsd.py @@ -7,9 +7,12 @@ from unicorn import UcError from unicorn.x86_const import UC_X86_INS_SYSCALL from qiling.arch.x86_utils import GDTManager, SegmentManager86 +from qiling.const import QL_OS from qiling.os.posix.posix import QlOsPosix class QlOsFreebsd(QlOsPosix): + type = QL_OS.FREEBSD + def __init__(self, ql): super(QlOsFreebsd, self).__init__(ql) @@ -27,7 +30,7 @@ class QlOsFreebsd(QlOsPosix): self.ql.hook_insn(self.hook_syscall, UC_X86_INS_SYSCALL) - def hook_syscall(self, intno= None): + def hook_syscall(self, ql): return self.load_syscall() diff --git a/qiling/os/linux/function_hook.py b/qiling/os/linux/function_hook.py index 7b07c2e5..aa2e6cf6 100644 --- a/qiling/os/linux/function_hook.py +++ b/qiling/os/linux/function_hook.py @@ -79,6 +79,10 @@ class HookFunc: elif self.ql.arch.type == QL_ARCH.ARM64: return self.ql.arch.regs.x30 + # PPC + elif self.ql.arch.type== QL_ARCH.PPC: + return self.ql.arch.regs.lr + # X86 elif self.ql.arch.type == QL_ARCH.X86: return self.ql.unpack(self.ql.mem.read(self.ql.arch.regs.esp, self.ql.arch.pointersize)) @@ -98,6 +102,10 @@ class HookFunc: elif self.ql.arch.type == QL_ARCH.MIPS: pass + # PPC + elif self.ql.arch.type== QL_ARCH.PPC: + pass + # ARM64 elif self.ql.arch.type == QL_ARCH.ARM64: pass @@ -121,6 +129,10 @@ class HookFunc: elif self.ql.arch.type == QL_ARCH.MIPS: self.ql.arch.regs.ra = addr + # PPC + elif self.ql.arch.type== QL_ARCH.PPC: + self.ql.arch.regs.lr = addr + # ARM64 elif self.ql.arch.type == QL_ARCH.ARM64: self.ql.arch.stack_write(0, addr) @@ -173,29 +185,6 @@ class HookFunc: else: self.context_fixup() - def ret(self): - # ARM - if self.ql.arch.type == QL_ARCH.ARM: - self.ql.arch.regs.arch_pc = self.ret_pc - - # MIPS32 - elif self.ql.arch.type == QL_ARCH.MIPS: - self.ql.arch.regs.arch_pc = self.ret_pc - - # ARM64 - elif self.ql.arch.type == QL_ARCH.ARM64: - self.ql.arch.regs.arch_pc = self.ret_pc - - # X86 - elif self.ql.arch.type == QL_ARCH.X86: - self.ql.arch.regs.arch_pc = self.ret_pc - - # X8664 - elif self.ql.arch.type == QL_ARCH.X8664: - self.ql.arch.regs.arch_pc = self.ret_pc - else: - raise - def call_exit(self): # if self.ql.arch.type == QL_ARCH.ARM or self.ql.arch.type == QL_ARCH.ARM64: # self.ql.arch.regs.arch_pc = self.ql.arch.regs.arch_pc + 4 @@ -211,7 +200,7 @@ class HookFunc: else: onexit_cb(self.ql, onexit_userdata) - self.ret() + self.ql.arch.regs.arch_pc = self.ret_pc class HookFuncRel(HookFunc): @@ -614,6 +603,14 @@ class FunctionHook: ins = b'\x00\x01' self.add_function_hook = self.add_function_hook_relocation + # PowerPC + elif self.ql.arch.type== QL_ARCH.PPC: + self.GLOB_DAT = 21 + self.JMP_SLOT = 22 + # nop + ins = b'\x60\x00\x00\x00' + self.add_function_hook = self.add_function_hook_relocation + self._parse() if self.rel != None: self.show_relocation(self.rel) diff --git a/qiling/os/linux/futex.py b/qiling/os/linux/futex.py index 1f17b9b2..ef78db4f 100644 --- a/qiling/os/linux/futex.py +++ b/qiling/os/linux/futex.py @@ -10,24 +10,25 @@ from gevent.event import Event from queue import Queue class QlLinuxFutexManagement: - + FUTEX_BITSET_MATCH_ANY = 0xffffffff - + def __init__(self): self._wait_list = {} - + @property def wait_list(self): return self._wait_list - + def futex_wait(self, ql, uaddr, t, val, bitset=FUTEX_BITSET_MATCH_ANY): + EAGAIN = 11 def _sched_wait_event(cur_thread): ql.log.debug(f"Wait for notifications.") event.wait() uaddr_value = ql.unpack32(ql.mem.read(uaddr, 4)) if uaddr_value != val: ql.log.debug(f"uaddr: {hex(uaddr_value)} != {hex(val)}") - return -1 + return -EAGAIN ql.emu_stop() if uaddr not in self.wait_list.keys(): self.wait_list[uaddr] = Queue() @@ -35,7 +36,7 @@ class QlLinuxFutexManagement: self.wait_list[uaddr].put((bitset, t, event)) t.sched_cb = _sched_wait_event return 0 - + def get_futex_wake_list(self, ql, addr, number, bitset=FUTEX_BITSET_MATCH_ANY): wakes = [] if addr not in self.wait_list or number == 0: diff --git a/qiling/os/linux/linux.py b/qiling/os/linux/linux.py index 8ce5f415..ed95a7ec 100644 --- a/qiling/os/linux/linux.py +++ b/qiling/os/linux/linux.py @@ -3,7 +3,6 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from typing import Callable from unicorn import UcError from unicorn.x86_const import UC_X86_INS_SYSCALL @@ -11,30 +10,32 @@ from qiling import Qiling from qiling.arch.x86_const import GS_SEGMENT_ADDR, GS_SEGMENT_SIZE from qiling.arch.x86_utils import GDTManager, SegmentManager86, SegmentManager64 from qiling.arch import arm_utils -from qiling.cc import QlCC, intel, arm, mips, riscv -from qiling.const import QL_ARCH, QL_INTERCEPT +from qiling.cc import QlCC, intel, arm, mips, riscv, ppc +from qiling.const import QL_ARCH, QL_OS from qiling.os.fcall import QlFunctionCall from qiling.os.const import * -from qiling.os.posix.const import NR_OPEN from qiling.os.posix.posix import QlOsPosix from . import futex from . import thread class QlOsLinux(QlOsPosix): + type = QL_OS.LINUX + def __init__(self, ql: Qiling): super(QlOsLinux, self).__init__(ql) self.ql = ql cc: QlCC = { - QL_ARCH.X86 : intel.cdecl, - QL_ARCH.X8664 : intel.amd64, - QL_ARCH.ARM : arm.aarch32, - QL_ARCH.ARM64 : arm.aarch64, - QL_ARCH.MIPS : mips.mipso32, - QL_ARCH.RISCV : riscv.riscv, - QL_ARCH.RISCV64: riscv.riscv, + QL_ARCH.X86 : intel.cdecl, + QL_ARCH.X8664 : intel.amd64, + QL_ARCH.ARM : arm.aarch32, + QL_ARCH.ARM64 : arm.aarch64, + QL_ARCH.MIPS : mips.mipso32, + QL_ARCH.RISCV : riscv.riscv, + QL_ARCH.RISCV64 : riscv.riscv, + QL_ARCH.PPC : ppc.ppc, }[ql.arch.type](ql.arch) self.fcall = QlFunctionCall(ql, cc) @@ -107,6 +108,11 @@ class QlOsLinux(QlOsPosix): self.ql.hook_intno(self.hook_syscall, 8) self.thread_class = None + elif self.ql.arch.type == QL_ARCH.PPC: + self.ql.arch.enable_float() + self.ql.hook_intno(self.hook_syscall, 8) + self.thread_class = None + # on fork or execve, do not inherit opened files tagged as 'close on exec' for i in range(len(self.fd)): if getattr(self.fd[i], 'close_on_exec', 0): @@ -116,10 +122,6 @@ class QlOsLinux(QlOsPosix): return self.load_syscall() - def add_function_hook(self, fn: str, cb: Callable, intercept: QL_INTERCEPT): - self.ql.os.function_hook.add_function_hook(fn, cb, intercept) - - def register_function_after_load(self, function): if function not in self.function_after_load_list: self.function_after_load_list.append(function) @@ -161,9 +163,9 @@ class QlOsLinux(QlOsPosix): self.ql.emu_start(self.ql.loader.elf_entry, self.exit_point, self.ql.timeout, self.ql.count) except UcError: - # TODO: this is bad We need a better approach for this - #if self.ql.output != QL_OUTPUT.DEBUG: - # return - self.emu_error() raise + + # display summary + for entry in self.stats.summary(): + self.ql.log.debug(entry) diff --git a/qiling/os/linux/map_syscall.py b/qiling/os/linux/map_syscall.py index 19bb4269..3d0bcead 100644 --- a/qiling/os/linux/map_syscall.py +++ b/qiling/os/linux/map_syscall.py @@ -18,8 +18,9 @@ def get_syscall_mapper(archtype: QL_ARCH): QL_ARCH.X8664 : x8664_syscall_table, QL_ARCH.X86 : x86_syscall_table, QL_ARCH.MIPS : mips_syscall_table, - QL_ARCH.RISCV : riscv32_syscall_table, - QL_ARCH.RISCV64 : riscv64_syscall_table + QL_ARCH.RISCV : riscv32_syscall_table, + QL_ARCH.RISCV64 : riscv64_syscall_table, + QL_ARCH.PPC : ppc_syscall_table }[archtype] def __mapper(syscall_num: int) -> str: @@ -2550,3 +2551,421 @@ riscv64_syscall_table = { 446: "landlock_restrict_self", 448: "process_mrelease", } + +ppc_syscall_table = { + 0: "restart_syscall", + 1: "exit", + 2: "fork", + 3: "read", + 4: "write", + 5: "open", + 6: "close", + 7: "waitpid", + 8: "creat", + 9: "link", + 10: "unlink", + 11: "execve", + 12: "chdir", + 13: "time", + 14: "mknod", + 15: "chmod", + 16: "lchown", + 17: "break", + 18: "oldstat", + 19: "lseek", + 20: "getpid", + 21: "mount", + 22: "umount", + 23: "setuid", + 24: "getuid", + 25: "stime", + 26: "ptrace", + 27: "alarm", + 28: "oldfstat", + 29: "pause", + 30: "utime", + 31: "stty", + 32: "gtty", + 33: "access", + 34: "nice", + 35: "ftime", + 36: "sync", + 37: "kill", + 38: "rename", + 39: "mkdir", + 40: "rmdir", + 41: "dup", + 42: "pipe", + 43: "times", + 44: "prof", + 45: "brk", + 46: "setgid", + 47: "getgid", + 48: "signal", + 49: "geteuid", + 50: "getegid", + 51: "acct", + 52: "umount2", + 53: "lock", + 54: "ioctl", + 55: "fcntl", + 56: "mpx", + 57: "setpgid", + 58: "ulimit", + 59: "oldolduname", + 60: "umask", + 61: "chroot", + 62: "ustat", + 63: "dup2", + 64: "getppid", + 65: "getpgrp", + 66: "setsid", + 67: "sigaction", + 68: "sgetmask", + 69: "ssetmask", + 70: "setreuid", + 71: "setregid", + 72: "sigsuspend", + 73: "sigpending", + 74: "sethostname", + 75: "setrlimit", + 76: "getrlimit", + 77: "getrusage", + 78: "gettimeofday", + 79: "settimeofday", + 80: "getgroups", + 81: "setgroups", + 82: "select", + 83: "symlink", + 84: "oldlstat", + 85: "readlink", + 86: "uselib", + 87: "swapon", + 88: "reboot", + 89: "readdir", + 90: "mmap", + 91: "munmap", + 92: "truncate", + 93: "ftruncate", + 94: "fchmod", + 95: "fchown", + 96: "getpriority", + 97: "setpriority", + 98: "profil", + 99: "statfs", + 100: "fstatfs", + 101: "ioperm", + 102: "socketcall", + 103: "syslog", + 104: "setitimer", + 105: "getitimer", + 106: "stat", + 107: "lstat", + 108: "fstat", + 109: "olduname", + 110: "iopl", + 111: "vhangup", + 112: "idle", + 113: "vm86", + 114: "wait4", + 115: "swapoff", + 116: "sysinfo", + 117: "ipc", + 118: "fsync", + 119: "sigreturn", + 120: "clone", + 121: "setdomainname", + 122: "uname", + 123: "modify_ldt", + 124: "adjtimex", + 125: "mprotect", + 126: "sigprocmask", + 127: "create_module", + 128: "init_module", + 129: "delete_module", + 130: "get_kernel_syms", + 131: "quotactl", + 132: "getpgid", + 133: "fchdir", + 134: "bdflush", + 135: "sysfs", + 136: "personality", + 137: "afs_syscall", + 138: "setfsuid", + 139: "setfsgid", + 140: "_llseek", + 141: "getdents", + 142: "_newselect", + 143: "flock", + 144: "msync", + 145: "readv", + 146: "writev", + 147: "getsid", + 148: "fdatasync", + 149: "_sysctl", + 150: "mlock", + 151: "munlock", + 152: "mlockall", + 153: "munlockall", + 154: "sched_setparam", + 155: "sched_getparam", + 156: "sched_setscheduler", + 157: "sched_getscheduler", + 158: "sched_yield", + 159: "sched_get_priority_max", + 160: "sched_get_priority_min", + 161: "sched_rr_get_interval", + 162: "nanosleep", + 163: "mremap", + 164: "setresuid", + 165: "getresuid", + 166: "query_module", + 167: "poll", + 168: "nfsservctl", + 169: "setresgid", + 170: "getresgid", + 171: "prctl", + 172: "rt_sigreturn", + 173: "rt_sigaction", + 174: "rt_sigprocmask", + 175: "rt_sigpending", + 176: "rt_sigtimedwait", + 177: "rt_sigqueueinfo", + 178: "rt_sigsuspend", + 179: "pread64", + 180: "pwrite64", + 181: "chown", + 182: "getcwd", + 183: "capget", + 184: "capset", + 185: "sigaltstack", + 186: "sendfile", + 187: "getpmsg", + 188: "putpmsg", + 189: "vfork", + 190: "ugetrlimit", + 191: "readahead", + 192: "mmap2", + 193: "truncate64", + 194: "ftruncate64", + 195: "stat64", + 196: "lstat64", + 197: "fstat64", + 198: "pciconfig_read", + 199: "pciconfig_write", + 200: "pciconfig_iobase", + 201: "multiplexer", + 202: "getdents64", + 203: "pivot_root", + 204: "fcntl64", + 205: "madvise", + 206: "mincore", + 207: "gettid", + 208: "tkill", + 209: "setxattr", + 210: "lsetxattr", + 211: "fsetxattr", + 212: "getxattr", + 213: "lgetxattr", + 214: "fgetxattr", + 215: "listxattr", + 216: "llistxattr", + 217: "flistxattr", + 218: "removexattr", + 219: "lremovexattr", + 220: "fremovexattr", + 221: "futex", + 222: "sched_setaffinity", + 223: "sched_getaffinity", + 225: "tuxcall", + 226: "sendfile64", + 227: "io_setup", + 228: "io_destroy", + 229: "io_getevents", + 230: "io_submit", + 231: "io_cancel", + 232: "set_tid_address", + 233: "fadvise64", + 234: "exit_group", + 235: "lookup_dcookie", + 236: "epoll_create", + 237: "epoll_ctl", + 238: "epoll_wait", + 239: "remap_file_pages", + 240: "timer_create", + 241: "timer_settime", + 242: "timer_gettime", + 243: "timer_getoverrun", + 244: "timer_delete", + 245: "clock_settime", + 246: "clock_gettime", + 247: "clock_getres", + 248: "clock_nanosleep", + 249: "swapcontext", + 250: "tgkill", + 251: "utimes", + 252: "statfs64", + 253: "fstatfs64", + 254: "fadvise64_64", + 255: "rtas", + 256: "sys_debug_setcontext", + 258: "migrate_pages", + 259: "mbind", + 260: "get_mempolicy", + 261: "set_mempolicy", + 262: "mq_open", + 263: "mq_unlink", + 264: "mq_timedsend", + 265: "mq_timedreceive", + 266: "mq_notify", + 267: "mq_getsetattr", + 268: "kexec_load", + 269: "add_key", + 270: "request_key", + 271: "keyctl", + 272: "waitid", + 273: "ioprio_set", + 274: "ioprio_get", + 275: "inotify_init", + 276: "inotify_add_watch", + 277: "inotify_rm_watch", + 278: "spu_run", + 279: "spu_create", + 280: "pselect6", + 281: "ppoll", + 282: "unshare", + 283: "splice", + 284: "tee", + 285: "vmsplice", + 286: "openat", + 287: "mkdirat", + 288: "mknodat", + 289: "fchownat", + 290: "futimesat", + 291: "fstatat64", + 292: "unlinkat", + 293: "renameat", + 294: "linkat", + 295: "symlinkat", + 296: "readlinkat", + 297: "fchmodat", + 298: "faccessat", + 299: "get_robust_list", + 300: "set_robust_list", + 301: "move_pages", + 302: "getcpu", + 303: "epoll_pwait", + 304: "utimensat", + 305: "signalfd", + 306: "timerfd_create", + 307: "eventfd", + 308: "sync_file_range2", + 309: "fallocate", + 310: "subpage_prot", + 311: "timerfd_settime", + 312: "timerfd_gettime", + 313: "signalfd4", + 314: "eventfd2", + 315: "epoll_create1", + 316: "dup3", + 317: "pipe2", + 318: "inotify_init1", + 319: "perf_event_open", + 320: "preadv", + 321: "pwritev", + 322: "rt_tgsigqueueinfo", + 323: "fanotify_init", + 324: "fanotify_mark", + 325: "prlimit64", + 326: "socket", + 327: "bind", + 328: "connect", + 329: "listen", + 330: "accept", + 331: "getsockname", + 332: "getpeername", + 333: "socketpair", + 334: "send", + 335: "sendto", + 336: "recv", + 337: "recvfrom", + 338: "shutdown", + 339: "setsockopt", + 340: "getsockopt", + 341: "sendmsg", + 342: "recvmsg", + 343: "recvmmsg", + 344: "accept4", + 345: "name_to_handle_at", + 346: "open_by_handle_at", + 347: "clock_adjtime", + 348: "syncfs", + 349: "sendmmsg", + 350: "setns", + 351: "process_vm_readv", + 352: "process_vm_writev", + 353: "finit_module", + 354: "kcmp", + 355: "sched_setattr", + 356: "sched_getattr", + 357: "renameat2", + 358: "seccomp", + 359: "getrandom", + 360: "memfd_create", + 361: "bpf", + 362: "execveat", + 363: "switch_endian", + 364: "userfaultfd", + 365: "membarrier", + 378: "mlock2", + 379: "copy_file_range", + 380: "preadv2", + 381: "pwritev2", + 382: "kexec_file_load", + 383: "statx", + 384: "pkey_alloc", + 385: "pkey_free", + 386: "pkey_mprotect", + 387: "rseq", + 388: "io_pgetevents", + 393: "semget", + 394: "semctl", + 395: "shmget", + 396: "shmctl", + 397: "shmat", + 398: "shmdt", + 399: "msgget", + 400: "msgsnd", + 401: "msgrcv", + 402: "msgctl", + 403: "clock_gettime64", + 404: "clock_settime64", + 405: "clock_adjtime64", + 406: "clock_getres_time64", + 407: "clock_nanosleep_time64", + 408: "timer_gettime64", + 409: "timer_settime64", + 410: "timerfd_gettime64", + 411: "timerfd_settime64", + 412: "utimensat_time64", + 413: "pselect6_time64", + 414: "ppoll_time64", + 416: "io_pgetevents_time64", + 417: "recvmmsg_time64", + 418: "mq_timedsend_time64", + 419: "mq_timedreceive_time64", + 420: "semtimedop_time64", + 421: "rt_sigtimedwait_time64", + 422: "futex_time64", + 423: "sched_rr_get_interval_time64", + 424: "pidfd_send_signal", + 425: "io_uring_setup", + 426: "io_uring_enter", + 427: "io_uring_register", + 428: "open_tree", + 429: "move_mount", + 430: "fsopen", + 431: "fsconfig", + 432: "fsmount", + 433: "fspick", +} diff --git a/qiling/os/linux/syscall.py b/qiling/os/linux/syscall.py index b3f10a7e..c61951c9 100644 --- a/qiling/os/linux/syscall.py +++ b/qiling/os/linux/syscall.py @@ -11,24 +11,29 @@ from datetime import datetime from math import floor import ctypes -class timespec(ctypes.Structure): - _fields_ = [ - ("tv_sec", ctypes.c_uint64), - ("tv_nsec", ctypes.c_int64) - ] +def __get_timespec_struct(archbits: int): + long = getattr(ctypes, f'c_int{archbits}') + ulong = getattr(ctypes, f'c_uint{archbits}') - _pack_ = 8 + class timespec(ctypes.Structure): + _pack_ = archbits // 8 + _fields_ = ( + ('tv_sec', ulong), + ('tv_nsec', long) + ) -# Temporary dirty fix. -# TODO: Pack ctypes.Structure according to ql.arch.type and ql.ostype? -class timespec32(ctypes.Structure): - _fields_ = [ - ("tv_sec", ctypes.c_uint32), - ("tv_nsec", ctypes.c_int32) - ] + return timespec + +def __get_timespec_obj(archbits: int): + now = datetime.now().timestamp() + + tv_sec = floor(now) + tv_nsec = floor((now - floor(now)) * 1e6) + ts_cls = __get_timespec_struct(archbits) + + return ts_cls(tv_sec=tv_sec, tv_nsec=tv_nsec) - _pack_ = 4 def ql_syscall_set_thread_area(ql: Qiling, u_info_addr: int): if ql.arch.type == QL_ARCH.X86: @@ -62,39 +67,25 @@ def ql_syscall_set_thread_area(ql: Qiling, u_info_addr: int): return 0 -def ql_syscall_set_tls(ql, address, *args, **kw): +def ql_syscall_set_tls(ql: Qiling, address: int): if ql.arch.type == QL_ARCH.ARM: ql.arch.regs.c13_c0_3 = address ql.mem.write_ptr(ql.arch.arm_get_tls_addr + 16, address, 4) ql.arch.regs.r0 = address ql.log.debug("settls(0x%x)" % address) -def ql_syscall_clock_gettime(ql, clock_gettime_clock_id, clock_gettime_timespec, *args, **kw): - now = datetime.now().timestamp() - tv_sec = floor(now) - tv_nsec = floor((now - floor(now)) * 1e6) - if ql.arch.type == QL_ARCH.X8664: - tp = timespec(tv_sec= tv_sec, tv_nsec=tv_nsec) - else: - tp = timespec32(tv_sec= tv_sec, tv_nsec=tv_nsec) - ql.mem.write(clock_gettime_timespec, bytes(tp)) +def ql_syscall_clock_gettime(ql: Qiling, clock_id: int, tp: int): + ts_obj = __get_timespec_obj(ql.arch.bits) + ql.mem.write(tp, bytes(ts_obj)) - ql.log.debug("clock_gettime(clock_id = %d, timespec = 0x%x)" % (clock_gettime_clock_id, clock_gettime_timespec)) - return 0 -def ql_syscall_gettimeofday(ql, gettimeofday_tv, gettimeofday_tz, *args, **kw): - now = datetime.now().timestamp() - tv_sec = floor(now) - tv_nsec = floor((now - floor(now)) * 1e6) - if ql.arch.type == QL_ARCH.X8664: - tp = timespec(tv_sec= tv_sec, tv_nsec=tv_nsec) - else: - tp = timespec32(tv_sec= tv_sec, tv_nsec=tv_nsec) +def ql_syscall_gettimeofday(ql: Qiling, tv: int, tz: int): + if tv: + ts_obj = __get_timespec_obj(ql.arch.bits) + ql.mem.write(tv, bytes(ts_obj)) - if gettimeofday_tv != 0: - ql.mem.write(gettimeofday_tv, bytes(tp)) - if gettimeofday_tz != 0: - ql.mem.write(gettimeofday_tz, b'\x00' * 8) - regreturn = 0 - return regreturn + if tz: + ql.mem.write(tz, b'\x00' * 8) + + return 0 diff --git a/qiling/os/linux/thread.py b/qiling/os/linux/thread.py index 04a6c1be..a5d0d8d2 100644 --- a/qiling/os/linux/thread.py +++ b/qiling/os/linux/thread.py @@ -14,7 +14,7 @@ from qiling import Qiling from qiling.os.thread import * from qiling.arch.x86_const import * from qiling.exception import QlErrorExecutionStop -from qiling.os.path import QlPathManager +from qiling.os.path import QlOsPath LINUX_THREAD_ID = 2000 @@ -140,8 +140,8 @@ class QlLinuxThread(QlThread): return self._path @path.setter - def path(self, p): - self._path = QlPathManager(self._ql, p.cwd) + def path(self, p: QlOsPath): + self._path = QlOsPath(self.ql.rootfs, p.cwd, self.ql.os.type) @property def log_file_fd(self): diff --git a/qiling/os/macos/macos.py b/qiling/os/macos/macos.py index ec826825..eb7f38ee 100644 --- a/qiling/os/macos/macos.py +++ b/qiling/os/macos/macos.py @@ -11,7 +11,7 @@ from unicorn.x86_const import UC_X86_INS_SYSCALL from qiling import Qiling from qiling.arch.x86_utils import GDTManager, SegmentManager64 from qiling.cc import intel -from qiling.const import QL_ARCH, QL_VERBOSE +from qiling.const import QL_ARCH, QL_OS, QL_VERBOSE from qiling.os.fcall import QlFunctionCall from qiling.os.posix.posix import QlOsPosix from qiling.os.macos.events.macos import QlMacOSEvManager @@ -20,6 +20,8 @@ from qiling.os.macos.events.macos_structs import mac_policy_list_t from qiling.os.macos.structs import kmod_info_t, POINTER64 class QlOsMacos(QlOsPosix): + type = QL_OS.MACOS + def __init__(self, ql: Qiling): super(QlOsMacos, self).__init__(ql) @@ -157,11 +159,11 @@ class QlOsMacos(QlOsPosix): self.ql.hook_insn(self.hook_syscall, UC_X86_INS_SYSCALL) - def hook_syscall(self, intno= None, int = None): + def hook_syscall(self, ql, intno = None): return self.load_syscall() - def hook_sigtrap(self, intno= None, int = None): + def hook_sigtrap(self, ql, intno): self.ql.log.info("Trap Found") self.emu_error() exit(1) diff --git a/qiling/os/macos/utils.py b/qiling/os/macos/utils.py index 351f0f0b..808fbe29 100644 --- a/qiling/os/macos/utils.py +++ b/qiling/os/macos/utils.py @@ -188,5 +188,5 @@ def page_align_end(addr, page_size): def set_eflags_cf(ql, target_cf): - ql.arch.regs.ef = ( ql.arch.regs.ef & 0xfffffffe ) | target_cf - return ql.arch.regs.ef + ql.arch.regs.eflags = ( ql.arch.regs.eflags & 0xfffffffe ) | target_cf + return ql.arch.regs.eflags diff --git a/qiling/os/mapper.py b/qiling/os/mapper.py index ed99baf9..6b79524f 100644 --- a/qiling/os/mapper.py +++ b/qiling/os/mapper.py @@ -7,7 +7,7 @@ import inspect import os from typing import Any, MutableMapping, Union -from .path import QlPathManager +from .path import QlOsPath from .filestruct import ql_file # All mapped objects should inherit this class. @@ -59,7 +59,7 @@ class QlFsMappedObject: class QlFsMapper: - def __init__(self, path: QlPathManager): + def __init__(self, path: QlOsPath): self._mapping: MutableMapping[str, Any] = {} self.path = path diff --git a/qiling/os/mcu/mcu.py b/qiling/os/mcu/mcu.py index ae3d9570..241823cc 100644 --- a/qiling/os/mcu/mcu.py +++ b/qiling/os/mcu/mcu.py @@ -3,8 +3,9 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from qiling.os.os import QlOs +from qiling.const import QL_OS from qiling.const import QL_ARCH +from qiling.os.os import QlOs from qiling.extensions.multitask import MultiTaskUnicorn, UnicornTask from unicorn import UC_ERR_OK @@ -31,9 +32,10 @@ class MCUTask(UnicornTask): class QlOsMcu(QlOs): + type = QL_OS.MCU + def __init__(self, ql): - super(QlOsMcu, self).__init__(ql) - + self.ql = ql self.runable = True self.fast_mode = False diff --git a/qiling/os/memory.py b/qiling/os/memory.py index a487e8b0..49b2bb5b 100644 --- a/qiling/os/memory.py +++ b/qiling/os/memory.py @@ -4,12 +4,11 @@ # import os, re -from typing import Any, Callable, List, MutableSequence, Optional, Sequence, Tuple +from typing import Any, Callable, Iterator, List, Mapping, MutableSequence, Optional, Pattern, Sequence, Tuple, Union from unicorn import UC_PROT_NONE, UC_PROT_READ, UC_PROT_WRITE, UC_PROT_EXEC, UC_PROT_ALL from qiling import Qiling -from qiling.const import * from qiling.exception import * # tuple: range start, range end, permissions mask, range label, is mmio? @@ -36,7 +35,7 @@ class QlMemoryManager: } if ql.arch.bits not in bit_stuff: - raise QlErrorStructConversion("Unsupported Qiling archtecture for memory manager") + raise QlErrorStructConversion("Unsupported Qiling architecture for memory manager") max_addr = bit_stuff[ql.arch.bits] @@ -128,8 +127,8 @@ class QlMemoryManager: self.map_info = tmp_map_info - def change_mapinfo(self, mem_s: int, mem_e: int, mem_p: int = None, mem_info: str = None): - tmp_map_info: MapInfoEntry = None + def change_mapinfo(self, mem_s: int, mem_e: int, mem_p: Optional[int] = None, mem_info: Optional[str] = None): + tmp_map_info: Optional[MapInfoEntry] = None info_idx: int = None for idx, map_info in enumerate(self.map_info): @@ -150,12 +149,12 @@ class QlMemoryManager: if mem_info is not None: self.map_info[info_idx] = (tmp_map_info[0], tmp_map_info[1], tmp_map_info[2], mem_info, tmp_map_info[4]) - def get_mapinfo(self) -> Sequence[Tuple[int, int, str, str, Optional[str]]]: + def get_mapinfo(self) -> Sequence[Tuple[int, int, str, str, str]]: """Get memory map info. Returns: A sequence of 5-tuples representing the memory map entries. Each tuple contains range start, range end, permissions, range label and path of - containing image (or None if not contained by any image) + containing image (or an empty string if not contained by any image) """ def __perms_mapping(ps: int) -> str: @@ -167,21 +166,21 @@ class QlMemoryManager: return ''.join(val if idx & ps else '-' for idx, val in perms_d.items()) - def __process(lbound: int, ubound: int, perms: int, label: str, is_mmio: bool) -> Tuple[int, int, str, str, Optional[str]]: + def __process(lbound: int, ubound: int, perms: int, label: str, is_mmio: bool) -> Tuple[int, int, str, str, str]: perms_str = __perms_mapping(perms) if hasattr(self.ql, 'loader'): image = self.ql.loader.find_containing_image(lbound) - container = image.path if image and not is_mmio else None + container = image.path if image and not is_mmio else '' else: - container = None + container = '' return (lbound, ubound, perms_str, label, container) return tuple(__process(*entry) for entry in self.map_info) - def show_mapinfo(self): - """Emit memory map info in a nicely formatted table. + def get_formatted_mapinfo(self) -> Sequence[str]: + """Get memory map info in a nicely formatted table. """ mapinfo = self.get_mapinfo() @@ -193,12 +192,17 @@ class QlMemoryManager: len_addr = max(grouped[0]) len_label = max(grouped[1]) - # emit title row - self.ql.log.info(f'{"Start":{len_addr}s} {"End":{len_addr}s} {"Perm":5s} {"Label":{len_label}s} {"Image"}') + # pre-allocate table + table = [''] * (len(mapinfo) + 1) - # emit table rows - for lbound, ubound, perms, label, container in mapinfo: - self.ql.log.info(f'{lbound:0{len_addr}x} - {ubound:0{len_addr}x} {perms:5s} {label:{len_label}s} {container or ""}') + # add title row + table[0] = f'{"Start":{len_addr}s} {"End":{len_addr}s} {"Perm":5s} {"Label":{len_label}s} {"Image"}' + + # add table rows + for i, (lbound, ubound, perms, label, container) in enumerate(mapinfo, 1): + table[i] = f'{lbound:0{len_addr}x} - {ubound:0{len_addr}x} {perms:5s} {label:{len_label}s} {container}' + + return table # TODO: relying on the label string is risky; find a more reliable method def get_lib_base(self, filename: str) -> Optional[int]: @@ -211,7 +215,7 @@ class QlMemoryManager: return next((lbound for lbound, info in stripped if os.path.basename(info) == filename), None) - def align(self, value: int, alignment: int = None) -> int: + def align(self, value: int, alignment: Optional[int] = None) -> int: """Align a value down to the specified alignment boundary. If `value` is already aligned, the same value is returned. Commonly used to determine the base address of the enclosing page. @@ -233,7 +237,7 @@ class QlMemoryManager: # round down to nearest alignment return value & ~(alignment - 1) - def align_up(self, value: int, alignment: int = None) -> int: + def align_up(self, value: int, alignment: Optional[int] = None) -> int: """Align a value up to the specified alignment boundary. If `value` is already aligned, the same value is returned. Commonly used to determine the end address of the enlosing page. @@ -281,7 +285,7 @@ class QlMemoryManager: self.ql.log.debug(f'restoring memory range: {lbound:#08x} {ubound:#08x} {label}') size = ubound - lbound - if not self.is_mapped(lbound, size): + if self.is_available(lbound, size): self.ql.log.debug(f'mapping {lbound:#08x} {ubound:#08x}, mapsize = {size:#x}') self.map(lbound, size, perms, label) @@ -306,13 +310,13 @@ class QlMemoryManager: return self.ql.uc.mem_read(addr, size) - def read_ptr(self, addr: int, size: int=None) -> int: + def read_ptr(self, addr: int, size: int = 0) -> int: """Read an integer value from a memory address. Bytes read will be unpacked using emulated architecture properties. Args: addr: memory address to read - size: pointer size (in bytes): either 1, 2, 4, 8, or None for arch native size + size: pointer size (in bytes): either 1, 2, 4, 8, or 0 for arch native size Returns: integer value stored at the specified memory address """ @@ -342,14 +346,14 @@ class QlMemoryManager: self.ql.uc.mem_write(addr, data) - def write_ptr(self, addr: int, value: int, size: int=None) -> None: + def write_ptr(self, addr: int, value: int, size: int = 0) -> None: """Write an integer value to a memory address. Bytes written will be packed using emulated architecture properties. Args: addr: target memory address value: integer value to write - size: pointer size (in bytes): either 1, 2, 4, 8, or None for arch native size + size: pointer size (in bytes): either 1, 2, 4, 8, or 0 for arch native size """ if not size: @@ -367,18 +371,18 @@ class QlMemoryManager: self.write(addr, __pack(value)) - def search(self, needle: bytes, begin: int = None, end: int = None) -> Sequence[int]: + def search(self, needle: Union[bytes, Pattern[bytes]], begin: Optional[int] = None, end: Optional[int] = None) -> Sequence[int]: """Search for a sequence of bytes in memory. Args: - needle: bytes sequence to look for + needle: bytes sequence or regex pattern to look for begin: search starting address (or None to start at lowest avaiable address) end: search ending address (or None to end at highest avaiable address) Returns: addresses of all matches """ - # if starting point not set, search from the first mapped region + # if starting point not set, search from the first mapped region if begin is None: begin = self.map_info[0][0] @@ -392,6 +396,10 @@ class QlMemoryManager: ranges = [(max(begin, lbound), min(ubound, end)) for lbound, ubound, _, _, is_mmio in self.map_info if not (end < lbound or ubound < begin or is_mmio)] results = [] + # if needle is a bytes sequence use it verbatim, not as a pattern + if type(needle) is bytes: + needle = re.escape(needle) + for lbound, ubound in ranges: haystack = self.read(lbound, ubound - lbound) local_results = (match.start(0) + lbound for match in re.finditer(needle, haystack)) @@ -414,17 +422,41 @@ class QlMemoryManager: if (addr, addr + size) in self.mmio_cbs: del self.mmio_cbs[(addr, addr+size)] - def unmap_all(self): + def unmap_all(self) -> None: """Reclaim the entire memory space. """ for begin, end, _ in self.ql.uc.mem_regions(): - if begin and end: - self.unmap(begin, end - begin + 1) + self.unmap(begin, end - begin + 1) + + def __mapped_regions(self) -> Iterator[Tuple[int, int]]: + """Iterate through all mapped memory regions, consolidating adjacent regions + together to a continuous one. Protection bits and labels are ignored. + """ + + if not self.map_info: + return + + iter_memmap = iter(self.map_info) + + p_lbound, p_ubound, _, _, _ = next(iter_memmap) + + # map_info is assumed to contain non-overlapping regions sorted by lbound + for lbound, ubound, _, _, _ in iter_memmap: + if lbound == p_ubound: + p_ubound = ubound + else: + yield (p_lbound, p_ubound) + + p_lbound = lbound + p_ubound = ubound + + yield (p_lbound, p_ubound) + def is_available(self, addr: int, size: int) -> bool: """Query whether the memory range starting at `addr` and is of length of `size` bytes - can be allocated. + is available for allocation. Returns: True if it can be allocated, False otherwise """ @@ -435,37 +467,23 @@ class QlMemoryManager: end = addr + size # make sure neither begin nor end are enclosed within a mapped range, or entirely enclosing one - return not any((lbound <= begin < ubound) or (lbound < end <= ubound) or (begin <= lbound < ubound <= end) for lbound, ubound, _, _, _ in self.map_info) + return not any((lbound <= begin < ubound) or (lbound < end <= ubound) or (begin <= lbound < ubound <= end) for lbound, ubound in self.__mapped_regions()) def is_mapped(self, addr: int, size: int) -> bool: """Query whether the memory range starting at `addr` and is of length of `size` bytes - is mapped, either partially or entirely. + is fully mapped. - Returns: True if any part of the specified memory range is taken, False otherwise + Returns: True if the specified memory range is taken fully, False otherwise """ - return not self.is_available(addr, size) + assert size > 0, 'expected a positive size value' - def is_free(self, address, size): - ''' - The main function of is_free first must fufull is_mapped condition. - then, check for is the mapped range empty, either fill with 0xFF or 0x00 - Returns true if mapped range is empty else return Flase - If not not mapped, map it and return true - ''' - if self.is_mapped(address, size) == True: - address_end = (address + size) - while address < address_end: - mem_read = self.ql.mem.read(address, 0x1) - if (mem_read[0] != 0x00) and (mem_read[0] != 0xFF): - return False - address += 1 - return True - else: - return True + begin = addr + end = addr + size + return any((lbound <= begin < end <= ubound) for lbound, ubound in self.__mapped_regions()) - def find_free_space(self, size: int, minaddr: int = None, maxaddr: int = None, align: int = None) -> int: + def find_free_space(self, size: int, minaddr: Optional[int] = None, maxaddr: Optional[int] = None, align: Optional[int] = None) -> int: """Locate an unallocated memory that is large enough to contain a range in size of `size` and based at `minaddr`. @@ -510,7 +528,7 @@ class QlMemoryManager: raise QlOutOfMemory('Out Of Memory') - def map_anywhere(self, size: int, minaddr: int = None, maxaddr: int = None, align: int = None, perms: int = UC_PROT_ALL, info: str = None) -> int: + def map_anywhere(self, size: int, minaddr: Optional[int] = None, maxaddr: Optional[int] = None, align: Optional[int] = None, perms: int = UC_PROT_ALL, info: Optional[str] = None) -> int: """Map a region anywhere in memory. Args: @@ -545,7 +563,7 @@ class QlMemoryManager: self.change_mapinfo(aligned_address, aligned_address + aligned_size, mem_p = perms) - def map(self, addr: int, size: int, perms: int = UC_PROT_ALL, info: str = None): + def map(self, addr: int, size: int, perms: int = UC_PROT_ALL, info: Optional[str] = None): """Map a new memory range. Args: @@ -724,7 +742,7 @@ class QlMemoryHeap: self.current_alloc = 0 self.current_use = 0 - def _find(self, addr: int, inuse: bool = None) -> Optional[Chunk]: + def _find(self, addr: int, inuse: Optional[bool] = None) -> Optional[Chunk]: """Find a chunk starting at a specified address. Args: diff --git a/qiling/os/os.py b/qiling/os/os.py index a06dacb4..93c4c012 100644 --- a/qiling/os/os.py +++ b/qiling/os/os.py @@ -4,7 +4,7 @@ # import sys -from typing import Any, Iterable, Optional, Callable, Mapping, Sequence, TextIO, Tuple +from typing import Any, Hashable, Iterable, Optional, Callable, Mapping, Sequence, TextIO, Tuple from unicorn import UcError @@ -15,10 +15,13 @@ from qiling.os.fcall import QlFunctionCall, TypedArg from .filestruct import ql_file from .mapper import QlFsMapper -from .utils import QlOsStats, QlOsUtils -from .path import QlPathManager +from .stats import QlOsStats +from .utils import QlOsUtils +from .path import QlOsPath class QlOs: + type: QL_OS + Resolver = Callable[[int], Any] def __init__(self, ql: Qiling, resolvers: Mapping[Any, Resolver] = {}): @@ -37,10 +40,10 @@ class QlOs: self.profile = self.ql.profile self.exit_code = 0 - if not ql.baremetal: + if self.type in QL_OS_POSIX + (QL_OS.WINDOWS, QL_OS.DOS): cwd = self.profile.get("MISC", "current_path") - self.path = QlPathManager(ql, cwd) + self.path = QlOsPath(ql.rootfs, cwd, self.type) self.fs_mapper = QlFsMapper(self.path) self.user_defined_api = { @@ -201,26 +204,19 @@ class QlOs: return retval - # TODO: separate this method into os-specific functionalities, instead of 'if-else' - def set_api(self, api_name: str, intercept_function: Callable, intercept: QL_INTERCEPT = QL_INTERCEPT.CALL): - """Either replace or hook OS API with a custom one. + def set_api(self, target: Hashable, handler: Callable, intercept: QL_INTERCEPT = QL_INTERCEPT.CALL): + """Either hook or replace an OS API with a custom one. Args: - api_name: target API name - intercept_function: function to call + target: target API identifier + handler: function to call intercept: `QL_INTERCEPT.CALL` : run handler instead of the existing target implementation `QL_INTERCEPT.ENTER`: run handler before the target API is called `QL_INTERCEPT.EXIT` : run handler after the target API is called """ - if self.ql.ostype == QL_OS.UEFI: - api_name = f'hook_{api_name}' - - if (self.ql.ostype in (QL_OS.WINDOWS, QL_OS.UEFI, QL_OS.DOS)) or (self.ql.ostype in (QL_OS_POSIX) and self.ql.loader.is_driver): - self.user_defined_api[intercept][api_name] = intercept_function - else: - self.add_function_hook(api_name, intercept_function, intercept) + self.user_defined_api[intercept][target] = handler # os main method; derivatives must implement one of their own def run(self) -> None: @@ -256,5 +252,6 @@ class QlOs: finally: self.ql.log.error(f'PC = {pc:#0{self.ql.arch.pointersize * 2 + 2}x}{pc_info}\n') - self.ql.log.info(f'Memory map:') - self.ql.mem.show_mapinfo() + self.ql.log.error(f'Memory map:') + for info_line in self.ql.mem.get_formatted_mapinfo(): + self.ql.log.error(info_line) diff --git a/qiling/os/path.py b/qiling/os/path.py index 85c67d9f..2cab1991 100644 --- a/qiling/os/path.py +++ b/qiling/os/path.py @@ -3,160 +3,330 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import os -from typing import Union -from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath +from typing import Optional, Union +from pathlib import Path, PurePosixPath, PureWindowsPath -from qiling import Qiling from qiling.const import QL_OS, QL_OS_POSIX -# OH-MY-WIN32 !!! -# Some codes from cygwin. -# -# Basic guide: -# We should only handle "normal" paths like "C:\Windows\System32" and "bin/a.exe" for users. -# For UNC paths like '\\.\PHYSICALDRIVE0" and "\\Server\Share", they should be implemented -# by users via fs mapping interface. -class QlPathManager: - def __init__(self, ql: Qiling, cwd: str): - self.ql = ql - self._cwd = cwd +AnyPurePath = Union[PurePosixPath, PureWindowsPath] + +class QlOsPath: + """Virtual to host path manipulations helper. + """ + + def __init__(self, rootfs: str, cwd: str, emulos: QL_OS) -> None: + """Initialize a path manipulation object. + + Args: + rootfs : host path to serve as the virtual root directory + cwd : virtual current working directory + emuls : emulated operating system + """ + + nt_path_os = (QL_OS.WINDOWS, QL_OS.DOS) + posix_path_os = QL_OS_POSIX + + # rootfs is a local directory on the host, and expected to exist + self._rootfs_path = Path(rootfs).resolve(strict=True) + + # determine how virtual paths should be handled + if emulos in nt_path_os: + self.PureVirtualPath = PureWindowsPath + + elif emulos in posix_path_os: + self.PureVirtualPath = PurePosixPath + + else: + raise ValueError(f'unexpected os type: {emulos}') + + self.cwd = cwd + + # + self.transform_to_relative_path = self.virtual_abspath + self.transform_to_real_path = self.virtual_to_host_path + self.transform_to_link_path = self.virtual_to_host_path + # + + @staticmethod + def __strip_parent_refs(path: AnyPurePath) -> AnyPurePath: + """Strip leading parent dir references, if any. + """ + + if path.parts: + pardir = r'..' + + while path.parts[0] == pardir: + path = path.relative_to(pardir) + + return path @property def cwd(self) -> str: - return self._cwd + return str(self._cwd_anchor / self._cwd_vpath) @cwd.setter - def cwd(self, c: str) -> None: - if not c.startswith('/'): - self.ql.log.warning(f'Sanity check: path does not start with a forward slash "/"') + def cwd(self, virtpath: str) -> None: + vpath = self.PureVirtualPath(virtpath) - self._cwd = c + if not vpath.is_absolute(): + raise ValueError(f'current working directory must be an absolute path: {virtpath}') - @staticmethod - def normalize(path: Union[Path, PurePath]) -> Union[Path, PurePath]: - # expected types: PosixPath, PurePosixPath, WindowsPath, PureWindowsPath - assert isinstance(path, (Path, PurePath)), f'did not expect {type(path).__name__!r} here' + # extract the virtual path anchor so we can append cwd path to rootfs later on. + # however, we will still need the anchor to provide full virtual paths + cwd_anchor = self.PureVirtualPath(vpath.anchor) + cwd_vpath = vpath.relative_to(cwd_anchor) - normalized_path = type(path)() + cwd_vpath = QlOsPath.__strip_parent_refs(cwd_vpath) - # remove anchor (necessary for Windows UNC paths) and convert to relative path - if path.is_absolute(): - path = path.relative_to(path.anchor) + self._cwd_anchor = cwd_anchor + self._cwd_vpath = cwd_vpath - for p in path.parts: - if p == '.': - continue + def __virtual_abspath(self, virtpath: Union[str, AnyPurePath]) -> AnyPurePath: + """Get the absolute virtual path representation of a virtual path. + This method does not follow symbolic links or parent directory references. - if p == '..': - normalized_path = normalized_path.parent - continue + Args: + virtpath: virtual path to resolve, either relative or absolute - normalized_path /= p + Returns: An absolute virtual path + """ - return normalized_path + vpath = self.PureVirtualPath(virtpath) - @staticmethod - def convert_win32_to_posix(rootfs: Union[str, Path], cwd: str, path: str) -> Path: - _rootfs = Path(rootfs) - _cwd = PurePosixPath(cwd[1:]) + if vpath.is_absolute(): + return vpath + + # rebase on top the current working directory. in case vpath is an absolute + # path, cwd will be discarded + absvpath = self._cwd_vpath / vpath + + # referencing root's parent directory should circle back to root. + # remove any leading parent dir references absvpath might have + absvpath = QlOsPath.__strip_parent_refs(absvpath) + + return self._cwd_anchor / absvpath + + def __resolved_vsymlink(self, basepath: Path, name: str): + """Attempt to resolve a virtual symbolic link. + + A virtual symbolic link points to a location within the virtual file + system, not to be confused with paths on the host file system. + + For example: + a vsymlink that points to '/var/tmp' should be resolved to + 'my/rootfs/path/var/tmp' on the host + """ + + fullpath = self._rootfs_path / basepath / name + vpath = None + + if fullpath.is_symlink(): + resolved = fullpath.resolve(strict=False) + + try: + # the resolve method turns fullpath into an absolute path on the host, + # but we need it to be an absolute virtual path. to convert the host + # path to virtual we have to make sure it resides within rootfs. + # + # if the host path is indeed under rootfs, we can rebase the virtual + # portion on top of rootfs and return an absolute virtual path. + # + # returning an absolute path will discard the accumulated vpath. + + vpath = self._cwd_anchor / resolved.relative_to(self._rootfs_path) + except ValueError: + # failing to convert the host path into a virtual one means that either + # the symbolic link is already an absolute virtual path, or it is pointing + # to an external directory on the host file system, which means it does + # not reflect a virtual path. + # + # on both cases we may return the path as-is, discarding the accumulated + # vpath. + # + # that, however, may not work in case the host filesystem and the virtual + # one are not of the same type (e.g. a virtual Windows path on top of a + # Linux host): the absolute path will not discard the accumulated vpath + # and will be appended - which is a corner case we do not know how to + # handle efficiently. + + vpath = resolved + + return vpath + + # this will work only if hosting os = virtual os + def __virtual_resolve(self, virtpath: Union[str, AnyPurePath]) -> AnyPurePath: + """Resolve a virtual path, including symbolic links and directory + references it might include. Path must not include circular symbolic + links. + + Args: + virtpath: virtual path to resolve, either relative or absolute + + Returns: An absolute virtual path + """ + + vpath = self.PureVirtualPath(virtpath) + + # if not already, turn vpath into an absolute path + if not vpath.is_absolute(): + vpath = self.__virtual_abspath(vpath) + + # accumulate paths as we progress through the resolution process. + # + # since symlink inspection and resolution can only be done on an + # actual file system, each step in the progress has to be translated + # into its correpsonding host path. that is the reason we keep track + # on the acumulated host path in parallel to the virtual one + # + # note: the reason we do not set acc_hpath to rootfs is to prevent + # parent dir refs from traversing beyond rootfs directory. + + acc_hpath = Path() + acc_vpath = self.PureVirtualPath(vpath.anchor) + + # eliminate virtual path's anchor to allow us accumulate its + # parts on top of rootfs + vpath = vpath.relative_to(vpath.anchor) + + for part in vpath.parts: + if part == '..': + acc_hpath = acc_hpath.parent + acc_vpath = acc_vpath.parent - # Things are complicated here. - # See https://docs.microsoft.com/zh-cn/windows/win32/fileio/naming-a-file?redirectedfrom=MSDN - if PureWindowsPath(path).is_absolute(): - if (len(path) >= 2 and path.startswith(r'\\')) or \ - (len(path) >= 3 and path[0].isalpha() and path[1:3] == ':\\'): # \\.\PhysicalDrive0 or \\Server\Share\Directory or X:\ - # UNC path should be handled in fs mapping. If not, append it to rootfs directly. - pw = PureWindowsPath(path) - result = _rootfs / QlPathManager.normalize(pw) else: - # code should never reach here. - result = _rootfs / QlPathManager.normalize(path) - else: - if len(path) >= 3 and path[:3] == r'\\?' or path[:3] == r'\??': # \??\ or \\?\ or \Device\.. - # Similair to \\.\, it should be handled in fs mapping. - pw = PureWindowsPath(path) - result = _rootfs / QlPathManager.normalize(_cwd / pw.relative_to(pw.anchor).as_posix()) - else: - # a normal relative path - result = _rootfs / QlPathManager.normalize(_cwd / PureWindowsPath(path).as_posix()) + # if this is a symlink attempt to resolve it + vtemp = self.__resolved_vsymlink(acc_hpath, part) - return result + # not a symlink; accumulate path part + if vtemp is None: + acc_hpath = acc_hpath / part + acc_vpath = acc_vpath / part + + else: + # rebase it on top of the accumulated virtual path + new_vpath = acc_vpath / vtemp + + # recursively resolve the new virtual path we got + vres = self.__virtual_resolve(new_vpath) + + acc_hpath = Path(vres) + acc_vpath = vres + + return acc_vpath + + def __virtual_to_host_path(self, virtpath: Union[str, AnyPurePath]) -> Path: + """Convert a virtual path to its corresponding path on the host. + + This method partialy normalizes the virtual path and does not resolve + references to parent directories neither virtual symbolic links + """ + + absvpath = self.__virtual_abspath(virtpath) + + # remove path anchor to allow path to be rebased + vpath = absvpath.relative_to(absvpath.anchor) + + # rebase virtual path on top of rootfs to get the host path + return self._rootfs_path / vpath + + def __is_safe_host_path(self, hostpath: Path, strict: bool = False) -> bool: + """Sanitize the specified host path and make sure it does not traverse out + of the rootfs directory hierarchy. + + Args: + hostpath : a local path to sanitize + strict : whether to raise an error if target path does not exist + + Returns: whether the path is safe to use + """ + + # canonicalization before assertion: resolve any relative path references and + # symbolic links that may exist. + # + # in case strict is set to True and the path does not exist, a FileNotFoundError + # is raised. this error is left for the user to catch and handle + hostpath = hostpath.resolve(strict=strict) + + try: + # to prevent path-traversal issues we have to make sure hostpath ended up + # as a subpath of rootfs. the following method will fail if that is not + # the case + _ = hostpath.relative_to(self._rootfs_path) + except ValueError: + return False + + else: + return True + + def virtual_abspath(self, virtpath: str) -> str: + """Convert a relative virtual path to an absolute virtual path based + on the current working directory. + + Args: + virtpath : relative virtual path + + Returns: the absolute virtual path + """ + + absvpath = self.__virtual_abspath(virtpath) + + return str(absvpath) + + def virtual_to_host_path(self, virtpath: str) -> str: + """Convert a virtual path to its corresponding path on the hosting system. + + Args: + virtpath : path on the emulated system. the path may be either absolute + or relative + + Returns: the corresponding path on the hosting system + """ + + absvpath = self.__virtual_resolve(virtpath) + hostpath = self.__virtual_to_host_path(absvpath) + + return str(hostpath) + + def is_safe_host_path(self, hostpath: str) -> bool: + hpath = Path(hostpath) + + return self.__is_safe_host_path(hpath, strict=False) @staticmethod - def convert_posix_to_win32(rootfs: Union[str, Path], cwd: str, path: str) -> Path: - _rootfs = Path(rootfs) - _cwd = PurePosixPath(cwd[1:]) - _path = PurePosixPath(path) + def __host_casefold_path(hostpath: str) -> Optional[str]: + # assuming posix host + p = PurePosixPath(hostpath) + norm = Path(p.anchor) - if _path.is_absolute(): - return _rootfs / QlPathManager.normalize(_path) - else: - return _rootfs / QlPathManager.normalize(_cwd / _path) + for elem in p.relative_to(norm).parts: + folded = elem.casefold() - @staticmethod - def convert_for_native_os(rootfs: Union[str, Path], cwd: str, path: str) -> Path: - _rootfs = Path(rootfs) - _cwd = PurePosixPath(cwd[1:]) - _path = Path(path) + try: + norm = next(entry for entry in norm.iterdir() if entry.name.casefold() == folded) + except StopIteration: + return None - if _path.is_absolute(): - return _rootfs / QlPathManager.normalize(_path) - else: - return _rootfs / QlPathManager.normalize(_cwd / _path.as_posix()) + return str(norm) - def convert_path(self, rootfs: Union[str, Path], cwd: str, path: str) -> Path: - emulated_os = self.ql.ostype - hosting_os = self.ql.host.os + def host_casefold_path(self, hostpath: str) -> Optional[str]: + """As opposed to POSIX paths, NT paths are case insensitive and may be specified + in multiple ways. When emulating an NT file system on top of POSIX one, virtual + NT paths and files might not be found because they are specified in a different + case than the one that is actually used on the hosting POSIX system. - # emulated os and hosting platform are of the same type - if (emulated_os == hosting_os) or (emulated_os in QL_OS_POSIX and hosting_os in QL_OS_POSIX): - return QlPathManager.convert_for_native_os(rootfs, cwd, path) + This method translates a case insensitive path into the actual case sensitive + name that is used on the hosting POSIX file system. - elif emulated_os in QL_OS_POSIX and hosting_os == QL_OS.WINDOWS: - return QlPathManager.convert_posix_to_win32(rootfs, cwd, path) + Args: + hostpath: a path on the host, case insensitive - elif emulated_os == QL_OS.WINDOWS and hosting_os in QL_OS_POSIX: - return QlPathManager.convert_win32_to_posix(rootfs, cwd, path) + Returns: the corresponding path on the host system, or None if the path does not + exist + """ - else: - return QlPathManager.convert_for_native_os(rootfs, cwd, path) + # only relevant if the emulated file system is NT-based + if self.PureVirtualPath is PureWindowsPath: + return QlOsPath.__host_casefold_path(hostpath) - def transform_to_link_path(self, path: str) -> str: - real_path = self.convert_path(self.ql.rootfs, self.cwd, path) + return hostpath - return str(real_path.absolute()) - - def transform_to_real_path(self, path: str) -> str: - # TODO: We really need a virtual file system. - real_path = self.convert_path(self.ql.rootfs, self.cwd, path) - - if os.path.islink(real_path): - link_path = Path(os.readlink(real_path)) - - real_path = self.convert_path(os.path.dirname(real_path), "/", link_path) - - if os.path.islink(real_path): - real_path = self.transform_to_real_path(real_path) - - # resolve multilevel symbolic link - if not os.path.exists(real_path): - path_dirs = link_path.parts - - if link_path.is_absolute(): - path_dirs = path_dirs[1:] - - for i in range(len(path_dirs) - 1): - path_prefix = os.path.sep.join(path_dirs[:i+1]) - real_path_prefix = self.transform_to_real_path(path_prefix) - path_remain = os.path.sep.join(path_dirs[i+1:]) - real_path = Path(os.path.join(real_path_prefix, path_remain)) - - if os.path.exists(real_path): - break - - return str(real_path.absolute()) - - # The `relative path` here refers to the path which is relative to the rootfs. - def transform_to_relative_path(self, path: str) -> str: - return str(Path(self.cwd) / path) diff --git a/qiling/os/posix/const.py b/qiling/os/posix/const.py index 51a03393..7340a758 100644 --- a/qiling/os/posix/const.py +++ b/qiling/os/posix/const.py @@ -515,6 +515,24 @@ linux_riscv_open_flags = { 'O_LARGEFILE': None, } +linux_ppc_open_flags = { + 'O_RDONLY': 0x0, + 'O_WRONLY': 0x1, + 'O_RDWR': 0x2, + 'O_NONBLOCK': 0x800, + 'O_APPEND': 0x400, + 'O_ASYNC': 0x2000, + 'O_SYNC': 0x101000, + 'O_NOFOLLOW': 0x8000, + 'O_CREAT': 0x40, + 'O_TRUNC': 0x200, + 'O_EXCL': 0x80, + 'O_NOCTTY': 0x100, + 'O_DIRECTORY': 0x4000, + 'O_BINARY' : None, + 'O_LARGEFILE': 0x10000, +} + freebsd_x86_open_flags = { 'O_RDONLY': 0x0, 'O_WRONLY': 0x1, diff --git a/qiling/os/posix/const_mapping.py b/qiling/os/posix/const_mapping.py index 02756b97..99fb31ca 100644 --- a/qiling/os/posix/const_mapping.py +++ b/qiling/os/posix/const_mapping.py @@ -3,11 +3,12 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from typing import MutableSequence -from .const import * +from typing import Mapping, MutableSequence from qiling import Qiling -from qiling.const import * +from qiling.const import QL_ARCH, QL_OS + +from .const import * def _invert_dict(d: Mapping) -> Mapping: return { v:k for k, v in d.items()} @@ -34,22 +35,27 @@ def _constant_mapping(bits: int, d_map: Mapping[str, int], ret: MutableSequence[ def ql_open_flag_mapping(ql: Qiling, flags): - def flag_mapping(flags, mapping_name, mapping_from, mapping_to): + def flag_mapping(flags, mapping_name, mapping_from, mapping_to, host_os, virt_os): ret = 0 for n in mapping_name: if mapping_from[n] is None or mapping_to[n] is None: continue if (flags & mapping_from[n]) == mapping_from[n]: ret = ret | mapping_to[n] + if (host_os == QL_OS.WINDOWS and virt_os != QL_OS.WINDOWS): + ret = ret | mapping_to['O_BINARY'] return ret f = {} t = {} - if ql.host.os == None: + host_os = ql.host.os + virt_os = ql.os.type + + if host_os is None: return flags - - if ql.ostype == QL_OS.LINUX: + + if virt_os == QL_OS.LINUX: if ql.arch.type in (QL_ARCH.X86, QL_ARCH.X8664): f = linux_x86_open_flags elif ql.arch.type in (QL_ARCH.ARM, QL_ARCH.ARM64): @@ -58,30 +64,32 @@ def ql_open_flag_mapping(ql: Qiling, flags): f = linux_mips_open_flags elif ql.arch.type in (QL_ARCH.RISCV, QL_ARCH.RISCV64): f = linux_riscv_open_flags + elif ql.arch.type == QL_ARCH.PPC: + f = linux_ppc_open_flags - elif ql.ostype == QL_OS.MACOS: + elif virt_os == QL_OS.MACOS: if ql.arch.type in (QL_ARCH.X86, QL_ARCH.X8664): f = macos_x86_open_flags - elif ql.ostype == QL_OS.FREEBSD: + elif virt_os == QL_OS.FREEBSD: f = freebsd_x86_open_flags - elif ql.ostype == QL_OS.WINDOWS: + elif virt_os == QL_OS.WINDOWS: f = windows_x86_open_flags - elif ql.ostype == QL_OS.QNX: + elif virt_os == QL_OS.QNX: f = qnx_arm64_open_flags - if ql.host.os == QL_OS.LINUX: + if host_os == QL_OS.LINUX: t = linux_x86_open_flags - elif ql.host.os == QL_OS.MACOS: + elif host_os == QL_OS.MACOS: t = macos_x86_open_flags - elif ql.host.os == QL_OS.FREEBSD: + elif host_os == QL_OS.FREEBSD: t = freebsd_x86_open_flags - elif ql.host.os == QL_OS.WINDOWS: + elif host_os == QL_OS.WINDOWS: t = windows_x86_open_flags if f == t: return flags - return flag_mapping(flags, open_flags_name, f, t) + return flag_mapping(flags, open_flags_name, f, t, host_os, virt_os) def mmap_flag_mapping(flags): diff --git a/qiling/os/posix/posix.py b/qiling/os/posix/posix.py index 651324df..6f5041c9 100644 --- a/qiling/os/posix/posix.py +++ b/qiling/os/posix/posix.py @@ -4,27 +4,30 @@ # from inspect import signature, Parameter -from typing import TextIO, Union, Callable +from typing import TextIO, Union, Callable, IO, List, Optional from unicorn.arm64_const import UC_ARM64_REG_X8, UC_ARM64_REG_X16 -from unicorn.arm_const import UC_ARM_REG_R7 +from unicorn.arm_const import ( + UC_ARM_REG_R0, UC_ARM_REG_R1, UC_ARM_REG_R2, UC_ARM_REG_R3, + UC_ARM_REG_R4, UC_ARM_REG_R5, UC_ARM_REG_R7, UC_ARM_REG_R12 +) from unicorn.mips_const import UC_MIPS_REG_V0 -from unicorn.x86_const import UC_X86_REG_EAX, UC_X86_REG_RAX +from unicorn.x86_const import ( + UC_X86_REG_EAX, UC_X86_REG_EBX, UC_X86_REG_ECX, UC_X86_REG_EDX, + UC_X86_REG_ESI, UC_X86_REG_EDI, UC_X86_REG_EBP, UC_X86_REG_RDI, + UC_X86_REG_RSI, UC_X86_REG_RDX, UC_X86_REG_R10, UC_X86_REG_R8, + UC_X86_REG_R9, UC_X86_REG_RAX +) from unicorn.riscv_const import UC_RISCV_REG_A7 +from unicorn.ppc_const import UC_PPC_REG_0 from qiling import Qiling -from qiling.cc import QlCC, intel, arm, mips, riscv +from qiling.cc import QlCC, intel, arm, mips, riscv, ppc from qiling.const import QL_ARCH, QL_OS, QL_INTERCEPT from qiling.exception import QlErrorSyscallNotFound from qiling.os.os import QlOs -from qiling.os.posix.const import errors -from qiling.utils import QlFileDes, ostype_convert_str, ql_get_module_function, ql_syscall_mapping_function - -from qiling.os.posix.syscall import * -from qiling.os.linux.syscall import * -from qiling.os.macos.syscall import * -from qiling.os.freebsd.syscall import * -from qiling.os.qnx.syscall import * +from qiling.os.posix.const import NR_OPEN, errors +from qiling.utils import ql_get_module, ql_get_module_function SYSCALL_PREF: str = f'ql_syscall_' @@ -58,6 +61,36 @@ class riscv32(riscv.riscv): class riscv64(riscv.riscv): pass +class ppc(ppc.ppc): + pass + + +class QlFileDes: + def __init__(self): + self.__fds: List[Optional[IO]] = [None] * NR_OPEN + + def __len__(self): + return len(self.__fds) + + def __getitem__(self, idx: Union[slice, int]): + return self.__fds[idx] + + def __setitem__(self, idx: int, val: Optional[IO]): + self.__fds[idx] = val + + def __iter__(self): + return iter(self.__fds) + + def __repr__(self): + return repr(self.__fds) + + def save(self): + return self.__fds + + def restore(self, fds): + self.__fds = fds + + class QlOsPosix(QlOs): def __init__(self, ql: Qiling): @@ -86,14 +119,15 @@ class QlOsPosix(QlOs): QL_ARCH.X86 : UC_X86_REG_EAX, QL_ARCH.X8664 : UC_X86_REG_RAX, QL_ARCH.RISCV : UC_RISCV_REG_A7, - QL_ARCH.RISCV64 : UC_RISCV_REG_A7 + QL_ARCH.RISCV64 : UC_RISCV_REG_A7, + QL_ARCH.PPC : UC_PPC_REG_0 }[self.ql.arch.type] # handle some special cases - if (self.ql.arch.type == QL_ARCH.ARM64) and (self.ql.ostype == QL_OS.MACOS): + if (self.ql.arch.type == QL_ARCH.ARM64) and (self.type == QL_OS.MACOS): self.__syscall_id_reg = UC_ARM64_REG_X16 - elif (self.ql.arch.type == QL_ARCH.ARM) and (self.ql.ostype == QL_OS.QNX): + elif (self.ql.arch.type == QL_ARCH.ARM) and (self.type == QL_OS.QNX): self.__syscall_id_reg = UC_ARM_REG_R12 # TODO: use abstract to access __syscall_cc and __syscall_id_reg by defining a system call class @@ -104,11 +138,12 @@ class QlOsPosix(QlOs): QL_ARCH.X86 : intel32, QL_ARCH.X8664 : intel64, QL_ARCH.RISCV : riscv32, - QL_ARCH.RISCV64 : riscv64 + QL_ARCH.RISCV64 : riscv64, + QL_ARCH.PPC : ppc }[self.ql.arch.type](self.ql.arch) # select syscall mapping function based on emulated OS and architecture - self.syscall_mapper = ql_syscall_mapping_function(self.ql.ostype, self.ql.arch.type) + self.syscall_mapper = self.__get_syscall_mapper(self.ql.arch.type) self._fd = QlFileDes() @@ -121,6 +156,14 @@ class QlOsPosix(QlOs): self._shms = {} + def __get_syscall_mapper(self, archtype: QL_ARCH): + qlos_path = f'.os.{self.type.name.lower()}.map_syscall' + qlos_func = 'get_syscall_mapper' + + func = ql_get_module_function(qlos_path, qlos_func) + + return func(archtype) + @QlOs.stdin.setter def stdin(self, stream: TextIO) -> None: self._stdin = stream @@ -162,6 +205,13 @@ class QlOsPosix(QlOs): self.posix_syscall_hooks[intercept][target] = handler + def set_api(self, target: str, handler: Callable, intercept: QL_INTERCEPT = QL_INTERCEPT.CALL): + if self.ql.loader.is_driver: + super().set_api(target, handler, intercept) + else: + self.function_hook.add_function_hook(target, handler, intercept) + + @staticmethod def getNameFromErrorCode(ret: int) -> str: """Return the hex representation of a return value and if possible @@ -195,9 +245,9 @@ class QlOsPosix(QlOs): if not syscall_hook: def __get_os_module(osname: str): - return ql_get_module_function(f'qiling.os.{osname.lower()}', 'syscall') + return ql_get_module(f'.os.{osname.lower()}.syscall') - os_syscalls = __get_os_module(ostype_convert_str(self.ql.ostype)) + os_syscalls = __get_os_module(self.type.name) posix_syscalls = __get_os_module('posix') # look in os-specific and posix syscall hooks diff --git a/qiling/os/posix/syscall/__init__.py b/qiling/os/posix/syscall/__init__.py index ef20cef5..4e87d9b6 100644 --- a/qiling/os/posix/syscall/__init__.py +++ b/qiling/os/posix/syscall/__init__.py @@ -7,9 +7,11 @@ from .futex import * from .ioctl import * from .mman import * from .net import * +from .personality import * from .poll import * from .prctl import * from .ptrace import * +from .random import * from .resource import * from .sched import * from .select import * @@ -25,4 +27,3 @@ from .uio import * from .unistd import * from .utsname import * from .wait import * -from .random import * diff --git a/qiling/os/posix/syscall/fcntl.py b/qiling/os/posix/syscall/fcntl.py index 97681f54..54b00555 100644 --- a/qiling/os/posix/syscall/fcntl.py +++ b/qiling/os/posix/syscall/fcntl.py @@ -27,7 +27,7 @@ def ql_syscall_open(ql: Qiling, filename: int, flags: int, mode: int): regreturn = -EMFILE else: try: - if ql.arch.type == QL_ARCH.ARM and ql.ostype != QL_OS.QNX: + if ql.arch.type == QL_ARCH.ARM and ql.os.type != QL_OS.QNX: mode = 0 flags = ql_open_flag_mapping(ql, flags) @@ -131,8 +131,8 @@ def ql_syscall_fcntl(ql: Qiling, fd: int, cmd: int, arg: int): if arg not in range(NR_OPEN): regreturn = -EINVAL - for idx, val in enumerate(ql.os.fd, arg): - if val is None: + for idx in range(arg, len(ql.os.fd)): + if ql.os.fd[idx] is None: ql.os.fd[idx] = f.dup() regreturn = idx break @@ -173,11 +173,13 @@ def ql_syscall_fcntl64(ql: Qiling, fd: int, cmd: int, arg: int): if arg not in range(NR_OPEN): regreturn = -1 - for idx, val in enumerate(ql.os.fd, arg): - if val is None: + for idx in range(arg, len(ql.os.fd)): + if ql.os.fd[idx] is None: ql.os.fd[idx] = f.dup() regreturn = idx break + else: + regreturn = -1 elif cmd == F_GETFL: regreturn = 2 diff --git a/qiling/os/posix/syscall/mman.py b/qiling/os/posix/syscall/mman.py index 0a09f1f8..5e3eae0b 100755 --- a/qiling/os/posix/syscall/mman.py +++ b/qiling/os/posix/syscall/mman.py @@ -73,7 +73,7 @@ def syscall_mmap_impl(ql: Qiling, addr: int, mlen: int, prot: int, flags: int, f if ver == 2: pgoffset = pgoffset * pagesize - elif ql.arch.type == QL_ARCH.ARM and ql.ostype == QL_OS.QNX: + elif ql.arch.type == QL_ARCH.ARM and ql.os.type == QL_OS.QNX: MAP_ANONYMOUS = 0x00080000 fd = ql.unpack32s(ql.pack32s(fd)) @@ -86,7 +86,7 @@ def syscall_mmap_impl(ql: Qiling, addr: int, mlen: int, prot: int, flags: int, f mmap_base = addr mmap_size = ql.mem.align_up(mlen - (addr & (pagesize - 1))) - if ql.ostype != QL_OS.QNX: + if ql.os.type != QL_OS.QNX: mmap_base = ql.mem.align(mmap_base) if (flags & MAP_FIXED) and mmap_base != addr: diff --git a/qiling/os/posix/syscall/personality.py b/qiling/os/posix/syscall/personality.py new file mode 100644 index 00000000..91a86925 --- /dev/null +++ b/qiling/os/posix/syscall/personality.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +import os + + +from qiling import Qiling +from qiling.const import * + +def ql_syscall_personality(ql: Qiling, persona: int): + regreturn = 0 + return regreturn diff --git a/qiling/os/posix/syscall/resource.py b/qiling/os/posix/syscall/resource.py index c244ad55..f04251bc 100644 --- a/qiling/os/posix/syscall/resource.py +++ b/qiling/os/posix/syscall/resource.py @@ -19,9 +19,16 @@ import os from qiling import Qiling def __getrlimit_common(ql: Qiling, res: int, rlim: int) -> int: - rlimit = resource.getrlimit(res) - ql.mem.write(rlim, ql.pack32s(rlimit[0]) + ql.pack32s(rlimit[1])) - + RLIMIT_STACK = 3 + if res == RLIMIT_STACK: + if ql.arch.bits == 64: + stack_size = int(ql.os.profile.get("OS64", "stack_size"), 16) + elif ql.arch.bits == 32: + stack_size = int(ql.os.profile.get("OS32", "stack_size"), 16) + rlimit = (stack_size, -1) + else: + rlimit = resource.getrlimit(res) + ql.mem.write(rlim, ql.pack64s(rlimit[0]) + ql.pack64s(rlimit[1])) return 0 def ql_syscall_ugetrlimit(ql: Qiling, res: int, rlim: int): @@ -30,23 +37,29 @@ def ql_syscall_ugetrlimit(ql: Qiling, res: int, rlim: int): def ql_syscall_getrlimit(ql: Qiling, res: int, rlim: int): return __getrlimit_common(ql, res, rlim) -def ql_syscall_setrlimit(ql: Qiling, setrlimit_resource: int, setrlimit_rlim: int): +def ql_syscall_setrlimit(ql: Qiling, res: int, rlim: int): # maybe we can nop the setrlimit - tmp_rlim = (ql.unpack32s(ql.mem.read(setrlimit_rlim, 4)), ql.unpack32s(ql.mem.read(setrlimit_rlim + 4, 4))) - resource.setrlimit(setrlimit_resource, tmp_rlim) + tmp_rlim = (ql.unpack32s(ql.mem.read(rlim, 4)), ql.unpack32s(ql.mem.read(rlim + 4, 4))) + resource.setrlimit(res, tmp_rlim) return 0 def ql_syscall_prlimit64(ql: Qiling, pid: int, res: int, new_limit: int, old_limit: int): # setrlimit() and getrlimit() if pid == 0 and new_limit == 0: - rlim = resource.getrlimit(res) - ql.mem.write(old_limit, ql.packs(rlim[0]) + ql.packs(rlim[1])) - - return 0 + try: + rlim = resource.getrlimit(res) + ql.mem.write(old_limit, ql.packs(rlim[0]) + ql.packs(rlim[1])) + return 0 + except: + return -1 # set other process which pid != 0 return -1 -def ql_syscall_getpriority(ql: Qiling, getpriority_which: int, getpriority_who: int): - return os.getpriority(getpriority_which, getpriority_who) +def ql_syscall_getpriority(ql: Qiling, which: int, who: int): + try: + regreturn = os.getpriority(which, who) + except: + regreturn = -1 + return regreturn diff --git a/qiling/os/posix/syscall/socket.py b/qiling/os/posix/syscall/socket.py index a27baa55..a80c7b99 100644 --- a/qiling/os/posix/syscall/socket.py +++ b/qiling/os/posix/syscall/socket.py @@ -75,7 +75,7 @@ def ql_syscall_socket(ql: Qiling, socket_domain, socket_type, socket_protocol): # ql_socket.open should use host platform based socket_type. try: emu_socket_value = socket_type - emu_socket_type = socket_type_mapping(socket_type, ql.arch.type, ql.ostype) + emu_socket_type = socket_type_mapping(socket_type, ql.arch.type, ql.os.type) socket_type = getattr(socket, emu_socket_type) ql.log.debug(f'Convert emu_socket_type {emu_socket_type}:{emu_socket_value} to host platform based socket_type {emu_socket_type}:{socket_type}') @@ -96,8 +96,8 @@ def ql_syscall_socket(ql: Qiling, socket_domain, socket_type, socket_protocol): ql.log.debug(f'{e}: {socket_domain=}, {socket_type=}, {socket_protocol=}') regreturn = -1 - socket_type = socket_type_mapping(socket_type, ql.arch.type, ql.ostype) - socket_domain = socket_domain_mapping(socket_domain, ql.arch.type, ql.ostype) + socket_type = socket_type_mapping(socket_type, ql.arch.type, ql.os.type) + socket_domain = socket_domain_mapping(socket_domain, ql.arch.type, ql.os.type) ql.log.debug("socket(%s, %s, %s) = %d" % (socket_domain, socket_type, socket_protocol, regreturn)) return regreturn @@ -151,7 +151,7 @@ def ql_syscall_getsockopt(ql: Qiling, sockfd, level, optname, optval_addr, optle try: emu_level = level - emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.ostype) + emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.os.type) level = getattr(socket, emu_level_name) ql.log.debug("Convert emu_level {}:{} to host platform based level {}:{}".format( emu_level_name, emu_level, emu_level_name, level)) @@ -168,12 +168,12 @@ def ql_syscall_getsockopt(ql: Qiling, sockfd, level, optname, optval_addr, optle try: emu_opt = optname - emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.ostype) + emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.os.type) # emu_opt_name is based on level if emu_level_name == "IPPROTO_IP": - emu_opt_name = socket_ip_option_mapping(emu_opt, ql.arch.type, ql.ostype) + emu_opt_name = socket_ip_option_mapping(emu_opt, ql.arch.type, ql.os.type) else: - emu_opt_name = socket_option_mapping(emu_opt, ql.arch.type, ql.ostype) + emu_opt_name = socket_option_mapping(emu_opt, ql.arch.type, ql.os.type) # Fix for mips if ql.arch.type == QL_ARCH.MIPS: @@ -212,7 +212,7 @@ def ql_syscall_setsockopt(ql: Qiling, sockfd, level, optname, optval_addr, optle try: try: emu_level = level - emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.ostype) + emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.os.type) level = getattr(socket, emu_level_name) ql.log.debug("Convert emu_level {}:{} to host platform based level {}:{}".format( emu_level_name, emu_level, emu_level_name, level)) @@ -229,12 +229,12 @@ def ql_syscall_setsockopt(ql: Qiling, sockfd, level, optname, optval_addr, optle try: emu_opt = optname - emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.ostype) + emu_level_name = socket_level_mapping(emu_level, ql.arch.type, ql.os.type) # emu_opt_name is based on level if emu_level_name == "IPPROTO_IP": - emu_opt_name = socket_ip_option_mapping(emu_opt, ql.arch.type, ql.ostype) + emu_opt_name = socket_ip_option_mapping(emu_opt, ql.arch.type, ql.os.type) else: - emu_opt_name = socket_option_mapping(emu_opt, ql.arch.type, ql.ostype) + emu_opt_name = socket_option_mapping(emu_opt, ql.arch.type, ql.os.type) # Fix for mips if ql.arch.type == QL_ARCH.MIPS: diff --git a/qiling/os/posix/syscall/stat.py b/qiling/os/posix/syscall/stat.py index d13873ae..efc80911 100644 --- a/qiling/os/posix/syscall/stat.py +++ b/qiling/os/posix/syscall/stat.py @@ -819,6 +819,108 @@ class LinuxRISCVStat(ctypes.Structure): _pack_ = 8 +# Srouce: https://elixir.bootlin.com/linux/latest/source/arch/powerpc/include/uapi/asm/stat.h#L30 +# struct stat { +# unsigned long st_dev; +# ino_t st_ino; +# #ifdef __powerpc64__ +# unsigned long st_nlink; +# mode_t st_mode; +# #else +# mode_t st_mode; +# unsigned short st_nlink; +# #endif +# uid_t st_uid; +# gid_t st_gid; +# unsigned long st_rdev; +# long st_size; +# unsigned long st_blksize; +# unsigned long st_blocks; +# unsigned long st_atime; +# unsigned long st_atime_nsec; +# unsigned long st_mtime; +# unsigned long st_mtime_nsec; +# unsigned long st_ctime; +# unsigned long st_ctime_nsec; +# unsigned long __unused4; +# unsigned long __unused5; +# #ifdef __powerpc64__ +# unsigned long __unused6; +# #endif +# }; + +class LinuxPPCStat(ctypes.BigEndianStructure): + _fields_ = [ + ("st_dev", ctypes.c_uint32), + ("st_ino", ctypes.c_uint32), + ("st_mode", ctypes.c_uint32), + ("st_nlink", ctypes.c_uint16), + ("st_uid", ctypes.c_uint32), + ("st_gid", ctypes.c_uint32), + ("st_rdev", ctypes.c_uint32), + ("st_size", ctypes.c_uint32), + ("st_blksize", ctypes.c_uint32), + ("st_blocks", ctypes.c_uint32), + ("st_atime", ctypes.c_uint32), + ("st_atime_ns", ctypes.c_uint32), + ("st_mtime", ctypes.c_uint32), + ("st_mtime_ns", ctypes.c_uint32), + ("st_ctime", ctypes.c_uint32), + ("st_ctime_ns", ctypes.c_uint32), + ("__unused4", ctypes.c_uint32), + ("__unused5", ctypes.c_uint32) + ] + + _pack_ = 8 + +# Srouce: https://elixir.bootlin.com/linux/latest/source/arch/powerpc/include/uapi/asm/stat.h#L60 +# struct stat64 { +# unsigned long long st_dev; /* Device. */ +# unsigned long long st_ino; /* File serial number. */ +# unsigned int st_mode; /* File mode. */ +# unsigned int st_nlink; /* Link count. */ +# unsigned int st_uid; /* User ID of the file's owner. */ +# unsigned int st_gid; /* Group ID of the file's group. */ +# unsigned long long st_rdev; /* Device number, if device. */ +# unsigned short __pad2; +# long long st_size; /* Size of file, in bytes. */ +# int st_blksize; /* Optimal block size for I/O. */ +# long long st_blocks; /* Number 512-byte blocks allocated. */ +# int st_atime; /* Time of last access. */ +# unsigned int st_atime_nsec; +# int st_mtime; /* Time of last modification. */ +# unsigned int st_mtime_nsec; +# int st_ctime; /* Time of last status change. */ +# unsigned int st_ctime_nsec; +# unsigned int __unused4; +# unsigned int __unused5; +# }; + +class LinuxPPCStat64(ctypes.BigEndianStructure): + _fields_ = [ + ("st_dev", ctypes.c_uint64), + ("st_ino", ctypes.c_uint64), + ("st_mode", ctypes.c_uint32), + ("st_nlink", ctypes.c_uint32), + ("st_uid", ctypes.c_uint32), + ("st_gid", ctypes.c_uint32), + ("st_rdev", ctypes.c_uint64), + ("__pad2", ctypes.c_uint16), + ("st_size", ctypes.c_uint64), + ("st_blksize", ctypes.c_uint32), + ("st_blocks", ctypes.c_uint64), + ("st_atime", ctypes.c_uint32), + ("st_atime_ns", ctypes.c_uint32), + ("st_mtime", ctypes.c_uint32), + ("st_mtime_ns", ctypes.c_uint32), + ("st_ctime", ctypes.c_uint32), + ("st_ctime_ns", ctypes.c_uint32), + ("__unused4", ctypes.c_uint32), + ("__unused5", ctypes.c_uint32) + ] + + _pack_ = 8 + # Source: openqnx lib/c/public/sys/stat.h # # struct stat { @@ -981,8 +1083,8 @@ class QNXARMStat64(ctypes.Structure): def get_stat64_struct(ql: Qiling): if ql.arch.bits == 64: - ql.log.warning(f"Trying to stat64 on a 64bit system with {ql.ostype} and {ql.arch.type}!") - if ql.ostype == QL_OS.LINUX: + ql.log.warning(f"Trying to stat64 on a 64bit system with {ql.os.type} and {ql.arch.type}!") + if ql.os.type == QL_OS.LINUX: if ql.arch.type == QL_ARCH.X86: return LinuxX86Stat64() elif ql.arch.type == QL_ARCH.MIPS: @@ -991,22 +1093,24 @@ def get_stat64_struct(ql: Qiling): return LinuxARMStat64() elif ql.arch.type in (QL_ARCH.RISCV, QL_ARCH.RISCV64): return LinuxRISCVStat() - elif ql.ostype == QL_OS.MACOS: + elif ql.arch.type == QL_ARCH.PPC: + return LinuxPPCStat64() + elif ql.os.type == QL_OS.MACOS: return MacOSStat64() - elif ql.ostype == QL_OS.QNX: + elif ql.os.type == QL_OS.QNX: return QNXARMStat64() - ql.log.warning(f"Unrecognized arch && os with {ql.arch.type} and {ql.ostype} for stat64! Fallback to Linux x86.") + ql.log.warning(f"Unrecognized arch && os with {ql.arch.type} and {ql.os.type} for stat64! Fallback to Linux x86.") return LinuxX86Stat64() def get_stat_struct(ql: Qiling): - if ql.ostype == QL_OS.FREEBSD: + if ql.os.type == QL_OS.FREEBSD: if ql.arch.type == QL_ARCH.X8664 or ql.arch.bits == 64: return FreeBSDX8664Stat() else: return FreeBSDX86Stat() - elif ql.ostype == QL_OS.MACOS: + elif ql.os.type == QL_OS.MACOS: return MacOSStat() - elif ql.ostype == QL_OS.LINUX: + elif ql.os.type == QL_OS.LINUX: if ql.arch.type == QL_ARCH.X8664: return LinuxX8664Stat() elif ql.arch.type == QL_ARCH.X86: @@ -1034,7 +1138,9 @@ def get_stat_struct(ql: Qiling): return LinuxARM64EBStat() elif ql.arch.type in (QL_ARCH.RISCV, QL_ARCH.RISCV64): return LinuxRISCVStat() - elif ql.ostype == QL_OS.QNX: + elif ql.archtype == QL_ARCH.PPC: + return LinuxPPCStat() + elif ql.os.type == QL_OS.QNX: if ql.arch.type == QL_ARCH.ARM64: return QNXARM64Stat() elif ql.arch.type == QL_ARCH.ARM: @@ -1042,7 +1148,7 @@ def get_stat_struct(ql: Qiling): return QNXARMStat() else: return QNXARMEBStat() - ql.log.warning(f"Unrecognized arch && os with {ql.arch.type} and {ql.ostype} for stat! Fallback to Linux x86.") + ql.log.warning(f"Unrecognized arch && os with {ql.arch.type} and {ql.os.type} for stat! Fallback to Linux x86.") return LinuxX86Stat() def __common_pack_stat_struct(stat, info) -> bytes: @@ -1102,11 +1208,11 @@ def transform_path(ql: Qiling, dirfd: int, path: int, flags: int = 0): then the target file is the one referred to by the file descriptor dirfd. """ - dirfd = ql.unpacks(ql.pack(dirfd)) + dirfd = ql.unpack32s(ql.pack32(dirfd & (1<<32)-1)) path = ql.os.utils.read_cstring(path) if path.startswith('/'): - return None, os.path.join(ql.rootfs, path) + return None, os.path.join(ql.rootfs, path.lstrip('/')) if dirfd == AT_FDCWD: return None, ql.os.path.transform_to_real_path(path) @@ -1119,15 +1225,26 @@ def transform_path(ql: Qiling, dirfd: int, path: int, flags: int = 0): def ql_syscall_chmod(ql: Qiling, filename: int, mode: int): + file_path = ql.os.utils.read_cstring(filename) + real_path = ql.os.path.transform_to_real_path(file_path) + try: + os.chmod(real_path, mode) + regreturn = 0 + except: + regreturn = -1 ql.log.debug(f'chmod("{ql.os.utils.read_cstring(filename)}", {mode:d}) = 0') - - return 0 + return regreturn def ql_syscall_fchmod(ql: Qiling, fd: int, mode: int): if fd not in range(NR_OPEN) or ql.os.fd[fd] is None: return -EBADF - - return 0 + try: + os.fchmod(ql.os.fd[fd].fileno(), mode) + regreturn = 0 + except: + regreturn = -1 + ql.log.debug("fchmod(%d, %d) = %d" % (fd, mode, regreturn)) + return regreturn def ql_syscall_fstatat64(ql: Qiling, dirfd: int, path: int, buf_ptr: int, flags: int): dirfd, real_path = transform_path(ql, dirfd, path, flags) @@ -1354,6 +1471,7 @@ def ql_syscall_mknodat(ql: Qiling, dirfd: int, path: int, mode: int, dev: int): except: regreturn = -1 + ql.log.debug("mknodat(%d, %s, 0%o, %d) = %d" % (dirfd, real_path, mode, dev, regreturn)) return regreturn @@ -1368,6 +1486,7 @@ def ql_syscall_mkdir(ql: Qiling, pathname: int, mode: int): except: regreturn = -1 + ql.log.debug("mkdir(%s, 0%o) = %d" % (real_path, mode, regreturn)) return regreturn def ql_syscall_rmdir(ql: Qiling, pathname: int): diff --git a/qiling/os/posix/syscall/unistd.py b/qiling/os/posix/syscall/unistd.py index 44b1a01b..fe0c01ca 100644 --- a/qiling/os/posix/syscall/unistd.py +++ b/qiling/os/posix/syscall/unistd.py @@ -12,7 +12,7 @@ from typing import Iterator from multiprocessing import Process from qiling import Qiling -from qiling.const import QL_ARCH, QL_OS, QL_VERBOSE +from qiling.const import QL_ARCH, QL_OS from qiling.os.posix.filestruct import ql_pipe from qiling.os.posix.const import * from qiling.os.posix.stat import Stat @@ -116,6 +116,27 @@ def ql_syscall_capset(ql: Qiling, hdrp: int, datap: int): def ql_syscall_kill(ql: Qiling, pid: int, sig: int): return 0 + +def ql_syscall_fsync(ql: Qiling, fd: int): + try: + os.fsync(ql.os.fd[fd].fileno()) + regreturn = 0 + except: + regreturn = -1 + ql.log.debug("fsync(%d) = %d" % (fd, regreturn)) + return regreturn + + +def ql_syscall_fdatasync(ql: Qiling, fd: int): + try: + os.fdatasync(ql.os.fd[fd].fileno()) + regreturn = 0 + except: + regreturn = -1 + ql.log.debug("fdatasync(%d) = %d" % (fd, regreturn)) + return regreturn + + def ql_syscall_faccessat(ql: Qiling, dfd: int, filename: int, mode: int): access_path = ql.os.utils.read_cstring(filename) real_path = ql.os.path.transform_to_real_path(access_path) @@ -159,12 +180,20 @@ def ql_syscall_lseek(ql: Qiling, fd: int, offset: int, origin: int): def ql_syscall__llseek(ql: Qiling, fd: int, offset_high: int, offset_low: int, result: int, whence: int): + if fd not in range(NR_OPEN): + return -EBADF + + f = ql.os.fd[fd] + + if f is None: + return -EBADF + # treat offset as a signed value offset = ql.unpack64s(ql.pack64((offset_high << 32) | offset_low)) origin = whence try: - ret = ql.os.fd[fd].seek(offset, origin) + ret = f.seek(offset, origin) except OSError: regreturn = -1 else: @@ -181,13 +210,14 @@ def ql_syscall_brk(ql: Qiling, inp: int): # otherwise, just return current brk_address if inp: - new_brk_addr = ((inp + 0xfff) // 0x1000) * 0x1000 + cur_brk_addr = ql.loader.brk_address + new_brk_addr = ql.mem.align_up(inp) - if inp > ql.loader.brk_address: # increase current brk_address if inp is greater - ql.mem.map(ql.loader.brk_address, new_brk_addr - ql.loader.brk_address, info="[brk]") + if inp > cur_brk_addr: # increase current brk_address if inp is greater + ql.mem.map(cur_brk_addr, new_brk_addr - cur_brk_addr, info="[brk]") - elif inp < ql.loader.brk_address: # shrink current bkr_address to inp if its smaller - ql.mem.unmap(new_brk_addr, ql.loader.brk_address - new_brk_addr) + elif inp < cur_brk_addr: # shrink current bkr_address to inp if its smaller + ql.mem.unmap(new_brk_addr, cur_brk_addr - new_brk_addr) ql.loader.brk_address = new_brk_addr @@ -278,6 +308,14 @@ def ql_syscall_read(ql: Qiling, fd, buf: int, length: int): def ql_syscall_write(ql: Qiling, fd: int, buf: int, count: int): + if fd not in range(NR_OPEN): + return -EBADF + + f = ql.os.fd[fd] + + if f is None: + return -EBADF + try: data = ql.mem.read(buf, count) except: @@ -285,12 +323,14 @@ def ql_syscall_write(ql: Qiling, fd: int, buf: int, count: int): else: ql.log.debug(f'write() CONTENT: {bytes(data)}') - if hasattr(ql.os.fd[fd], 'write'): - ql.os.fd[fd].write(data) + if hasattr(f, 'write'): + f.write(data) + + regreturn = count else: ql.log.warning(f'write failed since fd {fd:d} does not have a write method') + regreturn = -1 - regreturn = count return regreturn @@ -461,7 +501,6 @@ def ql_syscall_execve(ql: Qiling, pathname: int, argv: int, envp: int): if hasattr(ql.arch, 'msr'): ql.arch.msr.uc = uc - ql.uc = uc QlCoreHooks.__init__(ql, uc) ql.os.load() @@ -517,8 +556,6 @@ def ql_syscall_dup3(ql: Qiling, fd: int, newfd: int, flags: int): def ql_syscall_set_tid_address(ql: Qiling, tidptr: int): if ql.os.thread_management: - ql.os.thread_management.cur_thread.set_clear_child_tid_addr(tidptr) - regreturn = ql.os.thread_management.cur_thread.id else: regreturn = os.getpid() diff --git a/qiling/os/qnx/qnx.py b/qiling/os/qnx/qnx.py index 403533be..9f79f1d9 100644 --- a/qiling/os/qnx/qnx.py +++ b/qiling/os/qnx/qnx.py @@ -3,10 +3,8 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from typing import Callable import os -from typing import Callable from unicorn import UcError from qiling import Qiling @@ -16,27 +14,29 @@ from qiling.os.qnx.const import NTO_SIDE_CHANNEL, SYSMGR_PID, SYSMGR_CHID, SYSMG from qiling.os.qnx.helpers import QnxConn from qiling.os.qnx.structs import _thread_local_storage -from qiling.cc import QlCC, intel, arm, mips, riscv -from qiling.const import QL_ARCH, QL_INTERCEPT +from qiling.cc import QlCC, intel, arm, mips, riscv, ppc +from qiling.const import QL_ARCH, QL_OS from qiling.os.fcall import QlFunctionCall from qiling.os.const import * -from qiling.os.posix.const import NR_OPEN from qiling.os.posix.posix import QlOsPosix class QlOsQnx(QlOsPosix): + type = QL_OS.QNX + def __init__(self, ql: Qiling): super(QlOsQnx, self).__init__(ql) self.ql = ql cc: QlCC = { - QL_ARCH.X86 : intel.cdecl, - QL_ARCH.X8664 : intel.amd64, - QL_ARCH.ARM : arm.aarch32, - QL_ARCH.ARM64 : arm.aarch64, - QL_ARCH.MIPS : mips.mipso32, - QL_ARCH.RISCV : riscv.riscv, - QL_ARCH.RISCV64: riscv.riscv, + QL_ARCH.X86 : intel.cdecl, + QL_ARCH.X8664 : intel.amd64, + QL_ARCH.ARM : arm.aarch32, + QL_ARCH.ARM64 : arm.aarch64, + QL_ARCH.MIPS : mips.mipso32, + QL_ARCH.RISCV : riscv.riscv, + QL_ARCH.RISCV64 : riscv.riscv, + QL_ARCH.PPC : ppc.ppc, }[ql.arch.type](ql.arch) self.fcall = QlFunctionCall(ql, cc) @@ -75,15 +75,11 @@ class QlOsQnx(QlOsPosix): 'get_tls': 0xffff0fe0 }) - - def hook_syscall(self, intno= None, int = None): + + def hook_syscall(self, ql, intno): return self.load_syscall() - def add_function_hook(self, fn: str, cb: Callable, intercept: QL_INTERCEPT): - self.ql.os.function_hook.add_function_hook(fn, cb, intercept) - - def register_function_after_load(self, function): if function not in self.function_after_load_list: self.function_after_load_list.append(function) @@ -94,12 +90,6 @@ class QlOsQnx(QlOsPosix): f() - def hook_sigtrap(self, intno= None, int = None): - self.ql.log.info("Trap Found") - self.emu_error() - exit(1) - - def run(self): if self.ql.exit_point is not None: self.exit_point = self.ql.exit_point diff --git a/qiling/os/qnx/syscall.py b/qiling/os/qnx/syscall.py index 9a8d0200..2d22c968 100644 --- a/qiling/os/qnx/syscall.py +++ b/qiling/os/qnx/syscall.py @@ -198,7 +198,7 @@ def _msg_sendv(ql:Qiling, coid, smsg, sparts, rmsg, rparts, *args, **kw): type_ = ql.unpack16(sbody[:2]) msg_name = map_msgtype(ql, type_) - _msg_handler = ql_get_module_function(f"qiling.os.qnx", "message") + _msg_handler = ql_get_module_function(f".os.qnx", "message") if msg_name in dir(_msg_handler): msg_hook = eval(msg_name) diff --git a/qiling/os/stats.py b/qiling/os/stats.py new file mode 100644 index 00000000..0755872f --- /dev/null +++ b/qiling/os/stats.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +import json +from typing import Any, List, MutableMapping, Mapping, Optional, Set + +class QlOsStats: + """Record basic OS statistics, such as API calls and strings. + """ + + def __init__(self): + self.syscalls: MutableMapping[str, List] = {} + self.strings: MutableMapping[str, Set] = {} + + self.position = 0 + + def clear(self): + """Reset collected stats. + """ + + self.syscalls.clear() + self.strings.clear() + + self.position = 0 + + @staticmethod + def _banner(caption: str) -> List[str]: + bar = '-' * 24 + + return ['', caption, bar] + + def summary(self) -> List[str]: + ret = [] + + ret.extend(QlOsStats._banner('syscalls called')) + + for key, values in self.syscalls.items(): + ret.append(f'{key}:') + ret.extend(f' {json.dumps(value):s}' for value in values) + + ret.extend(QlOsStats._banner('strings ocurrences')) + + for key, values in self.strings.items(): + ret.append(f'{key}: {", ".join(str(word) for word in values)}') + + return ret + + def log_api_call(self, address: int, name: str, params: Mapping, retval: Any, retaddr: int) -> None: + """Record API calls along with their details. + + Args: + address : location of the calling instruction + name : api function name + params : mapping of the parameters name to their effective values + retval : value returned by the api function + retaddr : address to which the api function returned + """ + + if name.startswith('hook_'): + name = name[5:] + + self.syscalls.setdefault(name, []).append({ + 'params' : params, + 'retval' : retval, + 'address' : address, + 'retaddr' : retaddr, + 'position' : self.position + }) + + self.position += 1 + + def log_string(self, s: str) -> None: + """Record strings appearance as they are encountered during emulation. + + Args: + s : string to record + """ + + for token in s.split(' '): + self.strings.setdefault(token, set()).add(self.position) + + +class QlWinStats(QlOsStats): + """OS statistics object for Windows OS. Includes registry access stats. + """ + + def __init__(self): + super().__init__() + + self.registry: MutableMapping[str, List] = {} + + def clear(self): + super().clear() + + self.registry.clear() + + def summary(self) -> List[str]: + ret = super().summary() + + ret.extend(QlOsStats._banner('registry keys accessed')) + + for key, values in self.registry.items(): + ret.append(f'{key}:') + ret.extend(f' {json.dumps(value):s}' for value in values) + + return ret + + def log_reg_access(self, key: str, item: Optional[str], type: Optional[int], value: Any) -> None: + """Record registry access. + + Args: + key : accessed key name + name : sub item name (if provided) + type : sub item type (if provided) + value : value set to item, in case of a registry modification + """ + + self.registry.setdefault(key, []).append({ + 'item' : item, + 'type' : type, + 'value' : value, + 'position' : self.position + }) + + +class QlOsNullStats(QlOsStats): + """Nullified OS statistics object. + """ + + def clear(self): + pass + + def summary(self) -> List[str]: + return [] + + def log_api_call(self, address: int, name: str, params: Mapping, retval: Any, retaddr: int) -> None: + pass + + def log_string(self, s: str) -> None: + pass + + +class QlWinNullStats(QlOsNullStats): + """Nullified Windows statistics object. + """ + + def log_reg_access(self, key: str, item: Optional[str], type: Optional[int], value: Any) -> None: + pass diff --git a/qiling/os/uefi/__init__.py b/qiling/os/uefi/__init__.py index 5b8faeab..0e68ec3c 100644 --- a/qiling/os/uefi/__init__.py +++ b/qiling/os/uefi/__init__.py @@ -1,15 +1,15 @@ import csv from typing import Mapping -from os import path +import inspect +from pathlib import Path def __init_guids_db() -> Mapping[str, str]: """Initialize GUIDs dictionary from a local database. """ - csv_path = path.dirname(path.abspath(__file__)) - csv_path = path.join(csv_path, 'guids.csv') + csv_path = Path(inspect.getfile(inspect.currentframe())).parent / 'guids.csv' - with open(csv_path) as guids_file: + with csv_path.open('r') as guids_file: guids_reader = csv.reader(guids_file) return dict(tuple(entry) for entry in guids_reader) diff --git a/qiling/os/uefi/uefi.py b/qiling/os/uefi/uefi.py index 3923a3c2..67163e90 100644 --- a/qiling/os/uefi/uefi.py +++ b/qiling/os/uefi/uefi.py @@ -9,6 +9,7 @@ from unicorn import UcError from qiling import Qiling from qiling.cc import QlCC, intel +from qiling.const import QL_INTERCEPT, QL_OS from qiling.os.const import * from qiling.os.memory import QlMemoryHeap from qiling.os.os import QlOs, QlOsUtils @@ -18,6 +19,8 @@ from qiling.os.uefi import guids_db from qiling.os.uefi.smm import SmmEnv class QlOsUefi(QlOs): + type = QL_OS.UEFI + def __init__(self, ql: Qiling): super().__init__(ql) @@ -194,7 +197,12 @@ class QlOsUefi(QlOs): self.emit_stack() self.ql.log.error(f'Memory map:') - self.ql.mem.show_mapinfo() + for info_line in self.ql.mem.get_formatted_mapinfo(): + self.ql.log.error(info_line) + + + def set_api(self, target: str, handler: Callable, intercept: QL_INTERCEPT = QL_INTERCEPT.CALL): + super().set_api(f'hook_{target}', handler, intercept) def run(self): # TODO: this is not the right place for this diff --git a/qiling/os/utils.py b/qiling/os/utils.py index 8e8a6e28..1228df02 100644 --- a/qiling/os/utils.py +++ b/qiling/os/utils.py @@ -7,63 +7,12 @@ This module is intended for general purpose functions that are only used in qiling.os """ -from typing import Any, List, MutableMapping, Mapping, Set, Union, Sequence, MutableSequence, Tuple +from typing import MutableMapping, Union, Sequence, MutableSequence, Tuple from uuid import UUID from qiling import Qiling from qiling.const import QL_VERBOSE -class QlOsStats: - """Record basic OS statistics, such as API calls and strings. - """ - - def __init__(self): - self.syscalls: MutableMapping[str, List] = {} - self.syscalls_counter = 0 - self.appeared_strings: MutableMapping[str, Set] = {} - - def clear(self): - """Reset API and string appearance stats. - """ - - self.syscalls = {} - self.syscalls_counter = 0 - self.appeared_strings = {} - - def log_api_call(self, address: int, name: str, params: Mapping, retval: Any, retaddr: int) -> None: - """Record API calls along with their details. - - Args: - address : location of the calling instruction - name : api function name - params : mapping of the parameters name to their effective values - retval : value returned by the api function - retaddr : address to which the api function returned - """ - - if name.startswith('hook_'): - name = name[5:] - - self.syscalls.setdefault(name, []).append({ - 'params' : params, - 'retval' : retval, - 'address' : address, - 'retaddr' : retaddr, - 'position' : self.syscalls_counter - }) - - self.syscalls_counter += 1 - - def log_string(self, s: str) -> None: - """Record strings appearance as they are encountered during emulation. - - Args: - s : string to record - """ - - for token in s.split(' '): - self.appeared_strings.setdefault(token, set()).add(self.syscalls_counter) - class QlOsUtils: ELLIPSIS_PREF = r'__qlva_' diff --git a/qiling/os/windows/clipboard.py b/qiling/os/windows/clipboard.py index 02c1dc00..f7fecad9 100644 --- a/qiling/os/windows/clipboard.py +++ b/qiling/os/windows/clipboard.py @@ -4,69 +4,75 @@ # # A Simple Windows Clipboard Simulation -NOT_LOCKED = -1 +from typing import TYPE_CHECKING, Optional +if TYPE_CHECKING: + from qiling.os.windows.windows import QlOsWindows + +NOT_LOCKED = -1 +ERROR_CLIPBOARD_NOT_OPEN = 0x58a class Clipboard: - def __init__(self, os): - #super(Clipboard, self).__init__(ql) - + def __init__(self, os: 'QlOsWindows'): self.locked_by = NOT_LOCKED self.data = b"Default Clipboard Data" self.os = os + # Valid formats taken from https://doxygen.reactos.org/d8/dd6/base_ # 2applications_2mstsc_2constants_8h_source.html - self.formats = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 128, 129, 130, 131, 142, 512, 767, - 768, 1023] + self.formats = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 128, 129, 130, 131, 142, 512, 767, 768, 1023] - def open(self, h_wnd): - """ - Lock clipboard to hWnd if not already locked. + def open(self, h_wnd: int) -> bool: + """Lock clipboard to hWnd if not already locked. If hWnd is null default to current thead id """ + if h_wnd == 0: - hWnd = self.os.thread_manager.cur_thread.id + h_wnd = self.os.thread_manager.cur_thread.id if self.locked_by != NOT_LOCKED and self.locked_by != h_wnd: - return 0 - else: - self.locked_by = h_wnd - return 1 + return False - def format_available(self, fmt): - if fmt in self.formats: - return 1 - else: - return 0 + self.locked_by = h_wnd - def close(self): + return True + + def close(self) -> bool: if self.locked_by == NOT_LOCKED: - self.os.last_error = 0x58A # ERROR_CLIPBOARD_NOT_OPEN - return 0 - else: - self.locked_by = NOT_LOCKED - return 1 + self.os.last_error = ERROR_CLIPBOARD_NOT_OPEN + return False - def set_data(self, fmt, data): - hWnd = self.os.thread_manager.cur_thread.id - - if self.locked_by != hWnd: - self.os.last_error = 0x58A # ERROR_CLIPBOARD_NOT_OPEN - return 0 - else: - if fmt not in self.formats: - return 0 - self.data = data - return 1 + self.locked_by = NOT_LOCKED - def get_data(self, fmt): + return True + + def format_available(self, fmt: int) -> bool: + return fmt in self.formats + + def set_data(self, fmt: int, data: bytes) -> int: if fmt not in self.formats: return 0 + hWnd = self.os.thread_manager.cur_thread.id - + if self.locked_by != hWnd: - self.os.last_error = 0x58A # ERROR_CLIPBOARD_NOT_OPEN + self.os.last_error = ERROR_CLIPBOARD_NOT_OPEN return 0 - else: - return self.data + + self.data = data + + # BUG: this should be the handle of the clipboard object + return 1 + + def get_data(self, fmt: int) -> Optional[bytes]: + if fmt not in self.formats: + return None + + hWnd = self.os.thread_manager.cur_thread.id + + if self.locked_by != hWnd: + self.os.last_error = ERROR_CLIPBOARD_NOT_OPEN + return None + + return self.data diff --git a/qiling/os/windows/const.py b/qiling/os/windows/const.py index 2453319a..13884a29 100644 --- a/qiling/os/windows/const.py +++ b/qiling/os/windows/const.py @@ -612,6 +612,7 @@ PF_XSAVE_ENABLED = 0x17 # https://docs.microsoft.com/en-us/windows/win32/procthread/zwqueryinformationprocess +# https://www.pinvoke.net/default.aspx/ntdll/PROCESSINFOCLASS.html ProcessBasicInformation = 0 ProcessDebugPort = 7 ProcessExecuteFlags = 0x22 diff --git a/qiling/os/windows/dlls/advapi32.py b/qiling/os/windows/dlls/advapi32.py index baf247be..e9877989 100644 --- a/qiling/os/windows/dlls/advapi32.py +++ b/qiling/os/windows/dlls/advapi32.py @@ -13,13 +13,11 @@ def __RegOpenKey(ql: Qiling, address: int, params): hKey = params["hKey"] lpSubKey = params["lpSubKey"] phkResult = params["phkResult"] - ql.log.debug("Key %s %s" % (hKey, lpSubKey)) if hKey not in REG_KEYS: - ql.log.debug("Key %s %s not present" % (hKey, lpSubKey)) return ERROR_FILE_NOT_FOUND - else: - s_hKey = REG_KEYS[hKey] + + s_hKey = REG_KEYS[hKey] key = s_hKey + "\\" + lpSubKey # Keys in the profile are saved as KEY\PARAM = VALUE, so i just want to check that the key is the same @@ -39,48 +37,49 @@ def __RegOpenKey(ql: Qiling, address: int, params): ql.mem.write_ptr(phkResult, new_handle.id) return ERROR_SUCCESS -def __RegQueryValue(ql: Qiling, address: int, params): +def __RegQueryValue(ql: Qiling, address: int, params, wstring: bool): ret = ERROR_SUCCESS hKey = params["hKey"] - s_lpValueName = params["lpValueName"] + lpValueName = params["lpValueName"] lpType = params["lpType"] lpData = params["lpData"] lpcbData = params["lpcbData"] s_hKey = ql.os.handle_manager.get(hKey).obj params["hKey"] = s_hKey # read reg_type - reg_type = Registry.RegNone if lpType == 0 else ql.unpack32(ql.mem.read(lpType, 4)) + reg_type = Registry.RegNone if lpType == 0 else ql.mem.read_ptr(lpType, 4) - try: - # Keys in the profile are saved as KEY\PARAM = VALUE, so i just want to check that the key is the same - value = ql.os.profile["REGISTRY"][s_hKey + "\\" + s_lpValueName] - ql.log.debug("Using profile for value of key %s" % (s_hKey + "\\" + s_lpValueName,)) + # try reading the registry key value from profile first. + # if the key is not specified in profile, proceed to registry manager - # TODO i have no fucking idea on how to set a None value, fucking configparser - if value == "None": - return ERROR_FILE_NOT_FOUND + keyname = f'{s_hKey}\\{lpValueName}' + value = ql.os.profile["REGISTRY"].get(keyname) + + if value is None: + reg_type, value = ql.os.registry_manager.read(s_hKey, lpValueName, reg_type) + + else: + ql.log.debug(f'Value for {keyname} was read from profile') reg_type = Registry.RegSZ # set that the registry has been accessed - ql.os.registry_manager.access(s_hKey, s_lpValueName, value, reg_type) - - except KeyError: - # Read the registry - reg_type, value = ql.os.registry_manager.read(s_hKey, s_lpValueName, reg_type) + ql.os.registry_manager.access(s_hKey, lpValueName, reg_type, value) # error key if reg_type is None or value is None: ql.log.debug("Key value not found") return ERROR_FILE_NOT_FOUND - else: - # set lpData - length = ql.os.registry_manager.write_reg_value_into_mem(value, reg_type, lpData) - # set lpcbData - max_size = int.from_bytes(ql.mem.read(lpcbData, 4), byteorder="little") - ql.mem.write_ptr(lpcbData, length) - if max_size < length: - ret = ERROR_MORE_DATA + + # set lpData + length = ql.os.registry_manager.write_reg_value_into_mem(reg_type, lpData, value, wstring) + + # set lpcbData + max_size = ql.mem.read_ptr(lpcbData, 4) + ql.mem.write_ptr(lpcbData, length, 4) + + if max_size < length: + ret = ERROR_MORE_DATA return ret @@ -91,19 +90,21 @@ def __RegCreateKey(ql: Qiling, address: int, params): lpSubKey = params["lpSubKey"] phkResult = params["phkResult"] - if not (hKey in REG_KEYS): + if hKey not in REG_KEYS: return ERROR_FILE_NOT_FOUND - else: - s_hKey = REG_KEYS[hKey] - params["hKey"] = s_hKey - if not ql.os.registry_manager.exists(s_hKey + "\\" + lpSubKey): - ql.os.registry_manager.create(s_hKey + "\\" + lpSubKey) - ret = ERROR_SUCCESS + s_hKey = REG_KEYS[hKey] + params["hKey"] = s_hKey + + keyname = f'{s_hKey}\\{lpSubKey}' + + if not ql.os.registry_manager.exists(keyname): + ql.os.registry_manager.create(keyname) + ret = ERROR_SUCCESS # new handle if ret == ERROR_SUCCESS: - new_handle = Handle(obj=s_hKey + "\\" + lpSubKey) + new_handle = Handle(obj=keyname) ql.os.handle_manager.append(new_handle) if phkResult != 0: ql.mem.write_ptr(phkResult, new_handle.id) @@ -113,31 +114,33 @@ def __RegCreateKey(ql: Qiling, address: int, params): return ret -def __RegSetValue(ql: Qiling, address: int, params): +def __RegSetValue(ql: Qiling, address: int, params, wstring: bool): hKey = params["hKey"] lpSubKey = params["lpSubKey"] dwType = params["dwType"] lpData = params["lpData"] + cbData = params["cbData"] s_hKey = ql.os.handle_manager.get(hKey).obj # this is done so the print_function would print the correct value params["hKey"] = s_hKey - ql.os.registry_manager.write(s_hKey, lpSubKey, dwType, lpData) + # dwType is expected to be REG_SZ and lpData to point to a null-terminated string + ql.os.registry_manager.write(s_hKey, lpSubKey, dwType, lpData, cbData, wstring) return ERROR_SUCCESS -def __RegSetValueEx(ql: Qiling, address: int, params): +def __RegSetValueEx(ql: Qiling, address: int, params, wstring: bool): hKey = params["hKey"] lpValueName = params["lpValueName"] dwType = params["dwType"] lpData = params["lpData"] + cbData = params["cbData"] s_hKey = ql.os.handle_manager.get(hKey).obj params["hKey"] = s_hKey - # BUG: lpData should be handled according to the value in dwType - ql.os.registry_manager.write(s_hKey, lpValueName, dwType, lpData) + ql.os.registry_manager.write(s_hKey, lpValueName, dwType, lpData, cbData, wstring) return ERROR_SUCCESS @@ -240,7 +243,7 @@ def hook_RegOpenKeyA(ql: Qiling, address: int, params): 'lpcbData' : LPDWORD }) def hook_RegQueryValueExA(ql: Qiling, address: int, params): - return __RegQueryValue(ql, address, params) + return __RegQueryValue(ql, address, params, wstring=False) # LSTATUS RegQueryValueExW( # HKEY hKey, @@ -259,7 +262,7 @@ def hook_RegQueryValueExA(ql: Qiling, address: int, params): 'lpcbData' : LPDWORD }) def hook_RegQueryValueExW(ql: Qiling, address: int, params): - return __RegQueryValue(ql, address, params) + return __RegQueryValue(ql, address, params, wstring=True) # LSTATUS RegCloseKey( # HKEY hKey @@ -340,7 +343,7 @@ def hook_RegCreateKeyExW(ql: Qiling, address: int, params): 'cbData' : DWORD }) def hook_RegSetValueA(ql: Qiling, address: int, params): - return __RegSetValue(ql, address, params) + return __RegSetValue(ql, address, params, wstring=False) @winsdkapi(cc=STDCALL, params={ 'hKey' : HKEY, @@ -350,7 +353,7 @@ def hook_RegSetValueA(ql: Qiling, address: int, params): 'cbData' : DWORD }) def hook_RegSetValueW(ql: Qiling, address: int, params): - return __RegSetValue(ql, address, params) + return __RegSetValue(ql, address, params, wstring=False) # LSTATUS RegSetValueExA( # HKEY hKey, @@ -369,7 +372,7 @@ def hook_RegSetValueW(ql: Qiling, address: int, params): 'cbData' : DWORD }) def hook_RegSetValueExA(ql: Qiling, address: int, params): - return __RegSetValueEx(ql, address, params) + return __RegSetValueEx(ql, address, params, wstring=False) # LSTATUS RegSetValueExW( # HKEY hKey, @@ -388,7 +391,7 @@ def hook_RegSetValueExA(ql: Qiling, address: int, params): 'cbData' : DWORD }) def hook_RegSetValueExW(ql: Qiling, address: int, params): - return __RegSetValueEx(ql, address, params) + return __RegSetValueEx(ql, address, params, wstring=True) # LSTATUS RegDeleteKeyA( # HKEY hKey, @@ -457,8 +460,10 @@ def hook_GetTokenInformation(ql: Qiling, address: int, params): token = ql.os.handle_manager.get(TokenHandle).obj information_value = token.get(TokenInformationClass) - ql.mem.write(ReturnLength, len(information_value).to_bytes(4, byteorder="little")) - return_size = int.from_bytes(ql.mem.read(ReturnLength, 4), byteorder="little") + + ql.mem.write_ptr(ReturnLength, len(information_value), 4) + return_size = ql.mem.read_ptr(ReturnLength, 4) + ql.log.debug("The target is checking for its permissions") if return_size > TokenInformationLength: @@ -659,13 +664,9 @@ def hook_StartServiceA(ql: Qiling, address: int, params): }) def hook_AllocateAndInitializeSid(ql: Qiling, address: int, params): count = params["nSubAuthorityCount"] - subs = b"" + subs = b''.join(ql.pack32(params[f'nSubAuthority{i}']) for i in range(count)) - for i in range(count): - sub = params[f"nSubAuthority{i}"] - subs += sub.to_bytes(4, "little") - - sid = Sid(ql, revision=1, identifier=5, subs=subs, subs_count=count) + sid = Sid(ql, revision=1, subs_count=count, identifier=5, subs=subs) sid_addr = ql.os.heap.alloc(sid.size) sid.write(sid_addr) @@ -685,34 +686,46 @@ __poweruserssid = None # Power Users (S-1-5-32-547) def get_adminsid(ql): global __adminsid - if __adminsid == None: - # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20], nSubAuthority1 = DOMAIN_ALIAS_RID_ADMINS[0x220] + + if __adminsid is None: + # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20] + # nSubAuthority1 = DOMAIN_ALIAS_RID_ADMINS[0x220] subs = b"\x20\x00\x00\x00\x20\x02\x00\x00" __adminsid = Sid(ql, revision=1, identifier=5, subs=subs, subs_count=2) + return __adminsid def get_userssid(ql): global __userssid - if __userssid == None: - # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20], nSubAuthority1 = DOMAIN_ALIAS_RID_USERS[0x221] + + if __userssid is None: + # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20] + # nSubAuthority1 = DOMAIN_ALIAS_RID_USERS[0x221] subs = b"\x20\x00\x00\x00\x21\x02\x00\x00" __userssid = Sid(ql, revision=1, identifier=5, subs=subs, subs_count=2) + return __userssid def get_guestssid(ql): global __guestssid - if __guestssid == None: - # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20], nSubAuthority1 = DOMAIN_ALIAS_RID_GUESTS[0x222] + + if __guestssid is None: + # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20] + # nSubAuthority1 = DOMAIN_ALIAS_RID_GUESTS[0x222] subs = b"\x20\x00\x00\x00\x22\x02\x00\x00" __guestssid = Sid(ql, revision=1, identifier=5, subs=subs, subs_count=2) + return __guestssid def get_poweruserssid(ql): global __poweruserssid - if __poweruserssid == None: - # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20], nSubAuthority1 = DOMAIN_ALIAS_RID_POWER_USERS[0x223] + + if __poweruserssid is None: + # nSubAuthority0 = SECURITY_BUILTIN_DOMAIN_RID[0x20] + # nSubAuthority1 = DOMAIN_ALIAS_RID_POWER_USERS[0x223] subs = b"\x20\x00\x00\x00\x23\x02\x00\x00" __poweruserssid = Sid(ql, revision=1, identifier=5, subs=subs, subs_count=2) + return __poweruserssid diff --git a/qiling/os/windows/dlls/cng.py b/qiling/os/windows/dlls/cng.py deleted file mode 100644 index 96c4c541..00000000 --- a/qiling/os/windows/dlls/cng.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 -# -# Cross Platform and Multi Architecture Advanced Binary Emulation Framework -# - - -from qiling.os.windows.const import * -from qiling.os.windows.fncc import * -from qiling.os.const import * -from qiling.os.windows.utils import * -from qiling.os.windows.thread import * -from qiling.os.windows.handle import * -from qiling.exception import * - -dllname = 'cng_dll' - -# typedef struct _OSVERSIONINFOW { -# ULONG dwOSVersionInfoSize; -# ULONG dwMajorVersion; -# ULONG dwMinorVersion; -# ULONG dwBuildNumber; -# ULONG dwPlatformId; -# WCHAR szCSDVersion[128]; -# } -# NTSYSAPI NTSTATUS RtlGetVersion( -# PRTL_OSVERSIONINFOW lpVersionInformation -# ); -@winsdkapi(cc=CDECL, dllname=dllname, replace_params={"lpVersionInformation": POINTER}) -def hook_RtlGetVersion(ql, address, params): - pointer = params["lpVersionInformation"] - size = int.from_bytes(ql.mem.read(pointer, 4), byteorder="little") - os_version_info_asked = { - "dwOSVersionInfoSize": - size, - VER_MAJORVERSION: - int.from_bytes(ql.mem.read(pointer + 4, 4), byteorder="little"), - VER_MINORVERSION: - int.from_bytes(ql.mem.read(pointer + 8, 4), byteorder="little"), - VER_BUILDNUMBER: - int.from_bytes(ql.mem.read(pointer + 12, 4), byteorder="little"), - VER_PLATFORMID: - int.from_bytes(ql.mem.read(pointer + 16, 4), byteorder="little"), - "szCSDVersion": - int.from_bytes(ql.mem.read(pointer + 20, 128), byteorder="little"), - } - ql.mem.write( - pointer + 4, - ql.os.profile.getint("SYSTEM", - "majorVersion").to_bytes(4, byteorder="little")) - ql.mem.write( - pointer + 8, - ql.os.profile.getint("SYSTEM", - "minorVersion").to_bytes(4, byteorder="little")) - - ql.log.debug("The sample is checking the windows Version!") - return STATUS_SUCCESS diff --git a/qiling/os/windows/dlls/crypt32.py b/qiling/os/windows/dlls/crypt32.py index c1fb589f..7efa9119 100644 --- a/qiling/os/windows/dlls/crypt32.py +++ b/qiling/os/windows/dlls/crypt32.py @@ -19,9 +19,11 @@ def _CryptStringToBinary(ql: Qiling, address: int, params) -> int: string_dst = params["pbBinary"] flag_dst = params["pdwFlags"] - size_dst = int.from_bytes(ql.mem.read(size_dst_pointer, 4), byteorder="little") - if size_dst != 0 and size_dst < size_src: + size_dst = ql.mem.read_ptr(size_dst_pointer, 4) + + if size_dst and size_dst < size_src: raise QlErrorNotImplemented("API not implemented") + if flag_src == CRYPT_STRING_BASE64: # Had a padding error, hope this always works add_pad = 4 - (len(string_src) % 4) @@ -37,11 +39,13 @@ def _CryptStringToBinary(ql: Qiling, address: int, params) -> int: # Only wants the length return len(output) else: - if flag_dst != 0: + if flag_dst: # Is optional - ql.mem.write(flag_dst, flag_src.to_bytes(length=4, byteorder='little')) + ql.mem.write_ptr(flag_dst, flag_src, 4) + # Write size - ql.mem.write(size_dst_pointer, len(output).to_bytes(length=4, byteorder='little')) + ql.mem.write_ptr(size_dst_pointer, len(output), 4) + # Write result ql.mem.write(string_dst, bytes(output, encoding="utf-16le")) return 1 diff --git a/qiling/os/windows/dlls/kernel32/fibersapi.py b/qiling/os/windows/dlls/kernel32/fibersapi.py index 36e1b27f..1623fb11 100644 --- a/qiling/os/windows/dlls/kernel32/fibersapi.py +++ b/qiling/os/windows/dlls/kernel32/fibersapi.py @@ -14,7 +14,7 @@ from qiling.os.windows.fncc import * 'dwFlsIndex' : DWORD }) def hook_FlsFree(ql: Qiling, address: int, params): - return ql.os.fiber_manager.free(params['dwFlsIndex']) + return int(ql.os.fiber_manager.free(params['dwFlsIndex'])) # LPVOID FlsGetValue( # DWORD dwFlsIndex @@ -25,7 +25,7 @@ def hook_FlsFree(ql: Qiling, address: int, params): def hook_FlsGetValue(ql: Qiling, address: int, params): return ql.os.fiber_manager.get(params['dwFlsIndex']) -# LPVOID FlsSetValue( +# BOOL FlsSetValue( # DWORD dwFlsIndex # PVOID lpFlsData # ); @@ -34,7 +34,7 @@ def hook_FlsGetValue(ql: Qiling, address: int, params): 'lpFlsData' : PVOID }) def hook_FlsSetValue(ql: Qiling, address: int, params): - return ql.os.fiber_manager.set(params['dwFlsIndex'], params['lpFlsData']) + return int(ql.os.fiber_manager.set(params['dwFlsIndex'], params['lpFlsData'])) # DWORD FlsAlloc( # PFLS_CALLBACK_FUNCTION lpCallback @@ -43,10 +43,6 @@ def hook_FlsSetValue(ql: Qiling, address: int, params): 'lpCallback' : PFLS_CALLBACK_FUNCTION }) def hook_FlsAlloc(ql: Qiling, address: int, params): - # global cb = params['lpCallback'] cb = params['lpCallback'] - if cb: - return ql.os.fiber_manager.alloc(cb) - else: - return ql.os.fiber_manager.alloc() + return ql.os.fiber_manager.alloc(cb or None) diff --git a/qiling/os/windows/dlls/kernel32/fileapi.py b/qiling/os/windows/dlls/kernel32/fileapi.py index d4811027..0ccf37e0 100644 --- a/qiling/os/windows/dlls/kernel32/fileapi.py +++ b/qiling/os/windows/dlls/kernel32/fileapi.py @@ -3,6 +3,7 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +import ntpath import os from shutil import copyfile @@ -25,18 +26,13 @@ from qiling.os.windows.structs import Win32FindData def hook_GetFileType(ql: Qiling, address: int, params): hFile = params["hFile"] - if hFile in (STD_INPUT_HANDLE, STD_OUTPUT_HANDLE, STD_ERROR_HANDLE): - ret = FILE_TYPE_CHAR - else: - obj = ql.os.handle_manager.get(hFile) + handle = ql.os.handle_manager.get(hFile) - if obj is None: - raise QlErrorNotImplemented("API not implemented") - else: - # technically is not always a type_char but.. almost - ret = FILE_TYPE_CHAR + if handle is None: + raise QlErrorNotImplemented("API not implemented") - return ret + # technically is not always a type_char but.. almost + return FILE_TYPE_CHAR # HANDLE FindFirstFileA( # LPCSTR lpFileName, @@ -155,28 +151,16 @@ def hook_ReadFile(ql: Qiling, address: int, params): nNumberOfBytesToRead = params["nNumberOfBytesToRead"] lpNumberOfBytesRead = params["lpNumberOfBytesRead"] - if hFile == STD_INPUT_HANDLE: - if ql.os.automatize_input: - # TODO maybe insert a good random generation input - s = (b"A" * (nNumberOfBytesToRead - 1)) + b"\x00" - else: - ql.log.debug("Insert input") - s = ql.os.stdin.read(nNumberOfBytesToRead) + handle = ql.os.handle_manager.get(hFile) - slen = len(s) - read_len = slen + if handle is None: + ql.os.last_error = ERROR_INVALID_HANDLE + return 0 - if slen > nNumberOfBytesToRead: - s = s[:nNumberOfBytesToRead] - read_len = nNumberOfBytesToRead + data = handle.obj.read(nNumberOfBytesToRead) - ql.mem.write(lpBuffer, s) - ql.mem.write_ptr(lpNumberOfBytesRead, read_len, 4) - else: - f = ql.os.handle_manager.get(hFile).obj - data = f.read(nNumberOfBytesToRead) - ql.mem.write(lpBuffer, data) - ql.mem.write_ptr(lpNumberOfBytesRead, len(data), 4) + ql.mem.write(lpBuffer, data) + ql.mem.write_ptr(lpNumberOfBytesRead, len(data), 4) return 1 @@ -200,24 +184,20 @@ def hook_WriteFile(ql: Qiling, address: int, params): nNumberOfBytesToWrite = params["nNumberOfBytesToWrite"] lpNumberOfBytesWritten = params["lpNumberOfBytesWritten"] + handle = ql.os.handle_manager.get(hFile) + + if handle is None: + ql.os.last_error = ERROR_INVALID_HANDLE + return 0 + + fobj = handle.obj + data = ql.mem.read(lpBuffer, nNumberOfBytesToWrite) + if hFile == STD_OUTPUT_HANDLE: - s = ql.mem.read(lpBuffer, nNumberOfBytesToWrite) - ql.os.stdout.write(s) - ql.os.stats.log_string(s.decode()) - ql.mem.write_ptr(lpNumberOfBytesWritten, nNumberOfBytesToWrite, 4) - else: - f = ql.os.handle_manager.get(hFile) + ql.os.stats.log_string(data.decode()) - if f is None: - # Invalid handle - ql.os.last_error = ERROR_INVALID_HANDLE - return 0 - else: - f = f.obj - - buffer = ql.mem.read(lpBuffer, nNumberOfBytesToWrite) - nNumberOfBytesWritten = f.write(bytes(buffer)) - ql.mem.write_ptr(lpNumberOfBytesWritten, nNumberOfBytesWritten, 4) + written = fobj.write(bytes(data)) + ql.mem.write_ptr(lpNumberOfBytesWritten, written, 4) return 1 @@ -290,6 +270,27 @@ def hook_CreateFileA(ql: Qiling, address: int, params): def hook_CreateFileW(ql: Qiling, address: int, params): return _CreateFile(ql, address, params) +def _GetTempPath(ql: Qiling, address: int, params, wide: bool): + temp_path = ntpath.join(ql.rootfs, 'Windows', 'Temp') + + if not os.path.exists(temp_path): + os.makedirs(temp_path, 0o755) + + nBufferLength = params['nBufferLength'] + lpBuffer = params['lpBuffer'] + + enc = 'utf-16le' if wide else 'utf-8' + + # temp dir path has to end with a path separator + tmpdir = f'{ntpath.join(ql.os.windir, "Temp")}{ntpath.sep}'.encode(enc) + cstr = tmpdir + '\x00'.encode(enc) + + if nBufferLength >= len(cstr): + ql.mem.write(lpBuffer, cstr) + + # returned length does not include the null-terminator + return len(tmpdir) + # DWORD GetTempPathW( # DWORD nBufferLength, # LPWSTR lpBuffer @@ -299,16 +300,7 @@ def hook_CreateFileW(ql: Qiling, address: int, params): 'lpBuffer' : LPWSTR }) def hook_GetTempPathW(ql: Qiling, address: int, params): - temp_path = os.path.join(ql.rootfs, "Windows", "Temp") - - if not os.path.exists(temp_path): - os.makedirs(temp_path, 0o755) - - dest = params["lpBuffer"] - temp = (ql.os.windir + "Temp" + "\\\x00").encode('utf-16le') - ql.mem.write(dest, temp) - - return len(temp) + return _GetTempPath(ql, address, params, wide=True) # DWORD GetTempPathA( # DWORD nBufferLength, @@ -319,16 +311,7 @@ def hook_GetTempPathW(ql: Qiling, address: int, params): 'lpBuffer' : LPSTR }) def hook_GetTempPathA(ql: Qiling, address: int, params): - temp_path = os.path.join(ql.rootfs, "Windows", "Temp") - - if not os.path.exists(temp_path): - os.makedirs(temp_path, 0o755) - - dest = params["lpBuffer"] - temp = (ql.os.windir + "Temp" + "\\\x00").encode('utf-8') - ql.mem.write(dest, temp) - - return len(temp) + return _GetTempPath(ql, address, params, wide=False) # DWORD GetShortPathNameW( # LPCWSTR lpszLongPath, diff --git a/qiling/os/windows/dlls/kernel32/libloaderapi.py b/qiling/os/windows/dlls/kernel32/libloaderapi.py index 2d229ac8..65810713 100644 --- a/qiling/os/windows/dlls/kernel32/libloaderapi.py +++ b/qiling/os/windows/dlls/kernel32/libloaderapi.py @@ -10,7 +10,7 @@ from qiling.exception import QlErrorNotImplemented from qiling.os.windows.api import * from qiling.os.windows.const import * from qiling.os.windows.fncc import * -from qiling.os.windows.utils import * +from qiling.os.windows.utils import has_lib_ext def _GetModuleHandle(ql: Qiling, address: int, params): lpModuleName = params["lpModuleName"] @@ -20,13 +20,15 @@ def _GetModuleHandle(ql: Qiling, address: int, params): else: lpModuleName = lpModuleName.lower() - if not is_file_library(lpModuleName): - lpModuleName += ".dll" + if not has_lib_ext(lpModuleName): + lpModuleName = f'{lpModuleName}.dll' - if lpModuleName in ql.loader.dlls: - ret = ql.loader.dlls[lpModuleName] + image = ql.loader.get_image_by_name(lpModuleName) + + if image: + ret = image.base else: - ql.log.debug("Library %s not imported" % lpModuleName) + ql.log.debug(f'Library "{lpModuleName}" not imported') ret = 0 return ret @@ -148,33 +150,38 @@ def hook_GetModuleFileNameW(ql: Qiling, address: int, params): 'lpProcName' : POINTER # LPCSTR }) def hook_GetProcAddress(ql: Qiling, address: int, params): - if params["lpProcName"] > MAXUSHORT: + hModule = params['hModule'] + lpProcName = params['lpProcName'] + + if lpProcName > MAXUSHORT: # Look up by name - params["lpProcName"] = ql.os.utils.read_cstring(params["lpProcName"]) + params["lpProcName"] = ql.os.utils.read_cstring(lpProcName) lpProcName = bytes(params["lpProcName"], "ascii") else: # Look up by ordinal lpProcName = params["lpProcName"] # TODO fix for gandcrab - if params["lpProcName"] == "RtlComputeCrc32": + if lpProcName == "RtlComputeCrc32": return 0 # Check if dll is loaded - try: - dll_name = [key for key, value in ql.loader.dlls.items() if value == params['hModule']][0] - except IndexError as ie: - ql.log.info('Failed to import function "%s" with handle 0x%X' % (lpProcName, params['hModule'])) + dll_name = next((os.path.basename(image.path).casefold() for image in ql.loader.images if image.base == hModule), None) + + if dll_name is None: + ql.log.info('Failed to import function "%s" with handle 0x%X' % (lpProcName, hModule)) return 0 # Handle case where module is self - if dll_name == os.path.basename(ql.loader.path): + if dll_name == os.path.basename(ql.loader.path).casefold(): for addr, export in ql.loader.export_symbols.items(): if export['name'] == lpProcName: return addr - if lpProcName in ql.loader.import_address_table[dll_name]: - return ql.loader.import_address_table[dll_name][lpProcName] + iat = ql.loader.import_address_table[dll_name] + + if lpProcName in iat: + return iat[lpProcName] return 0 @@ -185,12 +192,12 @@ def _LoadLibrary(ql: Qiling, address: int, params): # Loading self return ql.loader.pe_image_address - return ql.loader.load_dll(lpLibFileName.encode()) + return ql.loader.load_dll(lpLibFileName) def _LoadLibraryEx(ql: Qiling, address: int, params): lpLibFileName = params["lpLibFileName"] - return ql.loader.load_dll(lpLibFileName.encode()) + return ql.loader.load_dll(lpLibFileName) # HMODULE LoadLibraryA( # LPCSTR lpLibFileName diff --git a/qiling/os/windows/dlls/kernel32/sysinfoapi.py b/qiling/os/windows/dlls/kernel32/sysinfoapi.py index 4b7bb2e3..fc2073eb 100644 --- a/qiling/os/windows/dlls/kernel32/sysinfoapi.py +++ b/qiling/os/windows/dlls/kernel32/sysinfoapi.py @@ -3,7 +3,7 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import os +import ntpath from datetime import datetime from qiling import Qiling @@ -94,7 +94,7 @@ def __GetWindowsDirectory(ql: Qiling, address: int, params, wstring: bool): lpBuffer = params["lpBuffer"] enc = 'utf-16le' if wstring else 'utf-8' - res = os.path.normpath(ql.os.windir) + res = ntpath.normpath(ql.os.windir) ql.mem.write(lpBuffer, f'{res}\x00'.encode(enc)) @@ -104,7 +104,7 @@ def __GetSystemDirectory(ql: Qiling, address: int, params, wstring: bool): lpBuffer = params["lpBuffer"] enc = 'utf-16le' if wstring else 'utf-8' - res = os.path.join(ql.os.windir, 'System32') + res = ql.os.winsys ql.mem.write(lpBuffer, f'{res}\x00'.encode(enc)) @@ -119,14 +119,14 @@ def __GetSystemDirectory(ql: Qiling, address: int, params, wstring: bool): 'uSize' : UINT }) def hook_GetWindowsDirectoryW(ql: Qiling, address: int, params): - return __GetWindowsDirectory(ql, address, params, True) + return __GetWindowsDirectory(ql, address, params, wstring=True) @winsdkapi(cc=STDCALL, params={ 'lpBuffer' : LPSTR, 'uSize' : UINT }) def hook_GetWindowsDirectoryA(ql: Qiling, address: int, params): - return __GetWindowsDirectory(ql, address, params, False) + return __GetWindowsDirectory(ql, address, params, wstring=False) # UINT GetSystemWindowsDirectoryW( # LPWSTR lpBuffer, diff --git a/qiling/os/windows/dlls/msvcrt.py b/qiling/os/windows/dlls/msvcrt.py index 52777f5a..73b84b3c 100644 --- a/qiling/os/windows/dlls/msvcrt.py +++ b/qiling/os/windows/dlls/msvcrt.py @@ -434,7 +434,7 @@ def hook_malloc(ql: Qiling, address: int, params): def __free(ql: Qiling, address: int, params): address = params['address'] - return ql.os.heap.free(address) + ql.os.heap.free(address) @winsdkapi(cc=CDECL, params={ 'address': POINTER @@ -573,7 +573,7 @@ def hook__time64(ql: Qiling, address: int, params): time_wasted = int(time.time()) - if dst != 0: - ql.mem.write(dst, time_wasted.to_bytes(8, "little")) + if dst: + ql.mem.write_ptr(dst, time_wasted, 8) return time_wasted diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index b077d4b9..5f424f77 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -3,6 +3,8 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +import os + from qiling import Qiling from qiling.os.windows.api import * from qiling.os.windows.fncc import * @@ -32,7 +34,7 @@ def hook_memcpy(ql: Qiling, address: int, params): try: data = bytes(ql.mem.read(src, count)) ql.mem.write(dest, data) - except Exception as e: + except Exception: ql.log.exception("") return dest @@ -54,7 +56,8 @@ def _QueryInformationProcess(ql: Qiling, address: int, params): elif flag == ProcessBasicInformation: pbi = structs.ProcessBasicInformation(ql, exitStatus=0, - pebBaseAddress=ql.os.heap_base_address, affinityMask=0, + pebBaseAddress=ql.loader.TEB.PebAddress, + affinityMask=0, basePriority=0, uniqueId=ql.os.profile.getint("KERNEL", "pid"), parentPid=ql.os.profile.getint("KERNEL", "parent_pid") @@ -62,7 +65,7 @@ def _QueryInformationProcess(ql: Qiling, address: int, params): addr = ql.os.heap.alloc(pbi.size) pbi.write(addr) - value = addr.to_bytes(ql.arch.pointersize, "little") + value = ql.pack(addr) else: ql.log.debug(str(flag)) raise QlErrorNotImplemented("API not implemented") @@ -70,8 +73,8 @@ def _QueryInformationProcess(ql: Qiling, address: int, params): ql.log.debug("The target is checking the debugger via QueryInformationProcess ") ql.mem.write(dst, value) - if pt_res != 0: - ql.mem.write(pt_res, 0x8.to_bytes(1, byteorder="little")) + if pt_res: + ql.mem.write_ptr(pt_res, 8, 1) return STATUS_SUCCESS @@ -144,11 +147,11 @@ def _QuerySystemInformation(ql: Qiling, address: int, params): if (bufferLength==sbi.size): sbi.write(dst) - if pt_res != 0: - ql.mem.write(pt_res, sbi.size.to_bytes(1, byteorder="little")) + if pt_res: + ql.mem.write_ptr(pt_res, sbi.size, 1) else: - if pt_res != 0: - ql.mem.write(pt_res, sbi.size.to_bytes(1, byteorder="little")) + if pt_res: + ql.mem.write_ptr(pt_res, sbi.size, 1) return STATUS_INFO_LENGTH_MISMATCH else: @@ -250,11 +253,11 @@ def hook_ZwQueryObject(ql: Qiling, address: int, params): else: raise QlErrorNotImplemented("API not implemented") - if dest != 0 and params["Handle"] != 0: + if dest and params["Handle"]: res.write(dest) - if size_dest != 0: - ql.mem.write(size_dest, res.size.to_bytes(4, "little")) + if size_dest: + ql.mem.write_ptr(size_dest, res.size, 4) return STATUS_SUCCESS @@ -300,17 +303,18 @@ def _SetInformationProcess(ql: Qiling, address: int, params): elif flag == ProcessBreakOnTermination: ql.log.debug("The target may be attempting modify a the 'critical' flag of the process") - elif flag == ProcessExecuteFlags: + elif flag == ProcessExecuteFlags: ql.log.debug("The target may be attempting to modify DEP for the process") - if dst != 0: - ql.mem.write(dst, 0x0.to_bytes(1, byteorder="little")) + if dst: + ql.mem.write_ptr(dst, 0, 1) elif flag == ProcessBasicInformation: pbi = structs.ProcessBasicInformation( ql, exitStatus=0, - pebBaseAddress=ql.os.heap_base_address, affinityMask=0, + pebBaseAddress=ql.loader.TEB.PebAddress, + affinityMask=0, basePriority=0, uniqueId=ql.os.profile.getint("KERNEL", "pid"), parentPid=ql.os.profile.getint("KERNEL", "parent_pid") @@ -319,7 +323,7 @@ def _SetInformationProcess(ql: Qiling, address: int, params): ql.log.debug("The target may be attempting to modify the PEB debug flag") addr = ql.os.heap.alloc(pbi.size) pbi.write(addr) - value = addr.to_bytes(ql.arch.pointersize, "little") + value = ql.pack(addr) else: ql.log.debug(str(flag)) raise QlErrorNotImplemented("API not implemented") @@ -356,17 +360,17 @@ def hook_LdrGetProcedureAddress(ql: Qiling, address: int, params): FunctionAddress = params['FunctionAddress'] # Check if dll is loaded - dll_name = next((key for key, value in ql.loader.dlls.items() if value == ModuleHandle), None) + dll_name = next((os.path.basename(path).casefold() for base, _, path in ql.loader.images if base == ModuleHandle), None) if dll_name is None: ql.log.debug(f'Could not find specified handle {ModuleHandle} in loaded DLL') return 0 identifier = bytes(FunctionName, 'ascii') if FunctionName else Ordinal + iat = ql.loader.import_address_table[dll_name] - if identifier in ql.loader.import_address_table[dll_name]: - addr = ql.loader.import_address_table[dll_name][identifier] - ql.mem.write(addr.to_bytes(length=ql.arch.pointersize, byteorder='little'), FunctionAddress) + if identifier in iat: + ql.mem.write_ptr(FunctionAddress, iat[identifier]) return 0 return 0xFFFFFFFF diff --git a/qiling/os/windows/dlls/ntoskrnl.py b/qiling/os/windows/dlls/ntoskrnl.py index 59ef9460..c54d274e 100644 --- a/qiling/os/windows/dlls/ntoskrnl.py +++ b/qiling/os/windows/dlls/ntoskrnl.py @@ -162,14 +162,7 @@ def __IoCreateDevice(ql: Qiling, address: int, params): DeviceCharacteristics = params['DeviceCharacteristics'] DeviceObject = params['DeviceObject'] - objcls = { - QL_ARCH.X86 : DEVICE_OBJECT32, - QL_ARCH.X8664 : DEVICE_OBJECT64 - }[ql.arch.type] - - addr = ql.os.heap.alloc(ctypes.sizeof(objcls)) - - device_object = objcls() + device_object = make_device_object(ql.arch.bits) device_object.Type = 3 # FILE_DEVICE_CD_ROM_FILE_SYSTEM ? device_object.DeviceExtension = ql.os.heap.alloc(DeviceExtensionSize) device_object.Size = ctypes.sizeof(device_object) + DeviceExtensionSize @@ -186,7 +179,9 @@ def __IoCreateDevice(ql: Qiling, address: int, params): device_object.Characteristics = DeviceCharacteristics - ql.mem.write(addr, bytes(device_object)[:]) + addr = ql.os.heap.alloc(ctypes.sizeof(device_object)) + + ql.mem.write(addr, bytes(device_object)) ql.mem.write_ptr(DeviceObject, addr) # update DriverObject.DeviceObject @@ -476,9 +471,11 @@ def hook_MmGetSystemRoutineAddress(ql: Qiling, address: int, params): index = hook_only_routine_address.index(SystemRoutineName) # found! for dll_name in ('ntoskrnl.exe', 'ntkrnlpa.exe', 'hal.dll'): - if dll_name in ql.loader.dlls: + image = ql.loader.get_image_by_name(dll_name) + + if image: # create fake address - new_function_address = ql.loader.dlls[dll_name] + index + 1 + new_function_address = image.base + index + 1 # update import address table ql.loader.import_symbols[new_function_address] = { 'name': SystemRoutineName, @@ -641,10 +638,7 @@ def hook_KeLeaveCriticalRegion(ql: Qiling, address: int, params): def hook_MmMapLockedPagesSpecifyCache(ql: Qiling, address: int, params): MemoryDescriptorList = params['MemoryDescriptorList'] - mdl_class: ctypes.Structure = { - QL_ARCH.X8664 : MDL64, - QL_ARCH.X86 : MDL32 - }[ql.arch.type] + mdl_class = make_mdl(ql.arch.bits).__class__ mdl_buffer = ql.mem.read(MemoryDescriptorList, ctypes.sizeof(mdl_class)) mdl = mdl_class.from_buffer(mdl_buffer) @@ -757,12 +751,11 @@ def _NtQuerySystemInformation(ql: Qiling, address: int, params): # if SystemInformationLength = 0, we return the total size in ReturnLength NumberOfModules = 1 - if ql.arch.bits == 64: - # only 1 module for ntoskrnl.exe - # FIXME: let users customize this? - size = 4 + ctypes.sizeof(RTL_PROCESS_MODULE_INFORMATION64) * NumberOfModules - else: - size = 4 + ctypes.sizeof(RTL_PROCESS_MODULE_INFORMATION32) * NumberOfModules + rpmi_class = make_rtl_process_module_info(ql.arch.bits).__class__ + + # only 1 module for ntoskrnl.exe + # FIXME: let users customize this? + size = 4 + ctypes.sizeof(rpmi_class) * NumberOfModules if params["ReturnLength"] != 0: ql.mem.write_ptr(params["ReturnLength"], size) @@ -771,16 +764,15 @@ def _NtQuerySystemInformation(ql: Qiling, address: int, params): return STATUS_INFO_LENGTH_MISMATCH else: # return all the loaded modules - if ql.arch.bits == 64: - module = RTL_PROCESS_MODULE_INFORMATION64() - else: - module = RTL_PROCESS_MODULE_INFORMATION32() - + module = make_rtl_process_module_info(ql.arch.bits) module.Section = 0 module.MappedBase = 0 - if ql.loader.is_driver == True: - module.ImageBase = ql.loader.dlls.get("ntoskrnl.exe") + if ql.loader.is_driver: + image = ql.loader.get_image_by_name("ntoskrnl.exe") + assert image, 'image is a driver, but ntoskrnl.exe was not loaded' + + module.ImageBase = image.base module.ImageSize = 0xab000 module.Flags = 0x8804000 @@ -874,7 +866,7 @@ def hook_IoAcquireCancelSpinLock(ql: Qiling, address: int, params): # PEPROCESS PsGetCurrentProcess(); @winsdkapi(cc=STDCALL, params={}) def hook_PsGetCurrentProcess(ql: Qiling, address: int, params): - return ql.eprocess_address + return ql.loader.eprocess_address # HANDLE PsGetCurrentProcessId(); @winsdkapi(cc=STDCALL, params={}) @@ -1073,12 +1065,9 @@ def hook_PsLookupProcessByProcessId(ql: Qiling, address: int, params): ProcessId = params["ProcessId"] Process = params["Process"] - if ql.arch.bits == 64: - obj = EPROCESS64 - else: - obj = EPROCESS32 + eprocess_obj = make_eprocess(ql.arch.bits) + addr = ql.os.heap.alloc(ctypes.sizeof(eprocess_obj)) - addr = ql.os.heap.alloc(ctypes.sizeof(obj)) ql.mem.write_ptr(Process, addr) ql.log.info(f'PID = {ProcessId:#x}, addrof(EPROCESS) == {addr:#x}') diff --git a/qiling/os/windows/dlls/shell32.py b/qiling/os/windows/dlls/shell32.py index abeae6eb..18bcc56c 100644 --- a/qiling/os/windows/dlls/shell32.py +++ b/qiling/os/windows/dlls/shell32.py @@ -3,6 +3,7 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +import ntpath import os from typing import Sequence @@ -146,7 +147,7 @@ def hook_SHGetSpecialFolderPathW(ql: Qiling, address: int, params): dst = params["pszPath"] if directory_id == CSIDL_COMMON_APPDATA: - path = str(ql.os.userprofile + "AppData\\") + path = ntpath.join(ql.os.userprofile, "AppData\\") # We always create the directory appdata_dir = path.split("C:\\")[1].replace("\\", "/") ql.log.debug("dir path: %s" % path) @@ -160,9 +161,10 @@ def hook_SHGetSpecialFolderPathW(ql: Qiling, address: int, params): if not os.path.exists(path_emulated): try: os.makedirs(path_emulated, 0o755) - ql.log.debug("os.makedirs completed") except OSError: ql.log.debug("os.makedirs failed") + else: + ql.log.debug("os.makedirs completed") else: raise QlErrorNotImplemented("API not implemented") diff --git a/qiling/os/windows/dlls/user32.py b/qiling/os/windows/dlls/user32.py index 61c23809..a761ded4 100644 --- a/qiling/os/windows/dlls/user32.py +++ b/qiling/os/windows/dlls/user32.py @@ -140,12 +140,12 @@ def hook_GetDesktopWindow(ql: Qiling, address: int, params): 'hWndNewOwner' : HWND }) def hook_OpenClipboard(ql: Qiling, address: int, params): - return ql.os.clipboard.open(params['hWndNewOwner']) + return int(ql.os.clipboard.open(params['hWndNewOwner'])) # BOOL CloseClipboard(); @winsdkapi(cc=STDCALL, params={}) def hook_CloseClipboard(ql: Qiling, address: int, params): - return ql.os.clipboard.close() + return int(ql.os.clipboard.close()) # HANDLE SetClipboardData( # UINT uFormat, @@ -190,7 +190,7 @@ def hook_GetClipboardData(ql: Qiling, address: int, params): 'format' : UINT }) def hook_IsClipboardFormatAvailable(ql: Qiling, address: int, params): - return ql.os.clipboard.format_available(params['uFormat']) + return int(ql.os.clipboard.format_available(params['uFormat'])) # UINT MapVirtualKeyW( # UINT uCode, @@ -866,8 +866,8 @@ def hook_GetWindowThreadProcessId(ql: Qiling, address: int, params): dst = params["lpdwProcessId"] - if dst != 0: - ql.mem.write(dst, pid.to_bytes(4, "little")) + if dst: + ql.mem.write_ptr(dst, pid, 4) return pid diff --git a/qiling/os/windows/fiber.py b/qiling/os/windows/fiber.py index e8bda865..09862076 100644 --- a/qiling/os/windows/fiber.py +++ b/qiling/os/windows/fiber.py @@ -2,11 +2,16 @@ # # Cross Platform and Multi Architecture Advanced Binary Emulation Framework +from typing import MutableMapping, Optional + from qiling import Qiling +from qiling.const import QL_ARCH +from qiling.os.windows.api import PVOID from qiling.os.windows.const import ERROR_INVALID_PARAMETER +from qiling.os.windows.fncc import CDECL class Fiber: - def __init__(self, idx, cb=None): + def __init__(self, idx: int, cb: Optional[int] = None): self.idx = idx self.data = 0 self.cb = cb @@ -14,61 +19,96 @@ class Fiber: class FiberManager: def __init__(self, ql: Qiling): - self.fibers = {} + self.fibers: MutableMapping[int, Fiber] = {} self.idx = 0 self.ql = ql - def alloc(self, cb=None): - rtn = self.idx - self.fibers[self.idx] = Fiber(self.idx, cb=cb) + def alloc(self, cb: Optional[int] = None) -> int: + idx = self.idx self.idx += 1 - return rtn - def free(self, idx): - if idx in self.fibers: - fiber = self.fibers[idx] + self.fibers[idx] = Fiber(idx, cb) - if fiber.cb: - self.ql.log.debug(f'Skipping emulation of callback function {fiber.cb:#x} for fiber {fiber.idx:#x}') + return idx - """ - ret_addr = self.ql.arch.regs.read(UC_X86_REG_RIP + 6 ) #FIXME, use capstone to get addr of next instr? + def free(self, idx: int) -> bool: + if idx not in self.fibers: + self.last_error = ERROR_INVALID_PARAMETER + return False - # Write Fls data to memory to be accessed by cb - addr = self.ql.os.heap.alloc(self.ql.arch.pointersize) - data = fiber.data.to_bytes(self.ql.arch.pointersize, byteorder='little') - self.ql.mem.write(addr, data) + fiber = self.fibers[idx] - # set up params and return address then jump to callback - if self.ql.arch.pointersize == 8: - self.ql.arch.regs.write(UC_X86_REG_RCX, addr) - else: - self.ql.stack_push(ret_addr) - self.ql.stack_push(ret_addr) - self.ql.log.debug("Jumping to callback @ 0x%X" % fiber.cb) - self.ql.arch.regs.write(UC_X86_REG_RIP, fiber.cb) - # All of this gets overwritten by the rest of the code in fncc.py - # Not sure how to actually make unicorn emulate the callback function due to that - """ + if fiber.cb is not None: + self.ql.log.debug(f'Skipping callback function of fiber {fiber.idx} at {fiber.cb:#010x}') - else: - del self.fibers[idx] - return 1 + # TODO: should figure out how to emulate the fiber callback and still return to complete + # the free api hook. + # + # details: normally the emulation flow is diverted by setting the architectural pc reg to + # the desired address. however that would only take effect when all hooks for the current + # address are done. here we want to call a native function and regain control once it is + # done to complete the 'free' api that was started. + # + # one way to do that it to use 'ql.emu_start' and emulate the callback from its entry point + # till it reaches its return address. that would indeed let us regain control and resume the + # 'free' api hook we started here, but doing that will cause uc to abandon the current + # emulation session -- effectively ending it. once the hooks for the current address are done, + # the program will go idle. + # + # if we choose to emulate till 'ql.os.exit_point' instead, the program will continue but the + # hook we are in will not resume and we will never "return" from it. using 'uc.context_save' + # and 'uc.context_restore' to maintain the current emulation properties does not seem to help + # here. + # + # we skip the fiber callback for now. - self.last_error = ERROR_INVALID_PARAMETER - return 0 + # + # self.ql.log.debug(f'Invoking callback function of fiber {fiber.idx} at {fiber.cb:#010x}') + # self.__invoke_callback(fiber) + # self.ql.log.debug(f'Callback function of fiber {fiber.idx} returned gracefully') + # - def set(self, idx, data): - if idx in self.fibers: - self.fibers[idx].data = data - return 1 + del self.fibers[idx] - self.last_error = ERROR_INVALID_PARAMETER - return 0 + return True - def get(self, idx): - if idx in self.fibers: - return self.fibers[idx].data + # TODO: this one is unused for now; see above + def __invoke_callback(self, fiber: Fiber): + assert fiber.cb is not None - self.last_error = ERROR_INVALID_PARAMETER - return 0 + # we are in an api hook. extract the return address of the free + # api to know where the callback should be returning to + retaddr = self.ql.stack_read(0) + + # one PVOID arg, set to fiber data + args = ((PVOID, fiber.data),) + + # set up call frame for callback + fcall = self.ql.os.fcall_select(CDECL) + fcall.call_native(fiber.cb, args, retaddr) + + # callback has to be invoked before returning from the free api + self.ql.emu_start(fiber.cb, retaddr) + + # unwind call frame + fcall.cc.unwind(len(args)) + + # ms64 cc needs also to unwind the reserved shadow slots on the stack + if self.ql.arch.type == QL_ARCH.X8664: + self.ql.arch.regs.arch_sp += (4 * self.ql.arch.pointersize) + + def set(self, idx: int, data: int) -> bool: + if idx not in self.fibers: + self.last_error = ERROR_INVALID_PARAMETER + return False + + self.fibers[idx].data = data + + return True + + def get(self, idx: int) -> int: + if idx not in self.fibers: + self.last_error = ERROR_INVALID_PARAMETER + return 0 + + return self.fibers[idx].data diff --git a/qiling/os/windows/handle.py b/qiling/os/windows/handle.py index 2a018ba1..b5776c96 100644 --- a/qiling/os/windows/handle.py +++ b/qiling/os/windows/handle.py @@ -4,68 +4,66 @@ # # A Simple Windows Handle Simulation +from typing import Any, MutableMapping, Optional + +from qiling.os.windows.const import STD_ERROR_HANDLE, STD_INPUT_HANDLE, STD_OUTPUT_HANDLE class Handle: ID = 0xa0000000 - def __init__(self, id=None, obj=None, - name=None, permissions=None): + def __init__(self, id: Optional[int] = None, obj: Any = None, name: Optional[str] = None, permissions: Optional[int] = None): if id is None: - self.id = Handle.ID + id = Handle.ID Handle.ID += 1 - else: - self.id = id + + self.id = id self.obj = obj self.name = name self.permissions = permissions - - # rewrite "=" - def __eq__(self, other): + + # overload "==" + def __eq__(self, other: 'Handle'): return self.id == other.id class HandleManager: # IO - STD_INPUT_HANDLE = Handle(id=0xfffffff6) - STD_OUTPUT_HANDLE = Handle(id=0xfffffff5) - STD_ERROR_HANDLE = Handle(id=0xfffffff4) + STDIN = Handle(id=STD_INPUT_HANDLE) + STDOUT = Handle(id=STD_OUTPUT_HANDLE) + STDERR = Handle(id=STD_ERROR_HANDLE) # Register - HKEY_CLASSES_ROOT = Handle(id=0x80000000) - HKEY_CURRENT_CONFIG = Handle(id=0x80000005) - HKEY_CURRENT_USER = Handle(id=0x80000001) + HKEY_CLASSES_ROOT = Handle(id=0x80000000) + HKEY_CURRENT_CONFIG = Handle(id=0x80000005) + HKEY_CURRENT_USER = Handle(id=0x80000001) HKEY_CURRENT_USER_LOCAL_SETTINGS = Handle(id=0x80000007) - HKEY_LOCAL_MACHINE = Handle(id=0x80000002) - HKEY_PERFORMANCE_DATA = Handle(id=0x80000004) + HKEY_LOCAL_MACHINE = Handle(id=0x80000002) + HKEY_PERFORMANCE_DATA = Handle(id=0x80000004) HKEY_PERFORMANCE_NLSTEXT = Handle(id=0x80000060) - HKEY_PERFORMANCE_TEXT = Handle(id=0x80000050) - HKEY_USERS = Handle(id=0x80000003) + HKEY_PERFORMANCE_TEXT = Handle(id=0x80000050) + HKEY_USERS = Handle(id=0x80000003) def __init__(self): - self.handles = {} - self.append(HandleManager.STD_INPUT_HANDLE) - self.append(HandleManager.STD_OUTPUT_HANDLE) - self.append(HandleManager.STD_ERROR_HANDLE) + self.handles: MutableMapping[int, Handle] = {} - def append(self, handle): + self.append(HandleManager.STDIN) + self.append(HandleManager.STDOUT) + self.append(HandleManager.STDERR) + + def append(self, handle: Handle) -> None: self.handles[handle.id] = handle - def get(self, id): + def get(self, id: int) -> Optional[Handle]: return self.handles.get(id, None) - def delete(self, id): - key = self.handles.get(id, None) + def delete(self, id: int) -> None: + key = self.get(id) + if key is not None: del self.handles[id] - def search(self, name): - for handle in self.handles.values(): - if handle.name == name: - return handle - return None + def search(self, name: str) -> Optional[Handle]: + return next((handle for handle in self.handles.values() if handle.name == name), None) - def search_by_obj(self, obj): - for handle in self.handles.values(): - if handle.obj == obj: - return handle - return None + def search_by_obj(self, obj: Any) -> Optional[Handle]: + return next((handle for handle in self.handles.values() if handle.obj == obj), None) diff --git a/qiling/os/windows/registry.py b/qiling/os/windows/registry.py index abdff9f3..70dbff3c 100644 --- a/qiling/os/windows/registry.py +++ b/qiling/os/windows/registry.py @@ -3,13 +3,13 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import json, os, sys - +import json, os from Registry import Registry +from typing import Any, MutableMapping, Optional, Tuple, Union -from qiling.os.windows.const import * +from qiling import Qiling +from qiling.os.windows.const import REG_TYPES from qiling.exception import * -from qiling.const import * # Registry Manager reads data from two places @@ -22,192 +22,265 @@ from qiling.const import * # Registry Manager will only write registry changes to config.json # and will not modify the hive file. - -class RegistryManager: - def __init__(self, ql, hive=None): - self.ql = ql - self.log_registry_dir = self.ql.rootfs - - if self.log_registry_dir == None: - self.log_registry_dir = "qlog" - - self.registry_diff = self.ql.targetname + "_diff.json" - self.regdiff = os.path.join(self.log_registry_dir, "registry", self.registry_diff) - - # hive dir - if hive: - self.hive = hive +class RegConf: + def __init__(self, fname: str): + try: + with open(fname, 'r') as infile: + data = infile.read() + except IOError: + config = {} else: - self.hive = os.path.join(ql.rootfs, "Windows", "registry") - ql.log.debug("Windows Registry PATH: %s" % self.hive) - if not os.path.exists(self.hive) and not self.ql.code: - raise QlErrorFileNotFound(f"Error: Registry files not found in '{self.hive}'!") + config = json.loads(data or '{}') - if not os.path.exists(self.regdiff): - self.registry_config = {} - try: - os.makedirs(os.path.join(self.log_registry_dir, "registry"), 0o755) - except Exception: - pass - else: - # read config - # use registry config first - self.f_config = open(self.regdiff, "rb") - data = self.f_config.read() - if data == b"": - self.registry_config = {} - self.f_config.close() - else: - try: - self.registry_config = json.loads(data) - except json.decoder.JSONDecodeError: - raise QlErrorJsonDecode("Windows Registry JSON decode error") - finally: - self.f_config.close() + self.conf: MutableMapping[str, dict[str, dict]] = config - # hkey local system - self.hklm = {} - try: - self.hklm['SECURITY'] = Registry.Registry(os.path.join(self.hive, 'SECURITY')) - self.hklm['SAM'] = Registry.Registry(os.path.join(self.hive, 'SAM')) - self.hklm['SOFTWARE'] = Registry.Registry(os.path.join(self.hive, 'SOFTWARE')) - self.hklm['SYSTEM'] = Registry.Registry(os.path.join(self.hive, 'SYSTEM')) - self.hklm['HARDWARE'] = Registry.Registry(os.path.join(self.hive, 'HARDWARE')) - # hkey current user - self.hkcu = Registry.Registry(os.path.join(self.hive, 'NTUSER.DAT')) - except FileNotFoundError: - if not ql.code: - QlErrorFileNotFound("WARNING: Registry files not found!") - except Exception: - if not ql.code: - QlErrorFileNotFound("WARNING: Registry files format error") - self.accessed = {} + def exists(self, key: str) -> bool: + return key in self.conf - def exists(self, key): - if key in self.regdiff: - return True - keys = key.split("\\") - self.access(key) - try: - if keys[0] == "HKEY_LOCAL_MACHINE": - reg = self.hklm[keys[1]] - sub = "\\".join(keys[2:]) - data = reg.open(sub) - elif keys[0] == "HKEY_CURRENT_USER": - reg = self.hkcu - sub = "\\".join(keys[1:]) - data = reg.open(sub) - else: - raise QlErrorNotImplemented("Windows Registry %s not implemented" % (keys[0])) - except Exception: - return False + def create(self, key: str) -> None: + if not self.exists(key): + self.conf[key] = {} - return True + def delete(self, key: str, subkey: str) -> None: + if self.exists(key): + del self.conf[key][subkey] - def read(self, key, subkey, reg_type): - # of the key, the subkey is the value checked + def read(self, key: str, subkey: str, reg_type: int) -> Tuple: + if key in self.conf: + subkeys = self.conf[key] - # read reg conf first - if key in self.regdiff and subkey in self.regdiff[key]: - if self.regdiff[key][subkey].type in REG_TYPES: - return REG_TYPES[self.regdiff[key][subkey].type], self.regdiff[key][subkey].value - else: - raise QlErrorNotImplemented( - "Windows Registry Type %s not implemented" % self.regdiff[key][subkey].type) + if subkey in subkeys: + subkey_item = subkeys[subkey] - # read hive - reg = None - data = None - keys = key.split('\\') - try: - if keys[0] == "HKEY_LOCAL_MACHINE": - reg = self.hklm[keys[1]] - sub = "\\".join(keys[2:]) - data = reg.open(sub) - elif keys[0] == "HKEY_CURRENT_USER": - reg = self.hkcu - sub = "\\".join(keys[1:]) - data = reg.open(sub) - else: - raise QlErrorNotImplemented("Windows Registry %s not implemented" % (keys[0])) + item_type = subkey_item['type'] + item_value = subkey_item['value'] - for value in data.values(): - if value.name() == subkey and (reg_type == Registry.RegNone or - value.value_type() == reg_type): + if item_type not in REG_TYPES: + raise QlErrorNotImplemented(f'Windows Registry Type {item_type} not implemented') - self.access(key, value_name=subkey, value=value.value(), type=value.value_type()) - return value.value_type(), value.value() - - except Registry.RegistryKeyNotFoundException: - pass - - self.access(key, value_name=subkey, value=None, type=None) + return REG_TYPES[item_type], item_value return None, None - def access(self, key, value_name=None, value=None, type=None): - if value_name is None: - if key not in self.accessed: - self.accessed[key] = [] - else: - self.accessed[key].append({ - "value_name": value_name, - "value": value, - "type": type, - "position": self.ql.os.stats.syscalls_counter - }) - # we don't have to increase the counter since we are technically inside a hook - - def create(self, key): - self.registry_config[key] = dict() - - def write(self, key, subkey, reg_type, data): - if key not in self.registry_config: + def write(self, key: str, subkey: str, reg_type: int, data: Union[str, bytes, int]) -> None: + if not self.exists(key): self.create(key) - # write registry changes to config.json - self.registry_config[key][subkey] = { - "type": REG_TYPES[reg_type], - "value": data + + self.conf[key][subkey] = { + 'type' : REG_TYPES[reg_type], + 'value' : data } - def delete(self, key, subkey): - del self.registry_config[key][subkey] + def save(self, fname: str): + if self.conf: + with open(fname, 'wb') as ofile: + data = json.dumps(self.conf) - @staticmethod - def _encode_binary_value(data): - # bytes(hex(data), 'ascii') - # TODO + ofile.write(data.encode('utf-8')) + + +class RegHive: + def __init__(self, hname: str): + def __make_reg(kname: str) -> Registry.Registry: + return Registry.Registry(os.path.join(hname, kname)) + + # hkey local system + self.hklm = { + 'SECURITY' : __make_reg('SECURITY'), + 'SAM' : __make_reg('SAM'), + 'SOFTWARE' : __make_reg('SOFTWARE'), + 'SYSTEM' : __make_reg('SYSTEM'), + 'HARDWARE' : __make_reg('HARDWARE') + } + + # hkey current user + self.hkcu = __make_reg('NTUSER.DAT') + + def __split_reg_path(self, key: str) -> Tuple[Optional[Registry.Registry], Optional[str]]: + regsep = '\\' + keys = key.split(regsep) + + if keys[0] == 'HKEY_LOCAL_MACHINE': + reg = self.hklm[keys[1].upper()] + sub = regsep.join(keys[2:]) + + elif keys[0] == 'HKEY_CURRENT_USER': + reg = self.hkcu + sub = regsep.join(keys[1:]) + + else: + reg = None + sub = None + + return reg, sub + + def exists(self, key: str) -> bool: + reg, sub = self.__split_reg_path(key) + + if reg is None: + return False + + try: + reg.open(sub) + except: + return False + else: + return True + + def create(self, key: str) -> None: pass - def write_reg_value_into_mem(self, reg_value, reg_type, address): - length = 0 - # string - if reg_type == Registry.RegSZ or reg_type == Registry.RegExpandSZ: - self.ql.mem.write(address, bytes(reg_value, "utf-16le") + b"\x00") - length = len(reg_value) - elif reg_type == Registry.RegBin: - # you can set REG_BINARY like '\x00\x01\x02' in config.json - if type(reg_value) == str: - self.ql.mem.write(address, bytes(reg_value)) - length = len(reg_value) - else: - raise QlErrorNotImplemented("Windows Registry Type not implemented") - elif reg_type == Registry.RegDWord: - data = self.ql.pack32(reg_value) - self.ql.mem.write(address, data) - length = len(data) - elif reg_type == Registry.RegQWord: - data = self.ql.pack64(reg_value) - self.ql.mem.write(address, data) - length = len(data) + def delete(self, key: str, subkey: str) -> None: + pass + + def read(self, key: str, subkey: str, reg_type: int) -> Tuple: + reg, sub = self.__split_reg_path(key) + + if reg is None: + raise QlErrorNotImplemented(f'registry root key not implemented') + + v_value = None + v_type = None + + try: + data = reg.open(sub) + except Registry.RegistryKeyNotFoundException: + pass else: - raise QlErrorNotImplemented( - "Windows Registry Type write to memory %s not implemented" % (REG_TYPES[reg_type])) + value = next((v for v in data.values() if v.name() == subkey and reg_type in (Registry.RegNone, v.value_type())), None) + + if value: + v_value = value.value() + v_type = value.value_type() + + return (v_type, v_value) + + def write(self, key: str, subkey: str, reg_type: int, data: Union[str, bytes, int]) -> None: + pass + + +class RegistryManager: + def __init__(self, ql: Qiling, hivedir: str): + self.ql = ql + self.regdiff = os.path.join(ql.rootfs, 'registry', f'{ql.targetname}_diff.json') + + # if conf file does not exist, create its directory to enable saving later on + if not os.path.exists(self.regdiff): + os.makedirs(os.path.dirname(self.regdiff), 0o755, exist_ok=True) + + if not ql.code: + if not os.path.exists(hivedir): + raise QlErrorFileNotFound(f'Windows registry directory not found: "{hivedir}"!') + + ql.log.debug(f'Loading Windows registry hive from {hivedir}') + + try: + self.reghive = RegHive(hivedir) + except FileNotFoundError: + if not ql.code: + QlErrorFileNotFound("Windows registry hive not found") + + except Exception: + if not ql.code: + QlErrorFileNotFound("Windows registry hive format error") + + try: + self.regconf = RegConf(self.regdiff) + except json.decoder.JSONDecodeError: + raise QlErrorJsonDecode("Windows registry JSON decode error") + + def exists(self, key: str) -> bool: + self.access(key) + + return self.regconf.exists(key) or self.reghive.exists(key) + + def read(self, key: str, subkey: str, reg_type: int) -> Tuple: + result = self.regconf.read(key, subkey, reg_type) + + if result == (None, None): + result = self.reghive.read(key, subkey, reg_type) + + self.access(key, subkey, *result) + + return result + + def access(self, key: str, name: Optional[str] = None, type: Optional[int] = None, value: Any = None): + self.ql.os.stats.log_reg_access(key, name, type, value) + + def create(self, key: str) -> None: + self.regconf.create(key) + self.reghive.create(key) + + def delete(self, key: str, subkey: str) -> None: + self.regconf.delete(key, subkey) + self.reghive.delete(key, subkey) + + def __reg_mem_read(self, data_type: int, data_addr: int, data_size: int, wide: bool) -> Optional[Union[str, bytes, int]]: + if data_type in (Registry.RegSZ, Registry.RegExpandSZ): + os_utils = self.ql.os.utils + read_string = os_utils.read_wstring if wide else os_utils.read_cstring + + data = read_string(data_addr) + + elif data_type == Registry.RegDWord: + data = self.ql.mem.read_ptr(data_addr, 4) + + elif data_type == Registry.RegQWord: + data = self.ql.mem.read_ptr(data_addr, 8) + + elif data_type == Registry.RegBin: + data = bytes(self.ql.mem.read(data_addr, data_size)) + + else: + data = None + + return data + + def __reg_mem_write(self, data_type: int, data_addr: int, data_val: Union[str, bytes, int], wide: bool) -> Optional[int]: + if data_type in (Registry.RegSZ, Registry.RegExpandSZ): + assert type(data_val) is str + + enc = 'utf-16le' if wide else 'utf-8' + data = f'{data_val}\x00'.encode(enc) + + elif data_type == Registry.RegDWord: + assert type(data_val) is int + + data = self.ql.pack32(data_val) + + elif data_type == Registry.RegQWord: + assert type(data_val) is int + + data = self.ql.pack64(data_val) + + elif data_type == Registry.RegBin: + assert type(data_val) is bytes + + data = data_val + + else: + return None + + self.ql.mem.write(data_addr, data) + + return len(data) + + def write(self, key: str, subkey: str, reg_type: int, data_addr: int, data_size: int, wide: bool) -> None: + data = self.__reg_mem_read(reg_type, data_addr, data_size, wide) + + if data is None: + raise QlErrorNotImplemented(f'registry type {REG_TYPES[reg_type]} not implemented') + + self.regconf.write(key, subkey, reg_type, data) + self.reghive.write(key, subkey, reg_type, data) + + def write_reg_value_into_mem(self, data_type: int, data_addr: int, data_val: Union[str, bytes, int], wide: bool) -> int: + length = self.__reg_mem_write(data_type, data_addr, data_val, wide) + + if length is None: + raise QlErrorNotImplemented(f'registry type {REG_TYPES[data_type]} not implemented') return length def save(self): - # write registry config to config file - if self.registry_config and len(self.registry_config) != 0: - with open(self.regdiff, "wb") as f: - f.write(bytes(json.dumps(self.registry_config), "utf-8")) + self.regconf.save(self.regdiff) diff --git a/qiling/os/windows/structs.py b/qiling/os/windows/structs.py index 4b7705fa..84412a57 100644 --- a/qiling/os/windows/structs.py +++ b/qiling/os/windows/structs.py @@ -3,486 +3,266 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import ctypes, struct +import ctypes from enum import IntEnum -from unicorn.x86_const import * +from qiling import Qiling +from qiling.os.windows.handle import Handle +from qiling.exception import QlErrorNotImplemented +from .wdk_const import IRP_MJ_MAXIMUM_FUNCTION, PROCESSOR_FEATURE_MAX + +def __make_struct(archbits: int): + """Provide a ctypes Structure base class based on the underlying + architecture properties. + """ + + class Struct(ctypes.LittleEndianStructure): + _pack_ = archbits // 8 + + return Struct -from qiling.const import * -from qiling.os.windows.handle import * -from qiling.exception import * -from .wdk_const import * +def __select_native_type(archbits: int): + """Select a ctypes integer type with the underlying architecture + native size. + """ + + __type = { + 32 : ctypes.c_uint32, + 64 : ctypes.c_uint64 + } + + return __type[archbits] -class POINTER32(ctypes.Structure): - _fields_ = [('value', ctypes.c_uint32)] +def __select_pointer_type(archbits: int): + """Provide a pointer base class based on the underlying + architecture properties. + """ + + native_type = __select_native_type(archbits) + Struct = __make_struct(archbits) + + class Pointer(Struct): + _fields_ = ( + ('value', native_type), + ) + + return Pointer -class POINTER64(ctypes.Structure): - _fields_ = [('value', ctypes.c_uint64)] +def make_teb(archbits: int, *args, **kwargs): + """Initialize a TEB structure. + + Additional arguments may be used to initialize specific TEB fields. + """ + + native_type = __select_native_type(archbits) + Struct = __make_struct(archbits) + + class TEB(Struct): + _fields_ = ( + ('CurrentSEH', native_type), + ('StackBase', native_type), + ('StackLimit', native_type), + ('SubSystemTib', native_type), + ('FiberData', native_type), + ('ArbitraryDataSlot', native_type), + ('TebAddress', native_type), + ('EnvironmentPointer', native_type), + ('ProcessID', native_type), + ('ThreadID', native_type), + ('RpcHandle', native_type), + ('TlsAddress', native_type), + ('PebAddress', native_type), + ('LastError', ctypes.c_int32), + ('CriticalSectionsCount', ctypes.c_int32), + ('CsrClientThreadAddress', native_type), + ('Win32ThreadInfo', native_type), + ('Win32ClientInfo', ctypes.c_byte * 124), + ('ReservedWow64', native_type), + ('CurrentLocale', ctypes.c_int32), + ('FpSwStatusReg', ctypes.c_int32), + ('ReservedOS', ctypes.c_byte * 216) + ) + + return TEB(*args, **kwargs) -class TEB: - def __init__(self, - ql, - base=0, - exception_list=0, - stack_base=0, - stack_limit=0, - sub_system_tib=0, - fiber_data=0, - arbitrary_user_pointer=0, - Self=0, - environment_pointer=0, - client_id_unique_process=0, - client_id_unique_thread=0, - rpc_handle=0, - tls_storage=0, - peb_address=0, - last_error_value=0, - last_status_value=0, - count_owned_locks=0, - hard_error_mode=0): - self.ql = ql - self.base = base - self.ExceptionList = exception_list - self.StackBase = stack_base - self.StackLimit = stack_limit - self.SubSystemTib = sub_system_tib - self.FiberData = fiber_data - self.ArbitraryUserPointer = arbitrary_user_pointer - self.Self = Self - self.EnvironmentPointer = environment_pointer - self.ClientIdUniqueProcess = client_id_unique_process - self.ClientIdUniqueThread = client_id_unique_thread - self.RpcHandle = rpc_handle - self.Tls_Storage = tls_storage - self.PEB_Address = peb_address - self.LastErrorValue = last_error_value - self.LastStatusValue = last_status_value - self.Count_Owned_Locks = count_owned_locks - self.HardErrorMode = hard_error_mode +def make_peb(archbits: int, *args, **kwargs): + """Initialize a PEB structure. - def bytes(self): - s = b'' - s += self.ql.pack(self.ExceptionList) # 0x00 - s += self.ql.pack(self.StackBase) # 0x04 - s += self.ql.pack(self.StackLimit) # 0x08 - s += self.ql.pack(self.SubSystemTib) # 0x0c - s += self.ql.pack(self.FiberData) # 0x10 - s += self.ql.pack(self.ArbitraryUserPointer) # 0x14 - s += self.ql.pack(self.Self) # 0x18 - s += self.ql.pack(self.EnvironmentPointer) # 0x1c - s += self.ql.pack(self.ClientIdUniqueProcess) # 0x20 - s += self.ql.pack(self.ClientIdUniqueThread) # 0x24 - s += self.ql.pack(self.RpcHandle) # 0x28 - s += self.ql.pack(self.Tls_Storage) # 0x2c - s += self.ql.pack(self.PEB_Address) # 0x30 - s += self.ql.pack(self.LastErrorValue) # 0x34 - s += self.ql.pack(self.LastStatusValue) # 0x38 - s += self.ql.pack(self.Count_Owned_Locks) # 0x3c - s += self.ql.pack(self.HardErrorMode) # 0x40 - return s + Additional arguments may be used to initialize specific PEB fields. + """ + + native_type = __select_native_type(archbits) + Struct = __make_struct(archbits) + + # https://www.geoffchappell.com/studies/windows/win32/ntdll/structs/peb/index.htm + class PEB(Struct): + _fields_ = ( + ('InheritedAddressSpace', ctypes.c_int8), + ('ReadImageFileExecOptions', ctypes.c_int8), + ('BeingDebugged', ctypes.c_int8), + ('BitField', ctypes.c_int8), + ('Mutant', native_type), + ('ImageBaseAddress', native_type), + ('LdrAddress', native_type), + ('ProcessParameters', native_type), + ('SubSystemData', native_type), + ('ProcessHeap', native_type), + ('FastPebLock', native_type), + ('AtlThunkSListPtr', native_type), + ('IFEOKey', native_type), + ('CrossProcessFlags', ctypes.c_int32), + ('KernelCallbackTable', native_type), + ('SystemReserved', ctypes.c_int32), + ('AtlThunkSListPtr32', ctypes.c_int32), + ('ApiSetMap', native_type), + ('TlsExpansionCounter', ctypes.c_int32), + ('TlsBitmap', native_type), + ('TlsBitmapBits', ctypes.c_int32 * 2), + ('ReadOnlySharedMemoryBase', native_type), + ('SharedData', native_type), + ('ReadOnlyStaticServerData', native_type), + ('AnsiCodePageData', native_type), + ('OemCodePageData', native_type), + ('UnicodeCaseTableData', native_type), + ('NumberOfProcessors', ctypes.c_int32), + ('NtGlobalFlag', ctypes.c_int32), + ('CriticalSectionTimeout', native_type) + # ... more + ) + + obj_size = { + 32: 0x047c, + 64: 0x07c8 + }[archbits] + + obj = PEB(*args, **kwargs) + ctypes.resize(obj, obj_size) + + return obj -# https://www.geoffchappell.com/studies/windows/win32/ntdll/structs/peb/index.htm +# https://docs.microsoft.com/en-us/windows/win32/api/subauth/ns-subauth-unicode_string +# +# typedef struct _UNICODE_STRING { +# USHORT Length; +# USHORT MaximumLength; +# PWSTR Buffer; +# } UNICODE_STRING, *PUNICODE_STRING; +def make_unicode_string(archbits: int, *args, **kwargs): + """Initialize a Unicode String structure. -class PEB: - def __init__(self, - ql, - base=0, - flag=0, - mutant=0, - image_base_address=0, - ldr_address=0, - process_parameters=0, - sub_system_data=0, - process_heap=0, - fast_peb_lock=0, - alt_thunk_s_list_ptr=0, - ifeo_key=0, - number_processors=0): - self.ql = ql - self.base = base - self.flag = flag - self.ImageBaseAddress = image_base_address - self.Mutant = mutant - self.LdrAddress = ldr_address - self.ProcessParameters = process_parameters - self.SubSystemData = sub_system_data - self.ProcessHeap = process_heap - self.FastPebLock = fast_peb_lock - self.AtlThunkSListPtr = alt_thunk_s_list_ptr - self.IFEOKey = ifeo_key - self.numberOfProcessors = number_processors - if self.ql.arch.type == 32: - self.size = 0x0468 - else: - self.size = 0x07B0 + Additional arguments may be used to initialize specific structure fields. + """ - def write(self, addr): - s = b'' - s += self.ql.pack(self.flag) # 0x0 / 0x0 - s += self.ql.pack(self.Mutant) # 0x4 / 0x8 - s += self.ql.pack(self.ImageBaseAddress) # 0x8 / 0x10 - s += self.ql.pack(self.LdrAddress) # 0xc / 0x18 - s += self.ql.pack(self.ProcessParameters) # 0x10 / 0x20 - s += self.ql.pack(self.SubSystemData) # 0x14 / 0x28 - s += self.ql.pack(self.ProcessHeap) # 0x18 / 0x30 - s += self.ql.pack(self.FastPebLock) # 0x1c / 0x38 - s += self.ql.pack(self.AtlThunkSListPtr) # 0x20 / 0x40 - s += self.ql.pack(self.IFEOKey) # 0x24 / 0x48 - self.ql.mem.write(addr, s) - # FIXME: understand how each attribute of the PEB works before adding it - self.ql.mem.write(addr + 0x64, self.ql.pack(self.numberOfProcessors)) + native_type = __select_native_type(archbits) + Struct = __make_struct(archbits) + class UNICODE_STRING(Struct): + _fields_ = ( + ('Length', ctypes.c_uint16), + ('MaximumLength', ctypes.c_uint16), + ('Buffer', native_type) + ) -class LDR_DATA: - def __init__(self, - ql, - base=0, - Length=0, - Initialized=0, - SsHandle=0, - InLoadOrderModuleList={ - 'Flink': 0, - 'Blink': 0 - }, - InMemoryOrderModuleList={ - 'Flink': 0, - 'Blink': 0 - }, - InInitializationOrderModuleList={ - 'Flink': 0, - 'Blink': 0 - }, - EntryInProgress=0, - ShutdownInProgress=0, - ShutdownThreadId=0): - self.ql = ql - self.base = base - self.Length = Length - self.Initialized = Initialized - self.SsHandle = SsHandle - self.InLoadOrderModuleList = InLoadOrderModuleList - self.InMemoryOrderModuleList = InMemoryOrderModuleList - self.InInitializationOrderModuleList = InInitializationOrderModuleList - self.EntryInProgress = EntryInProgress - self.ShutdownInProgress = ShutdownInProgress - self.selfShutdownThreadId = ShutdownThreadId + return UNICODE_STRING(*args, **kwargs) - def bytes(self): - s = b'' - s += self.ql.pack32(self.Length) # 0x0 - s += self.ql.pack32(self.Initialized) # 0x4 - s += self.ql.pack(self.SsHandle) # 0x8 - s += self.ql.pack(self.InLoadOrderModuleList['Flink']) # 0x0c - s += self.ql.pack(self.InLoadOrderModuleList['Blink']) - s += self.ql.pack(self.InMemoryOrderModuleList['Flink']) # 0x14 - s += self.ql.pack(self.InMemoryOrderModuleList['Blink']) - s += self.ql.pack( - self.InInitializationOrderModuleList['Flink']) # 0x1C - s += self.ql.pack(self.InInitializationOrderModuleList['Blink']) - s += self.ql.pack(self.EntryInProgress) - s += self.ql.pack(self.ShutdownInProgress) - s += self.ql.pack(self.selfShutdownThreadId) +# https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_driver_object +# +# typedef struct _DRIVER_OBJECT { +# CSHORT Type; +# CSHORT Size; +# PDEVICE_OBJECT DeviceObject; +# ULONG Flags; +# PVOID DriverStart; +# ULONG DriverSize; +# PVOID DriverSection; +# PDRIVER_EXTENSION DriverExtension; +# UNICODE_STRING DriverName; +# PUNICODE_STRING HardwareDatabase; +# PFAST_IO_DISPATCH FastIoDispatch; +# PDRIVER_INITIALIZE DriverInit; +# PDRIVER_STARTIO DriverStartIo; +# PDRIVER_UNLOAD DriverUnload; +# PDRIVER_DISPATCH MajorFunction[IRP_MJ_MAXIMUM_FUNCTION + 1]; +# } DRIVER_OBJECT, *PDRIVER_OBJECT; - return s +def make_driver_object(ql: Qiling, base: int, archbits: int): + native_type = __select_native_type(archbits) + pointer_type = __select_pointer_type(archbits) + Struct = __make_struct(archbits) + ucstrtype = make_unicode_string(archbits).__class__ -class LDR_DATA_TABLE_ENTRY: - def __init__(self, - ql, - base=0, - InLoadOrderLinks={ - 'Flink': 0, - 'Blink': 0 - }, - InMemoryOrderLinks={ - 'Flink': 0, - 'Blink': 0 - }, - InInitializationOrderLinks={ - 'Flink': 0, - 'Blink': 0 - }, - DllBase=0, - EntryPoint=0, - SizeOfImage=0, - FullDllName='', - BaseDllName='', - Flags=0, - LoadCount=0, - TlsIndex=0, - HashLinks=0, - SectionPointer=0, - CheckSum=0, - TimeDateStamp=0, - LoadedImports=0, - EntryPointActivationContext=0, - PatchInformation=0, - ForwarderLinks=0, - ServiceTagLinks=0, - StaticLinks=0, - ContextInformation=0, - OriginalBase=0, - LoadTime=0): - self.ql = ql - self.base = base - self.InLoadOrderLinks = InLoadOrderLinks - self.InMemoryOrderLinks = InMemoryOrderLinks - self.InInitializationOrderLinks = InInitializationOrderLinks - self.DllBase = DllBase - self.EntryPoint = EntryPoint - self.SizeOfImage = SizeOfImage + class DRIVER_OBJECT(Struct): + _fields_ = ( + ('_Type', ctypes.c_uint16), + ('_Size', ctypes.c_uint16), + ('_DeviceObject', pointer_type), + ('_Flags', ctypes.c_uint32), + ('_DriverStart', pointer_type), + ('_DriverSize', ctypes.c_uint32), + ('_DriverSection', pointer_type), + ('_DriverExtension', pointer_type), + ('_DriverName', ucstrtype), + ('_HardwareDatabase', pointer_type), + ('_FastIoDispatch', pointer_type), + ('_DriverInit', pointer_type), + ('_DriverStartIo', pointer_type), + ('_DriverUnload', pointer_type), + ('_MajorFunction', native_type * (IRP_MJ_MAXIMUM_FUNCTION + 1)) + ) - FullDllName = FullDllName.encode("utf-16le") - self.FullDllName = {} - self.FullDllName['Length'] = len(FullDllName) - self.FullDllName['MaximumLength'] = len(FullDllName) + 2 - self.FullDllName['BufferPtr'] = ql.heap.alloc( - self.FullDllName['MaximumLength']) - ql.mem.write(self.FullDllName['BufferPtr'], FullDllName + b"\x00\x00") + def __read_obj(self) -> 'DRIVER_OBJECT': + data = ql.mem.read(base, ctypes.sizeof(self)) - BaseDllName = BaseDllName.encode("utf-16le") - self.BaseDllName = {} - self.BaseDllName['Length'] = len(BaseDllName) - self.BaseDllName['MaximumLength'] = len(BaseDllName) + 2 - self.BaseDllName['BufferPtr'] = ql.heap.alloc( - self.BaseDllName['MaximumLength']) - ql.mem.write(self.BaseDllName['BufferPtr'], BaseDllName + b"\x00\x00") + return self.__class__.from_buffer(data) - self.Flags = Flags - self.LoadCount = LoadCount - self.TlsIndex = TlsIndex - self.HashLinks = HashLinks - self.SectionPointer = SectionPointer - self.CheckSum = CheckSum - self.TimeDateStamp = TimeDateStamp - self.LoadedImports = LoadedImports - self.EntryPointActivationContext = EntryPointActivationContext - self.PatchInformation = PatchInformation - self.ForwarderLinks = ForwarderLinks - self.ServiceTagLinks = ServiceTagLinks - self.StaticLinks = StaticLinks - self.ContextInformation = ContextInformation - self.OriginalBase = OriginalBase - self.LoadTime = LoadTime + def __write_obj(self) -> None: + ql.mem.write(base, bytes(self)) - def attrs(self): - return ", ".join("{}={}".format(k, getattr(self, k)) - for k in self.__dict__.keys()) + # get MajorFunction + @property + def MajorFunction(self): + obj = self.__read_obj() - def print(self): - return "[{}:{}]".format(self.__class__.__name__, self.attrs()) + return getattr(obj, '_MajorFunction') - def bytes(self): - s = b'' - s += self.ql.pack(self.InLoadOrderLinks['Flink']) # 0x0 - s += self.ql.pack(self.InLoadOrderLinks['Blink']) - s += self.ql.pack(self.InMemoryOrderLinks['Flink']) # 0x8 - s += self.ql.pack(self.InMemoryOrderLinks['Blink']) - s += self.ql.pack(self.InInitializationOrderLinks['Flink']) # 0x10 - s += self.ql.pack(self.InInitializationOrderLinks['Blink']) - s += self.ql.pack(self.DllBase) # 0x18 - s += self.ql.pack(self.EntryPoint) # 0x1c - s += self.ql.pack(self.SizeOfImage) # 0x20 - s += self.ql.pack16(self.FullDllName['Length']) # 0x24 - s += self.ql.pack16(self.FullDllName['MaximumLength']) # 0x26 + @property + def DeviceObject(self): + obj = self.__read_obj() - if self.ql.arch == QL_ARCH.X8664: - s += self.ql.pack32(0) + return getattr(obj, '_DeviceObject').value - s += self.ql.pack(self.FullDllName['BufferPtr']) # 0x28 - s += self.ql.pack16(self.BaseDllName['Length']) - s += self.ql.pack16(self.BaseDllName['MaximumLength']) + @DeviceObject.setter + def DeviceObject(self, value): + obj = self.__read_obj() + getattr(obj, '_DeviceObject').value = value - if self.ql.arch == QL_ARCH.X8664: - s += self.ql.pack32(0) + obj.__write_obj() - s += self.ql.pack(self.BaseDllName['BufferPtr']) - s += self.ql.pack(self.Flags) - s += self.ql.pack(self.LoadCount) - s += self.ql.pack(self.TlsIndex) - s += self.ql.pack(self.HashLinks) - s += self.ql.pack(self.SectionPointer) - s += self.ql.pack(self.CheckSum) - s += self.ql.pack(self.TimeDateStamp) - s += self.ql.pack(self.LoadedImports) - s += self.ql.pack(self.EntryPointActivationContext) - s += self.ql.pack(self.PatchInformation) - s += self.ql.pack(self.ForwarderLinks) - s += self.ql.pack(self.ServiceTagLinks) - s += self.ql.pack(self.StaticLinks) - s += self.ql.pack(self.ContextInformation) - s += self.ql.pack(self.OriginalBase) - s += self.ql.pack(self.LoadTime) - - return s - - -''' -https://docs.microsoft.com/en-us/windows/win32/api/subauth/ns-subauth-unicode_string - -typedef struct _UNICODE_STRING { - USHORT Length; - USHORT MaximumLength; - PWSTR Buffer; -} UNICODE_STRING, *PUNICODE_STRING; -''' - - -class UNICODE_STRING64(ctypes.Structure): - _pack_ = 8 - _fields_ = (('Length', ctypes.c_uint16), ('MaximumLength', ctypes.c_int16), - ('Buffer', ctypes.c_uint64)) - - -class UNICODE_STRING32(ctypes.Structure): - _pack_ = 4 - _fields_ = (('Length', ctypes.c_uint16), ('MaximumLength', ctypes.c_int16), - ('Buffer', ctypes.c_uint32)) - - -''' -https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/wdm/ns-wdm-_driver_object - -typedef struct _DRIVER_OBJECT { - CSHORT Type; - CSHORT Size; - PDEVICE_OBJECT DeviceObject; - ULONG Flags; - PVOID DriverStart; - ULONG DriverSize; - PVOID DriverSection; - PDRIVER_EXTENSION DriverExtension; - UNICODE_STRING DriverName; - PUNICODE_STRING HardwareDatabase; - PFAST_IO_DISPATCH FastIoDispatch; - PDRIVER_INITIALIZE DriverInit; - PDRIVER_STARTIO DriverStartIo; - PDRIVER_UNLOAD DriverUnload; - PDRIVER_DISPATCH MajorFunction[IRP_MJ_MAXIMUM_FUNCTION + 1]; -} DRIVER_OBJECT, *PDRIVER_OBJECT; -''' - - -class DRIVER_OBJECT64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ("_Type", ctypes.c_uint16), - ("_Size", ctypes.c_uint16), - ("_DeviceObject", POINTER64), - ("_Flags", ctypes.c_uint32), - ("_DriverStart", POINTER64), - ("_DriverSize", ctypes.c_uint32), - ("_DriverSection", POINTER64), - ("_DriverExtension", POINTER64), - ("_DriverName", UNICODE_STRING64), - ("_HardwareDatabase", POINTER64), - ("_FastIoDispatch", POINTER64), - ("_DriverInit", POINTER64), - ("_DriverStartIo", POINTER64), - ("_DriverUnload", POINTER64), - ("_MajorFunction", ctypes.c_uint64 * (IRP_MJ_MAXIMUM_FUNCTION + 1))) - - def __init__(self, ql, base): - self.ql = ql - self.base = base - - # get MajorFunction - @property - def MajorFunction(self): - data = self.ql.mem.read(self.base, ctypes.sizeof(self)) - obj = type(self).from_buffer(data) - return obj._MajorFunction - - @property - def DeviceObject(self): - # TODO: improve this code to avoid reading the whole object - data = self.ql.mem.read(self.base, ctypes.sizeof(self)) - obj = type(self).from_buffer(data) - return obj._DeviceObject.value - - @DeviceObject.setter - def DeviceObject(self, value): - # TODO: improve this code to avoid reading/writing the whole object - data = self.ql.mem.read(self.base, ctypes.sizeof(self)) - obj = type(self).from_buffer(data) - obj._DeviceObject.value = value - # update back to memory. - self.ql.mem.write(self.base, bytes(obj)) - - @property - def DriverUnload(self): - data = self.ql.mem.read(self.base, ctypes.sizeof(self)) - obj = type(self).from_buffer(data) - return obj._DriverUnload.value - - -class DRIVER_OBJECT32(ctypes.Structure): - _pack_ = 4 - _fields_ = ( - ("_Type", ctypes.c_uint16), - ("_Size", ctypes.c_uint16), - ("_DeviceObject", POINTER32), - ("_Flags", ctypes.c_uint32), - ("_DriverStart", POINTER32), - ("_DriverSize", ctypes.c_uint32), - ("_DriverSection", POINTER32), - ("_DriverExtension", POINTER32), - ("_DriverName", UNICODE_STRING32), - ("_HardwareDatabase", POINTER32), - ("_FastIoDispatch", POINTER32), - ("_DriverInit", POINTER32), - ("_DriverStartIo", POINTER32), - ("_DriverUnload", POINTER32), - ("_MajorFunction", ctypes.c_uint32 * (IRP_MJ_MAXIMUM_FUNCTION + 1))) - - def __init__(self, ql, base): - self.ql = ql - self.base = base - - # get MajorFunction - @property - def MajorFunction(self): - data = self.ql.mem.read(self.base, ctypes.sizeof(self)) - obj = type(self).from_buffer(data) - return obj._MajorFunction - - @property - def DeviceObject(self): - # TODO: improve this code to avoid reading the whole object - data = self.ql.mem.read(self.base, ctypes.sizeof(self)) - obj = type(self).from_buffer(data) - return obj._DeviceObject.value - - @DeviceObject.setter - def DeviceObject(self, value): - # TODO: improve this code to avoid reading/writing the whole object - data = self.ql.mem.read(self.base, ctypes.sizeof(self)) - obj = type(self).from_buffer(data) - obj._DeviceObject.value = value - # update back to memory. - self.ql.mem.write(self.base, bytes(obj)) - - @property - def DriverUnload(self): - data = self.ql.mem.read(self.base, ctypes.sizeof(self)) - obj = type(self).from_buffer(data) - return obj._DriverUnload.value + @property + def DriverUnload(self): + obj = self.__read_obj() + return getattr(obj, '_DriverUnload').value + return DRIVER_OBJECT() class KSYSTEM_TIME(ctypes.Structure): - _fields_ = (('LowPart', ctypes.c_uint32), ('High1Time', ctypes.c_int32), - ('High2Time', ctypes.c_int32)) + _fields_ = ( + ('LowPart', ctypes.c_uint32), + ('High1Time', ctypes.c_int32), + ('High2Time', ctypes.c_int32) + ) -class LARGE_INTEGER_DUMMYSTRUCTNAME(ctypes.Structure): +class LARGE_INTEGER_DUMMYSTRUCTNAME(ctypes.LittleEndianStructure): _fields_ = ( ('LowPart', ctypes.c_uint32), ('HighPart', ctypes.c_int32), @@ -495,8 +275,12 @@ class LARGE_INTEGER(ctypes.Union): ('QuadPart', ctypes.c_int64), ) +# https://www.geoffchappell.com/studies/windows/km/ntoskrnl/structs/kuser_shared_data/index.htm +# +# struct information: +# https://doxygen.reactos.org/d8/dae/modules_2rostests_2winetests_2ntdll_2time_8c_source.html -class KUSER_SHARED_DATA(ctypes.Structure): +class KUSER_SHARED_DATA(ctypes.LittleEndianStructure): _fields_ = ( ('TickCountLowDeprecated', ctypes.c_uint32), ('TickCountMultiplier', ctypes.c_uint32), @@ -539,216 +323,123 @@ class KUSER_SHARED_DATA(ctypes.Structure): ('TestRetInstruction', ctypes.c_uint8), ('_padding0', ctypes.c_uint8 * 0x2F8)) +def make_list_entry(archbits: int): + native_type = __select_native_type(archbits) + Struct = __make_struct(archbits) -class LIST_ENTRY32(ctypes.Structure): - _pack_ = 4 - _fields_ = ( - ('Flink', ctypes.c_uint32), - ('Blink', ctypes.c_uint32), - ) - - -class KDEVICE_QUEUE_ENTRY32(ctypes.Structure): - _pack_ = 4 - _fields_ = ( - ('DeviceListEntry', LIST_ENTRY32), - ('SortKey', ctypes.c_uint32), - ('Inserted', ctypes.c_uint8) + class LIST_ENTRY(Struct): + _fields_ = ( + ('Flink', native_type), + ('Blink', native_type) ) + return LIST_ENTRY -class WAIT_ENTRY32(ctypes.Structure): - _pack_ = 4 - _fields_ = ( - ('DmaWaitEntry', LIST_ENTRY32), - ('NumberOfChannels', ctypes.c_uint32), - ('DmaContext', ctypes.c_uint32) +def make_device_object(archbits: int): + native_type = __select_native_type(archbits) + pointer_type = __select_pointer_type(archbits) + Struct = __make_struct(archbits) + + LIST_ENTRY = make_list_entry(archbits) + + class KDEVICE_QUEUE_ENTRY(Struct): + _fields_ = ( + ('DeviceListEntry', LIST_ENTRY), + ('SortKey', ctypes.c_uint32), + ('Inserted', ctypes.c_uint8) ) + class WAIT_ENTRY(Struct): + _fields_ = ( + ('DmaWaitEntry', LIST_ENTRY), + ('NumberOfChannels', ctypes.c_uint32), + ('DmaContext', ctypes.c_uint32) + ) -class WAIT_QUEUE_UNION32(ctypes.Union): - _pack_ = 4 - _fields_ = ("WaitQueueEntry", KDEVICE_QUEUE_ENTRY32), ("Dma", WAIT_ENTRY32) + class WAIT_QUEUE_UNION(ctypes.Union): + _pack_ = archbits // 8 + _fields_ = ( + ("WaitQueueEntry", KDEVICE_QUEUE_ENTRY), + ("Dma", WAIT_ENTRY) + ) + class WAIT_CONTEXT_BLOCK(Struct): + _fields_ = ( + ('WaitQueue', WAIT_QUEUE_UNION), + ('DeviceRoutine', pointer_type), + ('DeviceContext', pointer_type), + ('NumberOfMapRegisters', ctypes.c_uint32), + ('DeviceObject', pointer_type), + ('CurrentIrp', pointer_type), + ('BufferChainingDpc', pointer_type) + ) -class WAIT_CONTEXT_BLOCK32(ctypes.Structure): - _pack_ = 4 - _fields_ = (('WaitQueue', WAIT_QUEUE_UNION32), - ('DeviceRoutine', POINTER32), - ('DeviceContext', POINTER32), - ('NumberOfMapRegisters', ctypes.c_uint32), - ('DeviceObject', POINTER32), - ('CurrentIrp', POINTER32), - ('BufferChainingDpc', POINTER32)) + class KDEVICE_QUEUE(Struct): + _fields_ = ( + ('Type', ctypes.c_int16), + ('Size', ctypes.c_int16), + ('DeviceListHead', LIST_ENTRY), + ('Lock', ctypes.c_uint32), + ('Busy', ctypes.c_uint8) + ) + # class SINGLE_LIST_ENTRY(Struct): + # _fields_ = ( + # ('Next', native_type), + # ) -class KDEVICE_QUEUE32(ctypes.Structure): - _pack_ = 4 - _fields_ = (('Type', ctypes.c_int16), ('Size', ctypes.c_int16), - ('DeviceListHead', LIST_ENTRY32), ('Lock', ctypes.c_uint32), - ('Busy', ctypes.c_uint8)) + # https://github.com/ntdiff/headers/blob/master/Win10_1507_TS1/x64/System32/hal.dll/Standalone/_KDPC.h + class KDPC(Struct): + _fields_ = ( + ('Type', ctypes.c_uint8), + ('Importance', ctypes.c_uint8), + ('Number', ctypes.c_uint16), + ('DpcListEntry', LIST_ENTRY), + ('DeferredRoutine', pointer_type), + ('DeferredContext', pointer_type), + ('SystemArgument1', pointer_type), + ('SystemArgument2', pointer_type), + ('DpcData', pointer_type) + ) + class DISPATCHER_HEADER(Struct): + _fields_ = ( + ('Lock', ctypes.c_int32), + ('SignalState', ctypes.c_int32), + ('WaitListHead', LIST_ENTRY) + ) -class SINGLE_LIST_ENTRY32(ctypes.Structure): - _fields_ = [(('Next', ctypes.c_uint32))] + # https://docs.microsoft.com/vi-vn/windows-hardware/drivers/ddi/wdm/ns-wdm-_device_object + class DEVICE_OBJECT(Struct): + _fields_ = ( + ('Type', ctypes.c_int16), + ('Size', ctypes.c_uint16), + ('ReferenceCount', ctypes.c_int32), + ('DriverObject', pointer_type), + ('NextDevice', pointer_type), + ('AttachedDevice', pointer_type), + ('CurrentIrp', pointer_type), + ('Timer', pointer_type), + ('Flags', ctypes.c_uint32), + ('Characteristics', ctypes.c_uint32), + ('Vpb', pointer_type), + ('DeviceExtension', native_type), + ('DeviceType', ctypes.c_uint32), + ('StackSize', ctypes.c_int16), + ('Queue', WAIT_CONTEXT_BLOCK), + ('AlignmentRequirement', ctypes.c_uint32), + ('DeviceQueue', KDEVICE_QUEUE), + ('Dpc', KDPC), + ('ActiveThreadCount', ctypes.c_uint32), + ('SecurityDescriptor', pointer_type), + ('DeviceLock', DISPATCHER_HEADER), + ('SectorSize', ctypes.c_uint16), + ('Spare1', ctypes.c_uint16), + ('DeviceObjectExtension', pointer_type), + ('Reserved', pointer_type) + ) - -# https://github.com/ntdiff/headers/blob/master/Win10_1507_TS1/x64/System32/hal.dll/Standalone/_KDPC.h -class KDPC32(ctypes.Structure): - _pack_ = 4 - _fields_ = ( - ('Type', ctypes.c_uint8), - ('Importance', ctypes.c_uint8), - ('Number', ctypes.c_uint16), - ('DpcListEntry', LIST_ENTRY32), - ('DeferredRoutine', POINTER32), - ('DeferredContext', POINTER32), - ('SystemArgument1', POINTER32), - ('SystemArgument2', POINTER32), - ('DpcData', POINTER32), - ) - - -class DISPATCHER_HEADER32(ctypes.Structure): - _fields_ = ( - ('Lock', ctypes.c_int32), - ('SignalState', ctypes.c_int32), - ('WaitListHead', LIST_ENTRY32), - ) - - -# https://docs.microsoft.com/vi-vn/windows-hardware/drivers/ddi/wdm/ns-wdm-_device_object -class DEVICE_OBJECT32(ctypes.Structure): - _pack_ = 4 - _fields_ = ( - ('Type', ctypes.c_int16), - ('Size', ctypes.c_uint16), - ('ReferenceCount', ctypes.c_int32), - ('DriverObject', POINTER32), - ('NextDevice', POINTER32), - ('AttachedDevice', POINTER32), - ('CurrentIrp', POINTER32), - ('Timer', POINTER32), - ('Flags', ctypes.c_uint32), - ('Characteristics', ctypes.c_uint32), - ('Vpb', POINTER32), - ('DeviceExtension', ctypes.c_uint32), - ('DeviceType', ctypes.c_uint32), - ('StackSize', ctypes.c_int16), - ('Queue', WAIT_CONTEXT_BLOCK32), - ('AlignmentRequirement', ctypes.c_uint32), - ('DeviceQueue', KDEVICE_QUEUE32), - ('Dpc', KDPC32), - ('ActiveThreadCount', ctypes.c_uint32), - ('SecurityDescriptor', POINTER32), - ('DeviceLock', DISPATCHER_HEADER32), - ('SectorSize', ctypes.c_uint16), - ('Spare1', ctypes.c_uint16), - ('DeviceObjectExtension', POINTER32), - ('Reserved', POINTER32), - ) - - -## 64bit structures -class LIST_ENTRY64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ('Flink', ctypes.c_uint64), - ('Blink', ctypes.c_uint64), - ) - - -class KDEVICE_QUEUE_ENTRY64(ctypes.Structure): - _pack_ = 8 - _fields_ = (('DeviceListEntry', LIST_ENTRY64), - ('SortKey', ctypes.c_uint32), ('Inserted', ctypes.c_uint8)) - - -class WAIT_ENTRY64(ctypes.Structure): - _pack_ = 8 - _fields_ = (('DmaWaitEntry', LIST_ENTRY64), - ('NumberOfChannels', ctypes.c_uint32), ('DmaContext', - ctypes.c_uint32)) - - -class WAIT_QUEUE_UNION64(ctypes.Union): - _fields_ = ("WaitQueueEntry", KDEVICE_QUEUE_ENTRY64), ("Dma", WAIT_ENTRY64) - - -class WAIT_CONTEXT_BLOCK64(ctypes.Structure): - _pack_ = 8 - _fields_ = (('WaitQueue', WAIT_QUEUE_UNION64), - ('DeviceRoutine', POINTER64), ('DeviceContext', POINTER64), - ('NumberOfMapRegisters', ctypes.c_uint32), ('DeviceObject', - POINTER64), - ('CurrentIrp', POINTER64), ('BufferChainingDpc', POINTER64)) - - -class KDEVICE_QUEUE64(ctypes.Structure): - _pack_ = 8 - _fields_ = (('Type', ctypes.c_int16), ('Size', ctypes.c_int16), - ('DeviceListHead', LIST_ENTRY64), ('Lock', ctypes.c_uint32), - ('Busy', ctypes.c_uint8)) - - -class SINGLE_LIST_ENTRY64(ctypes.Structure): - _pack_ = 8 - _fields_ = [(('Next', ctypes.c_uint64))] - - -class KDPC64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ('Type', ctypes.c_uint8), - ('Importance', ctypes.c_uint8), - ('Number', ctypes.c_uint16), - ('DpcListEntry', LIST_ENTRY64), - ('DeferredRoutine', POINTER64), - ('DeferredContext', POINTER64), - ('SystemArgument1', POINTER64), - ('SystemArgument2', POINTER64), - ('DpcData', POINTER64), - ) - - -class DISPATCHER_HEADER64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ('Lock', ctypes.c_int32), - ('SignalState', ctypes.c_int32), - ('WaitListHead', LIST_ENTRY64), - ) - - -class DEVICE_OBJECT64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ('Type', ctypes.c_int16), - ('Size', ctypes.c_uint16), - ('ReferenceCount', ctypes.c_int32), - ('DriverObject', POINTER64), - ('NextDevice', POINTER64), - ('AttachedDevice', POINTER64), - ('CurrentIrp', POINTER64), - ('Timer', POINTER64), - ('Flags', ctypes.c_uint32), - ('Characteristics', ctypes.c_uint32), - ('Vpb', POINTER64), - ('DeviceExtension', ctypes.c_uint64), - ('DeviceType', ctypes.c_uint32), - ('StackSize', ctypes.c_int16), - ('Queue', WAIT_CONTEXT_BLOCK64), - ('AlignmentRequirement', ctypes.c_uint32), - ('DeviceQueue', KDEVICE_QUEUE64), - ('Dpc', KDPC64), - ('ActiveThreadCount', ctypes.c_uint32), - ('SecurityDescriptor', POINTER64), - ('DeviceLock', DISPATCHER_HEADER64), - ('SectorSize', ctypes.c_uint16), - ('Spare1', ctypes.c_uint16), - ('DeviceObjectExtension', POINTER64), - ('Reserved', POINTER64), - ) + return DEVICE_OBJECT() # struct IO_STATUS_BLOCK { @@ -759,234 +450,102 @@ class DEVICE_OBJECT64(ctypes.Structure): # ULONG_PTR Information; # }; +def make_irp(archbits: int): + pointer_type = __select_pointer_type(archbits) + native_type = __select_native_type(archbits) + Struct = __make_struct(archbits) -class IO_STATUS_BLOCK_DUMMY64(ctypes.Union): - _pack_ = 8 - _fields_ = ( - ('Status', ctypes.c_int32), - ('Pointer', POINTER64), - ) + LIST_ENTRY = make_list_entry(archbits) - -class IO_STATUS_BLOCK64(ctypes.Structure): - _pack_ = 8 - _fields_ = (('Status', IO_STATUS_BLOCK_DUMMY64), ('Information', - POINTER64)) - - -class IO_STATUS_BLOCK_DUMMY32(ctypes.Union): - _fields_ = ( - ('Status', ctypes.c_int32), - ('Pointer', POINTER32), - ) - - -class IO_STATUS_BLOCK32(ctypes.Structure): - _pack_ = 4 - _fields_ = (('Status', IO_STATUS_BLOCK_DUMMY32), ('Information', - POINTER32)) - - -# struct IO_STACK_LOCATION { -# UCHAR MajorFunction; -# UCHAR MinorFunction; -# UCHAR Flags; -# UCHAR Control; -# union { -# struct { -# char _padding1[4]; -# ULONG OutputBufferLength; -# char _padding2[4]; -# ULONG POINTER_ALIGNMENT InputBufferLength; -# char _padding3[4]; -# ULONG POINTER_ALIGNMENT FsControlCode; -# char _padding4[4]; -# PVOID Type3InputBuffer; -# } FileSystemControl; -# struct { -# char _padding5[4]; -# ULONG OutputBufferLength; -# ULONG POINTER_ALIGNMENT InputBufferLength; // 10 -# char _padding7[4]; -# ULONG POINTER_ALIGNMENT IoControlCode; // 18 -# char _padding8[4]; -# PVOID Type3InputBuffer; // 20 -# } DeviceIoControl; -# } Parameters; -# }; -class IO_STACK_LOCATION_FILESYSTEMCONTROL64(ctypes.Structure): - _pack_ = 8 - _fields_ = (('OutputBufferLength', ctypes.c_uint32), ('_padding1', - ctypes.c_uint32), - ('InputBufferLength', ctypes.c_uint32), ('_padding2', - ctypes.c_uint32), - ('FsControlCode', ctypes.c_uint32), ('Type3InputBuffer', - POINTER64)) - - -class IO_STACK_LOCATION_FILESYSTEMCONTROL32(ctypes.Structure): - _pack_ = 4 - _fields_ = ( - ('OutputBufferLength', ctypes.c_uint32), - ('InputBufferLength', ctypes.c_uint32), - ('FsControlCode', ctypes.c_uint32), - ('Type3InputBuffer', POINTER32)) - - -class IO_STACK_LOCATION_DEVICEIOCONTROL64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ('OutputBufferLength', ctypes.c_uint32), - ('_padding1', ctypes.c_uint32), - ('InputBufferLength', ctypes.c_uint32), - ('_padding2', ctypes.c_uint32), - ('IoControlCode', ctypes.c_uint32), - ('Type3InputBuffer', POINTER64)) - - -class IO_STACK_LOCATION_DEVICEIOCONTROL32(ctypes.Structure): - _pack_ = 4 - _fields_ = ( - ('OutputBufferLength', ctypes.c_uint32), - ('InputBufferLength', ctypes.c_uint32), - ('IoControlCode', ctypes.c_uint32), - ('Type3InputBuffer', POINTER32) + class IO_STATUS_BLOCK_DUMMY(ctypes.Union): + _pack_ = archbits // 8 + _fields_ = ( + ('Status', ctypes.c_int32), + ('Pointer', pointer_type) ) -class IO_STACK_LOCATION_WRITE64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ('Length', ctypes.c_uint32), - ('_padding1', ctypes.c_uint32), - ('Key', ctypes.c_uint32), - ('Flags', ctypes.c_uint32), - ('ByteOffset', LARGE_INTEGER) - ) + class IO_STATUS_BLOCK(Struct): + _fields_ = ( + ('Status', IO_STATUS_BLOCK_DUMMY), + ('Information', pointer_type) + ) -class IO_STACK_LOCATION_WRITE32(ctypes.Structure): - _pack_ = 4 - _fields_ = ( - ('Length', ctypes.c_uint32), - ('Key', ctypes.c_uint32), - ('Flags', ctypes.c_uint32), - ('ByteOffset', LARGE_INTEGER) - ) + class IO_STACK_LOCATION_FILESYSTEMCONTROL(Struct): + _fields_ = ( + ('OutputBufferLength', native_type), # c_uint32 padded to native size + ('InputBufferLength', native_type), # c_uint32 padded to native size + ('FsControlCode', ctypes.c_uint32), + ('Type3InputBuffer', pointer_type) + ) -class IO_STACK_LOCATION_PARAM64(ctypes.Union): - _pack_ = 8 - _fields_ = (('FileSystemControl', IO_STACK_LOCATION_FILESYSTEMCONTROL64), - ('DeviceIoControl', IO_STACK_LOCATION_DEVICEIOCONTROL64), - ('Write', IO_STACK_LOCATION_WRITE64)) + class IO_STACK_LOCATION_DEVICEIOCONTROL(Struct): + _fields_ = ( + ('OutputBufferLength',native_type), # c_uint32 padded to native size + ('InputBufferLength', native_type), # c_uint32 padded to native size + ('IoControlCode', ctypes.c_uint32), + ('Type3InputBuffer', pointer_type) + ) + class IO_STACK_LOCATION_WRITE(Struct): + _fields_ = ( + ('Length', native_type), # c_uint32 padded to native size + ('Key', ctypes.c_uint32), + ('Flags', ctypes.c_uint32), + ('ByteOffset', LARGE_INTEGER) + ) -class IO_STACK_LOCATION_PARAM32(ctypes.Union): - _pack_ = 4 - _fields_ = (('FileSystemControl', IO_STACK_LOCATION_FILESYSTEMCONTROL32), - ('DeviceIoControl', IO_STACK_LOCATION_DEVICEIOCONTROL32), - ('Write', IO_STACK_LOCATION_WRITE32)) + class IO_STACK_LOCATION_PARAM(ctypes.Union): + _pack_ = archbits // 8 + _fields_ = ( + ('FileSystemControl', IO_STACK_LOCATION_FILESYSTEMCONTROL), + ('DeviceIoControl', IO_STACK_LOCATION_DEVICEIOCONTROL), + ('Write', IO_STACK_LOCATION_WRITE) + ) + class IO_STACK_LOCATION(Struct): + _fields_ = ( + ('MajorFunction', ctypes.c_byte), + ('MinorFunction', ctypes.c_byte), + ('Flags', ctypes.c_byte), + ('Control', ctypes.c_byte), + ('Parameters', IO_STACK_LOCATION_PARAM), + ('DeviceObject', pointer_type), + ('FileObject', pointer_type), + ('CompletionRoutine', pointer_type), + ('Context', pointer_type), + ) -class IO_STACK_LOCATION64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ('MajorFunction', ctypes.c_byte), - ('MinorFunction', ctypes.c_byte), - ('Flags', ctypes.c_byte), - ('Control', ctypes.c_byte), - ('_padding1', ctypes.c_byte * 0x4), - ('Parameters', IO_STACK_LOCATION_PARAM64), - ('DeviceObject', POINTER64), - ('FileObject', POINTER64), - ('CompletionRoutine', POINTER64), - ('Context', POINTER64), - ) + class AssociatedIrp(ctypes.Union): + _pack_ = archbits // 8 + _fields_ = ( + ('MasterIrp', pointer_type), + ('IrpCount', ctypes.c_uint32), + ('SystemBuffer', pointer_type) + ) + sz_factor = archbits // 32 -class IO_STACK_LOCATION32(ctypes.Structure): - _pack_ = 4 - _fields_ = ( - ('MajorFunction', ctypes.c_byte), - ('MinorFunction', ctypes.c_byte), - ('Flags', ctypes.c_byte), - ('Control', ctypes.c_byte), - ('Parameters', IO_STACK_LOCATION_PARAM32), - ('DeviceObject', POINTER32), - ('FileObject', POINTER32), - ('CompletionRoutine', POINTER32), - ('Context', POINTER32), - ) + class IRP(Struct): + _fields_ = ( + ('Type', ctypes.c_uint16), + ('Size', ctypes.c_uint16), + ('MdlAddress', pointer_type), + ('Flags', ctypes.c_uint32), + ('AssociatedIrp', AssociatedIrp), + ('ThreadListEntry', LIST_ENTRY), + ('IoStatus', IO_STATUS_BLOCK), + ('_padding1', ctypes.c_char * 8), + ('UserIosb', pointer_type), + ('UserEvent', pointer_type), + ('Overlay', ctypes.c_char * (8 * sz_factor)), + ('CancelRoutine', pointer_type), + ('UserBuffer', pointer_type), + ('_padding1', ctypes.c_char * (32 * sz_factor)), + ('irpstack', ctypes.POINTER(IO_STACK_LOCATION)), + ('_padding2', ctypes.c_char * (8 * sz_factor)) + ) - -# union { -# struct _IRP *MasterIrp; -# __volatile LONG IrpCount; -# PVOID SystemBuffer; -# } AssociatedIrp; - - -class AssociatedIrp64(ctypes.Union): - _fields_ = ( - ('MasterIrp', POINTER64), # ('MasterIrp', ctypes.POINTER(IRP64)), - ('IrpCount', ctypes.c_uint32), - ('SystemBuffer', POINTER64)) - - -class AssociatedIrp32(ctypes.Union): - _fields_ = ( - ('MasterIrp', POINTER32), # ('MasterIrp', ctypes.POINTER(IRP32)), - ('IrpCount', ctypes.c_uint32), - ('SystemBuffer', POINTER32)) - - -# struct _IRP { -# char _padding1[0x30]; -# IO_STATUS_BLOCK IoStatus; // distance is 0x30?? -# char _padding2[0x70 - 0x30 - sizeof(io_status_block)]; -# PVOID UserBuffer; // distance is 0x70 from _IRP -# char _padding3[0xB8 - 0x70 - sizeof(PVOID)]; -# IO_STACK_LOCATION *irpstack; -# }; -class IRP64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ('Type', ctypes.c_uint16), - ('Size', ctypes.c_uint16), - ('MdlAddress', POINTER64), - ('Flags', ctypes.c_uint32), - ('AssociatedIrp', AssociatedIrp64), - ('ThreadListEntry', LIST_ENTRY64), - ('IoStatus', IO_STATUS_BLOCK64), - ('_padding1', ctypes.c_char * 0x8), - ('UserIosb', POINTER64), - ('UserEvent', POINTER64), - ('Overlay', ctypes.c_char * 0x10), - ('CancelRoutine', POINTER64), - ('UserBuffer', POINTER64), - ('_padding1', ctypes.c_char * 0x40), - ('irpstack', ctypes.POINTER(IO_STACK_LOCATION64)), - ('_padding2', ctypes.c_char * 0x10), - ) - - -class IRP32(ctypes.Structure): - _fields_ = ( - ('Type', ctypes.c_uint16), - ('Size', ctypes.c_uint16), - ('MdlAddress', POINTER32), - ('Flags', ctypes.c_uint32), - ('AssociatedIrp', AssociatedIrp32), - ('ThreadListEntry', LIST_ENTRY32), - ('IoStatus', IO_STATUS_BLOCK32), - ('_padding1', ctypes.c_char * 0x8), - ('UserIosb', POINTER32), # 0x28 - ('UserEvent', POINTER32), - ('Overlay', ctypes.c_char * 8), - ('CancelRoutine', POINTER32), - ('UserBuffer', POINTER32), - ('_padding1', ctypes.c_char * 0x20), - ('irpstack', ctypes.POINTER(IO_STACK_LOCATION32)), - ('_padding2', ctypes.c_char * 8), - ) + return IRP() # typedef struct _MDL { @@ -1001,24 +560,26 @@ class IRP32(ctypes.Structure): # } MDL, *PMDL; -class MDL64(ctypes.Structure): - _pack_ = 8 - _fields_ = (('Next', POINTER64), ('Size', ctypes.c_uint16), - ('MdlFlags', ctypes.c_uint16), ('Process', POINTER64), - ('MappedSystemVa', POINTER64), ('StartVa', POINTER64), - ('ByteCount', ctypes.c_uint32), ('ByteOffset', - ctypes.c_uint32)) +def make_mdl(archbits: int): + pointer_type = __select_pointer_type(archbits) + Struct = __make_struct(archbits) + class MDL(Struct): + _fields_ = ( + ('Next', pointer_type), + ('Size', ctypes.c_uint16), + ('MdlFlags', ctypes.c_uint16), + ('Process', pointer_type), + ('MappedSystemVa', pointer_type), + ('StartVa', pointer_type), + ('ByteCount', ctypes.c_uint32), + ('ByteOffset', ctypes.c_uint32) + ) -class MDL32(ctypes.Structure): - _fields_ = (('Next', POINTER32), ('Size', ctypes.c_uint16), - ('MdlFlags', ctypes.c_uint16), ('Process', POINTER32), - ('MappedSystemVa', POINTER32), ('StartVa', POINTER32), - ('ByteCount', ctypes.c_uint32), ('ByteOffset', - ctypes.c_uint32)) - -#TODO: Repeated and might not be needed + return MDL() +# NOTE: the following classes are currently not needed +# # class DISPATCHER_HEADER64(ctypes.Structure): # _fields_ = ( # ('Lock', ctypes.c_int32), @@ -1029,8 +590,8 @@ class MDL32(ctypes.Structure): # ('SignalState', ctypes.c_int32), # ('WaitListHead', LIST_ENTRY64), # ) - - +# +# # class DISPATCHER_HEADER32(ctypes.Structure): # _fields_ = ( # ('Lock', ctypes.c_int32), @@ -1041,198 +602,198 @@ class MDL32(ctypes.Structure): # ('ThreadControlFlags', ctypes.c_uint8), # ('TimerMiscFlags', ctypes.c_uint8), # ) - - -class KAPC_STATE64(ctypes.Structure): - _fields_ = ( - ('ApcListHead', LIST_ENTRY64 * 2), - ('Process', POINTER64), - ('KernelApcInProgress', ctypes.c_uint8), - ('KernelApcPending', ctypes.c_uint8), - ('UserApcPending', ctypes.c_uint8), - ) - - -class KAPC_STATE32(ctypes.Structure): - _fields_ = ( - ('ApcListHead', LIST_ENTRY32 * 2), - ('Process', POINTER32), - ('KernelApcInProgress', ctypes.c_uint8), - ('KernelApcPending', ctypes.c_uint8), - ('UserApcPending', ctypes.c_uint8), - ) - - -class KTIMER64(ctypes.Structure): - _fields_ = ( - ('Header', DISPATCHER_HEADER64), - ('DueTime', LARGE_INTEGER), - ('TimerListEntry', LIST_ENTRY64), - ('Dpc', POINTER64), - ('Period', ctypes.c_uint32), - ) - - -class KTIMER32(ctypes.Structure): - _fields_ = ( - ('Header', DISPATCHER_HEADER32), - ('DueTime', LARGE_INTEGER), - ('TimerListEntry', LIST_ENTRY32), - ('Dpc', POINTER32), - ('Period', ctypes.c_uint32), - ) - - -class KWAIT_BLOCK64(ctypes.Structure): - _fields_ = ( - ('WaitListEntry', LIST_ENTRY64), - ('Thread', POINTER64), - ('Object', POINTER64), - ('NextWaitBlock', POINTER64), - ('WaitKey', ctypes.c_uint16), - ('WaitType', ctypes.c_uint8), - ('BlockState', ctypes.c_uint8), - ) - - -class KWAIT_BLOCK32(ctypes.Structure): - _fields_ = ( - ('WaitListEntry', LIST_ENTRY32), - ('Thread', POINTER32), - ('Object', POINTER32), - ('NextWaitBlock', POINTER32), - ('WaitKey', ctypes.c_uint16), - ('WaitType', ctypes.c_uint8), - ('BlockState', ctypes.c_uint8), - ) - - -class GROUP_AFFINITY64(ctypes.Structure): - _fields_ = (('Mask', ctypes.c_uint64), ('Group', ctypes.c_uint16), - ('Reserved', ctypes.c_uint16 * 3)) - - -class GROUP_AFFINITY32(ctypes.Structure): - _fields_ = (('Mask', ctypes.c_uint32), ('Group', ctypes.c_uint16), - ('Reserved', ctypes.c_uint16 * 3)) - - -class KAPC64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ('Type', ctypes.c_uint8), - ('SpareByte0', ctypes.c_uint8), - ('Size', ctypes.c_uint8), - ('SpareByte1', ctypes.c_uint8), - ('SpareLong0', ctypes.c_uint32), - ('Thread', POINTER64), - ('ApcListEntry', LIST_ENTRY64), - ('KernelRoutine', POINTER64), - ('RundownRoutine', POINTER64), - ('NormalRoutine', POINTER64), - ('NormalContext', POINTER64), - ('SystemArgument1', POINTER64), - ('SystemArgument2', POINTER64), - ('ApcStateIndex', ctypes.c_uint8), - ('ApcMode', ctypes.c_uint8), - ('Inserted', ctypes.c_uint8), - ) - - -class KAPC32(ctypes.Structure): - _fields_ = () - - -class KSEMAPHORE64(ctypes.Structure): - _pack_ = 8 - _fields_ = (("Header", DISPATCHER_HEADER64), ("Limit", ctypes.c_int32)) - - -class COUNTER_READING64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ("Type", ctypes.c_uint32), - ("Index", ctypes.c_uint32), - ("Start", ctypes.c_uint64), - ("Total", ctypes.c_uint64), - ) - - -class KTHREAD_COUNTERS64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ("WaitReasonBitMap", ctypes.c_int64), - ("UserData", POINTER64), - ("Flags", ctypes.c_uint32), - ("ContextSwitches", ctypes.c_uint32), - ("CycleTimeBias", ctypes.c_uint64), - ("HardwareCounters", ctypes.c_uint64), - ("HwCounter", COUNTER_READING64 * 16), - ) - - -class KTHREAD64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ('Header', DISPATCHER_HEADER64), - ('CycleTime', ctypes.c_uint64), - ('QuantumTarget', ctypes.c_uint64), - ('InitialStack', POINTER64), - ('StackLimit', POINTER64), - ('KernelStack', POINTER64), - ('ThreadLock', ctypes.c_uint64), - ('WaitRegister', ctypes.c_uint8), # _KWAIT_STATUS_REGISTER - ('Running', ctypes.c_uint8), - ('Alerted', ctypes.c_uint8 * 2), - ('MiscFlags', ctypes.c_uint32), - ('ApcState', KAPC_STATE64), - ('DeferredProcessor', ctypes.c_uint32), - ('ApcQueueLock', ctypes.c_uint64), - ('WaitStatus', ctypes.c_int64), - ('WaitBlockList', POINTER64), - ('WaitListEntry', LIST_ENTRY64), - ('Queue', POINTER64), - ('Teb', POINTER64), - ('Timer', KTIMER64), - ('ThreadFlags', ctypes.c_int32), - ('Spare0', ctypes.c_uint32), - ('WaitBlock', KWAIT_BLOCK64 * 4), - ('QueueListEntry', LIST_ENTRY64), - ('TrapFrame', POINTER64), - ('FirstArgument', POINTER64), - ('CallbackStack', POINTER64), - ('ApcStateIndex', ctypes.c_uint8), - ('BasePriority', ctypes.c_char), - ('PriorityDecrement', ctypes.c_char), - ('Preempted', ctypes.c_uint8), - ('AdjustReason', ctypes.c_uint8), - ('AdjustIncrement', ctypes.c_char), - ('PreviousMode', ctypes.c_char), - ('Saturation', ctypes.c_char), - ('SystemCallNumber', ctypes.c_uint32), - ('FreezeCount', ctypes.c_uint32), - ('UserAffinity', GROUP_AFFINITY64), - ('Process', POINTER64), - ('Affinity', GROUP_AFFINITY64), - ('IdealProcessor', ctypes.c_uint32), - ('UserIdealProcessor', ctypes.c_uint32), - ('ApcStatePointer', POINTER64 * 2), - ('SavedApcState', KAPC_STATE64), - ('Win32Thread', POINTER64), - ('StackBase', POINTER64), - ('SuspendApc', KAPC64), - ('SuspendSemaphore', KSEMAPHORE64), - ('ThreadListEntry', LIST_ENTRY64), - ('MutantListHead', LIST_ENTRY64), - ('SListFaultAddress', POINTER64), - ('ReadOperationCount', ctypes.c_int64), - ('WriteOperationCount', ctypes.c_int64), - ('OtherOperationCount', ctypes.c_int64), - ('ReadTransferCount', ctypes.c_int64), - ('WriteTransferCount', ctypes.c_int64), - ('OtherTransferCount', ctypes.c_int64), - ('ThreadCounters', POINTER64), - ('XStateSave', POINTER64)) +# +# +# class KAPC_STATE64(ctypes.Structure): +# _fields_ = ( +# ('ApcListHead', LIST_ENTRY64 * 2), +# ('Process', POINTER64), +# ('KernelApcInProgress', ctypes.c_uint8), +# ('KernelApcPending', ctypes.c_uint8), +# ('UserApcPending', ctypes.c_uint8), +# ) +# +# +# class KAPC_STATE32(ctypes.Structure): +# _fields_ = ( +# ('ApcListHead', LIST_ENTRY32 * 2), +# ('Process', POINTER32), +# ('KernelApcInProgress', ctypes.c_uint8), +# ('KernelApcPending', ctypes.c_uint8), +# ('UserApcPending', ctypes.c_uint8), +# ) +# +# +# class KTIMER64(ctypes.Structure): +# _fields_ = ( +# ('Header', DISPATCHER_HEADER64), +# ('DueTime', LARGE_INTEGER), +# ('TimerListEntry', LIST_ENTRY64), +# ('Dpc', POINTER64), +# ('Period', ctypes.c_uint32), +# ) +# +# +# class KTIMER32(ctypes.Structure): +# _fields_ = ( +# ('Header', DISPATCHER_HEADER32), +# ('DueTime', LARGE_INTEGER), +# ('TimerListEntry', LIST_ENTRY32), +# ('Dpc', POINTER32), +# ('Period', ctypes.c_uint32), +# ) +# +# +# class KWAIT_BLOCK64(ctypes.Structure): +# _fields_ = ( +# ('WaitListEntry', LIST_ENTRY64), +# ('Thread', POINTER64), +# ('Object', POINTER64), +# ('NextWaitBlock', POINTER64), +# ('WaitKey', ctypes.c_uint16), +# ('WaitType', ctypes.c_uint8), +# ('BlockState', ctypes.c_uint8), +# ) +# +# +# class KWAIT_BLOCK32(ctypes.Structure): +# _fields_ = ( +# ('WaitListEntry', LIST_ENTRY32), +# ('Thread', POINTER32), +# ('Object', POINTER32), +# ('NextWaitBlock', POINTER32), +# ('WaitKey', ctypes.c_uint16), +# ('WaitType', ctypes.c_uint8), +# ('BlockState', ctypes.c_uint8), +# ) +# +# +# class GROUP_AFFINITY64(ctypes.Structure): +# _fields_ = (('Mask', ctypes.c_uint64), ('Group', ctypes.c_uint16), +# ('Reserved', ctypes.c_uint16 * 3)) +# +# +# class GROUP_AFFINITY32(ctypes.Structure): +# _fields_ = (('Mask', ctypes.c_uint32), ('Group', ctypes.c_uint16), +# ('Reserved', ctypes.c_uint16 * 3)) +# +# +# class KAPC64(ctypes.Structure): +# _pack_ = 8 +# _fields_ = ( +# ('Type', ctypes.c_uint8), +# ('SpareByte0', ctypes.c_uint8), +# ('Size', ctypes.c_uint8), +# ('SpareByte1', ctypes.c_uint8), +# ('SpareLong0', ctypes.c_uint32), +# ('Thread', POINTER64), +# ('ApcListEntry', LIST_ENTRY64), +# ('KernelRoutine', POINTER64), +# ('RundownRoutine', POINTER64), +# ('NormalRoutine', POINTER64), +# ('NormalContext', POINTER64), +# ('SystemArgument1', POINTER64), +# ('SystemArgument2', POINTER64), +# ('ApcStateIndex', ctypes.c_uint8), +# ('ApcMode', ctypes.c_uint8), +# ('Inserted', ctypes.c_uint8), +# ) +# +# +# class KAPC32(ctypes.Structure): +# _fields_ = () +# +# +# class KSEMAPHORE64(ctypes.Structure): +# _pack_ = 8 +# _fields_ = (("Header", DISPATCHER_HEADER64), ("Limit", ctypes.c_int32)) +# +# +# class COUNTER_READING64(ctypes.Structure): +# _pack_ = 8 +# _fields_ = ( +# ("Type", ctypes.c_uint32), +# ("Index", ctypes.c_uint32), +# ("Start", ctypes.c_uint64), +# ("Total", ctypes.c_uint64), +# ) +# +# +# class KTHREAD_COUNTERS64(ctypes.Structure): +# _pack_ = 8 +# _fields_ = ( +# ("WaitReasonBitMap", ctypes.c_int64), +# ("UserData", POINTER64), +# ("Flags", ctypes.c_uint32), +# ("ContextSwitches", ctypes.c_uint32), +# ("CycleTimeBias", ctypes.c_uint64), +# ("HardwareCounters", ctypes.c_uint64), +# ("HwCounter", COUNTER_READING64 * 16), +# ) +# +# +# class KTHREAD64(ctypes.Structure): +# _pack_ = 8 +# _fields_ = ( +# ('Header', DISPATCHER_HEADER64), +# ('CycleTime', ctypes.c_uint64), +# ('QuantumTarget', ctypes.c_uint64), +# ('InitialStack', POINTER64), +# ('StackLimit', POINTER64), +# ('KernelStack', POINTER64), +# ('ThreadLock', ctypes.c_uint64), +# ('WaitRegister', ctypes.c_uint8), # _KWAIT_STATUS_REGISTER +# ('Running', ctypes.c_uint8), +# ('Alerted', ctypes.c_uint8 * 2), +# ('MiscFlags', ctypes.c_uint32), +# ('ApcState', KAPC_STATE64), +# ('DeferredProcessor', ctypes.c_uint32), +# ('ApcQueueLock', ctypes.c_uint64), +# ('WaitStatus', ctypes.c_int64), +# ('WaitBlockList', POINTER64), +# ('WaitListEntry', LIST_ENTRY64), +# ('Queue', POINTER64), +# ('Teb', POINTER64), +# ('Timer', KTIMER64), +# ('ThreadFlags', ctypes.c_int32), +# ('Spare0', ctypes.c_uint32), +# ('WaitBlock', KWAIT_BLOCK64 * 4), +# ('QueueListEntry', LIST_ENTRY64), +# ('TrapFrame', POINTER64), +# ('FirstArgument', POINTER64), +# ('CallbackStack', POINTER64), +# ('ApcStateIndex', ctypes.c_uint8), +# ('BasePriority', ctypes.c_char), +# ('PriorityDecrement', ctypes.c_char), +# ('Preempted', ctypes.c_uint8), +# ('AdjustReason', ctypes.c_uint8), +# ('AdjustIncrement', ctypes.c_char), +# ('PreviousMode', ctypes.c_char), +# ('Saturation', ctypes.c_char), +# ('SystemCallNumber', ctypes.c_uint32), +# ('FreezeCount', ctypes.c_uint32), +# ('UserAffinity', GROUP_AFFINITY64), +# ('Process', POINTER64), +# ('Affinity', GROUP_AFFINITY64), +# ('IdealProcessor', ctypes.c_uint32), +# ('UserIdealProcessor', ctypes.c_uint32), +# ('ApcStatePointer', POINTER64 * 2), +# ('SavedApcState', KAPC_STATE64), +# ('Win32Thread', POINTER64), +# ('StackBase', POINTER64), +# ('SuspendApc', KAPC64), +# ('SuspendSemaphore', KSEMAPHORE64), +# ('ThreadListEntry', LIST_ENTRY64), +# ('MutantListHead', LIST_ENTRY64), +# ('SListFaultAddress', POINTER64), +# ('ReadOperationCount', ctypes.c_int64), +# ('WriteOperationCount', ctypes.c_int64), +# ('OtherOperationCount', ctypes.c_int64), +# ('ReadTransferCount', ctypes.c_int64), +# ('WriteTransferCount', ctypes.c_int64), +# ('OtherTransferCount', ctypes.c_int64), +# ('ThreadCounters', POINTER64), +# ('XStateSave', POINTER64)) # struct _RTL_PROCESS_MODULE_INFORMATION { @@ -1247,36 +808,25 @@ class KTHREAD64(ctypes.Structure): # USHORT OffsetToFileName; # UCHAR FullPathName[256]; # } RTL_PROCESS_MODULE_INFORMATION, -class RTL_PROCESS_MODULE_INFORMATION64(ctypes.Structure): - _pack_ = 8 - _fields_ = ( - ('Section', ctypes.c_uint64), - ('MappedBase', ctypes.c_uint64), - ('ImageBase', ctypes.c_uint64), - ('ImageSize', ctypes.c_uint32), - ('Flags', ctypes.c_uint32), - ('LoadOrderIndex', ctypes.c_uint16), - ('InitOrderIndex', ctypes.c_uint16), - ('LoadCount', ctypes.c_uint16), - ('OffsetToFileName', ctypes.c_uint16), - ('FullPathName', ctypes.c_char * 256) - ) +def make_rtl_process_module_info(archbits: int): + native_type = __select_native_type(archbits) + Struct = __make_struct(archbits) + class RTL_PROCESS_MODULE_INFORMATION(Struct): + _fields_ = ( + ('Section', native_type), + ('MappedBase', native_type), + ('ImageBase', native_type), + ('ImageSize', ctypes.c_uint32), + ('Flags', ctypes.c_uint32), + ('LoadOrderIndex', ctypes.c_uint16), + ('InitOrderIndex', ctypes.c_uint16), + ('LoadCount', ctypes.c_uint16), + ('OffsetToFileName', ctypes.c_uint16), + ('FullPathName', ctypes.c_char * 256) + ) -class RTL_PROCESS_MODULE_INFORMATION32(ctypes.Structure): - _fields_ = ( - ('Section', ctypes.c_uint32), - ('MappedBase', ctypes.c_uint32), - ('ImageBase', ctypes.c_uint32), - ('ImageSize', ctypes.c_uint32), - ('Flags', ctypes.c_uint32), - ('LoadOrderIndex', ctypes.c_uint16), - ('InitOrderIndex', ctypes.c_uint16), - ('LoadCount', ctypes.c_uint16), - ('OffsetToFileName', ctypes.c_uint16), - ('FullPathName', ctypes.c_char * 256) - ) - + return RTL_PROCESS_MODULE_INFORMATION() # struct _EPROCESS { # struct _KPROCESS Pcb; //0x0 @@ -1427,205 +977,99 @@ class RTL_PROCESS_MODULE_INFORMATION32(ctypes.Structure): # ULONG SmallestTimerResolution; //0x4c0 # struct _PO_DIAG_STACK_RECORD* TimerResolutionStackRecord; //0x4c8 # }; -class EPROCESS64(ctypes.Structure): - _pack_ = 8 - _fields_ = (('dummy', ctypes.c_char * 0x4d0), ) - def __init__(self, ql, base): - self.ql = ql - self.base = base +def make_eprocess(archbits: int): + Struct = __make_struct(archbits) + + class EPROCESS(Struct): + _fields_ = ( + ('dummy', ctypes.c_uint8), + ) + + obj_size = { + 32: 0x2c0, + 64: 0x4d0 + }[archbits] + + obj = EPROCESS() + ctypes.resize(obj, obj_size) + + return obj -class EPROCESS32(ctypes.Structure): - _fields_ = (('dummy', ctypes.c_char * 0x2c0), ) +def make_ldr_data(archbits: int): + native_type = __select_native_type(archbits) + Struct = __make_struct(archbits) - def __init__(self, ql, base): - self.ql = ql - self.base = base + ListEntry = make_list_entry(archbits) + + class PEB_LDR_DATA(Struct): + _fields_ = ( + ('Length', ctypes.c_uint32), + ('Initialized', ctypes.c_uint32), + ('SsHandle', native_type), + ('InLoadOrderModuleList', ListEntry), + ('InMemoryOrderModuleList', ListEntry), + ('InInitializationOrderModuleList', ListEntry), + ('EntryInProgress', native_type), + ('ShutdownInProgress', native_type), + ('selfShutdownThreadId', native_type) + ) + + return PEB_LDR_DATA() -# FIXME: duplicate class -class LdrData: - def __init__(self, - ql, - base=0, - length=0, - initialized=0, - ss_handle=0, - in_load_order_module_list={ - 'Flink': 0, - 'Blink': 0 - }, - in_memory_order_module_list={ - 'Flink': 0, - 'Blink': 0 - }, - in_initialization_order_module_list={ - 'Flink': 0, - 'Blink': 0 - }, - entry_in_progress=0, - shutdown_in_progress=0, - shutdown_thread_id=0): - self.ql = ql - self.base = base - self.Length = length - self.Initialized = initialized - self.SsHandle = ss_handle - self.InLoadOrderModuleList = in_load_order_module_list - self.InMemoryOrderModuleList = in_memory_order_module_list - self.InInitializationOrderModuleList = in_initialization_order_module_list - self.EntryInProgress = entry_in_progress - self.ShutdownInProgress = shutdown_in_progress - self.selfShutdownThreadId = shutdown_thread_id +def make_ldr_data_table_entry(archbits: int): + pointer_type = __select_pointer_type(archbits) + native_type = __select_native_type(archbits) + Struct = __make_struct(archbits) - def bytes(self): - s = b'' - s += self.ql.pack32(self.Length) # 0x0 - s += self.ql.pack32(self.Initialized) # 0x4 - s += self.ql.pack(self.SsHandle) # 0x8 - s += self.ql.pack(self.InLoadOrderModuleList['Flink']) # 0x0c - s += self.ql.pack(self.InLoadOrderModuleList['Blink']) - s += self.ql.pack(self.InMemoryOrderModuleList['Flink']) # 0x14 - s += self.ql.pack(self.InMemoryOrderModuleList['Blink']) - s += self.ql.pack( - self.InInitializationOrderModuleList['Flink']) # 0x1C - s += self.ql.pack(self.InInitializationOrderModuleList['Blink']) - s += self.ql.pack(self.EntryInProgress) - s += self.ql.pack(self.ShutdownInProgress) - s += self.ql.pack(self.selfShutdownThreadId) - return s + ListEntry = make_list_entry(archbits) + UniString = make_unicode_string(archbits).__class__ + class RTL_BALANCED_NODE(Struct): + _fields_ = ( + ('Left', pointer_type), + ('Right', pointer_type), + ) -class LdrDataTableEntry: - def __init__(self, - ql, - base=0, - in_load_order_links={ - 'Flink': 0, - 'Blink': 0 - }, - in_memory_order_links={ - 'Flink': 0, - 'Blink': 0 - }, - in_initialization_order_links={ - 'Flink': 0, - 'Blink': 0 - }, - dll_base=0, - entry_point=0, - size_of_image=0, - full_dll_name='', - base_dll_name='', - flags=0, - load_count=0, - tls_index=0, - hash_links=0, - section_pointer=0, - check_sum=0, - time_date_stamp=0, - loaded_imports=0, - entry_point_activation_context=0, - patch_information=0, - forwarder_links=0, - service_tag_links=0, - static_links=0, - context_information=0, - original_base=0, - load_time=0): - self.ql = ql - self.base = base - self.InLoadOrderLinks = in_load_order_links - self.InMemoryOrderLinks = in_memory_order_links - self.InInitializationOrderLinks = in_initialization_order_links - self.DllBase = dll_base - self.EntryPoint = entry_point - self.SizeOfImage = size_of_image + class LdrDataTableEntry(Struct): + _fields_ = ( + ('InLoadOrderLinks', ListEntry), + ('InMemoryOrderLinks', ListEntry), + ('InInitializationOrderLinks', ListEntry), + ('DllBase', native_type), + ('EntryPoint', native_type), + ('SizeOfImage', native_type), + ('FullDllName', UniString), + ('BaseDllName', UniString), + ('Flags', native_type), + ('ObsoleteLoadCount', ctypes.c_uint16), + ('TlsIndex', ctypes.c_uint16), + ('HashLinks', ListEntry), + ('TimedateStamp', native_type), + ('EntryPointActivationContext', native_type), + ('Lock', native_type), + ('DdagNode', pointer_type), + ('NodeModuleLink', ListEntry), + ('LoadContext', native_type), + ('ParentDllBase', native_type), + ('SwitchBackContext', native_type), + ('BaseAddressIndexNode', RTL_BALANCED_NODE), + ('MappingInfoIndexNode', RTL_BALANCED_NODE), + ('OriginalBase', native_type), + ('LoadTime', LARGE_INTEGER), + ('BaseNameHashValue', native_type), + ('LoadReason', ctypes.c_uint32), + ('ImplicitPathOptions', native_type), + ('ReferenceCount', native_type), + # 1607+ + ('DependentLoadFlags', native_type), + # 1703+ + ('SigningLevel', ctypes.c_uint8) + ) - full_dll_name = full_dll_name.encode("utf-16le") - self.FullDllName = { - 'Length': len(full_dll_name), - 'MaximumLength': len(full_dll_name) + 2 - } - self.FullDllName['BufferPtr'] = self.ql.os.heap.alloc( - self.FullDllName['MaximumLength']) - ql.mem.write(self.FullDllName['BufferPtr'], - full_dll_name + b"\x00\x00") - - base_dll_name = base_dll_name.encode("utf-16le") - self.BaseDllName = { - 'Length': len(base_dll_name), - 'MaximumLength': len(base_dll_name) + 2 - } - self.BaseDllName['BufferPtr'] = self.ql.os.heap.alloc( - self.BaseDllName['MaximumLength']) - ql.mem.write(self.BaseDllName['BufferPtr'], - base_dll_name + b"\x00\x00") - - self.Flags = flags - self.LoadCount = load_count - self.TlsIndex = tls_index - self.HashLinks = hash_links - self.SectionPointer = section_pointer - self.CheckSum = check_sum - self.TimeDateStamp = time_date_stamp - self.LoadedImports = loaded_imports - self.EntryPointActivationContext = entry_point_activation_context - self.PatchInformation = patch_information - self.ForwarderLinks = forwarder_links - self.ServiceTagLinks = service_tag_links - self.StaticLinks = static_links - self.ContextInformation = context_information - self.OriginalBase = original_base - self.LoadTime = load_time - - def attrs(self): - return ", ".join("{}={}".format(k, getattr(self, k)) - for k in self.__dict__.keys()) - - def print(self): - return "[{}:{}]".format(self.__class__.__name__, self.attrs()) - - def bytes(self): - s = b'' - s += self.ql.pack(self.InLoadOrderLinks['Flink']) # 0x0 - s += self.ql.pack(self.InLoadOrderLinks['Blink']) - s += self.ql.pack(self.InMemoryOrderLinks['Flink']) # 0x8 - s += self.ql.pack(self.InMemoryOrderLinks['Blink']) - s += self.ql.pack(self.InInitializationOrderLinks['Flink']) # 0x10 - s += self.ql.pack(self.InInitializationOrderLinks['Blink']) - s += self.ql.pack(self.DllBase) # 0x18 - s += self.ql.pack(self.EntryPoint) # 0x1c - s += self.ql.pack(self.SizeOfImage) # 0x20 - s += self.ql.pack16(self.FullDllName['Length']) # 0x24 - s += self.ql.pack16(self.FullDllName['MaximumLength']) # 0x26 - if self.ql.arch.type == QL_ARCH.X8664: - s += self.ql.pack32(0) - s += self.ql.pack(self.FullDllName['BufferPtr']) # 0x28 - s += self.ql.pack16(self.BaseDllName['Length']) - s += self.ql.pack16(self.BaseDllName['MaximumLength']) - if self.ql.arch.type == QL_ARCH.X8664: - s += self.ql.pack32(0) - s += self.ql.pack(self.BaseDllName['BufferPtr']) - s += self.ql.pack(self.Flags) - s += self.ql.pack(self.LoadCount) - s += self.ql.pack(self.TlsIndex) - s += self.ql.pack(self.HashLinks) - s += self.ql.pack(self.SectionPointer) - s += self.ql.pack(self.CheckSum) - s += self.ql.pack(self.TimeDateStamp) - s += self.ql.pack(self.LoadedImports) - s += self.ql.pack(self.EntryPointActivationContext) - s += self.ql.pack(self.PatchInformation) - s += self.ql.pack(self.ForwarderLinks) - s += self.ql.pack(self.ServiceTagLinks) - s += self.ql.pack(self.StaticLinks) - s += self.ql.pack(self.ContextInformation) - s += self.ql.pack(self.OriginalBase) - s += self.ql.pack(self.LoadTime) - - return s + return LdrDataTableEntry() class WindowsStruct: @@ -1855,43 +1299,44 @@ class Sid(WindowsStruct): class Mutex: - def __init__(self, name, type): + def __init__(self, name: str, type: str): self.name = name self.locked = False self.type = type - def lock(self): + def lock(self) -> None: self.locked = True - def unlock(self): + def unlock(self) -> None: self.locked = False - def isFree(self): + def isFree(self) -> bool: return not self.locked -class IMAGE_IMPORT_DESCRIPTOR(ctypes.Structure): - _fields_ = ( - ('OriginalFirstThunk', ctypes.c_uint32), - ('TimeDateStamp', ctypes.c_uint32), - ('ForwarderChain', ctypes.c_uint32), - ('Name', ctypes.c_uint32), - ('FirstThunk', ctypes.c_uint32) - ) +# class IMAGE_IMPORT_DESCRIPTOR(ctypes.Structure): +# _fields_ = ( +# ('OriginalFirstThunk', ctypes.c_uint32), +# ('TimeDateStamp', ctypes.c_uint32), +# ('ForwarderChain', ctypes.c_uint32), +# ('Name', ctypes.c_uint32), +# ('FirstThunk', ctypes.c_uint32) +# ) +# +# +# class CLIENT_ID32(ctypes.Structure): +# _fields_ = ( +# ('UniqueProcess', ctypes.c_uint32), +# ('UniqueThread', ctypes.c_uint32) +# ) +# +# +# class CLIENT_ID64(ctypes.Structure): +# _fields_ = ( +# ('UniqueProcess', ctypes.c_uint64), +# ('UniqueThread', ctypes.c_uint64) +# ) - -class CLIENT_ID32(ctypes.Structure): - _fields_ = ( - ('UniqueProcess', ctypes.c_uint32), - ('UniqueThread', ctypes.c_uint32) - ) - - -class CLIENT_ID64(ctypes.Structure): - _fields_ = ( - ('UniqueProcess', ctypes.c_uint64), - ('UniqueThread', ctypes.c_uint64) - ) # typedef struct tagPOINT { # LONG x; # LONG y; diff --git a/qiling/os/windows/thread.py b/qiling/os/windows/thread.py index cbbb6651..ca36a477 100644 --- a/qiling/os/windows/thread.py +++ b/qiling/os/windows/thread.py @@ -3,62 +3,14 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from unicorn.x86_const import * +from typing import TYPE_CHECKING, cast -from qiling.exception import * -from qiling.os.thread import * -from .utils import * - - -def thread_scheduler(ql, address, size): - if ql.arch.regs.arch_pc == ql.os.thread_manager.THREAD_RET_ADDR: - ql.os.thread_manager.cur_thread.stop() - ql.os.thread_manager.do_schedule() - else: - ql.os.thread_manager.ins_count += 1 - ql.os.thread_manager.do_schedule() - - -# Simple Thread Manager -class QlWindowsThreadManagement(QlThread): - TIME_SLICE = 10 - - def __init__(self, ql, cur_thread): - super(QlWindowsThreadManagement, self).__init__(ql) - self.ql = ql - # main thread - self.cur_thread = cur_thread - self.threads = [self.cur_thread] - self.ins_count = 0 - self.THREAD_RET_ADDR = self.ql.os.heap.alloc(8) - # write nop to THREAD_RET_ADDR - self.ql.mem.write(self.THREAD_RET_ADDR, b"\x90"*8) - self.ql.hook_code(thread_scheduler) - - def append(self, thread): - self.threads.append(thread) - - def need_schedule(self): - return self.cur_thread.is_stop() or self.ins_count % QlWindowsThreadManagement.TIME_SLICE == 0 - - def do_schedule(self): - if self.cur_thread.is_stop() or self.ins_count % QlWindowsThreadManagement.TIME_SLICE == 0: - if len(self.threads) <= 1: - return - else: - for i in range(1, len(self.threads)): - next_id = (self.cur_thread.id + i) % len(self.threads) - next_thread = self.threads[next_id] - # find next thread - if next_thread.status == QlWindowsThread.RUNNING and (not next_thread.has_waitfor()): - if self.cur_thread.is_stop(): - pass - else: - self.cur_thread.suspend() - next_thread.resume() - self.cur_thread = next_thread - break +from qiling import Qiling +from qiling.const import QL_ARCH +from qiling.os.thread import QlThread +if TYPE_CHECKING: + from qiling.os.windows.windows import QlOsWindows class QlWindowsThread(QlThread): # static var @@ -67,8 +19,9 @@ class QlWindowsThread(QlThread): RUNNING = 1 TERMINATED = 2 - def __init__(self, ql, status=1, isFake=False): - super(QlWindowsThread, self).__init__(ql) + def __init__(self, ql: Qiling, status: int = 1, isFake: bool = False): + super().__init__(ql) + self.ql = ql self.id = QlWindowsThread.ID QlWindowsThread.ID += 1 @@ -79,55 +32,107 @@ class QlWindowsThread(QlThread): self.fake = isFake # create new thread - def create(self, func_addr, func_params, status): + def create(self, func_addr: int, func_params: int, status: int) -> int: + os = cast('QlOsWindows', self.ql.os) + # create new stack stack_size = 1024 - new_stack = self.ql.os.heap.alloc(stack_size) + stack_size - - self.saved_context = self.ql.arch.regs.save() + new_stack = os.heap.alloc(stack_size) + stack_size - # set return address, parameters + asize = self.ql.arch.pointersize + context = self.ql.arch.regs.save() + + # set return address + self.ql.mem.write_ptr(new_stack - asize, os.thread_manager.thread_ret_addr) + + # set parameters if self.ql.arch.type == QL_ARCH.X86: - self.ql.mem.write_ptr(new_stack - 4, self.ql.os.thread_manager.THREAD_RET_ADDR) self.ql.mem.write_ptr(new_stack, func_params) elif self.ql.arch.type == QL_ARCH.X8664: - self.ql.mem.write_ptr(new_stack - 8, self.ql.os.thread_manager.THREAD_RET_ADDR) - self.saved_context["rcx"] = func_params + context["rcx"] = func_params # set eip/rip, ebp/rbp, esp/rsp if self.ql.arch.type == QL_ARCH.X86: - self.saved_context["eip"] = func_addr - self.saved_context["ebp"] = new_stack - 4 - self.saved_context["esp"] = new_stack - 4 + context["eip"] = func_addr + context["ebp"] = new_stack - asize + context["esp"] = new_stack - asize + elif self.ql.arch.type == QL_ARCH.X8664: - self.saved_context["rip"] = func_addr - self.saved_context["rbp"] = new_stack - 8 - self.saved_context["rsp"] = new_stack - 8 + context["rip"] = func_addr + context["rbp"] = new_stack - asize + context["rsp"] = new_stack - asize + self.saved_context = context self.status = status - return self.id - self.status = status return self.id - def suspend(self): + def suspend(self) -> None: self.saved_context = self.ql.arch.regs.save() - def resume(self): + def resume(self) -> None: self.ql.arch.regs.restore(self.saved_context) self.status = QlWindowsThread.RUNNING - def stop(self): + def stop(self) -> None: self.status = QlWindowsThread.TERMINATED - def is_stop(self): + def is_stop(self) -> bool: return self.status == QlWindowsThread.TERMINATED - def waitfor(self, thread): + def waitfor(self, thread: 'QlWindowsThread') -> None: self.waitforthreads.append(thread) - def has_waitfor(self): - for each_thread in self.waitforthreads: - if not each_thread.is_stop(): - return True - return False + def has_waitfor(self) -> bool: + return any(not thread.is_stop() for thread in self.waitforthreads) + + +# Simple Thread Manager +class QlWindowsThreadManagement(QlThread): + TIME_SLICE = 10 + + def __init__(self, ql: Qiling, os: 'QlOsWindows', cur_thread: QlWindowsThread): + super().__init__(ql) + + self.ql = ql + # main thread + self.cur_thread = cur_thread + self.threads = [self.cur_thread] + self.icount = 0 + self.thread_ret_addr = os.heap.alloc(8) + + # write nop to thread_ret_addr + ql.mem.write(self.thread_ret_addr, b'\x90' * 8) + + def __thread_scheduler(ql: Qiling, address: int, size: int): + if ql.arch.regs.arch_pc == self.thread_ret_addr: + self.cur_thread.stop() + else: + self.icount += 1 + + self.do_schedule() + + ql.hook_code(__thread_scheduler) + + def append(self, thread: QlWindowsThread): + self.threads.append(thread) + + def need_schedule(self): + return self.cur_thread.is_stop() or (self.icount % QlWindowsThreadManagement.TIME_SLICE) == 0 + + def do_schedule(self) -> None: + if self.need_schedule(): + # if there is less than one thread, this loop won't run + for i in range(1, len(self.threads)): + next_id = (self.cur_thread.id + i) % len(self.threads) + next_thread = self.threads[next_id] + + # find next thread + if next_thread.status == QlWindowsThread.RUNNING and not next_thread.has_waitfor(): + if not self.cur_thread.is_stop(): + self.cur_thread.suspend() + + next_thread.resume() + self.cur_thread = next_thread + + break diff --git a/qiling/os/windows/utils.py b/qiling/os/windows/utils.py index 6ec9f6c7..bd13a763 100644 --- a/qiling/os/windows/utils.py +++ b/qiling/os/windows/utils.py @@ -4,8 +4,7 @@ # import ctypes -import ntpath -from typing import Tuple, TypeVar +from typing import Iterable, Tuple, TypeVar from unicorn import UcError @@ -24,39 +23,26 @@ def cmp(a: Comparable, b: Comparable) -> int: return (a > b) - (a < b) -def is_file_library(string: str) -> bool: - string = string.lower() - extension = string.rpartition('.')[-1] - return extension in ("dll", "exe", "sys", "drv") +def has_lib_ext(name: str) -> bool: + ext = name.lower().rpartition('.')[-1] - -def path_leaf(path): - head, tail = ntpath.split(path) - return tail or ntpath.basename(head) + return ext in ("dll", "exe", "sys", "drv") def io_Write(ql: Qiling, in_buffer: bytes): heap = ql.os.heap - if ql.ostype == QL_OS.WINDOWS: + if ql.loader.driver_object.MajorFunction[IRP_MJ_WRITE] == 0: + # raise error? + return (False, None) - if ql.loader.driver_object.MajorFunction[IRP_MJ_WRITE] == 0: - # raise error? - return (False, None) - - if ql.arch.bits == 32: - buf = ql.mem.read(ql.loader.driver_object.DeviceObject, ctypes.sizeof(DEVICE_OBJECT32)) - device_object = DEVICE_OBJECT32.from_buffer(buf) - else: - buf = ql.mem.read(ql.loader.driver_object.DeviceObject, ctypes.sizeof(DEVICE_OBJECT64)) - device_object = DEVICE_OBJECT64.from_buffer(buf) + driver_object_cls = ql.loader.driver_object.__class__ + buf = ql.mem.read(ql.loader.driver_object.DeviceObject, ctypes.sizeof(driver_object_cls)) + device_object = driver_object_cls.from_buffer(buf) alloc_addr = [] - def build_mdl(buffer_size, data=None): - if ql.arch.type == QL_ARCH.X8664: - mdl = MDL64() - else: - mdl = MDL32() + def build_mdl(buffer_size: int, data=None): + mdl = make_mdl(ql.arch.bits) mapped_address = heap.alloc(buffer_size) alloc_addr.append(mapped_address) @@ -69,27 +55,21 @@ def io_Write(ql: Qiling, in_buffer: bytes): ql.mem.write(mapped_address, written) return mdl + # allocate memory regions for IRP and IO_STACK_LOCATION - if ql.arch.type == QL_ARCH.X8664: - irp_addr = heap.alloc(ctypes.sizeof(IRP64)) - alloc_addr.append(irp_addr) - irpstack_addr = heap.alloc(ctypes.sizeof(IO_STACK_LOCATION64)) - alloc_addr.append(irpstack_addr) - # setup irp stack parameters - irpstack = IO_STACK_LOCATION64() - # setup IRP structure - irp = IRP64() - irp.irpstack = ctypes.cast(irpstack_addr, ctypes.POINTER(IO_STACK_LOCATION64)) - else: - irp_addr = heap.alloc(ctypes.sizeof(IRP32)) - alloc_addr.append(irp_addr) - irpstack_addr = heap.alloc(ctypes.sizeof(IO_STACK_LOCATION32)) - alloc_addr.append(irpstack_addr) - # setup irp stack parameters - irpstack = IO_STACK_LOCATION32() - # setup IRP structure - irp = IRP32() - irp.irpstack = ctypes.cast(irpstack_addr, ctypes.POINTER(IO_STACK_LOCATION32)) + irp = make_irp(ql.arch.bits) + irpstack_class = irp.irpstack._type_ + + irp_addr = heap.alloc(ctypes.sizeof(irp)) + alloc_addr.append(irp_addr) + + irpstack_addr = heap.alloc(ctypes.sizeof(irpstack_class)) + alloc_addr.append(irpstack_addr) + + # setup irp stack parameters + irpstack = irpstack_class() + # setup IRP structure + irp.irpstack = ctypes.cast(irpstack_addr, ctypes.POINTER(irpstack_class)) irpstack.MajorFunction = IRP_MJ_WRITE irpstack.Parameters.Write.Length = len(in_buffer) @@ -104,11 +84,7 @@ def io_Write(ql: Qiling, in_buffer: bytes): elif device_object.Flags & DO_DIRECT_IO: # DIRECT_IO mdl = build_mdl(len(in_buffer)) - if ql.arch.type == QL_ARCH.X8664: - mdl_addr = heap.alloc(ctypes.sizeof(MDL64)) - else: - mdl_addr = heap.alloc(ctypes.sizeof(MDL32)) - + mdl_addr = heap.alloc(ctypes.sizeof(mdl)) alloc_addr.append(mdl_addr) ql.mem.write(mdl_addr, bytes(mdl)) @@ -127,7 +103,10 @@ def io_Write(ql: Qiling, in_buffer: bytes): # set function args # TODO: make sure this is indeed STDCALL ql.os.fcall = ql.os.fcall_select(STDCALL) - ql.os.fcall.writeParams(((POINTER, ql.loader.driver_object.DeviceObject), (POINTER, irp_addr))) + ql.os.fcall.writeParams(( + (POINTER, ql.loader.driver_object.DeviceObject), + (POINTER, irp_addr) + )) try: # now emulate @@ -136,12 +115,8 @@ def io_Write(ql: Qiling, in_buffer: bytes): verify_ret(ql, err) # read current IRP state - if ql.arch.type == QL_ARCH.X8664: - irp_buffer = ql.mem.read(irp_addr, ctypes.sizeof(IRP64)) - irp = IRP64.from_buffer(irp_buffer) - else: - irp_buffer = ql.mem.read(irp_addr, ctypes.sizeof(IRP32)) - irp = IRP32.from_buffer(irp_buffer) + irp_buffer = ql.mem.read(irp_addr, ctypes.sizeof(irp)) + irp = irp.from_buffer(irp_buffer) io_status = irp.IoStatus # now free all alloc memory @@ -162,159 +137,130 @@ def io_Write(ql: Qiling, in_buffer: bytes): # LPDWORD lpBytesReturned, # LPOVERLAPPED lpOverlapped); def ioctl(ql: Qiling, params: Tuple[Tuple, int, bytes]) -> Tuple: - heap = ql.os.heap - def ioctl_code(DeviceType, Function, Method, Access): + allocations = [] + + def __heap_alloc(size: int) -> int: + address = ql.os.heap.alloc(size) + allocations.append(address) + + return address + + def __free_all(allocations: Iterable[int]) -> None: + for address in allocations: + ql.os.heap.free(address) + + def ioctl_code(DeviceType: int, Function: int, Method: int, Access: int) -> int: return (DeviceType << 16) | (Access << 14) | (Function << 2) | Method - alloc_addr = [] def build_mdl(buffer_size, data=None): - if ql.arch.type == QL_ARCH.X8664: - mdl = MDL64() - else: - mdl = MDL32() + mdl = make_mdl(ql.arch.bits) - mapped_address = heap.alloc(buffer_size) - alloc_addr.append(mapped_address) + mapped_address = __heap_alloc(buffer_size) mdl.MappedSystemVa.value = mapped_address mdl.StartVa.value = mapped_address mdl.ByteOffset = 0 mdl.ByteCount = buffer_size + if data: written = data if len(data) <= buffer_size else data[:buffer_size] ql.mem.write(mapped_address, written) return mdl - # quick simple way to manage all alloc memory - if ql.ostype == QL_OS.WINDOWS: - # print("DeviceControl callback is at 0x%x" % ql.loader.driver_object.MajorFunction[IRP_MJ_DEVICE_CONTROL]) - if ql.loader.driver_object.MajorFunction[IRP_MJ_DEVICE_CONTROL] == 0: - # raise error? - return (None, None, None) + if ql.loader.driver_object.MajorFunction[IRP_MJ_DEVICE_CONTROL] == 0: + # raise error? + return (None, None, None) - # create new memory region to store input data - _ioctl_code, output_buffer_size, in_buffer = params - # extract data transfer method - devicetype, function, ctl_method, access = _ioctl_code + # create new memory region to store input data + _ioctl_code, output_buffer_size, in_buffer = params + # extract data transfer method + devicetype, function, ctl_method, access = _ioctl_code - input_buffer_size = len(in_buffer) - input_buffer_addr = heap.alloc(input_buffer_size) - alloc_addr.append(input_buffer_addr) - ql.mem.write(input_buffer_addr, bytes(in_buffer)) + input_buffer_size = len(in_buffer) + input_buffer_addr = __heap_alloc(input_buffer_size) + ql.mem.write(input_buffer_addr, bytes(in_buffer)) - # create new memory region to store out data - output_buffer_addr = heap.alloc(output_buffer_size) - alloc_addr.append(output_buffer_addr) + # create new memory region to store out data + output_buffer_addr = __heap_alloc(output_buffer_size) - # allocate memory regions for IRP and IO_STACK_LOCATION - if ql.arch.type == QL_ARCH.X8664: - irp_addr = heap.alloc(ctypes.sizeof(IRP64)) - alloc_addr.append(irp_addr) - irpstack_addr = heap.alloc(ctypes.sizeof(IO_STACK_LOCATION64)) - alloc_addr.append(irpstack_addr) - # setup irp stack parameters - irpstack = IO_STACK_LOCATION64() - # setup IRP structure - irp = IRP64() - irp.irpstack = ctypes.cast(irpstack_addr, ctypes.POINTER(IO_STACK_LOCATION64)) - else: - irp_addr = heap.alloc(ctypes.sizeof(IRP32)) - alloc_addr.append(irp_addr) - irpstack_addr = heap.alloc(ctypes.sizeof(IO_STACK_LOCATION32)) - alloc_addr.append(irpstack_addr) - # setup irp stack parameters - irpstack = IO_STACK_LOCATION32() - # setup IRP structure - irp = IRP32() - irp.irpstack = ctypes.cast(irpstack_addr, ctypes.POINTER(IO_STACK_LOCATION32)) + # allocate memory regions for IRP and IO_STACK_LOCATION + irp = make_irp(ql.arch.bits) + irpstack_class = irp.irpstack._type_ - #print("32 stack location size = 0x%x" %ctypes.sizeof(IO_STACK_LOCATION32)) - #print("32 status block size = 0x%x" %ctypes.sizeof(IO_STATUS_BLOCK32)) - #print("32 irp size = 0x%x" %ctypes.sizeof(IRP32)) - #print("32 IoStatus offset = 0x%x" %IRP32.IoStatus.offset) - #print("32 UserIosb offset = 0x%x" %IRP32.UserIosb.offset) - #print("32 UserEvent offset = 0x%x" %IRP32.UserEvent.offset) - #print("32 UserBuffer offset = 0x%x" %IRP32.UserBuffer.offset) - #print("32 irpstack offset = 0x%x" %IRP32.irpstack.offset) - #print("irp at %x, irpstack at %x" %(irp_addr, irpstack_addr)) + irp_addr = __heap_alloc(ctypes.sizeof(irp)) + irpstack_addr = __heap_alloc(ctypes.sizeof(irpstack_class)) - ql.log.info("IRP is at 0x%x, IO_STACK_LOCATION is at 0x%x" %(irp_addr, irpstack_addr)) + # setup irp stack parameters + irpstack = irpstack_class() + # setup IRP structure + irp.irpstack = ctypes.cast(irpstack_addr, ctypes.POINTER(irpstack_class)) - irpstack.Parameters.DeviceIoControl.IoControlCode = ioctl_code(devicetype, function, ctl_method, access) - irpstack.Parameters.DeviceIoControl.OutputBufferLength = output_buffer_size - irpstack.Parameters.DeviceIoControl.InputBufferLength = input_buffer_size - irpstack.Parameters.DeviceIoControl.Type3InputBuffer.value = input_buffer_addr # used by IOCTL_METHOD_NEITHER - ql.mem.write(irpstack_addr, bytes(irpstack)) + ql.log.info("IRP is at 0x%x, IO_STACK_LOCATION is at 0x%x" %(irp_addr, irpstack_addr)) - if ctl_method == METHOD_NEITHER: - irp.UserBuffer.value = output_buffer_addr # used by IOCTL_METHOD_NEITHER + irpstack.Parameters.DeviceIoControl.IoControlCode = ioctl_code(devicetype, function, ctl_method, access) + irpstack.Parameters.DeviceIoControl.OutputBufferLength = output_buffer_size + irpstack.Parameters.DeviceIoControl.InputBufferLength = input_buffer_size + irpstack.Parameters.DeviceIoControl.Type3InputBuffer.value = input_buffer_addr # used by IOCTL_METHOD_NEITHER + ql.mem.write(irpstack_addr, bytes(irpstack)) - # allocate memory for AssociatedIrp.SystemBuffer - # used by IOCTL_METHOD_IN_DIRECT, IOCTL_METHOD_OUT_DIRECT and IOCTL_METHOD_BUFFERED - system_buffer_size = max(input_buffer_size, output_buffer_size) - system_buffer_addr = heap.alloc(system_buffer_size) - alloc_addr.append(system_buffer_addr) + if ctl_method == METHOD_NEITHER: + irp.UserBuffer.value = output_buffer_addr # used by IOCTL_METHOD_NEITHER - # init data from input buffer - ql.mem.write(system_buffer_addr, bytes(in_buffer)) - irp.AssociatedIrp.SystemBuffer.value = system_buffer_addr + # allocate memory for AssociatedIrp.SystemBuffer + # used by IOCTL_METHOD_IN_DIRECT, IOCTL_METHOD_OUT_DIRECT and IOCTL_METHOD_BUFFERED + system_buffer_size = max(input_buffer_size, output_buffer_size) + system_buffer_addr = __heap_alloc(system_buffer_size) + # init data from input buffer + ql.mem.write(system_buffer_addr, bytes(in_buffer)) + irp.AssociatedIrp.SystemBuffer.value = system_buffer_addr + + if ctl_method in (METHOD_IN_DIRECT, METHOD_OUT_DIRECT): + # Create MDL structure for output data + # used by both IOCTL_METHOD_IN_DIRECT and IOCTL_METHOD_OUT_DIRECT + mdl = build_mdl(output_buffer_size) + mdl_addr = __heap_alloc(ctypes.sizeof(mdl)) + + ql.mem.write(mdl_addr, bytes(mdl)) + irp.MdlAddress.value = mdl_addr + + # everything is done! Write IRP to memory + ql.mem.write(irp_addr, bytes(irp)) + + # set function args + ql.log.info("Executing IOCTL with DeviceObject = 0x%x, IRP = 0x%x" %(ql.loader.driver_object.DeviceObject, irp_addr)) + # TODO: make sure this is indeed STDCALL + ql.os.fcall = ql.os.fcall_select(STDCALL) + ql.os.fcall.writeParams(( + (POINTER, ql.loader.driver_object.DeviceObject), + (POINTER, irp_addr) + )) + + try: + ql.log.info(f"Executing from: {ql.loader.driver_object.MajorFunction[IRP_MJ_DEVICE_CONTROL]:#x}") + # now emulate IOCTL's DeviceControl + ql.run(ql.loader.driver_object.MajorFunction[IRP_MJ_DEVICE_CONTROL]) + except UcError as err: + verify_ret(ql, err) + + # read current IRP state + irp_buffer = ql.mem.read(irp_addr, ctypes.sizeof(irp)) + irp = irp.__class__.from_buffer(irp_buffer) + + io_status = irp.IoStatus + + # read output data + output_data = b'' + if io_status.Status.Status >= 0: + if ctl_method == METHOD_BUFFERED: + output_data = ql.mem.read(system_buffer_addr, io_status.Information.value) if ctl_method in (METHOD_IN_DIRECT, METHOD_OUT_DIRECT): - # Create MDL structure for output data - # used by both IOCTL_METHOD_IN_DIRECT and IOCTL_METHOD_OUT_DIRECT - mdl = build_mdl(output_buffer_size) - if ql.arch.type == QL_ARCH.X8664: - mdl_addr = heap.alloc(ctypes.sizeof(MDL64)) - else: - mdl_addr = heap.alloc(ctypes.sizeof(MDL32)) + output_data = ql.mem.read(mdl.MappedSystemVa.value, io_status.Information.value) + if ctl_method == METHOD_NEITHER: + output_data = ql.mem.read(output_buffer_addr, io_status.Information.value) - alloc_addr.append(mdl_addr) + # now free all alloc memory + __free_all(allocations) - ql.mem.write(mdl_addr, bytes(mdl)) - irp.MdlAddress.value = mdl_addr - - # everything is done! Write IRP to memory - ql.mem.write(irp_addr, bytes(irp)) - - # set function args - ql.log.info("Executing IOCTL with DeviceObject = 0x%x, IRP = 0x%x" %(ql.loader.driver_object.DeviceObject, irp_addr)) - # TODO: make sure this is indeed STDCALL - ql.os.fcall = ql.os.fcall_select(STDCALL) - ql.os.fcall.writeParams(((POINTER, ql.loader.driver_object.DeviceObject), (POINTER, irp_addr))) - - try: - # now emulate IOCTL's DeviceControl - ql.run(ql.loader.driver_object.MajorFunction[IRP_MJ_DEVICE_CONTROL]) - except UcError as err: - verify_ret(ql, err) - - # read current IRP state - if ql.arch.type == QL_ARCH.X8664: - irp_buffer = ql.mem.read(irp_addr, ctypes.sizeof(IRP64)) - irp = IRP64.from_buffer(irp_buffer) - else: - irp_buffer = ql.mem.read(irp_addr, ctypes.sizeof(IRP32)) - irp = IRP32.from_buffer(irp_buffer) - - io_status = irp.IoStatus - - # read output data - output_data = b'' - if io_status.Status.Status >= 0: - if ctl_method == METHOD_BUFFERED: - output_data = ql.mem.read(system_buffer_addr, io_status.Information.value) - if ctl_method in (METHOD_IN_DIRECT, METHOD_OUT_DIRECT): - output_data = ql.mem.read(mdl.MappedSystemVa.value, io_status.Information.value) - if ctl_method == METHOD_NEITHER: - output_data = ql.mem.read(output_buffer_addr, io_status.Information.value) - - # now free all alloc memory - for addr in alloc_addr: - # print("freeing heap memory at 0x%x" %addr) # FIXME: the output is not deterministic?? - heap.free(addr) - #print("\n") - - return io_status.Status.Status, io_status.Information.value, output_data - else: # TODO: IOCTL for non-Windows. - raise NotImplementedError + return io_status.Status.Status, io_status.Information.value, output_data diff --git a/qiling/os/windows/windows.py b/qiling/os/windows/windows.py index e7f69108..6c115d99 100644 --- a/qiling/os/windows/windows.py +++ b/qiling/os/windows/windows.py @@ -3,19 +3,21 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import json -from typing import Callable +import ntpath +from typing import Callable, TextIO, Type from unicorn import UcError from qiling import Qiling from qiling.arch.x86_const import GS_SEGMENT_ADDR, GS_SEGMENT_SIZE, FS_SEGMENT_ADDR, FS_SEGMENT_SIZE -from qiling.arch.x86_utils import GDTManager, SegmentManager86, SegmentManager64 +from qiling.arch.x86_utils import GDTManager, SegmentManager, SegmentManager86, SegmentManager64 from qiling.cc import intel -from qiling.const import QL_ARCH, QL_INTERCEPT -from qiling.exception import QlErrorSyscallError, QlErrorSyscallNotFound +from qiling.const import QL_ARCH, QL_OS, QL_INTERCEPT +from qiling.exception import QlErrorSyscallError, QlErrorSyscallNotFound, QlMemoryMappedError from qiling.os.fcall import QlFunctionCall +from qiling.os.memory import QlMemoryHeap from qiling.os.os import QlOs +from qiling.os.stats import QlWinStats from . import const from . import fncc @@ -24,13 +26,14 @@ from . import thread from . import clipboard from . import fiber from . import registry -from . import utils import qiling.os.windows.dlls as api class QlOsWindows(QlOs): + type = QL_OS.WINDOWS + def __init__(self, ql: Qiling): - super(QlOsWindows, self).__init__(ql) + super().__init__(ql) self.ql = ql @@ -58,26 +61,75 @@ class QlOsWindows(QlOs): return __selector[atype] self.fcall_select = __make_fcall_selector(ql.arch.type) - self.fcall = None + self.fcall = self.fcall_select(fncc.CDECL) - self.PE_RUN = True + self.stats = QlWinStats() + + ossection = f'OS{self.ql.arch.bits}' + heap_base = self.profile.getint(ossection, 'heap_address') + heap_size = self.profile.getint(ossection, 'heap_size') + + self.heap = QlMemoryHeap(self.ql, heap_base, heap_base + heap_size) + + sysdrv = self.profile.get('PATH', 'systemdrive') + windir = self.profile.get('PATH', 'windir') + username = self.profile.get('USER', 'username') + + self.windir = ntpath.join(sysdrv, windir) + self.winsys = ntpath.join(sysdrv, windir, 'System32') + self.userprofile = ntpath.join(sysdrv, 'Users', username) + self.username = username + + self.PE_RUN = False self.last_error = 0 # variables used inside hooks self.hooks_variables = {} self.syscall_count = {} self.argv = self.ql.argv self.env = self.ql.env - self.pid = self.profile.getint("KERNEL","pid") - self.automatize_input = self.profile.getboolean("MISC","automatize_input") - self.username = self.profile["USER"]["username"] - self.windir = self.profile["PATH"]["systemdrive"] + self.profile["PATH"]["windir"] - self.userprofile = self.profile["PATH"]["systemdrive"] + "Users\\" + self.profile["USER"]["username"] + "\\" + self.pid = self.profile.getint('KERNEL', 'pid') + self.services = {} self.load() + # only after handle manager has been set up we can assign the standard streams + self.stdin = self._stdin + self.stdout = self._stdout + self.stderr = self._stderr + + + @QlOs.stdin.setter + def stdin(self, stream: TextIO) -> None: + self._stdin = stream + + handle = self.handle_manager.get(const.STD_INPUT_HANDLE) + assert handle is not None + + handle.obj = stream + + @QlOs.stdout.setter + def stdout(self, stream: TextIO) -> None: + self._stdout = stream + + handle = self.handle_manager.get(const.STD_OUTPUT_HANDLE) + assert handle is not None + + handle.obj = stream + + @QlOs.stderr.setter + def stderr(self, stream: TextIO) -> None: + self._stderr = stream + + handle = self.handle_manager.get(const.STD_ERROR_HANDLE) + assert handle is not None + + handle.obj = stream + def load(self): self.setupGDT() + self.__setup_components() + # hook win api self.ql.hook_code(self.hook_winapi) @@ -85,7 +137,7 @@ class QlOsWindows(QlOs): def setupGDT(self): gdtm = GDTManager(self.ql) - segm_class = { + segm_class: Type[SegmentManager] = { 32 : SegmentManager86, 64 : SegmentManager64 }[self.ql.arch.bits] @@ -96,25 +148,27 @@ class QlOsWindows(QlOs): segm.setup_fs(FS_SEGMENT_ADDR, FS_SEGMENT_SIZE) segm.setup_gs(GS_SEGMENT_ADDR, GS_SEGMENT_SIZE) - if not self.ql.mem.is_mapped(FS_SEGMENT_ADDR, FS_SEGMENT_SIZE): - self.ql.mem.map(FS_SEGMENT_ADDR, FS_SEGMENT_SIZE, info='[FS]') + if not self.ql.mem.is_available(FS_SEGMENT_ADDR, FS_SEGMENT_SIZE): + raise QlMemoryMappedError('cannot map FS segment, memory location is taken') - if not self.ql.mem.is_mapped(GS_SEGMENT_ADDR, GS_SEGMENT_SIZE): - self.ql.mem.map(GS_SEGMENT_ADDR, GS_SEGMENT_SIZE, info='[GS]') + self.ql.mem.map(FS_SEGMENT_ADDR, FS_SEGMENT_SIZE, info='[FS]') + + if not self.ql.mem.is_available(GS_SEGMENT_ADDR, GS_SEGMENT_SIZE): + raise QlMemoryMappedError('cannot map GS segment, memory location is taken') + + self.ql.mem.map(GS_SEGMENT_ADDR, GS_SEGMENT_SIZE, info='[GS]') - def setupComponents(self): - # handle manager + def __setup_components(self): + reghive = self.path.transform_to_real_path(ntpath.join(self.windir, 'registry')) + self.handle_manager = handle.HandleManager() - # registry manger - self.registry_manager = registry.RegistryManager(self.ql) - # clipboard - self.clipboard = clipboard.Clipboard(self.ql.os) - # fibers + self.registry_manager = registry.RegistryManager(self.ql, reghive) + self.clipboard = clipboard.Clipboard(self) self.fiber_manager = fiber.FiberManager(self.ql) - # thread manager + main_thread = thread.QlWindowsThread(self.ql) - self.thread_manager = thread.QlWindowsThreadManagement(self.ql, main_thread) + self.thread_manager = thread.QlWindowsThreadManagement(self.ql, self, main_thread) # more handle manager new_handle = handle.Handle(obj=main_thread) @@ -148,32 +202,12 @@ class QlOsWindows(QlOs): raise QlErrorSyscallError("Windows API Implementation Error") else: - ql.log.warning(f'api {api_name} is not implemented') + ql.log.warning(f'api {api_name} ({entry["dll"]}) is not implemented') if ql.debug_stop: raise QlErrorSyscallNotFound("Windows API implementation not found") - def post_report(self): - self.ql.log.debug("Syscalls called:") - for key, values in self.stats.syscalls.items(): - self.ql.log.debug(f'{key}:') - - for value in values: - self.ql.log.debug(f' {json.dumps(value):s}') - - self.ql.log.debug("Registries accessed:") - for key, values in self.registry_manager.accessed.items(): - self.ql.log.debug(f'{key}:') - - for value in values: - self.ql.log.debug(f' {json.dumps(value):s}') - - self.ql.log.debug("Strings:") - for key, values in self.stats.appeared_strings.items(): - self.ql.log.debug(f'{key}: {" ".join(str(word) for word in values)}') - - def run(self): if self.ql.exit_point is not None: self.exit_point = self.ql.exit_point @@ -181,14 +215,19 @@ class QlOsWindows(QlOs): if self.ql.entry_point is not None: self.ql.loader.entry_point = self.ql.entry_point + entry_point = self.ql.loader.entry_point + exit_point = (self.ql.loader.entry_point + len(self.ql.code)) if self.ql.code else self.exit_point + + self.PE_RUN = True + try: - if self.ql.code: - self.ql.emu_start(self.ql.loader.entry_point, (self.ql.loader.entry_point + len(self.ql.code)), self.ql.timeout, self.ql.count) - else: - self.ql.emu_start(self.ql.loader.entry_point, self.exit_point, self.ql.timeout, self.ql.count) + self.ql.emu_start(entry_point, exit_point, self.ql.timeout, self.ql.count) except UcError: self.emu_error() raise self.registry_manager.save() - self.post_report() + + # display summary + for entry in self.stats.summary(): + self.ql.log.debug(entry) diff --git a/qiling/profiles/dos.ql b/qiling/profiles/dos.ql index 04542dd0..c1bf3edc 100644 --- a/qiling/profiles/dos.ql +++ b/qiling/profiles/dos.ql @@ -22,4 +22,4 @@ base_address = 0x7000 # usage: append = test1 append = automatize_input = False -current_path = / \ No newline at end of file +current_path = A:\ \ No newline at end of file diff --git a/qiling/profiles/linux.ql b/qiling/profiles/linux.ql index 5b53d982..f810f94b 100644 --- a/qiling/profiles/linux.ql +++ b/qiling/profiles/linux.ql @@ -19,7 +19,7 @@ stack_address = 0x7ff0d000 stack_size = 0x30000 load_address = 0x56555000 interp_address = 0x047ba000 -mmap_address = 0x774bf000 +mmap_address = 0x90000000 [KERNEL] @@ -48,4 +48,4 @@ current_path = / # To use IPv6 or not, to avoid binary double bind. ipv6 and ipv4 bind the same port at the same time bindtolocalhost = True # Bind to localhost -ipv6 = False \ No newline at end of file +ipv6 = False diff --git a/qiling/profiles/windows.ql b/qiling/profiles/windows.ql index d8046a12..17488330 100644 --- a/qiling/profiles/windows.ql +++ b/qiling/profiles/windows.ql @@ -3,18 +3,20 @@ heap_address = 0x500000000 heap_size = 0x5000000 stack_address = 0x7ffffffde000 stack_size = 0x40000 -image_address = 0x400000 +image_address = 0x400000 dll_address = 0x7ffff0000000 entry_point = 0x140000000 +KI_USER_SHARED_DATA = 0xfffff78000000000 [OS32] heap_address = 0x5000000 heap_size = 0x5000000 stack_address = 0xfffdd000 stack_size = 0x21000 -image_address = 0x400000 -dll_address = 0x10000000 -entry_point = 0x40000 +image_address = 0x400000 +dll_address = 0x10000000 +entry_point = 0x40000 +KI_USER_SHARED_DATA = 0xffdf0000 [CODE] # ram_size 0xa00000 is 10MB @@ -38,8 +40,7 @@ split = False # maily for multiple times Ql run with one file # usage: append = test1 append = -automatize_input = False -current_path = / +current_path = C:\ [SYSTEM] # Major Minor ProductType @@ -61,7 +62,7 @@ language = 1093 [PATH] systemdrive = C:\ -windir = Windows\ +windir = Windows [REGISTRY] registry_diff = registry_diff.json diff --git a/qiling/utils.py b/qiling/utils.py index 58064758..dc370305 100644 --- a/qiling/utils.py +++ b/qiling/utils.py @@ -9,209 +9,96 @@ thoughout the qiling framework """ from functools import partial -import importlib, os, copy, re, pefile, logging, yaml +from pathlib import Path +import importlib, inspect, os from configparser import ConfigParser -from logging import LogRecord -from typing import Any, Container, IO, List, Optional, Sequence, TextIO, Tuple, Type, Union -from enum import Enum +from types import ModuleType +from typing import TYPE_CHECKING, Any, Callable, Mapping, Optional, Tuple, TypeVar, Union from unicorn import UC_ERR_READ_UNMAPPED, UC_ERR_FETCH_UNMAPPED from qiling.exception import * -from qiling.const import QL_VERBOSE, QL_ARCH, QL_ENDIAN, QL_OS, QL_DEBUGGER -from qiling.const import debugger_map, arch_map, os_map, arch_os_map, loader_map -from qiling.os.posix.const import NR_OPEN +from qiling.const import QL_ARCH, QL_ENDIAN, QL_OS, QL_DEBUGGER +from qiling.const import debugger_map, arch_map, os_map, arch_os_map -FMT_STR = "%(levelname)s\t%(message)s" +if TYPE_CHECKING: + from qiling import Qiling + from qiling.arch.arch import QlArch + from qiling.debugger.debugger import QlDebugger + from qiling.loader.loader import QlLoader + from qiling.os.os import QlOs -# \033 -> ESC -# ESC [ -> CSI -# CSI %d;%d;... m -> SGR -class COLOR_CODE: - WHITE = '\033[37m' - CRIMSON = '\033[31m' - RED = '\033[91m' - GREEN = '\033[92m' - YELLOW = '\033[93m' - BLUE = '\033[94m' - MAGENTA = '\033[95m' - CYAN = '\033[96m' - ENDC = '\033[0m' +T = TypeVar('T') +QlClassInit = Callable[['Qiling'], T] -class QlBaseFormatter(logging.Formatter): - __level_tag = { - 'WARNING' : '[!]', - 'INFO' : '[=]', - 'DEBUG' : '[+]', - 'CRITICAL' : '[x]', - 'ERROR' : '[x]' - } - - def __init__(self, ql, *args, **kwargs): - super().__init__(*args, **kwargs) - self.ql = ql - - def get_level_tag(self, level: str) -> str: - return self.__level_tag[level] - - def get_thread_tag(self, thread: str) -> str: - return thread - - def format(self, record: LogRecord): - # In case we have multiple formatters, we have to keep a copy of the record. - record = copy.copy(record) - - # early logging may access ql.os when it is not yet set +def catch_KeyboardInterrupt(ql: 'Qiling', func: Callable): + def wrapper(*args, **kw): try: - cur_thread = self.ql.os.thread_management.cur_thread - except AttributeError: - tid = f'' - else: - tid = self.get_thread_tag(str(cur_thread)) + return func(*args, **kw) + except BaseException as e: + ql.stop() + ql._internal_exception = e - level = self.get_level_tag(record.levelname) - record.levelname = f'{level} {tid}' + return wrapper - return super().format(record) +def __name_to_enum(name: str, mapping: Mapping[str, T], aliases: Mapping[str, str] = {}) -> Optional[T]: + key = name.casefold() -class QlColoredFormatter(QlBaseFormatter): - __level_color = { - 'WARNING' : COLOR_CODE.YELLOW, - 'INFO' : COLOR_CODE.BLUE, - 'DEBUG' : COLOR_CODE.MAGENTA, - 'CRITICAL' : COLOR_CODE.CRIMSON, - 'ERROR' : COLOR_CODE.RED - } + return mapping.get(aliases.get(key) or key) - def get_level_tag(self, level: str) -> str: - s = super().get_level_tag(level) - - return f'{self.__level_color[level]}{s}{COLOR_CODE.ENDC}' - - def get_thread_tag(self, tid: str) -> str: - s = super().get_thread_tag(tid) - - return f'{COLOR_CODE.GREEN}{s}{COLOR_CODE.ENDC}' - -class RegexFilter(logging.Filter): - def __init__(self, regexp): - super(RegexFilter, self).__init__() - self.update_filter(regexp) - - def update_filter(self, regexp): - self._filter = re.compile(regexp) - - def filter(self, record: LogRecord): - msg = record.getMessage() - - return re.match(self._filter, msg) is not None - -class QlFileDes: - def __init__(self): - self.__fds: List[Optional[IO]] = [None] * NR_OPEN - - def __len__(self): - return len(self.__fds) - - def __getitem__(self, idx: int): - return self.__fds[idx] - - def __setitem__(self, idx: int, val: Optional[IO]): - self.__fds[idx] = val - - def __iter__(self): - return iter(self.__fds) - - def __repr__(self): - return repr(self.__fds) - - def save(self): - return self.__fds - - def restore(self, fds): - self.__fds = fds - - -def catch_KeyboardInterrupt(ql): - def decorator(func): - def wrapper(*args, **kw): - try: - return func(*args, **kw) - except BaseException as e: - ql.stop() - ql._internal_exception = e - - return wrapper - - return decorator - -def enum_values(e: Type[Enum]) -> Container: - return e.__members__.values() - -def ql_is_valid_ostype(ostype: QL_OS) -> bool: - return ostype in enum_values(QL_OS) - -def ql_is_valid_arch(arch: QL_ARCH) -> bool: - return arch in enum_values(QL_ARCH) - -def loadertype_convert_str(ostype: QL_OS) -> Optional[str]: - return loader_map.get(ostype) - -def __value_to_key(e: Type[Enum], val: Any) -> Optional[str]: - key = e._value2member_map_[val] - - return None if key is None else key.name - -def ostype_convert_str(ostype: QL_OS) -> Optional[str]: - return __value_to_key(QL_OS, ostype) - -def ostype_convert(ostype: str) -> Optional[QL_OS]: +def os_convert(os: str) -> Optional[QL_OS]: alias_map = { - "darwin": "macos", + 'darwin' : 'macos' } - return os_map.get(alias_map.get(ostype, ostype)) - -def arch_convert_str(arch: QL_ARCH) -> Optional[str]: - return __value_to_key(QL_ARCH, arch) + return __name_to_enum(os, os_map, alias_map) def arch_convert(arch: str) -> Optional[QL_ARCH]: alias_map = { - "x86_64": "x8664", - "riscv32": "riscv", + 'x86_64' : 'x8664', + 'riscv32' : 'riscv' } - - return arch_map.get(alias_map.get(arch, arch)) + + return __name_to_enum(arch, arch_map, alias_map) + +def debugger_convert(debugger: str) -> Optional[QL_DEBUGGER]: + return __name_to_enum(debugger, debugger_map) def arch_os_convert(arch: QL_ARCH) -> Optional[QL_OS]: return arch_os_map.get(arch) -def debugger_convert(debugger: str) -> Optional[QL_DEBUGGER]: - return debugger_map.get(debugger) - -def debugger_convert_str(debugger_id: QL_DEBUGGER) -> Optional[str]: - return __value_to_key(QL_DEBUGGER, debugger_id) - -# Call `function_name` in `module_name`. -# e.g. map_syscall in qiling.os.linux.map_syscall -def ql_get_module_function(module_name: str, function_name: str): - +def ql_get_module(module_name: str) -> ModuleType: try: - imp_module = importlib.import_module(module_name) - except ModuleNotFoundError: - raise QlErrorModuleNotFound(f'Unable to import module {module_name}') - except KeyError: + module = importlib.import_module(module_name, 'qiling') + except (ModuleNotFoundError, KeyError): raise QlErrorModuleNotFound(f'Unable to import module {module_name}') + return module + +def ql_get_module_function(module_name: str, member_name: str): + module = ql_get_module(module_name) + try: - module_function = getattr(imp_module, function_name) + member = getattr(module, member_name) except AttributeError: - raise QlErrorModuleFunctionNotFound(f'Unable to import {function_name} from {imp_module}') + raise QlErrorModuleFunctionNotFound(f'Unable to import {member_name} from {module_name}') - return module_function + return member -def ql_elf_parse_emu_env(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: +def __emu_env_from_pathname(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: + if os.path.isdir(path) and path.endswith('.kext'): + return QL_ARCH.X8664, QL_OS.MACOS, QL_ENDIAN.EL + + if os.path.isfile(path): + _, ext = os.path.splitext(path) + + if ext in ('.DOS_COM', '.DOS_MBR', '.DOS_EXE'): + return QL_ARCH.A8086, QL_OS.DOS, QL_ENDIAN.EL + + return None, None, None + +def __emu_env_from_elf(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: # instead of using full-blown elffile parsing, we perform a simple parsing to avoid # external dependencies for target systems that do not need them. # @@ -240,6 +127,7 @@ def ql_elf_parse_emu_env(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], EM_X86_64 = 62 EM_AARCH64 = 183 EM_RISCV = 243 + EM_PPC = 20 endianess = { ELFDATA2LSB : (QL_ENDIAN.EL, 'little'), @@ -250,7 +138,8 @@ def ql_elf_parse_emu_env(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], EM_386 : QL_ARCH.X86, EM_MIPS : QL_ARCH.MIPS, EM_ARM : QL_ARCH.ARM, - EM_RISCV : QL_ARCH.RISCV + EM_RISCV : QL_ARCH.RISCV, + EM_PPC : QL_ARCH.PPC } machines64 = { @@ -312,7 +201,7 @@ def ql_elf_parse_emu_env(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], return archtype, ostype, archendian -def ql_macho_parse_emu_env(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: +def __emu_env_from_macho(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: macho_macos_sig64 = b'\xcf\xfa\xed\xfe' macho_macos_sig32 = b'\xce\xfa\xed\xfe' macho_macos_fat = b'\xca\xfe\xba\xbe' # should be header for FAT @@ -340,8 +229,9 @@ def ql_macho_parse_emu_env(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS return arch, ostype, endian +def __emu_env_from_pe(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: + import pefile -def ql_pe_parse_emu_env(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: try: pe = pefile.PE(path, fast_load=True) except: @@ -379,83 +269,110 @@ def ql_pe_parse_emu_env(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], return arch, ostype, archendian - def ql_guess_emu_env(path: str) -> Tuple[Optional[QL_ARCH], Optional[QL_OS], Optional[QL_ENDIAN]]: - arch = None - ostype = None - endian = None + guessing_methods = ( + __emu_env_from_pathname, + __emu_env_from_elf, + __emu_env_from_macho, + __emu_env_from_pe + ) - if os.path.isdir(path) and path.endswith('.kext'): - return QL_ARCH.X8664, QL_OS.MACOS, QL_ENDIAN.EL + for gm in guessing_methods: + arch, ostype, endian = gm(path) - if os.path.isfile(path) and path.endswith('.DOS_COM'): - return QL_ARCH.A8086, QL_OS.DOS, QL_ENDIAN.EL - - if os.path.isfile(path) and path.endswith('.DOS_MBR'): - return QL_ARCH.A8086, QL_OS.DOS, QL_ENDIAN.EL - - if os.path.isfile(path) and path.endswith('.DOS_EXE'): - return QL_ARCH.A8086, QL_OS.DOS, QL_ENDIAN.EL - - arch, ostype, endian = ql_elf_parse_emu_env(path) - - if arch is None or ostype is None or endian is None: - arch, ostype, endian = ql_macho_parse_emu_env(path) - - if arch is None or ostype is None or endian is None: - arch, ostype, endian = ql_pe_parse_emu_env(path) + if None not in (arch, ostype, endian): + break + else: + arch, ostype, endian = (None, ) * 3 return arch, ostype, endian +def select_loader(ostype: QL_OS, libcache: bool) -> QlClassInit['QlLoader']: + if ostype == QL_OS.WINDOWS: + kwargs = {'libcache' : libcache} -def loader_setup(ql, ostype: QL_OS, libcache: bool): - args = [libcache] if ostype == QL_OS.WINDOWS else [] + else: + kwargs = {} - qlloader_name = loadertype_convert_str(ostype) - qlloader_path = f'qiling.loader.{qlloader_name.lower()}' - qlloader_class = f'QlLoader{qlloader_name.upper()}' + module = { + QL_OS.LINUX : r'elf', + QL_OS.FREEBSD : r'elf', + QL_OS.QNX : r'elf', + QL_OS.MACOS : r'macho', + QL_OS.WINDOWS : r'pe', + QL_OS.UEFI : r'pe_uefi', + QL_OS.DOS : r'dos', + QL_OS.EVM : r'evm', + QL_OS.MCU : r'mcu', + QL_OS.BLOB : r'blob' + }[ostype] + + qlloader_path = f'.loader.{module}' + qlloader_class = f'QlLoader{module.upper()}' obj = ql_get_module_function(qlloader_path, qlloader_class) - return obj(ql, *args) + return partial(obj, **kwargs) - -def component_setup(component_type: str, component_name: str, ql): - component_path = f'qiling.{component_type}.{component_name}' +def select_component(component_type: str, component_name: str, **kwargs) -> QlClassInit[Any]: + component_path = f'.{component_type}.{component_name}' component_class = f'Ql{component_name.capitalize()}Manager' obj = ql_get_module_function(component_path, component_class) - return obj(ql) + return partial(obj, **kwargs) - -def debugger_setup(options: Union[str, bool], ql): +def select_debugger(options: Union[str, bool]) -> Optional[QlClassInit['QlDebugger']]: if options is True: options = 'gdb' if type(options) is str: objname, *args = options.split(':') + dbgtype = debugger_convert(objname) - if debugger_convert(objname) not in enum_values(QL_DEBUGGER): + if dbgtype == QL_DEBUGGER.GDB: + kwargs = dict(zip(('ip', 'port'), args)) + + elif dbgtype == QL_DEBUGGER.QDB: + kwargs = {} + + def __int_nothrow(v: str, /) -> Optional[int]: + try: + return int(v, 0) + except ValueError: + return None + + # qdb init args are independent and may include any combination of: rr enable, init hook and script + for a in args: + if a == 'rr': + kwargs['rr'] = True + + elif __int_nothrow(a) is not None: + kwargs['init_hook'] = a + + else: + kwargs['script'] = a + + else: raise QlErrorOutput('Debugger not supported') - obj = ql_get_module_function(f'qiling.debugger.{objname}.{objname}', f'Ql{str.capitalize(objname)}') + obj = ql_get_module_function(f'.debugger.{objname}.{objname}', f'Ql{str.capitalize(objname)}') - return obj(ql, *args) + return partial(obj, **kwargs) return None -def arch_setup(archtype: QL_ARCH, endian: QL_ENDIAN, thumb: bool, ql): +def select_arch(archtype: QL_ARCH, endian: QL_ENDIAN, thumb: bool) -> QlClassInit['QlArch']: # set endianess and thumb mode for arm-based archs if archtype == QL_ARCH.ARM: - args = [endian, thumb] + kwargs = {'endian' : endian, 'thumb' : thumb} # set endianess for mips arch elif archtype == QL_ARCH.MIPS: - args = [endian] + kwargs = {'endian' : endian} else: - args = [] + kwargs = {} module = { QL_ARCH.A8086 : r'x86', @@ -467,43 +384,31 @@ def arch_setup(archtype: QL_ARCH, endian: QL_ENDIAN, thumb: bool, ql): QL_ARCH.EVM : r'evm.evm', QL_ARCH.CORTEX_M : r'cortex_m', QL_ARCH.RISCV : r'riscv', - QL_ARCH.RISCV64 : r'riscv64' + QL_ARCH.RISCV64 : r'riscv64', + QL_ARCH.PPC : r'ppc' }[archtype] - qlarch_path = f'qiling.arch.{module}' - qlarch_class = f'QlArch{arch_convert_str(archtype).upper()}' + qlarch_path = f'.arch.{module}' + qlarch_class = f'QlArch{archtype.name.upper()}' obj = ql_get_module_function(qlarch_path, qlarch_class) - return obj(ql, *args) + return partial(obj, **kwargs) - -# This function is extracted from os_setup (QlOsPosix) so I put it here. -def ql_syscall_mapping_function(ostype: QL_OS, archtype: QL_ARCH): - qlos_name = ostype_convert_str(ostype) - qlos_path = f'qiling.os.{qlos_name.lower()}.map_syscall' - qlos_func = 'get_syscall_mapper' - - func = ql_get_module_function(qlos_path, qlos_func) - - return func(archtype) - - -def os_setup(ostype: QL_OS, ql): - qlos_name = ostype_convert_str(ostype) - qlos_path = f'qiling.os.{qlos_name.lower()}.{qlos_name.lower()}' +def select_os(ostype: QL_OS) -> QlClassInit['QlOs']: + qlos_name = ostype.name + qlos_path = f'.os.{qlos_name.lower()}.{qlos_name.lower()}' qlos_class = f'QlOs{qlos_name.capitalize()}' obj = ql_get_module_function(qlos_path, qlos_class) - return obj(ql) - - -def profile_setup(ql, ostype: QL_OS, filename: Optional[str]): - ql.log.debug(f'Profile: {filename or "default"}') + return partial(obj) +def profile_setup(ostype: QL_OS, filename: Optional[str]): # mcu uses a yaml-based config if ostype == QL_OS.MCU: + import yaml + if filename: with open(filename) as f: config = yaml.load(f, Loader=yaml.Loader) @@ -511,8 +416,8 @@ def profile_setup(ql, ostype: QL_OS, filename: Optional[str]): config = {} else: - qiling_home = os.path.dirname(os.path.abspath(__file__)) - os_profile = os.path.join(qiling_home, 'profiles', f'{ostype_convert_str(ostype).lower()}.ql') + qiling_home = Path(inspect.getfile(inspect.currentframe())).parent + os_profile = qiling_home / 'profiles' / f'{ostype.name.lower()}.ql' profiles = [os_profile] @@ -527,111 +432,6 @@ def profile_setup(ql, ostype: QL_OS, filename: Optional[str]): return config -def ql_resolve_logger_level(verbose: QL_VERBOSE) -> int: - return { - QL_VERBOSE.DISABLED: logging.CRITICAL, - QL_VERBOSE.OFF : logging.WARNING, - QL_VERBOSE.DEFAULT : logging.INFO, - QL_VERBOSE.DEBUG : logging.DEBUG, - QL_VERBOSE.DISASM : logging.DEBUG, - QL_VERBOSE.DUMP : logging.DEBUG - }[verbose] - -QL_INSTANCE_ID = 114514 - -def __is_color_terminal(stream: TextIO) -> bool: - """Determine whether standard output is attached to a color terminal. - - see: https://stackoverflow.com/questions/53574442/how-to-reliably-test-color-capability-of-an-output-terminal-in-python3 - """ - - def __handle_nt(fd: int) -> bool: - import ctypes - import msvcrt - - ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 - - kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) - hstdout = msvcrt.get_osfhandle(fd) - mode = ctypes.c_ulong() - - return kernel32.GetConsoleMode(hstdout, ctypes.byref(mode)) and (mode.value & ENABLE_VIRTUAL_TERMINAL_PROCESSING != 0) - - def __handle_posix(fd: int) -> bool: - import curses - - try: - curses.setupterm(fd=fd) - except curses.error: - return True - else: - return curses.tigetnum('colors') > 0 - - def __default(_: int) -> bool: - return True - - handlers = { - 'nt' : __handle_nt, - 'posix' : __handle_posix - } - - handler = handlers.get(os.name, __default) - - return handler(stream.fileno()) - -# TODO: qltool compatibility -def ql_setup_logger(ql, log_file: Optional[str], console: bool, filters: Optional[Sequence], log_override: Optional[logging.Logger], log_plain: bool): - global QL_INSTANCE_ID - - # If there is an override for our logger, then use it. - if log_override is not None: - log = log_override - else: - # We should leave the root logger untouched. - log = logging.getLogger(f"qiling{QL_INSTANCE_ID}") - QL_INSTANCE_ID += 1 - - # Disable propagation to avoid duplicate output. - log.propagate = False - # Clear all handlers and filters. - log.handlers = [] - log.filters = [] - - # Do we have console output? - if console: - handler = logging.StreamHandler() - - if log_plain or not __is_color_terminal(handler.stream): - formatter = QlBaseFormatter(ql, FMT_STR) - else: - formatter = QlColoredFormatter(ql, FMT_STR) - - handler.setFormatter(formatter) - log.addHandler(handler) - else: - handler = logging.NullHandler() - log.addHandler(handler) - - # Do we have to write log to a file? - if log_file is not None: - handler = logging.FileHandler(log_file) - formatter = QlBaseFormatter(ql, FMT_STR) - handler.setFormatter(formatter) - log.addHandler(handler) - - # Remeber to add filters if necessary. - # If there aren't any filters, we do add the filters until users specify any. - log_filter = None - - if filters: - log_filter = RegexFilter(filters) - log.addFilter(log_filter) - - log.setLevel(logging.INFO) - - return log, log_filter - - # verify if emulator returns properly def verify_ret(ql, err): ql.log.debug("Got exception %u: init SP = %x, current SP = %x, PC = %x" %(err.errno, ql.os.init_sp, ql.arch.regs.arch_sp, ql.arch.regs.arch_pc)) @@ -640,7 +440,7 @@ def verify_ret(ql, err): # timeout is acceptable in this case if err.errno in (UC_ERR_READ_UNMAPPED, UC_ERR_FETCH_UNMAPPED): - if ql.ostype == QL_OS.MACOS: + if ql.os.type == QL_OS.MACOS: if ql.loader.kext_name: # FIXME: Should I push saved RIP before every method callings of IOKit object? if ql.os.init_sp == ql.arch.regs.arch_sp - 8: @@ -663,3 +463,21 @@ def verify_ret(ql, err): raise else: raise + +__all__ = [ + 'catch_KeyboardInterrupt', + 'os_convert', + 'arch_convert', + 'debugger_convert', + 'arch_os_convert', + 'ql_get_module', + 'ql_get_module_function', + 'ql_guess_emu_env', + 'select_os', + 'select_arch', + 'select_loader', + 'select_debugger', + 'select_component', + 'profile_setup', + 'verify_ret' +] diff --git a/qltool b/qltool index 5467c0dd..861bbbae 100755 --- a/qltool +++ b/qltool @@ -8,14 +8,15 @@ import os import sys import ast import pickle -import unicorn + +from unicorn import __version__ as uc_ver +from qiling import __version__ as ql_ver from qiling import Qiling from qiling.arch import utils as arch_utils from qiling.debugger.qdb import QlQdb from qiling.utils import arch_convert from qiling.const import QL_VERBOSE, QL_ENDIAN, os_map, arch_map, verbose_map -from qiling.__version__ import __version__ as ql_version from qiling.extensions.coverage import utils as cov_utils from qiling.extensions import report @@ -167,9 +168,9 @@ def handle_examples(parser: argparse.ArgumentParser): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument('--version', action='version', version=f'qltool for Qiling {ql_version}, using Unicorn {unicorn.__version__}') + parser.add_argument('--version', action='version', version=f'qltool for Qiling {ql_ver}, using Unicorn {uc_ver}') - commands = parser.add_subparsers(title='sub commands', description='select execution mode', dest='subcommand') + commands = parser.add_subparsers(title='sub commands', description='select execution mode', dest='subcommand', required=True) # set "run" subcommand options run_parser = commands.add_parser('run', help='run a program') @@ -186,7 +187,7 @@ if __name__ == '__main__': code_parser.add_argument('--thumb', action='store_true', help='specify thumb mode for ARM') code_parser.add_argument('--endian', choices=('little', 'big'), default='little', help='specify endianess for bi-endian archs') code_parser.add_argument('--os', required=True, choices=os_map) - code_parser.add_argument('--rootfs', help='emulated root filesystem, that is where all libraries reside') + code_parser.add_argument('--rootfs', default='.', help='emulated root filesystem, that is where all libraries reside') code_parser.add_argument('--format', choices=('asm', 'hex', 'bin'), default='bin', help='input file format') # set "examples" subcommand @@ -218,16 +219,11 @@ if __name__ == '__main__': comm_parser.add_argument('--libcache', action='store_true', help='enable dll caching for windows') options = parser.parse_args() - # subparser argument required=True is not supported in python 3.6 - # manually check whether the user did not specify a subcommand (execution mode) - if not options.subcommand: - parser.error('please select execution mode') - if options.subcommand == 'examples': handle_examples(parser) # ql file setup - if options.subcommand == 'run': + elif options.subcommand == 'run': ql = handle_run(options) # ql code setup diff --git a/setup.py b/setup.py index 18519bd0..0626555f 100644 --- a/setup.py +++ b/setup.py @@ -2,25 +2,22 @@ # # Python setup for Qiling framework -import sys, os from setuptools import setup, find_packages -here = os.path.abspath(os.path.dirname(__file__)) -gb = {} -with open(os.path.join(here, "qiling", "__version__.py"), "r+") as f: - exec(f.read(), gb) - -VERSION = gb['__version__'] +# NOTE: use "-dev" for dev branch +VERSION = "1.4.4" + "-dev" +#VERSION = "1.4.3" requirements = [ "capstone>=4.0.1", "unicorn>=2.0.0-rc7", - "pefile>=2021.9.3", + "pefile>=2022.5.30", "python-registry>=1.3.1", "keystone-engine>=0.9.2", "pyelftools>=0.28", "gevent>=20.9.0", "multiprocess>=0.70.12.2", + "windows-curses>=2.1.0;platform_system=='Windows'", "pyyaml>=6.0" ] @@ -43,22 +40,14 @@ extras = { "cmd2" ], "fuzz" : [ - + "unicornafl>=2.0.0;platform_system=='Windows'", + "fuzzercorn>=0.0.1;platform_system=='Linux'" ] } with open("README.md", "r", encoding="utf-8") as ld: long_description = ld.read() -if "win32" in sys.platform: - requirements += ["windows-curses>=2.1.0"] - -if "win32" not in sys.platform: - extras["fuzz"] += ["unicornafl>=2.0.0"] - -if "linux" in sys.platform: - extras["fuzz"] += ["fuzzercorn>=0.0.1"] - setup( name='qiling', version=VERSION, @@ -96,7 +85,12 @@ setup( packages=find_packages(), scripts=['qltool'], - include_package_data=True, + package_data={ + 'qiling': ['profiles/*.ql'], + 'qiling.debugger.gdb': ['xml/*/*'], + 'qiling.os.uefi': ['guids.csv'], + 'qiling.arch.evm.analysis': ['signatures.json'] + }, install_requires=requirements, extras_require=extras, ) diff --git a/tests/test_android.py b/tests/test_android.py index 79d18ea6..466bb0cf 100644 --- a/tests/test_android.py +++ b/tests/test_android.py @@ -4,28 +4,57 @@ # import os, platform, sys, unittest +from collections import defaultdict sys.path.append("..") from qiling import Qiling -from qiling.const import QL_VERBOSE +from qiling.os.mapper import QlFsMappedObject +from qiling.os.posix import syscall + + +class Fake_maps(QlFsMappedObject): + def __init__(self, ql): + self.ql = ql + def read(self, size): + stack = next(filter(lambda x : x[3]=='[stack]', self.ql.mem.map_info)) + return ('%x-%x %s\n' % (stack[0], stack[1], stack[3])).encode() + def fstat(self): + return defaultdict(int) + def close(self): + return 0 + +def my_syscall_close(ql, fd): + if fd in [0, 1, 2]: + return 0 + return syscall.ql_syscall_close(ql, fd) class TestAndroid(unittest.TestCase): + @unittest.skipUnless(platform.system() == 'Linux', 'run only on Linux') def test_android_arm64(self): - if platform.system() != "Linux": - return - test_binary = "../examples/rootfs/arm64_android/bin/arm64_android_hello" - rootfs = "../examples/rootfs/arm64_android" + test_binary = "../examples/rootfs/arm64_android6.0/bin/arm64_android_jniart" + rootfs = "../examples/rootfs/arm64_android6.0" + env = {"ANDROID_DATA":"/data", "ANDROID_ROOT":"/system"} - # FUTURE FIX: at this stage, need a file called /proc/self/exe in the rootfs - Android linker calls stat against /proc/self/exe and bails if it can't find it - # qiling handles readlink against /proc/self/exe, but doesn't handle it in stat - # https://cs.android.com/android/platform/superproject/+/master:bionic/linker/linker_main.cpp;l=221 - self.assertTrue(os.path.isfile(os.path.join(rootfs, "proc", "self", "exe")), rootfs + - "/proc/self/exe not found, Android linker will bail. Need a file at that location (empty is fine)") - - ql = Qiling([test_binary], rootfs, verbose=QL_VERBOSE.DEBUG, multithread=True) + ql = Qiling([test_binary], rootfs, env, multithread=True) + ql.os.set_syscall("close", my_syscall_close) + ql.add_fs_mapper("/proc/self/task/2000/maps", Fake_maps(ql)) ql.run() + del ql + + + #@unittest.skipUnless(platform.system() == 'Linux', 'run only on Linux') + #def test_android_arm(self): + # test_binary = "../examples/rootfs/arm64_android6.0/bin/arm_android_jniart" + # rootfs = "../examples/rootfs/arm64_android6.0" + # env = {"ANDROID_DATA":"/data", "ANDROID_ROOT":"/system"} + + # ql = Qiling([test_binary], rootfs, env, multithread=True) + # ql.os.set_syscall("close", my_syscall_close) + # ql.add_fs_mapper("/proc/self/task/2000/maps", Fake_maps(ql)) + # ql.run() + # del ql if __name__ == "__main__": diff --git a/tests/test_elf.py b/tests/test_elf.py index 141c2acc..7eb18fec 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -93,6 +93,8 @@ class ELFTest(unittest.TestCase): ql.run(begin=hook_address) del ql + os.remove(snapshot) + def test_elf_linux_x86_snapshot_restore_reg(self): self._test_elf_linux_x86_snapshot_restore_common(reg=True, ctx=False) @@ -820,6 +822,12 @@ class ELFTest(unittest.TestCase): del ql + def test_elf_linux_powerpc(self): + ql = Qiling(["../examples/rootfs/powerpc_linux/bin/powerpc_hello"], "../examples/rootfs/powerpc_linux", verbose=QL_VERBOSE.DEBUG) + ql.run() + del ql + + def test_elf_linux_arm_custom_syscall(self): def my_syscall_write(ql, write_fd, write_buf, write_count, *args, **kw): regreturn = 0 @@ -1008,12 +1016,6 @@ class ELFTest(unittest.TestCase): ql.run() del ql - def test_arm_directory_symlink(self): - ql = Qiling(["../examples/rootfs/arm_linux/bin/arm_hello"], "../examples/rootfs/arm_linux", verbose=QL_VERBOSE.DEBUG) - real_path = ql.os.path.transform_to_real_path("/lib/libsymlink_test.so") - self.assertTrue(real_path.endswith("/examples/rootfs/arm_linux/tmp/media/nand/symlink_test/libsymlink_test.so")) - del ql - def test_x8664_absolute_path(self): ql = Qiling(["../examples/rootfs/x8664_linux/bin/absolutepath"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG) @@ -1119,28 +1121,28 @@ class ELFTest(unittest.TestCase): ql.mem.write(0x1FFB, b"\x1f\x00\x07\x53\x03\x06\x07\x1f\x1b") # Needle not in haystack - self.assertEqual([], ql.mem.search(re.escape(b"\x3a\x01\x0b\x03\x53\x29\x1b\x1c\x04\x0d\x11"))) + self.assertEqual([], ql.mem.search(b"\x3a\x01\x0b\x03\x53\x29\x1b\x1c\x04\x0d\x11")) # Needle appears several times in haystack - self.assertEqual([0x1000 + 24, 0x2000 + 38, 0x3000 + 24], ql.mem.search(re.escape(b"\x4f\x53\x06\x0d\x1e\x0d\x1a"))) + self.assertEqual([0x1000 + 24, 0x2000 + 38, 0x3000 + 24], ql.mem.search(b"\x4f\x53\x06\x0d\x1e\x0d\x1a")) # Needle inside haystack - self.assertEqual([0x1000 + 13], ql.mem.search(re.escape(b"\x0f\x01\x1e\x0d\x53\x11\x07\x1d\x53\x1d\x18"), begin=0x1000 + 10, end=0x1000 + 30)) + self.assertEqual([0x1000 + 13], ql.mem.search(b"\x0f\x01\x1e\x0d\x53\x11\x07\x1d\x53\x1d\x18", begin=0x1000 + 10, end=0x1000 + 30)) # Needle before haystack - self.assertEqual([], ql.mem.search(re.escape(b"\x04\x0d\x1c\x53\x11\x07\x1d\x53\x0c\x07\x1f\x06"), begin=0x1337)) + self.assertEqual([], ql.mem.search(b"\x04\x0d\x1c\x53\x11\x07\x1d\x53\x0c\x07\x1f\x06", begin=0x1337)) # Needle after haystack - self.assertEqual([], ql.mem.search(re.escape(b"\x1b\x09\x11\x53\x0f\x07\x07\x0c\x0a\x11\x0d"), end=0x3000 + 13)) + self.assertEqual([], ql.mem.search(b"\x1b\x09\x11\x53\x0f\x07\x07\x0c\x0a\x11\x0d", end=0x3000 + 13)) # Needle exactly inside haystack - self.assertEqual([0x2000 + 13], ql.mem.search(re.escape(b"\x1a\x1d\x06\x53\x09\x1a\x07\x1d\x06\x0c"), begin=0x2000 + 13, end=0x2000 + 23)) + self.assertEqual([0x2000 + 13], ql.mem.search(b"\x1a\x1d\x06\x53\x09\x1a\x07\x1d\x06\x0c", begin=0x2000 + 13, end=0x2000 + 23)) # Needle 'tears' two mapped regions - self.assertEqual([], ql.mem.search(re.escape(b"\x1f\x00\x07\x53\x03\x06\x07\x1f\x1b"), begin=0x1F00, end=0x200F)) + self.assertEqual([], ql.mem.search(b"\x1f\x00\x07\x53\x03\x06\x07\x1f\x1b", begin=0x1F00, end=0x200F)) # Needle is a regex - self.assertEqual([0x1000 + 11, 0x2000 + 11, 0x3000 + 43], ql.mem.search(b"\x09\x53(\x0f|\x1a|\x04)[^\x0d]")) + self.assertEqual([0x1000 + 11, 0x2000 + 11, 0x3000 + 43], ql.mem.search(re.compile(b"\x09\x53(\x0f|\x1a|\x04)[^\x0d]"))) del ql diff --git a/tests/test_elf_ko.py b/tests/test_elf_ko.py index a9cc56c0..69c61a05 100644 --- a/tests/test_elf_ko.py +++ b/tests/test_elf_ko.py @@ -17,10 +17,8 @@ IS_FAST_TEST = 'QL_FAST_TEST' in os.environ class ELF_KO_Test(unittest.TestCase): + @unittest.skipIf(IS_FAST_TEST, 'fast test') def test_demigod_m0hamed_x86(self): - if IS_FAST_TEST: - self.skipTest('QL_FAST_TEST') - checklist = {} @linux_kernel_api(params={ diff --git a/tests/test_elf_multithread.py b/tests/test_elf_multithread.py index fd909f2f..482a3025 100644 --- a/tests/test_elf_multithread.py +++ b/tests/test_elf_multithread.py @@ -13,10 +13,8 @@ from qiling.os.filestruct import ql_file class ELFTest(unittest.TestCase): + @unittest.skipIf(platform.system() == "Darwin" and platform.machine() == "arm64", 'darwin host') def test_elf_linux_execve_x8664(self): - if platform.system() == "Darwin" and platform.machine() == "arm64": - return - ql = Qiling(["../examples/rootfs/x8664_linux/bin/posix_syscall_execve"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG) ql.run() @@ -31,9 +29,6 @@ class ELFTest(unittest.TestCase): def test_elf_linux_cloexec_x8664(self): - with open('../examples/rootfs/x8664_linux/testfile', 'wb') as f: - f.write(b'0123456789') - ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_cloexec_test"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG, diff --git a/tests/test_macho_kext.py b/tests/test_macho_kext.py index 879b676f..2bb23ccf 100644 --- a/tests/test_macho_kext.py +++ b/tests/test_macho_kext.py @@ -4,7 +4,7 @@ # -import sys, unittest +import os, sys, unittest from pathlib import Path from unicorn import UcError @@ -16,7 +16,10 @@ from qiling.os.const import STRING from qiling.os.macos.structs import * from qiling.os.macos.fncc import macos_kernel_api +IS_FAST_TEST = 'QL_FAST_TEST' in os.environ + class MACHOTest(unittest.TestCase): + @unittest.skipIf(IS_FAST_TEST, 'fast test') def test_macho_macos_superrootkit(self): # https://developer.apple.com/download/more # to download kernel.developmment diff --git a/tests/test_pathutils.py b/tests/test_pathutils.py index 3954203c..de56694e 100644 --- a/tests/test_pathutils.py +++ b/tests/test_pathutils.py @@ -3,127 +3,178 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import pathlib, sys, unittest +import sys, unittest +from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath + +sys.path.append('..') +from qiling.const import QL_OS +from qiling.os.path import QlOsPath + +is_nt_host = PurePath() == PureWindowsPath() +is_posix_host = PurePath() == PurePosixPath() + +def realpath(path: PurePath) -> Path: + return Path(path).resolve() + +def nt_to_native(rootfs: str, cwd: str, path: str) -> str: + p = QlOsPath(rootfs, cwd, QL_OS.WINDOWS) + + return p.virtual_to_host_path(path) + +def posix_to_native(rootfs: str, cwd: str, path: str) -> str: + p = QlOsPath(rootfs, cwd, QL_OS.LINUX) + + return p.virtual_to_host_path(path) -sys.path.append("..") -from qiling import Qiling -from qiling.const import QL_OS, QL_VERBOSE -from qiling.os.path import QlPathManager class TestPathUtils(unittest.TestCase): - def test_convert_win32_to_posix(self): - rootfs = pathlib.Path("../examples/rootfs/x8664_windows").resolve() + @unittest.skipUnless(is_posix_host, 'POSIX host only') + def test_convert_nt_to_posix(self): + rootfs = PurePosixPath(r'../examples/rootfs/x86_windows') - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\.\\PhysicalDrive0\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\.\\PhysicalDrive0\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\.\\PhysicalDrive0\\..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\.\\PhysicalDrive0\\..\\xxxx\\..\\test"))) + expected = str(realpath(rootfs) / 'test') - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\hostname\\share\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\hostname\\share\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\hostname\\share\\..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\hostname\\share\\..\\xxxx\\..\\test"))) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\PhysicalDrive0\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\PhysicalDrive0\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\PhysicalDrive0\\..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\PhysicalDrive0\\..\\xxxx\\..\\test')) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\.\\BootPartition\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\.\\BootPartition\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\.\\BootPartition\\..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\\\.\\BootPartition\\..\\xxxx\\..\\test"))) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\hostname\\share\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\hostname\\share\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\hostname\\share\\..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\hostname\\share\\..\\xxxx\\..\\test')) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "C:\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "C:\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "C:\\..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "C:\\..\\xxxx\\..\\test"))) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\BootPartition\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\BootPartition\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\BootPartition\\..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\BootPartition\\..\\xxxx\\..\\test')) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "\\..\\xxxx\\..\\test"))) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', 'C:\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', 'C:\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', 'C:\\..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', 'C:\\..\\xxxx\\..\\test')) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/", "..\\xxxx\\..\\test"))) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\..\\xxxx\\..\\test')) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/xxxx", "test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/xxxx/yyyy", "..\\test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/xxxx/yyyy/zzzz", "..\\..\\test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_win32_to_posix(rootfs, "/xxxx/yyyy", "..\\xxxx\\..\\test"))) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', 'test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '..\\xxxx\\..\\test')) - def test_convert_posix_to_win32(self): - rootfs = pathlib.Path("../examples/rootfs/x8664_linux").resolve() + expected = str(realpath(rootfs) / 'Windows' / 'test') - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/", "/test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/", "/../test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/", "/../../test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/", "/../xxxx/../test"))) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\Windows', 'test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\Windows\\System32', '..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\Windows\\System32\\drivers', '..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\Windows\\System32', '..\\xxxx\\..\\test')) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/", "test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/", "../test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/", "../../test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/", "../xxxx/../test"))) + @unittest.skipUnless(is_nt_host, 'NT host only') + def test_convert_posix_to_nt(self): + rootfs = PureWindowsPath(r'../examples/rootfs/x86_linux') - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/xxxx", "test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/xxxx/yyyy", "../test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/xxxx/yyyy/zzzz", "../../test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_posix_to_win32(rootfs, "/xxxx/yyyy", "../xxxx/../test"))) + expected = str(realpath(rootfs) / 'test') + + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '/test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '/../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '/../../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '/../xxxx/../test')) + + self.assertEqual(expected, posix_to_native(str(rootfs), '/', 'test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '../../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '../xxxx/../test')) + + expected = str(realpath(rootfs) / 'proc' / 'test') + + self.assertEqual(expected, posix_to_native(str(rootfs), '/proc', 'test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/proc/sys', '../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/proc/sys/net', '../../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/proc/sys', '../xxxx/../test')) def test_convert_for_native_os(self): - ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_hello_static"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG) - if ql.host.os == QL_OS.WINDOWS: - rootfs = pathlib.Path("../examples/rootfs/x8664_windows").resolve() - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\.\\PhysicalDrive0\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\.\\PhysicalDrive0\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\.\\PhysicalDrive0\\..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\.\\PhysicalDrive0\\..\\xxxx\\..\\test"))) + if is_nt_host: + rootfs = PureWindowsPath(r'../examples/rootfs/x86_windows') - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\hostname\\share\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\hostname\\share\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\hostname\\share\\..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\hostname\\share\\..\\xxxx\\..\\test"))) + expected = str(realpath(rootfs) / 'test') - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\.\\BootPartition\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\.\\BootPartition\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\.\\BootPartition\\..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\\\.\\BootPartition\\..\\xxxx\\..\\test"))) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\PhysicalDrive0\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\PhysicalDrive0\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\PhysicalDrive0\\..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\PhysicalDrive0\\..\\xxxx\\..\\test')) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "C:\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "C:\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "C:\\..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "C:\\..\\xxxx\\..\\test"))) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\hostname\\share\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\hostname\\share\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\hostname\\share\\..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\hostname\\share\\..\\xxxx\\..\\test')) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "\\..\\xxxx\\..\\test"))) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\BootPartition\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\BootPartition\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\BootPartition\\..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\\\.\\BootPartition\\..\\xxxx\\..\\test')) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "..\\..\\test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "..\\xxxx\\..\\test"))) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', 'C:\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', 'C:\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', 'C:\\..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', 'C:\\..\\xxxx\\..\\test')) + + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '\\..\\xxxx\\..\\test')) + + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', 'test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\', '..\\xxxx\\..\\test')) + + expected = str(realpath(rootfs) / 'Windows' / 'test') + + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\Windows', 'test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\Windows\\System32', '..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\Windows\\System32\\drivers', '..\\..\\test')) + self.assertEqual(expected, nt_to_native(str(rootfs), 'C:\\Windows\\System32', '..\\xxxx\\..\\test')) + + elif is_posix_host: + rootfs = PurePosixPath(r'../examples/rootfs/x86_linux') + + expected = str(realpath(rootfs) / 'test') + + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '/test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '/../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '/../../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '/../xxxx/../test')) + + self.assertEqual(expected, posix_to_native(str(rootfs), '/', 'test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '../../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '../xxxx/../test')) + + expected = str(realpath(rootfs) / 'proc' / 'test') + + self.assertEqual(expected, posix_to_native(str(rootfs), '/proc', 'test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/proc/sys', '../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/proc/sys/net', '../../test')) + self.assertEqual(expected, posix_to_native(str(rootfs), '/proc/sys', '../xxxx/../test')) + + # test virtual symlink: absolute virtual path + rootfs = PurePosixPath(r'../examples/rootfs/arm_linux') + expected = str(realpath(rootfs) / 'tmp' / 'media' / 'nand' / 'symlink_test' / 'libsymlink_test.so') + + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '/lib/libsymlink_test.so')) + + # test virtual symlink: relative virtual path + rootfs = PurePosixPath(r'../examples/rootfs/arm_qnx') + expected = str(realpath(rootfs) / 'lib' / 'libm.so.2') + + self.assertEqual(expected, posix_to_native(str(rootfs), '/', '/usr/lib/libm.so.2')) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/xxxx", "test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/xxxx/yyyy", "..\\test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/xxxx/yyyy/zzzz", "..\\..\\test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/xxxx/yyyy", "..\\xxxx\\..\\test"))) else: - rootfs = pathlib.Path("../examples/rootfs/x8664_linux").resolve() - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "/test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "/../test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "/../../test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "/../xxxx/../test"))) + self.fail('unexpected host os') - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "../test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "../../test"))) - self.assertEqual(str(rootfs / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/", "../xxxx/../test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/xxxx", "test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/xxxx/yyyy", "../test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/xxxx/yyyy/zzzz", "../../test"))) - self.assertEqual(str(rootfs / "xxxx" / "test"), str(QlPathManager.convert_for_native_os(rootfs, "/xxxx/yyyy", "../xxxx/../test"))) - - del ql - -if __name__ == "__main__": +if __name__ == '__main__': unittest.main() diff --git a/tests/test_pe.py b/tests/test_pe.py index ae6bf04f..176bb894 100644 --- a/tests/test_pe.py +++ b/tests/test_pe.py @@ -110,6 +110,7 @@ class PETest(unittest.TestCase): self.assertTrue(QLWinSingleTest(_t).run()) + @unittest.skipIf(IS_FAST_TEST, 'fast test') def test_pe_win_x86_uselessdisk(self): def _t(): class Fake_Drive(QlFsMappedObject): @@ -134,84 +135,84 @@ class PETest(unittest.TestCase): del ql return True - if IS_FAST_TEST: - self.skipTest('QL_FAST_TEST') - self.assertTrue(QLWinSingleTest(_t).run()) + @unittest.skipIf(IS_FAST_TEST, 'fast test') def test_pe_win_x86_gandcrab(self): def _t(): - def stop(ql, default_values): - print("Ok for now") + def stop(ql: Qiling): + ql.log.info("Ok for now") + ql.emu_stop() - def randomize_config_value(ql, key, subkey): - # https://en.wikipedia.org/wiki/Volume_serial_number - # https://www.digital-detective.net/documents/Volume%20Serial%20Numbers.pdf - if key == "VOLUME" and subkey == "serial_number": - month = random.randint(0, 12) - day = random.randint(0, 30) - first = hex(month)[2:] + hex(day)[2:] - seconds = random.randint(0, 60) - milli = random.randint(0, 100) - second = hex(seconds)[2:] + hex(milli)[2:] - first_half = int(first, 16) + int(second, 16) - hour = random.randint(0, 24) - minute = random.randint(0, 60) - third = hex(hour)[2:] + hex(minute)[2:] - year = random.randint(2000, 2020) - second_half = int(third, 16) + year - result = int(hex(first_half)[2:] + hex(second_half)[2:], 16) - ql.os.profile[key][subkey] = str(result) - elif key == "USER" and subkey == "username": - length = random.randint(0, 15) - new_name = "" - for i in range(length): - new_name += random.choice(st.ascii_lowercase + st.ascii_uppercase) - old_name = ql.os.profile[key][subkey] - # update paths - ql.os.profile[key][subkey] = new_name - for path in ql.os.profile["PATH"]: - val = ql.os.profile["PATH"][path].replace(old_name, new_name) - ql.os.profile["PATH"][path] = val - elif key == "SYSTEM" and subkey == "computername": - length = random.randint(0, 15) - new_name = "" - for i in range(length): - new_name += random.choice(st.ascii_lowercase + st.ascii_uppercase) - ql.os.profile[key][subkey] = new_name - else: - raise QlErrorNotImplemented("API not implemented") + def __rand_serialnum() -> str: + """ + see: https://en.wikipedia.org/wiki/Volume_serial_number + see: https://www.digital-detective.net/documents/Volume%20Serial%20Numbers.pdf + """ + + mon = random.randint(1, 12) + day = random.randint(0, 30) + word1 = (mon << 8) + day + + sec = random.randint(0, 59) + ms = random.randint(0, 99) + word2 = (sec << 8) + ms + + unified1 = word1 + word2 + + hrs = random.randint(0, 23) + mins = random.randint(0, 59) + word1 = (hrs << 8) + mins + + yr = random.randint(2000, 2020) + word2 = yr + + unified2 = word1 + word2 + + return f'{unified1:04x}-{unified2:04x}' + + def __rand_name(minlen: int, maxlen: int) -> str: + name_len = random.randint(minlen, maxlen) + + return ''.join(random.choices(st.ascii_lowercase + st.ascii_uppercase, k=name_len)) + ql = Qiling(["../examples/rootfs/x86_windows/bin/GandCrab502.bin"], "../examples/rootfs/x86_windows", verbose=QL_VERBOSE.DEBUG, profile="profiles/windows_gandcrab_admin.ql") - default_user = ql.os.profile["USER"]["username"] - default_computer = ql.os.profile["SYSTEM"]["computername"] - ql.hook_address(stop, 0x40860f, user_data=(default_user, default_computer)) - randomize_config_value(ql, "USER", "username") - randomize_config_value(ql, "SYSTEM", "computername") - randomize_config_value(ql, "VOLUME", "serial_number") - num_syscalls_admin = ql.os.stats.syscalls_counter + ql.hook_address(stop, 0x40860f) + + # randomize username + old_uname = ql.os.profile['USER']['username'] + new_uname = __rand_name(3, 10) + + # update paths accordingly + path_key = ql.os.profile['PATH'] + + for p in path_key: + path_key[p] = path_key[p].replace(old_uname, new_uname) + + ql.os.profile['USER']['username'] = new_uname + + # randomize computer name and serial number + ql.os.profile['SYSTEM']['computername'] = __rand_name(5, 15) + ql.os.profile['VOLUME']['serial_number'] = __rand_serialnum() + ql.run() + num_syscalls_admin = ql.os.stats.position del ql # RUN AS USER ql = Qiling(["../examples/rootfs/x86_windows/bin/GandCrab502.bin"], "../examples/rootfs/x86_windows", profile="profiles/windows_gandcrab_user.ql") ql.run() - num_syscalls_user = ql.os.stats.syscalls_counter + num_syscalls_user = ql.os.stats.position + del ql # let's check that gandcrab behave takes a different path if a different environment is found - if num_syscalls_admin == num_syscalls_user: - return False - - del ql - return True - - if IS_FAST_TEST: - self.skipTest('QL_FAST_TEST') + return num_syscalls_admin != num_syscalls_user self.assertTrue(QLWinSingleTest(_t).run()) @@ -326,6 +327,7 @@ class PETest(unittest.TestCase): self.assertTrue(QLWinSingleTest(_t).run()) + @unittest.skipIf(IS_FAST_TEST, 'fast test') def test_pe_win_x86_wannacry(self): def _t(): def stop(ql): @@ -340,9 +342,6 @@ class PETest(unittest.TestCase): del ql return True - if IS_FAST_TEST: - self.skipTest('QL_FAST_TEST') - self.assertTrue(QLWinSingleTest(_t).run()) @@ -358,6 +357,7 @@ class PETest(unittest.TestCase): self.assertTrue(QLWinSingleTest(_t).run()) + @unittest.skipIf(IS_FAST_TEST, 'fast test') def test_pe_win_al_khaser(self): def _t(): ql = Qiling(["../examples/rootfs/x86_windows/bin/al-khaser.bin"], "../examples/rootfs/x86_windows") @@ -387,9 +387,6 @@ class PETest(unittest.TestCase): del ql return True - if IS_FAST_TEST: - self.skipTest('QL_FAST_TEST') - self.assertTrue(QLWinSingleTest(_t).run()) diff --git a/tests/test_pe_sys.py b/tests/test_pe_sys.py index c608f205..f60c58ff 100644 --- a/tests/test_pe_sys.py +++ b/tests/test_pe_sys.py @@ -72,40 +72,14 @@ class PETest(unittest.TestCase): }) def hook_CreateFileA(ql: Qiling, address: int, params): lpFileName = params["lpFileName"] + if lpFileName.startswith("\\\\.\\"): - if ql.amsint32_driver: + if hasattr(ql, 'amsint32_driver'): return 0x13371337 - else: - return (-1) - else: - ret = _CreateFile(ql, address, params) - return ret + return -1 - def _WriteFile(ql: Qiling, address: int, params): - ret = 1 - hFile = params["hFile"] - lpBuffer = params["lpBuffer"] - nNumberOfBytesToWrite = params["nNumberOfBytesToWrite"] - lpNumberOfBytesWritten = params["lpNumberOfBytesWritten"] - #lpOverlapped = params["lpOverlapped"] - - if hFile == 0xfffffff5: - s = ql.mem.read(lpBuffer, nNumberOfBytesToWrite) - ql.os.stdout.write(s) - ql.os.stats.log_string(s.decode()) - ql.mem.write_ptr(lpNumberOfBytesWritten, nNumberOfBytesToWrite) - else: - f = ql.os.handle_manager.get(hFile) - if f is None: - # Invalid handle - ql.os.last_error = 0xffffffff - return 0 - - buffer = ql.mem.read(lpBuffer, nNumberOfBytesToWrite) - f.obj.write(bytes(buffer)) - ql.mem.write_ptr(lpNumberOfBytesWritten, nNumberOfBytesToWrite, 4) - return ret + return _CreateFile(ql, address, params) @winsdkapi(cc=STDCALL, params={ 'hFile' : HANDLE, @@ -120,21 +94,31 @@ class PETest(unittest.TestCase): nNumberOfBytesToWrite = params["nNumberOfBytesToWrite"] lpNumberOfBytesWritten = params["lpNumberOfBytesWritten"] - if hFile == 0x13371337: - buffer = ql.mem.read(lpBuffer, nNumberOfBytesToWrite) - try: - r, nNumberOfBytesToWrite = utils.io_Write(ql.amsint32_driver, buffer) - ql.mem.write_ptr(lpNumberOfBytesWritten, nNumberOfBytesToWrite, 4) - except Exception: - print("Error") - r = 1 - if r: - return 1 - else: - return 0 - else: - return _WriteFile(ql, address, params) + r = 1 + buffer = ql.mem.read(lpBuffer, nNumberOfBytesToWrite) + if hFile == 0x13371337: + success, nNumberOfBytesToWrite = utils.io_Write(ql.amsint32_driver, buffer) + r = int(success) + + elif hFile == 0xfffffff5: + s = buffer.decode() + + ql.os.stdout.write(s) + ql.os.stats.log_string(s) + + else: + f = ql.os.handle_manager.get(hFile) + + if f is None: + ql.os.last_error = 0xffffffff + return 0 + + f.obj.write(bytes(buffer)) + + ql.mem.write_ptr(lpNumberOfBytesWritten, nNumberOfBytesToWrite, 4) + + return r # BOOL StartServiceA( # SC_HANDLE hService, @@ -147,78 +131,88 @@ class PETest(unittest.TestCase): 'lpServiceArgVectors' : POINTER }) def hook_StartServiceA(ql: Qiling, address: int, params): + ql.test_set_api = True + hService = params["hService"] service_handle = ql.os.handle_manager.get(hService) - ql.test_set_api = True - if service_handle.name == "amsint32": - if service_handle.name in ql.os.services: - service_path = ql.os.services[service_handle.name] - service_path = ql.os.path.transform_to_real_path(service_path) - ql.amsint32_driver = Qiling([service_path], ql.rootfs, verbose=QL_VERBOSE.DEBUG) - init_unseen_symbols(ql.amsint32_driver, ql.amsint32_driver.loader.dlls["ntoskrnl.exe"]+0xb7695, b"NtTerminateProcess", 0, "ntoskrnl.exe") - print("load amsint32_driver") - try: - ql.amsint32_driver.run() - return 1 - except UcError as e: - print("Load driver error: ", e) - return 0 - else: - return 0 + if service_handle.name != "amsint32": + return 1 + + if service_handle.name not in ql.os.services: + return 0 + + service_path = ql.os.services[service_handle.name] + service_path = ql.os.path.transform_to_real_path(service_path) + + amsint32 = Qiling([service_path], ql.rootfs, verbose=QL_VERBOSE.DEBUG) + ntoskrnl = amsint32.loader.get_image_by_name("ntoskrnl.exe") + self.assertIsNotNone(ntoskrnl) + + init_unseen_symbols(amsint32, ntoskrnl.base + 0xb7695, b"NtTerminateProcess", 0, "ntoskrnl.exe") + amsint32.log.info('Loading amsint32 driver') + + setattr(ql, 'amsint32_driver', amsint32) + + try: + amsint32.run() + except UcError as e: + print("Load driver error: ", e) + return 0 else: return 1 def hook_first_stop_address(ql: Qiling, stops: List[bool]): - ql.log.info(f' >>>> First Stop address: {ql.arch.regs.arch_pc:#010x}') + ql.log.info(f' >>>> First stop address: {ql.arch.regs.arch_pc:#010x}') stops[0] = True ql.emu_stop() def hook_second_stop_address(ql: Qiling, stops: List[bool]): - ql.log.info(f' >>>> Second Stop address: {ql.arch.regs.arch_pc:#010x}') + ql.log.info(f' >>>> Second stop address: {ql.arch.regs.arch_pc:#010x}') stops[1] = True ql.emu_stop() def hook_third_stop_address(ql: Qiling, stops: List[bool]): - ql.log.info(f' >>>> Third Stop address: {ql.arch.regs.arch_pc:#010x}') + ql.log.info(f' >>>> Third stop address: {ql.arch.regs.arch_pc:#010x}') stops[2] = True ql.emu_stop() stops = [False, False, False] ql = Qiling(["../examples/rootfs/x86_windows/bin/sality.dll"], "../examples/rootfs/x86_windows", verbose=QL_VERBOSE.DEBUG) - # for this module - ql.amsint32_driver = None + # emulate some Windows API ql.os.set_api("CreateThread", hook_CreateThread) ql.os.set_api("CreateFileA", hook_CreateFileA) ql.os.set_api("WriteFile", hook_WriteFile) ql.os.set_api("StartServiceA", hook_StartServiceA) - #init sality + + # run until first stop ql.hook_address(hook_first_stop_address, 0x40EFFB, stops) ql.run() - # run driver thread # execution is about to resume from 0x4053B2, which essentially jumps to ExitThread (kernel32.dll). # Set ExitThread exit code to 0 - ql.os.fcall = ql.os.fcall_select(STDCALL) - ql.os.fcall.writeParams(((DWORD, 0),)) + fcall = ql.os.fcall_select(STDCALL) + fcall.writeParams(((DWORD, 0),)) + # run until second stop ql.hook_address(hook_second_stop_address, 0x4055FA, stops) ql.run(begin=0x4053B2) - print("test kill thread") - if ql.amsint32_driver: - utils.io_Write(ql.amsint32_driver, ql.pack32(0xdeadbeef)) - # TODO: Should stop at 0x10423, but for now just stop at 0x0001066a - stop_addr = 0x0001066a - ql.amsint32_driver.hook_address(hook_third_stop_address, stop_addr, stops) + # asmint32 driver should have been initialized by now. otherwise we get an exception + amsint32: Qiling = getattr(ql, 'amsint32_driver') - # TODO: not sure whether this one is really STDCALL - ql.amsint32_driver.os.fcall = ql.amsint32_driver.os.fcall_select(STDCALL) - ql.amsint32_driver.os.fcall.writeParams(((DWORD, 0),)) + utils.io_Write(amsint32, ql.pack32(0xdeadbeef)) - ql.amsint32_driver.run(begin=0x102D0) + # TODO: not sure whether this one is really STDCALL + fcall = amsint32.os.fcall_select(STDCALL) + fcall.writeParams(((DWORD, 0),)) + + # run until third stop + # TODO: Should stop at 0x10423, but for now just stop at 0x0001066a + amsint32.hook_address(hook_third_stop_address, 0x0001066a, stops) + amsint32.run(begin=0x102D0) self.assertTrue(stops[0]) self.assertTrue(stops[1]) diff --git a/tests/test_qdb.py b/tests/test_qdb.py index 077ecad9..0a0da506 100644 --- a/tests/test_qdb.py +++ b/tests/test_qdb.py @@ -3,9 +3,10 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import unittest -from qiling import Qiling +import sys, unittest +sys.path.append("..") +from qiling import Qiling class DebuggerTest(unittest.TestCase):