From a42fb2857955165f1c9e789d1888293e0a2368ea Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 16:02:08 +0800 Subject: [PATCH 01/21] Bump version to 1.4.11-dev and update classifiers --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index efaa4734..afa35a8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] urls = { homepage = "https://qiling.io" } name = "qiling" -version = "1.4.10" +version = "1.4.11-dev" description = "Qiling is an advanced binary emulation framework that cross-platform-architecture" authors = ["KaiJern Lau (xwings) "] license = "GPLv2" readme = "README.md" classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 3 - Beta", "Intended Audience :: Developers", "Topic :: Software Development :: Disassemblers", "Topic :: System :: Emulators", From 012726f27236ef8bf8f15af0b801d189cb67bfd2 Mon Sep 17 00:00:00 2001 From: retrocpugeek Date: Wed, 24 Jun 2026 23:47:52 +1000 Subject: [PATCH 02/21] Align getdents64 records to fix unaligned access on MIPS getdents64 wrote linux_dirent64 records with d_reclen = header + name, without rounding up to the alignment of the leading u64 d_ino. The next record's d_ino then lands unaligned, and a strict-alignment guest (e.g. MIPS/MIPS64) faults with UC_ERR_READ_UNALIGNED when walking the buffer. x86 tolerates the unaligned load, which is why this was never caught. Round d_reclen up to the d_ino alignment, matching the kernel. The rounding is scoped to the getdents64 (is_64) branch only: getdents64 places d_type before d_name so the trailing pad bytes are inert, whereas the legacy getdents layout stores d_type at offset d_reclen-1 and would break if padded the same way -- that is what got the earlier blanket fix (PR #1419) reverted. Co-Authored-By: Claude Opus 4.8 (1M context) --- qiling/os/posix/syscall/unistd.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/qiling/os/posix/syscall/unistd.py b/qiling/os/posix/syscall/unistd.py index 09b86c38..8076dcd7 100644 --- a/qiling/os/posix/syscall/unistd.py +++ b/qiling/os/posix/syscall/unistd.py @@ -936,6 +936,18 @@ def __getdents_common(ql: Qiling, fd: int, dirp: int, count: int, *, is_64: bool packed_d_ino = (ql.pack64(d_ino), n) if is_64: + # The kernel rounds each linux_dirent64 record up to the + # alignment of its leading u64 d_ino, so the next record's d_ino + # stays aligned. Without this, strict-alignment guests (e.g. + # MIPS) fault with an unaligned load when walking the buffer; + # x86 tolerates it, which is why it was never caught. This is + # safe here because getdents64 places d_type *before* d_name, so + # the trailing pad bytes are inert. The legacy getdents layout + # below stores d_type at offset d_reclen-1, so it can't be + # padded the same way without relocating d_type -- that mistake + # is what got the earlier blanket fix (PR #1419) reverted. + d_reclen = (d_reclen + (n - 1)) & ~(n - 1) + fields = ( (ql.pack64(d_ino), n), (ql.pack64(d_off), n), From 0e48da4be89d1d8a1708cef238c081840ecc917a Mon Sep 17 00:00:00 2001 From: retrocpugeek Date: Thu, 25 Jun 2026 00:31:16 +1000 Subject: [PATCH 03/21] Add regression test for getdents64 record alignment Calls ql_syscall_getdents64 on a directory and walks the returned linux_dirent64 records, asserting every record (and thus its leading u64 d_ino) starts on an 8-byte boundary. Before the fix d_reclen was not rounded up, so records were misaligned and a strict-alignment guest (e.g. MIPS) faulted with an unaligned load. Self-contained; runs on stock unicorn. Co-Authored-By: Claude Opus 4.8 (1M context) --- tests/test_elf.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/test_elf.py b/tests/test_elf.py index 2798028b..052ba80b 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -725,6 +725,43 @@ class ELFTest(unittest.TestCase): del ql + # Regression for getdents64 record alignment. Each linux_dirent64 record + # must be padded so the next record's leading u64 d_ino stays 8-byte + # aligned; otherwise a strict-alignment guest (e.g. MIPS) faults with an + # unaligned load while walking the buffer. x86 tolerates the unaligned read, + # which is why it was never caught. + def test_linux_getdents64_alignment(self): + from qiling.const import QL_ENDIAN + from qiling.os.posix.syscall.fcntl import ql_syscall_open + from qiling.os.posix.syscall.unistd import ql_syscall_getdents64 + + ql = Qiling(code=b"\x00\x00\x00\x00", archtype=QL_ARCH.MIPS, ostype=QL_OS.LINUX, + endian=QL_ENDIAN.EB, rootfs="../examples/rootfs/mips32_linux", + verbose=QL_VERBOSE.OFF) + + base = 0x100000 + ql.mem.map(base, 0x4000) + path_ptr, buf_ptr = base, base + 0x100 + ql.mem.write(path_ptr, b"/\x00") + + fd = ql_syscall_open(ql, path_ptr, 0, 0) # O_RDONLY on a directory + self.assertGreaterEqual(fd, 0) + + nbytes = ql_syscall_getdents64(ql, fd, buf_ptr, 0x2000) + self.assertGreater(nbytes, 0) + + # walk the linux_dirent64 records; every record (hence its leading u64 + # d_ino) must start at an 8-byte boundary. d_reclen is a u16 at offset 16. + buf = bytes(ql.mem.read(buf_ptr, nbytes)) + off = 0 + while off < nbytes: + self.assertEqual(off % 8, 0, f"dirent at offset {off} is not 8-byte aligned") + d_reclen = int.from_bytes(buf[off + 16:off + 18], "big") + self.assertNotEqual(d_reclen, 0) + off += d_reclen + + del ql + def test_memory_search(self): ql = Qiling(code=b"\xCC", archtype=QL_ARCH.X8664, ostype=QL_OS.LINUX, verbose=QL_VERBOSE.DEBUG) From ed25f7319fcc2b48559122e8688ecaffe184c10f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mikk=20Margus=20M=C3=B6ll?= Date: Thu, 25 Jun 2026 09:38:07 +0300 Subject: [PATCH 04/21] implement FlsGetValue2 --- qiling/os/windows/dlls/kernel32/fibersapi.py | 9 +++++++++ qiling/os/windows/fiber.py | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/qiling/os/windows/dlls/kernel32/fibersapi.py b/qiling/os/windows/dlls/kernel32/fibersapi.py index 1623fb11..f6df3d13 100644 --- a/qiling/os/windows/dlls/kernel32/fibersapi.py +++ b/qiling/os/windows/dlls/kernel32/fibersapi.py @@ -25,6 +25,15 @@ def hook_FlsFree(ql: Qiling, address: int, params): def hook_FlsGetValue(ql: Qiling, address: int, params): return ql.os.fiber_manager.get(params['dwFlsIndex']) +# LPVOID FlsGetValue2( +# DWORD dwFlsIndex +# ); +@winsdkapi(cc=STDCALL, params={ + 'dwFlsIndex' : DWORD +}) +def hook_FlsGetValue2(ql: Qiling, address: int, params): + return ql.os.fiber_manager.get_noerr(params['dwFlsIndex']) + # BOOL FlsSetValue( # DWORD dwFlsIndex # PVOID lpFlsData diff --git a/qiling/os/windows/fiber.py b/qiling/os/windows/fiber.py index 09862076..674153bd 100644 --- a/qiling/os/windows/fiber.py +++ b/qiling/os/windows/fiber.py @@ -112,3 +112,10 @@ class FiberManager: return 0 return self.fibers[idx].data + + # equivalent to get, but does not set errors; used for FlsGetValue2 + def get_noerr(self, idx: int) -> int: + if idx not in self.fibers: + return 0 + + return self.fibers[idx].data From f5c0afab5826d04132f37d986a1eed1069cf48a2 Mon Sep 17 00:00:00 2001 From: retrocpugeek Date: Thu, 25 Jun 2026 20:22:02 +1000 Subject: [PATCH 05/21] Implement the clone3 syscall Modern glibc (recent toolchains) issues clone3 from pthread_create rather than clone, so thread creation fails on any guest built against it: Qiling does not return -ENOSYS for unimplemented syscalls (it logs a warning and leaves the return register untouched), so glibc's clone3->clone fallback never fires. Add ql_syscall_clone3, which unpacks struct clone_args and delegates to the existing clone() handler. Translations: child_stack = stack + stack_size (clone3 passes the stack base plus a size; legacy clone wants the highest address), exit_signal folded into the flags' CSIGNAL byte, and an x8664 pre-swap that cancels ql_syscall_clone's arch-specific newtls<->child_tidptr swap. Add a self-contained regression test (test_clone3_translates_to_clone) that drives ql_syscall_clone3 directly and asserts the translation for both the generic path and the x8664 swap. Runs on stock unicorn; no clone3 binary needed. Co-Authored-By: Claude Opus 4.8 (1M context) --- qiling/os/posix/syscall/sched.py | 32 ++++++++++++++++++++ tests/test_elf_multithread.py | 52 +++++++++++++++++++++++++++++++- 2 files changed, 83 insertions(+), 1 deletion(-) diff --git a/qiling/os/posix/syscall/sched.py b/qiling/os/posix/syscall/sched.py index a7c297c1..b2073d17 100644 --- a/qiling/os/posix/syscall/sched.py +++ b/qiling/os/posix/syscall/sched.py @@ -130,6 +130,38 @@ def ql_syscall_clone(ql: Qiling, flags: int, child_stack: int, parent_tidptr: in return regreturn +def ql_syscall_clone3(ql: Qiling, cl_args: int, size: int): + # clone3(struct clone_args *uargs, size_t size). Translate the struct into + # the legacy clone() argument set and delegate. Modern glibc (used by recent + # toolchains) issues clone3 from pthread_create; without this it falls through + # to the "not implemented" path and thread creation fails. + # + # struct clone_args (all fields __aligned_u64): + # 0:flags 8:pidfd 16:child_tid 24:parent_tid 32:exit_signal + # 40:stack 48:stack_size 56:tls 64:set_tid 72:set_tid_size 80:cgroup + flags = ql.mem.read_ptr(cl_args + 0, 8) + child_tid = ql.mem.read_ptr(cl_args + 16, 8) + parent_tid = ql.mem.read_ptr(cl_args + 24, 8) + exit_signal = ql.mem.read_ptr(cl_args + 32, 8) + stack = ql.mem.read_ptr(cl_args + 40, 8) + stack_size = ql.mem.read_ptr(cl_args + 48, 8) + tls = ql.mem.read_ptr(cl_args + 56, 8) + + # legacy clone() takes the highest stack address; clone3 gives the base + size + child_stack = (stack + stack_size) if stack else 0 + + # clone3 keeps exit_signal in its own field; legacy clone packs it into the + # low CSIGNAL byte of flags + flags |= exit_signal & 0xff + + # ql_syscall_clone swaps newtls<->child_tidptr for x8664 to undo that arch's + # raw-syscall register order. clone3 hands us already-logical args, so on + # x8664 we pre-swap to cancel that out. + if ql.arch.type == QL_ARCH.X8664: + return ql_syscall_clone(ql, flags, child_stack, parent_tid, child_tid, tls) + + return ql_syscall_clone(ql, flags, child_stack, parent_tid, tls, child_tid) + def ql_syscall_sched_yield(ql: Qiling): def _sched_yield(cur_thread): gevent.sleep(0) diff --git a/tests/test_elf_multithread.py b/tests/test_elf_multithread.py index 8923efa1..8fedeff4 100644 --- a/tests/test_elf_multithread.py +++ b/tests/test_elf_multithread.py @@ -17,9 +17,10 @@ from typing import List sys.path.append("..") from qiling import Qiling from qiling.arch.models import X86_CPU_MODEL -from qiling.const import QL_VERBOSE, QL_INTERCEPT +from qiling.const import QL_VERBOSE, QL_INTERCEPT, QL_ARCH, QL_OS from qiling.os.filestruct import ql_file from qiling.os.stats import QlOsNullStats +import qiling.os.posix.syscall.sched as ql_sched BASE_ROOTFS = r'../examples/rootfs' @@ -201,6 +202,55 @@ class ELFTest(unittest.TestCase): self.assertTrue(logged[-2].startswith('thread 1 ret val is')) self.assertTrue(logged[-1].startswith('thread 2 ret val is')) + def test_clone3_translates_to_clone(self): + # clone3(struct clone_args *, size) must unpack the struct and delegate + # to the legacy clone() handler with translated args. Modern glibc issues + # clone3 from pthread_create, so without this thread creation fails. + # This drives ql_syscall_clone3 directly (stock unicorn, no clone3 binary + # needed) and asserts the translation. + CL = 0x100000 + FLAGS, CHILD_TID, PARENT_TID = 0x00010f00, 0x222000, 0x223000 + EXIT_SIGNAL, STACK, STACK_SIZE, TLS = 0x11, 0x7ff00000, 0x8000, 0x224000 + + def run_case(archtype: QL_ARCH, rootfs: str): + ql = Qiling(code=b"\x00\x00\x00\x00", archtype=archtype, ostype=QL_OS.LINUX, + rootfs=rootfs, verbose=QL_VERBOSE.OFF) + ql.mem.map(CL, 0x1000) + for off, val in ((0, FLAGS), (16, CHILD_TID), (24, PARENT_TID), + (32, EXIT_SIGNAL), (40, STACK), (48, STACK_SIZE), (56, TLS)): + ql.mem.write_ptr(CL + off, val, 8) + + captured = {} + + def fake_clone(ql, flags, child_stack, parent_tidptr, newtls, child_tidptr): + captured.update(flags=flags, child_stack=child_stack, parent_tidptr=parent_tidptr, + newtls=newtls, child_tidptr=child_tidptr) + return 0xabc + + orig = ql_sched.ql_syscall_clone + ql_sched.ql_syscall_clone = fake_clone + try: + ret = ql_sched.ql_syscall_clone3(ql, CL, 88) + finally: + ql_sched.ql_syscall_clone = orig + + return captured, ret + + # generic path (no x8664 register swap) + cap, ret = run_case(QL_ARCH.ARM, fr'{ARM_LINUX_ROOTFS}') + self.assertEqual(ret, 0xabc) + self.assertEqual(cap['child_stack'], STACK + STACK_SIZE) # base + size -> stack top + self.assertEqual(cap['flags'], FLAGS | EXIT_SIGNAL) # exit_signal folded in + self.assertEqual(cap['parent_tidptr'], PARENT_TID) + self.assertEqual(cap['newtls'], TLS) + self.assertEqual(cap['child_tidptr'], CHILD_TID) + + # x8664: clone3 pre-swaps newtls<->child_tid so ql_syscall_clone's own + # x8664 swap cancels out to the same logical mapping + cap, _ = run_case(QL_ARCH.X8664, fr'{X64_LINUX_ROOTFS}') + self.assertEqual(cap['newtls'], CHILD_TID) + self.assertEqual(cap['child_tidptr'], TLS) + def test_tcp_elf_linux_x86(self): logged: List[str] = [] From a4ca725e4ee9553df2de9b4b8aad7fa09d787392 Mon Sep 17 00:00:00 2001 From: retrocpugeek Date: Thu, 25 Jun 2026 21:55:21 +1000 Subject: [PATCH 06/21] Align legacy getdents records too Extend the record alignment to the legacy getdents path. The kernel rounds both getdents and getdents64 records up to the alignment of their leading d_ino (ALIGN(reclen, sizeof(long))); without it a strict-alignment guest (MIPS) faults walking the buffer, e.g. older glibc busybox 'ls' (which uses the legacy getdents syscall) crashes on directory listings. Legacy linux_dirent stores d_type in the record's last byte (offset d_reclen-1), so the alignment padding goes between d_name and d_type to keep d_type there -- padding without relocating d_type is what got the earlier blanket attempt (PR #1419) reverted. getdents64 (d_type before d_name) is unchanged. Add test_linux_getdents_alignment: walks legacy getdents records on a big-endian MIPS guest and asserts each is aligned, tiles the buffer, and keeps d_type (DT_DIR for '.'/'..'). Runs on stock unicorn. Co-Authored-By: Claude Opus 4.8 (1M context) --- qiling/os/posix/syscall/unistd.py | 31 +++++++++++--------- tests/test_elf.py | 47 +++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/qiling/os/posix/syscall/unistd.py b/qiling/os/posix/syscall/unistd.py index 8076dcd7..62cb6f61 100644 --- a/qiling/os/posix/syscall/unistd.py +++ b/qiling/os/posix/syscall/unistd.py @@ -926,7 +926,15 @@ def __getdents_common(ql: Qiling, fd: int, dirp: int, count: int, *, is_64: bool d_off = 0 d_name = (result.name if isinstance(result, os.DirEntry) else result._str).encode() + b'\x00' d_type = _type_mapping(result) - d_reclen = n + n + 2 + len(d_name) + 1 + + # The kernel rounds each record up to the alignment of its leading + # (pointer-sized) d_ino, so the next record's d_ino stays aligned. + # Without this, strict-alignment guests (e.g. MIPS) fault with an + # unaligned load when walking the buffer; x86 tolerates it, which is + # why it was never caught. Matches the kernel's ALIGN(reclen, + # sizeof(long)) for both getdents and getdents64. + unaligned_reclen = n + n + 2 + len(d_name) + 1 + d_reclen = (unaligned_reclen + (n - 1)) & ~(n - 1) # TODO: Dirty fix for X8664 MACOS 11.6 APFS # For some reason MACOS return int value is 64bit @@ -936,18 +944,8 @@ def __getdents_common(ql: Qiling, fd: int, dirp: int, count: int, *, is_64: bool packed_d_ino = (ql.pack64(d_ino), n) if is_64: - # The kernel rounds each linux_dirent64 record up to the - # alignment of its leading u64 d_ino, so the next record's d_ino - # stays aligned. Without this, strict-alignment guests (e.g. - # MIPS) fault with an unaligned load when walking the buffer; - # x86 tolerates it, which is why it was never caught. This is - # safe here because getdents64 places d_type *before* d_name, so - # the trailing pad bytes are inert. The legacy getdents layout - # below stores d_type at offset d_reclen-1, so it can't be - # padded the same way without relocating d_type -- that mistake - # is what got the earlier blanket fix (PR #1419) reverted. - d_reclen = (d_reclen + (n - 1)) & ~(n - 1) - + # getdents64 places d_type *before* d_name, so the trailing pad + # bytes are inert. fields = ( (ql.pack64(d_ino), n), (ql.pack64(d_off), n), @@ -956,11 +954,18 @@ def __getdents_common(ql: Qiling, fd: int, dirp: int, count: int, *, is_64: bool (d_name, len(d_name)) ) else: + # legacy linux_dirent stores d_type in the record's last byte + # (offset d_reclen-1), so the alignment padding goes *between* + # d_name and d_type to keep d_type there. padding without + # relocating d_type is what got the earlier blanket fix + # (PR #1419) reverted. + pad = d_reclen - unaligned_reclen fields = ( packed_d_ino, (ql.pack(d_off), n), (ql.pack16(d_reclen), 2), (d_name, len(d_name)), + (b'\x00' * pad, pad), (d_type, 1) ) diff --git a/tests/test_elf.py b/tests/test_elf.py index 052ba80b..42f70267 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -762,6 +762,53 @@ class ELFTest(unittest.TestCase): del ql + def test_linux_getdents_alignment(self): + # the legacy getdents record was not rounded up to the alignment of its + # word-sized d_ino either, so a strict-alignment guest faulted walking + # the buffer. unlike getdents64, legacy linux_dirent keeps d_type in the + # record's last byte (offset d_reclen-1), so the padding must sit between + # d_name and d_type -- verify the records are aligned AND d_type survives. + from qiling.const import QL_ENDIAN + from qiling.os.posix.syscall.fcntl import ql_syscall_open + from qiling.os.posix.syscall.unistd import ql_syscall_getdents + + ql = Qiling(code=b"\x00\x00\x00\x00", archtype=QL_ARCH.MIPS, ostype=QL_OS.LINUX, + endian=QL_ENDIAN.EB, rootfs="../examples/rootfs/mips32_linux", + verbose=QL_VERBOSE.OFF) + + base = 0x100000 + ql.mem.map(base, 0x4000) + path_ptr, buf_ptr = base, base + 0x100 + ql.mem.write(path_ptr, b"/\x00") + + fd = ql_syscall_open(ql, path_ptr, 0, 0) # O_RDONLY on a directory + self.assertGreaterEqual(fd, 0) + + nbytes = ql_syscall_getdents(ql, fd, buf_ptr, 0x2000) + self.assertGreater(nbytes, 0) + + # walk the linux_dirent records; on 32-bit MIPS d_ino is 4 bytes, so each + # record must start at a 4-byte boundary. d_reclen is a u16 at offset 8 + # (after the 4-byte d_ino + 4-byte d_off); d_name starts at offset 10. + buf = bytes(ql.mem.read(buf_ptr, nbytes)) + off = 0 + while off < nbytes: + self.assertEqual(off % 4, 0, f"dirent at offset {off} is not 4-byte aligned") + d_reclen = int.from_bytes(buf[off + 8:off + 10], "big") + self.assertNotEqual(d_reclen, 0) + + name = buf[off + 10:buf.index(b"\x00", off + 10)].decode() + # legacy getdents stores d_type in the record's last byte; '.' and + # '..' must still report DT_DIR (4) after the alignment padding + if name in (".", ".."): + self.assertEqual(buf[off + d_reclen - 1], 4, f"{name}: d_type not DT_DIR") + + off += d_reclen + + self.assertEqual(off, nbytes) # records tile the buffer exactly + + del ql + def test_memory_search(self): ql = Qiling(code=b"\xCC", archtype=QL_ARCH.X8664, ostype=QL_OS.LINUX, verbose=QL_VERBOSE.DEBUG) From 9e21eb9d6699ede45382939039c2d04b3635dae2 Mon Sep 17 00:00:00 2001 From: retrocpugeek Date: Thu, 25 Jun 2026 22:38:50 +1000 Subject: [PATCH 07/21] Fix MIPS socket option (SO_*) values linux_mips_socket_options used generic/incorrect values. MIPS has its own SO_* numbering (arch/mips/include/uapi/asm/socket.h) with the buffer/timeout/ type options in the 0x1000 range. The table listed SO_SNDBUF=0x01, SO_RCVBUF=0x02 (should be 0x1001/0x1002), SO_SND/RCVLOWAT and SO_SND/RCVTIMEO_OLD all wrong, SO_RCVLOWAT=0x04 silently collided with SO_REUSEADDR (Enum alias), and SO_OOBINLINE/SO_REUSEPORT were 0x00. A guest setsockopt(SOL_SOCKET, SO_RCVBUF, ...) (e.g. busybox ping) therefore raised 'Could not convert emulated socket option 4098'. Correct all values to the MIPS uapi and add the missing SO_TYPE/SO_ERROR/SO_ACCEPTCONN/SO_PROTOCOL/ SO_DOMAIN. Add test_elf.ELFTest.test_setsockopt_mips_so_rcvbuf: opens a socket on a MIPS guest and asserts setsockopt(SOL_SOCKET, SO_RCVBUF) succeeds. Self-contained, runs on stock unicorn; fails on dev, passes with the fix. Co-Authored-By: Claude Opus 4.8 (1M context) --- qiling/os/posix/const.py | 44 ++++++++++++++++++++++++---------------- tests/test_elf.py | 27 ++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 18 deletions(-) diff --git a/qiling/os/posix/const.py b/qiling/os/posix/const.py index df4c5b58..535cf6d7 100644 --- a/qiling/os/posix/const.py +++ b/qiling/os/posix/const.py @@ -380,24 +380,32 @@ class linux_mips_socket_level(Enum): # https://docs.huihoo.com/doxygen/linux/kernel/3.7/arch_2mips_2include_2uapi_2asm_2socket_8h_source.html # https://github.com/torvalds/linux/blob/master/arch/mips/include/uapi/asm/socket.h class linux_mips_socket_options(Enum): - SO_DEBUG = 0x01 - SO_REUSEADDR = 0x04 - SO_KEEPALIVE = 0x08 - SO_DONTROUTE = 0x10 - SO_BINDTODEVICE = 0x19 - SO_BROADCAST = 0x20 - SO_LINGER = 0x80 - SO_OOBINLINE = 0x00 - SO_REUSEPORT = 0x00 - SO_SNDBUF = 0x01 - SO_RCVBUF = 0x02 - SO_SNDLOWAT = 0x03 - SO_RCVLOWAT = 0x04 - SO_SNDTIMEO_OLD = 0x05 - SO_RCVTIMEO_OLD = 0x06 - SO_TIMESTAMP_OLD = 0x1d - # SO_TIMESTAMPNS_OLD = 0x23 - # SO_TIMESTAMPING_OLD = 0x25 + # MIPS uses its own SO_* numbering (arch/mips/include/uapi/asm/socket.h); + # the buffer/timeout/type options live in the 0x1000 range, unlike the + # generic asm-generic values. + SO_DEBUG = 0x0001 + SO_REUSEADDR = 0x0004 + SO_KEEPALIVE = 0x0008 + SO_DONTROUTE = 0x0010 + SO_BINDTODEVICE = 0x0019 + SO_BROADCAST = 0x0020 + SO_LINGER = 0x0080 + SO_OOBINLINE = 0x0100 + SO_REUSEPORT = 0x0200 + SO_TYPE = 0x1008 + SO_ERROR = 0x1007 + SO_SNDBUF = 0x1001 + SO_RCVBUF = 0x1002 + SO_SNDLOWAT = 0x1003 + SO_RCVLOWAT = 0x1004 + SO_SNDTIMEO_OLD = 0x1005 + SO_RCVTIMEO_OLD = 0x1006 + SO_ACCEPTCONN = 0x1009 + SO_PROTOCOL = 0x1028 + SO_DOMAIN = 0x1029 + SO_TIMESTAMP_OLD = 0x001d + # SO_TIMESTAMPNS_OLD = 0x0023 + # SO_TIMESTAMPING_OLD = 0x0025 SO_TIMESTAMP_NEW = 0x3f SO_TIMESTAMPNS_NEW = 0x40 SO_TIMESTAMPING_NEW = 0x41 diff --git a/tests/test_elf.py b/tests/test_elf.py index 2798028b..36287bff 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -404,6 +404,33 @@ class ELFTest(unittest.TestCase): ql.run() del ql + def test_setsockopt_mips_so_rcvbuf(self): + # MIPS uses its own SO_* numbering (SO_RCVBUF = 0x1002, not the generic + # 0x08). The MIPS socket-option table had the wrong values, so e.g. + # busybox ping's setsockopt(SOL_SOCKET, SO_RCVBUF, ...) raised + # NotImplementedError ("Could not convert emulated socket option 4098"). + from qiling.const import QL_ENDIAN + from qiling.os.posix.syscall.socket import ql_syscall_socket, ql_syscall_setsockopt + + ql = Qiling(code=b"\x00\x00\x00\x00", archtype=QL_ARCH.MIPS, ostype=QL_OS.LINUX, + endian=QL_ENDIAN.EB, rootfs="../examples/rootfs/mips32_linux", + verbose=QL_VERBOSE.OFF) + + AF_INET, SOCK_DGRAM = 2, 2 + SOL_SOCKET, SO_RCVBUF = 0xffff, 0x1002 # MIPS values + + fd = ql_syscall_socket(ql, AF_INET, SOCK_DGRAM, 0) + self.assertGreaterEqual(fd, 0) + + base = 0x100000 + ql.mem.map(base, 0x1000) + ql.mem.write_ptr(base, 16384, 4) # requested SO_RCVBUF size + + # must map SO_RCVBUF (0x1002) to the host option and succeed, not raise + self.assertEqual(ql_syscall_setsockopt(ql, fd, SOL_SOCKET, SO_RCVBUF, base, 4), 0) + + del ql + def test_elf_linux_arm_static(self): ql = Qiling(["../examples/rootfs/arm_linux/bin/arm_hello_static"], "../examples/rootfs/arm_linux", verbose=QL_VERBOSE.DEFAULT) all_mem = ql.mem.save() From abc4855388f32536d607b34a06553523e40d9043 Mon Sep 17 00:00:00 2001 From: xwings Date: Thu, 2 Jul 2026 08:52:20 +0800 Subject: [PATCH 08/21] Add documents for AGENTS --- AGENT.md | 1 + ARCHITECTURE.md | 183 +++++++++++++++++++++++++++++++++++ ARCHITECTURE/arch.md | 58 +++++++++++ ARCHITECTURE/cli.md | 48 +++++++++ ARCHITECTURE/core.md | 61 ++++++++++++ ARCHITECTURE/debugger.md | 52 ++++++++++ ARCHITECTURE/extensions.md | 61 ++++++++++++ ARCHITECTURE/hw.md | 53 ++++++++++ ARCHITECTURE/loader.md | 58 +++++++++++ ARCHITECTURE/os-baremetal.md | 52 ++++++++++ ARCHITECTURE/os-base.md | 60 ++++++++++++ ARCHITECTURE/os-posix.md | 58 +++++++++++ ARCHITECTURE/os-windows.md | 59 +++++++++++ CLAUDE.md | 1 + 14 files changed, 805 insertions(+) create mode 120000 AGENT.md create mode 100644 ARCHITECTURE.md create mode 100644 ARCHITECTURE/arch.md create mode 100644 ARCHITECTURE/cli.md create mode 100644 ARCHITECTURE/core.md create mode 100644 ARCHITECTURE/debugger.md create mode 100644 ARCHITECTURE/extensions.md create mode 100644 ARCHITECTURE/hw.md create mode 100644 ARCHITECTURE/loader.md create mode 100644 ARCHITECTURE/os-baremetal.md create mode 100644 ARCHITECTURE/os-base.md create mode 100644 ARCHITECTURE/os-posix.md create mode 100644 ARCHITECTURE/os-windows.md create mode 120000 CLAUDE.md diff --git a/AGENT.md b/AGENT.md new file mode 120000 index 00000000..708f0d2e --- /dev/null +++ b/AGENT.md @@ -0,0 +1 @@ +ARCHITECTURE.md \ No newline at end of file diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..56843aed --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,183 @@ +# Qiling Framework — Architecture + +This is the control center for agent-readable architecture docs. Cross-cutting +facts live here; each subsystem is documented once in `ARCHITECTURE/.md` +(see the [Index](#index)). + +## Mission + +Qiling is an advanced binary emulation framework: it emulates and sandboxes +code in an isolated environment across multiple platforms and architectures. +Built on top of Unicorn Engine, it adds what raw CPU emulation lacks — +operating-system context (syscalls, APIs, filesystems, registries), executable +format loading, and dynamic linking (`README.md:13`). + +Supported combinations are defined authoritatively in code: + +- Architectures — `QL_ARCH` (`qiling/const.py:15`): 8086, x86, x86-64, ARM, + ARM64, Cortex-M, MIPS, RISC-V 32/64, PowerPC. +- Operating systems — `QL_OS` (`qiling/const.py:28`): Linux, FreeBSD, macOS, + Windows, UEFI, DOS, QNX, MCU (bare-metal), BLOB. +- Formats: ELF, PE, Mach-O, COM/MBR, Intel HEX / raw firmware. Kernel-module + emulation for Windows `.sys`, Linux `.ko`, macOS `.kext`. + +Headline capabilities: fine-grained instrumentation hooks (instruction, basic +block, memory access, interrupt, syscall/API), VM state save/restore, dynamic +hot patching, cross-architecture debugging (GDB server and the built-in Qdb +with reverse debugging), and fuzzing integration (AFL++/unicornafl). + +## Target Environment + +- **Shape**: pure-Python library (`from qiling import Qiling`), plus the + `qltool` CLI and `qltui.py` TUI. Also shipped as a Docker image + (`Dockerfile`) and PyPI package. +- **Python**: 3.8+ (`pyproject.toml:30`). Packaging via Poetry; version 1.4.8 + (`pyproject.toml:4`), status Beta, license GPLv2. +- **Core dependencies**: `unicorn ==2.1.3` (hard-pinned CPU emulator), + `capstone` (disassembly), `keystone-engine` (assembly), `pefile`, + `pyelftools`, `python-registry`, `gevent` (multithread emulation), + `pyyaml` (MCU profiles). Extras: `fuzz` → unicornafl/fuzzercorn, + `RE` → r2libr. +- **Hosts**: Linux, Windows, macOS. Some test suites are host-gated (Windows + PE tests need real system DLLs collected on Windows; see + [os-windows.md](ARCHITECTURE/os-windows.md)). +- **Fixtures**: `examples/rootfs/` is a git submodule + (https://github.com/qilingframework/rootfs.git) holding target binaries for + tests and examples. + +## Workspace Layout + +| Path | Holds | +| ---- | ----- | +| `qiling/` | The framework package (see Index for per-subsystem docs) | +| `qiling/profiles/` | Default per-OS config profiles (`linux.ql`, `windows.ql`, …) | +| `examples/` | Curated demo scripts, `fuzzing/`, `mcu/`, `shellcodes/`, `extensions/`, `scripts/` (DLL collectors), and the `rootfs/` submodule | +| `jexamples/` | Legacy example set (not covered by CI) | +| `tests/` | CI test suite — standalone `unittest` files run from `tests/` (`tests/test_onlinux.sh`, `tests/test_pe.bat`, `tests/test_macho.sh`) | +| `docs/` | Stubs and assets; real documentation lives at https://docs.qiling.io | +| `qltool` | CLI entry point (see [cli.md](ARCHITECTURE/cli.md)) | +| `qltui.py` | Terminal UI invoked via `qltool qltui` | +| `Dockerfile` | Multi-stage Poetry wheel build on `python:3-slim` | +| `pyproject.toml` / `poetry.lock` | Packaging and dependency pins | +| `.github/workflows/` | CI (`build-ci.yml`: Windows + Ubuntu × Python 3.9/3.11), Docker publish, PyPI publish, Gitee mirror sync | + +## Boot / Entry Flow + +From CLI to emulated instructions (details live in the module docs): + +1. `qltool` parses args and builds kwargs; `run` and `code` subcommands end in + `Qiling(**ql_args)` (`qltool:276`) — see [cli.md](ARCHITECTURE/cli.md). +2. `Qiling.__init__` (`qiling/core.py:35`) is the composition root. Order: + guess arch/OS from the binary if not given → instantiate arch → init + struct/hook mixins → logger → profile → loader → memory manager → OS layer + → hardware manager (bare-metal only) → `loader.run()` maps the target into + emulated memory (`qiling/core.py:154-195`). All name→class resolution is in + `qiling/utils.py` — see [core.md](ARCHITECTURE/core.md). +3. `Qiling.run()` (`qiling/core.py:561`) attaches the debugger if configured, + applies queued patches, writes the exit trap, then delegates to + `os.run()` — see [debugger.md](ARCHITECTURE/debugger.md) and the OS docs. +4. The OS layer drives `Qiling.emu_start` (`qiling/core.py:743`), the thin + wrapper over Unicorn's `uc.emu_start`. Syscalls/APIs raised by the emulated + code are dispatched back into the OS layer + ([os-posix.md](ARCHITECTURE/os-posix.md), + [os-windows.md](ARCHITECTURE/os-windows.md), + [os-baremetal.md](ARCHITECTURE/os-baremetal.md)). + +## Roadmap + +Maturity-based — Qiling is a released project in maintenance/beta (v1.4.8). +There are no in-repo milestones; module Status is `done` when its test suite +proves it. Feature requests and the forward-looking wishlist are tracked in +GitHub issue [#333](https://github.com/qilingframework/qiling/issues/333) +(the `TODO` file is a pointer to it). Known cross-cutting gaps: + +- `ChangeLog` stops at 1.4.6 while `pyproject.toml` says 1.4.8. +- macOS CI job is commented out in `.github/workflows/build-ci.yml`. +- `jexamples/` is legacy and unexercised by CI. + +## Coding Discipline + +Behavioral guidelines to reduce common LLM coding mistakes. Merge with +project-specific instructions as needed. + +**Tradeoff:** These guidelines bias toward caution over speed. For +trivial tasks, use judgment. + +### 1. Think Before Coding + +**Don't assume. Don't hide confusion. Surface tradeoffs.** + +Before implementing: +- State your assumptions explicitly. If uncertain, ask. +- If multiple interpretations exist, present them - don't pick silently. +- If a simpler approach exists, say so. Push back when warranted. +- If something is unclear, stop. Name what's confusing. Ask. + +### 2. Simplicity First + +**Minimum code that solves the problem. Nothing speculative.** + +- No features beyond what was asked. +- No abstractions for single-use code. +- No "flexibility" or "configurability" that wasn't requested. +- No error handling for impossible scenarios. +- If you write 200 lines and it could be 50, rewrite it. + +Ask yourself: "Would a senior engineer say this is overcomplicated?" If +yes, simplify. + +### 3. Surgical Changes + +**Touch only what you must. Clean up only your own mess.** + +When editing existing code: +- Don't "improve" adjacent code, comments, or formatting. +- Don't refactor things that aren't broken. +- Match existing style, even if you'd do it differently. +- If you notice unrelated dead code, mention it - don't delete it. + +When your changes create orphans: +- Remove imports/variables/functions that YOUR changes made unused. +- Don't remove pre-existing dead code unless asked. + +The test: Every changed line should trace directly to the user's request. + +### 4. Goal-Driven Execution + +**Define success criteria. Loop until verified.** + +Transform tasks into verifiable goals: +- "Add validation" → "Write tests for invalid inputs, then make them pass" +- "Fix the bug" → "Write a test that reproduces it, then make it pass" +- "Refactor X" → "Ensure tests pass before and after" + +For multi-step tasks, state a brief plan: + +``` +1. [Step] → verify: [check] +2. [Step] → verify: [check] +3. [Step] → verify: [check] +``` + +Strong success criteria let you loop independently. Weak criteria ("make +it work") require constant clarification. + +--- + +**These guidelines are working if:** fewer unnecessary changes in diffs, +fewer rewrites due to overcomplication, and clarifying questions come +before implementation rather than after mistakes. + +## Index + +- [core.md](ARCHITECTURE/core.md) — the `Qiling` facade, hook engine, component selection, profiles, logging, exceptions +- [arch.md](ARCHITECTURE/arch.md) — CPU layer: registers, Unicorn instance, disassembler, calling conventions +- [loader.md](ARCHITECTURE/loader.md) — binary format loaders: ELF, PE, PE/UEFI, Mach-O, DOS, MCU firmware, raw blobs +- [os-base.md](ARCHITECTURE/os-base.md) — shared OS layer: `QlOs`, memory manager/heap, fcall marshalling, fs mapper, path virtualization +- [os-posix.md](ARCHITECTURE/os-posix.md) — POSIX syscall emulation: Linux, FreeBSD, macOS, QNX +- [os-windows.md](ARCHITECTURE/os-windows.md) — Windows API emulation, UEFI boot/runtime/SMM services, DOS interrupts +- [os-baremetal.md](ARCHITECTURE/os-baremetal.md) — MCU and raw-blob execution modes, cooperative multitasking +- [hw.md](ARCHITECTURE/hw.md) — MMIO peripheral emulation for bare-metal targets, board/chip definitions +- [debugger.md](ARCHITECTURE/debugger.md) — GDB remote-serial server and the built-in Qdb debugger +- [extensions.md](ARCHITECTURE/extensions.md) — AFL fuzzing, coverage/tracing, sanitizers, r2/IDA integration, pipes, reports +- [cli.md](ARCHITECTURE/cli.md) — `qltool` CLI and `qltui.py` TUI diff --git a/ARCHITECTURE/arch.md b/ARCHITECTURE/arch.md new file mode 100644 index 00000000..1bf638cd --- /dev/null +++ b/ARCHITECTURE/arch.md @@ -0,0 +1,58 @@ +# Arch — CPU architecture layer + +## Goal + +Own everything CPU-specific: the Unicorn `Uc` instance, register access, stack +primitives, disassembler/assembler, CPU models, and per-arch calling +conventions. This is the bottom layer — everything else reads `ql.arch`; arch +depends only on Unicorn/Capstone/Keystone. Mature released infrastructure; +maturity-based status. + +## Status + +`done` — all architectures exercised by the CI suite; CPU model selection +covered by `tests/test_cpu_models.py`. + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/arch/arch.py` | Abstract base `QlArch`: owns `uc`, `regs`, stack push/pop, save/restore, disassembler | +| `qiling/arch/x86.py` | `QlArchIntel` base + `QlArchA8086`/`QlArchX86`/`QlArchX8664`, GDT/MSR wiring | +| `qiling/arch/arm.py`, `arm64.py` | ARM/AArch64, thumb handling, coprocessor access | +| `qiling/arch/cortex_m.py` | Cortex-M on top of ARM: NVIC-style interrupt entry/exit for MCU mode | +| `qiling/arch/mips.py`, `riscv.py`, `riscv64.py`, `ppc.py` | Remaining architectures | +| `qiling/arch/register.py` | `QlRegisterManager` — attribute-style register read/write | +| `qiling/arch/models.py` | CPU model enums (`X86_CPU_MODEL` … `RISCV64_CPU_MODEL`) | +| `qiling/arch/msr.py`, `cpr.py`, `cpr64.py` | x86 MSRs, ARM/ARM64 coprocessor registers | +| `qiling/arch/utils.py` | `QlArchUtils`: disassembly output for verbose/trace modes | +| `qiling/cc/__init__.py` + `intel.py`, `arm.py`, `mips.py`, `ppc.py`, `riscv.py` | Calling conventions (arg/retval marshalling) consumed by `os/fcall.py` | + +## Key Types and Entry Points + +- `qiling/arch/arch.py:22` - `QlArch(ABC)` - cached properties `uc` (`:34`), `regs` (`:42`), `stack_push/stack_pop` (`:52`/`:66`), `save/restore` via UcContext (`:108`/`:112`), `disassembler` (`:117`). +- `qiling/arch/register.py:11` - `QlRegisterManager` - `ql.arch.regs.rax`-style access, backed by per-arch `*_const.py` tables. +- `qiling/arch/x86.py:22,53,79,111` - `QlArchIntel` / `QlArchA8086` / `QlArchX86` / `QlArchX8664`. +- `qiling/arch/cortex_m.py:67` - `QlArchCORTEX_M(QlArchARM)` - plus `QlInterruptContext` (`:25`) for exception entry/exit in MCU mode. +- `qiling/arch/models.py` - CPU model enums selected via the `cputype` kwarg (resolved in `select_arch`, `qiling/utils.py:376`). +- `qiling/cc/__init__.py:9` - `QlCC` - abstract calling convention; `QlCommonBaseCC` (`:104`); e.g. `cc/intel.py` defines `cdecl`/`stdcall`/`ms64`/`macosx64`. + +## Interactions + +- Instantiated first by [core.md](core.md) (`qiling/core.py:154`); `Qiling.uc` proxies `arch.uc` (`qiling/core.py:479`). +- [loader.md](loader.md) and the OS layers use `arch.regs` and stack primitives to set up entry state. +- `qiling/cc/` is consumed by `QlFunctionCall` in [os-base.md](os-base.md) for API argument marshalling. +- [debugger.md](debugger.md) reads/writes registers through this layer. + +## How to Test + +```sh +cd tests && python3 test_cpu_models.py # pass = unittest "OK", exit code 0 +``` + +- Broader arch coverage comes for free from `test_shellcode.py` (5 archs) and the per-OS suites. + +## Open Gaps / Roadmap + +- PowerPC and RISC-V have fewer OS-level tests than x86/ARM/MIPS (no dedicated POSIX suite beyond `tests/test_riscv.py`). +- Thumb state handling has a known fixup in `Qiling.emu_start` (`qiling/core.py:743`) rather than in the arch layer itself. diff --git a/ARCHITECTURE/cli.md b/ARCHITECTURE/cli.md new file mode 100644 index 00000000..40b13a98 --- /dev/null +++ b/ARCHITECTURE/cli.md @@ -0,0 +1,48 @@ +# CLI — qltool and qltui + +## Goal + +Give users a no-code way to run emulations: `qltool run` executes a binary +against a rootfs, `qltool code` runs shellcode (hex/asm/bin), `qltool +examples` prints usage samples, and `qltool qltui` launches an interactive +TUI that gathers the same options. Mature released infrastructure; +maturity-based status. + +## Status + +`done` — covered by `tests/test_qltool.py`, which shells out to `qltool` for +run/code subcommands including gdb attach and coverage output. + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qltool` | Executable argparse CLI; builds kwargs and drives `Qiling` | +| `qltui.py` | questionary/pyfx/termcolor TUI; collects options, returned to qltool | + +## Key Types and Entry Points + +- `qltool:189` - `run()` - argparse setup with subcommands `run`, `code`, `examples`, `qltui`; enum-mapping actions translate `--arch/--os/--endian/--verbose` strings to `QL_ARCH`/`QL_OS` enums (`qltool:59-75`). +- `qltool:129` - `handle_run(options)` - builds `{'argv': [file]+args, 'rootfs': ...}`. +- `qltool:78` - `handle_code(options)` - reads hex/asm/bin shellcode, assembling asm via `qiling.arch.utils.assembler` (`qltool:104`). +- `qltool:276` - `ql = Qiling(**ql_args)` - the single construction point; then optional Qdb (`:279`), gdbserver (`:285`), coverage-wrapped `ql.run()` (`:306-310`), JSON report (`:312`), exit with `ql.os.exit_code` (`:321`). + +## Interactions + +- Thin client of [core.md](core.md): constructs `Qiling` and calls `run()`. +- Attaches [debugger.md](debugger.md) via `--gdb host:port` / `--qdb [--rr]`. +- Uses [extensions.md](extensions.md) for `--coverage-file` (drcov) and `--json` report output. +- `qltool examples` mirrors scripts documented in `examples/README.md`. + +## How to Test + +```sh +cd tests && python3 test_qltool.py # pass = unittest "OK", exit 0 +``` + +- Manual smoke test: `./qltool run -f examples/rootfs/x8664_linux/bin/x8664_hello --rootfs examples/rootfs/x8664_linux` — pass = prints `Hello, World!`. + +## Open Gaps / Roadmap + +- `qltui.py` (TUI) has no automated tests — interactive only. +- `qltool` predates subcommand-style config files; complex setups (fs mappers, custom hooks) still require the Python API. diff --git a/ARCHITECTURE/core.md b/ARCHITECTURE/core.md new file mode 100644 index 00000000..4b16504e --- /dev/null +++ b/ARCHITECTURE/core.md @@ -0,0 +1,61 @@ +# Core — the Qiling facade and plumbing + +## Goal + +Own the public API and object lifecycle of an emulation: the `Qiling` class +composes arch, loader, memory, OS, and (for bare-metal) hardware components, +and exposes hooks, memory/register access, patching, and save/restore to +users. No roadmap milestone applies — this is mature released infrastructure; +status is maturity-based per the control center. + +## Status + +`done` — exercised by the whole test suite; the How to Test command below +boots shellcode end-to-end through `Qiling.__init__` → `run()` → `emu_start`. + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/core.py` | `Qiling` class: constructor/composition root, `run`, `emu_start/stop`, `save/restore`, `patch`, properties | +| `qiling/core_hooks.py` | `QlCoreHooks` mixin: wraps Unicorn hooks, dispatches to `Hook` lists | +| `qiling/core_hooks_types.py` | `Hook`, `HookAddr`, `HookIntr`, `HookRet` records | +| `qiling/core_struct.py` | `QlCoreStructs` mixin: endian/bit-width-aware pack/unpack helpers | +| `qiling/utils.py` | Name→class resolution: `select_arch/loader/os/component/debugger`, binary format sniffing, profile loading | +| `qiling/const.py` | Enums: `QL_ARCH`, `QL_OS`, `QL_VERBOSE`, `QL_INTERCEPT`, `QL_STATE`, groupings `QL_OS_POSIX`/`QL_OS_BAREMETAL` | +| `qiling/exception.py` | `QlErrorBase` and ~20 subclasses (`QlErrorArch`, `QlSyscallError`, …) | +| `qiling/host.py` | `QlHost`: describes the *hosting* platform (for pass-through features) | +| `qiling/log.py` | Logger setup, colored/plain formatters, regex filtering behind `Qiling.filter` | +| `qiling/profiles/*.ql` | Default per-OS config (memory layout, kernel uid/gid/pid) merged with user overrides | + +## Key Types and Entry Points + +- `qiling/core.py:35` - `Qiling(QlCoreHooks, QlCoreStructs)` - the facade; `__init__` composes components in fixed order: arch (`:154`) → mixins (`:157`) → logger → profile (`:178`) → loader (`:183`) → memory (`:188`) → OS (`:189`) → hw if bare-metal (`:191`) → `loader.run()` (`:195`). +- `qiling/core.py:561` - `Qiling.run(begin, end, timeout, count)` - attaches debugger, applies patches, writes exit trap, delegates to `os.run()`. +- `qiling/core.py:743` - `Qiling.emu_start(begin, end, timeout, count)` - thin wrapper over `uc.emu_start`; manages thumb bit, `QL_STATE`, exception re-raise. +- `qiling/core.py:609` / `:658` - `save()` / `restore()` - snapshot regs/mem/fd/os per-component. +- `qiling/core_hooks.py:150` - `QlCoreHooks` - hook registration API: `hook_code` (`:400`), `hook_block` (`:422`), `hook_address` (`:550`), `hook_intno` (`:575`), `hook_mem_read/write` (`:592`/`:610`), `hook_insn` (`:646`), `hook_del` (`:686`). +- `qiling/utils.py:278` - `ql_guess_emu_env(path)` - sniffs arch/OS/endian from pathname, ELF, Mach-O, or PE headers when not given. +- `qiling/utils.py:297,376,409,323,332` - `select_loader/arch/os/component/debugger` - dynamic-import factories; core never imports concrete subclasses. +- `qiling/utils.py:419` - `profile_setup(ostype, user_config)` - YAML for MCU, else ConfigParser over `qiling/profiles/.ql` + user overrides. + +## Interactions + +- Instantiates every other subsystem: [arch.md](arch.md), [loader.md](loader.md), [os-base.md](os-base.md) (memory + OS), [hw.md](hw.md) (bare-metal only). +- Lazily instantiates [debugger.md](debugger.md) inside `run()` via `select_debugger`. +- [extensions.md](extensions.md) and [cli.md](cli.md) consume only this public API. +- Hook dispatch protocol (`QL_HOOK_BLOCK`, `qiling/const.py:77`) is honored by the OS layers when they intercept syscalls/APIs. + +## How to Test + +```sh +cd tests && python3 test_shellcode.py # pass = unittest "OK", exit code 0 +``` + +- Exercises `Qiling(code=...)` construction and `run()` across x86/x86-64/ARM/ARM64/MIPS shellcode for Linux and Windows. + +## Open Gaps / Roadmap + +- `ChangeLog` lags the released version (1.4.6 vs 1.4.8 in `pyproject.toml`). +- `unicorn` is hard-pinned to 2.1.3; upgrading Unicorn is a project-wide event. +- Feature wishlist lives in GitHub issue [#333](https://github.com/qilingframework/qiling/issues/333). diff --git a/ARCHITECTURE/debugger.md b/ARCHITECTURE/debugger.md new file mode 100644 index 00000000..88289e3f --- /dev/null +++ b/ARCHITECTURE/debugger.md @@ -0,0 +1,52 @@ +# Debugger — GDB server and Qdb + +## Goal + +Let users debug emulated targets: a GDB remote-serial-protocol server so any +GDB/IDA/lldb front end can attach cross-architecture, and Qdb — a built-in +interactive CLI debugger with stepping, branch prediction, and record/replay +reverse debugging. Mature released infrastructure; maturity-based status. + +## Status + +`done` — Qdb covered by `tests/test_qdb.py`; the GDB server by +`tests/test_debugger.py` (spawns a real client session). + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/debugger/debugger.py` | Base `QlDebugger` | +| `qiling/debugger/gdb/gdb.py` | `QlGdb`: GDB remote-serial-protocol server | +| `qiling/debugger/gdb/xmlregs.py`, `gdb/xml/` | Target-description XML per arch for modern GDB clients | +| `qiling/debugger/qdb/qdb.py` | `QlQdb`: interactive Cmd-based debugger | +| `qiling/debugger/qdb/arch/` | Per-arch Qdb support (arm, intel, mips) | +| `qiling/debugger/qdb/branch_predictor/` | Predicts branch targets for step/next | +| `qiling/debugger/qdb/render/` | Register/stack/disasm view rendering | + +## Key Types and Entry Points + +- `qiling/debugger/debugger.py:13` - `QlDebugger` - base; `run()` starts the session. +- `qiling/debugger/gdb/gdb.py:68` - `QlGdb(QlDebugger)` - listens on ip:port, translates RSP packets to Qiling hook/mem/reg operations. +- `qiling/debugger/qdb/qdb.py:59` - `QlQdb(Cmd, QlDebugger)` - CLI loop; `rr` mode enables record/replay reverse debugging. +- Activation: set `ql.debugger = True | "gdb" | "gdb:0.0.0.0:9999" | "qdb" | "qdb:rr"` (`qiling/core.py:437`); instantiated lazily in `Qiling.run` via `select_debugger` (`qiling/utils.py:332`). +- `qltool` flags: `--gdb` and `--qdb` (see [cli.md](cli.md)). + +## Interactions + +- Instantiated by [core.md](core.md) at `Qiling.run` time, not construction. +- Reads/writes state exclusively through public APIs: registers via [arch.md](arch.md), memory via [os-base.md](os-base.md), breakpoints via `hook_address` ([core.md](core.md)). +- The IDA plugin in [extensions.md](extensions.md) offers an alternative front end over the same public API. + +## How to Test + +```sh +cd tests && python3 test_qdb.py # pass = unittest "OK", exit 0 +``` + +- GDB server: `cd tests && python3 test_debugger.py` — starts `QlGdb` and drives a scripted client. + +## Open Gaps / Roadmap + +- Qdb per-arch support covers arm/cortex-m/mips/intel; RISC-V and PPC lack Qdb arch modules (`qiling/debugger/qdb/arch/`). +- Record/replay (`qdb:rr`) stores full state per step — memory-heavy on long runs. diff --git a/ARCHITECTURE/extensions.md b/ARCHITECTURE/extensions.md new file mode 100644 index 00000000..a9f0e738 --- /dev/null +++ b/ARCHITECTURE/extensions.md @@ -0,0 +1,61 @@ +# Extensions — optional tooling on top of the core + +## Goal + +House everything that builds on the public `Qiling` API without being required +by it: fuzzing (AFL), code-coverage and execution-trace writers, a heap +sanitizer, radare2 and IDA Pro integration, fake stdio pipes, and JSON run +reports. Mature released infrastructure; maturity-based status. + +## Status + +`done` — history/coverage tracker tested in CI; AFL and r2 integrations need +optional extras (`fuzz`, `RE`); the IDA plugin needs IDA Pro (untested in CI). + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/extensions/afl/` | `ql_afl_fuzz` (unicornafl bridge) + `QlFuzzer` harness base class | +| `qiling/extensions/coverage/` | `collect_coverage` context manager; writers in `formats/` (drcov, drcov_exact, ezcov, history) | +| `qiling/extensions/tracing/` | Tenet-style execution trace writers (`formats/`) | +| `qiling/extensions/trace.py` | Disassembly tracing: full trace or ring-buffer history | +| `qiling/extensions/sanitizers/heap.py` | Canary-based heap sanitizer (UAF/OOB detection) | +| `qiling/extensions/r2/r2.py` | radare2 (r2libr) integration: sections/symbols/functions/xrefs of the loaded target | +| `qiling/extensions/idaplugin/qilingida.py` | IDA Pro plugin driving Qiling emulation from IDA | +| `qiling/extensions/pipe.py` | Fake stdio streams for hijacking emulated I/O (fuzzing staple) | +| `qiling/extensions/report/report.py` | `generate_report(ql)`: JSON summary of a run | +| `qiling/extensions/multitask.py` | Cooperative-multitask Unicorn wrapper (documented in [os-baremetal.md](os-baremetal.md)) | +| `qiling/extensions/mcu/` | Board/chip definitions (documented in [hw.md](hw.md)) | +| `qiling/extensions/winsdkapi.py` | Windows API signature decorator glue | + +## Key Types and Entry Points + +- `qiling/extensions/afl/afl.py:21` / `:87` - `ql_afl_fuzz` / `ql_afl_fuzz_custom` - hand control to AFL++ via unicornafl; harness base `QlFuzzer` (`qiling/extensions/afl/qlfuzzer.py:14`). +- `qiling/extensions/coverage/utils.py:48` - `collect_coverage(ql, name, file)` - context manager writing e.g. drcov files (used by `qltool --coverage-file`). +- `qiling/extensions/trace.py:145` / `:180` - `enable_full_trace` / `enable_history_trace` - disassembly tracing via `hook_code`. +- `qiling/extensions/sanitizers/heap.py:16` - `QlSanitizedMemoryHeap` - drop-in replacement for `ql.os.heap` with canaries and free-list checks. +- `qiling/extensions/r2/r2.py:135` - `R2(ql)` - rzpipe-backed analysis of the loaded image. +- `qiling/extensions/pipe.py` - `SimpleInStream`/`SimpleOutStream` (`:62`/`:69`) - assigned to `ql.os.stdin`/`stdout`. +- `qiling/extensions/report/report.py:56` - `generate_report(ql)` - JSON report (used by `qltool --json`). + +## Interactions + +- Everything here consumes only the public API of [core.md](core.md) (hooks, mem, regs) and [os-base.md](os-base.md) (heap, stdio, syscall/API overrides). +- The heap sanitizer wraps `QlMemoryHeap` from [os-base.md](os-base.md) (demo: `examples/uefi_sanitized_heap.py` with [os-windows.md](os-windows.md) UEFI). +- Fuzzing harnesses live in `examples/fuzzing/` and pair AFL with `pipe.py` and `set_syscall` from [os-posix.md](os-posix.md). +- `qltool` wires in coverage and report generation ([cli.md](cli.md)). + +## How to Test + +```sh +cd tests && python3 test_history.py # pass = unittest "OK", exit 0 +``` + +- r2 integration (needs `pip install qiling[RE]`): `cd tests && python3 test_r2.py`. +- AFL (needs `pip install qiling[fuzz]` + AFL++): run a harness from `examples/fuzzing/linux_x8664/`. + +## Open Gaps / Roadmap + +- `afl/` and `r2/` depend on optional extras not installed by default; the IDA plugin cannot be CI-tested (requires an IDA Pro license). +- Two overlapping trace mechanisms exist (`trace.py` and `tracing/`); no unified interface. diff --git a/ARCHITECTURE/hw.md b/ARCHITECTURE/hw.md new file mode 100644 index 00000000..346f6c3b --- /dev/null +++ b/ARCHITECTURE/hw.md @@ -0,0 +1,53 @@ +# HW — peripheral emulation for bare-metal targets + +## Goal + +Emulate memory-mapped peripherals (GPIO, UART/char, SPI, I2C, timers, DMA, +interrupt controllers, flash, …) so MCU firmware runs against realistic +hardware. Peripherals are instantiated from a YAML profile naming which +peripheral class sits at which MMIO base address. Only active in bare-metal +(MCU) mode. Mature released infrastructure; maturity-based status. + +## Status + +`done` — exercised by `tests/test_mcu.py` against STM32F1/F4 and GD32VF1 +firmware images (UART echo, freertos, blink, crc, dma_clock, i2c/spi/lcd). + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/hw/hw.py` | `QlHwManager`: creates peripherals from profile, maps MMIO, steps them each tick | +| `qiling/hw/peripheral.py` | Base `QlPeripheral` (+ `QlPeripheralUtils`) | +| `qiling/hw/analog/ char/ dma/ flash/ gpio/ i2c/ intc/ math/ mem/ misc/ net/ power/ sd/ spi/ timer/` | One directory per peripheral class, chip-specific implementations inside | +| `qiling/hw/const/` | Register layout constants per chip family | +| `qiling/hw/connectivity.py`, `external_device/` | External-interface plumbing (e.g. connecting a fake device to a bus) | +| `qiling/extensions/mcu/` | Board/chip definitions (stm32f4xx, gd32vf1, nxp, atmel, bes): memory maps naming peripheral class + base address, passed as `env=` to `Qiling` | + +## Key Types and Entry Points + +- `qiling/hw/hw.py:14` - `QlHwManager` - available as `ql.hw`; created by core only when `ql.baremetal` (`qiling/core.py:191`). +- `qiling/hw/hw.py:23` - `QlHwManager.create(label, struct, base)` - instantiates a peripheral from the profile entry and maps its MMIO region. +- `qiling/hw/hw.py:82` - `QlHwManager.step()` - advances every peripheral one tick; called from the MCU run loop. +- `qiling/hw/peripheral.py:132` - `QlPeripheral(QlPeripheralUtils)` - base class: a ctypes register struct + read/write handlers on the MMIO region. +- `qiling/extensions/mcu/stm32f4xx/stm32f407.py` (and siblings) - chip `env` dicts consumed at `Qiling(..., env=...)` construction. + +## Interactions + +- Created by [core.md](core.md) for bare-metal targets; driven by the MCU run loop in [os-baremetal.md](os-baremetal.md) (`QlHwManager.step()` between execution slices). +- Interrupt controller peripherals (`intc/`) raise exceptions delivered through `QlInterruptContext` in [arch.md](arch.md). +- MMIO regions are mapped through `QlMemoryManager` ([os-base.md](os-base.md)). +- Profile parsing (YAML) is in `profile_setup` ([core.md](core.md), `qiling/utils.py:419`). + +## How to Test + +```sh +cd tests && python3 test_mcu.py # pass = unittest "OK", exit 0 +``` + +- Exercises GPIO, UART, EXTI, I2C, SPI, DMA, CRC, RTC, watchdog peripherals on real firmware from `examples/rootfs/mcu/`. + +## Open Gaps / Roadmap + +- Peripheral fidelity is demand-driven: registers behave as observed firmware needs, not per full datasheets. +- Chip coverage limited to families under `qiling/extensions/mcu/`; adding a chip means writing its memory map + any missing peripheral classes. diff --git a/ARCHITECTURE/loader.md b/ARCHITECTURE/loader.md new file mode 100644 index 00000000..5cbb3d9f --- /dev/null +++ b/ARCHITECTURE/loader.md @@ -0,0 +1,58 @@ +# Loader — binary format loaders + +## Goal + +Map the target binary into emulated memory and prepare execution state: parse +the format, load segments/sections and dependencies (ld.so, DLLs, dylibs), +build stack/argv/auxv or OS structures (PEB/TEB, UEFI protocols), and set +entry/exit points. One loader per format, selected by OS type. Mature released +infrastructure; maturity-based status. + +## Status + +`done` — every format loader is exercised by its OS test suite; the How to +Test command proves the ELF path end-to-end. + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/loader/loader.py` | Base `QlLoader` + `Image` record; abstract `run()`, save/restore | +| `qiling/loader/elf.py` | ELF for Linux/FreeBSD/QNX: segments, interpreter (ld.so), stack, auxv | +| `qiling/loader/pe.py` | Windows PE: DLL resolution + `QlPeCache`, PEB/TEB/LDR via `Process` | +| `qiling/loader/pe_uefi.py` | UEFI DXE/SMM module loading, protocol installation | +| `qiling/loader/macho.py` | Mach-O (with its own `macho_parser/` package) | +| `qiling/loader/dos.py` | DOS COM/EXE/MBR | +| `qiling/loader/mcu.py` | MCU firmware: Intel HEX (`IhexParser`), ELF, or raw bin per YAML profile | +| `qiling/loader/blob.py` | Raw blobs loaded at a fixed address (e.g. u-boot) | + +## Key Types and Entry Points + +- `qiling/loader/loader.py:21` - `QlLoader` - base; `Image` NamedTuple (`:15`), abstract `run()` (`:62`), `skip_exit_check` (`:27`). +- `qiling/loader/elf.py:68` - `QlLoaderELF` - loads binary + interpreter, builds stack/auxv (`AUXV` enum `:32`), sets `entry_point`/`elf_entry`. +- `qiling/loader/pe.py:666` - `QlLoaderPE(QlLoader, Process)` - `Process` (`:69`) builds PEB/TEB/LDR; `QlPeCache` (`:41`) caches parsed DLLs behind the `libcache` kwarg (wired in `select_loader`, `qiling/utils.py:300`). +- `qiling/loader/pe_uefi.py:26` - `QlLoaderPE_UEFI` - loads DXE/SMM modules and installs protocols into the UEFI context. +- `qiling/loader/macho.py:70` - `QlLoaderMACHO`. +- `qiling/loader/mcu.py:57` - `QlLoaderMCU` - with `IhexParser` (`:15`). +- OS→loader mapping: `select_loader` (`qiling/utils.py:297`) — LINUX/FREEBSD/QNX→elf, MACOS→macho, WINDOWS→pe, UEFI→pe_uefi, DOS→dos, MCU→mcu, BLOB→blob. + +## Interactions + +- Instantiated by [core.md](core.md) (`qiling/core.py:183`); `loader.run()` fires at the end of `Qiling.__init__` (`qiling/core.py:195`). +- Uses `ql.mem` ([os-base.md](os-base.md)) for mapping and `ql.arch` ([arch.md](arch.md)) for initial register/stack state. +- The PE loader builds Windows process structures with [os-windows.md](os-windows.md) components (heap, handles); the UEFI loader populates the context in `qiling/os/uefi/`. +- IAT entries recorded here drive address-based API hooking in [os-windows.md](os-windows.md). +- The MCU loader reads the YAML profile owned by [core.md](core.md) and memory maps defined via [hw.md](hw.md). + +## How to Test + +```sh +cd tests && python3 -m unittest test_elf.ELFTest.test_elf_linux_x8664 # pass = "OK", exit 0 +``` + +- Full loader coverage: `test_elf.py`, `test_pe.py` (Windows host), `test_macho.py` (macOS host), `test_dos.py`, `test_mcu.py`, `test_blob.py`, `test_uefi.py`. + +## Open Gaps / Roadmap + +- PE and Mach-O loading depend on host-collected libraries (`examples/scripts/dllscollector.bat` / `dylibcollector.sh`); not runnable from a Linux checkout alone. +- `QlPeCache` cache invalidation is manual (delete cache files) — no versioning. diff --git a/ARCHITECTURE/os-baremetal.md b/ARCHITECTURE/os-baremetal.md new file mode 100644 index 00000000..2823b095 --- /dev/null +++ b/ARCHITECTURE/os-baremetal.md @@ -0,0 +1,52 @@ +# OS bare-metal — MCU firmware and raw blobs + +## Goal + +Execute code with no operating system. MCU mode runs microcontroller firmware +(STM32, GD32V, NXP, …) with peripheral emulation and interrupt scheduling on +a cooperative multitasking wrapper around Unicorn; BLOB mode runs raw binaries +(e.g. u-boot) at a fixed load address with no OS services at all. Mature +released infrastructure; maturity-based status. + +## Status + +`done` — MCU covered by `tests/test_mcu.py` (STM32F1/F4, GD32VF1 firmware), +BLOB by `tests/test_blob.py` (u-boot) and `tests/test_edl.py`. + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/os/mcu/mcu.py` | `QlOsMcu` run loop + `MCUTask` driving Unicorn and peripheral steps | +| `qiling/os/blob/blob.py` | `QlOsBlob`: minimal run loop for raw binaries | +| `qiling/extensions/multitask.py` | `UnicornTask` / `MultiTaskUnicorn`: cooperative multitasking over Unicorn | + +## Key Types and Entry Points + +- `qiling/os/mcu/mcu.py:37` - `QlOsMcu(QlOs)` - run loop; steps hardware between execution chunks and delivers interrupts. +- `qiling/os/mcu/mcu.py:17` - `MCUTask(UnicornTask)` - the firmware execution task. +- `qiling/extensions/multitask.py:26` / `:152` - `UnicornTask` / `MultiTaskUnicorn(Uc)` - task-switching Unicorn subclass MCU mode runs on. +- `qiling/os/blob/blob.py:12` - `QlOsBlob(QlOs)` - runs `entry_point` → `exit_point` with no OS services. +- MCU selection: `QL_OS.MCU` is the only member of `QL_OS_BAREMETAL` (`qiling/const.py:74`); `ql.baremetal` (`qiling/core.py:357`) gates hardware-manager creation. + +## Interactions + +- Both subclass [os-base.md](os-base.md) `QlOs`. +- MCU mode drives [hw.md](hw.md): `QlHwManager.step()` is called between execution slices; board/chip memory maps come from `qiling/extensions/mcu/` passed as `env=`. +- Interrupt entry/exit uses `QlInterruptContext` in [arch.md](arch.md) (`qiling/arch/cortex_m.py:25`). +- Firmware images are loaded by `QlLoaderMCU` / `QlLoaderBLOB` ([loader.md](loader.md)); MCU profiles are YAML (`profile_setup`, [core.md](core.md)). +- Fuzzing MCU firmware (e.g. `examples/fuzzing/stm32f429/`) combines this mode with [extensions.md](extensions.md) AFL support. + +## How to Test + +```sh +cd tests && python3 test_blob.py # pass = unittest "OK", exit 0 +``` + +- MCU (also proves [hw.md](hw.md)): `cd tests && python3 test_mcu.py`. +- Qualcomm EDL loader: `cd tests && python3 test_edl.py`. + +## Open Gaps / Roadmap + +- Supported chip families are those with board definitions in `qiling/extensions/mcu/` (STM32F1/F4, GD32VF1, NXP, Atmel, BES); new chips need new peripheral maps. +- BLOB mode provides no services by design — targets needing hardware must use MCU mode instead. diff --git a/ARCHITECTURE/os-base.md b/ARCHITECTURE/os-base.md new file mode 100644 index 00000000..7d71305d --- /dev/null +++ b/ARCHITECTURE/os-base.md @@ -0,0 +1,60 @@ +# OS base — shared OS-layer services + +## Goal + +Own the OS-agnostic services every OS personality builds on: the abstract +`QlOs` class, the emulated memory manager and heap, function-call argument +marshalling (via calling conventions), filesystem virtualization (rootfs +confinement + host mapping), file descriptor objects, green threads, and +API-call statistics. Mature released infrastructure; maturity-based status. + +## Status + +`done` — path virtualization and struct helpers have dedicated unit tests; +everything else is exercised by every OS suite. + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/os/os.py` | Abstract `QlOs`: utils/stats/path/fs-mapper composition, `set_api`, user-defined API dict, abstract `run()` | +| `qiling/os/memory.py` | `QlMemoryManager` (map/unmap/search, `ql.mem`) and `QlMemoryHeap` | +| `qiling/os/fcall.py` | `QlFunctionCall`: read args / write retval using `qiling/cc/` conventions | +| `qiling/os/mapper.py` | `QlFsMapper`/`QlFsMappedObject`: redirect emulated paths to host files/objects | +| `qiling/os/path.py` | `QlOsPath`: rootfs confinement and virtual→host path conversion | +| `qiling/os/filestruct.py`, `disk.py` | File-descriptor and raw-disk objects | +| `qiling/os/thread.py` | `QlThread(Greenlet)` base for multithread emulation | +| `qiling/os/stats.py` | `QlOsStats`: API/syscall call statistics | +| `qiling/os/struct.py`, `utils.py`, `const.py` | ctypes-based struct helpers, OS utils, shared constants | + +## Key Types and Entry Points + +- `qiling/os/os.py:24` - `QlOs` - composes utils/stats/path/fs-mapper; `user_defined_api` keyed by `QL_INTERCEPT`; abstract `run()` (`:239`). +- `qiling/os/os.py:224` - `QlOs.set_api(target, handler, intercept)` - user override of an emulated API (address- or name-based). +- `qiling/os/memory.py:23` - `QlMemoryManager` - `ql.mem`; instantiated by core *before* the OS (`qiling/core.py:188`). +- `qiling/os/memory.py:658` - `QlMemoryHeap` - heap used by Windows/UEFI APIs and the sanitizers extension. +- `qiling/os/fcall.py:21` - `QlFunctionCall` - argument/return marshalling on top of `qiling/cc/`. +- `qiling/os/mapper.py:64` - `QlFsMapper` - behind `ql.add_fs_mapper` (`qiling/core.py:701`). +- `qiling/os/path.py:14` - `QlOsPath` - rootfs-confined path resolution. +- `qiling/os/thread.py:11` - `QlThread(Greenlet)` - gevent-based thread base. + +## Interactions + +- Instantiated by [core.md](core.md): memory at `qiling/core.py:188`, the concrete OS at `:189` (via `select_os`). +- Uses [arch.md](arch.md) calling conventions (`qiling/cc/`) inside `QlFunctionCall`. +- Subclassed by [os-posix.md](os-posix.md) (`QlOsPosix`), [os-windows.md](os-windows.md) (`QlOsWindows`, `QlOsUefi`, `QlOsDos`), and [os-baremetal.md](os-baremetal.md) (`QlOsMcu`, `QlOsBlob`). +- [loader.md](loader.md) maps segments through `QlMemoryManager`. +- The heap is wrapped by `QlSanitizedMemoryHeap` in [extensions.md](extensions.md). + +## How to Test + +```sh +cd tests && python3 test_pathutils.py && python3 test_struct.py # pass = both print "OK", exit 0 +``` + +- `test_pathutils.py` proves rootfs path virtualization (incl. case-insensitive Windows semantics); `test_struct.py` proves the struct helpers. + +## Open Gaps / Roadmap + +- Multithread emulation (gevent) is opt-in via `multithread=True` and less battle-tested than single-threaded mode (see `tests/test_elf_multithread.py`). +- `QlOs.run()` contract is loose: each personality implements its own loop; no shared scheduler outside POSIX threads and MCU multitask. diff --git a/ARCHITECTURE/os-posix.md b/ARCHITECTURE/os-posix.md new file mode 100644 index 00000000..8c8dd84c --- /dev/null +++ b/ARCHITECTURE/os-posix.md @@ -0,0 +1,58 @@ +# OS POSIX — Linux, FreeBSD, macOS, QNX + +## Goal + +Emulate POSIX operating systems by intercepting syscalls: a shared +`QlOsPosix` layer owns the per-arch syscall ABI, number→name mapping, and the +file-descriptor table; per-OS subclasses add their personality (Linux futex +and procfs, macOS mach ports, QNX message passing). Mature released +infrastructure; maturity-based status. + +## Status + +`done` — Linux is the flagship target with the largest suite; FreeBSD/QNX +covered by dedicated tests; macOS tests are host-gated (see Open Gaps). + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/os/posix/posix.py` | `QlOsPosix`: syscall ABI per arch, dispatcher, `set_syscall`, fd table | +| `qiling/os/posix/syscall/` | Syscall implementations, one module per family (unistd, mman, socket, fcntl, signal, ioctl, …) | +| `qiling/os/posix/shm.py`, `msq.py` | SysV IPC (shared memory, message queues) | +| `qiling/os/linux/linux.py` | `QlOsLinux`: syscall hook wiring and the dynamic-ELF run loop | +| `qiling/os/linux/` | `futex.py`, `thread.py` (green threads), `procfs.py`, `map_syscall.py`/`syscall_nums.py`, `kernel_api/` (for .ko emulation) | +| `qiling/os/freebsd/freebsd.py` | `QlOsFreebsd` | +| `qiling/os/macos/macos.py` | `QlOsMacos` + mach ports/tasks/kext support | +| `qiling/os/qnx/qnx.py` | `QlOsQnx` + QNX message passing (`message.py`) | + +## Key Types and Entry Points + +- `qiling/os/posix/posix.py:48` - `QlOsPosix(QlOs)` - per-arch syscall ABI tables (`:69`), fd table, SysV IPC. +- `qiling/os/posix/posix.py:170` - `load_syscall()` - the dispatcher: syscall id → name (via per-OS `map_syscall`) → handler in `os/posix/syscall/*` or user override. +- `qiling/os/posix/posix.py:132` - `set_syscall(target, handler, intercept)` - user syscall hijack (by number or name, CALL/ENTER/EXIT). +- `qiling/os/posix/posix.py:100` - `__get_syscall_mapper(archtype)` - loads `qiling/os//map_syscall.py` dynamically. +- `qiling/os/linux/linux.py:26` - `QlOsLinux(QlOsPosix)` - `hook_syscall` (`:137`) registered on interrupt/insn hooks. +- `qiling/os/linux/linux.py:148` - `QlOsLinux.run()` - shellcode single-shot, gevent thread manager when `multithread=True`, or the common path: run ld.so to `elf_entry`, apply lib patches, then run to exit. + +## Interactions + +- Subclasses [os-base.md](os-base.md) `QlOs`; fd objects come from `qiling/os/filestruct.py`. +- Syscall entry is an arch hook: interrupt/insn hooks from [core.md](core.md) (`QlCoreHooks`) fire `hook_syscall`, using [arch.md](arch.md) registers per the syscall ABI. +- Runs binaries prepared by [loader.md](loader.md) (`QlLoaderELF`, `QlLoaderMACHO`). +- Path/fs access goes through `QlOsPath`/`QlFsMapper` in [os-base.md](os-base.md). +- Fuzzing harnesses in [extensions.md](extensions.md) commonly hijack fd 0 with `pipe.py` and syscalls via `set_syscall`. + +## How to Test + +```sh +cd tests && python3 test_posix.py # pass = unittest "OK", exit 0 +``` + +- Full POSIX coverage in CI: `tests/test_onlinux.sh` (adds `test_elf.py` targets, `test_elf_multithread.py`, `test_elf_ko.py`, `test_qnx.py`, `test_android.py`). + +## Open Gaps / Roadmap + +- macOS emulation tests (`test_macho.py`, `test_macho_kext.py`) need a macOS host with collected dylibs; the macOS CI job is commented out in `.github/workflows/build-ci.yml`. +- `test_elf_ko.py` requires a one-time `unzip -P infected m0hamed_rootkit.ko.zip` in `examples/rootfs/x86_linux/kernel/`. +- Syscall coverage is demand-driven — unimplemented syscalls raise/log and get added as targets need them. diff --git a/ARCHITECTURE/os-windows.md b/ARCHITECTURE/os-windows.md new file mode 100644 index 00000000..88a64276 --- /dev/null +++ b/ARCHITECTURE/os-windows.md @@ -0,0 +1,59 @@ +# OS Windows family — Windows, UEFI, DOS + +## Goal + +Emulate API-call-driven (rather than syscall-driven) environments. Windows: +Win32/NT API emulation with handles, registry, and PE process structures. +UEFI: boot/runtime/SMM services and protocol database for DXE/SMM modules. +DOS: BIOS/DOS interrupt services for COM/EXE/MBR targets. Mature released +infrastructure; maturity-based status. + +## Status + +`done` — Windows PE suite runs in CI on a Windows host; UEFI and DOS suites +run on Linux. Windows tests are platform-gated locally (need collected DLLs). + +## Code Structure + +| File | Role | +| ---- | ---- | +| `qiling/os/windows/windows.py` | `QlOsWindows`: component setup (handles, registry, clipboard, fiber), run loop | +| `qiling/os/windows/dlls/` | Win32 API implementations per DLL (kernel32/, ntdll.py, msvcrt.py, advapi32.py, …) | +| `qiling/os/windows/fncc.py`, `api.py` | Decorators declaring API signatures/calling conventions | +| `qiling/os/windows/handle.py`, `registry.py`, `clipboard.py`, `fiber.py`, `thread.py`, `structs.py` | Windows kernel-object emulation | +| `qiling/os/uefi/uefi.py` | `QlOsUefi` run loop | +| `qiling/os/uefi/bs.py`, `rt.py`, `ds.py`, `smm.py`, `protocols/` | Boot/runtime/DXE/SMM services and protocol implementations | +| `qiling/os/uefi/context.py` | UEFI execution contexts populated by the loader | +| `qiling/os/dos/dos.py` | `QlOsDos` run loop | +| `qiling/os/dos/interrupts/` | INT 10h/13h/15h/16h/21h… service handlers | + +## Key Types and Entry Points + +- `qiling/os/windows/windows.py:33` - `QlOsWindows(QlOs)` - `__setup_components` (`:156`) builds handle manager/registry/clipboard/fiber; `run()` (`:201`). +- `qiling/os/windows/fncc.py` - `@winsdkapi` decorator - declares an API's calling convention and typed params; implementations live in `dlls/`. +- API dispatch is address-based: IAT addresses recorded by the PE loader are hooked and marshalled through `QlFunctionCall` — user overrides via `QlOs.set_api` (`qiling/os/os.py:224`). +- `qiling/os/uefi/uefi.py:22` - `QlOsUefi(QlOs)` - executes DXE/SMM modules; services in `bs.py`/`rt.py`/`smm.py` are installed as callable tables. +- `qiling/os/dos/dos.py:33` - `QlOsDos(QlOs)` - dispatches BIOS/DOS interrupts from `interrupts/`. + +## Interactions + +- All three subclass [os-base.md](os-base.md) `QlOs` and allocate from `QlMemoryHeap`. +- [loader.md](loader.md): `QlLoaderPE` builds PEB/TEB and records IAT hook addresses; `QlLoaderPE_UEFI` installs protocols into `os/uefi/context.py`; `QlLoaderDOS` sets real-mode state. +- Argument marshalling uses `QlFunctionCall` + `qiling/cc/intel.py` conventions ([arch.md](arch.md)). +- The registry emulation reads hive files from the rootfs via [os-base.md](os-base.md) path services. +- `examples/uefi_sanitized_heap.py` pairs UEFI with the heap sanitizer from [extensions.md](extensions.md). + +## How to Test + +```sh +cd tests && python3 test_uefi.py # pass = unittest "OK", exit 0 (Linux-runnable) +``` + +- Windows host only: `cd tests && cmd.exe /C test_pe.bat` after running `examples/scripts/dllscollector.bat` (CI does this in `.github/workflows/build-ci.yml`). +- DOS: `cd tests && python3 test_dos.py` (uses scripted keyboard input). + +## Open Gaps / Roadmap + +- Windows tests require a Windows host: system DLLs/registry hives are not redistributable and must be collected locally. +- Win32 API surface is demand-driven; unimplemented APIs raise `QlErrorSyscallNotFound`-style errors and are added as samples need them. +- SMM emulation covers common protocols only (see `qiling/os/uefi/smm.py`). diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..708f0d2e --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +ARCHITECTURE.md \ No newline at end of file From ce6f08d71f6e41a0a3ff4377a3cc08db6fff9579 Mon Sep 17 00:00:00 2001 From: H1ghC0ntr4sT Date: Thu, 2 Jul 2026 22:51:45 +0700 Subject: [PATCH 09/21] Reset emu state after a failing DllMain call_dll_entrypoint() caught the DllMain UcError and restored registers but left ql._state at STARTED, since emu_start() only clears it on success. The 'emu_state is not STARTED' gate then silently skipped DllMain for every DLL loaded afterward. Reset the state to STOPPED on the exception path. --- qiling/loader/pe.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qiling/loader/pe.py b/qiling/loader/pe.py index 30959d0f..f9ac1bb6 100644 --- a/qiling/loader/pe.py +++ b/qiling/loader/pe.py @@ -462,6 +462,10 @@ class Process: self.ql.log.error(f'Error encountered while running {dll_name} DllMain, bailing') self.ql.arch.regs.restore(regs_state) + + # emu_start leaves the state at STARTED when it raises; reset it so a single + # failing DllMain does not gate every subsequently loaded DLL's entry point. + self.ql._state = QL_STATE.STOPPED else: fcall.cc.unwind(len(args)) From d8f904c44ae8550f6e73edc685d3fab49215476f Mon Sep 17 00:00:00 2001 From: w1tcher <10086+bowen.sun@users.noreply.github.com> Date: Tue, 7 Jul 2026 17:37:59 +0800 Subject: [PATCH 10/21] Fix FLS FiberManager writing last_error to a dead attribute FiberManager.free/set/get set `self.last_error` on the FiberManager instance, but nothing ever reads that attribute. The Windows last-error that GetLastError returns lives on `ql.os.last_error`, so FlsFree, FlsSetValue and FlsGetValue never actually reported ERROR_INVALID_PARAMETER for an invalid FLS index. Route the three writes to `self.ql.os.last_error`, consistent with every other kernel32 hook. Co-Authored-By: Claude Opus 4.8 (1M context) --- qiling/os/windows/fiber.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qiling/os/windows/fiber.py b/qiling/os/windows/fiber.py index 09862076..f75fad16 100644 --- a/qiling/os/windows/fiber.py +++ b/qiling/os/windows/fiber.py @@ -33,7 +33,7 @@ class FiberManager: def free(self, idx: int) -> bool: if idx not in self.fibers: - self.last_error = ERROR_INVALID_PARAMETER + self.ql.os.last_error = ERROR_INVALID_PARAMETER return False fiber = self.fibers[idx] @@ -99,7 +99,7 @@ class FiberManager: def set(self, idx: int, data: int) -> bool: if idx not in self.fibers: - self.last_error = ERROR_INVALID_PARAMETER + self.ql.os.last_error = ERROR_INVALID_PARAMETER return False self.fibers[idx].data = data @@ -108,7 +108,7 @@ class FiberManager: def get(self, idx: int) -> int: if idx not in self.fibers: - self.last_error = ERROR_INVALID_PARAMETER + self.ql.os.last_error = ERROR_INVALID_PARAMETER return 0 return self.fibers[idx].data From 88ca4d475e81a2fb82b834c19dde9e8a3a089f99 Mon Sep 17 00:00:00 2001 From: Jvle Date: Mon, 3 Aug 2026 13:43:04 +0800 Subject: [PATCH 11/21] feat: add riscv gdb support Signed-off-by: Jvle --- qiling/debugger/gdb/gdb.py | 5 +- qiling/debugger/gdb/xml/riscv/riscv-core.xml | 47 ++++++++++++++++ qiling/debugger/gdb/xml/riscv/riscv-fpu.xml | 54 +++++++++++++++++++ qiling/debugger/gdb/xml/riscv/target.xml | 15 ++++++ .../debugger/gdb/xml/riscv64/riscv64-core.xml | 47 ++++++++++++++++ .../debugger/gdb/xml/riscv64/riscv64-fpu.xml | 54 +++++++++++++++++++ qiling/debugger/gdb/xml/riscv64/target.xml | 15 ++++++ qiling/debugger/gdb/xmlregs.py | 10 +++- 8 files changed, 245 insertions(+), 2 deletions(-) create mode 100644 qiling/debugger/gdb/xml/riscv/riscv-core.xml create mode 100644 qiling/debugger/gdb/xml/riscv/riscv-fpu.xml create mode 100644 qiling/debugger/gdb/xml/riscv/target.xml create mode 100644 qiling/debugger/gdb/xml/riscv64/riscv64-core.xml create mode 100644 qiling/debugger/gdb/xml/riscv64/riscv64-fpu.xml create mode 100644 qiling/debugger/gdb/xml/riscv64/target.xml diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index f6d6498d..11d7cf7c 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -184,6 +184,7 @@ class QlGdb(QlDebugger): from unicorn.arm64_const import UC_ARM64_REG_X29 from unicorn.mips_const import UC_MIPS_REG_INVALID from unicorn.ppc_const import UC_PPC_REG_31 + from unicorn.riscv_const import UC_RISCV_REG_S0 arch_uc_bp = { QL_ARCH.X86 : UC_X86_REG_EBP, @@ -193,7 +194,9 @@ class QlGdb(QlDebugger): QL_ARCH.MIPS : UC_MIPS_REG_INVALID, # skipped QL_ARCH.A8086 : UC_X86_REG_EBP, QL_ARCH.CORTEX_M : UC_ARM_REG_R11, - QL_ARCH.PPC : UC_PPC_REG_31 + QL_ARCH.PPC : UC_PPC_REG_31, + QL_ARCH.RISCV : UC_RISCV_REG_S0, + QL_ARCH.RISCV64 : UC_RISCV_REG_S0 }[self.ql.arch.type] def __get_reg_idx(ucreg: int) -> int: diff --git a/qiling/debugger/gdb/xml/riscv/riscv-core.xml b/qiling/debugger/gdb/xml/riscv/riscv-core.xml new file mode 100644 index 00000000..76180b82 --- /dev/null +++ b/qiling/debugger/gdb/xml/riscv/riscv-core.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/qiling/debugger/gdb/xml/riscv/riscv-fpu.xml b/qiling/debugger/gdb/xml/riscv/riscv-fpu.xml new file mode 100644 index 00000000..1e1838e9 --- /dev/null +++ b/qiling/debugger/gdb/xml/riscv/riscv-fpu.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/qiling/debugger/gdb/xml/riscv/target.xml b/qiling/debugger/gdb/xml/riscv/target.xml new file mode 100644 index 00000000..1d0eb2b0 --- /dev/null +++ b/qiling/debugger/gdb/xml/riscv/target.xml @@ -0,0 +1,15 @@ + + + + + + riscv:rv32 + GNU/Linux + + + + diff --git a/qiling/debugger/gdb/xml/riscv64/riscv64-core.xml b/qiling/debugger/gdb/xml/riscv64/riscv64-core.xml new file mode 100644 index 00000000..61ec01e1 --- /dev/null +++ b/qiling/debugger/gdb/xml/riscv64/riscv64-core.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/qiling/debugger/gdb/xml/riscv64/riscv64-fpu.xml b/qiling/debugger/gdb/xml/riscv64/riscv64-fpu.xml new file mode 100644 index 00000000..1e1838e9 --- /dev/null +++ b/qiling/debugger/gdb/xml/riscv64/riscv64-fpu.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/qiling/debugger/gdb/xml/riscv64/target.xml b/qiling/debugger/gdb/xml/riscv64/target.xml new file mode 100644 index 00000000..3f51b935 --- /dev/null +++ b/qiling/debugger/gdb/xml/riscv64/target.xml @@ -0,0 +1,15 @@ + + + + + + riscv:rv64 + GNU/Linux + + + + diff --git a/qiling/debugger/gdb/xmlregs.py b/qiling/debugger/gdb/xmlregs.py index 6bc2371f..a84ec670 100644 --- a/qiling/debugger/gdb/xmlregs.py +++ b/qiling/debugger/gdb/xmlregs.py @@ -42,6 +42,12 @@ from qiling.arch.ppc_const import ( reg_map as ppc_regs ) +from qiling.arch.riscv_const import ( + reg_csr_map as riscv_regs_csr, + reg_float_map as riscv_regs_float, + reg_map as riscv_regs +) + from qiling.const import QL_ARCH, QL_OS RegEntry = Tuple[Optional[int], int, int] @@ -147,7 +153,9 @@ class QlGdbFeatures: QL_ARCH.CORTEX_M: dict(**cortex_m_regs), QL_ARCH.ARM64: dict(**arm64_regs, **arm64_regs_v, **arm64_reg_map_fp), QL_ARCH.MIPS: dict(**mips_regs_gpr), - QL_ARCH.PPC: dict(**ppc_regs) + QL_ARCH.PPC: dict(**ppc_regs), + QL_ARCH.RISCV: dict(**riscv_regs, **riscv_regs_float, **riscv_regs_csr), + QL_ARCH.RISCV64: dict(**riscv_regs, **riscv_regs_float, **riscv_regs_csr) }[archtype] regsinfo = sorted(QlGdbFeatures.__walk_xml_regs(xmltree)) From 493e1f0b82ab9f62a1b3acc2a23132bb72496b62 Mon Sep 17 00:00:00 2001 From: Jvle Date: Mon, 3 Aug 2026 13:43:24 +0800 Subject: [PATCH 12/21] fix: flush gdb tb cache Signed-off-by: Jvle --- qiling/debugger/gdb/utils.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/qiling/debugger/gdb/utils.py b/qiling/debugger/gdb/utils.py index 5e3b208b..4a836d52 100644 --- a/qiling/debugger/gdb/utils.py +++ b/qiling/debugger/gdb/utils.py @@ -20,10 +20,12 @@ class QlGdbUtils: self.exit_point = exit_point self.swbp = set() self.last_bp = None + self._need_flush_tb = False def __entry_point_hook(ql: Qiling): ql.hook_del(ep_hret) ql.hook_code(self.dbg_hook) + self._need_flush_tb = True ql.log.info(f'{PROMPT} stopped at entry point: {ql.arch.regs.arch_pc:#x}') ql.stop() @@ -32,6 +34,13 @@ class QlGdbUtils: # that hook will be used to set up the breakpoint handling hook ep_hret = ql.hook_address(__entry_point_hook, entry_point) + def flush_tb(self): + """Flush Unicorn translation blocks after GDB hook or breakpoint updates.""" + flush_tb = getattr(self.ql.uc, 'ctl_flush_tb', None) + + if flush_tb is not None: + flush_tb() + def dbg_hook(self, ql: Qiling, address: int, size: int): if getattr(ql.arch, 'is_thumb', False): address |= 1 @@ -56,6 +65,8 @@ class QlGdbUtils: for bp in targets: self.swbp.add(bp) + self._need_flush_tb = True + self.ql.log.info(f'{PROMPT} breakpoint added at {addr:#x}') return True @@ -69,6 +80,8 @@ class QlGdbUtils: for bp in targets: self.swbp.remove(bp) + self._need_flush_tb = True + self.ql.log.info(f'{PROMPT} breakpoint removed from {addr:#x}') return True @@ -80,6 +93,10 @@ class QlGdbUtils: if getattr(self.ql.arch, 'is_thumb', False): address |= 0b1 + if self._need_flush_tb: + self.flush_tb() + self._need_flush_tb = False + op = f'stepping {steps} instructions' if steps else 'resuming' self.ql.log.info(f'{PROMPT} {op} from {address:#x}') From b69128fee974af1de13248434fe11f12f8df083a Mon Sep 17 00:00:00 2001 From: Jvle Date: Tue, 4 Aug 2026 16:03:27 +0800 Subject: [PATCH 13/21] feat: add riscv qdb support Signed-off-by: Jvle --- qiling/debugger/qdb/arch/__init__.py | 1 + qiling/debugger/qdb/arch/arch_riscv.py | 48 +++++ .../debugger/qdb/branch_predictor/__init__.py | 4 +- .../branch_predictor_riscv.py | 168 ++++++++++++++++++ qiling/debugger/qdb/helper.py | 6 +- qiling/debugger/qdb/render/__init__.py | 1 + qiling/debugger/qdb/render/render_riscv.py | 24 +++ qiling/debugger/qdb/utils.py | 15 +- tests/qdb_scripts/riscv32.qdb | 6 + tests/qdb_scripts/riscv64.qdb | 6 + tests/test_qdb.py | 14 ++ 11 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 qiling/debugger/qdb/arch/arch_riscv.py create mode 100644 qiling/debugger/qdb/branch_predictor/branch_predictor_riscv.py create mode 100644 qiling/debugger/qdb/render/render_riscv.py create mode 100644 tests/qdb_scripts/riscv32.qdb create mode 100644 tests/qdb_scripts/riscv64.qdb diff --git a/qiling/debugger/qdb/arch/__init__.py b/qiling/debugger/qdb/arch/__init__.py index 12ed30d1..3c08d730 100644 --- a/qiling/debugger/qdb/arch/__init__.py +++ b/qiling/debugger/qdb/arch/__init__.py @@ -6,3 +6,4 @@ from .arch_arm import ArchARM, ArchCORTEX_M from .arch_intel import ArchIntel, ArchX86, ArchX64 from .arch_mips import ArchMIPS +from .arch_riscv import ArchRISCV, ArchRISCV64 diff --git a/qiling/debugger/qdb/arch/arch_riscv.py b/qiling/debugger/qdb/arch/arch_riscv.py new file mode 100644 index 00000000..a4cb987e --- /dev/null +++ b/qiling/debugger/qdb/arch/arch_riscv.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +from __future__ import annotations + +from .arch import Arch + + +class ArchRISCV(Arch): + def __init__(self) -> None: + regs = ( + 'zero', 'ra', 'sp', 'gp', 'tp', + 't0', 't1', 't2', 's0', 's1', + 'a0', 'a1', 'a2', 'a3', 'a4', 'a5', 'a6', 'a7', + 's2', 's3', 's4', 's5', 's6', 's7', 's8', 's9', 's10', 's11', + 't3', 't4', 't5', 't6', 'pc' + ) + + aliases = { + 's0': 'fp' + } + + super().__init__(regs, aliases, 4, 4) + + def unalias(self, name: str) -> str: + if name.startswith('x') and name[1:].isdigit(): + idx = int(name[1:]) + + xregs = ( + 'zero', 'ra', 'sp', 'gp', 'tp', + 't0', 't1', 't2', 's0', 's1', + 'a0', 'a1', 'a2', 'a3', 'a4', 'a5', 'a6', 'a7', + 's2', 's3', 's4', 's5', 's6', 's7', 's8', 's9', 's10', 's11', + 't3', 't4', 't5', 't6' + ) + + if idx < len(xregs): + return xregs[idx] + + return super().unalias(name) + + +class ArchRISCV64(ArchRISCV): + def __init__(self) -> None: + super().__init__() + self._asize = 8 diff --git a/qiling/debugger/qdb/branch_predictor/__init__.py b/qiling/debugger/qdb/branch_predictor/__init__.py index 670f6534..952bb44a 100644 --- a/qiling/debugger/qdb/branch_predictor/__init__.py +++ b/qiling/debugger/qdb/branch_predictor/__init__.py @@ -7,10 +7,12 @@ from .branch_predictor import BranchPredictor from .branch_predictor_arm import BranchPredictorARM, BranchPredictorCORTEX_M from .branch_predictor_intel import BranchPredictorX86, BranchPredictorX64 from .branch_predictor_mips import BranchPredictorMIPS +from .branch_predictor_riscv import BranchPredictorRISCV, BranchPredictorRISCV64 __all__ = [ 'BranchPredictor', 'BranchPredictorARM', 'BranchPredictorCORTEX_M', 'BranchPredictorX86', 'BranchPredictorX64', - 'BranchPredictorMIPS' + 'BranchPredictorMIPS', + 'BranchPredictorRISCV', 'BranchPredictorRISCV64' ] diff --git a/qiling/debugger/qdb/branch_predictor/branch_predictor_riscv.py b/qiling/debugger/qdb/branch_predictor/branch_predictor_riscv.py new file mode 100644 index 00000000..2f71ec52 --- /dev/null +++ b/qiling/debugger/qdb/branch_predictor/branch_predictor_riscv.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +from __future__ import annotations + +from typing import Callable, Dict, List, Optional, Set + +from capstone import CS_OP_IMM, CS_OP_MEM, CS_OP_REG + +from .branch_predictor import BranchPredictor, Prophecy +from ..arch import ArchRISCV, ArchRISCV64 +from ..misc import InvalidInsn + + +class BranchPredictorRISCV(BranchPredictor, ArchRISCV): + """Branch Predictor for RISC-V 32-bit. + """ + + stop = 'ebreak' + xlen = 32 + supports_c_jal = True + + def _unconditional_branches(self) -> Set[str]: + branches = {'j', 'jal', 'jalr', 'jr', 'ret'} + + if self.supports_c_jal: + branches.add('c.jal') + + return branches + + def _normalize_mnemonic(self, mnemonic: str) -> str: + return (mnemonic or '').lower() + + def _signed(self, val: int) -> int: + mask = (1 << self.xlen) - 1 + sign = 1 << (self.xlen - 1) + + val &= mask + return (val ^ sign) - sign + + def _unsigned(self, val: int) -> int: + return val & ((1 << self.xlen) - 1) + + def predict(self) -> Prophecy: + insn = self.disasm(self.cur_addr, True) + + going = False + where = 0 + + if isinstance(insn, InvalidInsn): + return Prophecy(going, where) + + mnemonic = self._normalize_mnemonic(insn.mnemonic) + base = mnemonic[2:] if mnemonic.startswith('c.') else mnemonic + operands: List[object] = list(insn.operands) + + conditional: Dict[str, Callable[..., bool]] = { + 'beq' : lambda a, b: a == b, + 'bne' : lambda a, b: a != b, + 'blt' : lambda a, b: a < b, + 'bge' : lambda a, b: a >= b, + 'bltu': lambda a, b: a < b, + 'bgeu': lambda a, b: a >= b, + 'beqz': lambda a: a == 0, + 'bnez': lambda a: a != 0, + 'bgez': lambda a: a >= 0, + 'bltz': lambda a: a < 0, + 'bgtz': lambda a: a > 0, + 'blez': lambda a: a <= 0, + } + + def __read_reg(reg: int) -> Optional[int]: + name = insn.reg_name(reg) + + return name and self.read_reg(self.unalias(name)) + + def __parse_op(op: object) -> Optional[int]: + if getattr(op, 'type', None) == CS_OP_REG: + return __read_reg(op.reg) + + if getattr(op, 'type', None) == CS_OP_IMM: + return op.imm + + if getattr(op, 'type', None) == CS_OP_MEM: + mem = op.mem + base_reg = __read_reg(mem.base) or 0 + index = __read_reg(mem.index) or 0 + return base_reg + index * mem.scale + mem.disp + + return None + + def __direct_target() -> Optional[int]: + imms = [op for op in operands if getattr(op, 'type', None) == CS_OP_IMM] + + if not imms: + return None + + return self.cur_addr + imms[-1].imm + + def __indirect_target() -> Optional[int]: + memop = next((op for op in operands if getattr(op, 'type', None) == CS_OP_MEM), None) + if memop is not None: + return __parse_op(memop) + + regs = [op for op in operands if getattr(op, 'type', None) == CS_OP_REG] + imms = [op for op in operands if getattr(op, 'type', None) == CS_OP_IMM] + + if regs and imms: + return (__parse_op(regs[-1]) or 0) + imms[-1].imm + + if regs: + return __parse_op(regs[-1]) + + return None + + unconditional = self._unconditional_branches() + is_unconditional = mnemonic in unconditional or base in unconditional + + if mnemonic == 'c.jal' and not self.supports_c_jal: + is_unconditional = False + + if is_unconditional: + going = True + + if base == 'ret' and not operands: + where = self.read_reg('ra') + elif base in {'jalr', 'jr', 'ret'}: + where = __indirect_target() or 0 + else: + where = __direct_target() or 0 + + if base in {'jalr', 'jr', 'ret'}: + where &= ~1 + + elif base in conditional: + predicate = conditional[base] + + if base in {'beqz', 'bnez', 'bgez', 'bltz', 'bgtz', 'blez'}: + reg = __parse_op(operands[0]) if operands else None + going = reg is not None and predicate(self._signed(reg)) + else: + lhs = __parse_op(operands[0]) if len(operands) > 0 else None + rhs = __parse_op(operands[1]) if len(operands) > 1 else None + + if base in {'blt', 'bge'}: + lhs = None if lhs is None else self._signed(lhs) + rhs = None if rhs is None else self._signed(rhs) + + if base in {'bltu', 'bgeu'}: + lhs = None if lhs is None else self._unsigned(lhs) + rhs = None if rhs is None else self._unsigned(rhs) + + going = lhs is not None and rhs is not None and predicate(lhs, rhs) + + if going: + where = __direct_target() or 0 + + return Prophecy(going, where) + + +class BranchPredictorRISCV64(BranchPredictorRISCV, ArchRISCV64): + """Branch Predictor for RISC-V 64-bit. + """ + + xlen = 64 + supports_c_jal = False diff --git a/qiling/debugger/qdb/helper.py b/qiling/debugger/qdb/helper.py index fd6c05bf..6245c806 100644 --- a/qiling/debugger/qdb/helper.py +++ b/qiling/debugger/qdb/helper.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, List, Tuple from qiling.const import QL_ARCH from .context import Context -from .arch import ArchCORTEX_M, ArchARM, ArchMIPS, ArchX86, ArchX64 +from .arch import ArchCORTEX_M, ArchARM, ArchMIPS, ArchRISCV, ArchRISCV64, ArchX86, ArchX64 if TYPE_CHECKING: @@ -26,7 +26,9 @@ def setup_command_helper(ql: Qiling): QL_ARCH.X8664: ArchX64, QL_ARCH.MIPS: ArchMIPS, QL_ARCH.ARM: ArchARM, - QL_ARCH.CORTEX_M: ArchCORTEX_M + QL_ARCH.CORTEX_M: ArchCORTEX_M, + QL_ARCH.RISCV: ArchRISCV, + QL_ARCH.RISCV64: ArchRISCV64, } ret = type('CommandHelper', (CommandHelper, atypes[ql.arch.type]), {}) diff --git a/qiling/debugger/qdb/render/__init__.py b/qiling/debugger/qdb/render/__init__.py index 0b7e6180..17e43008 100644 --- a/qiling/debugger/qdb/render/__init__.py +++ b/qiling/debugger/qdb/render/__init__.py @@ -7,3 +7,4 @@ from .render import ContextRender from .render_intel import ContextRenderX86, ContextRenderX64 from .render_mips import ContextRenderMIPS from .render_arm import ContextRenderARM, ContextRenderCORTEX_M +from .render_riscv import ContextRenderRISCV, ContextRenderRISCV64 diff --git a/qiling/debugger/qdb/render/render_riscv.py b/qiling/debugger/qdb/render/render_riscv.py new file mode 100644 index 00000000..ce95592a --- /dev/null +++ b/qiling/debugger/qdb/render/render_riscv.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +from .render import ContextRender +from ..arch import ArchRISCV, ArchRISCV64 + + +class ContextRenderRISCV(ContextRender, ArchRISCV): + """Context renderer for RISC-V architecture. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.regs_a_row = 5 + + def print_mode_info(self) -> None: + pass + + +class ContextRenderRISCV64(ContextRenderRISCV, ArchRISCV64): + """Context renderer for RISC-V 64-bit architecture. + """ diff --git a/qiling/debugger/qdb/utils.py b/qiling/debugger/qdb/utils.py index 03be0ba8..8da059c1 100644 --- a/qiling/debugger/qdb/utils.py +++ b/qiling/debugger/qdb/utils.py @@ -16,7 +16,9 @@ from .render import ( ContextRenderX64, ContextRenderARM, ContextRenderCORTEX_M, - ContextRenderMIPS + ContextRenderMIPS, + ContextRenderRISCV, + ContextRenderRISCV64, ) from .branch_predictor import ( @@ -26,6 +28,8 @@ from .branch_predictor import ( BranchPredictorARM, BranchPredictorCORTEX_M, BranchPredictorMIPS, + BranchPredictorRISCV, + BranchPredictorRISCV64, ) from .const import color, QDB_MSG @@ -111,13 +115,16 @@ def setup_branch_predictor(ql: Qiling) -> BranchPredictor: QL_ARCH.X8664: BranchPredictorX64, QL_ARCH.ARM: BranchPredictorARM, QL_ARCH.CORTEX_M: BranchPredictorCORTEX_M, - QL_ARCH.MIPS: BranchPredictorMIPS + QL_ARCH.MIPS: BranchPredictorMIPS, + QL_ARCH.RISCV: BranchPredictorRISCV, + QL_ARCH.RISCV64: BranchPredictorRISCV64, } p = preds[ql.arch.type] return p(ql) + def setup_context_render(ql: Qiling, predictor: BranchPredictor) -> ContextRender: """Setup context render according to arch. """ @@ -127,7 +134,9 @@ def setup_context_render(ql: Qiling, predictor: BranchPredictor) -> ContextRende QL_ARCH.X8664: ContextRenderX64, QL_ARCH.ARM: ContextRenderARM, QL_ARCH.CORTEX_M: ContextRenderCORTEX_M, - QL_ARCH.MIPS: ContextRenderMIPS + QL_ARCH.MIPS: ContextRenderMIPS, + QL_ARCH.RISCV: ContextRenderRISCV, + QL_ARCH.RISCV64: ContextRenderRISCV64, } r = rends[ql.arch.type] diff --git a/tests/qdb_scripts/riscv32.qdb b/tests/qdb_scripts/riscv32.qdb new file mode 100644 index 00000000..745aad55 --- /dev/null +++ b/tests/qdb_scripts/riscv32.qdb @@ -0,0 +1,6 @@ +x/8xw $sp +x/8i $pc +s 1 +info snapshot +p +q diff --git a/tests/qdb_scripts/riscv64.qdb b/tests/qdb_scripts/riscv64.qdb new file mode 100644 index 00000000..745aad55 --- /dev/null +++ b/tests/qdb_scripts/riscv64.qdb @@ -0,0 +1,6 @@ +x/8xw $sp +x/8i $pc +s 1 +info snapshot +p +q diff --git a/tests/test_qdb.py b/tests/test_qdb.py index 563dd840..8498d6c8 100644 --- a/tests/test_qdb.py +++ b/tests/test_qdb.py @@ -53,6 +53,20 @@ class DebuggerTest(unittest.TestCase): r'qdb_scripts/x86.qdb' ) + def test_qdb_riscv32_hello(self): + self.__test_common( + r'/bin/hello', + r'../examples/rootfs/riscv32_linux', + r'qdb_scripts/riscv32.qdb' + ) + + def test_qdb_riscv64_hello(self): + self.__test_common( + r'/bin/hello', + r'../examples/rootfs/riscv64_linux', + r'qdb_scripts/riscv64.qdb' + ) + if __name__ == '__main__': unittest.main() From 718280f68482534b62e97d982a8a9b781e377450 Mon Sep 17 00:00:00 2001 From: Jvle Date: Tue, 4 Aug 2026 16:03:56 +0800 Subject: [PATCH 14/21] fix: render trailing qdb register row Signed-off-by: Jvle --- qiling/debugger/qdb/render/render.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qiling/debugger/qdb/render/render.py b/qiling/debugger/qdb/render/render.py index b1d62b85..8d39008e 100644 --- a/qiling/debugger/qdb/render/render.py +++ b/qiling/debugger/qdb/render/render.py @@ -108,6 +108,9 @@ class Render: elements.clear() + if elements: + yield '\t'.join(elements) + for line in __render_regs_line(): print(line) From dbf1cab94aaee6770c82331c14ac7c1cec44206c Mon Sep 17 00:00:00 2001 From: Gigithecode Date: Fri, 7 Aug 2026 18:32:55 +0700 Subject: [PATCH 15/21] Fix broken __future__ import in IDA plugin custom_script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first line reads `from future import __annotations__`, with the module name and the imported symbol swapped. The intent was clearly the PEP 563 future statement `from __future__ import annotations`. As written, the line fails in both possible situations: - without the third-party `future` package installed (the common case), the import raises `ModuleNotFoundError`, so `QILING_IDA` is never defined; - with that package installed, the import silently succeeds but binds the module's empty `__annotations__` dict, so PEP 563 is *not* enabled — while the annotations in this file (`ql: Qiling`) rely on it, `Qiling` being imported under `TYPE_CHECKING` only. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Pru9dwqPVrDakERYZwzjUx --- examples/extensions/idaplugin/custom_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/extensions/idaplugin/custom_script.py b/examples/extensions/idaplugin/custom_script.py index 6461f45d..dc92b1b6 100644 --- a/examples/extensions/idaplugin/custom_script.py +++ b/examples/extensions/idaplugin/custom_script.py @@ -1,4 +1,4 @@ -from future import __annotations__ +from __future__ import annotations from typing import TYPE_CHECKING, List From de34db212da6e6295e86a0cdb9c2e1b9ad7d9bf8 Mon Sep 17 00:00:00 2001 From: retrocpugeek Date: Fri, 7 Aug 2026 22:28:08 +1000 Subject: [PATCH 16/21] gdb: report SIGTRAP (not SIGTERM) on single-step handle_s answered every single-step ('s') with a SIGTERM stop-reply whenever ql.emu_state was QL_STATE.STOPPED. But emu_start always leaves the state STOPPED after running the requested step count, so the guard was true on every step and gdb clients saw a spurious termination signal, disconnecting mid-debug (issues #1377, #1538). Give handle_s the same exit-vs-trap discrimination handle_c already uses: a step reports SIGTRAP unless it carried pc to the emulation exit point, in which case the guest has actually terminated and we reply W{exit_code}. Also wrap the step in the same UcError/KeyboardInterrupt handling as handle_c so a fault while stepping maps to a signal instead of crashing the stub, and hoist the shared uc-error->signal map to a module constant. Add a regression test that single-steps over the gdb stub and asserts the stop-reply is 'S05' (SIGTRAP), which fails as 'S0f' (SIGTERM) without the fix. Co-Authored-By: Claude Opus 4.8 --- qiling/debugger/gdb/gdb.py | 52 ++++++++++++++--------- tests/test_debugger.py | 86 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 19 deletions(-) diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index f6d6498d..d31544ee 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -30,7 +30,7 @@ from unicorn.unicorn_const import ( ) from qiling import Qiling -from qiling.const import QL_ARCH, QL_ENDIAN, QL_OS, QL_STATE +from qiling.const import QL_ARCH, QL_ENDIAN, QL_OS from qiling.debugger import QlDebugger from qiling.debugger.gdb.xmlregs import QlGdbFeatures from qiling.debugger.gdb.utils import QlGdbUtils @@ -56,6 +56,21 @@ SIGCHLD = 16 SIGCONT = 17 SIGSTOP = 18 +# translate a unicorn cpu fault into the closest posix signal, shared by the +# continue ('c') and single-step ('s') handlers when emulation faults +UC_ERROR_SIGMAP = { + UC_ERR_READ_UNMAPPED : SIGSEGV, + UC_ERR_WRITE_UNMAPPED : SIGSEGV, + UC_ERR_FETCH_UNMAPPED : SIGSEGV, + UC_ERR_WRITE_PROT : SIGSEGV, + UC_ERR_READ_PROT : SIGSEGV, + UC_ERR_FETCH_PROT : SIGSEGV, + UC_ERR_READ_UNALIGNED : SIGBUS, + UC_ERR_WRITE_UNALIGNED : SIGBUS, + UC_ERR_FETCH_UNALIGNED : SIGBUS, + UC_ERR_INSN_INVALID : SIGILL +} + # common replies REPLY_ACK = b'+' REPLY_EMPTY = b'' @@ -226,21 +241,8 @@ class QlGdb(QlDebugger): try: self.gdb.resume_emu() except UcError as err: - sigmap = { - UC_ERR_READ_UNMAPPED : SIGSEGV, - UC_ERR_WRITE_UNMAPPED : SIGSEGV, - UC_ERR_FETCH_UNMAPPED : SIGSEGV, - UC_ERR_WRITE_PROT : SIGSEGV, - UC_ERR_READ_PROT : SIGSEGV, - UC_ERR_FETCH_PROT : SIGSEGV, - UC_ERR_READ_UNALIGNED : SIGBUS, - UC_ERR_WRITE_UNALIGNED : SIGBUS, - UC_ERR_FETCH_UNALIGNED : SIGBUS, - UC_ERR_INSN_INVALID : SIGILL - } - # determine signal from uc error; default to SIGTERM - reply = f'S{sigmap.get(err.errno, SIGTERM):02x}' + reply = f'S{UC_ERROR_SIGMAP.get(err.errno, SIGTERM):02x}' except KeyboardInterrupt: # emulation was interrupted with ctrl+c @@ -678,11 +680,23 @@ class QlGdb(QlDebugger): """Perform a single step. """ - self.gdb.resume_emu(steps=1) + try: + self.gdb.resume_emu(steps=1) + except UcError as err: + # stepping faulted; report the closest posix signal + return f'S{UC_ERROR_SIGMAP.get(err.errno, SIGTERM):02x}' - # if emulation has been stopped, signal program termination - if self.ql.emu_state is QL_STATE.STOPPED: - return f'S{SIGTERM:02x}' + except KeyboardInterrupt: + # emulation was interrupted with ctrl+c + return f'S{SIGINT:02x}' + + # emu_start always leaves emu_state as STOPPED after a step, so that + # cannot tell an ordinary step apart from the guest exiting (see + # issues #1377 and #1538). instead, the guest has terminated only + # when the step carried pc all the way to the emulation exit point. + if getattr(self.ql.arch, 'effective_pc', self.ql.arch.regs.arch_pc) == self.gdb.exit_point: + # program terminated; report its exit code + return f'W{self.ql.os.exit_code:02x}' # otherwise, this is just single stepping return f'S{SIGTRAP:02x}' diff --git a/tests/test_debugger.py b/tests/test_debugger.py index c8df8ae0..792d8e2f 100644 --- a/tests/test_debugger.py +++ b/tests/test_debugger.py @@ -44,6 +44,55 @@ class SimpleGdbClient: self.__file.flush() +class ReadingGdbClient: + """A minimal gdb remote client that can also read replies, so tests can + assert on the stop-reply packets the server sends back. + """ + + def __init__(self, host: str, port: int): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((host, port)) + sock.settimeout(10) + + self.__sock = sock + self.__buf = b'' + + def __enter__(self): + return self + + def __exit__(self, ex_type, ex_value, ex_traceback): + self.__sock.close() + + @staticmethod + def checksum(data: str) -> int: + return sum(ord(c) for c in data) & 0xff + + def send(self, msg: str): + self.__sock.sendall(f'${msg}#{ReadingGdbClient.checksum(msg):02x}'.encode('latin')) + + def read_packet(self) -> str: + """Read a single '$#' reply, skipping any '+'/'-' acks. + """ + + # wait for a packet start marker + while b'$' not in self.__buf: + self.__buf += self.__sock.recv(4096) + + start = self.__buf.index(b'$') + + # wait until the terminating '#' and its two checksum digits arrived + while True: + end = self.__buf.find(b'#', start) + if end != -1 and len(self.__buf) >= end + 3: + break + self.__buf += self.__sock.recv(4096) + + data = self.__buf[start + 1:end] + self.__buf = self.__buf[end + 3:] + + return data.decode('latin') + + class DebuggerTest(unittest.TestCase): def test_gdbdebug_file_server(self): ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_hello"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG) @@ -168,6 +217,43 @@ class DebuggerTest(unittest.TestCase): ql.run() del ql + def test_gdbdebug_stepi_reports_sigtrap(self): + # regression for issues #1377 and #1538: a single-step ('s') used to be + # answered with a SIGTERM stop-reply because emu_state is always STOPPED + # after a step, which made gdb clients believe the program had died. the + # stop-reply for an ordinary step must be a SIGTRAP ('S05'), never a + # SIGTERM ('S0f'). + ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_hello"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.OFF) + ql.debugger = 'gdb:127.0.0.1:9996' + + replies = [] + + def gdb_test_client(): + # yield to allow ql to launch its gdbserver + time.sleep(1.337 * 2) + + with ReadingGdbClient('127.0.0.1', 9996) as client: + client.send('qSupported:multiprocess+;swbreak+;hwbreak+;vContSupported+;xmlRegisters=i386') + client.read_packet() + client.send('QStartNoAckMode') + client.read_packet() + + # step a few instructions; every reply must be a SIGTRAP stop + for _ in range(3): + client.send('s') + replies.append(client.read_packet()) + + client.send('k') + + thread = threading.Thread(target=gdb_test_client, daemon=True) + thread.start() + + ql.run() + thread.join(timeout=30) + del ql + + self.assertEqual(replies, ['S05', 'S05', 'S05']) + def test_gdbdebug_shellcode_server(self): X8664_LIN = bytes.fromhex('31c048bbd19d9691d08c97ff48f7db53545f995257545eb03b0f05') From c5a8a9b73ea4ceb40b424a9a3373a11beddc077f Mon Sep 17 00:00:00 2001 From: retrocpugeek Date: Sun, 12 Jul 2026 19:15:25 +1000 Subject: [PATCH 17/21] gdb: support async interrupt (ctrl-c) to break into a running guest The stub drove continue by calling emu_start synchronously and only read the socket again once the target stopped on its own, so the bare \x03 break byte a client sends to pause a running target was never seen. A guest that free-runs (e.g. an idle/event loop) could not be interrupted at all -- gdb/Ghidra reported 'Cannot execute this command while the target is running'. Poll the client socket for the break byte from the per-instruction run hook (dbg_hook), which does run on the emulation thread during emu_start: - GdbSerialConn.poll_interrupt(): non-blocking select+recv, True on \x03. - QlGdbUtils.dbg_hook: throttled (every INTR_POLL_INTERVAL insns) check of an installed check_interrupt callback; on a break, stop emulation and record it. - handle_c: reply SIGINT when the stop was an interrupt rather than a breakpoint or normal exit. Add a regression test that lets an infinite-loop guest free-run, sends the bare break byte and asserts the stop-reply is 'S02' (SIGINT). Without the fix no reply ever arrives, so the test stops the guest itself and fails rather than hanging the run. Also verified against a free-running MIPS64 BE guest: \x03 -> S02 in <1ms. Co-Authored-By: Claude Opus 4.8 (1M context) --- qiling/debugger/gdb/gdb.py | 31 +++++++++++++++++++- qiling/debugger/gdb/utils.py | 35 +++++++++++++++++++++++ tests/test_debugger.py | 55 ++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 1 deletion(-) diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index d31544ee..c6f591a5 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -15,6 +15,7 @@ import os import socket +import select import re import tempfile from functools import partial @@ -139,6 +140,10 @@ class QlGdb(QlDebugger): server = GdbSerialConn(self.ip, self.port, self.ql.log) killed = False + # let the run hook break into a free-running guest when the client sends + # an async interrupt (ctrl-c / \x03); see QlGdbUtils.dbg_hook. + self.gdb.check_interrupt = server.poll_interrupt + def __hexstr(value: int, nibbles: int = 0) -> str: """Encode a value into a hex string. """ @@ -249,7 +254,10 @@ class QlGdb(QlDebugger): reply = f'S{SIGINT:02x}' else: - if getattr(self.ql.arch, 'effective_pc', self.ql.arch.regs.arch_pc) == self.gdb.last_bp: + if self.gdb.interrupted: + # emulation was stopped by an async interrupt from the client + reply = f'S{SIGINT:02x}' + elif getattr(self.ql.arch, 'effective_pc', self.ql.arch.regs.arch_pc) == self.gdb.last_bp: # emulation stopped because it hit a breakpoint reply = f'S{SIGTRAP:02x}' else: @@ -837,6 +845,27 @@ class GdbSerialConn: self.client.close() self.sock.close() + def poll_interrupt(self) -> bool: + """Non-blocking check for an async interrupt from the client. + + While the target is running the only thing gdb sends is a bare ``\\x03`` + break byte (it waits for a stop reply before sending anything else), so + any readable bytes here are that interrupt or a stray protocol ack. + Returns True if a break was seen. Called from the run hook, so it must + never block. + """ + + readable, _, _ = select.select([self.client], [], [], 0) + if not readable: + return False + + try: + incoming = self.client.recv(self.BUFSIZE) + except (ConnectionError, OSError): + return False + + return b'\x03' in incoming + def readpackets(self) -> Iterator[bytes]: """Iterate through incoming packets in an active connection until it is terminated. diff --git a/qiling/debugger/gdb/utils.py b/qiling/debugger/gdb/utils.py index 5e3b208b..6413d1ee 100644 --- a/qiling/debugger/gdb/utils.py +++ b/qiling/debugger/gdb/utils.py @@ -14,6 +14,11 @@ PROMPT = r'gdb>' class QlGdbUtils: + # how often (in guest instructions) the run hook polls the client socket for + # an async interrupt. small enough to feel instant, large enough that the + # extra non-blocking socket check does not dominate the per-instruction hook. + INTR_POLL_INTERVAL = 200 + def __init__(self, ql: Qiling, entry_point: int, exit_point: int): self.ql = ql @@ -21,6 +26,14 @@ class QlGdbUtils: self.swbp = set() self.last_bp = None + # async-interrupt support: `check_interrupt` is a callable installed by the + # gdb stub that returns True when the client sent a break (ctrl-c / \x03) + # while the target was running. `interrupted` records that the last resume + # stopped for that reason (rather than a breakpoint or normal exit). + self.check_interrupt = None + self.interrupted = False + self._poll_counter = 0 + def __entry_point_hook(ql: Qiling): ql.hook_del(ep_hret) ql.hook_code(self.dbg_hook) @@ -36,6 +49,24 @@ class QlGdbUtils: if getattr(ql.arch, 'is_thumb', False): address |= 1 + # poll for an async interrupt from the client (gdb sends a bare \x03 while + # the target is running). throttled so the socket check stays off the hot + # path. this is the only way to break into a free-running guest, since the + # stub's packet loop is blocked inside emu_start until the target stops. + if self.check_interrupt is not None: + self._poll_counter += 1 + + if self._poll_counter >= self.INTR_POLL_INTERVAL: + self._poll_counter = 0 + + if self.check_interrupt(): + self.interrupted = True + self.last_bp = None + + ql.log.info(f'{PROMPT} interrupted by client, stopped at {address:#x}') + ql.stop() + return + # resuming emulation after hitting a breakpoint will re-enter this hook. # avoid an endless hooking loop by detecting and skipping this case if address == self.last_bp: @@ -83,4 +114,8 @@ class QlGdbUtils: op = f'stepping {steps} instructions' if steps else 'resuming' self.ql.log.info(f'{PROMPT} {op} from {address:#x}') + # clear any pending interrupt state from a previous resume + self.interrupted = False + self._poll_counter = 0 + self.ql.emu_start(address, self.exit_point, count=steps) diff --git a/tests/test_debugger.py b/tests/test_debugger.py index 792d8e2f..6219cca4 100644 --- a/tests/test_debugger.py +++ b/tests/test_debugger.py @@ -70,6 +70,12 @@ class ReadingGdbClient: def send(self, msg: str): self.__sock.sendall(f'${msg}#{ReadingGdbClient.checksum(msg):02x}'.encode('latin')) + def send_break(self): + """Send an async interrupt: a bare '\x03' byte, outside of packet framing. + """ + + self.__sock.sendall(b'\x03') + def read_packet(self) -> str: """Read a single '$#' reply, skipping any '+'/'-' acks. """ @@ -254,6 +260,55 @@ class DebuggerTest(unittest.TestCase): self.assertEqual(replies, ['S05', 'S05', 'S05']) + def test_gdbdebug_async_interrupt(self): + # a free-running guest could not be interrupted at all: the stub is blocked + # inside emu_start while the target runs, so the bare '\x03' break byte a + # client sends to pause it was never read, and clients reported 'Cannot + # execute this command while the target is running'. a break must stop the + # guest and yield a SIGINT ('S02') stop-reply. + INFINITE_LOOP = bytes.fromhex('90ebfd') # nop ; jmp -3 + + ql = Qiling(code=INFINITE_LOOP, archtype=QL_ARCH.X8664, ostype=QL_OS.LINUX, verbose=QL_VERBOSE.OFF) + ql.debugger = 'gdb:127.0.0.1:9994' + + replies = [] + + def gdb_test_client(): + # yield to allow ql to launch its gdbserver + time.sleep(1.337 * 2) + + with ReadingGdbClient('127.0.0.1', 9994) as client: + client.send('qSupported:multiprocess+;swbreak+;hwbreak+;vContSupported+;xmlRegisters=i386') + client.read_packet() + client.send('QStartNoAckMode') + client.read_packet() + + # let the guest free-run; it never stops on its own + client.send('c') + + # give it time to spin, then break into it + time.sleep(1.337) + client.send_break() + + try: + replies.append(client.read_packet()) + except OSError: + # the stub ignored the break and the guest is still spinning. + # stop it from here so the assertion below reports the failure + # instead of hanging the test run forever + ql.stop() + + client.send('k') + + thread = threading.Thread(target=gdb_test_client, daemon=True) + thread.start() + + ql.run() + thread.join(timeout=30) + del ql + + self.assertEqual(replies, ['S02']) + def test_gdbdebug_shellcode_server(self): X8664_LIN = bytes.fromhex('31c048bbd19d9691d08c97ff48f7db53545f995257545eb03b0f05') From a13a8c8c1faf2ed0575999803045d89ab5102c1e Mon Sep 17 00:00:00 2001 From: retrocpugeek Date: Sun, 23 Aug 2026 22:15:04 +1000 Subject: [PATCH 18/21] gdb: accept vCont C/S actions carrying any signal A client that resumes while a signal is pending sends a vCont action of the form 'C' (continue and deliver) or 'S' (step and deliver), e.g. 'vCont;S0f:pa410.1996;c:pa410.-1' -- the exact packet reported in issue #1377. handle_v matched only 'c'/'C05' and 's'/'S05', so any other signal fell through to an empty reply and clients aborted the session with 'Invalid remote reply:'. We do not deliver host signals to the guest, so the signal value carries no meaning for us: accept any of them and carry the action out as a plain resume or step, which is what the client asked for. This matters more now that the stub can stop with SIGINT on an async interrupt, since a client may well resume from such a stop with 'C02'. Also stop assuming os.exit_code exists when reporting termination: bare-metal os layers (QlOsMcu) do not define it, which turns the exit path into an AttributeError (seen in issue #1276). The underlying MCU interrupt handling of #1276 is out of scope here. Add a regression test asserting the '#1377' packet is answered with a SIGTRAP stop-reply and that a signalled continue runs the guest to termination; both replies are empty without the fix. Fixes #1377 Co-Authored-By: Claude Opus 5 (1M context) --- qiling/debugger/gdb/gdb.py | 18 ++++++++++---- tests/test_debugger.py | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index c6f591a5..77c0623f 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -261,8 +261,9 @@ class QlGdb(QlDebugger): # emulation stopped because it hit a breakpoint reply = f'S{SIGTRAP:02x}' else: - # emulation has completed successfully - reply = f'W{self.ql.os.exit_code:02x}' + # emulation has completed successfully. note bare-metal os layers + # do not have an exit code (issue #1276) + reply = f'W{getattr(self.ql.os, "exit_code", 0):02x}' return reply @@ -672,10 +673,17 @@ class QlGdb(QlDebugger): for grp in groups: cmd, *tid = grp.split(':', maxsplit=1) - if cmd in ('c', f'C{SIGTRAP:02x}'): + # 'C sig' and 'S sig' resume or step while delivering a signal + # to the guest. we do not deliver signals, so the signal value + # is ignored and the action is carried out as a plain resume or + # step. matching only 'C05' and 'S05' here made clients that + # resume with any other pending signal (e.g. 'S0f' after a stop + # reply we sent) receive an empty reply and bail out with + # 'Invalid remote reply' (issue #1377) + if cmd[:1] in ('c', 'C'): return handle_c('') - elif cmd in ('s', f'S{SIGTRAP:02x}'): + elif cmd[:1] in ('s', 'S'): return handle_s('') # FIXME: not sure how to handle multiple command @@ -704,7 +712,7 @@ class QlGdb(QlDebugger): # when the step carried pc all the way to the emulation exit point. if getattr(self.ql.arch, 'effective_pc', self.ql.arch.regs.arch_pc) == self.gdb.exit_point: # program terminated; report its exit code - return f'W{self.ql.os.exit_code:02x}' + return f'W{getattr(self.ql.os, "exit_code", 0):02x}' # otherwise, this is just single stepping return f'S{SIGTRAP:02x}' diff --git a/tests/test_debugger.py b/tests/test_debugger.py index 6219cca4..17ae7b8d 100644 --- a/tests/test_debugger.py +++ b/tests/test_debugger.py @@ -260,6 +260,55 @@ class DebuggerTest(unittest.TestCase): self.assertEqual(replies, ['S05', 'S05', 'S05']) + def test_gdbdebug_vcont_signal_actions(self): + # regression for issue #1377: a client resuming with a pending signal sends + # a 'vCont;S' or 'vCont;C' action (e.g. 'S0f' after a stop-reply we + # sent). the stub only recognized 'S05' and 'C05', replied empty to anything + # else, and the client bailed out with 'Invalid remote reply'. any signal must + # be accepted and carried out as a plain step or resume. + ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_hello"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.OFF) + ql.debugger = 'gdb:127.0.0.1:9995' + + replies = [] + + def gdb_test_client(): + # yield to allow ql to launch its gdbserver + time.sleep(1.337 * 2) + + with ReadingGdbClient('127.0.0.1', 9995) as client: + client.send('qSupported:multiprocess+;swbreak+;hwbreak+;vContSupported+;xmlRegisters=i386') + client.read_packet() + client.send('QStartNoAckMode') + client.read_packet() + + client.send('vCont?') + replies.append(client.read_packet()) + + # step while delivering SIGTERM; this is the packet reported in #1377 + client.send('vCont;S0f:p1.1;c:p1.-1') + replies.append(client.read_packet()) + + # resume while delivering SIGTERM; runs the guest to completion + client.send('vCont;C0f:p1.1') + replies.append(client.read_packet()) + + client.send('k') + + thread = threading.Thread(target=gdb_test_client, daemon=True) + thread.start() + + ql.run() + thread.join(timeout=30) + del ql + + self.assertEqual(replies[0], 'vCont;c;C;s;S') + + # the signalled step is an ordinary step: SIGTRAP, not an empty reply + self.assertEqual(replies[1], 'S05') + + # the signalled resume ran to termination and reported an exit code + self.assertTrue(replies[2].startswith('W'), f'unexpected reply to signalled resume: {replies[2]!r}') + def test_gdbdebug_async_interrupt(self): # a free-running guest could not be interrupted at all: the stub is blocked # inside emu_start while the target runs, so the bare '\x03' break byte a From 64054a0129ec5c88925706aa08ba600ffa608944 Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 1 Sep 2026 12:15:35 +0800 Subject: [PATCH 19/21] update new ARCHITECTURE docs --- ARCHITECTURE.md | 313 +++++++++++++++++++++++++++-------- ARCHITECTURE/arch.md | 4 +- ARCHITECTURE/debugger.md | 8 +- ARCHITECTURE/hw.md | 8 +- ARCHITECTURE/loader.md | 2 +- ARCHITECTURE/os-baremetal.md | 21 ++- ARCHITECTURE/os-base.md | 10 +- ARCHITECTURE/os-windows.md | 4 +- 8 files changed, 282 insertions(+), 88 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 56843aed..11f4437b 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -31,13 +31,13 @@ with reverse debugging), and fuzzing integration (AFL++/unicornafl). - **Shape**: pure-Python library (`from qiling import Qiling`), plus the `qltool` CLI and `qltui.py` TUI. Also shipped as a Docker image (`Dockerfile`) and PyPI package. -- **Python**: 3.8+ (`pyproject.toml:30`). Packaging via Poetry; version 1.4.8 - (`pyproject.toml:4`), status Beta, license GPLv2. -- **Core dependencies**: `unicorn ==2.1.3` (hard-pinned CPU emulator), - `capstone` (disassembly), `keystone-engine` (assembly), `pefile`, - `pyelftools`, `python-registry`, `gevent` (multithread emulation), - `pyyaml` (MCU profiles). Extras: `fuzz` → unicornafl/fuzzercorn, - `RE` → r2libr. +- **Python**: 3.10+ (`pyproject.toml:27`). Packaging via Poetry; version + 1.4.11-dev (`pyproject.toml:4`), status Beta, license GPLv2. +- **Core dependencies**: `unicorn ==2.1.3` (hard-pinned CPU emulator, + `pyproject.toml:29`), `capstone` (disassembly), `keystone-engine` + (assembly), `pefile`, `pyelftools`, `python-registry`, `gevent` + (multithread emulation), `pyyaml` (MCU profiles). Extras: `fuzz` → + unicornafl/fuzzercorn, `RE` → r2libr. - **Hosts**: Linux, Windows, macOS. Some test suites are host-gated (Windows PE tests need real system DLLs collected on Windows; see [os-windows.md](ARCHITECTURE/os-windows.md)). @@ -59,7 +59,7 @@ with reverse debugging), and fuzzing integration (AFL++/unicornafl). | `qltui.py` | Terminal UI invoked via `qltool qltui` | | `Dockerfile` | Multi-stage Poetry wheel build on `python:3-slim` | | `pyproject.toml` / `poetry.lock` | Packaging and dependency pins | -| `.github/workflows/` | CI (`build-ci.yml`: Windows + Ubuntu × Python 3.9/3.11), Docker publish, PyPI publish, Gitee mirror sync | +| `.github/workflows/` | CI (`build-ci.yml`: Windows + Ubuntu × Python 3.11/3.13, plus a Docker container job), Docker publish, PyPI publish, Gitee mirror sync | ## Boot / Entry Flow @@ -85,88 +85,269 @@ From CLI to emulated instructions (details live in the module docs): ## Roadmap -Maturity-based — Qiling is a released project in maintenance/beta (v1.4.8). -There are no in-repo milestones; module Status is `done` when its test suite -proves it. Feature requests and the forward-looking wishlist are tracked in +Maturity-based — Qiling is a released project in maintenance/beta +(v1.4.11-dev). There are no in-repo milestones; module Status is `done` when +its test suite proves it. Feature requests and the forward-looking wishlist are tracked in GitHub issue [#333](https://github.com/qilingframework/qiling/issues/333) (the `TODO` file is a pointer to it). Known cross-cutting gaps: -- `ChangeLog` stops at 1.4.6 while `pyproject.toml` says 1.4.8. -- macOS CI job is commented out in `.github/workflows/build-ci.yml`. +- `ChangeLog` stops at 1.4.6 (`ChangeLog:4`) while `pyproject.toml:4` says + 1.4.11-dev. +- macOS is dropped from the CI matrix and the macOS/kext job is commented out + (`.github/workflows/build-ci.yml:12`, `:84`). - `jexamples/` is legacy and unexercised by CI. +## Development Loop + +Coding Discipline governs writing; Review Checks govern review. This +loop connects them and defines when work is ready to release. + +```text +Frame → Write → Prove → Review → Gate + ▲ findings │ + └────────────────────────┘ +``` + +### The loop + +**1. Frame.** Convert the request into a goal with an observable check. +Inspect the request, code, docs, and repository conventions; record the +narrowest supported assumptions. Ask one focused question only when a +required decision cannot be discovered or safely inferred and guessing +would materially change the result. Once framed, continue without an +approval pause. + +**2. Write.** Make the smallest change that reaches the goal. Add no +unrequested features or abstractions, match local style, touch only +in-scope code, and remove only orphans created by the change. + +**3. Prove.** Run relevant tests and retain observable evidence. + +*Survey the suite before touching it.* Before adding, changing, merging, +or deleting any test, inventory the whole suite: enumerate every test +file and case name, then read in full each test whose subject, fixtures, +or assertions touch this change. Use a subagent for broad inventory when +supported. From that inventory decide the complete set of test edits at +once — what to change, what to add, what to merge, what to remove — each +backed by `file:line`, then execute only that plan. Never write a test +before the survey, and never discover existing coverage afterward. + +The plan obeys four rules: + +- **Reuse or extend first.** Add a case to the test that already owns + the behavior or shares its setup, fixtures, and subject. A new test + function or file is justified only when the survey found no existing + test owning the behavior, or when merging would hide which case + failed. +- **Add only what the goal needs.** A bug fix needs a reproducing + regression test; a new capability needs a test of its claimed + behavior. Nothing further. +- **Retire what this change made obsolete.** Delete tests whose behavior + no longer exists, and merge tests this change turned into duplicates, + citing the surviving test. Leave unrelated pre-existing tests alone; + record suspected redundancy under **Open Gaps / Roadmap**. +- **Never delete to reach green.** A failing test is a finding for + Write. Removal requires evidence that its behavior is gone or is still + covered elsewhere, cited by `file:line`. + +Coverage of claimed behavior must not decrease. A failure returns +directly to Write, never forward to Review. + +**4. Review.** Walk all seven Review Checks as separate passes. Read +whole affected files, not only the diff. Every finding needs `file:line` +evidence. Use an independent agent or isolated pass for Fit, +Dependencies, and Security when available. + +**5. Gate.** Apply the Definition of Done. Any unticked criterion, +`blocker`, or unresolved `major` returns its evidence to Write. All +criteria passing means the change is ready for public or production +release. There is no separate approval or reporting phase. + +### Definition of Done + +**Correctness** + +- The framed goal and its named check pass. +- Tests cover claimed behavior and pass; a bug fix has a regression test. +- The suite was surveyed before any test was written, changed, or + deleted; no added test duplicates coverage another test owns, and no + removal left claimed behavior uncovered. +- The owning module's **How to Test** command passes with evidence. +- The project builds and tests from a fresh clone without local-only + dependencies. + +**Review** + +- All seven Review Checks ran; none was skipped or assumed. +- No `blocker` or unresolved `major` remains. +- Nits were applied or consciously declined. + +**Legibility and contract** + +- A new maintainer can build, test, run, and understand public behavior + from the docs. +- Every changed line serves the goal; no drive-by formatting, debugging + remnants, commented-out code, secrets, tokens, or local paths remain. +- Public names, signatures, errors, and recovery are intelligible. +- Architecture docs and `file:line` references are current. +- Breaking changes, deprecations, dependencies, licenses, and attribution + are handled; commit or PR text explains why. + +### Iterating without thrashing + +- Every pass closes a named finding and touches only what it names. +- Nits alone do not trigger another pass. +- Re-run Prove after every fix. +- Two no-change passes force Gate re-evaluation: release if Done passes; + otherwise return the surviving evidence to Frame. +- Three passes on one finding return automatically to Frame for a new + approach. +- Never widen scope to satisfy a finding. Record out-of-scope work under + **Open Gaps / Roadmap**. + ## Coding Discipline -Behavioral guidelines to reduce common LLM coding mistakes. Merge with -project-specific instructions as needed. - -**Tradeoff:** These guidelines bias toward caution over speed. For -trivial tasks, use judgment. - ### 1. Think Before Coding -**Don't assume. Don't hide confusion. Surface tradeoffs.** - -Before implementing: -- State your assumptions explicitly. If uncertain, ask. -- If multiple interpretations exist, present them - don't pick silently. -- If a simpler approach exists, say so. Push back when warranted. -- If something is unclear, stop. Name what's confusing. Ask. +- Understand the request, code, goal, and repository conventions first. +- Record assumptions and choose the narrowest evidence-backed reading. +- Prefer the simpler approach when it reaches the same verified goal. +- Ask only during planning and only for a required answer that cannot be + discovered or safely inferred. ### 2. Simplicity First -**Minimum code that solves the problem. Nothing speculative.** - -- No features beyond what was asked. -- No abstractions for single-use code. -- No "flexibility" or "configurability" that wasn't requested. -- No error handling for impossible scenarios. -- If you write 200 lines and it could be 50, rewrite it. - -Ask yourself: "Would a senior engineer say this is overcomplicated?" If -yes, simplify. +- Implement only what was requested. +- Do not add single-use abstractions, speculative flexibility, or checks + for impossible conditions. +- If the implementation is materially larger than the problem, simplify + it. ### 3. Surgical Changes -**Touch only what you must. Clean up only your own mess.** - -When editing existing code: -- Don't "improve" adjacent code, comments, or formatting. -- Don't refactor things that aren't broken. -- Match existing style, even if you'd do it differently. -- If you notice unrelated dead code, mention it - don't delete it. - -When your changes create orphans: -- Remove imports/variables/functions that YOUR changes made unused. -- Don't remove pre-existing dead code unless asked. - -The test: Every changed line should trace directly to the user's request. +- Do not refactor, reformat, or clean up unrelated code. +- Match the surrounding style. +- Remove imports, variables, and functions made unused by this change; + leave pre-existing dead code alone unless requested. +- Every changed line must trace to the stated goal. ### 4. Goal-Driven Execution -**Define success criteria. Loop until verified.** +Turn work into verifiable outcomes, then loop until they pass: -Transform tasks into verifiable goals: -- "Add validation" → "Write tests for invalid inputs, then make them pass" -- "Fix the bug" → "Write a test that reproduces it, then make it pass" -- "Refactor X" → "Ensure tests pass before and after" +- Add validation → invalid inputs are rejected by a named passing test. +- Fix a bug → a regression test fails before the fix and passes after. +- Refactor → behavior tests pass before and after. -For multi-step tasks, state a brief plan: +Give every plan step its own check. Strengthen vague criteria from +repository evidence before implementation. -``` -1. [Step] → verify: [check] -2. [Step] → verify: [check] -3. [Step] → verify: [check] -``` +### Project-Specific Deviations -Strong success criteria let you loop independently. Weak criteria ("make -it work") require constant clarification. +- Emulation fidelity beats abstraction: syscall, API, and peripheral + implementations mirror the real platform's observable behavior even + when that means repetitive per-OS or per-chip code. Cross-OS + "unification" is a scope increase, not a simplification. +- Coverage is demand-driven by design (see the OS and HW module docs). + Adding an unrequested syscall, Win32 API, or peripheral register is + out of scope; record it under the owning module's **Open Gaps / + Roadmap**. +- `unicorn` is hard-pinned (`pyproject.toml:29`). Changing it, or any + behavior that depends on its version, is a project-wide event and + never an incidental part of another change. ---- +## Review Checks -**These guidelines are working if:** fewer unnecessary changes in diffs, -fewer rewrites due to overcomplication, and clarifying questions come -before implementation rather than after mistakes. +Run every check against every change before merge. Keep checks separate. + +Four rules bind all checks: + +- **Evidence or no finding.** Every finding cites `file:line`. +- **The repository is authoritative.** Demand only conventions visible + in the tree. +- **Read files, not only hunks.** Context can invalidate a finding or + reveal unreachable code, unused parameters, and hidden duplication. +- **Review the change, never the author.** Describe code and impact, not + how or by whom it was produced. + +### 1. Style + +Check indentation and local file conventions. Mixed indentation is +`major`; a consistent new file using the wrong local indent is `nit`. +Leave machine-checkable formatting to existing formatters and linters; +never demand unrelated reformatting. + +### 2. Naming + +Compare new names with nearby precedents before filing a finding. If the +repository is inconsistent, demand nothing. A local mismatch is `nit`; +an inconsistent public name is `major`. + +### 3. Duplication + +Search distinctive constants, errors, fields, and call sequences—not +only symbol names—for code performing the same job. Cite both sites and +the remedy. Cross-layer duplication is `major`; small local repetition +is `nit`. Similar code with meaningfully different branches is not +duplication. + +### 4. Quality + +Require followable control flow, errors handled where they occur, and +abstractions proportional to the problem. Swallowed errors, +inappropriate prints, unexplained magic values, and dead branches are +`major`. Remove unrequested configurability, one-caller wrappers, filler +comments, debugging remnants, and unrelated formatting. Missing tests +belong to Prove, not this check. + +### 5. Fit + +Read `ARCHITECTURE.md` and the owning module doc before the diff. Check +scope, layering, ownership, public-API growth, and performance claims. A +layering violation or unjustified public API is `major`. Architectural or +public-behavior changes must update the relevant docs in the same change. + +### 6. Dependencies + +Check manifests and imports, maintenance, supply-chain risk, advisories, +install-time behavior, license, transitive cost, and whether the standard +library is sufficient. An unjustified top-level dependency is `major`; +a live advisory or abandoned upstream is `blocker`. Incomplete evidence +does not pass. + +### 7. Security + +Check both defects and widened exposure: unsafe memory access, unchecked +sizes or offsets, integer overflow, path traversal, unsafe +deserialization, command construction, committed secrets, and unbounded +untrusted input. Trace input to impact; without a reachable path there is +no finding. A real defect is `major`; a trust-boundary break is `blocker`. +Describe the fix without publishing exploit steps. + +### Project-Specific Deviations + +- **Security, scope.** Qiling *emulates* untrusted binaries; guest code + doing something hostile inside the sandbox is the product working, not + a finding. Findings target the host boundary: rootfs escape via path + handling (`qiling/os/path.py`), unchecked guest-controlled sizes or + offsets reaching host allocations or `struct` unpacking, and parser + input in `qiling/loader/` reachable from an untrusted image. +- **Dependencies.** `pyproject.toml` is the only manifest. A new + top-level runtime dependency is `blocker` absent an explicit request; + optional integrations belong in an extra (`fuzz`, `RE`). + +### Severity and the merge threshold + +| Severity | Effect | +| -------- | ------ | +| `blocker` | Must not merge. | +| `major` | Must be resolved before merge. | +| `nit` | Apply or consciously decline. | +| `info` | Context or a question; no action implied. | + +Merge only with no `blocker` and no unresolved `major`. A check that did +not run does not pass. Findings feed Write and Gate directly; they do not +create a reporting phase. ## Index diff --git a/ARCHITECTURE/arch.md b/ARCHITECTURE/arch.md index 1bf638cd..75e9d3b0 100644 --- a/ARCHITECTURE/arch.md +++ b/ARCHITECTURE/arch.md @@ -26,7 +26,7 @@ covered by `tests/test_cpu_models.py`. | `qiling/arch/models.py` | CPU model enums (`X86_CPU_MODEL` … `RISCV64_CPU_MODEL`) | | `qiling/arch/msr.py`, `cpr.py`, `cpr64.py` | x86 MSRs, ARM/ARM64 coprocessor registers | | `qiling/arch/utils.py` | `QlArchUtils`: disassembly output for verbose/trace modes | -| `qiling/cc/__init__.py` + `intel.py`, `arm.py`, `mips.py`, `ppc.py`, `riscv.py` | Calling conventions (arg/retval marshalling) consumed by `os/fcall.py` | +| `qiling/cc/__init__.py` + `intel.py`, `arm.py`, `mips.py`, `ppc.py`, `riscv.py` | Calling conventions (arg/retval marshalling) consumed by `qiling/os/fcall.py` | ## Key Types and Entry Points @@ -35,7 +35,7 @@ covered by `tests/test_cpu_models.py`. - `qiling/arch/x86.py:22,53,79,111` - `QlArchIntel` / `QlArchA8086` / `QlArchX86` / `QlArchX8664`. - `qiling/arch/cortex_m.py:67` - `QlArchCORTEX_M(QlArchARM)` - plus `QlInterruptContext` (`:25`) for exception entry/exit in MCU mode. - `qiling/arch/models.py` - CPU model enums selected via the `cputype` kwarg (resolved in `select_arch`, `qiling/utils.py:376`). -- `qiling/cc/__init__.py:9` - `QlCC` - abstract calling convention; `QlCommonBaseCC` (`:104`); e.g. `cc/intel.py` defines `cdecl`/`stdcall`/`ms64`/`macosx64`. +- `qiling/cc/__init__.py:9` - `QlCC` - abstract calling convention; `QlCommonBaseCC` (`:110`); e.g. `qiling/cc/intel.py` defines `cdecl`/`stdcall`/`ms64`/`macosx64`. ## Interactions diff --git a/ARCHITECTURE/debugger.md b/ARCHITECTURE/debugger.md index 88289e3f..52156ec9 100644 --- a/ARCHITECTURE/debugger.md +++ b/ARCHITECTURE/debugger.md @@ -17,7 +17,8 @@ reverse debugging. Mature released infrastructure; maturity-based status. | File | Role | | ---- | ---- | | `qiling/debugger/debugger.py` | Base `QlDebugger` | -| `qiling/debugger/gdb/gdb.py` | `QlGdb`: GDB remote-serial-protocol server | +| `qiling/debugger/gdb/gdb.py` | `QlGdb`: GDB remote-serial-protocol server, plus the `GdbSerialConn` transport | +| `qiling/debugger/gdb/utils.py` | `QlGdbUtils`: breakpoint table and the per-instruction `dbg_hook` that services breakpoints, stepping, and async interrupts | | `qiling/debugger/gdb/xmlregs.py`, `gdb/xml/` | Target-description XML per arch for modern GDB clients | | `qiling/debugger/qdb/qdb.py` | `QlQdb`: interactive Cmd-based debugger | | `qiling/debugger/qdb/arch/` | Per-arch Qdb support (arm, intel, mips) | @@ -27,7 +28,10 @@ reverse debugging. Mature released infrastructure; maturity-based status. ## Key Types and Entry Points - `qiling/debugger/debugger.py:13` - `QlDebugger` - base; `run()` starts the session. -- `qiling/debugger/gdb/gdb.py:68` - `QlGdb(QlDebugger)` - listens on ip:port, translates RSP packets to Qiling hook/mem/reg operations. +- `qiling/debugger/gdb/gdb.py:84` - `QlGdb(QlDebugger)` - listens on ip:port, translates RSP packets to Qiling hook/mem/reg operations; `run()` (`:139`) serves the session. +- `qiling/debugger/gdb/gdb.py:817` - `GdbSerialConn` - the socket transport; `poll_interrupt()` (`:856`) is a non-blocking check for a client `\x03`, wired into `QlGdbUtils.check_interrupt` (`qiling/debugger/gdb/gdb.py:145`) so a running guest can be broken into asynchronously. +- `qiling/debugger/gdb/utils.py:16` - `QlGdbUtils` - `dbg_hook` (`:48`) runs per instruction to service breakpoints, single-step, and interrupts; `bp_insert`/`bp_remove` (`:81`/`:94`); `resume_emu` (`:107`). +- Stop replies report `SIGTRAP` (`qiling/debugger/gdb/gdb.py:49`) for both single-step (`:243`) and async-interrupt stops (`:262`). - `qiling/debugger/qdb/qdb.py:59` - `QlQdb(Cmd, QlDebugger)` - CLI loop; `rr` mode enables record/replay reverse debugging. - Activation: set `ql.debugger = True | "gdb" | "gdb:0.0.0.0:9999" | "qdb" | "qdb:rr"` (`qiling/core.py:437`); instantiated lazily in `Qiling.run` via `select_debugger` (`qiling/utils.py:332`). - `qltool` flags: `--gdb` and `--qdb` (see [cli.md](cli.md)). diff --git a/ARCHITECTURE/hw.md b/ARCHITECTURE/hw.md index 346f6c3b..c1466aaa 100644 --- a/ARCHITECTURE/hw.md +++ b/ARCHITECTURE/hw.md @@ -26,11 +26,11 @@ firmware images (UART echo, freertos, blink, crc, dma_clock, i2c/spi/lcd). ## Key Types and Entry Points -- `qiling/hw/hw.py:14` - `QlHwManager` - available as `ql.hw`; created by core only when `ql.baremetal` (`qiling/core.py:191`). -- `qiling/hw/hw.py:23` - `QlHwManager.create(label, struct, base)` - instantiates a peripheral from the profile entry and maps its MMIO region. -- `qiling/hw/hw.py:82` - `QlHwManager.step()` - advances every peripheral one tick; called from the MCU run loop. +- `qiling/hw/hw.py:60` - `QlHwManager` - available as `ql.hw`; created by core only when `ql.baremetal` (`qiling/core.py:191`). +- `qiling/hw/hw.py:67` - `QlHwManager.create(label, struct, base)` - instantiates a peripheral from the profile entry and maps its MMIO region. +- `qiling/hw/hw.py:135` - `QlHwManager.step()` - advances every peripheral one tick; called from the MCU run loop. - `qiling/hw/peripheral.py:132` - `QlPeripheral(QlPeripheralUtils)` - base class: a ctypes register struct + read/write handlers on the MMIO region. -- `qiling/extensions/mcu/stm32f4xx/stm32f407.py` (and siblings) - chip `env` dicts consumed at `Qiling(..., env=...)` construction. +- `qiling/extensions/mcu/stm32f4/stm32f407.py` (and siblings) - chip `env` dicts consumed at `Qiling(..., env=...)` construction. ## Interactions diff --git a/ARCHITECTURE/loader.md b/ARCHITECTURE/loader.md index 5cbb3d9f..da4b7196 100644 --- a/ARCHITECTURE/loader.md +++ b/ARCHITECTURE/loader.md @@ -30,7 +30,7 @@ Test command proves the ELF path end-to-end. - `qiling/loader/loader.py:21` - `QlLoader` - base; `Image` NamedTuple (`:15`), abstract `run()` (`:62`), `skip_exit_check` (`:27`). - `qiling/loader/elf.py:68` - `QlLoaderELF` - loads binary + interpreter, builds stack/auxv (`AUXV` enum `:32`), sets `entry_point`/`elf_entry`. -- `qiling/loader/pe.py:666` - `QlLoaderPE(QlLoader, Process)` - `Process` (`:69`) builds PEB/TEB/LDR; `QlPeCache` (`:41`) caches parsed DLLs behind the `libcache` kwarg (wired in `select_loader`, `qiling/utils.py:300`). +- `qiling/loader/pe.py:817` - `QlLoaderPE(QlLoader, Process)` - `Process` (`:77`) builds PEB/TEB/LDR; `QlPeCache` (`:49`) caches parsed DLLs behind the `libcache` kwarg (wired in `select_loader`, `qiling/utils.py:300`). - `qiling/loader/pe_uefi.py:26` - `QlLoaderPE_UEFI` - loads DXE/SMM modules and installs protocols into the UEFI context. - `qiling/loader/macho.py:70` - `QlLoaderMACHO`. - `qiling/loader/mcu.py:57` - `QlLoaderMCU` - with `IhexParser` (`:15`). diff --git a/ARCHITECTURE/os-baremetal.md b/ARCHITECTURE/os-baremetal.md index 2823b095..0b7a68a7 100644 --- a/ARCHITECTURE/os-baremetal.md +++ b/ARCHITECTURE/os-baremetal.md @@ -11,7 +11,8 @@ released infrastructure; maturity-based status. ## Status `done` — MCU covered by `tests/test_mcu.py` (STM32F1/F4, GD32VF1 firmware), -BLOB by `tests/test_blob.py` (u-boot) and `tests/test_edl.py`. +BLOB by `test_blob.BlobTest.test_uboot_arm` and `tests/test_edl.py`. The other +case in `tests/test_blob.py` is blocked on a missing fixture (see Open Gaps). ## Code Structure @@ -23,10 +24,10 @@ BLOB by `tests/test_blob.py` (u-boot) and `tests/test_edl.py`. ## Key Types and Entry Points -- `qiling/os/mcu/mcu.py:37` - `QlOsMcu(QlOs)` - run loop; steps hardware between execution chunks and delivers interrupts. +- `qiling/os/mcu/mcu.py:41` - `QlOsMcu(QlOs)` - run loop; steps hardware between execution chunks and delivers interrupts. - `qiling/os/mcu/mcu.py:17` - `MCUTask(UnicornTask)` - the firmware execution task. - `qiling/extensions/multitask.py:26` / `:152` - `UnicornTask` / `MultiTaskUnicorn(Uc)` - task-switching Unicorn subclass MCU mode runs on. -- `qiling/os/blob/blob.py:12` - `QlOsBlob(QlOs)` - runs `entry_point` → `exit_point` with no OS services. +- `qiling/os/blob/blob.py:14` - `QlOsBlob(QlOs)` - runs `entry_point` → `exit_point` with no OS services. - MCU selection: `QL_OS.MCU` is the only member of `QL_OS_BAREMETAL` (`qiling/const.py:74`); `ql.baremetal` (`qiling/core.py:357`) gates hardware-manager creation. ## Interactions @@ -40,13 +41,21 @@ BLOB by `tests/test_blob.py` (u-boot) and `tests/test_edl.py`. ## How to Test ```sh -cd tests && python3 test_blob.py # pass = unittest "OK", exit 0 +cd tests && python3 -m unittest test_blob.BlobTest.test_uboot_arm # pass = "OK", exit 0 ``` -- MCU (also proves [hw.md](hw.md)): `cd tests && python3 test_mcu.py`. -- Qualcomm EDL loader: `cd tests && python3 test_edl.py`. +- MCU (also proves [hw.md](hw.md)): `cd tests && python3 test_mcu.py` — pass = `Ran 18 tests … OK`. +- Qualcomm EDL loader: `cd tests && python3 test_edl.py` — pass = `Ran 1 test … OK`. +- The whole `test_blob.py` file does **not** pass from a clean checkout; see + Open Gaps. ## Open Gaps / Roadmap - Supported chip families are those with board definitions in `qiling/extensions/mcu/` (STM32F1/F4, GD32VF1, NXP, Atmel, BES); new chips need new peripheral maps. - BLOB mode provides no services by design — targets needing hardware must use MCU mode instead. +- `test_blob.BlobTest.test_blob_raw` (`tests/test_blob.py:85`) errors with + `FileNotFoundError` on a clean checkout: it reads + `examples/rootfs/blob/example_raw.bin` (`tests/test_blob.py:96`), but the + pinned `examples/rootfs` submodule ships only `u-boot.bin.img`. Either the + fixture must be added upstream and the submodule bumped, or the test skipped + when the fixture is absent. diff --git a/ARCHITECTURE/os-base.md b/ARCHITECTURE/os-base.md index 7d71305d..756a0e04 100644 --- a/ARCHITECTURE/os-base.md +++ b/ARCHITECTURE/os-base.md @@ -29,13 +29,13 @@ everything else is exercised by every OS suite. ## Key Types and Entry Points -- `qiling/os/os.py:24` - `QlOs` - composes utils/stats/path/fs-mapper; `user_defined_api` keyed by `QL_INTERCEPT`; abstract `run()` (`:239`). -- `qiling/os/os.py:224` - `QlOs.set_api(target, handler, intercept)` - user override of an emulated API (address- or name-based). -- `qiling/os/memory.py:23` - `QlMemoryManager` - `ql.mem`; instantiated by core *before* the OS (`qiling/core.py:188`). -- `qiling/os/memory.py:658` - `QlMemoryHeap` - heap used by Windows/UEFI APIs and the sanitizers extension. +- `qiling/os/os.py:24` - `QlOs` - composes utils/stats/path/fs-mapper; `user_defined_api` keyed by `QL_INTERCEPT`; abstract `run()` (`:240`). +- `qiling/os/os.py:225` - `QlOs.set_api(target, handler, intercept)` - user override of an emulated API (address- or name-based). +- `qiling/os/memory.py:40` - `QlMemoryManager` - `ql.mem`; instantiated by core *before* the OS (`qiling/core.py:188`). +- `qiling/os/memory.py:678` - `QlMemoryHeap` - heap used by Windows/UEFI APIs and the sanitizers extension. - `qiling/os/fcall.py:21` - `QlFunctionCall` - argument/return marshalling on top of `qiling/cc/`. - `qiling/os/mapper.py:64` - `QlFsMapper` - behind `ql.add_fs_mapper` (`qiling/core.py:701`). -- `qiling/os/path.py:14` - `QlOsPath` - rootfs-confined path resolution. +- `qiling/os/path.py:16` - `QlOsPath` - rootfs-confined path resolution. - `qiling/os/thread.py:11` - `QlThread(Greenlet)` - gevent-based thread base. ## Interactions diff --git a/ARCHITECTURE/os-windows.md b/ARCHITECTURE/os-windows.md index 88a64276..cf418779 100644 --- a/ARCHITECTURE/os-windows.md +++ b/ARCHITECTURE/os-windows.md @@ -31,14 +31,14 @@ run on Linux. Windows tests are platform-gated locally (need collected DLLs). - `qiling/os/windows/windows.py:33` - `QlOsWindows(QlOs)` - `__setup_components` (`:156`) builds handle manager/registry/clipboard/fiber; `run()` (`:201`). - `qiling/os/windows/fncc.py` - `@winsdkapi` decorator - declares an API's calling convention and typed params; implementations live in `dlls/`. -- API dispatch is address-based: IAT addresses recorded by the PE loader are hooked and marshalled through `QlFunctionCall` — user overrides via `QlOs.set_api` (`qiling/os/os.py:224`). +- API dispatch is address-based: IAT addresses recorded by the PE loader are hooked and marshalled through `QlFunctionCall` — user overrides via `QlOs.set_api` (`qiling/os/os.py:225`). - `qiling/os/uefi/uefi.py:22` - `QlOsUefi(QlOs)` - executes DXE/SMM modules; services in `bs.py`/`rt.py`/`smm.py` are installed as callable tables. - `qiling/os/dos/dos.py:33` - `QlOsDos(QlOs)` - dispatches BIOS/DOS interrupts from `interrupts/`. ## Interactions - All three subclass [os-base.md](os-base.md) `QlOs` and allocate from `QlMemoryHeap`. -- [loader.md](loader.md): `QlLoaderPE` builds PEB/TEB and records IAT hook addresses; `QlLoaderPE_UEFI` installs protocols into `os/uefi/context.py`; `QlLoaderDOS` sets real-mode state. +- [loader.md](loader.md): `QlLoaderPE` builds PEB/TEB and records IAT hook addresses; `QlLoaderPE_UEFI` installs protocols into `qiling/os/uefi/context.py`; `QlLoaderDOS` sets real-mode state. - Argument marshalling uses `QlFunctionCall` + `qiling/cc/intel.py` conventions ([arch.md](arch.md)). - The registry emulation reads hive files from the rootfs via [os-base.md](os-base.md) path services. - `examples/uefi_sanitized_heap.py` pairs UEFI with the heap sanitizer from [extensions.md](extensions.md). From 206b90fd891cee4bc0056d2198ddf85678b55f96 Mon Sep 17 00:00:00 2001 From: xwings Date: Sat, 5 Sep 2026 16:57:28 +0800 Subject: [PATCH 20/21] minor refactor and pypi fix --- .github/workflows/pythonpublish.yml | 53 +++-- ARCHITECTURE.md | 34 ++- ARCHITECTURE/cli.md | 21 +- README.md | 4 + pyproject.toml | 17 +- qiling/cli.py | 324 ++++++++++++++++++++++++++++ qltool | 316 +-------------------------- tests/test_qltool.py | 42 ++++ 8 files changed, 464 insertions(+), 347 deletions(-) create mode 100644 qiling/cli.py diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 4b83c2ba..07cb4f0f 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -1,44 +1,73 @@ name: PyPI 📦 Distribution -on: [push] +on: [push, pull_request] + +permissions: + contents: read jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: - python-version: "^3.8" + python-version: "3.13" - name: Install dependencies - run: pipx install poetry + run: python -m pip install "poetry>=2,<3" build twine + - name: Check package metadata + run: poetry check --lock + - name: Check release tag matches package version + if: startsWith(github.ref, 'refs/tags/') + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + python - <<'PY' + import os + import tomllib + from pathlib import Path + from packaging.version import Version + + tag = os.environ['RELEASE_TAG'].removeprefix('v') + version = tomllib.loads(Path('pyproject.toml').read_text())['tool']['poetry']['version'] + if Version(tag) != Version(version): + raise SystemExit(f'Release tag {tag!r} does not match package version {version!r}') + PY - name: Build distribution 📦 - run: poetry build -n + run: python -m build + - name: Check distribution metadata + run: python -m twine check --strict dist/* + - name: Test installed wheel + run: | + python -m pip install dist/*.whl + python -I tests/test_qltool.py InstalledQltool_Test -v - uses: actions/upload-artifact@v4 with: + name: distributions path: ${{ github.workspace }}/dist/* + if-no-files-found: error publish: needs: [build] runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags') + if: startsWith(github.ref, 'refs/tags/') steps: - uses: actions/download-artifact@v4 with: - name: artifact + name: distributions path: dist - name: Publish distribution 📦 to test PyPI - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.testpypi_pass }} - repository_url: https://test.pypi.org/legacy/ + repository-url: https://test.pypi.org/legacy/ + skip-existing: true - name: Publish distribution 📦 to PyPI - if: ${{ success() }} - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.pypi_pass }} diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 11f4437b..dd0822d7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -31,10 +31,10 @@ with reverse debugging), and fuzzing integration (AFL++/unicornafl). - **Shape**: pure-Python library (`from qiling import Qiling`), plus the `qltool` CLI and `qltui.py` TUI. Also shipped as a Docker image (`Dockerfile`) and PyPI package. -- **Python**: 3.10+ (`pyproject.toml:27`). Packaging via Poetry; version - 1.4.11-dev (`pyproject.toml:4`), status Beta, license GPLv2. +- **Python**: 3.10+ (`pyproject.toml:37`). Packaging via Poetry; version + 1.4.11.dev0 (`pyproject.toml:4`), status Beta, license GPL-2.0-or-later. - **Core dependencies**: `unicorn ==2.1.3` (hard-pinned CPU emulator, - `pyproject.toml:29`), `capstone` (disassembly), `keystone-engine` + `pyproject.toml:39`), `capstone` (disassembly), `keystone-engine` (assembly), `pefile`, `pyelftools`, `python-registry`, `gevent` (multithread emulation), `pyyaml` (MCU profiles). Extras: `fuzz` → unicornafl/fuzzercorn, `RE` → r2libr. @@ -55,7 +55,7 @@ with reverse debugging), and fuzzing integration (AFL++/unicornafl). | `jexamples/` | Legacy example set (not covered by CI) | | `tests/` | CI test suite — standalone `unittest` files run from `tests/` (`tests/test_onlinux.sh`, `tests/test_pe.bat`, `tests/test_macho.sh`) | | `docs/` | Stubs and assets; real documentation lives at https://docs.qiling.io | -| `qltool` | CLI entry point (see [cli.md](ARCHITECTURE/cli.md)) | +| `qltool` / `qiling/cli.py` | Checkout launcher / installed CLI implementation (see [cli.md](ARCHITECTURE/cli.md)) | | `qltui.py` | Terminal UI invoked via `qltool qltui` | | `Dockerfile` | Multi-stage Poetry wheel build on `python:3-slim` | | `pyproject.toml` / `poetry.lock` | Packaging and dependency pins | @@ -65,8 +65,9 @@ with reverse debugging), and fuzzing integration (AFL++/unicornafl). From CLI to emulated instructions (details live in the module docs): -1. `qltool` parses args and builds kwargs; `run` and `code` subcommands end in - `Qiling(**ql_args)` (`qltool:276`) — see [cli.md](ARCHITECTURE/cli.md). +1. `qltool` calls `qiling.cli.run`, which parses args and builds kwargs; + `run` and `code` subcommands end in `Qiling(**ql_args)` + (`qiling/cli.py:276`) — see [cli.md](ARCHITECTURE/cli.md). 2. `Qiling.__init__` (`qiling/core.py:35`) is the composition root. Order: guess arch/OS from the binary if not given → instantiate arch → init struct/hook mixins → logger → profile → loader → memory manager → OS layer @@ -86,17 +87,32 @@ From CLI to emulated instructions (details live in the module docs): ## Roadmap Maturity-based — Qiling is a released project in maintenance/beta -(v1.4.11-dev). There are no in-repo milestones; module Status is `done` when +(v1.4.11.dev0). There are no in-repo milestones; module Status is `done` when its test suite proves it. Feature requests and the forward-looking wishlist are tracked in GitHub issue [#333](https://github.com/qilingframework/qiling/issues/333) (the `TODO` file is a pointer to it). Known cross-cutting gaps: - `ChangeLog` stops at 1.4.6 (`ChangeLog:4`) while `pyproject.toml:4` says - 1.4.11-dev. + 1.4.11.dev0. - macOS is dropped from the CI matrix and the macOS/kext job is commented out (`.github/workflows/build-ci.yml:12`, `:84`). - `jexamples/` is legacy and unexercised by CI. +## Packaging and Releases + +`.github/workflows/pythonpublish.yml` validates metadata and the lock file, +builds a source distribution and a wheel from it, checks both with Twine, +and runs `InstalledQltool_Test` against the installed wheel outside the +checkout. Builds run on pushes and pull requests; publishing runs on tags. + +Before tagging a release, set its version with `poetry version ` +and commit the change. The tag must match the package version under PEP 440 +normalization, with an optional `v` prefix. Development versions remain +prereleases; a stable release requires a stable version in `pyproject.toml`. +Publishing uploads to TestPyPI, then PyPI, using the existing +`testpypi_pass` and `pypi_pass` API-token secrets. TestPyPI skips existing +files so a production upload can be retried after TestPyPI succeeds. + ## Development Loop Coding Discipline governs writing; Review Checks govern review. This @@ -252,7 +268,7 @@ repository evidence before implementation. Adding an unrequested syscall, Win32 API, or peripheral register is out of scope; record it under the owning module's **Open Gaps / Roadmap**. -- `unicorn` is hard-pinned (`pyproject.toml:29`). Changing it, or any +- `unicorn` is hard-pinned (`pyproject.toml:39`). Changing it, or any behavior that depends on its version, is a project-wide event and never an incidental part of another change. diff --git a/ARCHITECTURE/cli.md b/ARCHITECTURE/cli.md index 40b13a98..44ed56ba 100644 --- a/ARCHITECTURE/cli.md +++ b/ARCHITECTURE/cli.md @@ -11,21 +11,24 @@ maturity-based status. ## Status `done` — covered by `tests/test_qltool.py`, which shells out to `qltool` for -run/code subcommands including gdb attach and coverage output. +run/code subcommands and coverage output. `InstalledQltool_Test` checks the +installed command, shellcode exit status, bundled profiles, and TUI import +outside the checkout. ## Code Structure | File | Role | | ---- | ---- | -| `qltool` | Executable argparse CLI; builds kwargs and drives `Qiling` | +| `qltool` | Checkout launcher for `qiling.cli.run` | +| `qiling/cli.py` | Argparse CLI; installed as `qltool`, builds kwargs and drives `Qiling` | | `qltui.py` | questionary/pyfx/termcolor TUI; collects options, returned to qltool | ## Key Types and Entry Points -- `qltool:189` - `run()` - argparse setup with subcommands `run`, `code`, `examples`, `qltui`; enum-mapping actions translate `--arch/--os/--endian/--verbose` strings to `QL_ARCH`/`QL_OS` enums (`qltool:59-75`). -- `qltool:129` - `handle_run(options)` - builds `{'argv': [file]+args, 'rootfs': ...}`. -- `qltool:78` - `handle_code(options)` - reads hex/asm/bin shellcode, assembling asm via `qiling.arch.utils.assembler` (`qltool:104`). -- `qltool:276` - `ql = Qiling(**ql_args)` - the single construction point; then optional Qdb (`:279`), gdbserver (`:285`), coverage-wrapped `ql.run()` (`:306-310`), JSON report (`:312`), exit with `ql.os.exit_code` (`:321`). +- `qiling/cli.py:189` - `run()` - argparse setup with subcommands `run`, `code`, `examples`, `qltui`; enum-mapping actions translate `--arch/--os/--endian/--verbose` strings to `QL_ARCH`/`QL_OS` enums (`qiling/cli.py:59-75`). +- `qiling/cli.py:129` - `handle_run(options)` - builds `{'argv': [file]+args, 'rootfs': ...}`. +- `qiling/cli.py:78` - `handle_code(options)` - reads hex/asm/bin shellcode, assembling asm via `qiling.arch.utils.assembler` (`qiling/cli.py:104`). +- `qiling/cli.py:276` - `ql = Qiling(**ql_args)` - the single construction point; then optional Qdb (`:279`), gdbserver (`:285`), coverage-wrapped `ql.run()` (`:306-310`), JSON report (`:312`), exit with `ql.os.exit_code` (`:321`). ## Interactions @@ -37,12 +40,16 @@ run/code subcommands including gdb attach and coverage output. ## How to Test ```sh +python3 -m pip install . cd tests && python3 test_qltool.py # pass = unittest "OK", exit 0 ``` +- Packaging regression checks, from the repository root after installing + the built wheel: `python3 -I tests/test_qltool.py InstalledQltool_Test -v`. - Manual smoke test: `./qltool run -f examples/rootfs/x8664_linux/bin/x8664_hello --rootfs examples/rootfs/x8664_linux` — pass = prints `Hello, World!`. ## Open Gaps / Roadmap -- `qltui.py` (TUI) has no automated tests — interactive only. +- `qltui.py` has an installed import smoke test; its interactive flows are + not covered by automated tests. - `qltool` predates subcommand-style config files; complex setups (fs mappers, custom hooks) still require the Python API. diff --git a/README.md b/README.md index 05d6945b..668d335e 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,10 @@ Qiling Framework emulates MBR Qiling also provides a friendly tool named `qltool` to quickly emulate shellcode & executable binaries. +Installing Qiling with pip also installs the `qltool` command, including +`qltool qltui` for the terminal UI. In a source checkout, use `./qltool` as +shown below. Example binaries and rootfs files must be downloaded separately. + With qltool, easy execution can be performed: diff --git a/pyproject.toml b/pyproject.toml index afa35a8d..da3e4b52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] urls = { homepage = "https://qiling.io" } name = "qiling" -version = "1.4.11-dev" +version = "1.4.11.dev0" description = "Qiling is an advanced binary emulation framework that cross-platform-architecture" authors = ["KaiJern Lau (xwings) "] -license = "GPLv2" +license = "GPL-2.0-or-later" readme = "README.md" classifiers = [ - "Development Status :: 3 - Beta", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Topic :: Software Development :: Disassemblers", "Topic :: System :: Emulators", @@ -24,6 +24,15 @@ keywords = [ "IoT", ] +packages = [ + { include = "qiling" }, + { include = "qltui.py" }, +] +include = [{ path = "qltool", format = "sdist" }] + +[tool.poetry.scripts] +qltool = "qiling.cli:run" + [tool.poetry.dependencies] python = "^3.10" capstone = "^5" @@ -50,5 +59,5 @@ fuzz = ["unicornafl", "fuzzercorn"] RE = ["r2libr"] [build-system] -requires = ["poetry-core"] +requires = ["poetry-core>=2.0,<3.0"] build-backend = "poetry.core.masonry.api" diff --git a/qiling/cli.py b/qiling/cli.py new file mode 100644 index 00000000..a8e0a0df --- /dev/null +++ b/qiling/cli.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +import argparse +import os +import sys +import ast +import pickle + +from pprint import pprint +from typing import TYPE_CHECKING, Any, AnyStr, Dict, Literal, Mapping, Type, overload + +from unicorn import __version__ as uc_ver +from qiling import __version__ as ql_ver + +from qiling import Qiling +from qiling.arch import utils as arch_utils +from qiling.debugger.qdb import QlQdb +from qiling.const import QL_ENDIAN, QL_VERBOSE, endian_map, os_map, arch_map, verbose_map +from qiling.extensions.coverage import utils as cov_utils +from qiling.extensions import report + + +if TYPE_CHECKING: + from enum import Enum + + +@overload +def read_file(fname: AnyStr) -> str: ... + + +@overload +def read_file(fname: AnyStr, mode: Literal['b']) -> bytes: ... + + +def read_file(fname: AnyStr, mode: Literal['b', ''] = ''): + """Read file contents. + """ + + with open(fname, f'r{mode}') as f: + content = f.read() + + return content + + +class __arg_env(argparse.Action): + def __call__(self, parser, namespace, values: str, option_string): + if os.path.exists(values): + with open(values, 'rb') as f: + env = pickle.load(f) + else: + env = ast.literal_eval(values) + + setattr(namespace, self.dest, env or {}) + + +def __make_enum_arg(enum_rmap: Mapping[str, 'Enum'], aliases: Mapping[str, str] = {}) -> Type[argparse.Action]: + class __enum_arg(argparse.Action): + def __call__(self, parser, namespace, values: str, option_string): + values = values.casefold() + + if values in aliases: + values = aliases[values] + + setattr(namespace, self.dest, enum_rmap[values]) + + return __enum_arg + + +__arg_endian = __make_enum_arg(endian_map) +__arg_archtype = __make_enum_arg(arch_map, {'x86_64': 'x8664', 'riscv32': 'riscv'}) +__arg_ostype = __make_enum_arg(os_map, {'darwin': 'macos'}) +__arg_verbose = __make_enum_arg(verbose_map) + + +def handle_code(options: argparse.Namespace) -> Dict[str, Any]: + if options.format == 'hex': + if options.input is not None: + print("Loading HEX from ARGV") + payload = str(options.input) + + elif options.filename is not None: + print("Loading HEX from FILE") + payload = read_file(options.filename) + + else: + print("Error: please specify an input source for hex") + exit(1) + + code = bytes.fromhex(payload.replace(r'\x', '')) + + elif options.format == 'asm': + print("Loading ASM from FILE") + + if options.filename is None: + print("Error: please specify an input source for asm") + exit(1) + + payload = read_file(options.filename) + + # assemble the payload and turn it into opcodes bytes + assembler = arch_utils.assembler(options.arch, options.endian, options.thumb) + code, _ = assembler.asm(payload) + code = bytes(code) + + elif options.format == 'bin': + print("Loading BIN from FILE") + + if options.filename is None: + print("Error: please specify an input source for bin") + exit(1) + + code = read_file(options.filename, 'b') + + ql_args = { + 'rootfs': options.rootfs, + 'code': code, + 'ostype': options.os, + 'archtype': options.arch, + 'endian': options.endian, + 'thumb': options.thumb + } + + return ql_args + + +def handle_run(options: argparse.Namespace) -> Dict[str, Any]: + effective_argv = [] + + # with argv + if options.filename is not None and options.run_args == []: + effective_argv = [options.filename] + options.args + + # Without argv + elif options.filename is None and options.args == [] and options.run_args != []: + effective_argv = options.run_args + + else: + print("ERROR: Command error!") + + ql_args = { + 'argv': effective_argv, + 'rootfs': options.rootfs + } + + return ql_args + + +def handle_examples(parser: argparse.ArgumentParser): + prog = parser.prog + + __ql_examples = f"""Examples: + + With code: + {prog} code --os linux --arch arm --format hex -f examples/shellcodes/linarm32_tcp_reverse_shell.hex + {prog} code --os linux --arch x86 --format asm -f examples/shellcodes/lin32_execve.asm + + With binary file: + {prog} run -f examples/rootfs/x8664_linux/bin/x8664_hello --rootfs examples/rootfs/x8664_linux + {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux + + With binary file and Qdb: + {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --qdb + {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --qdb --rr + + With binary file and gdbserver: + {prog} run -f examples/rootfs/x8664_linux/bin/x8664_hello --gdb 127.0.0.1:9999 --rootfs examples/rootfs/x8664_linux + + With binary file and additional argv: + {prog} run -f examples/rootfs/x8664_linux/bin/x8664_args --rootfs examples/rootfs/x8664_linux --args test1 test2 test3 + + With binary file and various output format: + {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --verbose disasm + {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --filter ^open + + With UEFI file: + {prog} run -f examples/rootfs/x8664_efi/bin/TcgPlatformSetupPolicy --rootfs examples/rootfs/x8664_efi --env examples/rootfs/x8664_efi/rom2_nvar.pickel + + With binary file and json output: + {prog} run -f examples/rootfs/x86_windows/bin/x86_hello.exe --rootfs examples/rootfs/x86_windows --no-console --json + +""" + + parser.exit(0, __ql_examples) + + +def run(): + parser = argparse.ArgumentParser() + parser.add_argument('--version', action='version', version=f'qltool for Qiling {ql_ver}, using Unicorn {uc_ver}') + + commands = parser.add_subparsers(title='sub commands', description='select execution mode', dest='subcommand', required=True) + + # set "run" subcommand options + run_parser = commands.add_parser('run', help='run a program') + run_parser.add_argument('-f', '--filename', default=None, metavar="FILE", help="filename") + run_parser.add_argument('--rootfs', required=True, help='emulated rootfs') + run_parser.add_argument('--args', default=[], nargs=argparse.REMAINDER, dest="args", help="args") + run_parser.add_argument('run_args', default=[], nargs=argparse.REMAINDER) + + # set "code" subcommand options + code_parser = commands.add_parser('code', help='execute a shellcode') + code_parser.add_argument('-f', '--filename', metavar="FILE", help="filename") + code_parser.add_argument('-i', '--input', metavar="INPUT", dest="input", help='input hex value') + code_parser.add_argument('--arch', required=True, choices=arch_map, action=__arg_archtype) + code_parser.add_argument('--thumb', action='store_true', default=False, help='specify thumb mode for ARM') + code_parser.add_argument('--endian', choices=endian_map, default=QL_ENDIAN.EL, action=__arg_endian, help='specify endianness for bi-endian archs') + code_parser.add_argument('--os', required=True, choices=os_map, action=__arg_ostype) + code_parser.add_argument('--rootfs', default='.', help='emulated root filesystem, that is where all libraries reside') + code_parser.add_argument('--format', choices=('asm', 'hex', 'bin'), default='bin', help='input file format') + + # set "examples" subcommand + expl_parser = commands.add_parser('examples', help='show examples and exit', add_help=False) + + # set "qltui" subcommand + qltui_parser = commands.add_parser('qltui', help='show qiling Terminal User Interface', add_help=False) + + comm_parser = run_parser + + if len(sys.argv) > 1 and sys.argv[1] == 'code': + comm_parser = code_parser + + # set common options + comm_parser.add_argument('-v', '--verbose', choices=verbose_map, default=QL_VERBOSE.DEFAULT, action=__arg_verbose, help='set verbosity level') + comm_parser.add_argument('--env', metavar="FILE", action=__arg_env, default={}, help="pickle file containing an environment dictionary") + comm_parser.add_argument('-g', '--gdb', nargs='?', metavar='SERVER:PORT', const='gdb', help='enable gdb server') + comm_parser.add_argument('--qdb', action='store_true', help='attach Qdb at entry point, it\'s MIPS, ARM(THUMB) supported only for now') + comm_parser.add_argument('--rr', action='store_true', help='switch on record and replay feature in qdb, only works with --qdb') + comm_parser.add_argument('--profile', help="define a customized profile") + comm_parser.add_argument('--no-console', action='store_false', dest='console', help='do not emit output to console') + comm_parser.add_argument('-e', '--filter', metavar='REGEXP', default=None, help="apply a filtering regexp on log output") + comm_parser.add_argument('--log-file', help="write log to a file") + comm_parser.add_argument('--log-plain', action='store_true', help="do not use colors in log output") + comm_parser.add_argument('--root', action='store_true', help='enable sudo required mode') + comm_parser.add_argument('--debug-stop', action='store_true', help='stop running on error; requires verbose to be set to either "debug" or "dump"') + comm_parser.add_argument('-m', '--multithread', action='store_true', help='run in multithread mode') + comm_parser.add_argument('--timeout', type=int, default=0, help='set emulation timeout') + comm_parser.add_argument('-c', '--coverage-file', default=None, help='code coverage file name') + comm_parser.add_argument('--coverage-format', default='drcov', choices=cov_utils.factory.formats, help='code coverage file format') + comm_parser.add_argument('--json', action='store_true', help='print a json report of the emulation') + comm_parser.add_argument('--libcache', action='store_true', help='enable dll caching for windows') + options = parser.parse_args() + + qltui_enabled = False + ql_args = {} + + if options.subcommand == 'qltui': + import qltui + options = qltui.get_data() + qltui_enabled = True + + if options.subcommand == 'examples': + handle_examples(parser) + + # ql file setup + elif options.subcommand == 'run': + ql_args = handle_run(options) + + # ql code setup + elif options.subcommand == 'code': + ql_args = handle_code(options) + + ql_args.update({ + 'env': options.env, + 'verbose': options.verbose, + 'profile': options.profile, + 'console': options.console, + 'filter': options.filter, + 'log_devices': options.log_file and [options.log_file], + 'log_plain': options.log_plain, + 'multithread': options.multithread, + 'libcache': options.libcache + }) + + ql = Qiling(**ql_args) + + # attach Qdb at entry point + if options.qdb: + QlQdb(ql, rr=bool(options.rr)).run() + + parser.exit(0) + + # ql execute additional options + if options.gdb: + argval = options.gdb + + if argval != 'gdb': + argval = f'gdb:{argval}' + + ql.debugger = argval + + if options.debug_stop: + if options.verbose not in (QL_VERBOSE.DEBUG, QL_VERBOSE.DUMP): + parser.error('the debug_stop option requires verbose to be set to either "debug" or "dump"') + + ql.debug_stop = True + + if options.root: + ql.os.root = True + + if qltui_enabled: + hook_dictionary = qltui.hook(ql) + + # ql run + if options.coverage_file: + with cov_utils.collect_coverage(ql, options.coverage_format, options.coverage_file): + ql.run(timeout=options.timeout) + else: + ql.run(timeout=options.timeout) + + if options.json: + rep = report.generate_report(ql) + + if qltui_enabled: + rep["syscalls"] = qltui.transform_syscalls(ql.os.stats.syscalls) + qltui.show_report(ql, rep, hook_dictionary) + else: + pprint(rep) + + exit(ql.os.exit_code) + +if __name__ == '__main__': + run() diff --git a/qltool b/qltool index 5164468c..6d5ec489 100755 --- a/qltool +++ b/qltool @@ -3,322 +3,8 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import argparse -import os -import sys -import ast -import pickle +from qiling.cli import run -from pprint import pprint -from typing import TYPE_CHECKING, Any, AnyStr, Dict, Literal, Mapping, Type, overload - -from unicorn import __version__ as uc_ver -from qiling import __version__ as ql_ver - -from qiling import Qiling -from qiling.arch import utils as arch_utils -from qiling.debugger.qdb import QlQdb -from qiling.const import QL_ENDIAN, QL_VERBOSE, endian_map, os_map, arch_map, verbose_map -from qiling.extensions.coverage import utils as cov_utils -from qiling.extensions import report - - -if TYPE_CHECKING: - from enum import Enum - - -@overload -def read_file(fname: AnyStr) -> str: ... - - -@overload -def read_file(fname: AnyStr, mode: Literal['b']) -> bytes: ... - - -def read_file(fname: AnyStr, mode: Literal['b', ''] = ''): - """Read file contents. - """ - - with open(fname, f'r{mode}') as f: - content = f.read() - - return content - - -class __arg_env(argparse.Action): - def __call__(self, parser, namespace, values: str, option_string): - if os.path.exists(values): - with open(values, 'rb') as f: - env = pickle.load(f) - else: - env = ast.literal_eval(values) - - setattr(namespace, self.dest, env or {}) - - -def __make_enum_arg(enum_rmap: Mapping[str, 'Enum'], aliases: Mapping[str, str] = {}) -> Type[argparse.Action]: - class __enum_arg(argparse.Action): - def __call__(self, parser, namespace, values: str, option_string): - values = values.casefold() - - if values in aliases: - values = aliases[values] - - setattr(namespace, self.dest, enum_rmap[values]) - - return __enum_arg - - -__arg_endian = __make_enum_arg(endian_map) -__arg_archtype = __make_enum_arg(arch_map, {'x86_64': 'x8664', 'riscv32': 'riscv'}) -__arg_ostype = __make_enum_arg(os_map, {'darwin': 'macos'}) -__arg_verbose = __make_enum_arg(verbose_map) - - -def handle_code(options: argparse.Namespace) -> Dict[str, Any]: - if options.format == 'hex': - if options.input is not None: - print("Loading HEX from ARGV") - payload = str(options.input) - - elif options.filename is not None: - print("Loading HEX from FILE") - payload = read_file(options.filename) - - else: - print("Error: please specify an input source for hex") - exit(1) - - code = bytes.fromhex(payload.replace(r'\x', '')) - - elif options.format == 'asm': - print("Loading ASM from FILE") - - if options.filename is None: - print("Error: please specify an input source for asm") - exit(1) - - payload = read_file(options.filename) - - # assemble the payload and turn it into opcodes bytes - assembler = arch_utils.assembler(options.arch, options.endian, options.thumb) - code, _ = assembler.asm(payload) - code = bytes(code) - - elif options.format == 'bin': - print("Loading BIN from FILE") - - if options.filename is None: - print("Error: please specify an input source for bin") - exit(1) - - code = read_file(options.filename, 'b') - - ql_args = { - 'rootfs': options.rootfs, - 'code': code, - 'ostype': options.os, - 'archtype': options.arch, - 'endian': options.endian, - 'thumb': options.thumb - } - - return ql_args - - -def handle_run(options: argparse.Namespace) -> Dict[str, Any]: - effective_argv = [] - - # with argv - if options.filename is not None and options.run_args == []: - effective_argv = [options.filename] + options.args - - # Without argv - elif options.filename is None and options.args == [] and options.run_args != []: - effective_argv = options.run_args - - else: - print("ERROR: Command error!") - - ql_args = { - 'argv': effective_argv, - 'rootfs': options.rootfs - } - - return ql_args - - -def handle_examples(parser: argparse.ArgumentParser): - prog = os.path.basename(__file__) - - __ql_examples = f"""Examples: - - With code: - {prog} code --os linux --arch arm --format hex -f examples/shellcodes/linarm32_tcp_reverse_shell.hex - {prog} code --os linux --arch x86 --format asm -f examples/shellcodes/lin32_execve.asm - - With binary file: - {prog} run -f examples/rootfs/x8664_linux/bin/x8664_hello --rootfs examples/rootfs/x8664_linux - {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux - - With binary file and Qdb: - {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --qdb - {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --qdb --rr - - With binary file and gdbserver: - {prog} run -f examples/rootfs/x8664_linux/bin/x8664_hello --gdb 127.0.0.1:9999 --rootfs examples/rootfs/x8664_linux - - With binary file and additional argv: - {prog} run -f examples/rootfs/x8664_linux/bin/x8664_args --rootfs examples/rootfs/x8664_linux --args test1 test2 test3 - - With binary file and various output format: - {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --verbose disasm - {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --filter ^open - - With UEFI file: - {prog} run -f examples/rootfs/x8664_efi/bin/TcgPlatformSetupPolicy --rootfs examples/rootfs/x8664_efi --env examples/rootfs/x8664_efi/rom2_nvar.pickel - - With binary file and json output: - {prog} run -f examples/rootfs/x86_windows/bin/x86_hello.exe --rootfs examples/rootfs/x86_windows --no-console --json - -""" - - parser.exit(0, __ql_examples) - - -def run(): - parser = argparse.ArgumentParser() - parser.add_argument('--version', action='version', version=f'qltool for Qiling {ql_ver}, using Unicorn {uc_ver}') - - commands = parser.add_subparsers(title='sub commands', description='select execution mode', dest='subcommand', required=True) - - # set "run" subcommand options - run_parser = commands.add_parser('run', help='run a program') - run_parser.add_argument('-f', '--filename', default=None, metavar="FILE", help="filename") - run_parser.add_argument('--rootfs', required=True, help='emulated rootfs') - run_parser.add_argument('--args', default=[], nargs=argparse.REMAINDER, dest="args", help="args") - run_parser.add_argument('run_args', default=[], nargs=argparse.REMAINDER) - - # set "code" subcommand options - code_parser = commands.add_parser('code', help='execute a shellcode') - code_parser.add_argument('-f', '--filename', metavar="FILE", help="filename") - code_parser.add_argument('-i', '--input', metavar="INPUT", dest="input", help='input hex value') - code_parser.add_argument('--arch', required=True, choices=arch_map, action=__arg_archtype) - code_parser.add_argument('--thumb', action='store_true', default=False, help='specify thumb mode for ARM') - code_parser.add_argument('--endian', choices=endian_map, default=QL_ENDIAN.EL, action=__arg_endian, help='specify endianness for bi-endian archs') - code_parser.add_argument('--os', required=True, choices=os_map, action=__arg_ostype) - code_parser.add_argument('--rootfs', default='.', help='emulated root filesystem, that is where all libraries reside') - code_parser.add_argument('--format', choices=('asm', 'hex', 'bin'), default='bin', help='input file format') - - # set "examples" subcommand - expl_parser = commands.add_parser('examples', help='show examples and exit', add_help=False) - - # set "qltui" subcommand - qltui_parser = commands.add_parser('qltui', help='show qiling Terminal User Interface', add_help=False) - - comm_parser = run_parser - - if len(sys.argv) > 1 and sys.argv[1] == 'code': - comm_parser = code_parser - - # set common options - comm_parser.add_argument('-v', '--verbose', choices=verbose_map, default=QL_VERBOSE.DEFAULT, action=__arg_verbose, help='set verbosity level') - comm_parser.add_argument('--env', metavar="FILE", action=__arg_env, default={}, help="pickle file containing an environment dictionary") - comm_parser.add_argument('-g', '--gdb', nargs='?', metavar='SERVER:PORT', const='gdb', help='enable gdb server') - comm_parser.add_argument('--qdb', action='store_true', help='attach Qdb at entry point, it\'s MIPS, ARM(THUMB) supported only for now') - comm_parser.add_argument('--rr', action='store_true', help='switch on record and replay feature in qdb, only works with --qdb') - comm_parser.add_argument('--profile', help="define a customized profile") - comm_parser.add_argument('--no-console', action='store_false', dest='console', help='do not emit output to console') - comm_parser.add_argument('-e', '--filter', metavar='REGEXP', default=None, help="apply a filtering regexp on log output") - comm_parser.add_argument('--log-file', help="write log to a file") - comm_parser.add_argument('--log-plain', action='store_true', help="do not use colors in log output") - comm_parser.add_argument('--root', action='store_true', help='enable sudo required mode') - comm_parser.add_argument('--debug-stop', action='store_true', help='stop running on error; requires verbose to be set to either "debug" or "dump"') - comm_parser.add_argument('-m', '--multithread', action='store_true', help='run in multithread mode') - comm_parser.add_argument('--timeout', type=int, default=0, help='set emulation timeout') - comm_parser.add_argument('-c', '--coverage-file', default=None, help='code coverage file name') - comm_parser.add_argument('--coverage-format', default='drcov', choices=cov_utils.factory.formats, help='code coverage file format') - comm_parser.add_argument('--json', action='store_true', help='print a json report of the emulation') - comm_parser.add_argument('--libcache', action='store_true', help='enable dll caching for windows') - options = parser.parse_args() - - qltui_enabled = False - ql_args = {} - - if options.subcommand == 'qltui': - import qltui - options = qltui.get_data() - qltui_enabled = True - - if options.subcommand == 'examples': - handle_examples(parser) - - # ql file setup - elif options.subcommand == 'run': - ql_args = handle_run(options) - - # ql code setup - elif options.subcommand == 'code': - ql_args = handle_code(options) - - ql_args.update({ - 'env': options.env, - 'verbose': options.verbose, - 'profile': options.profile, - 'console': options.console, - 'filter': options.filter, - 'log_devices': options.log_file and [options.log_file], - 'log_plain': options.log_plain, - 'multithread': options.multithread, - 'libcache': options.libcache - }) - - ql = Qiling(**ql_args) - - # attach Qdb at entry point - if options.qdb: - QlQdb(ql, rr=bool(options.rr)).run() - - parser.exit(0) - - # ql execute additional options - if options.gdb: - argval = options.gdb - - if argval != 'gdb': - argval = f'gdb:{argval}' - - ql.debugger = argval - - if options.debug_stop: - if options.verbose not in (QL_VERBOSE.DEBUG, QL_VERBOSE.DUMP): - parser.error('the debug_stop option requires verbose to be set to either "debug" or "dump"') - - ql.debug_stop = True - - if options.root: - ql.os.root = True - - if qltui_enabled: - hook_dictionary = qltui.hook(ql) - - # ql run - if options.coverage_file: - with cov_utils.collect_coverage(ql, options.coverage_format, options.coverage_file): - ql.run(timeout=options.timeout) - else: - ql.run(timeout=options.timeout) - - if options.json: - rep = report.generate_report(ql) - - if qltui_enabled: - rep["syscalls"] = qltui.transform_syscalls(ql.os.stats.syscalls) - qltui.show_report(ql, rep, hook_dictionary) - else: - pprint(rep) - - exit(ql.os.exit_code) if __name__ == '__main__': run() diff --git a/tests/test_qltool.py b/tests/test_qltool.py index 79f25738..9949b90f 100644 --- a/tests/test_qltool.py +++ b/tests/test_qltool.py @@ -7,7 +7,11 @@ import os import subprocess import re import sys +import sysconfig +import tempfile import unittest +from importlib.metadata import version +from pathlib import Path class Qltool_Test(unittest.TestCase): @@ -48,5 +52,43 @@ class Qltool_Test(unittest.TestCase): self.assertTrue(all(p.search(entry) is not None for entry in log_entries)) +class InstalledQltool_Test(unittest.TestCase): + """Exercise the installed distribution outside the source checkout.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.cwd = directory.name + self.qltool = Path(sysconfig.get_path('scripts')) / ('qltool.exe' if os.name == 'nt' else 'qltool') + + def __run(self, *args: str, expected_exit: int = 0) -> str: + result = subprocess.run( + [str(self.qltool), *args], cwd=self.cwd, + env={**os.environ, 'PYTHONPATH': ''}, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, timeout=30 + ) + self.assertEqual(result.returncode, expected_exit, result.stdout) + return result.stdout + + def test_qltool_version_and_examples(self): + self.assertIn(f'Qiling {version("qiling")}', self.__run('--version')) + self.assertIn('qltool code --os linux', self.__run('examples')) + + def test_qltool_shellcode_exit_status(self): + # Linux x86 exit(7), using the profiles shipped in the installed wheel. + self.__run('code', '--os', 'linux', '--arch', 'x86', '--format', 'hex', + '--input', 'b801000000bb07000000cd80', '--verbose', 'off', + expected_exit=7) + + def test_qltui_import(self): + result = subprocess.run( + [sys.executable, '-I', '-c', 'import qltui; assert callable(qltui.get_data)'], + cwd=self.cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, timeout=30 + ) + self.assertEqual(result.returncode, 0, result.stdout) + + if __name__ == "__main__": unittest.main() From 2dbbf4858841ca2778e376080bdd9f30d389ff53 Mon Sep 17 00:00:00 2001 From: xwings Date: Sat, 5 Sep 2026 16:59:26 +0800 Subject: [PATCH 21/21] bump to 1.4.11 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index da3e4b52..98da381d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] urls = { homepage = "https://qiling.io" } name = "qiling" -version = "1.4.11.dev0" +version = "1.4.11" description = "Qiling is an advanced binary emulation framework that cross-platform-architecture" authors = ["KaiJern Lau (xwings) "] license = "GPL-2.0-or-later"