From f141b73c7dc00f3a127e0ab8c4eb4c800c2145ab Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Mon, 24 Mar 2025 00:43:47 +0100 Subject: [PATCH 001/138] Add HeapReAlloc hook --- qiling/os/windows/dlls/kernel32/heapapi.py | 30 ++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/qiling/os/windows/dlls/kernel32/heapapi.py b/qiling/os/windows/dlls/kernel32/heapapi.py index c48e5aa8..6746c6d8 100644 --- a/qiling/os/windows/dlls/kernel32/heapapi.py +++ b/qiling/os/windows/dlls/kernel32/heapapi.py @@ -70,6 +70,36 @@ def hook_HeapAlloc(ql: Qiling, address: int, params): return ptr +# DECLSPEC_ALLOCATOR LPVOID HeapReAlloc( +# HANDLE hHeap, +# DWORD dwFlags, +# _Frees_ptr_opt_ LPVOID lpMem, +# SIZE_T dwBytes +# ); +@winsdkapi(cc=STDCALL, params={ + 'hHeap' : HANDLE, + 'dwFlags' : DWORD, + 'lpMem': LPVOID, + 'dwBytes' : SIZE_T +}) +def hook_HeapReAlloc(ql: Qiling, address: int, params): + base = params["lpMem"] + newSize = params["dwBytes"] + + oldSize = ql.os.heap.size(base) + oldData = bytes(ql.mem.read(base, oldSize)) + + ql.os.heap.free(base) + + if newSize < oldSize: + oldData = oldData[0:newSize] + + newBase = ql.os.heap.alloc(newSize) + if newBase: + ql.mem.write(newBase, oldData) + + return newBase + # SIZE_T HeapSize( # HANDLE hHeap, # DWORD dwFlags, From 742630f724f4e1faecc329bff275bb78d373a886 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Mon, 24 Mar 2025 00:45:33 +0100 Subject: [PATCH 002/138] Add _realloc_base hook --- qiling/os/windows/dlls/msvcrt.py | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/qiling/os/windows/dlls/msvcrt.py b/qiling/os/windows/dlls/msvcrt.py index 2db18455..fffc58c0 100644 --- a/qiling/os/windows/dlls/msvcrt.py +++ b/qiling/os/windows/dlls/msvcrt.py @@ -498,6 +498,42 @@ def hook_malloc(ql: Qiling, address: int, params): return ql.os.heap.alloc(size) +def __realloc(ql: Qiling, address: int, params): + block = params['block'] + size = params['size'] + + if not block: + return ql.os.heap.alloc(size) + + if size == 0: + ql.os.heap.free(block) + return 0 + + oldSize = ql.os.heap.size(block) + oldData = bytes(ql.mem.read(block, size)) + ql.os.heap.free(block) + + if size < oldSize: + oldData = oldData[0:size] + + newBase = ql.os.heap.alloc(size) + + if newBase: + ql.mem.write(newBase, oldData) + + return newBase + +# void* __cdecl _realloc_base( +# void* const block, +# size_t const size +# ) +@winsdkapi(cc=CDECL, params={ + 'block' : POINTER, + 'size' : UINT +}) +def hook__realloc_base(ql: Qiling, address: int, params): + return __realloc(ql, address, params) + def __free(ql: Qiling, address: int, params): address = params['address'] From 8101061622a40f35508731fdc22aa8e660a60fe9 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Mon, 24 Mar 2025 00:46:29 +0100 Subject: [PATCH 003/138] Add RtlPcToFileHeader hook --- qiling/os/windows/dlls/ntdll.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index 92ba4a84..5560308c 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -452,4 +452,26 @@ def hook_wcsstr(ql: Qiling, address: int, params): @winsdkapi(cc=STDCALL, params={}) def hook_CsrGetProcessId(ql: Qiling, address: int, params): pid = ql.os.profile["PROCESSES"].getint("csrss.exe", fallback=12345) - return pid \ No newline at end of file + return pid + +# NTSYSAPI PVOID RtlPcToFileHeader( +# [in] PVOID PcValue, +# [out] PVOID *BaseOfImage +# ); +@winsdkapi(cc=STDCALL, params={ + 'PcValue' : PVOID, + 'BaseOfImage': PVOID +}) +def hook_RtlPcToFileHeader(ql: Qiling, address: int, params): + pc = params["PcValue"] + base_of_image_ptr = params["BaseOfImage"] + + containing_image = ql.loader.find_containing_image(pc) + + if containing_image: + base_addr = containing_image.base + else: + base_addr = 0 + + ql.mem.write_ptr(base_of_image_ptr, base_addr) + return base_addr From fb8128f42c20bcddecca5c83216a1d164257d98b Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Mon, 24 Mar 2025 00:47:37 +0100 Subject: [PATCH 004/138] Remove _initterm and _initterm_e hooks --- qiling/os/windows/dlls/msvcrt.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/qiling/os/windows/dlls/msvcrt.py b/qiling/os/windows/dlls/msvcrt.py index fffc58c0..48cad23d 100644 --- a/qiling/os/windows/dlls/msvcrt.py +++ b/qiling/os/windows/dlls/msvcrt.py @@ -174,17 +174,6 @@ def hook_puts(ql: Qiling, address: int, params): def hook__cexit(ql: Qiling, address: int, params): pass -# void __cdecl _initterm( -# PVFV *, -# PVFV * -# ); -@winsdkapi(cc=CDECL, params={ - 'pfbegin' : POINTER, - 'pfend' : POINTER -}) -def hook__initterm(ql: Qiling, address: int, params): - return 0 - # void exit( # int const status # ); @@ -194,17 +183,6 @@ def hook__initterm(ql: Qiling, address: int, params): def hook_exit(ql: Qiling, address: int, params): ql.emu_stop() -# int __cdecl _initterm_e( -# PVFV *, -# PVFV * -# ); -@winsdkapi(cc=CDECL, params={ - 'pfbegin' : POINTER, - 'pfend' : POINTER -}) -def hook__initterm_e(ql: Qiling, address: int, params): - return 0 - # char*** __cdecl __p___argv (void); @winsdkapi(cc=CDECL, params={}) def hook___p___argv(ql: Qiling, address: int, params): From 84b02eeb1ab2474be9502803290d79be58fc9d78 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Mon, 24 Mar 2025 00:48:45 +0100 Subject: [PATCH 005/138] Remove __acrt_iob_func hook --- qiling/os/windows/dlls/msvcrt.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/qiling/os/windows/dlls/msvcrt.py b/qiling/os/windows/dlls/msvcrt.py index 48cad23d..9b9e1bba 100644 --- a/qiling/os/windows/dlls/msvcrt.py +++ b/qiling/os/windows/dlls/msvcrt.py @@ -281,13 +281,6 @@ def hook_wprintf(ql: Qiling, address: int, params): return count -# MSVCRT_FILE * CDECL MSVCRT___acrt_iob_func(unsigned idx) -@winsdkapi(cc=CDECL, params={ - 'idx': UINT -}) -def hook___acrt_iob_func(ql: Qiling, address: int, params): - return 0 - def __stdio_common_vfprintf(ql: Qiling, address: int, params, wstring: bool): format = params['_Format'] arglist = params['_ArgList'] From 69a6906c0b8960e73186b808dead92a254d50a28 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Thu, 27 Mar 2025 21:32:25 +0100 Subject: [PATCH 006/138] Add function table parsing and lookup to the PE loader --- qiling/loader/pe.py | 75 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/qiling/loader/pe.py b/qiling/loader/pe.py index 7e6746de..8a4b94a8 100644 --- a/qiling/loader/pe.py +++ b/qiling/loader/pe.py @@ -79,6 +79,12 @@ class Process: export_symbols: MutableMapping[int, Dict[str, Any]] libcache: Optional[QlPeCache] + # maps image base to RVA of its function table + function_table_lookup: MutableMapping[int, int] + + # maps image base to its list of function table entries + function_tables: MutableMapping[int, list] + def __init__(self, ql: Qiling): self.ql = ql @@ -105,6 +111,67 @@ class Process: vpath = ntpath.join(dirname, basename) return self.ql.os.path.virtual_to_host_path(vpath), basename.casefold() + + def init_function_tables(self, pe: pefile.PE, image_base: int): + """Parse function table data for the given PE file. + Only works for x64 images. + + Args: + pe: the PE image whose function data should be parsed + image_base: the absolute address at which the image was loaded + """ + if self.ql.arch.type == QL_ARCH.X8664: + + # Check if the PE file has an exception directory + if hasattr(pe, 'DIRECTORY_ENTRY_EXCEPTION'): + exception_dir = pe.OPTIONAL_HEADER.DATA_DIRECTORY[ + pefile.DIRECTORY_ENTRY['IMAGE_DIRECTORY_ENTRY_EXCEPTION'] + ] + + self.function_table_lookup[image_base] = exception_dir.VirtualAddress + + runtime_function_list = [] + + for _, exception_entry in enumerate(pe.DIRECTORY_ENTRY_EXCEPTION, start=1): + runtime_function_list.append(exception_entry) + + if self.function_tables.get(image_base) is None: + self.function_tables[image_base] = [] + + self.function_tables[image_base].extend(runtime_function_list) + + self.ql.log.debug(f'Parsed {len(runtime_function_list)} exception directory entries') + + else: + self.ql.log.debug(f'Image has no exception directory; skipping exception data') + + def lookup_function_entry(self, base_addr: int, control_pc: int): + """Look up a RUNTIME_FUNCTION entry and its index in a module's + function table, such that the given program counter falls within + the entry's begin and end range. + + Args: + base_addr: The base address of the image whose exception directory to search. + control_pc: The program counter. + + Returns: + A tuple (index, runtime_function) + """ + function_table = self.function_tables[base_addr] + + # Initiate a search of the function table for a RUNTIME_FUNCTION + # entry such that the provided PC falls within its start and end range. + for i, runtime_function in enumerate(function_table): + + # Begin and end addresses exist in the entry as RVAs, + # convert them to absolute addresses. + begin_addr = base_addr + runtime_function.struct.BeginAddress + end_addr = base_addr + runtime_function.struct.EndAddress + + if begin_addr <= control_pc < end_addr: + return i, runtime_function + + return None, None def load_dll(self, name: str, is_driver: bool = False) -> int: dll_path, dll_name = self.__get_path_elements(name) @@ -195,6 +262,9 @@ class Process: with ShowProgress(self.ql.log, 0.1337): dll.relocate_image(image_base) + # initialize the function tables only after possible relocation + self.init_function_tables(dll, image_base) + data = bytearray(dll.get_memory_mapped_image()) assert image_size >= len(data) @@ -709,6 +779,8 @@ class QlLoaderPE(QlLoader, Process): self.export_symbols = {} self.import_address_table = {} self.ldr_list = [] + self.function_tables = {} + self.function_table_lookup = {} self.pe_image_address = 0 self.pe_image_size = 0 self.dll_size = 0 @@ -841,6 +913,9 @@ class QlLoaderPE(QlLoader, Process): # set up call frame for DllMain self.ql.os.fcall.call_native(self.entry_point, args, None) + # Initialize the function tables + super().init_function_tables(pe, image_base) + elif pe is None: self.ql.mem.map(self.entry_point, self.ql.os.code_ram_size, info="[shellcode]") From 1f45f9bda36e6177f0d840e3a850d58bd4817e6f Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Thu, 27 Mar 2025 21:35:36 +0100 Subject: [PATCH 007/138] Adjust segment descriptors for x86_64 --- qiling/arch/x86_utils.py | 51 +++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/qiling/arch/x86_utils.py b/qiling/arch/x86_utils.py index 4726ef74..dfd45623 100644 --- a/qiling/arch/x86_utils.py +++ b/qiling/arch/x86_utils.py @@ -5,6 +5,7 @@ from typing import Optional from qiling import Qiling from qiling.arch.x86 import QlArchIntel from qiling.arch.x86_const import * +from qiling.const import QL_ARCH from qiling.exception import QlGDTError, QlMemoryMappedError from qiling.os.memory import QlMemoryManager @@ -60,6 +61,11 @@ class GDTManager: # setup GDT by writing to GDTR ql.arch.regs.write(UC_X86_REG_GDTR, (0, base, limit, 0x0)) + if ql.arch.type == QL_ARCH.X8664: + self.is_long_mode = True + else: + self.is_long_mode = False + self.array = GDTArray(ql.mem, base, num_entries) @staticmethod @@ -93,7 +99,18 @@ class GDTManager: return (idx << 3) | QL_X86_SEGSEL_TI_GDT | rpl def register_gdt_segment(self, index: int, seg_base: int, seg_limit: int, access: int) -> int: - flags = QL_X86_F_OPSIZE_32 + is_code = access & QL_X86_A_CODE + + if is_code and self.is_long_mode: + # If this is a code segment and 64-bit long mode is enabled, + # then set the long segment descriptor bit. + # This prevents some strange CPU errors encountered with + # intra-privilege level IRET instructions used for + # context switching on 64-bit Windows. + flags = QL_X86_F_LONG + else: + # Otherwise, OPSIZE_32 should be set. + flags = QL_X86_F_OPSIZE_32 # is this a huge segment? if seg_limit > (1 << 16): @@ -138,16 +155,21 @@ class SegmentManager: class SegmentManager86(SegmentManager): def setup_cs_ds_ss_es(self, base: int, size: int) -> None: - # While debugging the linux kernel segment, the cs segment was found on the third segment of gdt. + # TODO: 64-bit code segment access bits were adjusted, removing the conforming bit. + # Perhaps make the same change for x86? access = QL_X86_A_PRESENT | QL_X86_A_PRIV_3 | QL_X86_A_DESC_CODE | QL_X86_A_CODE | QL_X86_A_CODE_C | QL_X86_A_CODE_R + # While debugging the linux kernel segment, the cs segment was found on the third segment of gdt. selector = self.gdtm.register_gdt_segment(3, base, size - 1, access) self.arch.regs.cs = selector # TODO : The section permission here should be QL_X86_A_PRIV_3, but I do n’t know why it can only be set to QL_X86_A_PRIV_0. + # TODO: 64-bit data segment access bits were adjusted, removing the direction bit. + # After this change, there were no problems changing the privilege level to ring 3. + # Perhaps make the same change for x86? + access = QL_X86_A_PRESENT | QL_X86_A_PRIV_0 | QL_X86_A_DESC_DATA | QL_X86_A_DATA | QL_X86_A_DATA_E | QL_X86_A_DATA_W # While debugging the Linux kernel segment, I found that the three segments DS, SS, and ES all point to the same location in the GDT table. # This position is the fifth segment table of GDT. - access = QL_X86_A_PRESENT | QL_X86_A_PRIV_0 | QL_X86_A_DESC_DATA | QL_X86_A_DATA | QL_X86_A_DATA_E | QL_X86_A_DATA_W selector = self.gdtm.register_gdt_segment(5, base, size - 1, access) self.arch.regs.ds = selector @@ -169,15 +191,32 @@ class SegmentManager86(SegmentManager): class SegmentManager64(SegmentManager): def setup_cs_ds_ss_es(self, base: int, size: int) -> None: + # Code segment access bits: + # * QL_X86_A_PRESENT : Present + # * QL_X86_A_PRIV_3 : Ring 3 (user-mode) + # * QL_X86_A_DESC_CODE : Segment describes a code segment + # * QL_X86_A_CODE : Executable bit set + # * QL_X86_A_CODE_R : Readable + # Not set: + # * QL_X86_A_CODE_C : Conforming bit + # -> unset means code in this segment can only be executed from the ring set in DPL. + access = QL_X86_A_PRESENT | QL_X86_A_PRIV_3 | QL_X86_A_DESC_CODE | QL_X86_A_CODE | QL_X86_A_CODE_R # While debugging the linux kernel segment, the cs segment was found on the sixth segment of gdt. - access = QL_X86_A_PRESENT | QL_X86_A_PRIV_3 | QL_X86_A_DESC_CODE | QL_X86_A_CODE | QL_X86_A_CODE_C | QL_X86_A_CODE_R selector = self.gdtm.register_gdt_segment(6, base, size - 1, access) self.arch.regs.cs = selector - # TODO : The section permission here should be QL_X86_A_PRIV_3, but I do n’t know why it can only be set to QL_X86_A_PRIV_0. + # Data segment access bits: + # * QL_X86_A_PRESENT : Present + # * QL_X86_A_PRIV_3 : Ring 3 (user-mode) + # * QL_X86_A_DESC_DATA : Segment describes a data segment + # * QL_X86_A_DATA : Executable bit NOT set + # * QL_X86_A_DATA_W : Writable + # Not set: + # * QL_X86_A_DATA_E : Direction bit + # -> unset means the data segment grows upward, rather than downward. + access = QL_X86_A_PRESENT | QL_X86_A_PRIV_3 | QL_X86_A_DESC_DATA | QL_X86_A_DATA | QL_X86_A_DATA_W # When I debug the Linux kernel, I find that only the SS is set to the fifth segment table, and the rest are not set. - access = QL_X86_A_PRESENT | QL_X86_A_PRIV_0 | QL_X86_A_DESC_DATA | QL_X86_A_DATA | QL_X86_A_DATA_E | QL_X86_A_DATA_W selector = self.gdtm.register_gdt_segment(5, base, size - 1, access) # self.arch.regs.ds = selector From 27db58de8615d9d3fc3875c603ad72a70cea2b08 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Thu, 27 Mar 2025 21:37:10 +0100 Subject: [PATCH 008/138] Add hooks for function table lookup functions in ntdll --- qiling/os/windows/dlls/ntdll.py | 123 ++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index 5560308c..f1e95fa7 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -17,6 +17,7 @@ from qiling.os.windows.handle import * from qiling.os.windows import structs from qiling.os.windows import utils +from unicorn.x86_const import * # void *memcpy( # void *dest, @@ -475,3 +476,125 @@ def hook_RtlPcToFileHeader(ql: Qiling, address: int, params): ql.mem.write_ptr(base_of_image_ptr, base_addr) return base_addr + +# NTSYSAPI PRUNTIME_FUNCTION RtlLookupFunctionEntry( +# [in] DWORD64 ControlPc, +# [out] PDWORD64 ImageBase, +# [out] PUNWIND_HISTORY_TABLE HistoryTable +# ); +@winsdkapi(cc=STDCALL, params={ + 'ControlPc': PVOID, + 'ImageBase': PVOID, + 'HistoryTable': PVOID +}) +def hook_RtlLookupFunctionEntry(ql: Qiling, address: int, params): + control_pc = params["ControlPc"] + image_base_ptr = params["ImageBase"] + + # TODO: Make use of the history table to optimize this function. + # Alternatively, we could add caching to the loader, seeing as the + # loader is responsible for lookups in the function table. + + # For simplicity, we are going to ignore the history table. + # history_table_ptr = params["HistoryTable"] + + # This function should not be getting called on x86. + if ql.arch.type != QL_ARCH.X8664: + return QlErrorNotImplemented("RtlLookupFunctionEntry is not implemented for x86") + + containing_image = ql.loader.find_containing_image(control_pc) + + if containing_image: + base_addr = containing_image.base + else: + base_addr = 0 + + return 0 + + # If we got a valid location to write the image base ptr, + # copy it there, and proceed. + if image_base_ptr != 0: + ql.mem.write_ptr(image_base_ptr, base_addr) + + # Get the base address of the function table. + function_table_addr = base_addr + ql.loader.function_table_lookup[base_addr] + + # Look up the RUNTIME_FUNCTION entry; we are interested in the index in the table + # so that we can compute the address. + runtime_function_idx, runtime_function = ql.loader.lookup_function_entry(base_addr, control_pc) + + # If a suitable function entry was found, + # compute its address and return. + if runtime_function: + return function_table_addr + runtime_function_idx * 12 # sizeof(RUNTIME_FUNCTION) + + return 0 + +# NTSYSAPI +# PRUNTIME_FUNCTION +# RtlLookupFunctionTable ( +# IN PVOID ControlPc, +# OUT PVOID *ImageBase, +# OUT PULONG SizeOfTable +# ); +@winsdkapi(cc=STDCALL, params={ + 'ControlPc': PVOID, + 'ImageBase': PVOID, + 'SizeOfTable': PVOID +}) +def hook_RtlLookupFunctionTable(ql: Qiling, address: int, params): + control_pc = params["ControlPc"] + image_base_ptr = params["ImageBase"] + size_of_table_ptr = params["SizeOfTable"] + + # This function should not be getting called on x86. + if ql.arch.type != QL_ARCH.X8664: + return QlErrorNotImplemented("RtlLookupFunctionTable is not implemented for x86") + + containing_image = ql.loader.find_containing_image(control_pc) + + if containing_image: + base_addr = containing_image.base + else: + base_addr = 0 + + return 0 + + # If we got a valid location to write the image base ptr, + # copy it there, and proceed. + if image_base_ptr != 0: + ql.mem.write_ptr(image_base_ptr, base_addr) + + # If image base was 0, we are not going to find a valid function + # table anyway, so just return. + if base_addr == 0: + return 0 + + # Look up the RVA of the function table. + function_table_rva = ql.loader.function_table_lookup[base_addr] + + # The caller is expecting a pointer, so convert the RVA + # to an absolute address. + function_table_addr = int(base_addr + function_table_rva) + + # If a valid pointer for the size was provided, + # we want to figure out the size of the table. + if size_of_table_ptr != 0: + # Look up the function table from the loader, + # and get the number of entries. + function_table = ql.loader.function_tables[base_addr] + + # compute the total size of the table + size_of_table = len(function_table) * 12 # sizeof(RUNTIME_FUNCTION) + + # Write the size to memory at the provided pointer. + ql.mem.write_ptr(size_of_table_ptr, size_of_table, 4) + + return function_table_addr + +@winsdkapi(cc=STDCALL, params={}) +def hook_LdrControlFlowGuardEnforced(ql: Qiling, address: int, params): + # There are some checks in ntdll for whether CFG is enabled. + # We simply bypass these checks by returning 0. + # May not be necessary, but we do it just in case. + return 0 From 22a4b498ffce7dca7868d0320f37e7d0fef10b00 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Thu, 27 Mar 2025 21:37:29 +0100 Subject: [PATCH 009/138] Make RaiseException hook passthru --- qiling/os/windows/dlls/kernel32/errhandlingapi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiling/os/windows/dlls/kernel32/errhandlingapi.py b/qiling/os/windows/dlls/kernel32/errhandlingapi.py index 4dba7efb..6206d589 100644 --- a/qiling/os/windows/dlls/kernel32/errhandlingapi.py +++ b/qiling/os/windows/dlls/kernel32/errhandlingapi.py @@ -74,7 +74,7 @@ def hook_SetErrorMode(ql: Qiling, address: int, params): 'dwExceptionFlags' : DWORD, 'nNumberOfArguments' : DWORD, 'lpArguments' : POINTER -}) +}, passthru=True) def hook_RaiseException(ql: Qiling, address: int, params): nNumberOfArguments = params['nNumberOfArguments'] lpArguments = params['lpArguments'] From ce03a8a113dc24c2e5d725a1411f7606b791cd05 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Fri, 28 Mar 2025 22:56:15 +0100 Subject: [PATCH 010/138] Add ProcessCookie case for NtQueryInformationProcess hook --- qiling/os/windows/const.py | 1 + qiling/os/windows/dlls/ntdll.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/qiling/os/windows/const.py b/qiling/os/windows/const.py index d001925e..8fe29023 100644 --- a/qiling/os/windows/const.py +++ b/qiling/os/windows/const.py @@ -638,6 +638,7 @@ ProcessProtectionInformation = 61 ProcessDebugObjectHandle = 30 ProcessDebugFlags = 31 ProcessExecuteFlags = 34 +ProcessCookie = 36 ProcessImageInformation = 37 ProcessMitigationPolicy = 52 ProcessFaultInformation = 63 diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index f1e95fa7..34f18404 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -40,6 +40,7 @@ def hook_memcpy(ql: Qiling, address: int, params): return dest def _QueryInformationProcess(ql: Qiling, address: int, params): + handle = params["ProcessHandle"] flag = params["ProcessInformationClass"] obuf_ptr = params["ProcessInformation"] obuf_len = params['ProcessInformationLength'] @@ -69,6 +70,27 @@ def _QueryInformationProcess(ql: Qiling, address: int, params): res_data = bytes(pci_obj) + + elif flag == ProcessCookie: + hCurrentProcess = { + QL_ARCH.X86 : 0xFFFFFFFF, + QL_ARCH.X8664: 0xFFFFFFFFFFFFFFFF + }[ql.arch.type] + + if handle != hCurrentProcess: + # If a process attempts to query the cookie of another + # process, then QueryInformationProcess returns an error. + return STATUS_INVALID_PARAMETER + + # TODO: Change this to something else, + # maybe a static randomly generated value. + res_data = ql.pack32(0x00000001) + + if obuf_len != len(res_data): + # If the buffer length is not ULONG size + # then QueryInformationProcess returns an error. + return STATUS_INFO_LENGTH_MISMATCH + else: # TODO: support more info class ("flag") values ql.log.info(f'QueryInformationProcess: no implementation for info class {flag:#04x}') From 4ac6458dcc9c2597c760e722587bcbb40b2846ac Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Fri, 28 Mar 2025 22:56:51 +0100 Subject: [PATCH 011/138] Remove Encode/DecodePointer hooks --- qiling/os/windows/dlls/kernel32/winbase.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/qiling/os/windows/dlls/kernel32/winbase.py b/qiling/os/windows/dlls/kernel32/winbase.py index 6fe624b3..0c5c1d12 100644 --- a/qiling/os/windows/dlls/kernel32/winbase.py +++ b/qiling/os/windows/dlls/kernel32/winbase.py @@ -159,24 +159,6 @@ def hook__lwrite(ql: Qiling, address: int, params): def hook_FatalExit(ql: Qiling, address: int, params): ql.emu_stop() -# PVOID EncodePointer( -# _In_ PVOID Ptr -# ); -@winsdkapi(cc=STDCALL, params={ - 'Ptr' : PVOID -}) -def hook_EncodePointer(ql: Qiling, address: int, params): - return params['Ptr'] - -# PVOID DecodePointer( -# _In_ PVOID Ptr -# ); -@winsdkapi(cc=STDCALL, params={ - 'Ptr' : PVOID -}) -def hook_DecodePointer(ql: Qiling, address: int, params): - return params['Ptr'] - # UINT WinExec( # LPCSTR lpCmdLine, # UINT uCmdShow From 54564b00be5a9e1ac2737f564d6a92f6c47e1793 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Fri, 28 Mar 2025 23:18:35 +0100 Subject: [PATCH 012/138] Add support for forwarded exports to the PE loader --- qiling/loader/pe.py | 89 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/qiling/loader/pe.py b/qiling/loader/pe.py index 8a4b94a8..442a61d5 100644 --- a/qiling/loader/pe.py +++ b/qiling/loader/pe.py @@ -29,6 +29,16 @@ if TYPE_CHECKING: from logging import Logger from qiling import Qiling +class ForwardedExport: + def __init__(self, + source_dll: str, source_ordinal: str, source_symbol: str, + target_dll: str, target_symbol: str): + self.source_dll = source_dll + self.source_ordinal = source_ordinal + self.source_symbol = source_symbol + self.target_dll = target_dll + self.target_symbol = target_symbol + class QlPeCacheEntry(NamedTuple): ba: int @@ -85,6 +95,10 @@ class Process: # maps image base to its list of function table entries function_tables: MutableMapping[int, list] + # List of exports which have been forwarded from + # one DLL to another. + forwarded_exports: list[ForwardedExport] + def __init__(self, ql: Qiling): self.ql = ql @@ -172,6 +186,49 @@ class Process: return i, runtime_function return None, None + + def resolve_forwarded_exports(self): + while self.forwarded_exports: + forwarded_export = self.forwarded_exports.pop() + + source_dll = forwarded_export.source_dll + source_ordinal = forwarded_export.source_ordinal + source_symbol = forwarded_export.source_symbol + target_dll = forwarded_export.target_dll + target_symbol = forwarded_export.target_symbol + + target_iat = self.import_address_table.get(target_dll) + + if target_iat: + # If we have an existing entry in the process IAT for the code + # this entry forwards to, then we will point the symbol there + # rather than the symbol string in the exporter's data section. + forward_ea = target_iat.get(target_symbol) + + if forward_ea: + self.import_address_table[source_dll][source_symbol] = forward_ea + self.import_address_table[source_dll][source_ordinal] = forward_ea + + # Register the new address as having the source symbol/ordinal. + # This way, hooks on forward source symbols will function + # correctly. + + self.import_symbols[forward_ea] = { + 'name' : source_symbol, + 'ordinal' : source_ordinal, + 'dll' : source_dll.split('.')[0] + } + + # TODO: With the above code, hooks on functions which are + # forward targets may not work correctly. + # The most correct way to resolve this would be to add + # support for addresses to be associated with multiple symbols. + + self.ql.log.debug(f"Forwarding symbol {source_dll}.{source_symbol} to {target_dll}.{target_symbol}: Resolved symbol to ({forward_ea:#x})") + else: + self.ql.log.warning(f"Forwarding symbol {source_dll}.{source_symbol} to {target_dll}.{target_symbol}: Failed to resolve address") + else: + pass # If IAT was not found, it is probably a virtual library. def load_dll(self, name: str, is_driver: bool = False) -> int: dll_path, dll_name = self.__get_path_elements(name) @@ -273,6 +330,31 @@ class Process: for sym in dll.DIRECTORY_ENTRY_EXPORT.symbols: ea = image_base + sym.address + if sym.forwarder: + # Some exports are forwarders, meaning they + # actually refer to code in other libraries. + # + # For example, calls to + # kernel32.InterlockedPushEntrySList + # should be forwarded to + # ntdll.RtlInterlockedPushEntrySList + # + # If we do not properly account for forwarders then + # calls to these symbols will land in the exporter's + # data section and cause a lot of problems. + forward_str = sym.forwarder + + if b'.' in forward_str: + target_dll_name, target_symbol_name = forward_str.split(b'.', 1) + + target_dll_filename = (target_dll_name.lower() + b'.dll').decode() + + # Remember the forwarded export for later. + forwarded_export = ForwardedExport(dll_name, sym.ordinal, sym.name, + target_dll_filename, target_symbol_name) + + self.forwarded_exports.append(forwarded_export) + import_symbols[ea] = { 'name' : sym.name, 'ordinal' : sym.ordinal, @@ -297,6 +379,8 @@ class Process: self.import_address_table[dll_name] = import_table self.import_symbols.update(import_symbols) + self.resolve_forwarded_exports() + dll_base = image_base dll_len = image_size @@ -744,12 +828,14 @@ class QlLoaderPE(QlLoader, Process): def run(self): self.init_dlls = ( 'ntdll.dll', - 'kernel32.dll', + 'kernelbase.dll', # kernel32 forwards some exports to kernelbase + 'kernel32.dll', # for efficiency, load kernelbase first 'user32.dll' ) self.sys_dlls = ( 'ntdll.dll', + 'kernelbase.dll', 'kernel32.dll', 'mscoree.dll', 'ucrtbase.dll' @@ -781,6 +867,7 @@ class QlLoaderPE(QlLoader, Process): self.ldr_list = [] self.function_tables = {} self.function_table_lookup = {} + self.forwarded_exports = [] self.pe_image_address = 0 self.pe_image_size = 0 self.dll_size = 0 From 37288e6d40db2c6cd40731285e6ed4230cb081a3 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Fri, 28 Mar 2025 23:19:45 +0100 Subject: [PATCH 013/138] Add user32 to DllMain blacklist --- qiling/loader/pe.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiling/loader/pe.py b/qiling/loader/pe.py index 442a61d5..8c2924cd 100644 --- a/qiling/loader/pe.py +++ b/qiling/loader/pe.py @@ -435,8 +435,8 @@ class Process: # the blacklist may be revisited from time to time to see if any of the file # can be safely unlisted. blacklist = { - 32 : ('gdi32.dll',), - 64 : ('gdi32.dll',) + 32 : ('gdi32.dll','user32.dll',), + 64 : ('gdi32.dll','user32.dll',) }[self.ql.arch.bits] if dll_name in blacklist: From 52a3910d02b16a99ca13d5b99b138b182064c2a5 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Fri, 28 Mar 2025 23:52:08 +0100 Subject: [PATCH 014/138] Fix some typos in ntdll hook code --- qiling/os/windows/dlls/ntdll.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index 34f18404..94b952f7 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -522,7 +522,7 @@ def hook_RtlLookupFunctionEntry(ql: Qiling, address: int, params): # This function should not be getting called on x86. if ql.arch.type != QL_ARCH.X8664: - return QlErrorNotImplemented("RtlLookupFunctionEntry is not implemented for x86") + raise QlErrorNotImplemented("RtlLookupFunctionEntry is not implemented for x86") containing_image = ql.loader.find_containing_image(control_pc) @@ -571,7 +571,7 @@ def hook_RtlLookupFunctionTable(ql: Qiling, address: int, params): # This function should not be getting called on x86. if ql.arch.type != QL_ARCH.X8664: - return QlErrorNotImplemented("RtlLookupFunctionTable is not implemented for x86") + raise QlErrorNotImplemented("RtlLookupFunctionTable is not implemented for x86") containing_image = ql.loader.find_containing_image(control_pc) From 9661ad6326bebbd343cf81c2c12623a0bc44321a Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Sat, 29 Mar 2025 15:33:00 +0100 Subject: [PATCH 015/138] Add ZwRaiseException hook, move unhandled exception logic --- .../windows/dlls/kernel32/errhandlingapi.py | 36 -------- qiling/os/windows/dlls/ntdll.py | 90 +++++++++++++++++++ qiling/os/windows/structs.py | 16 ++++ 3 files changed, 106 insertions(+), 36 deletions(-) diff --git a/qiling/os/windows/dlls/kernel32/errhandlingapi.py b/qiling/os/windows/dlls/kernel32/errhandlingapi.py index 6206d589..084ffabc 100644 --- a/qiling/os/windows/dlls/kernel32/errhandlingapi.py +++ b/qiling/os/windows/dlls/kernel32/errhandlingapi.py @@ -44,15 +44,6 @@ def hook_GetLastError(ql: Qiling, address: int, params): def hook_SetLastError(ql: Qiling, address: int, params): ql.os.last_error = params['dwErrCode'] -# LONG UnhandledExceptionFilter( -# _EXCEPTION_POINTERS *ExceptionInfo -# ); -@winsdkapi(cc=STDCALL, params={ - 'ExceptionInfo' : POINTER -}) -def hook_UnhandledExceptionFilter(ql: Qiling, address: int, params): - return 1 - # UINT SetErrorMode( # UINT uMode # ); @@ -63,33 +54,6 @@ def hook_SetErrorMode(ql: Qiling, address: int, params): # TODO maybe this need a better implementation return 0 -# __analysis_noreturn VOID RaiseException( -# DWORD dwExceptionCode, -# DWORD dwExceptionFlags, -# DWORD nNumberOfArguments, -# const ULONG_PTR *lpArguments -# ); -@winsdkapi(cc=STDCALL, params={ - 'dwExceptionCode' : DWORD, - 'dwExceptionFlags' : DWORD, - 'nNumberOfArguments' : DWORD, - 'lpArguments' : POINTER -}, passthru=True) -def hook_RaiseException(ql: Qiling, address: int, params): - nNumberOfArguments = params['nNumberOfArguments'] - lpArguments = params['lpArguments'] - - handle = ql.os.handle_manager.search("TopLevelExceptionHandler") - - if handle is None: - ql.log.warning(f'RaiseException: top level exception handler not found') - return - - exception_handler = handle.obj - args = [(PARAM_INTN, ql.mem.read_ptr(lpArguments + i * ql.arch.pointersize)) for i in range(nNumberOfArguments)] if lpArguments else [] - - ql.os.fcall.call_native(exception_handler, args, None) - # PVOID AddVectoredExceptionHandler( # ULONG First, # PVECTORED_EXCEPTION_HANDLER Handler diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index 94b952f7..09b5d175 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -620,3 +620,93 @@ def hook_LdrControlFlowGuardEnforced(ql: Qiling, address: int, params): # We simply bypass these checks by returning 0. # May not be necessary, but we do it just in case. return 0 + +# NTSYSAPI +# NTSTATUS +# ZwRaiseException ( +# IN PEXCEPTION_RECORD ExceptionRecord, +# IN PCONTEXT ContextRecord, +# IN BOOLEAN FirstChance +# ); +@winsdkapi(cc=STDCALL, params={ + 'ExceptionRecord': PVOID, + 'ContextRecord': PVOID, + 'FirstChance': BOOLEAN +}) +def hook_ZwRaiseException(ql: Qiling, address: int, params): + exception_ptr = params['ExceptionRecord'] + context_ptr = params['ContextRecord'] + first_chance = params['FirstChance'] + + # In Windows, an unhandled exception triggers the + # top-level unhandled exception filter, after which the process + # is terminated and error reporting services are called. + # Regardless of whether an unhandled exception filter is present, + # the process terminates with the same error code that was raised. + + # Our strategy for this hook is to forward second-chance exceptions + # to the registered unhandled exception filter, if one exists. + + if first_chance: + raise QlErrorNotImplemented("ZwRaiseException is not implemented for first-chance exceptions.") + + if exception_ptr: + exception_code = ql.mem.read_ptr(exception_ptr, 4) # exception code is always DWORD + ql.log.debug(f"[ZwRaiseException] ExceptionCode: 0x{exception_code:08X}") + else: + ql.log.debug("[ZwRaiseException] ExceptionRecord is NULL") + + ql.log.debug(f" ContextRecord: 0x{context_ptr:016X}") + ql.log.debug(f" FirstChance: {first_chance}") + + handle = ql.os.handle_manager.search("TopLevelExceptionHandler") + + if handle is None: + ql.log.debug(f'[ZwRaiseException] No top-level exception filter was found.') + ql.log.info(f'The process exited with code 0x{exception_code:08X}.') + + ql.os.exit_code = exception_code + + ql.emu_stop() + + ret_addr = ql.stack_read(0) + + exception_filter = handle.obj + + # allocate some memory for the EXCEPTION_POINTERS struct + epointers_struct = structs.make_exception_pointers(ql.arch.bits) + exception_pointers_ptr = ql.os.heap.alloc(epointers_struct.sizeof()) + + with epointers_struct.ref(ql.mem, exception_pointers_ptr) as epointers_obj: + epointers_obj.ExceptionRecord = exception_ptr + epointers_obj.ContextRecord = context_ptr + + exception_filter = handle.obj + ql.log.debug(f'[ZwRaiseException] Resuming execution at the top-level exception filter at 0x{exception_filter:08X}.') + + # Hack: We are going to fake that the caller of ZwRaiseException + # actually called the unhandled exception filter instead. + + # We will create a hook which will be triggered when the unhandled + # exception filter returns, so that we may terminate execution. + def __post_exception_filter(ql: Qiling): + # Free the exception pointers struct we allocated earlier. + # Might not be needed, since we are going to terminate the process + # soon, but we might as well free it. + ql.os.heap.free(exception_pointers_ptr) + + ql.log.debug(f'[ZwRaiseException] Returned from unhandled exception filter at 0x{exception_filter:08X}.') + ql.log.info(f'The process exited with code 0x{exception_code:08X}.') + + ql.os.exit_code = exception_code + + ql.emu_stop() + + ql.hook_address(__post_exception_filter, ret_addr) + + exception_filter_args = [(POINTER, exception_pointers_ptr)] + + # Resume execution at the registered unhandled exception filter. + # If a program is using a custom unhandled exception filter as an anti-debugging + # trick, then the exception filter might not return. + ql.os.fcall.call_native(exception_filter, exception_filter_args, ret_addr) \ No newline at end of file diff --git a/qiling/os/windows/structs.py b/qiling/os/windows/structs.py index 56d685fc..ea35c74c 100644 --- a/qiling/os/windows/structs.py +++ b/qiling/os/windows/structs.py @@ -1545,3 +1545,19 @@ def make_win32_find_data(archbits: int, *, wide: bool): ) return WIN32_FIND_DATA + +# https://learn.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-exception_pointers +def make_exception_pointers(archbits: int): + """Generate an EXCEPTION_POINTERS structure class. + """ + + native_type = struct.get_native_type(archbits) + Struct = struct.get_aligned_struct(archbits) + + class EXCEPTION_POINTERS(Struct): + _fields_ = ( + ('ExceptionRecord', native_type), + ('ContextRecord', native_type) + ) + + return EXCEPTION_POINTERS \ No newline at end of file From 32e9fe345f5e62a5052782650358f7aa8cf7e2d7 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Sun, 30 Mar 2025 13:16:45 +0200 Subject: [PATCH 016/138] Fix unhandled exception filter not being called correctly --- qiling/os/windows/dlls/ntdll.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index 09b5d175..fb8fbe4d 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -632,7 +632,7 @@ def hook_LdrControlFlowGuardEnforced(ql: Qiling, address: int, params): 'ExceptionRecord': PVOID, 'ContextRecord': PVOID, 'FirstChance': BOOLEAN -}) +}, passthru=True) def hook_ZwRaiseException(ql: Qiling, address: int, params): exception_ptr = params['ExceptionRecord'] context_ptr = params['ContextRecord'] From 0ed88851b65580cc3a0b2d7d58dbc1cfbd0fb2d4 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Sun, 30 Mar 2025 17:43:48 +0200 Subject: [PATCH 017/138] Add 64-bit msvcp140 DLLs to dllscollector script --- examples/scripts/dllscollector.bat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/scripts/dllscollector.bat b/examples/scripts/dllscollector.bat index 2b85a83e..9433d1e2 100644 --- a/examples/scripts/dllscollector.bat +++ b/examples/scripts/dllscollector.bat @@ -131,6 +131,9 @@ CALL :collect_dll64 win32u.dll CALL :collect_dll64 winhttp.dll CALL :collect_dll64 wininet.dll CALL :collect_dll64 ws2_32.dll +CALL :collect_dll64 msvcp140.dll +CALL :collect_dll64 msvcp140_1.dll +CALL :collect_dll64 msvcp140_2.dll CALL :collect_dll64 downlevel\api-ms-win-crt-heap-l1-1-0.dll CALL :collect_dll64 downlevel\api-ms-win-crt-locale-l1-1-0.dll From 51fc085261beffb9d447cddc2382d0f6175b82b3 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Sun, 30 Mar 2025 17:45:49 +0200 Subject: [PATCH 018/138] Add abort hook --- qiling/os/windows/const.py | 1 + qiling/os/windows/dlls/msvcrt.py | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/qiling/os/windows/const.py b/qiling/os/windows/const.py index 8fe29023..6cf06d0e 100644 --- a/qiling/os/windows/const.py +++ b/qiling/os/windows/const.py @@ -38,6 +38,7 @@ STATUS_INVALID_HANDLE = 0xC0000008 STATUS_PROCEDURE_NOT_FOUND = 0xC000007A STATUS_DLL_NOT_FOUND = 0xC0000135 STATUS_PORT_NOT_SET = 0xC0000353 +STATUS_STACK_BUFFER_OVERRUN = 0xC0000409 STATUS_NO_YIELD_PERFORMED = 0x40000024 # ... diff --git a/qiling/os/windows/dlls/msvcrt.py b/qiling/os/windows/dlls/msvcrt.py index 9b9e1bba..b209b20d 100644 --- a/qiling/os/windows/dlls/msvcrt.py +++ b/qiling/os/windows/dlls/msvcrt.py @@ -10,7 +10,7 @@ from qiling import Qiling from qiling.exception import QlErrorNotImplemented from qiling.os.const import * from qiling.os.windows.fncc import * -from qiling.os.windows.const import LOCALE +from qiling.os.windows.const import * from qiling.os.windows.handle import Handle # void __set_app_type ( @@ -651,3 +651,18 @@ def hook__time64(ql: Qiling, address: int, params): ql.mem.write_ptr(dst, time_wasted, 8) return time_wasted + +# void abort( void ); +@winsdkapi(cc=CDECL, params={}) +def hook_abort(ql: Qiling, address: int, params): + # During testing, it was found that programs terminating abnormally + # via abort() terminated with exit code=STATUS_STACK_BUFFER_OVERRUN. + # According to Microsoft's devblog, this does not necessarily mean + # that a stack buffer overrun occurred. + # Rather, it can indicate abnormal program termination in a variety of + # situations, including abort(). + # https://devblogs.microsoft.com/oldnewthing/20190108-00/?p=100655 + # + ql.os.exit_code = STATUS_STACK_BUFFER_OVERRUN + + ql.emu_stop() \ No newline at end of file From 273b46b3674342c3aaa51ad09c0b0c0c307abe55 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Sun, 30 Mar 2025 18:03:10 +0200 Subject: [PATCH 019/138] Add 32-bit msvcp140 DLLs to dllscollector script --- examples/scripts/dllscollector.bat | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/scripts/dllscollector.bat b/examples/scripts/dllscollector.bat index 9433d1e2..b0707bba 100644 --- a/examples/scripts/dllscollector.bat +++ b/examples/scripts/dllscollector.bat @@ -94,6 +94,9 @@ CALL :collect_dll32 wininet.dll CALL :collect_dll32 winmm.dll CALL :collect_dll32 ws2_32.dll CALL :collect_dll32 wsock32.dll +CALL :collect_dll32 msvcp140.dll +CALL :collect_dll32 msvcp140_1.dll +CALL :collect_dll32 msvcp140_2.dll CALL :collect_dll32 downlevel\api-ms-win-core-fibers-l1-1-1.dll CALL :collect_dll32 downlevel\api-ms-win-core-localization-l1-2-1.dll From cf23ef9f81f40dcdda9abac0b6d1930d671176d0 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Sun, 30 Mar 2025 19:18:50 +0200 Subject: [PATCH 020/138] Small changes in ZwRaiseException hook --- qiling/os/windows/dlls/ntdll.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index fb8fbe4d..86c14df0 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -638,6 +638,17 @@ def hook_ZwRaiseException(ql: Qiling, address: int, params): context_ptr = params['ContextRecord'] first_chance = params['FirstChance'] + # The native ZwRaiseException simply uses a syscall to start + # the kernel exception dispatcher. However, Windows syscalls + # are not really working in Qiling right now. + # For now, we just provide a workaround for second-chance + # exceptions to work. + # TODO: Get some kind of solution for kernel exception + # dispatching. This is also needed for first-chance exceptions + # to work properly on 32-bit Windows. + if first_chance: + raise QlErrorNotImplemented("ZwRaiseException is not implemented for first-chance exceptions.") + # In Windows, an unhandled exception triggers the # top-level unhandled exception filter, after which the process # is terminated and error reporting services are called. @@ -647,9 +658,6 @@ def hook_ZwRaiseException(ql: Qiling, address: int, params): # Our strategy for this hook is to forward second-chance exceptions # to the registered unhandled exception filter, if one exists. - if first_chance: - raise QlErrorNotImplemented("ZwRaiseException is not implemented for first-chance exceptions.") - if exception_ptr: exception_code = ql.mem.read_ptr(exception_ptr, 4) # exception code is always DWORD ql.log.debug(f"[ZwRaiseException] ExceptionCode: 0x{exception_code:08X}") @@ -668,6 +676,7 @@ def hook_ZwRaiseException(ql: Qiling, address: int, params): ql.os.exit_code = exception_code ql.emu_stop() + return ret_addr = ql.stack_read(0) From 2b4246684aaf5e88e354a55e1ad26dc01a342f4f Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Sun, 30 Mar 2025 19:34:11 +0200 Subject: [PATCH 021/138] Make some requested changes in ntdll hooks --- qiling/os/windows/dlls/ntdll.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index 86c14df0..eff4a924 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -72,10 +72,7 @@ def _QueryInformationProcess(ql: Qiling, address: int, params): elif flag == ProcessCookie: - hCurrentProcess = { - QL_ARCH.X86 : 0xFFFFFFFF, - QL_ARCH.X8664: 0xFFFFFFFFFFFFFFFF - }[ql.arch.type] + hCurrentProcess = (1 << ql.arch.bits) - 1 if handle != hCurrentProcess: # If a process attempts to query the cookie of another @@ -491,10 +488,7 @@ def hook_RtlPcToFileHeader(ql: Qiling, address: int, params): containing_image = ql.loader.find_containing_image(pc) - if containing_image: - base_addr = containing_image.base - else: - base_addr = 0 + base_addr = containing_image.base if containing_image else 0 ql.mem.write_ptr(base_of_image_ptr, base_addr) return base_addr @@ -521,7 +515,7 @@ def hook_RtlLookupFunctionEntry(ql: Qiling, address: int, params): # history_table_ptr = params["HistoryTable"] # This function should not be getting called on x86. - if ql.arch.type != QL_ARCH.X8664: + if ql.arch.type is QL_ARCH.X86: raise QlErrorNotImplemented("RtlLookupFunctionEntry is not implemented for x86") containing_image = ql.loader.find_containing_image(control_pc) From 7dd9fcdc74aba6ac8e190ba71799d46881600cf1 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Sun, 30 Mar 2025 21:27:53 +0200 Subject: [PATCH 022/138] Add C++ runtime and exception-related tests --- tests/test_windows_cpp_x86.py | 64 +++++++++++ tests/test_windows_cpp_x8664.py | 181 ++++++++++++++++++++++++++++++++ 2 files changed, 245 insertions(+) create mode 100644 tests/test_windows_cpp_x86.py create mode 100644 tests/test_windows_cpp_x8664.py diff --git a/tests/test_windows_cpp_x86.py b/tests/test_windows_cpp_x86.py new file mode 100644 index 00000000..ba45466e --- /dev/null +++ b/tests/test_windows_cpp_x86.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +import sys, unittest + +sys.path.append("..") +from qiling import Qiling +from qiling.const import QL_VERBOSE +from qiling.extensions import pipe + + +def good_bad_count(test_str: str, good_str="GOOD", bad_str="BAD"): + good_count = test_str.count(good_str) + bad_count = test_str.count(bad_str) + + return good_count, bad_count + + +class CppTests_x86(unittest.TestCase): + + def test_cpp_helloworld(self): + """ Test a basic C++ Hello World program which prints "Hello World!" + to the console using std::cout. + """ + ql = Qiling(["../examples/rootfs/x86_windows/bin/except/CppHelloWorld_x86.exe"], "../examples/rootfs/x86_windows/", verbose=QL_VERBOSE.DEFAULT) + + ql.os.stdout = pipe.SimpleStringBuffer() + + ql.run() + + conout = ql.os.stdout.read() + self.assertEqual(conout, b"Hello World!\x0d\x0a") + + del ql + + def test_cpp_types(self): + """ This program tests several C++ type-related runtime features. + - typeid + - dynamic_cast + - virtual methods + - virtual destructors + """ + ql = Qiling(["../examples/rootfs/x86_windows/bin/except/TestCppTypes_x86.exe"], "../examples/rootfs/x86_windows/", verbose=QL_VERBOSE.DEFAULT) + + ql.os.stdout = pipe.SimpleStringBuffer() + + ql.run() + + conout = ql.os.stdout.read().decode('utf-8') + good_count, bad_count = good_bad_count(conout) + + # the test program should print + # - 'GOOD' 12 times + # - 'BAD' 0 times + self.assertEqual(good_count, 12) + self.assertEqual(bad_count, 0) + + del ql + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/tests/test_windows_cpp_x8664.py b/tests/test_windows_cpp_x8664.py new file mode 100644 index 00000000..8f89a47e --- /dev/null +++ b/tests/test_windows_cpp_x8664.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +import sys, unittest + +sys.path.append("..") +from qiling import Qiling +from qiling.const import QL_VERBOSE +from qiling.extensions import pipe + + +def good_bad_count(test_str: str, good_str="GOOD", bad_str="BAD"): + good_count = test_str.count(good_str) + bad_count = test_str.count(bad_str) + + return good_count, bad_count + + +class CppTests_x8664(unittest.TestCase): + + def test_cpp_helloworld(self): + """ Test a basic C++ Hello World program which prints "Hello World!" + to the console using std::cout. + """ + ql = Qiling(["../examples/rootfs/x8664_windows/bin/except/CppHelloWorld.exe"], "../examples/rootfs/x8664_windows/", verbose=QL_VERBOSE.DEFAULT) + + ql.os.stdout = pipe.SimpleStringBuffer() + + ql.run() + + conout = ql.os.stdout.read() + self.assertEqual(conout, b"Hello World!\x0d\x0a") + + del ql + + def test_cpp_types(self): + """ This program tests several C++ type-related runtime features. + - typeid + - dynamic_cast + - virtual methods + - virtual destructors + """ + ql = Qiling(["../examples/rootfs/x8664_windows/bin/except/TestCppTypes.exe"], "../examples/rootfs/x8664_windows/", verbose=QL_VERBOSE.DEFAULT) + + ql.os.stdout = pipe.SimpleStringBuffer() + + ql.run() + + conout = ql.os.stdout.read().decode('utf-8') + good_count, bad_count = good_bad_count(conout) + + # the test program should print + # - 'GOOD' 12 times + # - 'BAD' 0 times + self.assertEqual(good_count, 12) + self.assertEqual(bad_count, 0) + + del ql + + def test_soft_seh(self): + """ Test software SEH. + This test program uses __try..__catch and calls RaiseException with + a custom code. If software SEH is functioning correctly, the program + should be able to invoke its __catch-block and continue execution after. + """ + ql = Qiling(["../examples/rootfs/x8664_windows/bin/except/TestSoftSEH.exe"], "../examples/rootfs/x8664_windows/", verbose=QL_VERBOSE.DEFAULT) + + ql.os.stdout = pipe.SimpleStringBuffer() + + ql.run() + + conout = ql.os.stdout.read().decode('utf-8') + good_count, bad_count = good_bad_count(conout) + + # the test program should print + # - 'GOOD' 4 times + # - 'BAD' 0 times + self.assertEqual(good_count, 4) + self.assertEqual(bad_count, 0) + + # If the exception handler was not invoked for some reason, + # the program may terminate abnormally with a non-zero exit + # code. + self.assertEqual(ql.os.exit_code, 0) + + del ql + + def test_soft_cppex(self): + """ Test software C++ exceptions. + This test program tests try..catch in various ways. If exception dispatching + and stack unwinding are functioning correctly, the program will run to completion. + - Simple try..catch + - Try..catch with throw data + - Nested try..catch with throw data + """ + ql = Qiling(["../examples/rootfs/x8664_windows/bin/except/TestCppEx.exe"], "../examples/rootfs/x8664_windows/", verbose=QL_VERBOSE.DEFAULT) + + ql.os.stdout = pipe.SimpleStringBuffer() + + ql.run() + + conout = ql.os.stdout.read().decode('utf-8') + good_count, bad_count = good_bad_count(conout, 'y', 'n') + + # the test program should print + # - 'y' 14 times + # - 'n' 0 times + self.assertEqual(good_count, 14) + self.assertEqual(bad_count, 0) + + # If the exception handler was not invoked for some reason, + # the program may terminate abnormally with a non-zero exit + # code. + self.assertEqual(ql.os.exit_code, 0) + + del ql + + def test_cppex_unhandled_filtered(self): + """ Test unhandled C++ exceptions. + This program registers its own unhandled exception filter via + SetUnhandledExceptionFilter, then throws an uncaught exception. + If unhandled exception filters are functioning correctly, + the program's custom exception filter will be reached, but + execution will NOT resume after the exception. + Instead, the program is expected to terminate abnormally + with status code 0xE06D7363 (C++ runtime exception). + """ + ql = Qiling(["../examples/rootfs/x8664_windows/bin/except/TestCppExUnhandled.exe"], "../examples/rootfs/x8664_windows/", verbose=QL_VERBOSE.DEFAULT) + + ql.os.stdout = pipe.SimpleStringBuffer() + + ql.run() + + conout = ql.os.stdout.read().decode('utf-8') + good_count, bad_count = good_bad_count(conout) + + # the test program should print + # - 'GOOD' 3 times + # - 'BAD' 0 times + self.assertEqual(good_count, 3) + self.assertEqual(bad_count, 0) + + # The program should have terminated abnormally + # with status code 0xE06D7363 (C++ runtime exception). + self.assertEqual(ql.os.exit_code, 0xE06D7363) + + del ql + + def test_cppex_unhandled_unfiltered(self): + """ Test unhandled C++ exceptions. + This program throws an uncaught C++ exception. + The program is expected to terminate abnormally + with status code 0xC0000409 (STATUS_STACK_BUFFER_OVERRUN). + """ + ql = Qiling(["../examples/rootfs/x8664_windows/bin/except/TestCppExUnhandled2.exe"], "../examples/rootfs/x8664_windows/", verbose=QL_VERBOSE.DEFAULT) + + ql.os.stdout = pipe.SimpleStringBuffer() + + ql.run() + + conout = ql.os.stdout.read().decode('utf-8') + good_count, bad_count = good_bad_count(conout) + + # the test program should print + # - 'GOOD' 1 time + # - 'BAD' 0 times + self.assertEqual(good_count, 1) + self.assertEqual(bad_count, 0) + + # The program is expected to terminate abnormally + # with status code 0xC0000409 (STATUS_STACK_BUFFER_OVERRUN) + # https://devblogs.microsoft.com/oldnewthing/20190108-00/?p=100655 + # + self.assertEqual(ql.os.exit_code, 0xC0000409) + + del ql + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 4ea1a0ada6584a4b4cb0a5840a4fa20ee33aac78 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Sun, 30 Mar 2025 21:36:09 +0200 Subject: [PATCH 023/138] Make requested change in GDTManager --- qiling/arch/x86_utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qiling/arch/x86_utils.py b/qiling/arch/x86_utils.py index dfd45623..1bc9f895 100644 --- a/qiling/arch/x86_utils.py +++ b/qiling/arch/x86_utils.py @@ -61,10 +61,7 @@ class GDTManager: # setup GDT by writing to GDTR ql.arch.regs.write(UC_X86_REG_GDTR, (0, base, limit, 0x0)) - if ql.arch.type == QL_ARCH.X8664: - self.is_long_mode = True - else: - self.is_long_mode = False + self.is_long_mode = ql.arch.type is QL_ARCH.X8664 self.array = GDTArray(ql.mem, base, num_entries) From b895748a312451de278499fd88b968ae668f7fbc Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Sun, 30 Mar 2025 21:56:38 +0200 Subject: [PATCH 024/138] Make requested changes in PE loader --- qiling/loader/pe.py | 47 +++++++++++++++------------------------------ 1 file changed, 16 insertions(+), 31 deletions(-) diff --git a/qiling/loader/pe.py b/qiling/loader/pe.py index 8c2924cd..6cbf3781 100644 --- a/qiling/loader/pe.py +++ b/qiling/loader/pe.py @@ -10,7 +10,8 @@ import pefile import pickle import secrets import ntpath -from typing import TYPE_CHECKING, Any, Dict, MutableMapping, NamedTuple, Optional, Mapping, Sequence, Tuple, Union +from collections import namedtuple +from typing import TYPE_CHECKING, Any, Dict, List, MutableMapping, NamedTuple, Optional, Mapping, Sequence, Tuple, Union from unicorn import UcError from unicorn.x86_const import UC_X86_REG_CR4, UC_X86_REG_CR8 @@ -29,15 +30,10 @@ if TYPE_CHECKING: from logging import Logger from qiling import Qiling -class ForwardedExport: - def __init__(self, - source_dll: str, source_ordinal: str, source_symbol: str, - target_dll: str, target_symbol: str): - self.source_dll = source_dll - self.source_ordinal = source_ordinal - self.source_symbol = source_symbol - self.target_dll = target_dll - self.target_symbol = target_symbol +ForwardedExport = namedtuple('ForwardedExport', [ + 'source_dll', 'source_ordinal', 'source_symbol', + 'target_dll', 'target_symbol' +]) class QlPeCacheEntry(NamedTuple): @@ -90,14 +86,14 @@ class Process: libcache: Optional[QlPeCache] # maps image base to RVA of its function table - function_table_lookup: MutableMapping[int, int] + function_table_lookup: Dict[int, int] # maps image base to its list of function table entries - function_tables: MutableMapping[int, list] + function_tables: MutableMapping[int, List] # List of exports which have been forwarded from # one DLL to another. - forwarded_exports: list[ForwardedExport] + forwarded_exports: List[ForwardedExport] def __init__(self, ql: Qiling): self.ql = ql @@ -128,13 +124,13 @@ class Process: def init_function_tables(self, pe: pefile.PE, image_base: int): """Parse function table data for the given PE file. - Only works for x64 images. + Only really relevant for non-x86 images. Args: pe: the PE image whose function data should be parsed image_base: the absolute address at which the image was loaded """ - if self.ql.arch.type == QL_ARCH.X8664: + if self.ql.arch.type is not QL_ARCH.X86: # Check if the PE file has an exception directory if hasattr(pe, 'DIRECTORY_ENTRY_EXCEPTION'): @@ -144,12 +140,9 @@ class Process: self.function_table_lookup[image_base] = exception_dir.VirtualAddress - runtime_function_list = [] + runtime_function_list = list(pe.DIRECTORY_ENTRY_EXCEPTION) - for _, exception_entry in enumerate(pe.DIRECTORY_ENTRY_EXCEPTION, start=1): - runtime_function_list.append(exception_entry) - - if self.function_tables.get(image_base) is None: + if image_base not in self.function_tables: self.function_tables[image_base] = [] self.function_tables[image_base].extend(runtime_function_list) @@ -175,17 +168,9 @@ class Process: # Initiate a search of the function table for a RUNTIME_FUNCTION # entry such that the provided PC falls within its start and end range. - for i, runtime_function in enumerate(function_table): - - # Begin and end addresses exist in the entry as RVAs, - # convert them to absolute addresses. - begin_addr = base_addr + runtime_function.struct.BeginAddress - end_addr = base_addr + runtime_function.struct.EndAddress - - if begin_addr <= control_pc < end_addr: - return i, runtime_function - - return None, None + return next(((i, rtfunc) for i, rtfunc in enumerate(function_table) + if rtfunc.struct.BeginAddress <= control_pc - base_addr < rtfunc.struct.EndAddress), + (None, None)) def resolve_forwarded_exports(self): while self.forwarded_exports: From c0daea100314c975c823a5314cd90a82357c9d84 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Mon, 31 Mar 2025 22:42:21 +0200 Subject: [PATCH 025/138] Refactor hooks in ntdll --- qiling/os/windows/dlls/ntdll.py | 75 +++++++++++++++++---------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index eff4a924..04b52abc 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -493,6 +493,36 @@ def hook_RtlPcToFileHeader(ql: Qiling, address: int, params): ql.mem.write_ptr(base_of_image_ptr, base_addr) return base_addr +def _FindImageBaseAndFunctionTable(ql: Qiling, control_pc: int, image_base_ptr: int): + """ + Helper function to locate a containing image for `control_pc` as well as its + function table, while writing the image base to `image_base_ptr` (if non-zero). + Returns: + (base_addr, function_table_addr) + if no image is found, otherwise + (0, 0) + """ + containing_image = ql.loader.find_containing_image(control_pc) + + if containing_image: + base_addr = containing_image.base + else: + base_addr = 0 + + # Write base address to the ImageBase pointer, if provided + if image_base_ptr != 0: + ql.mem.write_ptr(image_base_ptr, base_addr) + + # If we don’t have a valid base, abort now + if base_addr == 0: + return 0, 0 + + # Look up the function-table RVA and compute the absolute address + function_table_rva = ql.loader.function_table_lookup.get(base_addr) + function_table_addr = base_addr + function_table_rva if function_table_rva else 0 + + return base_addr, function_table_addr + # NTSYSAPI PRUNTIME_FUNCTION RtlLookupFunctionEntry( # [in] DWORD64 ControlPc, # [out] PDWORD64 ImageBase, @@ -518,22 +548,11 @@ def hook_RtlLookupFunctionEntry(ql: Qiling, address: int, params): if ql.arch.type is QL_ARCH.X86: raise QlErrorNotImplemented("RtlLookupFunctionEntry is not implemented for x86") - containing_image = ql.loader.find_containing_image(control_pc) - - if containing_image: - base_addr = containing_image.base - else: - base_addr = 0 + base_addr, function_table_addr = _FindImageBaseAndFunctionTable(ql, control_pc, image_base_ptr) + # If no function table was found, abort. + if function_table_addr == 0: return 0 - - # If we got a valid location to write the image base ptr, - # copy it there, and proceed. - if image_base_ptr != 0: - ql.mem.write_ptr(image_base_ptr, base_addr) - - # Get the base address of the function table. - function_table_addr = base_addr + ql.loader.function_table_lookup[base_addr] # Look up the RUNTIME_FUNCTION entry; we are interested in the index in the table # so that we can compute the address. @@ -564,35 +583,17 @@ def hook_RtlLookupFunctionTable(ql: Qiling, address: int, params): size_of_table_ptr = params["SizeOfTable"] # This function should not be getting called on x86. - if ql.arch.type != QL_ARCH.X8664: + if ql.arch.type is QL_ARCH.X86: raise QlErrorNotImplemented("RtlLookupFunctionTable is not implemented for x86") - containing_image = ql.loader.find_containing_image(control_pc) + base_addr, function_table_addr = _FindImageBaseAndFunctionTable(ql, control_pc, image_base_ptr) - if containing_image: - base_addr = containing_image.base - else: - base_addr = 0 + # If no function table was found, abort. + if function_table_addr == 0: + ql.mem.write_ptr(size_of_table_ptr, 0, 4) return 0 - # If we got a valid location to write the image base ptr, - # copy it there, and proceed. - if image_base_ptr != 0: - ql.mem.write_ptr(image_base_ptr, base_addr) - - # If image base was 0, we are not going to find a valid function - # table anyway, so just return. - if base_addr == 0: - return 0 - - # Look up the RVA of the function table. - function_table_rva = ql.loader.function_table_lookup[base_addr] - - # The caller is expecting a pointer, so convert the RVA - # to an absolute address. - function_table_addr = int(base_addr + function_table_rva) - # If a valid pointer for the size was provided, # we want to figure out the size of the table. if size_of_table_ptr != 0: From 8f92c73b4076b5c6cc5da64f3e53a966bae37fcf Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Mon, 31 Mar 2025 22:50:25 +0200 Subject: [PATCH 026/138] Add source for C++ and exception-related test programs --- examples/src/windows/except/CppHelloWorld.cpp | 11 +++ examples/src/windows/except/README | 3 + examples/src/windows/except/TestCppEx.cpp | 95 +++++++++++++++++++ .../src/windows/except/TestCppExUnhandled.cpp | 46 +++++++++ .../windows/except/TestCppExUnhandled2.cpp | 21 ++++ examples/src/windows/except/TestCppTypes.cpp | 93 ++++++++++++++++++ examples/src/windows/except/TestSoftSEH.cpp | 45 +++++++++ 7 files changed, 314 insertions(+) create mode 100644 examples/src/windows/except/CppHelloWorld.cpp create mode 100644 examples/src/windows/except/README create mode 100644 examples/src/windows/except/TestCppEx.cpp create mode 100644 examples/src/windows/except/TestCppExUnhandled.cpp create mode 100644 examples/src/windows/except/TestCppExUnhandled2.cpp create mode 100644 examples/src/windows/except/TestCppTypes.cpp create mode 100644 examples/src/windows/except/TestSoftSEH.cpp diff --git a/examples/src/windows/except/CppHelloWorld.cpp b/examples/src/windows/except/CppHelloWorld.cpp new file mode 100644 index 00000000..4b78ac15 --- /dev/null +++ b/examples/src/windows/except/CppHelloWorld.cpp @@ -0,0 +1,11 @@ +// This is the default Hello World program generated by Visual Studio 2022. + +#include + +int main() +{ + std::cout << "Hello World!\n"; + + return 0; +} + diff --git a/examples/src/windows/except/README b/examples/src/windows/except/README new file mode 100644 index 00000000..8dfda022 --- /dev/null +++ b/examples/src/windows/except/README @@ -0,0 +1,3 @@ +In this folder: Sources for programs intended to help test C++ features and software exceptions. + +Compile with MSVC (Visual Studio 2022) \ No newline at end of file diff --git a/examples/src/windows/except/TestCppEx.cpp b/examples/src/windows/except/TestCppEx.cpp new file mode 100644 index 00000000..bd6fa46e --- /dev/null +++ b/examples/src/windows/except/TestCppEx.cpp @@ -0,0 +1,95 @@ +#include +#include + +/* + * Test simple try..catch. + */ +void test1() +{ + std::cout << "y"; + + try { + std::cout << "y"; + throw (unsigned int)0x12345678; + std::cout << "n"; + } + catch(unsigned int n) { + n; + std::cout << "y"; + } + + std::cout << "y"; +} + +/* + * Test simple try..catch with throw. + */ +void test2() +{ + std::cout << "y"; + + try { + std::cout << "y"; + throw (unsigned int)0x12345679; + std::cout << "n"; + } + catch (unsigned int n) { + n; + if (n == 0x12345679) { + std::cout << "y"; + } + else { + std::cout << "n"; + } + } + + std::cout << "y"; +} + +/* + * Test nested try..catch with throw. + */ +void test3() +{ + std::cout << "y"; + + try { + std::cout << "y"; + + try { + std::cout << "y"; + throw (unsigned int)0x1234567A; + std::cout << "n"; + } + catch (unsigned int n) { + n; + if (n == 0x1234567A) { + std::cout << "y"; + } + else { + std::cout << "n"; + } + } + + std::cout << "y"; + } + catch (unsigned int n) { + n; + std::cout << "n"; + } + + std::cout << "y"; +} + +int main() +{ + /* + * For this program, all subtests successful will print: + * - 14 'y' + * - 0 'n' + */ + + test1(); + test2(); + test3(); +} diff --git a/examples/src/windows/except/TestCppExUnhandled.cpp b/examples/src/windows/except/TestCppExUnhandled.cpp new file mode 100644 index 00000000..0074d1f4 --- /dev/null +++ b/examples/src/windows/except/TestCppExUnhandled.cpp @@ -0,0 +1,46 @@ +#include +#include + +LONG WINAPI CustomExceptionFilter(EXCEPTION_POINTERS* ExceptionInfo) { + printf("Inside exception filter (GOOD)\n"); + DWORD exceptionCode = (DWORD)ExceptionInfo->ExceptionRecord->ExceptionCode; + printf("Exception Code: 0x%X\n", exceptionCode); + + if (exceptionCode == 0xE06D7363) { // code for C++ exception + printf("Exception code DOES match, GOOD\n"); + } + else { + printf("Exception code DOES NOT match, BAD\n"); + } + + printf("Exception Address: 0x%llx\n", (ULONGLONG)ExceptionInfo->ExceptionRecord->ExceptionAddress); + + printf("After printing exception: (GOOD)\n"); + + return EXCEPTION_EXECUTE_HANDLER; +} + +int main() { + /* + * For this program, all subtests successful will print: + * - 3 'GOOD' + * - 0 'BAD' + * + * It is expected that the program terminates abnormally + * with status code 0xE06D7363 (C++ exception) + */ + + // Set the custom top-level exception filter + SetUnhandledExceptionFilter(CustomExceptionFilter); + + // Throw an unhandled exception. + // It should be caught by our filter. + throw (unsigned int)5; + + // We should never reach this point, because the exception + // dispatcher should terminate the program after our unhandled + // exception filter is called. + printf("After exception filter (BAD)\n"); + + return 0; +} \ No newline at end of file diff --git a/examples/src/windows/except/TestCppExUnhandled2.cpp b/examples/src/windows/except/TestCppExUnhandled2.cpp new file mode 100644 index 00000000..600855cc --- /dev/null +++ b/examples/src/windows/except/TestCppExUnhandled2.cpp @@ -0,0 +1,21 @@ +#include +#include + +int main() +{ + /* + * For this program, all subtests successful will print: + * - 1 'GOOD' + * - 0 'BAD' + * + * It is expected that the program terminates abnormally + * with status code 0xC0000409 (stack buffer overrun/security + * check failure) + */ + + printf("Before throw (GOOD)\n"); + + throw (unsigned int)5; + + printf("After throw (BAD)\n"); +} diff --git a/examples/src/windows/except/TestCppTypes.cpp b/examples/src/windows/except/TestCppTypes.cpp new file mode 100644 index 00000000..42b8da21 --- /dev/null +++ b/examples/src/windows/except/TestCppTypes.cpp @@ -0,0 +1,93 @@ +#include + +struct TestStruct { + float q; +}; + +class TestClass { +public: + int x, y; + virtual ~TestClass() { + std::cout << "TestClass destructor, GOOD" << std::endl; + }; + void yyy() { + std::cout << "REALLY GOOD" << std::endl; + } +}; + +class Something { +public: + char z; + virtual ~Something() { + std::cout << "Something destructor, GOOD" << std::endl; + }; + virtual void zzz() { + std::cout << "BAD" << std::endl; + }; +}; + +class TestClass2 : public TestClass, public Something { +public: + int z; + virtual ~TestClass2() { + std::cout << "TestClass2 destructor, GOOD" << std::endl; + }; + virtual void zzz() { + std::cout << "GOOD" << std::endl; + }; +}; + +int main() +{ + /* + * For this program, all subtests successful will print: + * - 12 'GOOD' + * - 0 'BAD' + */ + + int x = 5; + TestClass p; + TestStruct s; + + std::cout << typeid(x).name() << std::endl; + if (strcmp(typeid(x).name(), "int") == 0) { + std::cout << "typeid(x) is int, GOOD" << std::endl; + } + else { + std::cout << "typeid(x) is NOT int, BAD" << std::endl; + } + + std::cout << typeid(p).name() << std::endl; + if (strcmp(typeid(p).name(), "class TestClass") == 0) { + std::cout << "typeid(p) is \"class TestClass\", GOOD" << std::endl; + } + else { + std::cout << "typeid(p) is NOT \"class TestClass\", BAD" << std::endl; + } + + std::cout << typeid(s).name() << std::endl; + if (strcmp(typeid(s).name(), "struct TestStruct") == 0) { + std::cout << "typeid(s) is \"struct TestStruct\", GOOD" << std::endl; + } + else { + std::cout << "typeid(s) is NOT \"struct TestStruct\", BAD" << std::endl; + } + + std::cout << "Reached virtual methods and dynamic_cast test. GOOD" << std::endl; + + TestClass2* kz = new TestClass2; + + Something* ks = static_cast(kz); + + ks->zzz(); + + TestClass* pk = dynamic_cast(ks); + + pk->yyy(); + + std::cout << "Reached virtual destructor test. GOOD" << std::endl; + + delete pk; + + std::cout << "Finished all tests. GOOD" << std::endl; +} diff --git a/examples/src/windows/except/TestSoftSEH.cpp b/examples/src/windows/except/TestSoftSEH.cpp new file mode 100644 index 00000000..2578b7aa --- /dev/null +++ b/examples/src/windows/except/TestSoftSEH.cpp @@ -0,0 +1,45 @@ +#include +#include + +void test1() { + __try { + printf("Inside __try block. (GOOD)\n"); + + RaiseException( + 0xE0000001, + 0, + 0, + nullptr + ); + + printf("After RaiseException. (BAD)\n"); + } + __except (EXCEPTION_EXECUTE_HANDLER) { + printf("In __except block. (GOOD)\n"); + + unsigned long excepCode = GetExceptionCode(); + + printf("Exception code=0x%x\n", excepCode); + + if (excepCode == 0xE0000001) { + printf("Exception code IS same, GOOD\n"); + } + else { + printf("Exception code DOES NOT MATCH, BAD\n"); + } + } + + printf("After __except block. (GOOD)\n"); +} + +int main() { + /* + * For this program, all subtests successful will print: + * - 4 'GOOD' + * - 0 'BAD' + */ + + test1(); + + return 0; +} From 462f68a749039c4d6987343e0477649b0e977b4c Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Mon, 31 Mar 2025 23:04:20 +0200 Subject: [PATCH 027/138] Add note to ZwRaiseException hook --- qiling/os/windows/dlls/ntdll.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index 04b52abc..9d84b5d9 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -713,4 +713,9 @@ def hook_ZwRaiseException(ql: Qiling, address: int, params): # Resume execution at the registered unhandled exception filter. # If a program is using a custom unhandled exception filter as an anti-debugging # trick, then the exception filter might not return. + + # TODO: This relies on the hook being marked 'passthru' so that Qiling + # doesn't rewind after it returns. However, this is not entirely intended + # behavior of passthru, so this is a bit of a hack. Maybe find some + # way to rewrite without passthru. ql.os.fcall.call_native(exception_filter, exception_filter_args, ret_addr) \ No newline at end of file From 15e3c009cbb936440a2c84bf3db86f7f8fbf6316 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Tue, 1 Apr 2025 22:53:11 +0200 Subject: [PATCH 028/138] Add hook for EtwNotificationRegister --- qiling/os/windows/dlls/ntdll.py | 39 ++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index 9d84b5d9..33056c74 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -718,4 +718,41 @@ def hook_ZwRaiseException(ql: Qiling, address: int, params): # doesn't rewind after it returns. However, this is not entirely intended # behavior of passthru, so this is a bit of a hack. Maybe find some # way to rewrite without passthru. - ql.os.fcall.call_native(exception_filter, exception_filter_args, ret_addr) \ No newline at end of file + ql.os.fcall.call_native(exception_filter, exception_filter_args, ret_addr) + +# NTSTATUS EtwNotificationRegister( +# LPCGUID ProviderGuid, +# ULONG Type, +# PVOID CallbackFunction, +# PVOID CallbackContext, +# PVOID* RegistrationHandle +# ); +@winsdkapi(cc=STDCALL, params={ + 'ProviderGuid': PVOID, + 'Type': DWORD, + 'CallbackFunction': PVOID, + 'CallbackContext': PVOID, + 'RegistrationHandle': PVOID +}) +def hook_EtwNotificationRegister(ql: Qiling, address: int, params): + reg_handle_ptr = params['RegistrationHandle'] + + # It is very important to have a hook for this function + # because it is called by some Windows DLLs (sechost.dll, + # advapi32.dll) during initialization when the global + # CRT lock is held. + # If a DllMain aborts here, then the global CRT lock is never + # freed and any attempt to lock the global CRT lock *anywhere* + # will crash us. + + # TODO: See if a more thorough implementation + # is needed for this function. + + # For now, just create a dummy handle, and return it. + handle = Handle() + ql.os.handle_manager.append(handle) + + if reg_handle_ptr: + ql.mem.write_ptr(reg_handle_ptr, handle.id) + + return STATUS_SUCCESS From 92cd2d41db3a962a1538144d3c987d4b7889b8b2 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Wed, 2 Apr 2025 00:09:25 +0200 Subject: [PATCH 029/138] Fix an issue with forwarded symbols, and improve readability --- qiling/loader/pe.py | 59 ++++++++++++++++++++++++++------------------- 1 file changed, 34 insertions(+), 25 deletions(-) diff --git a/qiling/loader/pe.py b/qiling/loader/pe.py index 6cbf3781..ee1ed8a8 100644 --- a/qiling/loader/pe.py +++ b/qiling/loader/pe.py @@ -182,38 +182,47 @@ class Process: target_dll = forwarded_export.target_dll target_symbol = forwarded_export.target_symbol + if not source_symbol: + # Some DLLs (shlwapi.dll) have a bunch of forwarded + # exports with ordinals but no symbols. + # These are really annoying to deal with, but they are + # used extremely rarely, so we will ignore them. + continue + target_iat = self.import_address_table.get(target_dll) - if target_iat: - # If we have an existing entry in the process IAT for the code - # this entry forwards to, then we will point the symbol there - # rather than the symbol string in the exporter's data section. - forward_ea = target_iat.get(target_symbol) + if not target_iat: + # If IAT was not found, it is probably a virtual library. + continue - if forward_ea: - self.import_address_table[source_dll][source_symbol] = forward_ea - self.import_address_table[source_dll][source_ordinal] = forward_ea + # If we have an existing entry in the process IAT for the code + # this entry forwards to, then we will point the symbol there + # rather than the symbol string in the exporter's data section. + forward_ea = target_iat.get(target_symbol) - # Register the new address as having the source symbol/ordinal. - # This way, hooks on forward source symbols will function - # correctly. + if not forward_ea: + self.ql.log.warning(f"Forwarding symbol {source_dll}.{source_symbol} to {target_dll}.{target_symbol}: Failed to resolve address") + continue - self.import_symbols[forward_ea] = { - 'name' : source_symbol, - 'ordinal' : source_ordinal, - 'dll' : source_dll.split('.')[0] - } + self.import_address_table[source_dll][source_symbol] = forward_ea + self.import_address_table[source_dll][source_ordinal] = forward_ea - # TODO: With the above code, hooks on functions which are - # forward targets may not work correctly. - # The most correct way to resolve this would be to add - # support for addresses to be associated with multiple symbols. + # Register the new address as having the source symbol/ordinal. + # This way, hooks on forward source symbols will function + # correctly. - self.ql.log.debug(f"Forwarding symbol {source_dll}.{source_symbol} to {target_dll}.{target_symbol}: Resolved symbol to ({forward_ea:#x})") - else: - self.ql.log.warning(f"Forwarding symbol {source_dll}.{source_symbol} to {target_dll}.{target_symbol}: Failed to resolve address") - else: - pass # If IAT was not found, it is probably a virtual library. + self.import_symbols[forward_ea] = { + 'name' : source_symbol, + 'ordinal' : source_ordinal, + 'dll' : source_dll.split('.')[0] + } + + # TODO: With the above code, hooks on functions which are + # forward targets may not work correctly. + # The most correct way to resolve this would be to add + # support for addresses to be associated with multiple symbols. + + self.ql.log.debug(f"Forwarding symbol {source_dll}.{source_symbol} to {target_dll}.{target_symbol}: Resolved symbol to ({forward_ea:#x})") def load_dll(self, name: str, is_driver: bool = False) -> int: dll_path, dll_name = self.__get_path_elements(name) From 6adafd598b4ee659cdfd37cb1a2ee8a08447f40a Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Wed, 2 Apr 2025 16:47:04 +0200 Subject: [PATCH 030/138] Unify heap API hooks, address debug CRT init issues --- qiling/os/windows/dlls/kernel32/heapapi.py | 55 +++++-- qiling/os/windows/dlls/msvcrt.py | 163 +++++++++------------ 2 files changed, 119 insertions(+), 99 deletions(-) diff --git a/qiling/os/windows/dlls/kernel32/heapapi.py b/qiling/os/windows/dlls/kernel32/heapapi.py index 6746c6d8..871a933f 100644 --- a/qiling/os/windows/dlls/kernel32/heapapi.py +++ b/qiling/os/windows/dlls/kernel32/heapapi.py @@ -49,6 +49,17 @@ def hook_HeapCreate(ql: Qiling, address: int, params): return ql.os.heap.alloc(dwInitialSize) +def _HeapAlloc(ql: Qiling, address: int, params): + dwFlags = params["dwFlags"] + dwBytes = params["dwBytes"] + + ptr = ql.os.heap.alloc(dwBytes) + + if ptr and (dwFlags & HEAP_ZERO_MEMORY): + __zero_mem(ql.mem, ptr, dwBytes) + + return ptr + # DECLSPEC_ALLOCATOR LPVOID HeapAlloc( # HANDLE hHeap, # DWORD dwFlags, @@ -60,15 +71,7 @@ def hook_HeapCreate(ql: Qiling, address: int, params): 'dwBytes' : SIZE_T }) def hook_HeapAlloc(ql: Qiling, address: int, params): - dwFlags = params["dwFlags"] - dwBytes = params["dwBytes"] - - ptr = ql.os.heap.alloc(dwBytes) - - if ptr and (dwFlags & HEAP_ZERO_MEMORY): - __zero_mem(ql.mem, ptr, dwBytes) - - return ptr + return _HeapAlloc(ql, address, params) # DECLSPEC_ALLOCATOR LPVOID HeapReAlloc( # HANDLE hHeap, @@ -86,6 +89,14 @@ def hook_HeapReAlloc(ql: Qiling, address: int, params): base = params["lpMem"] newSize = params["dwBytes"] + if not base: + return _HeapAlloc(ql, address, params) + + if newSize == 0: + ql.os.heap.free(base) + + return 0 + oldSize = ql.os.heap.size(base) oldData = bytes(ql.mem.read(base, oldSize)) @@ -150,3 +161,29 @@ def hook_HeapSetInformation(ql: Qiling, address: int, params): @winsdkapi(cc=STDCALL, params={}) def hook_GetProcessHeap(ql: Qiling, address: int, params): return ql.os.heap.start_address + +# BOOL HeapValidate( +# HANDLE hHeap, +# DWORD dwFlags, +# LPCVOID lpMem +# ); +@winsdkapi(cc=STDCALL, params={ + 'hHeap': PVOID, + 'dwFlags': DWORD, + 'lpMem': PVOID +}) +def hook_HeapValidate(ql: Qiling, address: int, params): + hHeap = params['hHeap'] + lpMem = params['lpMem'] + + if not hHeap: + return 0 + + # TODO: Maybe _find is a heap manager implementation + # detail, in which case we shouldn't rely on it. + chunk = ql.os.heap._find(lpMem) + + if not chunk: + return 0 + + return chunk.inuse diff --git a/qiling/os/windows/dlls/msvcrt.py b/qiling/os/windows/dlls/msvcrt.py index b209b20d..763e74bd 100644 --- a/qiling/os/windows/dlls/msvcrt.py +++ b/qiling/os/windows/dlls/msvcrt.py @@ -135,10 +135,9 @@ def hook__controlfp(ql: Qiling, address: int, params): # ); @winsdkapi(cc=CDECL, params={ 'func' : POINTER -}) +}, passthru=True) def hook_atexit(ql: Qiling, address: int, params): - ret = 0 - return ret + return # char*** __p__environ(void) @winsdkapi(cc=CDECL, params={}) @@ -211,20 +210,12 @@ def hook___p___argc(ql: Qiling, address: int, params): return ret # TODO: this one belongs to ucrtbase.dll -@winsdkapi(cc=CDECL, params={}) +@winsdkapi(cc=CDECL, params={}, passthru=True) def hook__get_initial_narrow_environment(ql: Qiling, address: int, params): - ret = 0 - - for i, (k, v) in enumerate(ql.env.items()): - entry = bytes(f'{k}={v}', 'ascii') + b'\x00' - p_entry = ql.os.heap.alloc(len(entry)) - - ql.mem.write(p_entry, entry) - - if i == 0: - ret = p_entry - - return ret + # If the native version of this function does not + # get to run, then debug versions of the CRT DLLs can fail + # their initialization. + return # int sprintf ( char * str, const char * format, ... ); @winsdkapi(cc=CDECL, params={ @@ -339,6 +330,18 @@ def __stdio_common_vsprintf(ql: Qiling, address: int, params, wstring: bool): def hook___stdio_common_vsprintf(ql: Qiling, address: int, params): return __stdio_common_vsprintf(ql, address, params, False) +@winsdkapi(cc=CDECL, params={ + '_Options' : PARAM_INT64, + '_Buffer' : POINTER, + '_BufferCount' : SIZE_T, + '_MaxCount' : SIZE_T, + '_Format' : STRING, + '_Locale' : DWORD, + '_ArgList' : POINTER +}) +def hook___stdio_common_vsnprintf(ql: Qiling, address: int, params): + return __stdio_common_vsprintf(ql, address, params, False) + @winsdkapi(cc=CDECL, params={ '_Options' : PARAM_INT64, '_Buffer' : POINTER, @@ -350,6 +353,18 @@ def hook___stdio_common_vsprintf(ql: Qiling, address: int, params): def hook___stdio_common_vswprintf(ql: Qiling, address: int, params): return __stdio_common_vsprintf(ql, address, params, True) +@winsdkapi(cc=CDECL, params={ + '_Options' : PARAM_INT64, + '_Buffer' : POINTER, + '_BufferCount' : SIZE_T, + '_MaxCount' : SIZE_T, + '_Format' : WSTRING, + '_Locale' : DWORD, + '_ArgList' : POINTER +}) +def hook___stdio_common_vsnwprintf(ql: Qiling, address: int, params): + return __stdio_common_vsprintf(ql, address, params, True) + # all the "_s" versions are aliases to their non-"_s" counterparts @winsdkapi(cc=CDECL, params={ @@ -383,6 +398,18 @@ def hook___stdio_common_vfwprintf_s(ql: Qiling, address: int, params): def hook___stdio_common_vsprintf_s(ql: Qiling, address: int, params): return hook___stdio_common_vsprintf.__wrapped__(ql, address, params) +@winsdkapi(cc=CDECL, params={ + '_Options' : PARAM_INT64, + '_Buffer' : POINTER, + '_BufferCount' : SIZE_T, + '_MaxCount' : SIZE_T, + '_Format' : STRING, + '_Locale' : DWORD, + '_ArgList' : POINTER +}) +def hook___stdio_common_vsnprintf_s(ql: Qiling, address: int, params): + return hook___stdio_common_vsnprintf.__wrapped__(ql, address, params) + @winsdkapi(cc=CDECL, params={ '_Options' : PARAM_INT64, '_Buffer' : POINTER, @@ -394,6 +421,18 @@ def hook___stdio_common_vsprintf_s(ql: Qiling, address: int, params): def hook___stdio_common_vswprintf_s(ql: Qiling, address: int, params): return hook___stdio_common_vswprintf.__wrapped__(ql, address, params) +@winsdkapi(cc=CDECL, params={ + '_Options' : PARAM_INT64, + '_Buffer' : POINTER, + '_BufferCount' : SIZE_T, + '_MaxCount' : SIZE_T, + '_Format' : WSTRING, + '_Locale' : DWORD, + '_ArgList' : POINTER +}) +def hook___stdio_common_vsnwprintf_s(ql: Qiling, address: int, params): + return hook___stdio_common_vsnwprintf.__wrapped__(ql, address, params) + @winsdkapi(cc=CDECL, params={}) def hook___lconv_init(ql: Qiling, address: int, params): return 0 @@ -449,50 +488,18 @@ def hook_strncmp(ql: Qiling, address: int, params): return result -def __malloc(ql: Qiling, address: int, params): - size = params['size'] - - return ql.os.heap.alloc(size) - @winsdkapi(cc=CDECL, params={ 'size' : UINT -}) +}, passthru=True) def hook__malloc_base(ql: Qiling, address: int, params): - return __malloc(ql, address, params) + return # void* malloc（unsigned int size) @winsdkapi(cc=CDECL, params={ 'size' : UINT -}) +}, passthru=True) def hook_malloc(ql: Qiling, address: int, params): - size = params['size'] - - return ql.os.heap.alloc(size) - -def __realloc(ql: Qiling, address: int, params): - block = params['block'] - size = params['size'] - - if not block: - return ql.os.heap.alloc(size) - - if size == 0: - ql.os.heap.free(block) - return 0 - - oldSize = ql.os.heap.size(block) - oldData = bytes(ql.mem.read(block, size)) - ql.os.heap.free(block) - - if size < oldSize: - oldData = oldData[0:size] - - newBase = ql.os.heap.alloc(size) - - if newBase: - ql.mem.write(newBase, oldData) - - return newBase + return # void* __cdecl _realloc_base( # void* const block, @@ -501,27 +508,22 @@ def __realloc(ql: Qiling, address: int, params): @winsdkapi(cc=CDECL, params={ 'block' : POINTER, 'size' : UINT -}) +}, passthru=True) def hook__realloc_base(ql: Qiling, address: int, params): - return __realloc(ql, address, params) - -def __free(ql: Qiling, address: int, params): - address = params['address'] - - ql.os.heap.free(address) + return @winsdkapi(cc=CDECL, params={ 'address': POINTER -}) +}, passthru=True) def hook__free_base(ql: Qiling, address: int, params): - return __free(ql, address, params) + return # void* free（void *address) @winsdkapi(cc=CDECL, params={ 'address': POINTER -}) +}, passthru=True) def hook_free(ql: Qiling, address: int, params): - return __free(ql, address, params) + return # _onexit_t _onexit( # _onexit_t function @@ -546,32 +548,16 @@ def hook__onexit(ql: Qiling, address: int, params): 'dest' : POINTER, 'c' : INT, 'count' : SIZE_T -}) +}, passthru=True) def hook_memset(ql: Qiling, address: int, params): - dest = params["dest"] - c = params["c"] - count = params["count"] - - ql.mem.write(dest, bytes([c] * count)) - - return dest - -def __calloc(ql: Qiling, address: int, params): - num = params['num'] - size = params['size'] - - count = num * size - ret = ql.os.heap.alloc(count) - ql.mem.write(ret, bytes([0] * count)) - - return ret + return @winsdkapi(cc=CDECL, params={ 'num' : SIZE_T, 'size' : SIZE_T -}) +}, passthru=True) def hook__calloc_base(ql: Qiling, address: int, params): - return __calloc(ql, address, params) + return # void *calloc( # size_t num, @@ -580,9 +566,9 @@ def hook__calloc_base(ql: Qiling, address: int, params): @winsdkapi(cc=CDECL, params={ 'num' : SIZE_T, 'size' : SIZE_T -}) +}, passthru=True) def hook_calloc(ql: Qiling, address: int, params): - return __calloc(ql, address, params) + return # void * memmove( # void *dest, @@ -593,12 +579,9 @@ def hook_calloc(ql: Qiling, address: int, params): 'dest' : POINTER, 'src' : POINTER, 'num' : SIZE_T -}) +}, passthru=True) def hook_memmove(ql: Qiling, address: int, params): - data = ql.mem.read(params['src'], params['num']) - ql.mem.write(params['dest'], bytes(data)) - - return params['dest'] + return # int _ismbblead( # unsigned int c From a3bccf5bc7d5816c60b6c628127320f2a7ec2560 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Wed, 2 Apr 2025 16:50:16 +0200 Subject: [PATCH 031/138] Fix buffer overrun issue in LCMapString implementation --- qiling/os/windows/dlls/kernel32/winnls.py | 25 ++++++++++++----------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/qiling/os/windows/dlls/kernel32/winnls.py b/qiling/os/windows/dlls/kernel32/winnls.py index 296fe999..7acebefe 100644 --- a/qiling/os/windows/dlls/kernel32/winnls.py +++ b/qiling/os/windows/dlls/kernel32/winnls.py @@ -80,18 +80,19 @@ def hook_IsValidCodePage(ql: Qiling, address: int, params): return 1 def __LCMapString(ql: Qiling, address: int, params, wstring: bool): - lpSrcStr: str = params["lpSrcStr"] + lpSrcStr: int = params["lpSrcStr"] + cchSrc: int = params["cchSrc"] lpDestStr: int = params["lpDestStr"] cchDest: int = params["cchDest"] - enc = "utf-16le" if wstring else "utf-8" - res = f'{lpSrcStr}\x00' + char_size = 2 if wstring else 1 + byte_count = cchSrc * char_size if cchDest and lpDestStr: - # TODO maybe do some other check, for now is working - ql.mem.write(lpDestStr, res.encode(enc)) + source_bytes = ql.mem.read(lpSrcStr, byte_count) + ql.mem.write(lpDestStr, bytes(source_bytes)) - return len(res) + return cchSrc # int LCMapStringW( # LCID Locale, @@ -104,9 +105,9 @@ def __LCMapString(ql: Qiling, address: int, params, wstring: bool): @winsdkapi(cc=STDCALL, params={ 'Locale' : LCID, 'dwMapFlags' : DWORD, - 'lpSrcStr' : LPCWSTR, + 'lpSrcStr' : POINTER, 'cchSrc' : INT, - 'lpDestStr' : LPWSTR, + 'lpDestStr' : POINTER, 'cchDest' : INT }) def hook_LCMapStringW(ql: Qiling, address: int, params): @@ -123,9 +124,9 @@ def hook_LCMapStringW(ql: Qiling, address: int, params): @winsdkapi(cc=STDCALL, params={ 'Locale' : LCID, 'dwMapFlags' : DWORD, - 'lpSrcStr' : LPCSTR, + 'lpSrcStr' : POINTER, 'cchSrc' : INT, - 'lpDestStr' : LPSTR, + 'lpDestStr' : POINTER, 'cchDest' : INT }) def hook_LCMapStringA(ql: Qiling, address: int, params): @@ -145,9 +146,9 @@ def hook_LCMapStringA(ql: Qiling, address: int, params): @winsdkapi(cc=STDCALL, params={ 'lpLocaleName' : LPCWSTR, 'dwMapFlags' : DWORD, - 'lpSrcStr' : LPCWSTR, + 'lpSrcStr' : POINTER, 'cchSrc' : INT, - 'lpDestStr' : LPWSTR, + 'lpDestStr' : POINTER, 'cchDest' : INT, 'lpVersionInformation' : LPNLSVERSIONINFO, 'lpReserved' : LPVOID, From fe51b28ea85e4823a648e546d313b561b4ca7aaf Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Wed, 2 Apr 2025 16:55:17 +0200 Subject: [PATCH 032/138] Make requested change in PE loader --- qiling/loader/pe.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/qiling/loader/pe.py b/qiling/loader/pe.py index ee1ed8a8..8ea5884b 100644 --- a/qiling/loader/pe.py +++ b/qiling/loader/pe.py @@ -30,10 +30,12 @@ if TYPE_CHECKING: from logging import Logger from qiling import Qiling -ForwardedExport = namedtuple('ForwardedExport', [ - 'source_dll', 'source_ordinal', 'source_symbol', - 'target_dll', 'target_symbol' -]) +class ForwardedExport(NamedTuple): + source_dll: str + source_ordinal: str + source_symbol: str + target_dll: str + target_symbol: str class QlPeCacheEntry(NamedTuple): From 68aae4d7211eaa64b3525bc22bb9e5b91812f64a Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Wed, 2 Apr 2025 17:06:31 +0200 Subject: [PATCH 033/138] Add __dllonexit hook --- qiling/os/windows/dlls/msvcrt.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/qiling/os/windows/dlls/msvcrt.py b/qiling/os/windows/dlls/msvcrt.py index 763e74bd..e0ad4beb 100644 --- a/qiling/os/windows/dlls/msvcrt.py +++ b/qiling/os/windows/dlls/msvcrt.py @@ -539,6 +539,27 @@ def hook__onexit(ql: Qiling, address: int, params): return addr +# _onexit_t __dllonexit( +# _onexit_t func, +# _PVFV ** pbegin, +# _PVFV ** pend +# ); +@winsdkapi(cc=STDCALL, params={ + 'function': POINTER, + 'pbegin': POINTER, + 'pend': POINTER +}) +def hook___dllonexit(ql: Qiling, address: int, params): + function = params['function'] + + if function: + addr = ql.os.heap.alloc(ql.arch.pointersize) + ql.mem.write_ptr(addr, function) + + return addr + + return 0 + # void *memset( # void *dest, # int c, From 4bd74596bbcc00a3ee2a1e882e35ae9d165ca68e Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Wed, 2 Apr 2025 17:34:10 +0200 Subject: [PATCH 034/138] Add passthru exception-related hooks --- qiling/os/windows/dlls/ntdll.py | 142 ++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/qiling/os/windows/dlls/ntdll.py b/qiling/os/windows/dlls/ntdll.py index 33056c74..16956f73 100644 --- a/qiling/os/windows/dlls/ntdll.py +++ b/qiling/os/windows/dlls/ntdll.py @@ -756,3 +756,145 @@ def hook_EtwNotificationRegister(ql: Qiling, address: int, params): ql.mem.write_ptr(reg_handle_ptr, handle.id) return STATUS_SUCCESS + +# NTSYSAPI +# VOID RtlRaiseException( +# PEXCEPTION_RECORD ExceptionRecord +# ); +@winsdkapi(cc=STDCALL, params={ + 'ExceptionRecord': PVOID +}, passthru=True) +def hook_RtlRaiseException(ql: Qiling, address: int, params): + return + +# NTSYSAPI +# PRUNTIME_FUNCTION RtlVirtualUnwind( +# DWORD HandlerType, +# DWORD64 ImageBase, +# DWORD64 ControlPc, +# PRUNTIME_FUNCTION FunctionEntry, +# PCONTEXT ContextRecord, +# PVOID* HandlerData, +# PDWORD64 EstablisherFrame, +# PKNONVOLATILE_CONTEXT_POINTERS ContextPointers +# ); +@winsdkapi(cc=STDCALL, params={ + 'HandlerType': DWORD, + 'ImageBase': PVOID, + 'ControlPc': PVOID, + 'FunctionEntry': PVOID, + 'ContextRecord': PVOID, + 'HandlerData': PVOID, + 'EstablisherFrame': PVOID, + 'ContextPointers': PVOID +}, passthru=True) +def hook_RtlVirtualUnwind(ql: Qiling, address: int, params): + return + +# NTSYSAPI +# VOID RtlUnwindEx( +# PVOID TargetFrame, +# PVOID TargetIp, +# PEXCEPTION_RECORD ExceptionRecord, +# PVOID ReturnValue, +# PCONTEXT OriginalContext, +# PUNWIND_HISTORY_TABLE HistoryTable +# ); +@winsdkapi(cc=STDCALL, params={ + 'TargetFrame': PVOID, + 'TargetIp': PVOID, + 'ExceptionRecord': PVOID, + 'ReturnValue': PVOID, + 'OriginalContext': PVOID, + 'HistoryTable': PVOID +}, passthru=True) +def hook_RtlUnwindEx(ql: Qiling, address: int, params): + return + +# NTSYSAPI +# BOOLEAN RtlDispatchException( +# PEXCEPTION_RECORD ExceptionRecord, +# PCONTEXT ContextRecord +# ); +@winsdkapi(cc=STDCALL, params={ + 'ExceptionRecord': PVOID, + 'ContextRecord': PVOID +}, passthru=True) +def hook_RtlDispatchException(ql: Qiling, address: int, params): + return + +# NTSYSAPI +# VOID RtlRestoreContext( +# PCONTEXT ContextRecord, +# PEXCEPTION_RECORD ExceptionRecord +# ); +@winsdkapi(cc=CDECL, params={ + 'ContextRecord': PVOID, + 'ExceptionRecord': PVOID +}, passthru=True) +def hook_RtlRestoreContext(ql: Qiling, address: int, params): + return + +# NTSYSAPI +# VOID RtlCaptureContext( +# PCONTEXT ContextRecord +# ); +@winsdkapi(cc=STDCALL, params={ + 'ContextRecord': PVOID +}, passthru=True) +def hook_RtlCaptureContext(ql: Qiling, address: int, params): + return + +# NTSYSAPI +# VOID RtlCaptureContext2( +# PCONTEXT ContextRecord, +# ULONG Flags +# ); +@winsdkapi(cc=STDCALL, params={ + 'ContextRecord': PVOID, + 'Flags': DWORD +}, passthru=True) +def hook_RtlCaptureContext2(ql: Qiling, address: int, params): + return + +# NTSYSAPI +# NTSTATUS RtlInitializeExtendedContext2( +# USHORT Version, +# USHORT ContextFlags, +# ULONG ExtensionCount, +# ULONG *ExtensionSizes, +# ULONG BufferSize, +# PVOID Buffer, +# PCONTEXT Context, +# ULONG *LengthReturned +# ); +@winsdkapi(cc=STDCALL, params={ + 'Version': WORD, + 'ContextFlags': WORD, + 'ExtensionCount': DWORD, + 'ExtensionSizes': PVOID, + 'BufferSize': DWORD, + 'Buffer': PVOID, + 'Context': PVOID, + 'LengthReturned': PVOID +}, passthru=True) +def hook_RtlInitializeExtendedContext2(ql: Qiling, address: int, params): + return + +# NTSYSAPI +# NTSTATUS RtlGetExtendedContextLength2( +# USHORT Version, +# USHORT ContextFlags, +# ULONG ExtensionCount, +# ULONG *ExtensionSizes, +# PULONG Length +# ); +@winsdkapi(cc=STDCALL, params={ + 'Version': WORD, + 'ContextFlags': WORD, + 'ExtensionCount': DWORD, + 'ExtensionSizes': PVOID, + 'Length': PVOID +}, passthru=True) +def hook_RtlGetExtendedContextLength2(ql: Qiling, address: int, params): + return From f390b476516d91e20ebdffc87e4330d0f5580f61 Mon Sep 17 00:00:00 2001 From: Jacob Farnsworth <6502enthusiast@gmail.com> Date: Wed, 2 Apr 2025 18:39:53 +0200 Subject: [PATCH 035/138] Restore old RaiseException hook, add special case for x86 --- .../windows/dlls/kernel32/errhandlingapi.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/qiling/os/windows/dlls/kernel32/errhandlingapi.py b/qiling/os/windows/dlls/kernel32/errhandlingapi.py index 084ffabc..ece53aa8 100644 --- a/qiling/os/windows/dlls/kernel32/errhandlingapi.py +++ b/qiling/os/windows/dlls/kernel32/errhandlingapi.py @@ -115,3 +115,47 @@ def hook_RemoveVectoredExceptionHandler(ql: Qiling, address: int, params): hook.remove() return 0 + +# VOID RaiseException( +# DWORD dwExceptionCode, +# DWORD dwExceptionFlags, +# DWORD nNumberOfArguments, +# CONST ULONG_PTR* lpArguments +# ); +@winsdkapi(cc=STDCALL, params={ + 'dwExceptionCode': DWORD, + 'dwExceptionFlags': DWORD, + 'nNumberOfArguments': DWORD, + 'lpArguments': PVOID +}, passthru=True) +def hook_RaiseException(ql: Qiling, address: int, params): + # On x86_64, RaiseException will call RtlRaiseException, + # which calls the exception dispatcher directly. The native + # exception dispatching code mostly works correctly + # for software exceptions, so we shall simply continue + # through to the native dispatcher in this case. + if ql.arch.type is not QL_ARCH.X86: + return + + # On x86, the situation is different. RtlRaiseException + # will call ZwRaiseException, which uses a syscall. + # However, Qiling doesn't really support Windows syscalls + # right now. + # We will treat all exceptions as unhandled exceptions, + # which is better than nothing. + # TODO: Get kernel exception dispatching working properly, + # then first-chance software exceptions, SEH, and C++ + # exceptions can work on 32-bit Windows too. + nNumberOfArguments = params['nNumberOfArguments'] + lpArguments = params['lpArguments'] + + handle = ql.os.handle_manager.search("TopLevelExceptionHandler") + + if handle is None: + ql.log.warning(f'RaiseException: top level exception handler not found') + return + + exception_handler = handle.obj + args = [(PARAM_INTN, ql.mem.read_ptr(lpArguments + i * ql.arch.pointersize)) for i in range(nNumberOfArguments)] if lpArguments else [] + + ql.os.fcall.call_native(exception_handler, args, None) From 1ee9ccb8e15a4544312206a9d1c6107a8cf16697 Mon Sep 17 00:00:00 2001 From: libumem Date: Fri, 4 Apr 2025 02:59:35 +0000 Subject: [PATCH 036/138] Initial commit for epoll --- qiling/os/posix/const.py | 19 +++ qiling/os/posix/syscall/epoll.py | 232 +++++++++++++++++++++++++++++++ 2 files changed, 251 insertions(+) create mode 100644 qiling/os/posix/syscall/epoll.py diff --git a/qiling/os/posix/const.py b/qiling/os/posix/const.py index 6fcea41b..88110166 100644 --- a/qiling/os/posix/const.py +++ b/qiling/os/posix/const.py @@ -1064,3 +1064,22 @@ SHMAT = 21 SHMDT = 22 SHMGET = 23 SHMCTL = 24 + +# epoll syscall +EPOLL_OPS = {0x001: "EPOLL_CTL_ADD", 0x002: "EPOLL_CTL_DEL", 0x003: "EPOLL_CTL_MOD"} + +EPOLLIN = 0x001 +EPOLLPRI = 0x002 +EPOLLOUT = 0x004 +EPOLLRDNORM = 0x040 +EPOLLRDBAND = 0x080 +EPOLLWRNORM = 0x100 +EPOLLWRBAND = 0x200 +EPOLLMSG = 0x400 +EPOLLERR = 0x008 +EPOLLHUP = 0x010 +EPOLLRDHUP = 0x2000 +EPOLLEXCLUSIVE = 1 << 28 +EPOLLWAKEUP = 1 << 29 +EPOLLONESHOT = 1 << 30 +EPOLLET = 1 << 31 diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py new file mode 100644 index 00000000..cbfcdb1e --- /dev/null +++ b/qiling/os/posix/syscall/epoll.py @@ -0,0 +1,232 @@ +class QlEpollObj: + def __init__(self, epoll_object): + self._epoll_object = epoll_object + self._fds = {} # key: fd, value: eventmask + # keep track of which fds have what eventmasks, + # since this isn't directly supported in select.epoll + + @property + def get_epoll_instance(self): + return self._epoll_object + + @property + def get_eventmask(self, fd): + return self._fds[fd] + + @property + def get_fds(self): + if len(self._fds.keys()) == 0: + return [] + return list(self._fds.keys()) + + def set_eventmask(self, fd, newmask): + # the mask for an FD shouldn't ever be undefined + # as it is set whenever an FD is added for a QlEpollObj instance + newmask = self.get_eventmask() | newmask # or with new eventmask value + self._epoll_object.modify(fd, newmask) + + def monitor_fd(self, fd, eventmask): + self._epoll_object.register( + fd, eventmask + ) # tell the epoll object to watch the fd arg, looking for events matching the eventmask + self._fds[fd] = eventmask + + def delist_fd(self, fd): + self._fds.pop(fd) + self._epoll_object.unregister(fd) + + def close(self): + self.get_epoll_instance.close() + + def is_present(self, fd): + if fd not in self.get_fds: + return 0 + return 1 + + +def check_epoll_depth(ql_fd_list, epolls_list, depth): + if depth == 6: + return 1 + new_epolls_list = [] + flag = 0 + for ent in list(epolls_list): + watched = ent.get_fds + for w in watched: + if isinstance(ql_fd_list[w], QlEpollObj): + flag = 1 + new_epolls_list.append(ql_fd_list[w]) + if flag: + check_epoll_depth(ql_fd_list, new_epolls_list, depth + 1) + return 0 + + +def ql_epoll_ctl(ql: qiling.Qiling, epfd: int, op: int, fd: int, event: POINTER): + # Basic sanity checks first + if event != 0: + ql_event = ql.unpack32(ql.mem.read(event, 4)) # events list is uint32_t + else: + ql_event = ( + 0 # event can be null, for example, when deleting a fd from interest list + ) + ql_op = "" + epoll_obj = -1 + try: + epoll_parent_obj = ql.os.fd[epfd] + epoll_obj = epoll_parent_obj.get_epoll_instance + except KeyError as k: + ql.log.debug("Unable to grab epoll object, something wrong with ql.os.fd!") + ql.log.debug(k) + return EINVAL + try: + ql_op = EPOLL_OPS[op] + except KeyError as k: + ql.log.debug("Warning, invalid epoll op detected") + ql.log.debug(k) + return EINVAL # not clear from man page, but to be safe don't support 'undefined' ops. + """ + Qiling doesn't check process capabilities right now, so this case isn't explicitly handled yet + EPOLLWAKEUP (since Linux 3.5) + If EPOLLONESHOT and EPOLLET are clear and the process has the CAP_BLOCK_SUSPEND capability + """ + + # Unclear if qiling supports a way to determine + # if the target file descriptor is a directory + # Check against PersistentQlFile is to ensure + # that polling stdin, stdout, stderr is supported + fd_obj = ql.os.fd[fd] + if isinstance(fd_obj, ql_file) and not isinstance( + fd_obj, PersistentQlFile + ): # EPERM The target file fd does not support epoll. This error can occur if fd refers to, for example, a regular file or a directory. + return EPERM + + if isinstance(ql.os.fd[fd], QlEpollObj) and (op & EPOLLEXCLUSIVE): + # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance + return EINVAL + + # Necessary to iterate over all possible qiling fds + # to determine if we have a chain of more than five epolls monitoring each other + # This may be removed in the future if the QlOsLinux class had a separate + # field for reserved for tracking epoll objects. + epolls_list = [] + for f in ql.os.fd: + if isinstance(f, QlEpollObj): + epolls_list.append(f) + level_check = check_epoll_depth(ql.os.fd, epolls_list, 1) + if level_check: # more than five detected + return ELOOP + if epoll_obj is None or fd_obj is None: + # epfd or fd is not a valid file descriptor. + return EBADF + if epfd == fd: + return EINVAL + if epoll_obj.fileno() == fd: + return ELOOP # ELOOP ...or a nesting depth of epoll instances greater than 5. + match ql_op: + case "EPOLL_CTL_ADD": + if epoll_parent_obj.is_present( + fd + ): # can't add an fd that's already being waited on + return EEXIST # op was EPOLL_CTL_ADD, and the supplied file descriptor fd is already registered with this epoll instance. + epoll_parent_obj.monitor_fd( + fd, ql_event + ) # add to list of fds to be monitored with per-fd eventmask + # register will actual epoll instance + # and add eventmask accordingly + case "EPOLL_CTL_DEL": + if not epoll_parent_obj.is_present( + fd + ): # op was EPOLL_CTL_MOD or EPOLL_CTL_DEL, and fd is not registered with this epoll instance. + return ENOENT + epoll_parent_obj.delist_fd(fd) # remove from fds list and do so in the + # underlying epoll instance + case "EPOLL_CTL_MOD": + if not epoll_parent_obj.is_present( + fd + ): # ENOENT op was EPOLL_CTL_MOD or EPOLL_CTL_DEL, and fd is not registered with this epoll instance + return ENOENT + # EINVAL op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE. + if op & EPOLLEXCLUSIVE and fd in epoll_obj.get_fds: + return EINVAL # EINVAL op was EPOLL_CTL_MOD and the EPOLLEXCLUSIVE flag has previously been applied to this epfd, fd pair. + epoll_parent_obj.set_eventmask(ql_event) + + return 0 + + +def ql_epoll_wait( + ql: qiling.Qiling, epfd: int, epoll_events: POINTER, maxevents: int, timeout: int +): + if maxevents <= 0: + return EINVAL + # default value is 0xffffffff, but + # this fails when passing to epoll.poll() + if timeout == 0xFFFFFFFF: + timeout = None + + try: + epoll_parent_obj = ql.os.fd[epfd] + epoll_obj = epoll_parent_obj.get_epoll_instance + if not isinstance(epoll_parent_obj, QlEpollObj): + return EINVAL + except KeyError: + ql.log.debug(f"FD {epfd} doesn't appear to be a valid epoll file descriptor") + return EBADF + try: + ql_event = ql.unpack(ql.mem.read(epoll_events, ql.arch.pointersize)) + except Exception: + ql.log.debug("Can't read from epoll_events pointer") + return EFAULT + ready_fds = list(epoll_obj.poll(timeout, maxevents)) + + # Each tuple in ready_fds consists of + # (file descriptor, eventmask) + # so we iterate through these to indicate which fds + # are ready and 'why' + ret_val = len(ready_fds) + for i in range(0, ret_val): + fd = ready_fds[i][0] + interest_mask = ready_fds[i][1] + if ( + interest_mask & EPOLLONESHOT + ): # no longer interested in this fd, so remove from list + epoll_parent_obj.delist_fd(fd) + + counter = ( + ql.arch.pointersize + 4 + ) * i # use ql.arch.pointersize to be compatible with 32-bit + data = ql.pack32(interest_mask) # uint32_t eventfds + data += ql.pack(fd) # need fd only, use pack() to handle endianness + size + ql.mem.write(epoll_events + counter, data) + return ret_val + + +""" +Use select.epoll for underlying implementation, +just as select.poll is used for emulating poll() +""" + + +def ql_epoll_create1(ql: qiling.Qiling, flags: int): + if flags != select.EPOLL_CLOEXEC and flags != 0: + return EINVAL + ret = select.epoll(sizehint=-1, flags=flags) + fd = ret.fileno() + ql_obj = QlEpollObj(ret) + ql.os.fd[fd] = ql_obj + return fd + + +""" +Almost identical to above, but can't simply wrap +because of the slightly different args and the different +syscall number +""" + + +def ql_epoll_create(ql: qiling.Qiling, size: int): + if size < 0: + return EINVAL + ret = select.epoll(sizehint=size, flags=0) + fd = ret.fileno() + ql_obj = QlEpollObj(ret) + ql.os.fd[fd] = ql_obj + return fd \ No newline at end of file From 18381f42a3d48682e0d89642657916e5f026411d Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Fri, 4 Apr 2025 03:07:09 +0000 Subject: [PATCH 037/138] Simple test case --- tests/test_elf.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/test_elf.py b/tests/test_elf.py index 7977e3f1..0af4c90d 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -770,6 +770,13 @@ class ELFTest(unittest.TestCase): del ql - + def test_elf_linux_x8664_epoll_simple(self): + # TODO: Get the example in rootfs + rootfs = "../examples/rootfs/x8664_linux" + argv = r"../examples/rootfs/x8664_linux/epoll-0".split() + ql = qiling.Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) + ql.os.stdin = pipe.SimpleInStream(0) + ql.os.stdin.write(b"stop\n") + ql.run() if __name__ == "__main__": unittest.main() From 1f12da52c40fc26648325eb1470a4273732cf470 Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Fri, 4 Apr 2025 03:10:22 +0000 Subject: [PATCH 038/138] Add del ql --- tests/test_elf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_elf.py b/tests/test_elf.py index 0af4c90d..f12f3e01 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -778,5 +778,6 @@ class ELFTest(unittest.TestCase): ql.os.stdin = pipe.SimpleInStream(0) ql.os.stdin.write(b"stop\n") ql.run() + del ql if __name__ == "__main__": unittest.main() From e208a34520da92a5e7e409fa4bf2ec791e0f2bcf Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Fri, 4 Apr 2025 03:13:05 +0000 Subject: [PATCH 039/138] Skeleton for graph --- tests/test_elf.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_elf.py b/tests/test_elf.py index f12f3e01..07b533a8 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -779,5 +779,9 @@ class ELFTest(unittest.TestCase): ql.os.stdin.write(b"stop\n") ql.run() del ql + + def test_elf_linux_x8664_epoll_server(self): + pass + del ql if __name__ == "__main__": unittest.main() From 2479b912bce2dd0a13c76411289e7944aa2b4440 Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Mon, 7 Apr 2025 21:03:00 +0000 Subject: [PATCH 040/138] Flesh out tests, clean up --- qiling/os/posix/const.py | 1 + qiling/os/posix/syscall/epoll.py | 40 +++++++++++++++++++++-------- tests/test_elf.py | 43 ++++++++++++++++++++++++++------ tests/test_elf_multithread.py | 2 -- 4 files changed, 67 insertions(+), 19 deletions(-) diff --git a/qiling/os/posix/const.py b/qiling/os/posix/const.py index 88110166..db1bdf69 100644 --- a/qiling/os/posix/const.py +++ b/qiling/os/posix/const.py @@ -1083,3 +1083,4 @@ EPOLLEXCLUSIVE = 1 << 28 EPOLLWAKEUP = 1 << 29 EPOLLONESHOT = 1 << 30 EPOLLET = 1 << 31 +EPOLL_CLOEXEC = 0x02000000 \ No newline at end of file diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index cbfcdb1e..54ff0e5a 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -1,3 +1,16 @@ +import qiling +from qiling.const import * +from qiling.os.posix.const import * +from qiling.os.const import * +from qiling.os.filestruct import ql_file +import select +from ctypes import * +from qiling.os import struct +import struct +from qiling.os.filestruct import PersistentQlFile +from qiling.extensions import pipe +import sys + class QlEpollObj: def __init__(self, epoll_object): self._epoll_object = epoll_object @@ -43,9 +56,14 @@ class QlEpollObj: return 0 return 1 - +''' +Recursively checks each epoll instance's 'watched' +fds for an instance of epoll being watched. +If a chain of over 5 levels is detected, return +1, which will return ELOOP in ql_epoll_wait +''' def check_epoll_depth(ql_fd_list, epolls_list, depth): - if depth == 6: + if depth == 7: return 1 new_epolls_list = [] flag = 0 @@ -59,7 +77,10 @@ def check_epoll_depth(ql_fd_list, epolls_list, depth): check_epoll_depth(ql_fd_list, new_epolls_list, depth + 1) return 0 - +''' +Modify an existing epoll +man 7 epoll for more details +''' def ql_epoll_ctl(ql: qiling.Qiling, epfd: int, op: int, fd: int, event: POINTER): # Basic sanity checks first if event != 0: @@ -152,6 +173,10 @@ def ql_epoll_ctl(ql: qiling.Qiling, epfd: int, op: int, fd: int, event: POINTER) return 0 +''' +Wait on an existing epoll for events specified +earlier. man 7 epoll_wait for more info +''' def ql_epoll_wait( ql: qiling.Qiling, epfd: int, epoll_events: POINTER, maxevents: int, timeout: int ): @@ -203,8 +228,6 @@ def ql_epoll_wait( Use select.epoll for underlying implementation, just as select.poll is used for emulating poll() """ - - def ql_epoll_create1(ql: qiling.Qiling, flags: int): if flags != select.EPOLL_CLOEXEC and flags != 0: return EINVAL @@ -217,11 +240,8 @@ def ql_epoll_create1(ql: qiling.Qiling, flags: int): """ Almost identical to above, but can't simply wrap -because of the slightly different args and the different -syscall number +because of the slightly different prototype """ - - def ql_epoll_create(ql: qiling.Qiling, size: int): if size < 0: return EINVAL @@ -229,4 +249,4 @@ def ql_epoll_create(ql: qiling.Qiling, size: int): fd = ret.fileno() ql_obj = QlEpollObj(ret) ql.os.fd[fd] = ql_obj - return fd \ No newline at end of file + return fd diff --git a/tests/test_elf.py b/tests/test_elf.py index 07b533a8..b3be53da 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -9,8 +9,9 @@ import random import os import io import re - +import socket import sys +import time sys.path.append("..") from typing import Any, Sequence @@ -771,17 +772,45 @@ class ELFTest(unittest.TestCase): del ql def test_elf_linux_x8664_epoll_simple(self): - # TODO: Get the example in rootfs + # TODO: Get the example in rootfs, see https://github.com/qilingframework/rootfs/pull/35 + # epoll-0 source: https://github.com/maxasm/epoll-c/blob/main/main.c rootfs = "../examples/rootfs/x8664_linux" - argv = r"../examples/rootfs/x8664_linux/epoll-0".split() + argv = r"../examples/rootfs/x8664_linux/bin/epoll-0".split() ql = qiling.Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) - ql.os.stdin = pipe.SimpleInStream(0) - ql.os.stdin.write(b"stop\n") + ql.os.stdin = pipe.InteractiveInStream(0) + ql.os.stdin.write(b'echo\n') + ql.os.stdin.write(b"stop\n") # signal to exit gracefully ql.run() + self.assertIn("echo", ql.os.stdout.read().decode("utf-8")) del ql - def test_elf_linux_x8664_epoll_server(self): - pass + # TODO: https://github.com/qilingframework/rootfs/pull/35 must be merged + # Source for onestraw server: https://github.com/onestraw/epoll-example + # with a slight change to exit after the first request + def client(): + time.sleep(3) # give time for the server to listen + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + dest = ("127.0.0.1", 8000) + try: + s.connect(dest) + except Exception as e: + ql.log.debug('test failed') + ql.log.debug(e) + + test = b"hello world" + s.send(test) + s.close() + # use threads here to test how the server + # handles the request + client_thread = threading.Thread(target=client, daemon=True) + client_thread.start() + rootfs = "../examples/rootfs/" + argv = r"../examples/rootfs/x8664_linux/bin/onestraw-server s".split() # s means 'server mode' + ql = qiling.Qiling(argv, rootfs, multithread=False, verbose=QL_VERBOSE.DEBUG) + ql.os.stdout = pipe.SimpleOutStream(1) # server prints data received to stdout + ql.filter = '^data:' + ql.run() + self.assertIn('hello world', ql.os.stdout.read().decode("utf-8")) del ql if __name__ == "__main__": unittest.main() diff --git a/tests/test_elf_multithread.py b/tests/test_elf_multithread.py index 9b60e8d1..3841a011 100644 --- a/tests/test_elf_multithread.py +++ b/tests/test_elf_multithread.py @@ -6,7 +6,6 @@ import http.client import platform import re -import socket import sys import os import threading @@ -636,6 +635,5 @@ class ELFTest(unittest.TestCase): feedback = response.read() self.assertEqual('httpd_test_successful', feedback.decode()) - if __name__ == "__main__": unittest.main() From 56dd77b6608698bfe54f4bde01981a40609c9532 Mon Sep 17 00:00:00 2001 From: rliebig Date: Thu, 17 Apr 2025 16:21:32 +0200 Subject: [PATCH 041/138] disable getauxval HWCAP value for ARM64, which indicates Atomics support, which is however not implemented in current Unicorn versions --- qiling/loader/elf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiling/loader/elf.py b/qiling/loader/elf.py index 81ea096c..a5a8ac6f 100644 --- a/qiling/loader/elf.py +++ b/qiling/loader/elf.py @@ -330,7 +330,7 @@ class QlLoaderELF(QlLoader): hwcap_values = { (QL_ARCH.ARM, QL_ENDIAN.EL, 32): 0x001fb8d7, (QL_ARCH.ARM, QL_ENDIAN.EB, 32): 0xd7b81f00, - (QL_ARCH.ARM64, QL_ENDIAN.EL, 64): 0x078bfbfd + (QL_ARCH.ARM64, QL_ENDIAN.EL, 64): 0x078bfafd } # determine hwcap value by arch properties; if not found default to 0 From 290f11671a3d1a1cf8f9eea120e8e2e44de71930 Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Fri, 25 Apr 2025 02:03:53 +0000 Subject: [PATCH 042/138] Begin fixing this PR --- examples/rootfs | 2 +- examples/src/linux/x8664_linux_epoll.c | 54 ++++++ examples/src/linux/x8664_linux_onestraw.c | 199 ++++++++++++++++++++++ qiling/os/posix/syscall/__init__.py | 1 + qiling/os/posix/syscall/epoll.py | 67 ++++---- tests/test_elf.py | 3 +- 6 files changed, 290 insertions(+), 36 deletions(-) create mode 100644 examples/src/linux/x8664_linux_epoll.c create mode 100644 examples/src/linux/x8664_linux_onestraw.c diff --git a/examples/rootfs b/examples/rootfs index f71f45fe..6d4d654f 160000 --- a/examples/rootfs +++ b/examples/rootfs @@ -1 +1 @@ -Subproject commit f71f45fe1a39d58d8b8cae717f55cebeb37f63c7 +Subproject commit 6d4d654fdc2892490d98c433eca3efa5c6d062c7 diff --git a/examples/src/linux/x8664_linux_epoll.c b/examples/src/linux/x8664_linux_epoll.c new file mode 100644 index 00000000..afd0c8a4 --- /dev/null +++ b/examples/src/linux/x8664_linux_epoll.c @@ -0,0 +1,54 @@ +#define MAX_EVENTS 5 +#define READ_SIZE 10 +#include // for fprintf() +#include // for close(), read() +#include // for epoll_create1(), epoll_ctl(), struct epoll_event +#include // for strncmp + +int main() +{ + //setvbuf(stdin, NULL, _IONBF, 0); + int running = 1, event_count, i; + size_t bytes_read; + char read_buffer[READ_SIZE + 1]; + struct epoll_event event, events[MAX_EVENTS]; + int epoll_fd = epoll_create1(0); + + if (epoll_fd == -1) { + fprintf(stderr, "Failed to create epoll file descriptor\n"); + return 1; + } + + event.events = EPOLLIN; + event.data.fd = 0; + + if(epoll_ctl(epoll_fd, EPOLL_CTL_ADD, 0, &event)) + { + fprintf(stderr, "Failed to add file descriptor to epoll\n"); + close(epoll_fd); + return 1; + } + + while (running) { + printf("\nPolling for input...\n"); + event_count = epoll_wait(epoll_fd, events, MAX_EVENTS, 30000); + printf("%d ready events\n", event_count); + for (i = 0; i < event_count; i++) { + printf("Reading file descriptor '%d' -- ", events[i].data.fd); + bytes_read = read(events[i].data.fd, read_buffer, READ_SIZE); + printf("%zd bytes read.\n", bytes_read); + read_buffer[bytes_read] = '\0'; + printf("Read '%s'\n", read_buffer); + + if(!strncmp(read_buffer, "stop\n", 5)) + running = 0; + } + } + + if (close(epoll_fd)) { + fprintf(stderr, "Failed to close epoll file descriptor\n"); + return 1; + } + + return 0; +} diff --git a/examples/src/linux/x8664_linux_onestraw.c b/examples/src/linux/x8664_linux_onestraw.c new file mode 100644 index 00000000..43a93df8 --- /dev/null +++ b/examples/src/linux/x8664_linux_onestraw.c @@ -0,0 +1,199 @@ +/* + * Attention: + * To keep things simple, do not handle socket/bind/listen/.../epoll_create/epoll_wait API error + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define DEFAULT_PORT 8000 +#define MAX_CONN 16 +#define MAX_EVENTS 32 +#define BUF_SIZE 16 +#define MAX_LINE 256 + +void server_run(); +void client_run(); + +int main(int argc, char *argv[]) +{ + int opt; + char role = 's'; + while ((opt = getopt(argc, argv, "cs")) != -1) { + switch (opt) { + case 'c': + role = 'c'; + break; + case 's': + break; + default: + printf("usage: %s [-cs]\n", argv[0]); + exit(1); + } + } + if (role == 's') { + server_run(); + } else { + client_run(); + } + return 0; +} + +/* + * register events of fd to epfd + */ +static void epoll_ctl_add(int epfd, int fd, uint32_t events) +{ + struct epoll_event ev; + ev.events = events; + ev.data.fd = fd; + if (epoll_ctl(epfd, EPOLL_CTL_ADD, fd, &ev) == -1) { + perror("epoll_ctl()\n"); + exit(1); + } +} + +static void set_sockaddr(struct sockaddr_in *addr) +{ + bzero((char *)addr, sizeof(struct sockaddr_in)); + addr->sin_family = AF_INET; + addr->sin_addr.s_addr = INADDR_ANY; + addr->sin_port = htons(DEFAULT_PORT); +} + +static int setnonblocking(int sockfd) +{ + if (fcntl(sockfd, F_SETFL, fcntl(sockfd, F_GETFL, 0) | O_NONBLOCK) == + -1) { + return -1; + } + return 0; +} + +/* + * epoll echo server + */ +void server_run() +{ + int i; + int n; + int epfd; + int nfds; + int listen_sock; + int conn_sock; + int socklen; + char buf[BUF_SIZE]; + struct sockaddr_in srv_addr; + struct sockaddr_in cli_addr; + struct epoll_event events[MAX_EVENTS]; + + listen_sock = socket(AF_INET, SOCK_STREAM, 0); + + set_sockaddr(&srv_addr); + bind(listen_sock, (struct sockaddr *)&srv_addr, sizeof(srv_addr)); + + setnonblocking(listen_sock); + listen(listen_sock, MAX_CONN); + + epfd = epoll_create(1); + epoll_ctl_add(epfd, listen_sock, EPOLLIN | EPOLLOUT | EPOLLET); + + socklen = sizeof(cli_addr); + int a; + for (a = 0; a < 2; a++) { + nfds = epoll_wait(epfd, events, MAX_EVENTS, -1); + for (i = 0; i < nfds; i++) { + if (events[i].data.fd == listen_sock) { + /* handle new connection */ + conn_sock = + accept(listen_sock, + (struct sockaddr *)&cli_addr, + &socklen); + + inet_ntop(AF_INET, (char *)&(cli_addr.sin_addr), + buf, sizeof(cli_addr)); + printf("[+] connected with %s:%d\n", buf, + ntohs(cli_addr.sin_port)); + + setnonblocking(conn_sock); + epoll_ctl_add(epfd, conn_sock, + EPOLLIN | EPOLLET | EPOLLRDHUP | + EPOLLHUP); + } else if (events[i].events & EPOLLIN) { + /* handle EPOLLIN event */ + for (;;) { + bzero(buf, sizeof(buf)); + n = read(events[i].data.fd, buf, + sizeof(buf)); + if (n <= 0 /* || errno == EAGAIN */ ) { + break; + } else { + printf("[+] data: %s\n", buf); + write(events[i].data.fd, buf, + strlen(buf)); + } + } + } else { + printf("[+] unexpected\n"); + } + /* check if the connection is closing */ + if (events[i].events & (EPOLLRDHUP | EPOLLHUP)) { + printf("[+] connection closed\n"); + epoll_ctl(epfd, EPOLL_CTL_DEL, + events[i].data.fd, NULL); + close(events[i].data.fd); + continue; + } + } + } +} + +/* + * test clinet + */ +void client_run() +{ + int n; + int c; + int sockfd; + char buf[MAX_LINE]; + struct sockaddr_in srv_addr; + + sockfd = socket(AF_INET, SOCK_STREAM, 0); + + set_sockaddr(&srv_addr); + + if (connect(sockfd, (struct sockaddr *)&srv_addr, sizeof(srv_addr)) < 0) { + perror("connect()"); + exit(1); + } + + for (;;) { + printf("input: "); + fgets(buf, sizeof(buf), stdin); + c = strlen(buf) - 1; + buf[c] = '\0'; + write(sockfd, buf, c + 1); + + bzero(buf, sizeof(buf)); + while (errno != EAGAIN + && (n = read(sockfd, buf, sizeof(buf))) > 0) { + printf("echo: %s\n", buf); + bzero(buf, sizeof(buf)); + + c -= n; + if (c <= 0) { + break; + } + } + } + close(sockfd); +} diff --git a/qiling/os/posix/syscall/__init__.py b/qiling/os/posix/syscall/__init__.py index 38b10e64..fec37e52 100644 --- a/qiling/os/posix/syscall/__init__.py +++ b/qiling/os/posix/syscall/__init__.py @@ -30,3 +30,4 @@ from .uio import * from .unistd import * from .utsname import * from .wait import * +from .epoll import * \ No newline at end of file diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 54ff0e5a..b67c4314 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -1,4 +1,4 @@ -import qiling +from qiling import * from qiling.const import * from qiling.os.posix.const import * from qiling.os.const import * @@ -60,7 +60,7 @@ class QlEpollObj: Recursively checks each epoll instance's 'watched' fds for an instance of epoll being watched. If a chain of over 5 levels is detected, return -1, which will return ELOOP in ql_epoll_wait +1, which will return ELOOP in ql_syscall_epoll_wait ''' def check_epoll_depth(ql_fd_list, epolls_list, depth): if depth == 7: @@ -81,7 +81,7 @@ def check_epoll_depth(ql_fd_list, epolls_list, depth): Modify an existing epoll man 7 epoll for more details ''' -def ql_epoll_ctl(ql: qiling.Qiling, epfd: int, op: int, fd: int, event: POINTER): +def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: POINTER): # Basic sanity checks first if event != 0: ql_event = ql.unpack32(ql.mem.read(event, 4)) # events list is uint32_t @@ -142,33 +142,32 @@ def ql_epoll_ctl(ql: qiling.Qiling, epfd: int, op: int, fd: int, event: POINTER) return EINVAL if epoll_obj.fileno() == fd: return ELOOP # ELOOP ...or a nesting depth of epoll instances greater than 5. - match ql_op: - case "EPOLL_CTL_ADD": - if epoll_parent_obj.is_present( - fd - ): # can't add an fd that's already being waited on - return EEXIST # op was EPOLL_CTL_ADD, and the supplied file descriptor fd is already registered with this epoll instance. - epoll_parent_obj.monitor_fd( - fd, ql_event - ) # add to list of fds to be monitored with per-fd eventmask - # register will actual epoll instance - # and add eventmask accordingly - case "EPOLL_CTL_DEL": - if not epoll_parent_obj.is_present( - fd - ): # op was EPOLL_CTL_MOD or EPOLL_CTL_DEL, and fd is not registered with this epoll instance. - return ENOENT - epoll_parent_obj.delist_fd(fd) # remove from fds list and do so in the - # underlying epoll instance - case "EPOLL_CTL_MOD": - if not epoll_parent_obj.is_present( - fd - ): # ENOENT op was EPOLL_CTL_MOD or EPOLL_CTL_DEL, and fd is not registered with this epoll instance - return ENOENT - # EINVAL op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE. - if op & EPOLLEXCLUSIVE and fd in epoll_obj.get_fds: - return EINVAL # EINVAL op was EPOLL_CTL_MOD and the EPOLLEXCLUSIVE flag has previously been applied to this epfd, fd pair. - epoll_parent_obj.set_eventmask(ql_event) + if ql_op == "EPOLL_CTL_ADD": + if epoll_parent_obj.is_present( + fd + ): # can't add an fd that's already being waited on + return EEXIST # op was EPOLL_CTL_ADD, and the supplied file descriptor fd is already registered with this epoll instance. + epoll_parent_obj.monitor_fd( + fd, ql_event + ) # add to list of fds to be monitored with per-fd eventmask + # register will actual epoll instance + # and add eventmask accordingly + elif ql_op == "EPOLL_CTL_DEL": + if not epoll_parent_obj.is_present( + fd + ): # op was EPOLL_CTL_MOD or EPOLL_CTL_DEL, and fd is not registered with this epoll instance. + return ENOENT + epoll_parent_obj.delist_fd(fd) # remove from fds list and do so in the + # underlying epoll instance + elif ql_op == "EPOLL_CTL_MOD": + if not epoll_parent_obj.is_present( + fd + ): # ENOENT op was EPOLL_CTL_MOD or EPOLL_CTL_DEL, and fd is not registered with this epoll instance + return ENOENT + # EINVAL op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE. + if op & EPOLLEXCLUSIVE and fd in epoll_obj.get_fds: + return EINVAL # EINVAL op was EPOLL_CTL_MOD and the EPOLLEXCLUSIVE flag has previously been applied to this epfd, fd pair. + epoll_parent_obj.set_eventmask(ql_event) return 0 @@ -177,8 +176,8 @@ def ql_epoll_ctl(ql: qiling.Qiling, epfd: int, op: int, fd: int, event: POINTER) Wait on an existing epoll for events specified earlier. man 7 epoll_wait for more info ''' -def ql_epoll_wait( - ql: qiling.Qiling, epfd: int, epoll_events: POINTER, maxevents: int, timeout: int +def ql_syscall_epoll_wait( + ql: Qiling, epfd: int, epoll_events: POINTER, maxevents: int, timeout: int ): if maxevents <= 0: return EINVAL @@ -228,7 +227,7 @@ def ql_epoll_wait( Use select.epoll for underlying implementation, just as select.poll is used for emulating poll() """ -def ql_epoll_create1(ql: qiling.Qiling, flags: int): +def ql_syscall_epoll_create1(ql: Qiling, flags: int): if flags != select.EPOLL_CLOEXEC and flags != 0: return EINVAL ret = select.epoll(sizehint=-1, flags=flags) @@ -242,7 +241,7 @@ def ql_epoll_create1(ql: qiling.Qiling, flags: int): Almost identical to above, but can't simply wrap because of the slightly different prototype """ -def ql_epoll_create(ql: qiling.Qiling, size: int): +def ql_syscall_epoll_create(ql: Qiling, size: int): if size < 0: return EINVAL ret = select.epoll(sizehint=size, flags=0) diff --git a/tests/test_elf.py b/tests/test_elf.py index b3be53da..ab9ffa6c 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -770,7 +770,7 @@ class ELFTest(unittest.TestCase): self.assertNotIn("root\n", ql.os.stdout.read().decode("utf-8")) del ql - + @unittest.skip('See PR') def test_elf_linux_x8664_epoll_simple(self): # TODO: Get the example in rootfs, see https://github.com/qilingframework/rootfs/pull/35 # epoll-0 source: https://github.com/maxasm/epoll-c/blob/main/main.c @@ -783,6 +783,7 @@ class ELFTest(unittest.TestCase): ql.run() self.assertIn("echo", ql.os.stdout.read().decode("utf-8")) del ql + @unittest.skip('See PR') def test_elf_linux_x8664_epoll_server(self): # TODO: https://github.com/qilingframework/rootfs/pull/35 must be merged # Source for onestraw server: https://github.com/onestraw/epoll-example From c6eee99422412d5034c85990f2b3a243f25d644a Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Fri, 25 Apr 2025 02:07:36 +0000 Subject: [PATCH 043/138] Add socket import back, was not supposed to be removed --- tests/test_elf_multithread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_elf_multithread.py b/tests/test_elf_multithread.py index 3841a011..8923efa1 100644 --- a/tests/test_elf_multithread.py +++ b/tests/test_elf_multithread.py @@ -11,7 +11,7 @@ import os import threading import time import unittest - +import socket from typing import List sys.path.append("..") From 033fb4b4bc79a2a86dfca84ecd4cd1b3119c36f3 Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Sat, 26 Apr 2025 01:50:44 +0000 Subject: [PATCH 044/138] Address PR comments --- qiling/os/posix/const.py | 16 +++++++---- qiling/os/posix/syscall/epoll.py | 48 ++++++++++++-------------------- tests/test_elf.py | 8 +----- 3 files changed, 29 insertions(+), 43 deletions(-) diff --git a/qiling/os/posix/const.py b/qiling/os/posix/const.py index db1bdf69..6739c03a 100644 --- a/qiling/os/posix/const.py +++ b/qiling/os/posix/const.py @@ -1065,8 +1065,12 @@ SHMDT = 22 SHMGET = 23 SHMCTL = 24 -# epoll syscall -EPOLL_OPS = {0x001: "EPOLL_CTL_ADD", 0x002: "EPOLL_CTL_DEL", 0x003: "EPOLL_CTL_MOD"} +# epoll syscall, see https://codebrowser.dev/glibc/glibc/sysdeps/unix/sysv/linux/sys/epoll.h.html +EPOLL_OPS = { +0x001: "EPOLL_CTL_ADD", +0x002: "EPOLL_CTL_DEL", +0x003: "EPOLL_CTL_MOD" +} EPOLLIN = 0x001 EPOLLPRI = 0x002 @@ -1079,8 +1083,8 @@ EPOLLMSG = 0x400 EPOLLERR = 0x008 EPOLLHUP = 0x010 EPOLLRDHUP = 0x2000 -EPOLLEXCLUSIVE = 1 << 28 -EPOLLWAKEUP = 1 << 29 -EPOLLONESHOT = 1 << 30 -EPOLLET = 1 << 31 +EPOLLEXCLUSIVE = 268435456 #1 << 28 +EPOLLWAKEUP = 536870912 #1 << 29 +EPOLLONESHOT = 1073741824 #1 << 30 +EPOLLET = 2147483648 #1 << 31 EPOLL_CLOEXEC = 0x02000000 \ No newline at end of file diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index b67c4314..1a36ed04 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -19,23 +19,21 @@ class QlEpollObj: # since this isn't directly supported in select.epoll @property - def get_epoll_instance(self): + def epoll_instance(self): return self._epoll_object @property - def get_eventmask(self, fd): + def eventmask(self, fd): return self._fds[fd] @property - def get_fds(self): - if len(self._fds.keys()) == 0: - return [] + def fds(self): return list(self._fds.keys()) - def set_eventmask(self, fd, newmask): + def eventmask(self, fd: int, newmask: int): # the mask for an FD shouldn't ever be undefined # as it is set whenever an FD is added for a QlEpollObj instance - newmask = self.get_eventmask() | newmask # or with new eventmask value + newmask = self.eventmask() | newmask # or with new eventmask value self._epoll_object.modify(fd, newmask) def monitor_fd(self, fd, eventmask): @@ -49,31 +47,28 @@ class QlEpollObj: self._epoll_object.unregister(fd) def close(self): - self.get_epoll_instance.close() + self.epoll_instance.close() - def is_present(self, fd): - if fd not in self.get_fds: - return 0 - return 1 + def is_present(self, fd: int) -> bool: + return fd in self.get_fds -''' +""" Recursively checks each epoll instance's 'watched' fds for an instance of epoll being watched. If a chain of over 5 levels is detected, return 1, which will return ELOOP in ql_syscall_epoll_wait -''' +""" def check_epoll_depth(ql_fd_list, epolls_list, depth): if depth == 7: return 1 new_epolls_list = [] flag = 0 for ent in list(epolls_list): - watched = ent.get_fds + watched = ent.fds for w in watched: if isinstance(ql_fd_list[w], QlEpollObj): - flag = 1 new_epolls_list.append(ql_fd_list[w]) - if flag: + if new_epolls_list: check_epoll_depth(ql_fd_list, new_epolls_list, depth + 1) return 0 @@ -83,17 +78,12 @@ man 7 epoll for more details ''' def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: POINTER): # Basic sanity checks first - if event != 0: - ql_event = ql.unpack32(ql.mem.read(event, 4)) # events list is uint32_t - else: - ql_event = ( - 0 # event can be null, for example, when deleting a fd from interest list - ) + ql_event = event and ql.mem.read_ptr(event, 4) ql_op = "" epoll_obj = -1 try: epoll_parent_obj = ql.os.fd[epfd] - epoll_obj = epoll_parent_obj.get_epoll_instance + epoll_obj = epoll_parent_obj.epoll_instance except KeyError as k: ql.log.debug("Unable to grab epoll object, something wrong with ql.os.fd!") ql.log.debug(k) @@ -101,8 +91,6 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: POINTER try: ql_op = EPOLL_OPS[op] except KeyError as k: - ql.log.debug("Warning, invalid epoll op detected") - ql.log.debug(k) return EINVAL # not clear from man page, but to be safe don't support 'undefined' ops. """ Qiling doesn't check process capabilities right now, so this case isn't explicitly handled yet @@ -165,9 +153,9 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: POINTER ): # ENOENT op was EPOLL_CTL_MOD or EPOLL_CTL_DEL, and fd is not registered with this epoll instance return ENOENT # EINVAL op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE. - if op & EPOLLEXCLUSIVE and fd in epoll_obj.get_fds: + if op & EPOLLEXCLUSIVE and fd in epoll_obj.fds: return EINVAL # EINVAL op was EPOLL_CTL_MOD and the EPOLLEXCLUSIVE flag has previously been applied to this epfd, fd pair. - epoll_parent_obj.set_eventmask(ql_event) + epoll_parent_obj.eventmask(ql_event) return 0 @@ -188,14 +176,14 @@ def ql_syscall_epoll_wait( try: epoll_parent_obj = ql.os.fd[epfd] - epoll_obj = epoll_parent_obj.get_epoll_instance + epoll_obj = epoll_parent_obj.epoll_instance if not isinstance(epoll_parent_obj, QlEpollObj): return EINVAL except KeyError: ql.log.debug(f"FD {epfd} doesn't appear to be a valid epoll file descriptor") return EBADF try: - ql_event = ql.unpack(ql.mem.read(epoll_events, ql.arch.pointersize)) + ql_event = ql.mem.read_ptr(epoll_events, ql.arch.pointersize) except Exception: ql.log.debug("Can't read from epoll_events pointer") return EFAULT diff --git a/tests/test_elf.py b/tests/test_elf.py index ab9ffa6c..dc2fc700 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -785,19 +785,13 @@ class ELFTest(unittest.TestCase): del ql @unittest.skip('See PR') def test_elf_linux_x8664_epoll_server(self): - # TODO: https://github.com/qilingframework/rootfs/pull/35 must be merged # Source for onestraw server: https://github.com/onestraw/epoll-example # with a slight change to exit after the first request def client(): time.sleep(3) # give time for the server to listen s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) dest = ("127.0.0.1", 8000) - try: - s.connect(dest) - except Exception as e: - ql.log.debug('test failed') - ql.log.debug(e) - + s.connect(dest) test = b"hello world" s.send(test) s.close() From fd32154ca174a181a6ee4b0d8000c7ffd36bf3aa Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Sat, 26 Apr 2025 03:38:56 +0000 Subject: [PATCH 045/138] Begin working on test issues --- examples/rootfs | 2 +- qiling/os/posix/syscall/epoll.py | 2 +- tests/test_elf.py | 15 ++++++++++----- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/examples/rootfs b/examples/rootfs index 6d4d654f..f71f45fe 160000 --- a/examples/rootfs +++ b/examples/rootfs @@ -1 +1 @@ -Subproject commit 6d4d654fdc2892490d98c433eca3efa5c6d062c7 +Subproject commit f71f45fe1a39d58d8b8cae717f55cebeb37f63c7 diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 1a36ed04..cdd1d3c3 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -50,7 +50,7 @@ class QlEpollObj: self.epoll_instance.close() def is_present(self, fd: int) -> bool: - return fd in self.get_fds + return fd in self.fds """ Recursively checks each epoll instance's 'watched' diff --git a/tests/test_elf.py b/tests/test_elf.py index dc2fc700..e93bdd0e 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -770,14 +770,19 @@ class ELFTest(unittest.TestCase): self.assertNotIn("root\n", ql.os.stdout.read().decode("utf-8")) del ql + + """ + This tests a sample binary that (e)polls on stdin + and echos back the output. Upon receiving 'stop', it + will exit. + """ @unittest.skip('See PR') def test_elf_linux_x8664_epoll_simple(self): - # TODO: Get the example in rootfs, see https://github.com/qilingframework/rootfs/pull/35 # epoll-0 source: https://github.com/maxasm/epoll-c/blob/main/main.c rootfs = "../examples/rootfs/x8664_linux" - argv = r"../examples/rootfs/x8664_linux/bin/epoll-0".split() - ql = qiling.Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) - ql.os.stdin = pipe.InteractiveInStream(0) + argv = r"../examples/rootfs/x8664_linux/bin/x8664_linux_epoll_0".split() + ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) + ql.os.stdin = pipe.SimpleBufferedStream() ql.os.stdin.write(b'echo\n') ql.os.stdin.write(b"stop\n") # signal to exit gracefully ql.run() @@ -801,7 +806,7 @@ class ELFTest(unittest.TestCase): client_thread.start() rootfs = "../examples/rootfs/" argv = r"../examples/rootfs/x8664_linux/bin/onestraw-server s".split() # s means 'server mode' - ql = qiling.Qiling(argv, rootfs, multithread=False, verbose=QL_VERBOSE.DEBUG) + ql = Qiling(argv, rootfs, multithread=False, verbose=QL_VERBOSE.DEBUG) ql.os.stdout = pipe.SimpleOutStream(1) # server prints data received to stdout ql.filter = '^data:' ql.run() From f2861b7f38d563830d6662de048c34b105f667ac Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Sat, 26 Apr 2025 16:36:05 +0000 Subject: [PATCH 046/138] Nominally working epoll server test --- qiling/os/posix/syscall/epoll.py | 21 +++++++++++++++++---- tests/test_elf.py | 26 ++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index cdd1d3c3..8a3d7ea2 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -11,6 +11,7 @@ from qiling.os.filestruct import PersistentQlFile from qiling.extensions import pipe import sys + class QlEpollObj: def __init__(self, epoll_object): self._epoll_object = epoll_object @@ -52,12 +53,15 @@ class QlEpollObj: def is_present(self, fd: int) -> bool: return fd in self.fds + """ Recursively checks each epoll instance's 'watched' fds for an instance of epoll being watched. If a chain of over 5 levels is detected, return 1, which will return ELOOP in ql_syscall_epoll_wait """ + + def check_epoll_depth(ql_fd_list, epolls_list, depth): if depth == 7: return 1 @@ -72,10 +76,13 @@ def check_epoll_depth(ql_fd_list, epolls_list, depth): check_epoll_depth(ql_fd_list, new_epolls_list, depth + 1) return 0 -''' + +""" Modify an existing epoll man 7 epoll for more details -''' +""" + + def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: POINTER): # Basic sanity checks first ql_event = event and ql.mem.read_ptr(event, 4) @@ -160,10 +167,12 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: POINTER return 0 -''' +""" Wait on an existing epoll for events specified earlier. man 7 epoll_wait for more info -''' +""" + + def ql_syscall_epoll_wait( ql: Qiling, epfd: int, epoll_events: POINTER, maxevents: int, timeout: int ): @@ -215,6 +224,8 @@ def ql_syscall_epoll_wait( Use select.epoll for underlying implementation, just as select.poll is used for emulating poll() """ + + def ql_syscall_epoll_create1(ql: Qiling, flags: int): if flags != select.EPOLL_CLOEXEC and flags != 0: return EINVAL @@ -229,6 +240,8 @@ def ql_syscall_epoll_create1(ql: Qiling, flags: int): Almost identical to above, but can't simply wrap because of the slightly different prototype """ + + def ql_syscall_epoll_create(ql: Qiling, size: int): if size < 0: return EINVAL diff --git a/tests/test_elf.py b/tests/test_elf.py index e93bdd0e..4673262b 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -12,6 +12,8 @@ import re import socket import sys import time +import threading +from ctypes import * sys.path.append("..") from typing import Any, Sequence @@ -788,10 +790,27 @@ class ELFTest(unittest.TestCase): ql.run() self.assertIn("echo", ql.os.stdout.read().decode("utf-8")) del ql - @unittest.skip('See PR') + """ + This tests a simple server that uses epoll + to wait for data, then prints it out. It has been + modified to exit after data has been received; instead + of a typical server operation that reads requests indefinitely. + + It listens on port 8000, and a separate thread is spawned in + order to test how the server handles a 'hello world' input. + The server prints out whatever it receives, so the assert + statement checks the input is present as expected. + """ + #@unittest.skip('See PR') def test_elf_linux_x8664_epoll_server(self): # Source for onestraw server: https://github.com/onestraw/epoll-example - # with a slight change to exit after the first request + + """ + FIXME: Without a hook for this syscall, this error fires: + TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType + """ + def hook_newfstatat(ql: Qiling, dirfd: int, pathname: POINTER, statbuf: POINTER, flags:int): + return 0 def client(): time.sleep(3) # give time for the server to listen s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) @@ -800,13 +819,12 @@ class ELFTest(unittest.TestCase): test = b"hello world" s.send(test) s.close() - # use threads here to test how the server - # handles the request client_thread = threading.Thread(target=client, daemon=True) client_thread.start() rootfs = "../examples/rootfs/" argv = r"../examples/rootfs/x8664_linux/bin/onestraw-server s".split() # s means 'server mode' ql = Qiling(argv, rootfs, multithread=False, verbose=QL_VERBOSE.DEBUG) + ql.os.set_syscall("newfstatat",hook_newfstatat, QL_INTERCEPT.CALL) ql.os.stdout = pipe.SimpleOutStream(1) # server prints data received to stdout ql.filter = '^data:' ql.run() From f99e07ac0bb24c77843b9271a1ab0cb28aca769c Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Sat, 26 Apr 2025 16:38:22 +0000 Subject: [PATCH 047/138] Type annotations for args --- qiling/os/posix/syscall/epoll.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 8a3d7ea2..48c3cbc5 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -24,7 +24,7 @@ class QlEpollObj: return self._epoll_object @property - def eventmask(self, fd): + def eventmask(self, fd: int): return self._fds[fd] @property @@ -37,13 +37,13 @@ class QlEpollObj: newmask = self.eventmask() | newmask # or with new eventmask value self._epoll_object.modify(fd, newmask) - def monitor_fd(self, fd, eventmask): + def monitor_fd(self, fd: int, eventmask: int) -> None: self._epoll_object.register( fd, eventmask ) # tell the epoll object to watch the fd arg, looking for events matching the eventmask self._fds[fd] = eventmask - def delist_fd(self, fd): + def delist_fd(self, fd: int) -> None: self._fds.pop(fd) self._epoll_object.unregister(fd) From 0022b8d2d4b0a7f55269f6ee6f409c1dc56c313f Mon Sep 17 00:00:00 2001 From: elicn Date: Mon, 28 Apr 2025 13:28:12 +0300 Subject: [PATCH 048/138] Refactor and fixes --- qiling/os/posix/const.py | 46 ++-- qiling/os/posix/syscall/epoll.py | 362 ++++++++++++++++--------------- 2 files changed, 206 insertions(+), 202 deletions(-) diff --git a/qiling/os/posix/const.py b/qiling/os/posix/const.py index 6739c03a..df4c5b58 100644 --- a/qiling/os/posix/const.py +++ b/qiling/os/posix/const.py @@ -1065,26 +1065,28 @@ SHMDT = 22 SHMGET = 23 SHMCTL = 24 -# epoll syscall, see https://codebrowser.dev/glibc/glibc/sysdeps/unix/sysv/linux/sys/epoll.h.html -EPOLL_OPS = { -0x001: "EPOLL_CTL_ADD", -0x002: "EPOLL_CTL_DEL", -0x003: "EPOLL_CTL_MOD" -} +# see: https://elixir.bootlin.com/linux/v5.19.17/source/include/uapi/linux/eventpoll.h +EPOLL_CTL_ADD = 1 +EPOLL_CTL_DEL = 2 +EPOLL_CTL_MOD = 3 -EPOLLIN = 0x001 -EPOLLPRI = 0x002 -EPOLLOUT = 0x004 -EPOLLRDNORM = 0x040 -EPOLLRDBAND = 0x080 -EPOLLWRNORM = 0x100 -EPOLLWRBAND = 0x200 -EPOLLMSG = 0x400 -EPOLLERR = 0x008 -EPOLLHUP = 0x010 -EPOLLRDHUP = 0x2000 -EPOLLEXCLUSIVE = 268435456 #1 << 28 -EPOLLWAKEUP = 536870912 #1 << 29 -EPOLLONESHOT = 1073741824 #1 << 30 -EPOLLET = 2147483648 #1 << 31 -EPOLL_CLOEXEC = 0x02000000 \ No newline at end of file +EPOLLIN = 0x00000001 +EPOLLPRI = 0x00000002 +EPOLLOUT = 0x00000004 +EPOLLERR = 0x00000008 +EPOLLHUP = 0x00000010 +EPOLLNVAL = 0x00000020 +EPOLLRDNORM = 0x00000040 +EPOLLRDBAND = 0x00000080 +EPOLLWRNORM = 0x00000100 +EPOLLWRBAND = 0x00000200 +EPOLLMSG = 0x00000400 +EPOLLRDHUP = 0x00002000 + +# TODO: should be aligned to emulated system's close-on-exec value +EPOLL_CLOEXEC = 0o2000000 + +EPOLLEXCLUSIVE = 0b1 << 28 +EPOLLWAKEUP = 0b1 << 29 +EPOLLONESHOT = 0b1 << 30 +EPOLLET = 0b1 << 31 diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 48c3cbc5..bd237879 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -1,252 +1,254 @@ +import select + +from typing import TYPE_CHECKING, Dict, List + from qiling import * from qiling.const import * from qiling.os.posix.const import * from qiling.os.const import * from qiling.os.filestruct import ql_file -import select -from ctypes import * -from qiling.os import struct -import struct from qiling.os.filestruct import PersistentQlFile -from qiling.extensions import pipe -import sys + + +if TYPE_CHECKING: + from qiling.os.posix.posix import QlFileDes class QlEpollObj: - def __init__(self, epoll_object): + def __init__(self, epoll_object: select.epoll): self._epoll_object = epoll_object - self._fds = {} # key: fd, value: eventmask + + # maps fd to eventmask # keep track of which fds have what eventmasks, # since this isn't directly supported in select.epoll + self._fds: Dict[int, int] = {} @property - def epoll_instance(self): - return self._epoll_object - - @property - def eventmask(self, fd: int): - return self._fds[fd] - - @property - def fds(self): + def fds(self) -> List[int]: return list(self._fds.keys()) - def eventmask(self, fd: int, newmask: int): + @property + def epoll_instance(self) -> select.epoll: + return self._epoll_object + + def get_eventmask(self, fd: int) -> int: + return self._fds[fd] + + def set_eventmask(self, fd: int, newmask: int): # the mask for an FD shouldn't ever be undefined # as it is set whenever an FD is added for a QlEpollObj instance - newmask = self.eventmask() | newmask # or with new eventmask value + + # elicn: don't we need to update self._fds[fd] with the new mask just like in monitor_fd? + + newmask = self.get_eventmask(fd) | newmask self._epoll_object.modify(fd, newmask) def monitor_fd(self, fd: int, eventmask: int) -> None: - self._epoll_object.register( - fd, eventmask - ) # tell the epoll object to watch the fd arg, looking for events matching the eventmask + # tell the epoll object to watch the fd arg, looking for events matching the eventmask + self._epoll_object.register(fd, eventmask) self._fds[fd] = eventmask def delist_fd(self, fd: int) -> None: self._fds.pop(fd) self._epoll_object.unregister(fd) - def close(self): + def close(self) -> None: self.epoll_instance.close() def is_present(self, fd: int) -> bool: return fd in self.fds -""" -Recursively checks each epoll instance's 'watched' -fds for an instance of epoll being watched. -If a chain of over 5 levels is detected, return -1, which will return ELOOP in ql_syscall_epoll_wait -""" +def check_epoll_depth(ql_fd_list: QlFileDes, epolls_list: List[QlEpollObj], depth: int = 0) -> None: + # Recursively checks each epoll instance's 'watched' fds for an instance of + # epoll being watched. If a chain of over 5 levels is detected, return 1, + # which will return ELOOP in ql_syscall_epoll_wait + if depth >= 5: + raise RecursionError -def check_epoll_depth(ql_fd_list, epolls_list, depth): - if depth == 7: - return 1 new_epolls_list = [] - flag = 0 - for ent in list(epolls_list): + + for ent in epolls_list: watched = ent.fds + for w in watched: - if isinstance(ql_fd_list[w], QlEpollObj): - new_epolls_list.append(ql_fd_list[w]) + obj = ql_fd_list[w] + + if isinstance(obj, QlEpollObj): + new_epolls_list.append(obj) + + # elicn: new_epolls_list is not cleared between loop iterations, rather it keeps + # aggregating items from previous iterations. is this what we want? + if new_epolls_list: check_epoll_depth(ql_fd_list, new_epolls_list, depth + 1) - return 0 -""" -Modify an existing epoll -man 7 epoll for more details -""" - - -def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: POINTER): - # Basic sanity checks first - ql_event = event and ql.mem.read_ptr(event, 4) - ql_op = "" - epoll_obj = -1 - try: - epoll_parent_obj = ql.os.fd[epfd] - epoll_obj = epoll_parent_obj.epoll_instance - except KeyError as k: - ql.log.debug("Unable to grab epoll object, something wrong with ql.os.fd!") - ql.log.debug(k) - return EINVAL - try: - ql_op = EPOLL_OPS[op] - except KeyError as k: - return EINVAL # not clear from man page, but to be safe don't support 'undefined' ops. +def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): + """Modify an existing epoll. """ - Qiling doesn't check process capabilities right now, so this case isn't explicitly handled yet - EPOLLWAKEUP (since Linux 3.5) - If EPOLLONESHOT and EPOLLET are clear and the process has the CAP_BLOCK_SUSPEND capability - """ - # Unclear if qiling supports a way to determine - # if the target file descriptor is a directory - # Check against PersistentQlFile is to ensure - # that polling stdin, stdout, stderr is supported - fd_obj = ql.os.fd[fd] - if isinstance(fd_obj, ql_file) and not isinstance( - fd_obj, PersistentQlFile - ): # EPERM The target file fd does not support epoll. This error can occur if fd refers to, for example, a regular file or a directory. - return EPERM + # not clear from man page, but to be safe don't support 'undefined' ops. + if op not in (EPOLL_CTL_ADD, EPOLL_CTL_DEL, EPOLL_CTL_MOD): + return -EINVAL - if isinstance(ql.os.fd[fd], QlEpollObj) and (op & EPOLLEXCLUSIVE): - # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance - return EINVAL - - # Necessary to iterate over all possible qiling fds - # to determine if we have a chain of more than five epolls monitoring each other - # This may be removed in the future if the QlOsLinux class had a separate - # field for reserved for tracking epoll objects. - epolls_list = [] - for f in ql.os.fd: - if isinstance(f, QlEpollObj): - epolls_list.append(f) - level_check = check_epoll_depth(ql.os.fd, epolls_list, 1) - if level_check: # more than five detected - return ELOOP - if epoll_obj is None or fd_obj is None: - # epfd or fd is not a valid file descriptor. - return EBADF if epfd == fd: - return EINVAL + return -EINVAL + + if epfd not in range(NR_OPEN): + return -EBADF + + epoll_parent_obj = ql.os.fd[epfd] + + if not isinstance(epoll_parent_obj, QlEpollObj): + return -EINVAL + + epoll_obj = epoll_parent_obj.epoll_instance + + if epoll_obj is None: + return -EBADF + if epoll_obj.fileno() == fd: - return ELOOP # ELOOP ...or a nesting depth of epoll instances greater than 5. - if ql_op == "EPOLL_CTL_ADD": - if epoll_parent_obj.is_present( - fd - ): # can't add an fd that's already being waited on - return EEXIST # op was EPOLL_CTL_ADD, and the supplied file descriptor fd is already registered with this epoll instance. - epoll_parent_obj.monitor_fd( - fd, ql_event - ) # add to list of fds to be monitored with per-fd eventmask - # register will actual epoll instance - # and add eventmask accordingly - elif ql_op == "EPOLL_CTL_DEL": - if not epoll_parent_obj.is_present( - fd - ): # op was EPOLL_CTL_MOD or EPOLL_CTL_DEL, and fd is not registered with this epoll instance. - return ENOENT - epoll_parent_obj.delist_fd(fd) # remove from fds list and do so in the - # underlying epoll instance - elif ql_op == "EPOLL_CTL_MOD": - if not epoll_parent_obj.is_present( - fd - ): # ENOENT op was EPOLL_CTL_MOD or EPOLL_CTL_DEL, and fd is not registered with this epoll instance - return ENOENT + return -ELOOP + + # Qiling doesn't check process capabilities right now, so this case isn't explicitly handled yet + # EPOLLWAKEUP (since Linux 3.5) + # If EPOLLONESHOT and EPOLLET are clear and the process has the CAP_BLOCK_SUSPEND capability + + # TODO: not sure if qiling supports a way to determine if the target file descriptor is a + # directory Check against PersistentQlFile is to ensure that polling stdin, stdout, stderr + # is supported + + fd_obj = ql.os.fd[fd] + + if fd_obj is None: + return -EBADF + + # The target file fd does not support epoll. This error can occur if fd refers to, for + # example, a regular file or a directory. + if isinstance(fd_obj, ql_file) and not isinstance(fd_obj, PersistentQlFile): + return -EPERM + + # elicn: not sure how the following condition even possible after we checked that op can + # be only one of EPOLL_CTL_{ADD,DEL,MOD} (originally checked with a dict) + + # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance + if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): + return -EINVAL + + # Necessary to iterate over all possible qiling fds to determine if we have a chain of more + # than five epolls monitoring each other This may be removed in the future if the QlOsLinux + # class had a separate field for reserved for tracking epoll objects. + epolls_list = [fobj for fobj in ql.os.fd if isinstance(fobj, QlEpollObj)] + + try: + check_epoll_depth(ql.os.fd, epolls_list) + # more than five detected? + except RecursionError: + return -ELOOP + + ql_event = event and ql.mem.read_ptr(event, 4) + + if op == EPOLL_CTL_ADD: + # can't add an fd that's already being waited on + if epoll_parent_obj.is_present(fd): + return -EEXIST + + # add to list of fds to be monitored with per-fd eventmask register will actual epoll + # instance and add eventmask accordingly + epoll_parent_obj.monitor_fd(fd, ql_event) + + elif op == EPOLL_CTL_DEL: + if not epoll_parent_obj.is_present(fd): + return -ENOENT + + # remove from fds list and do so in the underlying epoll instance + epoll_parent_obj.delist_fd(fd) + + elif op == EPOLL_CTL_MOD: + if not epoll_parent_obj.is_present(fd): + return -ENOENT + # EINVAL op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE. if op & EPOLLEXCLUSIVE and fd in epoll_obj.fds: - return EINVAL # EINVAL op was EPOLL_CTL_MOD and the EPOLLEXCLUSIVE flag has previously been applied to this epfd, fd pair. - epoll_parent_obj.eventmask(ql_event) + return -EINVAL + + epoll_parent_obj.set_eventmask(fd, ql_event) return 0 -""" -Wait on an existing epoll for events specified -earlier. man 7 epoll_wait for more info -""" +def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: int, timeout: int): + """Wait on an existing epoll for specific events. + """ - -def ql_syscall_epoll_wait( - ql: Qiling, epfd: int, epoll_events: POINTER, maxevents: int, timeout: int -): if maxevents <= 0: - return EINVAL - # default value is 0xffffffff, but - # this fails when passing to epoll.poll() + return -EINVAL + + # default value is 0xffffffff, but this fails when passing to epoll.poll() if timeout == 0xFFFFFFFF: timeout = None - try: - epoll_parent_obj = ql.os.fd[epfd] - epoll_obj = epoll_parent_obj.epoll_instance - if not isinstance(epoll_parent_obj, QlEpollObj): - return EINVAL - except KeyError: - ql.log.debug(f"FD {epfd} doesn't appear to be a valid epoll file descriptor") - return EBADF - try: - ql_event = ql.mem.read_ptr(epoll_events, ql.arch.pointersize) - except Exception: - ql.log.debug("Can't read from epoll_events pointer") - return EFAULT - ready_fds = list(epoll_obj.poll(timeout, maxevents)) + if epfd not in range(NR_OPEN): + return -EBADF - # Each tuple in ready_fds consists of - # (file descriptor, eventmask) - # so we iterate through these to indicate which fds - # are ready and 'why' - ret_val = len(ready_fds) - for i in range(0, ret_val): - fd = ready_fds[i][0] - interest_mask = ready_fds[i][1] - if ( - interest_mask & EPOLLONESHOT - ): # no longer interested in this fd, so remove from list + epoll_parent_obj = ql.os.fd[epfd] + + if not isinstance(epoll_parent_obj, QlEpollObj): + return -EINVAL + + epoll_obj = epoll_parent_obj.epoll_instance + + if epoll_obj is None: + return -EBADF + + # elicn: ql_event is not used, not sure why we need that here + # try: + # ql_event = ql.mem.read_ptr(epoll_events) + # except Exception: + # return -EFAULT + + ready_fds = epoll_obj.poll(timeout, maxevents) + + # Each tuple in ready_fds consists of (file descriptor, eventmask) so we iterate + # through these to indicate which fds are ready and 'why' + + for i, (fd, interest_mask) in enumerate(ready_fds): + # if no longer interested in this fd, remove from list + if interest_mask & EPOLLONESHOT: epoll_parent_obj.delist_fd(fd) - counter = ( - ql.arch.pointersize + 4 - ) * i # use ql.arch.pointersize to be compatible with 32-bit - data = ql.pack32(interest_mask) # uint32_t eventfds - data += ql.pack(fd) # need fd only, use pack() to handle endianness + size - ql.mem.write(epoll_events + counter, data) - return ret_val + data = ql.pack32(interest_mask) + ql.pack(fd) + offset = len(data) * i + + # elicn: maybe we need to use ql_event instead of epoll_events here..? + ql.mem.write(epoll_events + offset, data) + + return len(ready_fds) -""" -Use select.epoll for underlying implementation, -just as select.poll is used for emulating poll() -""" +def __epoll_create(ql: Qiling, sizehint: int, flags: int) -> int: + # Use select.epoll for underlying implementation, just as select.poll is + # used for emulating poll() + + ret = select.epoll(sizehint, flags) + + fd = ret.fileno() + ql.os.fd[fd] = QlEpollObj(ret) + + return fd def ql_syscall_epoll_create1(ql: Qiling, flags: int): if flags != select.EPOLL_CLOEXEC and flags != 0: - return EINVAL - ret = select.epoll(sizehint=-1, flags=flags) - fd = ret.fileno() - ql_obj = QlEpollObj(ret) - ql.os.fd[fd] = ql_obj - return fd + return -EINVAL - -""" -Almost identical to above, but can't simply wrap -because of the slightly different prototype -""" + return __epoll_create(ql, -1, flags) def ql_syscall_epoll_create(ql: Qiling, size: int): if size < 0: - return EINVAL - ret = select.epoll(sizehint=size, flags=0) - fd = ret.fileno() - ql_obj = QlEpollObj(ret) - ql.os.fd[fd] = ql_obj - return fd + return -EINVAL + + return __epoll_create(ql, size, 0) From 74380b26d5e757aa403d4f01b57d58390f45b7e2 Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Wed, 30 Apr 2025 00:18:41 +0000 Subject: [PATCH 049/138] Address a few elicn comments, fix root for server test --- examples/src/linux/Makefile | 3 +++ qiling/os/posix/syscall/epoll.py | 17 ++++++----------- tests/test_elf.py | 12 ++++++------ 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/examples/src/linux/Makefile b/examples/src/linux/Makefile index 1c81f67b..8a26e589 100644 --- a/examples/src/linux/Makefile +++ b/examples/src/linux/Makefile @@ -33,6 +33,7 @@ TARGETS = \ x8664_hello_cpp \ x8664_hello_cpp_static \ x8664_cloexec_test \ + x8664_linux_onestraw \ patch_test.bin .PHONY: all clean @@ -125,6 +126,8 @@ libpatch_test.so: patch_test.so.h patch_test.so.c patch_test.bin: patch_test.bin.c libpatch_test.so $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $^ +x8664_onestraw_server: x8664_linux_onestraw.c + $(CC) $(CPPFLAGS) $(CFLAGS) -m64 -o $@ $< $(OBJS):%.o:%.c $(CC) $(CFLAGS) -c $< -o $@ diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index bd237879..7b318bd4 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -12,7 +12,7 @@ from qiling.os.filestruct import PersistentQlFile if TYPE_CHECKING: from qiling.os.posix.posix import QlFileDes - +from qiling.os.posix.posix import QlFileDes class QlEpollObj: def __init__(self, epoll_object: select.epoll): @@ -38,9 +38,9 @@ class QlEpollObj: # the mask for an FD shouldn't ever be undefined # as it is set whenever an FD is added for a QlEpollObj instance - # elicn: don't we need to update self._fds[fd] with the new mask just like in monitor_fd? - + # libumem: resolved elicn feedback newmask = self.get_eventmask(fd) | newmask + self._fds[fd] = newmask self._epoll_object.modify(fd, newmask) def monitor_fd(self, fd: int, eventmask: int) -> None: @@ -117,7 +117,8 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): # If EPOLLONESHOT and EPOLLET are clear and the process has the CAP_BLOCK_SUSPEND capability # TODO: not sure if qiling supports a way to determine if the target file descriptor is a - # directory Check against PersistentQlFile is to ensure that polling stdin, stdout, stderr + # directory. + # Here, check against PersistentQlFile is to ensure that polling stdin, stdout, stderr # is supported fd_obj = ql.os.fd[fd] @@ -203,11 +204,6 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i if epoll_obj is None: return -EBADF - # elicn: ql_event is not used, not sure why we need that here - # try: - # ql_event = ql.mem.read_ptr(epoll_events) - # except Exception: - # return -EFAULT ready_fds = epoll_obj.poll(timeout, maxevents) @@ -221,8 +217,7 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i data = ql.pack32(interest_mask) + ql.pack(fd) offset = len(data) * i - - # elicn: maybe we need to use ql_event instead of epoll_events here..? + # Resolved elicn remark, ql_event was dead code ql.mem.write(epoll_events + offset, data) return len(ready_fds) diff --git a/tests/test_elf.py b/tests/test_elf.py index 4673262b..aae73898 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -778,17 +778,17 @@ class ELFTest(unittest.TestCase): and echos back the output. Upon receiving 'stop', it will exit. """ - @unittest.skip('See PR') + @unittest.skip("TODO: Stdin hijacking doesn't work as expected") def test_elf_linux_x8664_epoll_simple(self): # epoll-0 source: https://github.com/maxasm/epoll-c/blob/main/main.c rootfs = "../examples/rootfs/x8664_linux" argv = r"../examples/rootfs/x8664_linux/bin/x8664_linux_epoll_0".split() ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) - ql.os.stdin = pipe.SimpleBufferedStream() + #ql.os.stdin = pipe.SimpleInStream(0) ql.os.stdin.write(b'echo\n') ql.os.stdin.write(b"stop\n") # signal to exit gracefully ql.run() - self.assertIn("echo", ql.os.stdout.read().decode("utf-8")) + self.assertIn("echo\n", ql.os.stdout.read().decode("utf-8")) del ql """ This tests a simple server that uses epoll @@ -806,7 +806,7 @@ class ELFTest(unittest.TestCase): # Source for onestraw server: https://github.com/onestraw/epoll-example """ - FIXME: Without a hook for this syscall, this error fires: + Note: Without a hook for this syscall, this error fires: TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType """ def hook_newfstatat(ql: Qiling, dirfd: int, pathname: POINTER, statbuf: POINTER, flags:int): @@ -821,8 +821,8 @@ class ELFTest(unittest.TestCase): s.close() client_thread = threading.Thread(target=client, daemon=True) client_thread.start() - rootfs = "../examples/rootfs/" - argv = r"../examples/rootfs/x8664_linux/bin/onestraw-server s".split() # s means 'server mode' + rootfs = "../examples/rootfs/x8664_linux_glibc2.39" + argv = r"../examples/rootfs/x8664_linux/bin/x8664_onestraw_server s".split() # s means 'server mode' ql = Qiling(argv, rootfs, multithread=False, verbose=QL_VERBOSE.DEBUG) ql.os.set_syscall("newfstatat",hook_newfstatat, QL_INTERCEPT.CALL) ql.os.stdout = pipe.SimpleOutStream(1) # server prints data received to stdout From dbf213846bb9e309afe51ca40d8c7ddfb966695d Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:47:38 +0300 Subject: [PATCH 050/138] Turn check_epoll_depth into a prefix visitor --- qiling/os/posix/syscall/epoll.py | 45 ++++++++++++-------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 7b318bd4..423c75d9 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -59,30 +59,22 @@ class QlEpollObj: return fd in self.fds -def check_epoll_depth(ql_fd_list: QlFileDes, epolls_list: List[QlEpollObj], depth: int = 0) -> None: - # Recursively checks each epoll instance's 'watched' fds for an instance of - # epoll being watched. If a chain of over 5 levels is detected, return 1, - # which will return ELOOP in ql_syscall_epoll_wait +def check_epoll_depth(ql_fd_list: QlFileDes) -> None: + """Recursively check each epoll instance's 'watched' fds for an instance of + epoll being watched. If a chain of over 5 levels is detected, raise an exception + """ - if depth >= 5: - raise RecursionError + def __visit_obj(obj: QlEpollObj, depth: int): + if depth >= 5: + raise RecursionError - new_epolls_list = [] + for fd in obj.fds: + if isinstance(ql_fd_list[fd], QlEpollObj): + __visit_obj(obj, depth + 1) - for ent in epolls_list: - watched = ent.fds - - for w in watched: - obj = ql_fd_list[w] - - if isinstance(obj, QlEpollObj): - new_epolls_list.append(obj) - - # elicn: new_epolls_list is not cleared between loop iterations, rather it keeps - # aggregating items from previous iterations. is this what we want? - - if new_epolls_list: - check_epoll_depth(ql_fd_list, new_epolls_list, depth + 1) + for obj in ql_fd_list: + if isinstance(obj, QlEpollObj): + __visit_obj(obj, 1) def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): @@ -138,14 +130,11 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): return -EINVAL - # Necessary to iterate over all possible qiling fds to determine if we have a chain of more - # than five epolls monitoring each other This may be removed in the future if the QlOsLinux - # class had a separate field for reserved for tracking epoll objects. - epolls_list = [fobj for fobj in ql.os.fd if isinstance(fobj, QlEpollObj)] - try: - check_epoll_depth(ql.os.fd, epolls_list) - # more than five detected? + # Necessary to iterate over all possible qiling fds to determine if we have a chain of more + # than five epolls monitoring each other This may be removed in the future if the QlOsLinux + # class had a separate field reserved for tracking epoll objects. + check_epoll_depth(ql.os.fd) except RecursionError: return -ELOOP From 96674e1e7aba182ed62b6a62c7fa98a0f120adcc Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:49:32 +0300 Subject: [PATCH 051/138] Use container semantics instead of is_present --- qiling/os/posix/syscall/epoll.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 423c75d9..5fa898ca 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -55,7 +55,10 @@ class QlEpollObj: def close(self) -> None: self.epoll_instance.close() - def is_present(self, fd: int) -> bool: + def __contains__(self, fd: int) -> bool: + """Test whether a specific fd is already being watched by this epoll instance. + """ + return fd in self.fds @@ -142,7 +145,7 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if op == EPOLL_CTL_ADD: # can't add an fd that's already being waited on - if epoll_parent_obj.is_present(fd): + if fd in epoll_parent_obj: return -EEXIST # add to list of fds to be monitored with per-fd eventmask register will actual epoll @@ -150,14 +153,14 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): epoll_parent_obj.monitor_fd(fd, ql_event) elif op == EPOLL_CTL_DEL: - if not epoll_parent_obj.is_present(fd): + if fd not in epoll_parent_obj: return -ENOENT # remove from fds list and do so in the underlying epoll instance epoll_parent_obj.delist_fd(fd) elif op == EPOLL_CTL_MOD: - if not epoll_parent_obj.is_present(fd): + if fd not in epoll_parent_obj: return -ENOENT # EINVAL op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE. From f1081275e5d193b44cf1d9e8eda63a1b15f4cf7f Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:50:17 +0300 Subject: [PATCH 052/138] Fix events pointer handling --- qiling/os/posix/syscall/epoll.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 5fa898ca..08d96a43 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -141,16 +141,24 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): except RecursionError: return -ELOOP - ql_event = event and ql.mem.read_ptr(event, 4) - if op == EPOLL_CTL_ADD: # can't add an fd that's already being waited on if fd in epoll_parent_obj: return -EEXIST + if not event: + return -EINVAL + + event_ptr = ql.mem.read_ptr(event) + events = ql.mem.read_ptr(event_ptr, 4) + + # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance + if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): + return -EINVAL + # add to list of fds to be monitored with per-fd eventmask register will actual epoll # instance and add eventmask accordingly - epoll_parent_obj.monitor_fd(fd, ql_event) + epoll_parent_obj.monitor_fd(fd, events) elif op == EPOLL_CTL_DEL: if fd not in epoll_parent_obj: @@ -163,11 +171,17 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if fd not in epoll_parent_obj: return -ENOENT - # EINVAL op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE. - if op & EPOLLEXCLUSIVE and fd in epoll_obj.fds: + if not event: return -EINVAL - epoll_parent_obj.set_eventmask(fd, ql_event) + event_ptr = ql.mem.read_ptr(event) + events = ql.mem.read_ptr(event_ptr, 4) + + # EPOLLEXCLUSIVE cannot be set on MOD operation, only on ADD + if events & EPOLLEXCLUSIVE: + return -EINVAL + + epoll_parent_obj.set_eventmask(fd, events) return 0 From 0c96871d35210cbe97009aee324931eaeff087dc Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:51:41 +0300 Subject: [PATCH 053/138] Fix returned events array --- qiling/os/posix/syscall/epoll.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 08d96a43..8dadd212 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -221,7 +221,9 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i if interest_mask & EPOLLONESHOT: epoll_parent_obj.delist_fd(fd) - data = ql.pack32(interest_mask) + ql.pack(fd) + # FIXME: the data packed after events should be the one passed on epoll_ctl + # for that specific fd. currently this does not align with the spec + data = ql.pack32(interest_mask) + ql.pack64(fd) offset = len(data) * i # Resolved elicn remark, ql_event was dead code ql.mem.write(epoll_events + offset, data) From bf83e9c2cf6bef787ce80a6470eb6e4a1d55d12d Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:52:16 +0300 Subject: [PATCH 054/138] Cleanup and minor cosmetics --- qiling/os/posix/syscall/epoll.py | 48 +++++++++++++------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 8dadd212..d3eb0abe 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -1,18 +1,17 @@ +from __future__ import annotations + import select -from typing import TYPE_CHECKING, Dict, List +from typing import TYPE_CHECKING, Dict, KeysView -from qiling import * -from qiling.const import * from qiling.os.posix.const import * -from qiling.os.const import * -from qiling.os.filestruct import ql_file -from qiling.os.filestruct import PersistentQlFile +from qiling.os.filestruct import PersistentQlFile, ql_file if TYPE_CHECKING: + from qiling import Qiling from qiling.os.posix.posix import QlFileDes -from qiling.os.posix.posix import QlFileDes + class QlEpollObj: def __init__(self, epoll_object: select.epoll): @@ -24,8 +23,8 @@ class QlEpollObj: self._fds: Dict[int, int] = {} @property - def fds(self) -> List[int]: - return list(self._fds.keys()) + def fds(self) -> KeysView[int]: + return self._fds.keys() @property def epoll_instance(self) -> select.epoll: @@ -38,10 +37,10 @@ class QlEpollObj: # the mask for an FD shouldn't ever be undefined # as it is set whenever an FD is added for a QlEpollObj instance - # libumem: resolved elicn feedback newmask = self.get_eventmask(fd) | newmask - self._fds[fd] = newmask + self._epoll_object.modify(fd, newmask) + self._fds[fd] = newmask def monitor_fd(self, fd: int, eventmask: int) -> None: # tell the epoll object to watch the fd arg, looking for events matching the eventmask @@ -53,7 +52,7 @@ class QlEpollObj: self._epoll_object.unregister(fd) def close(self) -> None: - self.epoll_instance.close() + self._epoll_object.close() def __contains__(self, fd: int) -> bool: """Test whether a specific fd is already being watched by this epoll instance. @@ -111,28 +110,20 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): # EPOLLWAKEUP (since Linux 3.5) # If EPOLLONESHOT and EPOLLET are clear and the process has the CAP_BLOCK_SUSPEND capability - # TODO: not sure if qiling supports a way to determine if the target file descriptor is a - # directory. - # Here, check against PersistentQlFile is to ensure that polling stdin, stdout, stderr - # is supported - fd_obj = ql.os.fd[fd] if fd_obj is None: return -EBADF + # TODO: not sure if qiling supports a way to determine if the target file descriptor is a + # directory. Here, check against PersistentQlFile is to ensure that polling stdin, stdout, + # stderr is supported + # The target file fd does not support epoll. This error can occur if fd refers to, for # example, a regular file or a directory. if isinstance(fd_obj, ql_file) and not isinstance(fd_obj, PersistentQlFile): return -EPERM - # elicn: not sure how the following condition even possible after we checked that op can - # be only one of EPOLL_CTL_{ADD,DEL,MOD} (originally checked with a dict) - - # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance - if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): - return -EINVAL - try: # Necessary to iterate over all possible qiling fds to determine if we have a chain of more # than five epolls monitoring each other This may be removed in the future if the QlOsLinux @@ -210,22 +201,21 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i if epoll_obj is None: return -EBADF - ready_fds = epoll_obj.poll(timeout, maxevents) # Each tuple in ready_fds consists of (file descriptor, eventmask) so we iterate # through these to indicate which fds are ready and 'why' - for i, (fd, interest_mask) in enumerate(ready_fds): + for i, (fd, events) in enumerate(ready_fds): # if no longer interested in this fd, remove from list - if interest_mask & EPOLLONESHOT: + if events & EPOLLONESHOT: epoll_parent_obj.delist_fd(fd) # FIXME: the data packed after events should be the one passed on epoll_ctl # for that specific fd. currently this does not align with the spec - data = ql.pack32(interest_mask) + ql.pack64(fd) + data = ql.pack32(events) + ql.pack64(fd) offset = len(data) * i - # Resolved elicn remark, ql_event was dead code + ql.mem.write(epoll_events + offset, data) return len(ready_fds) From 5a87323df352fc4925e1b7c61169cddbbd71781c Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:52:40 +0300 Subject: [PATCH 055/138] Tidy up tests --- examples/src/linux/Makefile | 3 +- tests/test_elf.py | 78 +++++++++++++++++++------------------ 2 files changed, 43 insertions(+), 38 deletions(-) diff --git a/examples/src/linux/Makefile b/examples/src/linux/Makefile index 8a26e589..006ea490 100644 --- a/examples/src/linux/Makefile +++ b/examples/src/linux/Makefile @@ -33,7 +33,7 @@ TARGETS = \ x8664_hello_cpp \ x8664_hello_cpp_static \ x8664_cloexec_test \ - x8664_linux_onestraw \ + x8664_linux_onestraw \ patch_test.bin .PHONY: all clean @@ -125,6 +125,7 @@ libpatch_test.so: patch_test.so.h patch_test.so.c $(CC) $(CPPFLAGS) -Wall -s -O0 -shared -fpic -o $@ patch_test.so.c patch_test.bin: patch_test.bin.c libpatch_test.so + $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $^ x8664_onestraw_server: x8664_linux_onestraw.c $(CC) $(CPPFLAGS) $(CFLAGS) -m64 -o $@ $< diff --git a/tests/test_elf.py b/tests/test_elf.py index aae73898..8928e7a5 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -772,63 +772,67 @@ class ELFTest(unittest.TestCase): self.assertNotIn("root\n", ql.os.stdout.read().decode("utf-8")) del ql - - """ - This tests a sample binary that (e)polls on stdin - and echos back the output. Upon receiving 'stop', it - will exit. - """ - @unittest.skip("TODO: Stdin hijacking doesn't work as expected") + + @unittest.skip("stdin hijacking doesn't work as expected") def test_elf_linux_x8664_epoll_simple(self): - # epoll-0 source: https://github.com/maxasm/epoll-c/blob/main/main.c + # This tests a sample binary that (e)polls on stdin and echos back the output. Upon + # receiving 'stop', it will exit. + # + # epoll-0 tkaen from: https://github.com/maxasm/epoll-c/blob/main/main.c + rootfs = "../examples/rootfs/x8664_linux" argv = r"../examples/rootfs/x8664_linux/bin/x8664_linux_epoll_0".split() ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) + #ql.os.stdin = pipe.SimpleInStream(0) ql.os.stdin.write(b'echo\n') - ql.os.stdin.write(b"stop\n") # signal to exit gracefully + ql.os.stdin.write(b'stop\n') # signal to exit gracefully ql.run() - self.assertIn("echo\n", ql.os.stdout.read().decode("utf-8")) + + self.assertIn(b'echo\n', ql.os.stdout.read()) del ql - """ - This tests a simple server that uses epoll - to wait for data, then prints it out. It has been - modified to exit after data has been received; instead - of a typical server operation that reads requests indefinitely. - It listens on port 8000, and a separate thread is spawned in - order to test how the server handles a 'hello world' input. - The server prints out whatever it receives, so the assert - statement checks the input is present as expected. - """ - #@unittest.skip('See PR') def test_elf_linux_x8664_epoll_server(self): - # Source for onestraw server: https://github.com/onestraw/epoll-example + # This tests a simple server that uses epoll to wait for data, then prints it out. It has + # been modified to exit after data has been received; instead of a typical server operation + # that reads requests indefinitely. + # + # It listens on port 8000, and a separate thread is spawned in order to test how the server + # handles a 'hello world' input. The server prints out whatever it receives, so the assert + # statement checks the input is present as expected. + # + # onestraw server taken from: https://github.com/onestraw/epoll-example - """ - Note: Without a hook for this syscall, this error fires: - TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType - """ - def hook_newfstatat(ql: Qiling, dirfd: int, pathname: POINTER, statbuf: POINTER, flags:int): + # Note: Without a hook for this syscall, this error fires: + # TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType + def hook_newfstatat(ql: Qiling, dirfd: int, pathname: int, statbuf: int, flags: int): return 0 + def client(): - time.sleep(3) # give time for the server to listen + # give time for the server to listen + time.sleep(3) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - dest = ("127.0.0.1", 8000) - s.connect(dest) - test = b"hello world" - s.send(test) + s.connect(("127.0.0.1", 8000)) + s.send(b"hello world") s.close() - client_thread = threading.Thread(target=client, daemon=True) - client_thread.start() + rootfs = "../examples/rootfs/x8664_linux_glibc2.39" argv = r"../examples/rootfs/x8664_linux/bin/x8664_onestraw_server s".split() # s means 'server mode' - ql = Qiling(argv, rootfs, multithread=False, verbose=QL_VERBOSE.DEBUG) - ql.os.set_syscall("newfstatat",hook_newfstatat, QL_INTERCEPT.CALL) + + ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) + ql.os.set_syscall("newfstatat", hook_newfstatat, QL_INTERCEPT.CALL) ql.os.stdout = pipe.SimpleOutStream(1) # server prints data received to stdout ql.filter = '^data:' + + client_thread = threading.Thread(target=client, daemon=True) + client_thread.start() + ql.run() - self.assertIn('hello world', ql.os.stdout.read().decode("utf-8")) + + self.assertIn(b'hello world', ql.os.stdout.read()) del ql + + if __name__ == "__main__": unittest.main() From 12d3c58162665294730873a4468c32904a0a8f9b Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Wed, 7 May 2025 00:39:23 +0000 Subject: [PATCH 056/138] Address last of feedback --- qiling/os/posix/syscall/epoll.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 7b318bd4..c1103378 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -12,7 +12,7 @@ from qiling.os.filestruct import PersistentQlFile if TYPE_CHECKING: from qiling.os.posix.posix import QlFileDes -from qiling.os.posix.posix import QlFileDes +#from qiling.os.posix.posix import QlFileDes class QlEpollObj: def __init__(self, epoll_object: select.epoll): @@ -59,10 +59,10 @@ class QlEpollObj: return fd in self.fds -def check_epoll_depth(ql_fd_list: QlFileDes, epolls_list: List[QlEpollObj], depth: int = 0) -> None: +def check_epoll_depth(ql_fd_list, epolls_list: List[QlEpollObj], depth: int = 0) -> None: # Recursively checks each epoll instance's 'watched' fds for an instance of - # epoll being watched. If a chain of over 5 levels is detected, return 1, - # which will return ELOOP in ql_syscall_epoll_wait + # epoll being watched. If a chain of over 5 levels is detected, raise + # an exception if depth >= 5: raise RecursionError @@ -78,11 +78,9 @@ def check_epoll_depth(ql_fd_list: QlFileDes, epolls_list: List[QlEpollObj], dept if isinstance(obj, QlEpollObj): new_epolls_list.append(obj) - # elicn: new_epolls_list is not cleared between loop iterations, rather it keeps - # aggregating items from previous iterations. is this what we want? - if new_epolls_list: check_epoll_depth(ql_fd_list, new_epolls_list, depth + 1) + new_epolls_list = [] def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): @@ -131,8 +129,6 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if isinstance(fd_obj, ql_file) and not isinstance(fd_obj, PersistentQlFile): return -EPERM - # elicn: not sure how the following condition even possible after we checked that op can - # be only one of EPOLL_CTL_{ADD,DEL,MOD} (originally checked with a dict) # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): From e4242ca741116c66fdde120e003b608559ef0b92 Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Thu, 8 May 2025 00:10:34 +0000 Subject: [PATCH 057/138] Fix mem read issue --- qiling/os/posix/syscall/epoll.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index bd5a4acc..dd8fccf1 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -154,7 +154,6 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): return -EINVAL event_ptr = ql.mem.read_ptr(event) - events = ql.mem.read_ptr(event_ptr, 4) # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): @@ -162,7 +161,7 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): # add to list of fds to be monitored with per-fd eventmask register will actual epoll # instance and add eventmask accordingly - epoll_parent_obj.monitor_fd(fd, events) + epoll_parent_obj.monitor_fd(fd, event_ptr) elif op == EPOLL_CTL_DEL: if fd not in epoll_parent_obj: @@ -179,13 +178,12 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): return -EINVAL event_ptr = ql.mem.read_ptr(event) - events = ql.mem.read_ptr(event_ptr, 4) # EPOLLEXCLUSIVE cannot be set on MOD operation, only on ADD if events & EPOLLEXCLUSIVE: return -EINVAL - epoll_parent_obj.set_eventmask(fd, events) + epoll_parent_obj.set_eventmask(fd, event_ptr) return 0 @@ -226,7 +224,7 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i # FIXME: the data packed after events should be the one passed on epoll_ctl # for that specific fd. currently this does not align with the spec - data = ql.pack32(events) + ql.pack64(fd) + data = ql.pack32(events) + ql.pack(fd) offset = len(data) * i ql.mem.write(epoll_events + offset, data) From f9497bc7d6998c02bef5dd4cddf18e9132b24c7a Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Fri, 9 May 2025 03:26:43 +0000 Subject: [PATCH 058/138] Address more feedback --- qiling/os/posix/syscall/epoll.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index dd8fccf1..659faf43 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -107,6 +107,18 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if epoll_obj.fileno() == fd: return -ELOOP + events = 0 + if event: + events = ql.mem.read_ptr(event,4) + """ + event is of type epoll_event. run man epoll_event for more info + struct epoll_event { + uint32_t events; /* Epoll events */ + epoll_data_t data; /* User data variable */ + }; + so, read 4 bytes for the events field + """ + # Qiling doesn't check process capabilities right now, so this case isn't explicitly handled yet # EPOLLWAKEUP (since Linux 3.5) @@ -153,7 +165,7 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if not event: return -EINVAL - event_ptr = ql.mem.read_ptr(event) + # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): @@ -161,7 +173,7 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): # add to list of fds to be monitored with per-fd eventmask register will actual epoll # instance and add eventmask accordingly - epoll_parent_obj.monitor_fd(fd, event_ptr) + epoll_parent_obj.monitor_fd(fd, events) elif op == EPOLL_CTL_DEL: if fd not in epoll_parent_obj: @@ -177,13 +189,12 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if not event: return -EINVAL - event_ptr = ql.mem.read_ptr(event) # EPOLLEXCLUSIVE cannot be set on MOD operation, only on ADD if events & EPOLLEXCLUSIVE: return -EINVAL - epoll_parent_obj.set_eventmask(fd, event_ptr) + epoll_parent_obj.set_eventmask(fd, events) return 0 @@ -222,9 +233,8 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i if events & EPOLLONESHOT: epoll_parent_obj.delist_fd(fd) - # FIXME: the data packed after events should be the one passed on epoll_ctl - # for that specific fd. currently this does not align with the spec - data = ql.pack32(events) + ql.pack(fd) + # https://elixir.bootlin.com/linux/v6.14.4/source/include/uapi/linux/eventpoll.h#L83 + data = ql.pack32(events) + ql.pack64(fd) offset = len(data) * i ql.mem.write(epoll_events + offset, data) From c363dab2fb5cf3487ccc256f69ed219dd6b28707 Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 14 May 2025 13:04:15 +0300 Subject: [PATCH 059/138] Tidy up coverage classes and methods --- qiling/extensions/coverage/formats/base.py | 19 ++++--- qiling/extensions/coverage/formats/drcov.py | 49 ++++++++++--------- .../coverage/formats/drcov_exact.py | 7 +-- qiling/extensions/coverage/formats/ezcov.py | 35 +++++++------ qiling/extensions/tracing/formats/base.py | 18 +++---- 5 files changed, 64 insertions(+), 64 deletions(-) diff --git a/qiling/extensions/coverage/formats/base.py b/qiling/extensions/coverage/formats/base.py index d9fe7c34..32a6fc55 100644 --- a/qiling/extensions/coverage/formats/base.py +++ b/qiling/extensions/coverage/formats/base.py @@ -4,8 +4,11 @@ # from abc import ABC, abstractmethod +from typing import TYPE_CHECKING -from qiling import Qiling + +if TYPE_CHECKING: + from qiling import Qiling class QlBaseCoverage(ABC): @@ -15,25 +18,21 @@ class QlBaseCoverage(ABC): all the methods marked with the @abstractmethod decorator. """ + FORMAT_NAME: str + def __init__(self, ql: Qiling): super().__init__() self.ql = ql - @property - @staticmethod @abstractmethod - def FORMAT_NAME() -> str: - raise NotImplementedError - - @abstractmethod - def activate(self): + def activate(self) -> None: pass @abstractmethod - def deactivate(self): + def deactivate(self) -> None: pass @abstractmethod - def dump_coverage(self, coverage_file: str): + def dump_coverage(self, coverage_file: str) -> None: pass diff --git a/qiling/extensions/coverage/formats/drcov.py b/qiling/extensions/coverage/formats/drcov.py index 51a42194..d541837d 100644 --- a/qiling/extensions/coverage/formats/drcov.py +++ b/qiling/extensions/coverage/formats/drcov.py @@ -3,12 +3,18 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from ctypes import Structure -from ctypes import c_uint32, c_uint16 +from __future__ import annotations + +from ctypes import Structure, c_uint32, c_uint16 +from typing import TYPE_CHECKING, BinaryIO from .base import QlBaseCoverage +if TYPE_CHECKING: + from qiling import Qiling + + # Adapted from https://www.ayrx.me/drcov-file-format class bb_entry(Structure): _fields_ = [ @@ -29,7 +35,7 @@ class QlDrCoverage(QlBaseCoverage): FORMAT_NAME = "drcov" - def __init__(self, ql): + def __init__(self, ql: Qiling): super().__init__(ql) self.drcov_version = 2 @@ -37,28 +43,27 @@ class QlDrCoverage(QlBaseCoverage): self.basic_blocks = [] self.bb_callback = None - @staticmethod - def block_callback(ql, address, size, self): - for mod_id, mod in enumerate(ql.loader.images): - if mod.base <= address <= mod.end: - ent = bb_entry(address - mod.base, size, mod_id) - self.basic_blocks.append(ent) - break + def activate(self) -> None: + self.bb_callback = self.ql.hook_block(self.block_callback) - def activate(self): - self.bb_callback = self.ql.hook_block(self.block_callback, user_data=self) + def deactivate(self) -> None: + if self.bb_callback: + self.ql.hook_del(self.bb_callback) - def deactivate(self): - self.ql.hook_del(self.bb_callback) + def dump_coverage(self, coverage_file: str) -> None: + def __write_line(bio: BinaryIO, line: str) -> None: + bio.write(f'{line}\n'.encode()) - def dump_coverage(self, coverage_file): with open(coverage_file, "wb") as cov: - cov.write(f"DRCOV VERSION: {self.drcov_version}\n".encode()) - cov.write(f"DRCOV FLAVOR: {self.drcov_flavor}\n".encode()) - cov.write(f"Module Table: version {self.drcov_version}, count {len(self.ql.loader.images)}\n".encode()) - cov.write("Columns: id, base, end, entry, checksum, timestamp, path\n".encode()) + __write_line(cov, f"DRCOV VERSION: {self.drcov_version}") + __write_line(cov, f"DRCOV FLAVOR: {self.drcov_flavor}") + __write_line(cov, f"Module Table: version {self.drcov_version}, count {len(self.ql.loader.images)}") + __write_line(cov, "Columns: id, base, end, entry, checksum, timestamp, path") + for mod_id, mod in enumerate(self. ql.loader.images): - cov.write(f"{mod_id}, {mod.base}, {mod.end}, 0, 0, 0, {mod.path}\n".encode()) - cov.write(f"BB Table: {len(self.basic_blocks)} bbs\n".encode()) - for bb in self.basic_blocks: + __write_line(cov, f"{mod_id}, {mod.base}, {mod.end}, 0, 0, 0, {mod.path}") + + __write_line(cov, f"BB Table: {len(self.basic_blocks)} bbs") + + for bb in self.basic_blocks.values(): cov.write(bytes(bb)) diff --git a/qiling/extensions/coverage/formats/drcov_exact.py b/qiling/extensions/coverage/formats/drcov_exact.py index 685c6c04..afb6e844 100644 --- a/qiling/extensions/coverage/formats/drcov_exact.py +++ b/qiling/extensions/coverage/formats/drcov_exact.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -17,10 +17,7 @@ class QlDrCoverageExact(QlDrCoverage): FORMAT_NAME = "drcov_exact" - def __init__(self, ql): - super().__init__(ql) - - def activate(self): + def activate(self) -> None: # We treat every instruction as a block on its own. self.bb_callback = self.ql.hook_code(self.block_callback, user_data=self) \ No newline at end of file diff --git a/qiling/extensions/coverage/formats/ezcov.py b/qiling/extensions/coverage/formats/ezcov.py index b2521869..2fab67e9 100644 --- a/qiling/extensions/coverage/formats/ezcov.py +++ b/qiling/extensions/coverage/formats/ezcov.py @@ -1,14 +1,20 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from __future__ import annotations from collections import namedtuple from os.path import basename +from typing import TYPE_CHECKING, List from .base import QlBaseCoverage +if TYPE_CHECKING: + from qiling import Qiling + + # Adapted from https://github.com/nccgroup/Cartographer/blob/main/EZCOV.md#coverage-data class bb_entry(namedtuple('bb_entry', 'offset size mod_id')): def csvline(self): @@ -27,27 +33,24 @@ class QlEzCoverage(QlBaseCoverage): FORMAT_NAME = "ezcov" - def __init__(self, ql): + def __init__(self, ql: Qiling): super().__init__(ql) + self.ezcov_version = 1 - self.ezcov_flavor = 'ezcov' - self.basic_blocks = [] - self.bb_callback = None + self.ezcov_flavor = 'ezcov' + self.basic_blocks: List[bb_entry] = [] + self.bb_callback = None - @staticmethod - def block_callback(ql, address, size, self): - mod = ql.loader.find_containing_image(address) - if mod is not None: - ent = bb_entry(address - mod.base, size, basename(mod.path)) - self.basic_blocks.append(ent) + def block_callback(self, ql: Qiling, address: int, size: int): - def activate(self): - self.bb_callback = self.ql.hook_block(self.block_callback, user_data=self) + def activate(self) -> None: + self.bb_callback = self.ql.hook_block(self.block_callback) - def deactivate(self): - self.ql.hook_del(self.bb_callback) + def deactivate(self) -> None: + if self.bb_callback: + self.ql.hook_del(self.bb_callback) - def dump_coverage(self, coverage_file): + def dump_coverage(self, coverage_file: str) -> None: with open(coverage_file, "w") as cov: cov.write(f"EZCOV VERSION: {self.ezcov_version}\n") cov.write("# Qiling EZCOV exporter tool\n") diff --git a/qiling/extensions/tracing/formats/base.py b/qiling/extensions/tracing/formats/base.py index 14594434..dbb9c78a 100644 --- a/qiling/extensions/tracing/formats/base.py +++ b/qiling/extensions/tracing/formats/base.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # This code structure is copied and modified from the coverage extension @@ -12,24 +12,20 @@ class QlBaseTrace(ABC): To add support for a new coverage format, just derive from this class and implement all the methods marked with the @abstractmethod decorator. """ - + + FORMAT_NAME: str + def __init__(self): super().__init__() - @property - @staticmethod @abstractmethod - def FORMAT_NAME(): - raise NotImplementedError - - @abstractmethod - def activate(self): + def activate(self) -> None: pass @abstractmethod - def deactivate(self): + def deactivate(self) -> None: pass @abstractmethod - def dump_trace(self, trace_file): + def dump_trace(self, trace_file: str) -> None: pass \ No newline at end of file From be824ad0d4671347c6750331c54212933f6aaf1c Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 14 May 2025 13:05:53 +0300 Subject: [PATCH 060/138] Remove drcove bb dups and speed it up --- qiling/extensions/coverage/formats/drcov.py | 32 +++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/qiling/extensions/coverage/formats/drcov.py b/qiling/extensions/coverage/formats/drcov.py index d541837d..bed0f870 100644 --- a/qiling/extensions/coverage/formats/drcov.py +++ b/qiling/extensions/coverage/formats/drcov.py @@ -6,13 +6,15 @@ from __future__ import annotations from ctypes import Structure, c_uint32, c_uint16 -from typing import TYPE_CHECKING, BinaryIO +from functools import lru_cache +from typing import TYPE_CHECKING, BinaryIO, Dict, Tuple from .base import QlBaseCoverage if TYPE_CHECKING: from qiling import Qiling + from qiling.loader.loader import QlLoader # Adapted from https://www.ayrx.me/drcov-file-format @@ -40,9 +42,35 @@ class QlDrCoverage(QlBaseCoverage): self.drcov_version = 2 self.drcov_flavor = 'drcov' - self.basic_blocks = [] + self.basic_blocks: Dict[int, bb_entry] = {} self.bb_callback = None + @lru_cache(maxsize=64) + def _get_img_base(self, loader: QlLoader, address: int) -> Tuple[int, int]: + """Retrieve the containing image of a given address. + + Addresses are expected to be aligned to page boundary, and cached for faster retrieval. + """ + + return next((i, img.base) for i, img in enumerate(loader.images) if img.base <= address < img.end) + + def block_callback(self, ql: Qiling, address: int, size: int): + if address not in self.basic_blocks: + try: + # we rely on the fact that images are allocated on page size boundary and + # use it to speed up image retrieval. we align the basic block address to + # page boundary, knowing basic blocks within the same page belong to the + # same image. then we use the aligned address to retreive the containing + # image. returned values are cached so subsequent retrievals for basic + # blocks within the same page will return the cached value instead of + # going through the retreival process again (up to maxsize cached pages) + + i, img_base = self._get_img_base(ql.loader, address & ~(0x1000 - 1)) + except StopIteration: + pass + else: + self.basic_blocks[address] = bb_entry(address - img_base, size, i) + def activate(self) -> None: self.bb_callback = self.ql.hook_block(self.block_callback) From 104a3f3399c13cdf9f4eb487fcad8d3d5aa8eded Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 14 May 2025 13:06:07 +0300 Subject: [PATCH 061/138] Additional housekeeping --- .../coverage/formats/drcov_exact.py | 3 +-- qiling/extensions/coverage/formats/ezcov.py | 25 +++++++++++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/qiling/extensions/coverage/formats/drcov_exact.py b/qiling/extensions/coverage/formats/drcov_exact.py index afb6e844..4f708278 100644 --- a/qiling/extensions/coverage/formats/drcov_exact.py +++ b/qiling/extensions/coverage/formats/drcov_exact.py @@ -19,5 +19,4 @@ class QlDrCoverageExact(QlDrCoverage): def activate(self) -> None: # We treat every instruction as a block on its own. - self.bb_callback = self.ql.hook_code(self.block_callback, user_data=self) - \ No newline at end of file + self.bb_callback = self.ql.hook_code(self.block_callback) diff --git a/qiling/extensions/coverage/formats/ezcov.py b/qiling/extensions/coverage/formats/ezcov.py index 2fab67e9..46290e4c 100644 --- a/qiling/extensions/coverage/formats/ezcov.py +++ b/qiling/extensions/coverage/formats/ezcov.py @@ -4,9 +4,9 @@ # from __future__ import annotations -from collections import namedtuple -from os.path import basename -from typing import TYPE_CHECKING, List + +import os +from typing import Any, TYPE_CHECKING, List, NamedTuple from .base import QlBaseCoverage @@ -16,10 +16,15 @@ if TYPE_CHECKING: # Adapted from https://github.com/nccgroup/Cartographer/blob/main/EZCOV.md#coverage-data -class bb_entry(namedtuple('bb_entry', 'offset size mod_id')): - def csvline(self): - offset = '0x{:08x}'.format(self.offset) +class bb_entry(NamedTuple): + offset: int + size: int + mod_id: Any + + def as_csv(self) -> str: + offset = f'{self.offset:#010x}' mod_id = f"[ {self.mod_id if self.mod_id is not None else ''} ]" + return f"{offset},{self.size},{mod_id}\n" class QlEzCoverage(QlBaseCoverage): @@ -42,6 +47,10 @@ class QlEzCoverage(QlBaseCoverage): self.bb_callback = None def block_callback(self, ql: Qiling, address: int, size: int): + img = ql.loader.find_containing_image(address) + + if img is not None: + self.basic_blocks.append(bb_entry(address - img.base, size, os.path.basename(img.path))) def activate(self) -> None: self.bb_callback = self.ql.hook_block(self.block_callback) @@ -54,5 +63,5 @@ class QlEzCoverage(QlBaseCoverage): with open(coverage_file, "w") as cov: cov.write(f"EZCOV VERSION: {self.ezcov_version}\n") cov.write("# Qiling EZCOV exporter tool\n") - for bb in self.basic_blocks: - cov.write(bb.csvline()) \ No newline at end of file + + cov.writelines(bb.as_csv() for bb in self.basic_blocks) From 43b8b3c5088bb69dd58b8319142f46039b2a39e9 Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 14 May 2025 13:23:44 +0300 Subject: [PATCH 062/138] Add forgotten import --- qiling/extensions/coverage/formats/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qiling/extensions/coverage/formats/base.py b/qiling/extensions/coverage/formats/base.py index 32a6fc55..4ca162cb 100644 --- a/qiling/extensions/coverage/formats/base.py +++ b/qiling/extensions/coverage/formats/base.py @@ -3,6 +3,8 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from __future__ import annotations + from abc import ABC, abstractmethod from typing import TYPE_CHECKING From 70255afd5a5a31ea36b69fd20b832065ab9070fb Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 14 May 2025 13:24:16 +0300 Subject: [PATCH 063/138] Have blob loader contain an image --- qiling/loader/blob.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/qiling/loader/blob.py b/qiling/loader/blob.py index 382dbb33..b8831a55 100644 --- a/qiling/loader/blob.py +++ b/qiling/loader/blob.py @@ -4,9 +4,10 @@ # from qiling import Qiling -from qiling.loader.loader import QlLoader +from qiling.loader.loader import QlLoader, Image from qiling.os.memory import QlMemoryHeap + class QlLoaderBLOB(QlLoader): def __init__(self, ql: Qiling): super().__init__(ql) @@ -16,13 +17,19 @@ class QlLoaderBLOB(QlLoader): def run(self): self.load_address = self.ql.os.entry_point # for consistency - self.ql.mem.map(self.ql.os.entry_point, self.ql.os.code_ram_size, info="[code]") - self.ql.mem.write(self.ql.os.entry_point, self.ql.code) + code_begins = self.load_address + code_size = self.ql.os.code_ram_size + code_ends = code_begins + code_size - heap_address = self.ql.os.entry_point + self.ql.os.code_ram_size + self.ql.mem.map(code_begins, code_size, info="[code]") + self.ql.mem.write(code_begins, self.ql.code) + + # allow image-related functionalities + self.images.append(Image(code_begins, code_ends, 'blob_code')) + + # FIXME: heap starts above end of ram?? + heap_base = code_ends heap_size = int(self.ql.os.profile.get("CODE", "heap_size"), 16) - self.ql.os.heap = QlMemoryHeap(self.ql, heap_address, heap_address + heap_size) + self.ql.os.heap = QlMemoryHeap(self.ql, heap_base, heap_base + heap_size) - self.ql.arch.regs.arch_sp = heap_address - 0x1000 - - return + self.ql.arch.regs.arch_sp = code_ends - 0x1000 From 47c3df1eb5cf3fccb71319137ae88bbea42a153b Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Fri, 16 May 2025 02:23:58 +0000 Subject: [PATCH 064/138] Fix root for test case --- tests/test_elf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_elf.py b/tests/test_elf.py index 8928e7a5..3a5af4cd 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -817,7 +817,7 @@ class ELFTest(unittest.TestCase): s.send(b"hello world") s.close() - rootfs = "../examples/rootfs/x8664_linux_glibc2.39" + rootfs = "../examples/rootfs/x8664_linux" argv = r"../examples/rootfs/x8664_linux/bin/x8664_onestraw_server s".split() # s means 'server mode' ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) From 4d0b939da5b5e626c8f979402bda469a25a0820a Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Mon, 2 Jun 2025 23:27:36 +0000 Subject: [PATCH 065/138] Revert "Fix root for test case" This reverts commit 47c3df1eb5cf3fccb71319137ae88bbea42a153b. --- tests/test_elf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_elf.py b/tests/test_elf.py index 3a5af4cd..8928e7a5 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -817,7 +817,7 @@ class ELFTest(unittest.TestCase): s.send(b"hello world") s.close() - rootfs = "../examples/rootfs/x8664_linux" + rootfs = "../examples/rootfs/x8664_linux_glibc2.39" argv = r"../examples/rootfs/x8664_linux/bin/x8664_onestraw_server s".split() # s means 'server mode' ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) From 34fcfb56b54aae46a1b3019923191b8190a42fc9 Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Mon, 2 Jun 2025 23:35:17 +0000 Subject: [PATCH 066/138] Revert "Address more feedback" This reverts commit f9497bc7d6998c02bef5dd4cddf18e9132b24c7a. --- qiling/os/posix/syscall/epoll.py | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 659faf43..dd8fccf1 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -107,18 +107,6 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if epoll_obj.fileno() == fd: return -ELOOP - events = 0 - if event: - events = ql.mem.read_ptr(event,4) - """ - event is of type epoll_event. run man epoll_event for more info - struct epoll_event { - uint32_t events; /* Epoll events */ - epoll_data_t data; /* User data variable */ - }; - so, read 4 bytes for the events field - """ - # Qiling doesn't check process capabilities right now, so this case isn't explicitly handled yet # EPOLLWAKEUP (since Linux 3.5) @@ -165,7 +153,7 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if not event: return -EINVAL - + event_ptr = ql.mem.read_ptr(event) # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): @@ -173,7 +161,7 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): # add to list of fds to be monitored with per-fd eventmask register will actual epoll # instance and add eventmask accordingly - epoll_parent_obj.monitor_fd(fd, events) + epoll_parent_obj.monitor_fd(fd, event_ptr) elif op == EPOLL_CTL_DEL: if fd not in epoll_parent_obj: @@ -189,12 +177,13 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if not event: return -EINVAL + event_ptr = ql.mem.read_ptr(event) # EPOLLEXCLUSIVE cannot be set on MOD operation, only on ADD if events & EPOLLEXCLUSIVE: return -EINVAL - epoll_parent_obj.set_eventmask(fd, events) + epoll_parent_obj.set_eventmask(fd, event_ptr) return 0 @@ -233,8 +222,9 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i if events & EPOLLONESHOT: epoll_parent_obj.delist_fd(fd) - # https://elixir.bootlin.com/linux/v6.14.4/source/include/uapi/linux/eventpoll.h#L83 - data = ql.pack32(events) + ql.pack64(fd) + # FIXME: the data packed after events should be the one passed on epoll_ctl + # for that specific fd. currently this does not align with the spec + data = ql.pack32(events) + ql.pack(fd) offset = len(data) * i ql.mem.write(epoll_events + offset, data) From 82de6f9d3a45eb0ef6da1d50d6053c6bb0b8517c Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Mon, 2 Jun 2025 23:35:23 +0000 Subject: [PATCH 067/138] Revert "Fix mem read issue" This reverts commit e4242ca741116c66fdde120e003b608559ef0b92. --- qiling/os/posix/syscall/epoll.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index dd8fccf1..bd5a4acc 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -154,6 +154,7 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): return -EINVAL event_ptr = ql.mem.read_ptr(event) + events = ql.mem.read_ptr(event_ptr, 4) # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): @@ -161,7 +162,7 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): # add to list of fds to be monitored with per-fd eventmask register will actual epoll # instance and add eventmask accordingly - epoll_parent_obj.monitor_fd(fd, event_ptr) + epoll_parent_obj.monitor_fd(fd, events) elif op == EPOLL_CTL_DEL: if fd not in epoll_parent_obj: @@ -178,12 +179,13 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): return -EINVAL event_ptr = ql.mem.read_ptr(event) + events = ql.mem.read_ptr(event_ptr, 4) # EPOLLEXCLUSIVE cannot be set on MOD operation, only on ADD if events & EPOLLEXCLUSIVE: return -EINVAL - epoll_parent_obj.set_eventmask(fd, event_ptr) + epoll_parent_obj.set_eventmask(fd, events) return 0 @@ -224,7 +226,7 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i # FIXME: the data packed after events should be the one passed on epoll_ctl # for that specific fd. currently this does not align with the spec - data = ql.pack32(events) + ql.pack(fd) + data = ql.pack32(events) + ql.pack64(fd) offset = len(data) * i ql.mem.write(epoll_events + offset, data) From 5ad49c0fdc2ded8a4ece5b6aa669eaa8adbd73c8 Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Mon, 2 Jun 2025 23:37:38 +0000 Subject: [PATCH 068/138] Revert "Revert "Fix root for test case"" This reverts commit 4d0b939da5b5e626c8f979402bda469a25a0820a. --- tests/test_elf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_elf.py b/tests/test_elf.py index 8928e7a5..3a5af4cd 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -817,7 +817,7 @@ class ELFTest(unittest.TestCase): s.send(b"hello world") s.close() - rootfs = "../examples/rootfs/x8664_linux_glibc2.39" + rootfs = "../examples/rootfs/x8664_linux" argv = r"../examples/rootfs/x8664_linux/bin/x8664_onestraw_server s".split() # s means 'server mode' ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) From ca0d35a51db43c161556a9bd9d3352307cb59273 Mon Sep 17 00:00:00 2001 From: elicn Date: Tue, 3 Jun 2025 13:19:21 +0300 Subject: [PATCH 069/138] Introduce packed struct --- qiling/os/struct.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/qiling/os/struct.py b/qiling/os/struct.py index 9e156777..3b94d7be 100644 --- a/qiling/os/struct.py +++ b/qiling/os/struct.py @@ -222,6 +222,26 @@ class BaseStructEB(BaseStruct, ctypes.BigEndianStructure): pass +@cache +def get_packed_struct(endian: QL_ENDIAN = QL_ENDIAN.EL) -> Type[BaseStruct]: + """Provide a packed version of BaseStruct based on the emulated + architecture endianess. + + Args: + archbits: required alignment in bits + """ + + Struct = { + QL_ENDIAN.EL: BaseStructEL, + QL_ENDIAN.EB: BaseStructEB + }[endian] + + class PackedStruct(Struct): + _pack_ = 1 + + return PackedStruct + + @cache def get_aligned_struct(archbits: int, endian: QL_ENDIAN = QL_ENDIAN.EL) -> Type[BaseStruct]: """Provide an aligned version of BaseStruct based on the emulated @@ -229,6 +249,7 @@ def get_aligned_struct(archbits: int, endian: QL_ENDIAN = QL_ENDIAN.EL) -> Type[ Args: archbits: required alignment in bits + endian: required endianness """ Struct = { From 31720cfca67414f0c0409fe26232437583bc25a0 Mon Sep 17 00:00:00 2001 From: elicn Date: Tue, 3 Jun 2025 13:20:02 +0300 Subject: [PATCH 070/138] Refactor epoll to rely on ctypess tructure --- qiling/os/posix/syscall/epoll.py | 124 ++++++++++++++++++++----------- 1 file changed, 81 insertions(+), 43 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index bd5a4acc..50b70c6c 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -1,17 +1,48 @@ from __future__ import annotations +import ctypes import select -from typing import TYPE_CHECKING, Dict, KeysView +from typing import TYPE_CHECKING, Dict, KeysView, NamedTuple +from qiling.os import struct from qiling.os.posix.const import * from qiling.os.filestruct import PersistentQlFile, ql_file if TYPE_CHECKING: from qiling import Qiling + from qiling.arch.arch import QlArch from qiling.os.posix.posix import QlFileDes + +class QlEpollEntry(NamedTuple): + """A named tuple to represent an epoll entry. + + This is used to store the events mask and the data for each entry in + the epoll instance. + """ + + events: int + data: int + + +@struct.cache +def __make_epoll_event(arch: QlArch): + """Create a structure to represent an epoll event. + """ + + Struct = struct.get_packed_struct(arch.endian) + + class epoll_event(Struct): + _fields_ = ( + ('events', ctypes.c_uint32), + ('data', ctypes.c_uint64) + ) + + return epoll_event + + class QlEpollObj: def __init__(self, epoll_object: select.epoll): self._epoll_object = epoll_object @@ -19,7 +50,7 @@ class QlEpollObj: # maps fd to eventmask # keep track of which fds have what eventmasks, # since this isn't directly supported in select.epoll - self._fds: Dict[int, int] = {} + self._fds: Dict[int, QlEpollEntry] = {} @property def fds(self) -> KeysView[int]: @@ -29,30 +60,29 @@ class QlEpollObj: def epoll_instance(self) -> select.epoll: return self._epoll_object - def get_eventmask(self, fd: int) -> int: - return self._fds[fd] - - def set_eventmask(self, fd: int, newmask: int): - # the mask for an FD shouldn't ever be undefined - # as it is set whenever an FD is added for a QlEpollObj instance - - newmask = self.get_eventmask(fd) | newmask - - self._epoll_object.modify(fd, newmask) - self._fds[fd] = newmask - - def monitor_fd(self, fd: int, eventmask: int) -> None: - # tell the epoll object to watch the fd arg, looking for events matching the eventmask - self._epoll_object.register(fd, eventmask) - self._fds[fd] = eventmask - - def delist_fd(self, fd: int) -> None: - self._fds.pop(fd) - self._epoll_object.unregister(fd) - def close(self) -> None: self._epoll_object.close() + def __getitem__(self, fd: int) -> QlEpollEntry: + return self._fds[fd] + + def __setitem__(self, fd: int, entry: QlEpollEntry) -> None: + # if fd is already being watched, modify its eventmask. + if fd in self: + self._epoll_object.modify(fd, entry.events) + + # otherwise, register it with the epoll object + else: + self._epoll_object.register(fd, entry.events) + + self._fds[fd] = entry + + def __delitem__(self, fd: int) -> None: + """Remove an fd from the epoll instance. + """ + + self._fds.pop(fd) + self._epoll_object.unregister(fd) def __contains__(self, fd: int) -> bool: """Test whether a specific fd is already being watched by this epoll instance. @@ -153,23 +183,25 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if not event: return -EINVAL - event_ptr = ql.mem.read_ptr(event) - events = ql.mem.read_ptr(event_ptr, 4) + # dereference the event pointer to get structure fields + epoll_event_cls = __make_epoll_event(ql.arch) + epoll_event = epoll_event_cls.load_from(ql.mem, event) # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance - if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): + if isinstance(fd_obj, QlEpollObj) and (epoll_event.events & EPOLLEXCLUSIVE): return -EINVAL - # add to list of fds to be monitored with per-fd eventmask register will actual epoll - # instance and add eventmask accordingly - epoll_parent_obj.monitor_fd(fd, events) + epoll_parent_obj[fd] = QlEpollEntry( + epoll_event.events, + epoll_event.data + ) elif op == EPOLL_CTL_DEL: if fd not in epoll_parent_obj: return -ENOENT # remove from fds list and do so in the underlying epoll instance - epoll_parent_obj.delist_fd(fd) + del epoll_parent_obj[fd] elif op == EPOLL_CTL_MOD: if fd not in epoll_parent_obj: @@ -178,14 +210,18 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if not event: return -EINVAL - event_ptr = ql.mem.read_ptr(event) - events = ql.mem.read_ptr(event_ptr, 4) + # dereference the event pointer to get structure fields + epoll_event_cls = __make_epoll_event(ql.arch) + epoll_event = epoll_event_cls.load_from(ql.mem, event) # EPOLLEXCLUSIVE cannot be set on MOD operation, only on ADD - if events & EPOLLEXCLUSIVE: + if epoll_event.events & EPOLLEXCLUSIVE: return -EINVAL - epoll_parent_obj.set_eventmask(fd, events) + epoll_parent_obj[fd] = QlEpollEntry( + epoll_event.events, + epoll_event.data + ) return 0 @@ -216,20 +252,22 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i ready_fds = epoll_obj.poll(timeout, maxevents) + epoll_event_cls = __make_epoll_event(ql.arch) + # Each tuple in ready_fds consists of (file descriptor, eventmask) so we iterate # through these to indicate which fds are ready and 'why' - + # + # FIXME: emulated system fds are not the same as hosted system fds for i, (fd, events) in enumerate(ready_fds): + entry = epoll_parent_obj[fd] + epoll_event = epoll_event_cls(events, entry.data) + + offset = epoll_event_cls.sizeof() * i + ql.mem.write(epoll_events + offset, bytes(epoll_event)) + # if no longer interested in this fd, remove from list if events & EPOLLONESHOT: - epoll_parent_obj.delist_fd(fd) - - # FIXME: the data packed after events should be the one passed on epoll_ctl - # for that specific fd. currently this does not align with the spec - data = ql.pack32(events) + ql.pack64(fd) - offset = len(data) * i - - ql.mem.write(epoll_events + offset, data) + del epoll_parent_obj[fd] return len(ready_fds) From 92e43a35527ff7e4ca31694959c10d66ddfa118f Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Wed, 4 Jun 2025 00:40:36 +0000 Subject: [PATCH 071/138] Fix glibc issue with new test root --- tests/test_elf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_elf.py b/tests/test_elf.py index 3a5af4cd..82c386c0 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -817,7 +817,7 @@ class ELFTest(unittest.TestCase): s.send(b"hello world") s.close() - rootfs = "../examples/rootfs/x8664_linux" + rootfs = "../examples/rootfs/x8664_linux_glibc2.39" # fix GLIBC version requirement argv = r"../examples/rootfs/x8664_linux/bin/x8664_onestraw_server s".split() # s means 'server mode' ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) From fdfec5b6d1b1abffaa1d815038cf512bd19ea82f Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Tue, 17 Jun 2025 22:39:45 +0000 Subject: [PATCH 072/138] Attempt #1 to fix test case root --- tests/test_elf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_elf.py b/tests/test_elf.py index 82c386c0..3a5af4cd 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -817,7 +817,7 @@ class ELFTest(unittest.TestCase): s.send(b"hello world") s.close() - rootfs = "../examples/rootfs/x8664_linux_glibc2.39" # fix GLIBC version requirement + rootfs = "../examples/rootfs/x8664_linux" argv = r"../examples/rootfs/x8664_linux/bin/x8664_onestraw_server s".split() # s means 'server mode' ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) From b1ad033720ca6df4659acfbb3eae2f3d050181f4 Mon Sep 17 00:00:00 2001 From: elicn Date: Sat, 21 Jun 2025 22:39:08 +0300 Subject: [PATCH 073/138] Have qdb use xPSR for Cortex M instead of CPSR --- qiling/debugger/qdb/arch/arch_arm.py | 6 +++++- .../debugger/qdb/branch_predictor/branch_predictor_arm.py | 2 +- qiling/debugger/qdb/render/render_arm.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/qiling/debugger/qdb/arch/arch_arm.py b/qiling/debugger/qdb/arch/arch_arm.py index cbf63c2a..72a2979d 100644 --- a/qiling/debugger/qdb/arch/arch_arm.py +++ b/qiling/debugger/qdb/arch/arch_arm.py @@ -3,12 +3,14 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from typing import Dict, Optional +from typing import ClassVar, Dict, Optional from .arch import Arch class ArchARM(Arch): + _flags_reg: ClassVar[str] = 'cpsr' + def __init__(self) -> None: regs = ( 'r0', 'r1', 'r2', 'r3', @@ -134,6 +136,8 @@ class ArchARM(Arch): class ArchCORTEX_M(ArchARM): + _flags_reg: ClassVar[str] = 'xpsr' + def __init__(self): super().__init__() diff --git a/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py b/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py index ea5dde0e..eda5163a 100644 --- a/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py +++ b/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py @@ -31,7 +31,7 @@ class BranchPredictorARM(BranchPredictor, ArchARM): """Get flags map of CPSR. """ - cpsr = self.read_reg('cpsr') + cpsr = self.read_reg(self._flags_reg) return ( (cpsr & (0b1 << 28)) != 0, # V, overflow flag diff --git a/qiling/debugger/qdb/render/render_arm.py b/qiling/debugger/qdb/render/render_arm.py index 5f5adb50..0e042f7f 100644 --- a/qiling/debugger/qdb/render/render_arm.py +++ b/qiling/debugger/qdb/render/render_arm.py @@ -15,7 +15,7 @@ class ContextRenderARM(ContextRender, ArchARM): """ def print_mode_info(self) -> None: - cpsr = self.read_reg('cpsr') + cpsr = self.read_reg(self._flags_reg) flags = ArchARM.get_flags(cpsr) mode = ArchARM.get_mode(cpsr) From 95d7b8f6d9087727219231660f7bc658c7cc2a2f Mon Sep 17 00:00:00 2001 From: elicn Date: Sat, 21 Jun 2025 22:39:37 +0300 Subject: [PATCH 074/138] Generalize code to meet both Cortex A and M --- .../qdb/branch_predictor/branch_predictor_arm.py | 12 ++++++------ qiling/debugger/qdb/render/render_arm.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py b/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py index eda5163a..f3cd0529 100644 --- a/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py +++ b/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py @@ -27,8 +27,8 @@ class BranchPredictorARM(BranchPredictor, ArchARM): stop = 'udf' - def get_cpsr(self) -> Tuple[bool, bool, bool, bool]: - """Get flags map of CPSR. + def get_cond_flags(self) -> Tuple[bool, bool, bool, bool]: + """Get condition status flags from CPSR / xPSR. """ cpsr = self.read_reg(self._flags_reg) @@ -122,9 +122,9 @@ class BranchPredictorARM(BranchPredictor, ArchARM): def __is_taken(cc: int) -> Tuple[bool, Tuple[bool, ...]]: pred = predicate[cc] - cpsr = self.get_cpsr() + flags = self.get_cond_flags() - return pred(*cpsr), cpsr + return pred(*flags), flags # conditions predicate selector predicate: Dict[int, Callable[..., bool]] = { @@ -215,13 +215,13 @@ class BranchPredictorARM(BranchPredictor, ArchARM): where = __parse_op(operands[1], **msize[suffix]) elif iname in binop: - going, cpsr = __is_taken(insn.cc) + going, flags = __is_taken(insn.cc) if going: operator = binop[iname] op1 = __parse_op(operands[1]) op2 = __parse_op(operands[2]) - carry = int(cpsr[1]) + carry = int(flags[1]) where = (op1 and op2) and operator(op1, op2, carry) diff --git a/qiling/debugger/qdb/render/render_arm.py b/qiling/debugger/qdb/render/render_arm.py index 0e042f7f..f08e39fa 100644 --- a/qiling/debugger/qdb/render/render_arm.py +++ b/qiling/debugger/qdb/render/render_arm.py @@ -3,7 +3,7 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from typing import Iterator, Optional +from typing import Iterator from .render import Render, ContextRender from ..arch import ArchARM, ArchCORTEX_M From d5b51dd7247fa82f49ab01301dad9b703625640e Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Sun, 22 Jun 2025 23:57:28 +0000 Subject: [PATCH 075/138] Remove dup'd comment, locally tested fix for error during ELFTest suite --- qiling/os/posix/syscall/epoll.py | 4 ---- tests/test_elf.py | 5 ++--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 50b70c6c..46911678 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -88,7 +88,6 @@ class QlEpollObj: """Test whether a specific fd is already being watched by this epoll instance. """ - return fd in self.fds @@ -168,9 +167,6 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): epolls_list = [fobj for fobj in ql.os.fd if isinstance(fobj, QlEpollObj)] try: - # Necessary to iterate over all possible qiling fds to determine if we have a chain of more - # than five epolls monitoring each other This may be removed in the future if the QlOsLinux - # class had a separate field reserved for tracking epoll objects. check_epoll_depth(ql.os.fd) except RecursionError: return -ELOOP diff --git a/tests/test_elf.py b/tests/test_elf.py index 3a5af4cd..6afd4974 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -783,8 +783,7 @@ class ELFTest(unittest.TestCase): rootfs = "../examples/rootfs/x8664_linux" argv = r"../examples/rootfs/x8664_linux/bin/x8664_linux_epoll_0".split() ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) - - #ql.os.stdin = pipe.SimpleInStream(0) + ql.os.stdin = pipe.SimpleInStream(0) ql.os.stdin.write(b'echo\n') ql.os.stdin.write(b'stop\n') # signal to exit gracefully ql.run() @@ -830,7 +829,7 @@ class ELFTest(unittest.TestCase): ql.run() - self.assertIn(b'hello world', ql.os.stdout.read()) + self.assertIn(b'hello world', ql.os.stdout.read(200)) # 200 is arbitrary--"good enough" for this task del ql From aefef5ffafef4a49bc9feec4a81c22ce9a5ef244 Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Mon, 23 Jun 2025 23:40:35 +0000 Subject: [PATCH 076/138] Disable test case, for now --- tests/test_elf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_elf.py b/tests/test_elf.py index 6afd4974..52db6719 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -790,7 +790,7 @@ class ELFTest(unittest.TestCase): self.assertIn(b'echo\n', ql.os.stdout.read()) del ql - + @unittest.skip("See comment in https://github.com/qilingframework/qiling/pull/1558") def test_elf_linux_x8664_epoll_server(self): # This tests a simple server that uses epoll to wait for data, then prints it out. It has # been modified to exit after data has been received; instead of a typical server operation From 58487ee10e7bd8f40cb9b4b4d22e6c36682263b2 Mon Sep 17 00:00:00 2001 From: elicn Date: Tue, 24 Jun 2025 17:35:13 +0300 Subject: [PATCH 077/138] Add missing CortexM base class --- qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py b/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py index f3cd0529..553c5ed7 100644 --- a/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py +++ b/qiling/debugger/qdb/branch_predictor/branch_predictor_arm.py @@ -17,7 +17,7 @@ from capstone.arm_const import ( from unicorn.arm_const import UC_ARM_REG_PC from .branch_predictor import BranchPredictor, Prophecy -from ..arch import ArchARM +from ..arch import ArchARM, ArchCORTEX_M from ..misc import InvalidInsn @@ -261,6 +261,6 @@ class BranchPredictorARM(BranchPredictor, ArchARM): return Prophecy(going, where) -class BranchPredictorCORTEX_M(BranchPredictorARM): +class BranchPredictorCORTEX_M(BranchPredictorARM, ArchCORTEX_M): """Branch Predictor for ARM Cortex-M. """ From 327f475ba3d7fb9bed89c3c516b3ad1c8bd10d86 Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 6 Jul 2025 18:23:07 +0300 Subject: [PATCH 078/138] Decouple BLOB entry point and load address --- examples/uboot_bin.ql | 1 + qiling/loader/blob.py | 5 ++++- qiling/os/blob/blob.py | 12 ++++++++---- qiling/os/os.py | 1 + tests/profiles/uboot_bin.ql | 1 + 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/examples/uboot_bin.ql b/examples/uboot_bin.ql index b7f7216c..c33a7d23 100644 --- a/examples/uboot_bin.ql +++ b/examples/uboot_bin.ql @@ -1,5 +1,6 @@ [CODE] ram_size = 0xa00000 +load_address = 0x80800000 entry_point = 0x80800000 heap_size = 0x300000 diff --git a/qiling/loader/blob.py b/qiling/loader/blob.py index b8831a55..f17b80a9 100644 --- a/qiling/loader/blob.py +++ b/qiling/loader/blob.py @@ -15,7 +15,8 @@ class QlLoaderBLOB(QlLoader): self.load_address = 0 def run(self): - self.load_address = self.ql.os.entry_point # for consistency + self.load_address = self.ql.os.load_address + self.entry_point = self.ql.os.entry_point code_begins = self.load_address code_size = self.ql.os.code_ram_size @@ -28,8 +29,10 @@ class QlLoaderBLOB(QlLoader): self.images.append(Image(code_begins, code_ends, 'blob_code')) # FIXME: heap starts above end of ram?? + # FIXME: heap should be allocated by OS, not loader heap_base = code_ends heap_size = int(self.ql.os.profile.get("CODE", "heap_size"), 16) self.ql.os.heap = QlMemoryHeap(self.ql, heap_base, heap_base + heap_size) + # FIXME: stack pointer should be a configurable profile setting self.ql.arch.regs.arch_sp = code_ends - 0x1000 diff --git a/qiling/os/blob/blob.py b/qiling/os/blob/blob.py index 02e6f94d..e4a02256 100644 --- a/qiling/os/blob/blob.py +++ b/qiling/os/blob/blob.py @@ -9,6 +9,7 @@ from qiling.const import QL_ARCH, QL_OS from qiling.os.fcall import QlFunctionCall from qiling.os.os import QlOs + class QlOsBlob(QlOs): """ QlOsBlob for bare barines. @@ -21,7 +22,7 @@ class QlOsBlob(QlOs): type = QL_OS.BLOB def __init__(self, ql: Qiling): - super(QlOsBlob, self).__init__(ql) + super().__init__(ql) self.ql = ql @@ -39,11 +40,14 @@ class QlOsBlob(QlOs): self.fcall = QlFunctionCall(ql, cc) def run(self): - if self.ql.entry_point: + # if entry point was set explicitly, override the default one + if self.ql.entry_point is not None: self.entry_point = self.ql.entry_point - self.exit_point = self.ql.loader.load_address + len(self.ql.code) - if self.ql.exit_point: + self.exit_point = self.load_address + len(self.ql.code) + + # if exit point was set explicitly, override the default one + if self.ql.exit_point is not None: self.exit_point = self.ql.exit_point self.ql.emu_start(self.entry_point, self.exit_point, self.ql.timeout, self.ql.count) diff --git a/qiling/os/os.py b/qiling/os/os.py index 636e089c..dd9f3856 100644 --- a/qiling/os/os.py +++ b/qiling/os/os.py @@ -89,6 +89,7 @@ class QlOs: if self.ql.code: # this shellcode entrypoint does not work for windows # windows shellcode entry point will comes from pe loader + self.load_address = self.profile.getint('CODE', 'load_address') self.entry_point = self.profile.getint('CODE', 'entry_point') self.code_ram_size = self.profile.getint('CODE', 'ram_size') diff --git a/tests/profiles/uboot_bin.ql b/tests/profiles/uboot_bin.ql index b7f7216c..c33a7d23 100644 --- a/tests/profiles/uboot_bin.ql +++ b/tests/profiles/uboot_bin.ql @@ -1,5 +1,6 @@ [CODE] ram_size = 0xa00000 +load_address = 0x80800000 entry_point = 0x80800000 heap_size = 0x300000 From d2b4867f21c9c913cce38da1248a61834132d9b7 Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 6 Jul 2025 18:23:36 +0300 Subject: [PATCH 079/138] Adjust test and example --- examples/hello_arm_uboot.py | 104 ++++++++++++++++++++---------------- tests/test_blob.py | 40 +++++++++----- 2 files changed, 87 insertions(+), 57 deletions(-) diff --git a/examples/hello_arm_uboot.py b/examples/hello_arm_uboot.py index 9544fe0e..f97ff6ef 100644 --- a/examples/hello_arm_uboot.py +++ b/examples/hello_arm_uboot.py @@ -8,68 +8,82 @@ sys.path.append("..") from qiling.core import Qiling from qiling.const import QL_ARCH, QL_OS, QL_VERBOSE -from qiling.os.const import STRING +from qiling.os.const import STRING, SIZE_T, POINTER -def get_kaimendaji_password(): - def my_getenv(ql: Qiling): - env = { - "ID" : b"000000000000000", - "ethaddr" : b"11:22:33:44:55:66" - } +def my_getenv(ql: Qiling): + env = { + "ID" : b"000000000000000", + "ethaddr" : b"11:22:33:44:55:66" + } - params = ql.os.resolve_fcall_params({'key': STRING}) - value = env.get(params["key"], b"") + params = ql.os.resolve_fcall_params({'key': STRING}) + value = env.get(params["key"], b"") - value_addr = ql.os.heap.alloc(len(value)) - ql.mem.write(value_addr, value) + value_addr = ql.os.heap.alloc(len(value)) + ql.mem.write(value_addr, value) - ql.arch.regs.r0 = value_addr - ql.arch.regs.arch_pc = ql.arch.regs.lr + ql.arch.regs.r0 = value_addr + ql.arch.regs.arch_pc = ql.arch.regs.lr - def get_password(ql: Qiling): - password_raw = ql.mem.read(ql.arch.regs.r0, ql.arch.regs.r2) - password = '' - for item in password_raw: - if 0 <= item <= 9: - password += chr(item + 48) - else: - password += chr(item + 87) +def get_password(ql: Qiling): + # we land on a memcmp call, where the real password is being compared to + # the one provided by the user. we can follow the arguments to read the + # real password - print("The password is: %s" % password) + params = ql.os.resolve_fcall_params({ + 'ptr1': POINTER, # points to real password + 'ptr2': POINTER, # points to user provided password + 'size': SIZE_T # comparison length + }) - def partial_run_init(ql: Qiling): - # argv prepare - ql.arch.regs.arch_sp -= 0x30 - arg0_ptr = ql.arch.regs.arch_sp - ql.mem.write(arg0_ptr, b"kaimendaji") + ptr1 = params['ptr1'] + size = params['size'] - ql.arch.regs.arch_sp -= 0x10 - arg1_ptr = ql.arch.regs.arch_sp - ql.mem.write(arg1_ptr, b"000000") # arbitrary password + password_raw = ql.mem.read(ptr1, size) - ql.arch.regs.arch_sp -= 0x20 - argv_ptr = ql.arch.regs.arch_sp - ql.mem.write_ptr(argv_ptr, arg0_ptr) - ql.mem.write_ptr(argv_ptr + ql.arch.pointersize, arg1_ptr) + def __hex_digit(ch: int) -> str: + off = ord('0') if ch in range(10) else ord('a') - 10 - ql.arch.regs.r2 = 2 - ql.arch.regs.r3 = argv_ptr + return chr(ch + off) - with open("../examples/rootfs/blob/u-boot.bin.img", "rb") as f: - uboot_code = f.read() + # should be: "013f1f" + password = "".join(__hex_digit(ch) for ch in password_raw) - ql = Qiling(code=uboot_code[0x40:], archtype=QL_ARCH.ARM, ostype=QL_OS.BLOB, profile="uboot_bin.ql", verbose=QL_VERBOSE.OFF) + print(f'The password is: {password}') - image_base_addr = ql.loader.load_address - ql.hook_address(my_getenv, image_base_addr + 0x13AC0) - ql.hook_address(get_password, image_base_addr + 0x48634) - partial_run_init(ql) +def partial_run_init(ql: Qiling): + # argv prepare + ql.arch.regs.arch_sp -= 0x30 + arg0_ptr = ql.arch.regs.arch_sp + ql.mem.write(arg0_ptr, b"kaimendaji") - ql.run(image_base_addr + 0x486B4, image_base_addr + 0x48718) + ql.arch.regs.arch_sp -= 0x10 + arg1_ptr = ql.arch.regs.arch_sp + ql.mem.write(arg1_ptr, b"000000") # arbitrary password + + ql.arch.regs.arch_sp -= 0x20 + argv_ptr = ql.arch.regs.arch_sp + ql.mem.write_ptr(argv_ptr, arg0_ptr) + ql.mem.write_ptr(argv_ptr + ql.arch.pointersize, arg1_ptr) + + ql.arch.regs.r2 = 2 + ql.arch.regs.r3 = argv_ptr if __name__ == "__main__": - get_kaimendaji_password() + with open("../examples/rootfs/blob/u-boot.bin.img", "rb") as f: + uboot_code = f.read() + + ql = Qiling(code=uboot_code[0x40:], archtype=QL_ARCH.ARM, ostype=QL_OS.BLOB, profile="uboot_bin.ql", verbose=QL_VERBOSE.DEBUG) + + imgbase = ql.loader.images[0].base + + ql.hook_address(my_getenv, imgbase + 0x13AC0) + ql.hook_address(get_password, imgbase + 0x48634) + + partial_run_init(ql) + + ql.run(imgbase + 0x486B4, imgbase + 0x48718) diff --git a/tests/test_blob.py b/tests/test_blob.py index bc191dc1..33e35751 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -10,13 +10,17 @@ sys.path.append("..") from qiling.core import Qiling from qiling.const import QL_ARCH, QL_OS, QL_VERBOSE -from qiling.os.const import STRING +from qiling.os.const import STRING, POINTER, SIZE_T class BlobTest(unittest.TestCase): def test_uboot_arm(self): - def my_getenv(ql, *args, **kwargs): - env = {"ID": b"000000000000000", "ethaddr": b"11:22:33:44:55:66"} + def my_getenv(ql: Qiling): + env = { + "ID": b"000000000000000", + "ethaddr": b"11:22:33:44:55:66" + } + params = ql.os.resolve_fcall_params({'key': STRING}) value = env.get(params["key"], b"") @@ -26,12 +30,23 @@ class BlobTest(unittest.TestCase): ql.arch.regs.r0 = value_addr ql.arch.regs.arch_pc = ql.arch.regs.lr - def check_password(ql, *args, **kwargs): - passwd_output = ql.mem.read(ql.arch.regs.r0, ql.arch.regs.r2) - passwd_input = ql.mem.read(ql.arch.regs.r1, ql.arch.regs.r2) - self.assertEqual(passwd_output, passwd_input) + def check_password(ql: Qiling): + params = ql.os.resolve_fcall_params({ + 'ptr1': POINTER, # points to real password + 'ptr2': POINTER, # points to user provided password + 'size': SIZE_T # comparison length + }) - def partial_run_init(ql): + ptr1 = params['ptr1'] + ptr2 = params['ptr2'] + size = params['size'] + + real_password = ql.mem.read(ptr1, size) + user_password = ql.mem.read(ptr2, size) + + self.assertSequenceEqual(real_password, user_password, seq_type=bytearray) + + def partial_run_init(ql: Qiling): # argv prepare ql.arch.regs.arch_sp -= 0x30 arg0_ptr = ql.arch.regs.arch_sp @@ -56,13 +71,14 @@ class BlobTest(unittest.TestCase): ql = Qiling(code=uboot_code[0x40:], archtype=QL_ARCH.ARM, ostype=QL_OS.BLOB, profile="profiles/uboot_bin.ql", verbose=QL_VERBOSE.DEBUG) - image_base_addr = ql.loader.load_address - ql.hook_address(my_getenv, image_base_addr + 0x13AC0) - ql.hook_address(check_password, image_base_addr + 0x48634) + imgbase = ql.loader.images[0].base + + ql.hook_address(my_getenv, imgbase + 0x13AC0) + ql.hook_address(check_password, imgbase + 0x48634) partial_run_init(ql) - ql.run(image_base_addr + 0x486B4, image_base_addr + 0x48718) + ql.run(imgbase + 0x486B4, imgbase + 0x48718) del ql From dacc8e001201471fb1e4ff0d8dcfbb56c0e6cac0 Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 6 Jul 2025 18:24:01 +0300 Subject: [PATCH 080/138] Remove redundant BLOB case --- qiling/debugger/qdb/qdb.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/qiling/debugger/qdb/qdb.py b/qiling/debugger/qdb/qdb.py index ae942139..7182b46d 100644 --- a/qiling/debugger/qdb/qdb.py +++ b/qiling/debugger/qdb/qdb.py @@ -136,10 +136,7 @@ class QlQdb(Cmd, QlDebugger): with self.__set_temp(self.ql, 'verbose', QL_VERBOSE.DISABLED): self.ql.os.run() - if self.ql.os.type is QL_OS.BLOB: - self.ql.loader.entry_point = self.ql.loader.load_address - - elif init_hook: + if init_hook: for each_hook in init_hook: self.do_breakpoint(each_hook) From bb21658e32eeb1ceaa45894126282ffbb1ff3601 Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 6 Jul 2025 18:25:00 +0300 Subject: [PATCH 081/138] Fix QDB crash on allocation boundaries --- qiling/debugger/qdb/context.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiling/debugger/qdb/context.py b/qiling/debugger/qdb/context.py index 34919754..344f7563 100644 --- a/qiling/debugger/qdb/context.py +++ b/qiling/debugger/qdb/context.py @@ -57,7 +57,7 @@ class Context: """Helper function for disassembling. """ - insn_bytes = self.read_insn(address) + insn_bytes = self.read_insn(address) or b'' insn = None if insn_bytes: @@ -75,7 +75,7 @@ class Context: A tuple of: instruction address, size, mnemonic and operands """ - insn_bytes = self.read_insn(address) + insn_bytes = self.read_insn(address) or b'' insn = None if insn_bytes: From e68c923abd685aeeb5212bff6ef204a0b6ad7c10 Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 6 Jul 2025 18:42:59 +0300 Subject: [PATCH 082/138] Patch profiles to meet new required key --- qiling/profiles/linux.ql | 1 + qiling/profiles/windows.ql | 1 + 2 files changed, 2 insertions(+) diff --git a/qiling/profiles/linux.ql b/qiling/profiles/linux.ql index eac82348..de828a32 100644 --- a/qiling/profiles/linux.ql +++ b/qiling/profiles/linux.ql @@ -1,6 +1,7 @@ [CODE] # ram_size 0xa00000 is 10MB ram_size = 0xa00000 +load_address = 0x1000000 entry_point = 0x1000000 diff --git a/qiling/profiles/windows.ql b/qiling/profiles/windows.ql index 15cc2f39..ac2cc668 100644 --- a/qiling/profiles/windows.ql +++ b/qiling/profiles/windows.ql @@ -23,6 +23,7 @@ KI_USER_SHARED_DATA = 0x7ffe0000 [CODE] # ram_size 0xa00000 is 10MB ram_size = 0xa00000 +load_address = 0x1000000 entry_point = 0x1000000 [KERNEL] From 7088d22a73eb165ac196cffdefab9ed68099de79 Mon Sep 17 00:00:00 2001 From: elicn Date: Sun, 6 Jul 2025 18:44:59 +0300 Subject: [PATCH 083/138] Allow tests to import relatively --- tests/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/__init__.py diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b From 13e4569e276fdd9d1fe8f1394f9ef57b82a54083 Mon Sep 17 00:00:00 2001 From: elicn Date: Tue, 8 Jul 2025 18:08:33 +0300 Subject: [PATCH 084/138] Fix gdb regs reference for Cortex-M --- .../gdb/xml/{arm => cortex_m}/arm-m-profile.xml | 8 +++++++- qiling/debugger/gdb/xml/cortex_m/target.xml | 12 ++++++++++++ qiling/debugger/gdb/xmlregs.py | 9 ++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) rename qiling/debugger/gdb/xml/{arm => cortex_m}/arm-m-profile.xml (81%) create mode 100644 qiling/debugger/gdb/xml/cortex_m/target.xml diff --git a/qiling/debugger/gdb/xml/arm/arm-m-profile.xml b/qiling/debugger/gdb/xml/cortex_m/arm-m-profile.xml similarity index 81% rename from qiling/debugger/gdb/xml/arm/arm-m-profile.xml rename to qiling/debugger/gdb/xml/cortex_m/arm-m-profile.xml index f0584a20..a0707150 100644 --- a/qiling/debugger/gdb/xml/arm/arm-m-profile.xml +++ b/qiling/debugger/gdb/xml/cortex_m/arm-m-profile.xml @@ -25,4 +25,10 @@ - + + + + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/cortex_m/target.xml b/qiling/debugger/gdb/xml/cortex_m/target.xml new file mode 100644 index 00000000..63591239 --- /dev/null +++ b/qiling/debugger/gdb/xml/cortex_m/target.xml @@ -0,0 +1,12 @@ + + + + + + armv7-m + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xmlregs.py b/qiling/debugger/gdb/xmlregs.py index f569cd22..983ac6ef 100644 --- a/qiling/debugger/gdb/xmlregs.py +++ b/qiling/debugger/gdb/xmlregs.py @@ -13,14 +13,21 @@ from qiling.arch.arm_const import ( reg_map_q as arm_regs_q, reg_map_s as arm_regs_s ) + +from qiling.arch.cortex_m_const import ( + reg_map as conretx_m_regs +) + from qiling.arch.arm64_const import ( reg_map as arm64_regs, reg_map_v as arm64_regs_v, reg_map_fp as arm64_reg_map_fp ) + from qiling.arch.mips_const import ( reg_map as mips_regs_gpr ) + from qiling.arch.x86_const import ( reg_map_32 as x86_regs_32, reg_map_64 as x86_regs_64, @@ -133,7 +140,7 @@ class QlGdbFeatures: QL_ARCH.X86: dict(**x86_regs_32, **x86_regs_misc, **x86_regs_cr, **x86_regs_st, **x86_regs_xmm), QL_ARCH.X8664: dict(**x86_regs_64, **x86_regs_misc, **x86_regs_cr, **x86_regs_st, **x86_regs_xmm, **x86_regs_ymm), QL_ARCH.ARM: dict(**arm_regs, **arm_regs_vfp, **arm_regs_q, **arm_regs_s), - QL_ARCH.CORTEX_M: arm_regs, + QL_ARCH.CORTEX_M: dict(**conretx_m_regs), QL_ARCH.ARM64: dict(**arm64_regs, **arm64_regs_v, **arm64_reg_map_fp), QL_ARCH.MIPS: dict(**mips_regs_gpr) }[archtype] From 66f0fa366964b25dcd21f135aa8554fc9599d33d Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 9 Jul 2025 13:11:14 +0300 Subject: [PATCH 085/138] Typo fix --- qiling/debugger/gdb/xmlregs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiling/debugger/gdb/xmlregs.py b/qiling/debugger/gdb/xmlregs.py index 983ac6ef..1477d693 100644 --- a/qiling/debugger/gdb/xmlregs.py +++ b/qiling/debugger/gdb/xmlregs.py @@ -15,7 +15,7 @@ from qiling.arch.arm_const import ( ) from qiling.arch.cortex_m_const import ( - reg_map as conretx_m_regs + reg_map as coretx_m_regs ) from qiling.arch.arm64_const import ( @@ -140,7 +140,7 @@ class QlGdbFeatures: QL_ARCH.X86: dict(**x86_regs_32, **x86_regs_misc, **x86_regs_cr, **x86_regs_st, **x86_regs_xmm), QL_ARCH.X8664: dict(**x86_regs_64, **x86_regs_misc, **x86_regs_cr, **x86_regs_st, **x86_regs_xmm, **x86_regs_ymm), QL_ARCH.ARM: dict(**arm_regs, **arm_regs_vfp, **arm_regs_q, **arm_regs_s), - QL_ARCH.CORTEX_M: dict(**conretx_m_regs), + QL_ARCH.CORTEX_M: dict(**coretx_m_regs), QL_ARCH.ARM64: dict(**arm64_regs, **arm64_regs_v, **arm64_reg_map_fp), QL_ARCH.MIPS: dict(**mips_regs_gpr) }[archtype] From a2542f1dbca0c5761853a6a618cb95dd7cb7d174 Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 9 Jul 2025 14:50:11 +0300 Subject: [PATCH 086/138] Typo fix --- qiling/debugger/gdb/xmlregs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiling/debugger/gdb/xmlregs.py b/qiling/debugger/gdb/xmlregs.py index 1477d693..e1abf08d 100644 --- a/qiling/debugger/gdb/xmlregs.py +++ b/qiling/debugger/gdb/xmlregs.py @@ -15,7 +15,7 @@ from qiling.arch.arm_const import ( ) from qiling.arch.cortex_m_const import ( - reg_map as coretx_m_regs + reg_map as cortex_m_regs ) from qiling.arch.arm64_const import ( @@ -140,7 +140,7 @@ class QlGdbFeatures: QL_ARCH.X86: dict(**x86_regs_32, **x86_regs_misc, **x86_regs_cr, **x86_regs_st, **x86_regs_xmm), QL_ARCH.X8664: dict(**x86_regs_64, **x86_regs_misc, **x86_regs_cr, **x86_regs_st, **x86_regs_xmm, **x86_regs_ymm), QL_ARCH.ARM: dict(**arm_regs, **arm_regs_vfp, **arm_regs_q, **arm_regs_s), - QL_ARCH.CORTEX_M: dict(**coretx_m_regs), + QL_ARCH.CORTEX_M: dict(**cortex_m_regs), QL_ARCH.ARM64: dict(**arm64_regs, **arm64_regs_v, **arm64_reg_map_fp), QL_ARCH.MIPS: dict(**mips_regs_gpr) }[archtype] From d3cc47dc7ff5513c2b4d14695c2b4aa52a3283a1 Mon Sep 17 00:00:00 2001 From: technikelly <11539105+technikelly@users.noreply.github.com> Date: Tue, 12 Aug 2025 10:55:50 -0400 Subject: [PATCH 087/138] adding support for raw binary blobs --- examples/blob_raw.ql | 4 ++ examples/hello_arm_blob_raw.py | 81 +++++++++++++++++++++++++++ examples/src/blob/Makefile | 57 +++++++++++++++++++ examples/src/blob/example_raw.c | 61 ++++++++++++++++++++ examples/src/blob/linker.ld | 45 +++++++++++++++ qiling/loader/blob.py | 44 +++++++++------ qiling/os/blob/blob.py | 7 ++- tests/profiles/blob_raw.ql | 4 ++ tests/test_blob.py | 99 +++++++++++++++++++++++++++++++++ 9 files changed, 382 insertions(+), 20 deletions(-) create mode 100644 examples/blob_raw.ql create mode 100644 examples/hello_arm_blob_raw.py create mode 100644 examples/src/blob/Makefile create mode 100644 examples/src/blob/example_raw.c create mode 100644 examples/src/blob/linker.ld create mode 100644 tests/profiles/blob_raw.ql diff --git a/examples/blob_raw.ql b/examples/blob_raw.ql new file mode 100644 index 00000000..164219a9 --- /dev/null +++ b/examples/blob_raw.ql @@ -0,0 +1,4 @@ +[BLOB_RAW] +load_address = 0x10000000 +image_size = 0xbc +image_name = example_raw.bin \ No newline at end of file diff --git a/examples/hello_arm_blob_raw.py b/examples/hello_arm_blob_raw.py new file mode 100644 index 00000000..6e44d6bf --- /dev/null +++ b/examples/hello_arm_blob_raw.py @@ -0,0 +1,81 @@ +############################################################################## +# Added example for raw binary blob +# Kelly Patterson - Cisco Talos +# Copyright (C) 2025 Cisco Systems Inc +############################################################################## +from qiling import Qiling +from qiling.const import QL_ARCH, QL_OS, QL_VERBOSE +from qiling.extensions.coverage import utils as cov_utils + +BASE_ADDRESS = 0x10000000 +CHECKSUM_FUNC_ADDR = BASE_ADDRESS + 0x8 +END_ADDRESS = 0x100000ba +DATA_ADDR = 0xa0000000 # Arbitrary address for data +STACK_ADDR = 0xb0000000 # Arbitrary address for stack + +# Python implementation of the checksum function being emulated +def checksum_function(input_data_buffer: bytes): + expected_checksum_python = 0 + input_data_len = len(input_data_buffer) + if input_data_len >= 1 and input_data_buffer[0] == 0xDE: # MAGIC_VALUE_1 + for i in range(min(input_data_len, 4)): + expected_checksum_python += input_data_buffer[i] + expected_checksum_python += 0x10 + elif input_data_len >= 2 and input_data_buffer[1] == 0xAD: # MAGIC_VALUE_2 + for i in range(input_data_len): + expected_checksum_python ^= input_data_buffer[i] + expected_checksum_python += 0x20 + else: + for i in range(input_data_len): + expected_checksum_python += input_data_buffer[i] + expected_checksum_python &= 0xFF # Ensure it's a single byte + +def unmapped_handler(ql, type, addr, size, value): + + print(f"Unmapped Memory R/W, trying to access {hex(size)} bytes at {hex(addr)} from {hex(ql.arch.regs.pc)}") + +def emulate_checksum_function(input_data_buffer: bytes): + print(f"\n--- Testing with input: {input_data_buffer.hex()} ---") + + ql = Qiling(archtype=QL_ARCH.ARM, ostype=QL_OS.BLOB, profile="blob_raw.ql", verbose=QL_VERBOSE.DEBUG, thumb=True) + + input_data_len = len(input_data_buffer) + + # Map memory for the binary, data and stack + ql.mem.map(BASE_ADDRESS, 0x10000) + ql.mem.map(STACK_ADDR, 0x2000) + ql.mem.map(DATA_ADDR, ql.mem.align_up(input_data_len + 0x100)) # Map enough space for data + + # Write the binary into memory + ql.mem.write(BASE_ADDRESS, open("rootfs/blob/example_raw.bin", "rb").read()) + + # Write input data + ql.mem.write(DATA_ADDR, input_data_buffer) + + # Set up the stack pointer + ql.arch.regs.sp = STACK_ADDR + 0x2000 - 4 + # Set up argument registers + ql.arch.regs.r0 = DATA_ADDR + ql.arch.regs.r1 = input_data_len + + # Set the program counter to the function's entry point + ql.arch.regs.pc = CHECKSUM_FUNC_ADDR + + # Set the return address (LR) to a dummy address. + ql.arch.regs.lr = 0xbebebebe + + ql.hook_mem_unmapped(unmapped_handler) + #ql.debugger="gdb:127.0.0.1:9999" + + # Start emulation + print(f"Starting emulation at PC: {hex(ql.arch.regs.pc)}") + try: + ql.run(begin=CHECKSUM_FUNC_ADDR, end=END_ADDRESS) + except Exception as e: + print(f"Emulation error: {e}") + + print(f"Emulated checksum: {hex(ql.arch.regs.r0)}") + +if __name__ == "__main__": + data = b"\x01\x02\x03\x04\x05" # Example input data + emulate_checksum_function(data) \ No newline at end of file diff --git a/examples/src/blob/Makefile b/examples/src/blob/Makefile new file mode 100644 index 00000000..7f4252f7 --- /dev/null +++ b/examples/src/blob/Makefile @@ -0,0 +1,57 @@ +############################################################################## +# Added example for raw binary blob +# Kelly Patterson - Cisco Talos +# Copyright (C) 2025 Cisco Systems Inc +############################################################################## +# Makefile for Bare-Metal ARM Hash Calculator + +# --- Toolchain Definitions --- +TOOLCHAIN_PREFIX = arm-none-eabi + +# Compiler, Linker, and Objcopy executables +CC = $(TOOLCHAIN_PREFIX)-gcc +LD = $(TOOLCHAIN_PREFIX)-gcc +OBJCOPY = $(TOOLCHAIN_PREFIX)-objcopy + +# --- Source and Output Files --- +SRCS = example_raw.c +OBJS = $(SRCS:.c=.o) # Convert .c to .o +ELF = example_raw.elf +BIN = example_raw.bin + +# --- Linker Script --- +LDSCRIPT = linker.ld + +# --- Compiler Flags --- +CFLAGS = -c -O0 -mcpu=cortex-a7 -mthumb -ffreestanding -nostdlib + +# --- Linker Flags --- +LDFLAGS = -T $(LDSCRIPT) -nostdlib + +# --- Objcopy Flags --- +OBJCOPYFLAGS = -O binary + +# --- Default Target --- +.PHONY: all clean + +all: $(BIN) + +# Rule to build the raw binary (.bin) from the ELF file +$(BIN): $(ELF) + $(OBJCOPY) $(OBJCOPYFLAGS) $< $@ + @echo "Successfully created $(BIN)" + +# Rule to link the object file into an ELF executable +$(ELF): $(OBJS) $(LDSCRIPT) + $(LD) $(LDFLAGS) $(OBJS) -o $@ + @echo "Successfully linked $(ELF)" + +# Rule to compile the C source file into an object file +%.o: %.c + $(CC) $(CFLAGS) $< -o $@ + @echo "Successfully compiled $<" + +# --- Clean Rule --- +clean: + rm -f $(OBJS) $(ELF) $(BIN) + @echo "Cleaned build artifacts." diff --git a/examples/src/blob/example_raw.c b/examples/src/blob/example_raw.c new file mode 100644 index 00000000..5ac71cb3 --- /dev/null +++ b/examples/src/blob/example_raw.c @@ -0,0 +1,61 @@ + /* + * Added example for raw binary blob + * Kelly Patterson - Cisco Talos + * Copyright (C) 2025 Cisco Systems Inc + * + */ +// example_raw.c + +// Define some magic values +#define MAGIC_VALUE_1 0xDE +#define MAGIC_VALUE_2 0xAD + +// This function calculates a checksum with branches based on input data +// It takes a pointer to data and its length +// Returns the checksum (unsigned char to fit in a byte) +unsigned char calculate_checksum(const unsigned char *data, unsigned int length) { + unsigned char checksum = 0; + + // Branch 1: Check for MAGIC_VALUE_1 at the start + if (length >= 1 && data[0] == MAGIC_VALUE_1) { + // If first byte is MAGIC_VALUE_1, do a simple sum of first 4 bytes + // (or up to length if less than 4) + for (unsigned int i = 0; i < length && i < 4; i++) { + checksum += data[i]; + } + // Add a fixed offset to make this path distinct + checksum += 0x10; + } + // Branch 2: Check for MAGIC_VALUE_2 at the second byte + else if (length >= 2 && data[1] == MAGIC_VALUE_2) { + // If second byte is MAGIC_VALUE_2, do a XOR sum of all bytes + for (unsigned int i = 0; i < length; i++) { + checksum ^= data[i]; + } + // Add a fixed offset to make this path distinct + checksum += 0x20; + } + // Default Branch: Standard byte sum checksum + else { + for (unsigned int i = 0; i < length; i++) { + checksum += data[i]; + } + } + + return checksum; +} + +// Minimal entry point for bare-metal. +// This function will not be called directly during Qiling emulation, +// but it's needed for the linker to have an entry point. +__attribute__((section(".text.startup"))) +void _start() { + // In a real bare-metal application, this would initialize hardware, + // set up stacks, etc. For this example, it's just a placeholder. + // We'll call calculate_checksum directly from our Qiling script. + + while (1) { + // Do nothing, or perhaps put the CPU to sleep + asm volatile ("wfi"); // Wait For Interrupt (ARM instruction) + } +} \ No newline at end of file diff --git a/examples/src/blob/linker.ld b/examples/src/blob/linker.ld new file mode 100644 index 00000000..ea57d85e --- /dev/null +++ b/examples/src/blob/linker.ld @@ -0,0 +1,45 @@ +/* linker.ld */ + /* + * Added example for raw binary blob + * Kelly Patterson - Cisco Talos + * Copyright (C) 2025 Cisco Systems Inc + * + */ + +ENTRY(_start) /* Define the entry point of our program */ + +/* Define memory regions - simple RAM region for this example */ +MEMORY +{ + ram (rwx) : ORIGIN = 0x10000000, LENGTH = 64K /* 64KB of RAM for our program */ +} + +SECTIONS +{ + /* Define the start of our program in memory. + */ + . = 0x10000000; + + .text : { + KEEP(*(.text.startup)) /* Keep the _start function */ + *(.text) /* All other code */ + *(.text.*) + *(.rodata) /* Read-only data */ + *(.rodata.*) + . = ALIGN(4); + } > ram /* Place .text section in the 'ram' region */ + + .data : { + . = ALIGN(4); + *(.data) /* Initialized data */ + *(.data.*) + . = ALIGN(4); + } > ram + + .bss : { + . = ALIGN(4); + *(.bss) + *(.bss.*) + . = ALIGN(4); + } > ram +} \ No newline at end of file diff --git a/qiling/loader/blob.py b/qiling/loader/blob.py index f17b80a9..96168538 100644 --- a/qiling/loader/blob.py +++ b/qiling/loader/blob.py @@ -2,6 +2,9 @@ # # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +# Added support for raw binary blob emulation +# Kelly Patterson - Cisco Talos +# Copyright (C) 2025 Cisco Systems Inc from qiling import Qiling from qiling.loader.loader import QlLoader, Image @@ -12,27 +15,32 @@ class QlLoaderBLOB(QlLoader): def __init__(self, ql: Qiling): super().__init__(ql) - self.load_address = 0 - def run(self): - self.load_address = self.ql.os.load_address - self.entry_point = self.ql.os.entry_point + if self.ql.os.profile.has_section("BLOB_RAW"): + # For raw binary blobs, user will handle memory mapping + self.load_address = int(self.ql.os.profile.get("BLOB_RAW", "load_address"), 16) + image_size = int(self.ql.os.profile.get("BLOB_RAW", "image_size"), 16) + image_name = self.ql.os.profile.get("BLOB_RAW", "image_name", fallback="blob.raw") + self.images.append(Image(self.load_address, self.load_address+image_size, image_name)) # used to collect coverage + else: + self.load_address = self.ql.os.load_address + self.entry_point = self.ql.os.entry_point - code_begins = self.load_address - code_size = self.ql.os.code_ram_size - code_ends = code_begins + code_size + code_begins = self.load_address + code_size = self.ql.os.code_ram_size + code_ends = code_begins + code_size - self.ql.mem.map(code_begins, code_size, info="[code]") - self.ql.mem.write(code_begins, self.ql.code) + self.ql.mem.map(code_begins, code_size, info="[code]") + self.ql.mem.write(code_begins, self.ql.code) - # allow image-related functionalities - self.images.append(Image(code_begins, code_ends, 'blob_code')) + # allow image-related functionalities + self.images.append(Image(code_begins, code_ends, 'blob_code')) - # FIXME: heap starts above end of ram?? - # FIXME: heap should be allocated by OS, not loader - heap_base = code_ends - heap_size = int(self.ql.os.profile.get("CODE", "heap_size"), 16) - self.ql.os.heap = QlMemoryHeap(self.ql, heap_base, heap_base + heap_size) + # FIXME: heap starts above end of ram?? + # FIXME: heap should be allocated by OS, not loader + heap_base = code_ends + heap_size = int(self.ql.os.profile.get("CODE", "heap_size"), 16) + self.ql.os.heap = QlMemoryHeap(self.ql, heap_base, heap_base + heap_size) - # FIXME: stack pointer should be a configurable profile setting - self.ql.arch.regs.arch_sp = code_ends - 0x1000 + # FIXME: stack pointer should be a configurable profile setting + self.ql.arch.regs.arch_sp = code_ends - 0x1000 diff --git a/qiling/os/blob/blob.py b/qiling/os/blob/blob.py index e4a02256..f2056e55 100644 --- a/qiling/os/blob/blob.py +++ b/qiling/os/blob/blob.py @@ -2,6 +2,9 @@ # # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +# Added support for raw binary blob emulation +# Kelly Patterson - Cisco Talos +# Copyright (C) 2025 Cisco Systems Inc from qiling import Qiling from qiling.cc import QlCC, intel, arm, mips, riscv, ppc @@ -44,10 +47,10 @@ class QlOsBlob(QlOs): if self.ql.entry_point is not None: self.entry_point = self.ql.entry_point - self.exit_point = self.load_address + len(self.ql.code) - # if exit point was set explicitly, override the default one if self.ql.exit_point is not None: self.exit_point = self.ql.exit_point + elif self.ql.code is not None: # self.ql.code might not always be provided + self.exit_point = self.load_address + len(self.ql.code) self.ql.emu_start(self.entry_point, self.exit_point, self.ql.timeout, self.ql.count) diff --git a/tests/profiles/blob_raw.ql b/tests/profiles/blob_raw.ql new file mode 100644 index 00000000..164219a9 --- /dev/null +++ b/tests/profiles/blob_raw.ql @@ -0,0 +1,4 @@ +[BLOB_RAW] +load_address = 0x10000000 +image_size = 0xbc +image_name = example_raw.bin \ No newline at end of file diff --git a/tests/test_blob.py b/tests/test_blob.py index 33e35751..d284bbac 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -2,6 +2,9 @@ # # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +# Added test for raw binary blob emulation +# Kelly Patterson - Cisco Talos +# Copyright (C) 2025 Cisco Systems Inc import unittest @@ -82,6 +85,102 @@ class BlobTest(unittest.TestCase): del ql + @unittest.skip("Temporarily disabled") + def test_blob_checksum_calculations(self): + def run_checksum_emu(input_data_buffer: bytes) -> int: + """ + Callable function that takes input data buffer and returns the checksum. + """ + BASE_ADDRESS = 0x10000000 + CHECKSUM_FUNC_ADDR = BASE_ADDRESS + 0x8 + END_ADDRESS = 0x100000ba + DATA_ADDR = 0xa0000000 + STACK_ADDR = 0xb0000000 + + ql = Qiling(archtype=QL_ARCH.ARM, ostype=QL_OS.BLOB, profile="profiles/blob_raw.ql", verbose=QL_VERBOSE.DEBUG, thumb=True) + + input_data_len = len(input_data_buffer) + + # Map memory for the binary, data and stack + ql.mem.map(BASE_ADDRESS, 0x10000) + ql.mem.map(STACK_ADDR, 0x2000) + ql.mem.map(DATA_ADDR, ql.mem.align_up(input_data_len + 0x100)) + + # Write the binary into memory + ql.mem.write(BASE_ADDRESS, open("../examples/rootfs/blob/example_raw.bin", "rb").read()) + + # Write input data + ql.mem.write(DATA_ADDR, input_data_buffer) + + # Set up registers + ql.arch.regs.sp = STACK_ADDR + 0x2000 - 4 + ql.arch.regs.r0 = DATA_ADDR + ql.arch.regs.r1 = input_data_len + ql.arch.regs.pc = CHECKSUM_FUNC_ADDR + ql.arch.regs.lr = 0xbebebebe + + ql.run(begin=CHECKSUM_FUNC_ADDR, end=END_ADDRESS) + result = ql.arch.regs.r0 + + return result + + def calculate_expected_checksum(input_data_buffer: bytes) -> int: + """ + Python implementation of the expected checksum calculation. + """ + input_data_len = len(input_data_buffer) + expected_checksum = 0 + + if input_data_len >= 1 and input_data_buffer[0] == 0xDE: # MAGIC_VALUE_1 + for i in range(min(input_data_len, 4)): + expected_checksum += input_data_buffer[i] + expected_checksum += 0x10 + elif input_data_len >= 2 and input_data_buffer[1] == 0xAD: # MAGIC_VALUE_2 + for i in range(input_data_len): + expected_checksum ^= input_data_buffer[i] + expected_checksum += 0x20 + else: + for i in range(input_data_len): + expected_checksum += input_data_buffer[i] + + return expected_checksum & 0xFF + + # Test cases with descriptions + test_cases = { + "default_path": { + "data": b"\x01\x02\x03\x04\x05", + "description": "Default path - simple sum of all bytes" + }, + "magic_value_1": { + "data": b"\xDE\x01\x02\x03\x04\x05", + "description": "Magic Value 1 path (0xDE at data[0]) - sum first 4 bytes + 0x10" + }, + "magic_value_2": { + "data": b"\x01\xAD\x02\x03\x04\x05", + "description": "Magic Value 2 path (0xAD at data[1]) - XOR all bytes + 0x20" + }, + "edge_magic1_short": { + "data": b"\xDE\x01", + "description": "Edge case: Magic Value 1, but short data (only 2 bytes)" + }, + "edge_magic2_too_short": { + "data": b"\xAD", + "description": "Edge case: Magic Value 2, but too short (fallback to default)" + }, + "both_magic_values": { + "data": b"\xDE\xAD\x01\x02", + "description": "Both magic values present, DE at [0] takes precedence" + } + } + + # Assertions with descriptions - directly call functions with test data + self.assertEqual(run_checksum_emu(test_cases["default_path"]["data"]), calculate_expected_checksum(test_cases["default_path"]["data"])) # Default path + self.assertEqual(run_checksum_emu(test_cases["magic_value_1"]["data"]), calculate_expected_checksum(test_cases["magic_value_1"]["data"])) # Magic Value 1 path + self.assertEqual(run_checksum_emu(test_cases["magic_value_2"]["data"]), calculate_expected_checksum(test_cases["magic_value_2"]["data"])) # Magic Value 2 path + self.assertEqual(run_checksum_emu(test_cases["edge_magic1_short"]["data"]), calculate_expected_checksum(test_cases["edge_magic1_short"]["data"])) # Edge case: Magic Value 1, short data + self.assertEqual(run_checksum_emu(test_cases["edge_magic2_too_short"]["data"]), calculate_expected_checksum(test_cases["edge_magic2_too_short"]["data"])) # Edge case: Magic Value 2, too short + self.assertEqual(run_checksum_emu(test_cases["both_magic_values"]["data"]), calculate_expected_checksum(test_cases["both_magic_values"]["data"])) # Both magic values, DE takes precedence + if __name__ == "__main__": unittest.main() From 46866adb17a47a893a311f7464191bfdae861203 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 15 Aug 2025 17:09:30 +0300 Subject: [PATCH 088/138] Redesign MMIO support to be pickle-friendly --- qiling/hw/hw.py | 165 ++++++++++++++++++++++++++++--------------- qiling/loader/mcu.py | 21 +++--- qiling/os/memory.py | 36 +++++++--- 3 files changed, 145 insertions(+), 77 deletions(-) diff --git a/qiling/hw/hw.py b/qiling/hw/hw.py index 33081a05..e558584e 100644 --- a/qiling/hw/hw.py +++ b/qiling/hw/hw.py @@ -3,24 +3,68 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import ctypes +from functools import cached_property +from typing import Any, Dict, List, Optional, Tuple -from qiling.core import Qiling +from qiling import Qiling from qiling.hw.peripheral import QlPeripheral from qiling.utils import ql_get_module_function from qiling.exception import QlErrorModuleFunctionNotFound +# should adhere to the QlMmioHandler interface, but not extend it directly to +# avoid potential pickling issues +class QlPripheralHandler: + def __init__(self, hwman: "QlHwManager", base: int, size: int, label: str) -> None: + self._hwman = hwman + self._base = base + self._size = size + self._label = label + + def __getstate__(self): + state = self.__dict__.copy() + del state['_hwman'] # remove non-pickleable reference + + return state + + @cached_property + def _mmio(self) -> bytearray: + """Get memory buffer used to back non-mapped hardware mmio regions. + """ + + return bytearray(self._size) + + def read(self, ql: Qiling, offset: int, size: int) -> int: + address = self._base + offset + hardware = self._hwman.find(address) + + if hardware: + return hardware.read(address - hardware.base, size) + + else: + ql.log.debug('[%s] read non-mapped hardware [%#010x]', self._label, address) + return int.from_bytes(self._mmio[offset:offset + size], byteorder='little') + + def write(self, ql: Qiling, offset: int, size: int, value: int) -> None: + address = self._base + offset + hardware = self._hwman.find(address) + + if hardware: + hardware.write(address - hardware.base, size, value) + + else: + ql.log.debug('[%s] write non-mapped hardware [%#010x] = %#010x', self._label, address, value) + self._mmio[offset:offset + size] = value.to_bytes(size, 'little') + + class QlHwManager: def __init__(self, ql: Qiling): self.ql = ql - self.entity = {} - self.region = {} + self.entity: Dict[str, QlPeripheral] = {} + self.region: Dict[str, List[Tuple[int, int]]] = {} - self.stepable = {} - - def create(self, label: str, struct: str=None, base: int=None, kwargs: dict={}) -> "QlPeripheral": + def create(self, label: str, struct: Optional[str] = None, base: Optional[int] = None, kwargs: Optional[Dict[str, Any]] = None) -> QlPeripheral: """ Create the peripheral accroding the label and envs. struct: Structure of the peripheral. Use defualt ql structure if not provide. @@ -30,39 +74,45 @@ class QlHwManager: if struct is None: struct, base, kwargs = self.load_env(label.upper()) - try: - - entity = ql_get_module_function('qiling.hw', struct)(self.ql, label, **kwargs) - - self.entity[label] = entity - if hasattr(entity, 'step'): - self.stepable[label] = entity + if kwargs is None: + kwargs = {} + try: + entity = ql_get_module_function('qiling.hw', struct)(self.ql, label, **kwargs) + + except QlErrorModuleFunctionNotFound: + self.ql.log.warning(f'could not create {struct}({label}): implementation not found') + + else: + assert isinstance(entity, QlPeripheral) + assert isinstance(base, int) + + self.entity[label] = entity self.region[label] = [(lbound + base, rbound + base) for (lbound, rbound) in entity.region] - return entity - except QlErrorModuleFunctionNotFound: - self.ql.log.debug(f'The {struct}({label}) has not been implemented') - def delete(self, label: str): + # FIXME: what should we do if struct is not implemented? is it OK to return None , or we fail? + + def delete(self, label: str) -> None: """ Remove the peripheral """ - if label in self.entity: - self.entity.pop(label) - self.region.pop(label) - if label in self.stepable: - self.stepable.pop(label) - def load_env(self, label: str): + if label in self.entity: + del self.entity[label] + + if label in self.region: + del self.region[label] + + def load_env(self, label: str) -> Tuple[str, int, Dict[str, Any]]: """ Get peripheral information (structure, base address, initialization list) from env. Args: label (str): Peripheral Label - + """ args = self.ql.env[label] - + return args['struct'], args['base'], args.get("kwargs", {}) def load_all(self): @@ -70,48 +120,30 @@ class QlHwManager: if args['type'] == 'peripheral': self.create(label.lower(), args['struct'], args['base'], args.get("kwargs", {})) - def find(self, address: int): + # TODO: this is wasteful. device mapping is known at creation time. at least we could cache lru entries + def find(self, address: int) -> Optional[QlPeripheral]: """ Find the peripheral at `address` """ - + for label in self.entity.keys(): for lbound, rbound in self.region[label]: if lbound <= address < rbound: return self.entity[label] + return None + def step(self): - """ Update all peripheral's state + """ Update all peripheral's state """ - for entity in self.stepable.values(): - entity.step() - def setup_mmio(self, begin, size, info=""): - mmio = ctypes.create_string_buffer(size) + for ent in self.entity.values(): + if hasattr(ent, 'step'): + ent.step() - def mmio_read_cb(ql, offset, size): - address = begin + offset - hardware = self.find(address) - - if hardware: - return hardware.read(address - hardware.base, size) - else: - ql.log.debug('%s Read non-mapped hardware [0x%08x]' % (info, address)) - - buf = ctypes.create_string_buffer(size) - ctypes.memmove(buf, ctypes.addressof(mmio) + offset, size) - return int.from_bytes(buf.raw, byteorder='little') + def setup_mmio(self, begin: int, size: int, info: str) -> None: + dev = QlPripheralHandler(self, begin, size, info) - def mmio_write_cb(ql, offset, size, value): - address = begin + offset - hardware = self.find(address) - - if hardware: - hardware.write(address - hardware.base, size, value) - else: - ql.log.debug('%s Write non-mapped hardware [0x%08x] = 0x%08x' % (info, address, value)) - ctypes.memmove(ctypes.addressof(mmio) + offset, (value).to_bytes(size, 'little'), size) - - self.ql.mem.map_mmio(begin, size, mmio_read_cb, mmio_write_cb, info=info) + self.ql.mem.map_mmio(begin, size, dev, info) def show_info(self): self.ql.log.info(f'{"Start":8s} {"End":8s} {"Label":8s} {"Class"}') @@ -131,8 +163,25 @@ class QlHwManager: return self.entity.get(key) def save(self): - return {label : entity.save() for label, entity in self.entity.items()} + return { + 'entity': {label: entity.save() for label, entity in self.entity.items()}, + 'region': self.region + } def restore(self, saved_state): - for label, data in saved_state.items(): + entity = saved_state['entity'] + assert isinstance(entity, dict) + + region = saved_state['region'] + assert isinstance(region, dict) + + for label, data in entity.items(): self.entity[label].restore(data) + + self.region = region + + # a dirty hack to rehydrate non-pickleable hwman + # a proper fix would require a deeper refactoring to how peripherals are created and managed + for ph in self.ql.mem.mmio_cbs.values(): + if isinstance(ph, QlPripheralHandler): + setattr(ph, '_hwman', self) diff --git a/qiling/loader/mcu.py b/qiling/loader/mcu.py index 8a91d633..204d17e4 100644 --- a/qiling/loader/mcu.py +++ b/qiling/loader/mcu.py @@ -114,20 +114,23 @@ class QlLoaderMCU(QlLoader): base = args['base'] self.ql.mem.map(base, size, info=f'[{name}]') - if memtype == 'remap': + # elif memtype == 'remap': + # size = args['size'] + # base = args['base'] + # alias = args['alias'] + # self.ql.hw.setup_remap(alias, base, size, info=f'[{name}]') + + elif memtype == 'mmio': size = args['size'] base = args['base'] - alias = args['alias'] - self.ql.hw.setup_remap(alias, base, size, info=f'[{name}]') + self.ql.hw.setup_mmio(base, size, name) - if memtype == 'mmio': - size = args['size'] - base = args['base'] - self.ql.hw.setup_mmio(base, size, info=f'[{name}]') - - if memtype == 'core': + elif memtype == 'core': self.ql.hw.create(name.lower()) + else: + self.ql.log.error(f'Unknown memory type "{memtype}" for {name}') + def run(self): self.load_profile() self.load_env() diff --git a/qiling/os/memory.py b/qiling/os/memory.py index 0939fc27..ec7aef19 100644 --- a/qiling/os/memory.py +++ b/qiling/os/memory.py @@ -6,7 +6,7 @@ import bisect import os import re -from typing import Any, Callable, Iterator, List, Mapping, Optional, Pattern, Sequence, Tuple, Union +from typing import Any, Callable, Dict, Iterator, List, Mapping, Optional, Pattern, Protocol, Sequence, Tuple, Union from unicorn import UC_PROT_NONE, UC_PROT_READ, UC_PROT_WRITE, UC_PROT_EXEC, UC_PROT_ALL @@ -20,6 +20,22 @@ MmioReadCallback = Callable[[Qiling, int, int], int] MmioWriteCallback = Callable[[Qiling, int, int, int], None] +class QlMmioHandler(Protocol): + """A simple MMIO handler boilerplate that can be used to implement memory mapped devices. + + This should be extended to implement mapped devices state machines. Note that the read and write + methods are optional, where their existance indicates whether the device supports the corresponding + operation. That is, an unimplemented method means the corresponding operation will be silently + dropped. + """ + + def read(self, ql: Qiling, offset: int, size: int) -> int: + ... + + def write(self, ql: Qiling, offset: int, size: int, value: int) -> None: + ... + + class QlMemoryManager: """ some ideas and code from: @@ -29,7 +45,7 @@ class QlMemoryManager: def __init__(self, ql: Qiling, pagesize: int = 0x1000): self.ql = ql self.map_info: List[MapInfoEntry] = [] - self.mmio_cbs = {} + self.mmio_cbs: Dict[Tuple[int, int], QlMmioHandler] = {} bit_stuff = { 64: (1 << 64) - 1, @@ -272,7 +288,7 @@ class QlMemoryManager: for lbound, ubound, perm, label, is_mmio in self.map_info: if is_mmio: - mem_dict['mmio'].append((lbound, ubound, perm, label, *self.mmio_cbs[(lbound, ubound)])) + mem_dict['mmio'].append((lbound, ubound, perm, label, self.mmio_cbs[(lbound, ubound)])) else: data = self.read(lbound, ubound - lbound) mem_dict['ram'].append((lbound, ubound, perm, label, bytes(data))) @@ -294,12 +310,12 @@ class QlMemoryManager: self.ql.log.debug(f'writing {len(data):#x} bytes at {lbound:#08x}') self.write(lbound, data) - for lbound, ubound, perms, label, read_cb, write_cb in mem_dict['mmio']: + for lbound, ubound, perms, label, handler in mem_dict['mmio']: self.ql.log.debug(f"restoring mmio range: {lbound:#08x} {ubound:#08x} {label}") size = ubound - lbound if not self.is_mapped(lbound, size): - self.map_mmio(lbound, size, read_cb, write_cb, info=label) + self.map_mmio(lbound, size, handler, label) def read(self, addr: int, size: int) -> bytearray: """Read bytes from memory. @@ -619,15 +635,15 @@ class QlMemoryManager: self.ql.uc.mem_map(addr, size, perms) self.add_mapinfo(addr, addr + size, perms, info or '[mapped]', is_mmio=False) - def map_mmio(self, addr: int, size: int, read_cb: Optional[MmioReadCallback], write_cb: Optional[MmioWriteCallback], info: str = '[mmio]'): + def map_mmio(self, addr: int, size: int, handler: QlMmioHandler, info: str = '[mmio]'): # TODO: mmio memory overlap with ram? Is that possible? # TODO: Can read_cb or write_cb be None? How uc handle that access? prot = UC_PROT_NONE - if read_cb: + if hasattr(handler, 'read'): prot |= UC_PROT_READ - if write_cb: + if hasattr(handler, 'write'): prot |= UC_PROT_WRITE # generic mmio read wrapper @@ -642,10 +658,10 @@ class QlMemoryManager: cb(self.ql, offset, size, value) - self.ql.uc.mmio_map(addr, size, __mmio_read, read_cb, __mmio_write, write_cb) + self.ql.uc.mmio_map(addr, size, __mmio_read, handler.read, __mmio_write, handler.write) self.add_mapinfo(addr, addr + size, prot, info, is_mmio=True) - self.mmio_cbs[(addr, addr + size)] = (read_cb, write_cb) + self.mmio_cbs[(addr, addr + size)] = handler class Chunk: From c23928a2bf4843638718b03d72f0c025c317d1dd Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 15 Aug 2025 17:10:03 +0300 Subject: [PATCH 089/138] Cosmetics --- qiling/hw/hw.py | 2 +- qiling/loader/mcu.py | 41 +++++++++++++++++++++-------------------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/qiling/hw/hw.py b/qiling/hw/hw.py index e558584e..3d869d56 100644 --- a/qiling/hw/hw.py +++ b/qiling/hw/hw.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # diff --git a/qiling/loader/mcu.py b/qiling/loader/mcu.py index 204d17e4..08f63558 100644 --- a/qiling/loader/mcu.py +++ b/qiling/loader/mcu.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework -# Built on top of Unicorn emulator (www.unicorn-engine.org) +# Built on top of Unicorn emulator (www.unicorn-engine.org) import io @@ -27,7 +27,7 @@ class IhexParser: if addr != begin + len(stream): self.segments.append((begin, stream)) begin, stream = addr, data - + else: stream += data @@ -36,13 +36,13 @@ class IhexParser: def parse_line(self, line): if len(line) < 9: return - + desc = line[7: 9] - size = int(line[1: 3], 16) - + size = int(line[1: 3], 16) + addr = bytes.fromhex(line[3: 7]) - data = bytes.fromhex(line[9: 9 + size * 2]) - + data = bytes.fromhex(line[9: 9 + size * 2]) + if desc == '00': # Data offset = int.from_bytes(addr, byteorder='big') self.mem.append((self.base + offset, data)) @@ -52,20 +52,20 @@ class IhexParser: elif desc == '04': # Extended Linear Address self.base = int.from_bytes(data, byteorder='big') * 0x10000 - + class QlLoaderMCU(QlLoader): def __init__(self, ql:Qiling): - super().__init__(ql) - + super().__init__(ql) + self.entry_point = 0 self.load_address = 0 self.filetype = self.guess_filetype() - + if self.filetype == 'elf': with open(self.ql.path, 'rb') as infile: self.elf = ELFFile(io.BytesIO(infile.read())) - + elif self.filetype == 'bin': self.map_address = self.argv[1] @@ -74,8 +74,8 @@ class QlLoaderMCU(QlLoader): def guess_filetype(self): if self.ql.path.endswith('.elf'): - return 'elf' - + return 'elf' + if self.ql.path.endswith('.bin'): return 'bin' @@ -83,7 +83,7 @@ class QlLoaderMCU(QlLoader): return 'hex' return 'elf' - + def reset(self): if self.filetype == 'elf': for segment in self.elf.iter_segments(type='PT_LOAD'): @@ -99,7 +99,7 @@ class QlLoaderMCU(QlLoader): for begin, data in self.ihex.segments: self.ql.mem.write(begin, data) - + self.ql.arch.init_context() self.entry_point = self.ql.arch.regs.read('pc') @@ -109,11 +109,12 @@ class QlLoaderMCU(QlLoader): def load_env(self): for name, args in self.env.items(): memtype = args['type'] + if memtype == 'memory': size = args['size'] base = args['base'] self.ql.mem.map(base, size, info=f'[{name}]') - + # elif memtype == 'remap': # size = args['size'] # base = args['base'] @@ -134,8 +135,8 @@ class QlLoaderMCU(QlLoader): def run(self): self.load_profile() self.load_env() - + ## Handle interrupt from instruction execution self.ql.hook_intr(self.ql.arch.unicorn_exception_handler) - + self.reset() From 7b98c01840065111abc781e3e810e5421ff92eaf Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 15 Aug 2025 17:39:03 +0300 Subject: [PATCH 090/138] Make MCU loader less noisy --- qiling/loader/mcu.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiling/loader/mcu.py b/qiling/loader/mcu.py index 08f63558..3ad64c3b 100644 --- a/qiling/loader/mcu.py +++ b/qiling/loader/mcu.py @@ -130,7 +130,7 @@ class QlLoaderMCU(QlLoader): self.ql.hw.create(name.lower()) else: - self.ql.log.error(f'Unknown memory type "{memtype}" for {name}') + self.ql.log.debug(f'ignoring unknown memory type "{memtype}" for {name}') def run(self): self.load_profile() From 81c2a39f1f39fb4117db20ad5d931bcb20d5ba47 Mon Sep 17 00:00:00 2001 From: technikelly <11539105+technikelly@users.noreply.github.com> Date: Wed, 20 Aug 2025 14:55:37 -0400 Subject: [PATCH 091/138] minimizing blob changes --- examples/blob_raw.ql | 6 ++--- examples/hello_arm_blob_raw.py | 26 ++++++++++++++------- qiling/loader/blob.py | 42 +++++++++++++++++----------------- qiling/os/blob/blob.py | 7 ++---- tests/profiles/blob_raw.ql | 6 ++--- tests/test_blob.py | 11 ++++----- 6 files changed, 52 insertions(+), 46 deletions(-) diff --git a/examples/blob_raw.ql b/examples/blob_raw.ql index 164219a9..23390130 100644 --- a/examples/blob_raw.ql +++ b/examples/blob_raw.ql @@ -1,4 +1,4 @@ -[BLOB_RAW] +[CODE] load_address = 0x10000000 -image_size = 0xbc -image_name = example_raw.bin \ No newline at end of file +entry_point = 0x10000008 +ram_size = 0xa00000 \ No newline at end of file diff --git a/examples/hello_arm_blob_raw.py b/examples/hello_arm_blob_raw.py index 6e44d6bf..f260a4e0 100644 --- a/examples/hello_arm_blob_raw.py +++ b/examples/hello_arm_blob_raw.py @@ -6,6 +6,7 @@ from qiling import Qiling from qiling.const import QL_ARCH, QL_OS, QL_VERBOSE from qiling.extensions.coverage import utils as cov_utils +from qiling.loader.loader import Image BASE_ADDRESS = 0x10000000 CHECKSUM_FUNC_ADDR = BASE_ADDRESS + 0x8 @@ -29,6 +30,7 @@ def checksum_function(input_data_buffer: bytes): for i in range(input_data_len): expected_checksum_python += input_data_buffer[i] expected_checksum_python &= 0xFF # Ensure it's a single byte + return expected_checksum_python def unmapped_handler(ql, type, addr, size, value): @@ -37,18 +39,24 @@ def unmapped_handler(ql, type, addr, size, value): def emulate_checksum_function(input_data_buffer: bytes): print(f"\n--- Testing with input: {input_data_buffer.hex()} ---") - ql = Qiling(archtype=QL_ARCH.ARM, ostype=QL_OS.BLOB, profile="blob_raw.ql", verbose=QL_VERBOSE.DEBUG, thumb=True) + with open("rootfs/blob/example_raw.bin", "rb") as f: + raw_code = f.read() + + ql = Qiling(code=raw_code, archtype=QL_ARCH.ARM, ostype=QL_OS.BLOB, profile="blob_raw.ql", verbose=QL_VERBOSE.DEBUG, thumb=True) + + # monkeypatch - Correcting the loader image name, used for coverage collection + # Remove all images with name 'blob_code' that were created by the blob loader + ql.loader.images = [img for img in ql.loader.images if img.path != 'blob_code'] + # Add image back with correct info + ql.loader.images.append(Image(ql.loader.load_address, ql.loader.load_address + ql.os.code_ram_size, 'example_raw.bin')) + input_data_len = len(input_data_buffer) - # Map memory for the binary, data and stack - ql.mem.map(BASE_ADDRESS, 0x10000) + # Map memory for the data and stack ql.mem.map(STACK_ADDR, 0x2000) ql.mem.map(DATA_ADDR, ql.mem.align_up(input_data_len + 0x100)) # Map enough space for data - # Write the binary into memory - ql.mem.write(BASE_ADDRESS, open("rootfs/blob/example_raw.bin", "rb").read()) - # Write input data ql.mem.write(DATA_ADDR, input_data_buffer) @@ -70,7 +78,8 @@ def emulate_checksum_function(input_data_buffer: bytes): # Start emulation print(f"Starting emulation at PC: {hex(ql.arch.regs.pc)}") try: - ql.run(begin=CHECKSUM_FUNC_ADDR, end=END_ADDRESS) + with cov_utils.collect_coverage(ql, 'drcov', 'output.cov'): + ql.run(begin=CHECKSUM_FUNC_ADDR, end=END_ADDRESS) except Exception as e: print(f"Emulation error: {e}") @@ -78,4 +87,5 @@ def emulate_checksum_function(input_data_buffer: bytes): if __name__ == "__main__": data = b"\x01\x02\x03\x04\x05" # Example input data - emulate_checksum_function(data) \ No newline at end of file + emulate_checksum_function(data) + print(hex(checksum_function(data))) \ No newline at end of file diff --git a/qiling/loader/blob.py b/qiling/loader/blob.py index 96168538..c4a69de8 100644 --- a/qiling/loader/blob.py +++ b/qiling/loader/blob.py @@ -9,38 +9,38 @@ from qiling import Qiling from qiling.loader.loader import QlLoader, Image from qiling.os.memory import QlMemoryHeap +import configparser class QlLoaderBLOB(QlLoader): def __init__(self, ql: Qiling): super().__init__(ql) + self.load_address = 0 + def run(self): - if self.ql.os.profile.has_section("BLOB_RAW"): - # For raw binary blobs, user will handle memory mapping - self.load_address = int(self.ql.os.profile.get("BLOB_RAW", "load_address"), 16) - image_size = int(self.ql.os.profile.get("BLOB_RAW", "image_size"), 16) - image_name = self.ql.os.profile.get("BLOB_RAW", "image_name", fallback="blob.raw") - self.images.append(Image(self.load_address, self.load_address+image_size, image_name)) # used to collect coverage - else: - self.load_address = self.ql.os.load_address - self.entry_point = self.ql.os.entry_point + self.load_address = self.ql.os.load_address + self.entry_point = self.ql.os.entry_point - code_begins = self.load_address - code_size = self.ql.os.code_ram_size - code_ends = code_begins + code_size + code_begins = self.load_address + code_size = self.ql.os.code_ram_size + code_ends = code_begins + code_size - self.ql.mem.map(code_begins, code_size, info="[code]") - self.ql.mem.write(code_begins, self.ql.code) + self.ql.mem.map(code_begins, code_size, info="[code]") + self.ql.mem.write(code_begins, self.ql.code) - # allow image-related functionalities - self.images.append(Image(code_begins, code_ends, 'blob_code')) + # allow image-related functionalities + self.images.append(Image(code_begins, code_ends, 'blob_code')) - # FIXME: heap starts above end of ram?? - # FIXME: heap should be allocated by OS, not loader - heap_base = code_ends + # FIXME: heap starts above end of ram?? + # FIXME: heap should be allocated by OS, not loader + heap_base = code_ends + # if heap_size is defined, create the heap + try: heap_size = int(self.ql.os.profile.get("CODE", "heap_size"), 16) self.ql.os.heap = QlMemoryHeap(self.ql, heap_base, heap_base + heap_size) + except (configparser.NoSectionError, configparser.NoOptionError): + pass # heap_size is not required - # FIXME: stack pointer should be a configurable profile setting - self.ql.arch.regs.arch_sp = code_ends - 0x1000 + # FIXME: stack pointer should be a configurable profile setting + self.ql.arch.regs.arch_sp = code_ends - 0x1000 diff --git a/qiling/os/blob/blob.py b/qiling/os/blob/blob.py index f2056e55..e4a02256 100644 --- a/qiling/os/blob/blob.py +++ b/qiling/os/blob/blob.py @@ -2,9 +2,6 @@ # # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -# Added support for raw binary blob emulation -# Kelly Patterson - Cisco Talos -# Copyright (C) 2025 Cisco Systems Inc from qiling import Qiling from qiling.cc import QlCC, intel, arm, mips, riscv, ppc @@ -47,10 +44,10 @@ class QlOsBlob(QlOs): if self.ql.entry_point is not None: self.entry_point = self.ql.entry_point + self.exit_point = self.load_address + len(self.ql.code) + # if exit point was set explicitly, override the default one if self.ql.exit_point is not None: self.exit_point = self.ql.exit_point - elif self.ql.code is not None: # self.ql.code might not always be provided - self.exit_point = self.load_address + len(self.ql.code) self.ql.emu_start(self.entry_point, self.exit_point, self.ql.timeout, self.ql.count) diff --git a/tests/profiles/blob_raw.ql b/tests/profiles/blob_raw.ql index 164219a9..23390130 100644 --- a/tests/profiles/blob_raw.ql +++ b/tests/profiles/blob_raw.ql @@ -1,4 +1,4 @@ -[BLOB_RAW] +[CODE] load_address = 0x10000000 -image_size = 0xbc -image_name = example_raw.bin \ No newline at end of file +entry_point = 0x10000008 +ram_size = 0xa00000 \ No newline at end of file diff --git a/tests/test_blob.py b/tests/test_blob.py index d284bbac..bd0b9ecf 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -97,18 +97,17 @@ class BlobTest(unittest.TestCase): DATA_ADDR = 0xa0000000 STACK_ADDR = 0xb0000000 - ql = Qiling(archtype=QL_ARCH.ARM, ostype=QL_OS.BLOB, profile="profiles/blob_raw.ql", verbose=QL_VERBOSE.DEBUG, thumb=True) + with open("../examples/rootfs/blob/example_raw.bin", "rb") as f: + raw_code = f.read() + + ql = Qiling(code=raw_code, archtype=QL_ARCH.ARM, ostype=QL_OS.BLOB, profile="profiles/blob_raw.ql", verbose=QL_VERBOSE.DEBUG, thumb=True) input_data_len = len(input_data_buffer) - # Map memory for the binary, data and stack - ql.mem.map(BASE_ADDRESS, 0x10000) + # Map memory for data and stack ql.mem.map(STACK_ADDR, 0x2000) ql.mem.map(DATA_ADDR, ql.mem.align_up(input_data_len + 0x100)) - # Write the binary into memory - ql.mem.write(BASE_ADDRESS, open("../examples/rootfs/blob/example_raw.bin", "rb").read()) - # Write input data ql.mem.write(DATA_ADDR, input_data_buffer) From 6d697b5d7fe0ebf8e28474fe737fe76594f43b0d Mon Sep 17 00:00:00 2001 From: technikelly <11539105+technikelly@users.noreply.github.com> Date: Wed, 20 Aug 2025 15:12:54 -0400 Subject: [PATCH 092/138] updating copyright statements with license info --- examples/hello_arm_blob_raw.py | 1 + examples/src/blob/Makefile | 1 + examples/src/blob/example_raw.c | 1 + examples/src/blob/linker.ld | 1 + qiling/loader/blob.py | 1 + tests/test_blob.py | 1 + 6 files changed, 6 insertions(+) diff --git a/examples/hello_arm_blob_raw.py b/examples/hello_arm_blob_raw.py index f260a4e0..6096bbd9 100644 --- a/examples/hello_arm_blob_raw.py +++ b/examples/hello_arm_blob_raw.py @@ -2,6 +2,7 @@ # Added example for raw binary blob # Kelly Patterson - Cisco Talos # Copyright (C) 2025 Cisco Systems Inc +# Licensed under the GNU General Public License v2.0 or later ############################################################################## from qiling import Qiling from qiling.const import QL_ARCH, QL_OS, QL_VERBOSE diff --git a/examples/src/blob/Makefile b/examples/src/blob/Makefile index 7f4252f7..66fe0b43 100644 --- a/examples/src/blob/Makefile +++ b/examples/src/blob/Makefile @@ -2,6 +2,7 @@ # Added example for raw binary blob # Kelly Patterson - Cisco Talos # Copyright (C) 2025 Cisco Systems Inc +# Licensed under the GNU General Public License v2.0 or later ############################################################################## # Makefile for Bare-Metal ARM Hash Calculator diff --git a/examples/src/blob/example_raw.c b/examples/src/blob/example_raw.c index 5ac71cb3..eae56b2b 100644 --- a/examples/src/blob/example_raw.c +++ b/examples/src/blob/example_raw.c @@ -2,6 +2,7 @@ * Added example for raw binary blob * Kelly Patterson - Cisco Talos * Copyright (C) 2025 Cisco Systems Inc + * Licensed under the GNU General Public License v2.0 or later * */ // example_raw.c diff --git a/examples/src/blob/linker.ld b/examples/src/blob/linker.ld index ea57d85e..9c9e7485 100644 --- a/examples/src/blob/linker.ld +++ b/examples/src/blob/linker.ld @@ -3,6 +3,7 @@ * Added example for raw binary blob * Kelly Patterson - Cisco Talos * Copyright (C) 2025 Cisco Systems Inc + * Licensed under the GNU General Public License v2.0 or later * */ diff --git a/qiling/loader/blob.py b/qiling/loader/blob.py index c4a69de8..7fa5e50a 100644 --- a/qiling/loader/blob.py +++ b/qiling/loader/blob.py @@ -5,6 +5,7 @@ # Added support for raw binary blob emulation # Kelly Patterson - Cisco Talos # Copyright (C) 2025 Cisco Systems Inc +# Licensed under the GNU General Public License v2.0 or later from qiling import Qiling from qiling.loader.loader import QlLoader, Image diff --git a/tests/test_blob.py b/tests/test_blob.py index bd0b9ecf..310d202a 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -5,6 +5,7 @@ # Added test for raw binary blob emulation # Kelly Patterson - Cisco Talos # Copyright (C) 2025 Cisco Systems Inc +# Licensed under the GNU General Public License v2.0 or later import unittest From a2223abb5a9eb6c601748e8f60e18f516b509b40 Mon Sep 17 00:00:00 2001 From: technikelly <11539105+technikelly@users.noreply.github.com> Date: Wed, 20 Aug 2025 15:15:17 -0400 Subject: [PATCH 093/138] update description --- qiling/loader/blob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qiling/loader/blob.py b/qiling/loader/blob.py index 7fa5e50a..7a7e3325 100644 --- a/qiling/loader/blob.py +++ b/qiling/loader/blob.py @@ -2,7 +2,7 @@ # # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -# Added support for raw binary blob emulation +# Heaps are optional for blobs # Kelly Patterson - Cisco Talos # Copyright (C) 2025 Cisco Systems Inc # Licensed under the GNU General Public License v2.0 or later From 313b907bdecb0787aea642af077d90f9b73fa808 Mon Sep 17 00:00:00 2001 From: rliebig Date: Sat, 23 Aug 2025 12:14:38 -0400 Subject: [PATCH 094/138] return EISDIR in case ql_file object has a directory path and a read syscall is attempted on it --- qiling/os/posix/syscall/unistd.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qiling/os/posix/syscall/unistd.py b/qiling/os/posix/syscall/unistd.py index 8931f95e..09b86c38 100644 --- a/qiling/os/posix/syscall/unistd.py +++ b/qiling/os/posix/syscall/unistd.py @@ -415,6 +415,8 @@ def ql_syscall_read(ql: Qiling, fd: int, buf: int, length: int): try: data = f.read(length) + except IsADirectoryError: + return -EISDIR except ConnectionError: ql.log.debug('read failed due to a connection error') return -EIO From 5ec4a3b57a0cdf6e638026e04ab94cc60b1aabb5 Mon Sep 17 00:00:00 2001 From: technikelly <11539105+technikelly@users.noreply.github.com> Date: Mon, 25 Aug 2025 13:34:45 -0400 Subject: [PATCH 095/138] addressing review comments, removing copyright notices --- examples/hello_arm_blob_raw.py | 47 ++++++++++++++++++++------------- examples/src/blob/Makefile | 8 +----- examples/src/blob/example_raw.c | 8 +----- examples/src/blob/linker.ld | 7 ----- examples/uboot_bin.ql | 1 + qiling/loader/blob.py | 16 ----------- qiling/os/blob/blob.py | 9 ++++++- tests/profiles/uboot_bin.ql | 1 + tests/test_blob.py | 43 +++--------------------------- 9 files changed, 43 insertions(+), 97 deletions(-) diff --git a/examples/hello_arm_blob_raw.py b/examples/hello_arm_blob_raw.py index 6096bbd9..4c257166 100644 --- a/examples/hello_arm_blob_raw.py +++ b/examples/hello_arm_blob_raw.py @@ -1,13 +1,13 @@ ############################################################################## -# Added example for raw binary blob -# Kelly Patterson - Cisco Talos -# Copyright (C) 2025 Cisco Systems Inc -# Licensed under the GNU General Public License v2.0 or later +# This example is meant to demonstrate the modifications necessary +# to enable code coverage when emulating small code snippets or bare-metal +# code. ############################################################################## from qiling import Qiling from qiling.const import QL_ARCH, QL_OS, QL_VERBOSE from qiling.extensions.coverage import utils as cov_utils from qiling.loader.loader import Image +import os BASE_ADDRESS = 0x10000000 CHECKSUM_FUNC_ADDR = BASE_ADDRESS + 0x8 @@ -16,6 +16,8 @@ DATA_ADDR = 0xa0000000 # Arbitrary address for data STACK_ADDR = 0xb0000000 # Arbitrary address for stack # Python implementation of the checksum function being emulated +# This checksum function is intended to have different code paths based on the input +# which is useful for observing code coverage def checksum_function(input_data_buffer: bytes): expected_checksum_python = 0 input_data_len = len(input_data_buffer) @@ -33,26 +35,34 @@ def checksum_function(input_data_buffer: bytes): expected_checksum_python &= 0xFF # Ensure it's a single byte return expected_checksum_python -def unmapped_handler(ql, type, addr, size, value): +def unmapped_handler(ql: Qiling, type: int, addr: int, size: int, value: int) -> None: + print(f"Unmapped Memory R/W, trying to access {size:d} bytes at {addr:#010x} from {ql.arch.regs.pc:#010x}") - print(f"Unmapped Memory R/W, trying to access {hex(size)} bytes at {hex(addr)} from {hex(ql.arch.regs.pc)}") - -def emulate_checksum_function(input_data_buffer: bytes): +def emulate_checksum_function(input_data_buffer: bytes) -> None: print(f"\n--- Testing with input: {input_data_buffer.hex()} ---") - with open("rootfs/blob/example_raw.bin", "rb") as f: - raw_code = f.read() + test_file = "rootfs/blob/example_raw.bin" - ql = Qiling(code=raw_code, archtype=QL_ARCH.ARM, ostype=QL_OS.BLOB, profile="blob_raw.ql", verbose=QL_VERBOSE.DEBUG, thumb=True) + with open(test_file, "rb") as f: + raw_code: bytes = f.read() - # monkeypatch - Correcting the loader image name, used for coverage collection - # Remove all images with name 'blob_code' that were created by the blob loader + ql: Qiling = Qiling( + code=raw_code, + archtype=QL_ARCH.ARM, + ostype=QL_OS.BLOB, + profile="blob_raw.ql", + verbose=QL_VERBOSE.DEBUG, + thumb=True + ) + + ''' monkeypatch - Correcting the loader image name, used for coverage collection + removing all images with name 'blob_code' that were created by the blob loader. + This is necessary because some code coverage visualization tools require the + module name to match that of the input file ''' ql.loader.images = [img for img in ql.loader.images if img.path != 'blob_code'] - # Add image back with correct info - ql.loader.images.append(Image(ql.loader.load_address, ql.loader.load_address + ql.os.code_ram_size, 'example_raw.bin')) + ql.loader.images.append(Image(ql.loader.load_address, ql.loader.load_address + ql.os.code_ram_size, os.path.basename(test_file))) - - input_data_len = len(input_data_buffer) + input_data_len: int = len(input_data_buffer) # Map memory for the data and stack ql.mem.map(STACK_ADDR, 0x2000) @@ -88,5 +98,4 @@ def emulate_checksum_function(input_data_buffer: bytes): if __name__ == "__main__": data = b"\x01\x02\x03\x04\x05" # Example input data - emulate_checksum_function(data) - print(hex(checksum_function(data))) \ No newline at end of file + emulate_checksum_function(data) \ No newline at end of file diff --git a/examples/src/blob/Makefile b/examples/src/blob/Makefile index 66fe0b43..74966f26 100644 --- a/examples/src/blob/Makefile +++ b/examples/src/blob/Makefile @@ -1,10 +1,4 @@ -############################################################################## -# Added example for raw binary blob -# Kelly Patterson - Cisco Talos -# Copyright (C) 2025 Cisco Systems Inc -# Licensed under the GNU General Public License v2.0 or later -############################################################################## -# Makefile for Bare-Metal ARM Hash Calculator +# Makefile for Bare-Metal ARM Checksum Calculator # --- Toolchain Definitions --- TOOLCHAIN_PREFIX = arm-none-eabi diff --git a/examples/src/blob/example_raw.c b/examples/src/blob/example_raw.c index eae56b2b..13cd7077 100644 --- a/examples/src/blob/example_raw.c +++ b/examples/src/blob/example_raw.c @@ -1,10 +1,4 @@ - /* - * Added example for raw binary blob - * Kelly Patterson - Cisco Talos - * Copyright (C) 2025 Cisco Systems Inc - * Licensed under the GNU General Public License v2.0 or later - * - */ +// example checksum algorithm to demonstrate raw binary code coverage in qiling // example_raw.c // Define some magic values diff --git a/examples/src/blob/linker.ld b/examples/src/blob/linker.ld index 9c9e7485..ae31f2fa 100644 --- a/examples/src/blob/linker.ld +++ b/examples/src/blob/linker.ld @@ -1,11 +1,4 @@ /* linker.ld */ - /* - * Added example for raw binary blob - * Kelly Patterson - Cisco Talos - * Copyright (C) 2025 Cisco Systems Inc - * Licensed under the GNU General Public License v2.0 or later - * - */ ENTRY(_start) /* Define the entry point of our program */ diff --git a/examples/uboot_bin.ql b/examples/uboot_bin.ql index c33a7d23..1e95311f 100644 --- a/examples/uboot_bin.ql +++ b/examples/uboot_bin.ql @@ -2,6 +2,7 @@ ram_size = 0xa00000 load_address = 0x80800000 entry_point = 0x80800000 +heap_address = 0xa0000000 heap_size = 0x300000 diff --git a/qiling/loader/blob.py b/qiling/loader/blob.py index 7a7e3325..72844339 100644 --- a/qiling/loader/blob.py +++ b/qiling/loader/blob.py @@ -2,15 +2,9 @@ # # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -# Heaps are optional for blobs -# Kelly Patterson - Cisco Talos -# Copyright (C) 2025 Cisco Systems Inc -# Licensed under the GNU General Public License v2.0 or later from qiling import Qiling from qiling.loader.loader import QlLoader, Image -from qiling.os.memory import QlMemoryHeap -import configparser class QlLoaderBLOB(QlLoader): @@ -33,15 +27,5 @@ class QlLoaderBLOB(QlLoader): # allow image-related functionalities self.images.append(Image(code_begins, code_ends, 'blob_code')) - # FIXME: heap starts above end of ram?? - # FIXME: heap should be allocated by OS, not loader - heap_base = code_ends - # if heap_size is defined, create the heap - try: - heap_size = int(self.ql.os.profile.get("CODE", "heap_size"), 16) - self.ql.os.heap = QlMemoryHeap(self.ql, heap_base, heap_base + heap_size) - except (configparser.NoSectionError, configparser.NoOptionError): - pass # heap_size is not required - # FIXME: stack pointer should be a configurable profile setting self.ql.arch.regs.arch_sp = code_ends - 0x1000 diff --git a/qiling/os/blob/blob.py b/qiling/os/blob/blob.py index e4a02256..af52fa74 100644 --- a/qiling/os/blob/blob.py +++ b/qiling/os/blob/blob.py @@ -8,6 +8,7 @@ from qiling.cc import QlCC, intel, arm, mips, riscv, ppc from qiling.const import QL_ARCH, QL_OS from qiling.os.fcall import QlFunctionCall from qiling.os.os import QlOs +from qiling.os.memory import QlMemoryHeap class QlOsBlob(QlOs): @@ -49,5 +50,11 @@ class QlOsBlob(QlOs): # if exit point was set explicitly, override the default one if self.ql.exit_point is not None: self.exit_point = self.ql.exit_point - + + # if heap info is provided in profile, create heap + heap_base = self.profile.getint('CODE', 'heap_address', fallback=None) + heap_size = self.profile.getint('CODE', 'heap_size', fallback=None) + if heap_base is not None and heap_size is not None: + self.heap = QlMemoryHeap(self.ql, heap_base, heap_base + heap_size) + self.ql.emu_start(self.entry_point, self.exit_point, self.ql.timeout, self.ql.count) diff --git a/tests/profiles/uboot_bin.ql b/tests/profiles/uboot_bin.ql index c33a7d23..1e95311f 100644 --- a/tests/profiles/uboot_bin.ql +++ b/tests/profiles/uboot_bin.ql @@ -2,6 +2,7 @@ ram_size = 0xa00000 load_address = 0x80800000 entry_point = 0x80800000 +heap_address = 0xa0000000 heap_size = 0x300000 diff --git a/tests/test_blob.py b/tests/test_blob.py index 310d202a..753e0ef5 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -2,10 +2,6 @@ # # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -# Added test for raw binary blob emulation -# Kelly Patterson - Cisco Talos -# Copyright (C) 2025 Cisco Systems Inc -# Licensed under the GNU General Public License v2.0 or later import unittest @@ -87,7 +83,7 @@ class BlobTest(unittest.TestCase): del ql @unittest.skip("Temporarily disabled") - def test_blob_checksum_calculations(self): + def test_blob_raw(self): def run_checksum_emu(input_data_buffer: bytes) -> int: """ Callable function that takes input data buffer and returns the checksum. @@ -145,41 +141,8 @@ class BlobTest(unittest.TestCase): return expected_checksum & 0xFF - # Test cases with descriptions - test_cases = { - "default_path": { - "data": b"\x01\x02\x03\x04\x05", - "description": "Default path - simple sum of all bytes" - }, - "magic_value_1": { - "data": b"\xDE\x01\x02\x03\x04\x05", - "description": "Magic Value 1 path (0xDE at data[0]) - sum first 4 bytes + 0x10" - }, - "magic_value_2": { - "data": b"\x01\xAD\x02\x03\x04\x05", - "description": "Magic Value 2 path (0xAD at data[1]) - XOR all bytes + 0x20" - }, - "edge_magic1_short": { - "data": b"\xDE\x01", - "description": "Edge case: Magic Value 1, but short data (only 2 bytes)" - }, - "edge_magic2_too_short": { - "data": b"\xAD", - "description": "Edge case: Magic Value 2, but too short (fallback to default)" - }, - "both_magic_values": { - "data": b"\xDE\xAD\x01\x02", - "description": "Both magic values present, DE at [0] takes precedence" - } - } - - # Assertions with descriptions - directly call functions with test data - self.assertEqual(run_checksum_emu(test_cases["default_path"]["data"]), calculate_expected_checksum(test_cases["default_path"]["data"])) # Default path - self.assertEqual(run_checksum_emu(test_cases["magic_value_1"]["data"]), calculate_expected_checksum(test_cases["magic_value_1"]["data"])) # Magic Value 1 path - self.assertEqual(run_checksum_emu(test_cases["magic_value_2"]["data"]), calculate_expected_checksum(test_cases["magic_value_2"]["data"])) # Magic Value 2 path - self.assertEqual(run_checksum_emu(test_cases["edge_magic1_short"]["data"]), calculate_expected_checksum(test_cases["edge_magic1_short"]["data"])) # Edge case: Magic Value 1, short data - self.assertEqual(run_checksum_emu(test_cases["edge_magic2_too_short"]["data"]), calculate_expected_checksum(test_cases["edge_magic2_too_short"]["data"])) # Edge case: Magic Value 2, too short - self.assertEqual(run_checksum_emu(test_cases["both_magic_values"]["data"]), calculate_expected_checksum(test_cases["both_magic_values"]["data"])) # Both magic values, DE takes precedence + test_input = b"\x01\x02\x03\x04\x05" + self.assertEqual(run_checksum_emu(test_input), calculate_expected_checksum(test_input)) if __name__ == "__main__": From 04236d77b1941a505da9fe19f4abe755c15f9a0e Mon Sep 17 00:00:00 2001 From: technikelly <11539105+technikelly@users.noreply.github.com> Date: Wed, 27 Aug 2025 11:26:39 -0400 Subject: [PATCH 096/138] enabling test and updating rootfs submodule --- examples/rootfs | 2 +- tests/test_blob.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/rootfs b/examples/rootfs index f71f45fe..120fb6d3 160000 --- a/examples/rootfs +++ b/examples/rootfs @@ -1 +1 @@ -Subproject commit f71f45fe1a39d58d8b8cae717f55cebeb37f63c7 +Subproject commit 120fb6d37700a2d4c0e35ced599aaee7a8f98723 diff --git a/tests/test_blob.py b/tests/test_blob.py index 753e0ef5..0bd9a662 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -82,7 +82,6 @@ class BlobTest(unittest.TestCase): del ql - @unittest.skip("Temporarily disabled") def test_blob_raw(self): def run_checksum_emu(input_data_buffer: bytes) -> int: """ From f45eb7042ace11358d242814d8a6cea8ee7c1ab7 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:35:20 +0300 Subject: [PATCH 097/138] Cache Struct instances for faster conversions --- qiling/core_struct.py | 50 +++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/qiling/core_struct.py b/qiling/core_struct.py index 6c0d99cc..f10fd42f 100644 --- a/qiling/core_struct.py +++ b/qiling/core_struct.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -25,14 +25,14 @@ class QlCoreStructs: QL_ENDIAN.EB: '>' }[endian] - self._fmt8 = f'{modifier}B' - self._fmt8s = f'{modifier}b' - self._fmt16 = f'{modifier}H' - self._fmt16s = f'{modifier}h' - self._fmt32 = f'{modifier}I' - self._fmt32s = f'{modifier}i' - self._fmt64 = f'{modifier}Q' - self._fmt64s = f'{modifier}q' + self._fmt8 = struct.Struct(f'{modifier}B') + self._fmt8s = struct.Struct(f'{modifier}b') + self._fmt16 = struct.Struct(f'{modifier}H') + self._fmt16s = struct.Struct(f'{modifier}h') + self._fmt32 = struct.Struct(f'{modifier}I') + self._fmt32s = struct.Struct(f'{modifier}i') + self._fmt64 = struct.Struct(f'{modifier}Q') + self._fmt64s = struct.Struct(f'{modifier}q') handlers = { 64 : (self.pack64, self.pack64s, self.unpack64, self.unpack64s), @@ -51,49 +51,49 @@ class QlCoreStructs: self.unpacks = ups def pack64(self, x: int, /) -> bytes: - return struct.pack(self._fmt64, x) + return self._fmt64.pack(x) def pack64s(self, x: int, /) -> bytes: - return struct.pack(self._fmt64s, x) + return self._fmt64s.pack(x) def unpack64(self, x: ReadableBuffer, /) -> int: - return struct.unpack(self._fmt64, x)[0] + return self._fmt64.unpack(x)[0] def unpack64s(self, x: ReadableBuffer, /) -> int: - return struct.unpack(self._fmt64s, x)[0] + return self._fmt64s.unpack(x)[0] def pack32(self, x: int, /) -> bytes: - return struct.pack(self._fmt32, x) + return self._fmt32.pack(x) def pack32s(self, x: int, /) -> bytes: - return struct.pack(self._fmt32s, x) + return self._fmt32s.pack(x) def unpack32(self, x: ReadableBuffer, /) -> int: - return struct.unpack(self._fmt32, x)[0] + return self._fmt32.unpack(x)[0] def unpack32s(self, x: ReadableBuffer, /) -> int: - return struct.unpack(self._fmt32s, x)[0] + return self._fmt32s.unpack(x)[0] def pack16(self, x: int, /) -> bytes: - return struct.pack(self._fmt16, x) + return self._fmt16.pack(x) def pack16s(self, x: int, /) -> bytes: - return struct.pack(self._fmt16s, x) + return self._fmt16s.pack(x) def unpack16(self, x: ReadableBuffer, /) -> int: - return struct.unpack(self._fmt16, x)[0] + return self._fmt16.unpack(x)[0] def unpack16s(self, x: ReadableBuffer, /) -> int: - return struct.unpack(self._fmt16s, x)[0] + return self._fmt16s.unpack(x)[0] def pack8(self, x: int, /) -> bytes: - return struct.pack(self._fmt8, x) + return self._fmt8.pack(x) def pack8s(self, x: int, /) -> bytes: - return struct.pack(self._fmt8s, x) + return self._fmt8s.pack(x) def unpack8(self, x: ReadableBuffer, /) -> int: - return struct.unpack(self._fmt8, x)[0] + return self._fmt8.unpack(x)[0] def unpack8s(self, x: ReadableBuffer, /) -> int: - return struct.unpack(self._fmt8s, x)[0] + return self._fmt8s.unpack(x)[0] From 59ebfdd0fd300e9038a0b9341e896ad12a495356 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:36:42 +0300 Subject: [PATCH 098/138] Cache memory accessors for faster access --- qiling/os/memory.py | 61 ++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/qiling/os/memory.py b/qiling/os/memory.py index ec7aef19..4c0f8654 100644 --- a/qiling/os/memory.py +++ b/qiling/os/memory.py @@ -64,6 +64,31 @@ class QlMemoryManager: # make sure pagesize is a power of 2 assert self.pagesize & (self.pagesize - 1) == 0, 'pagesize has to be a power of 2' + self._packers = { + (1, True): ql.pack8s, + (2, True): ql.pack16s, + (4, True): ql.pack32s, + (8, True): ql.pack64s, + + (1, False): ql.pack8, + (2, False): ql.pack16, + (4, False): ql.pack32, + (8, False): ql.pack64 + } + + self._unpackers = { + (1, True): ql.unpack8s, + (2, True): ql.unpack16s, + (4, True): ql.unpack32s, + (8, True): ql.unpack64s, + + (1, False): ql.unpack8, + (2, False): ql.unpack16, + (4, False): ql.unpack32, + (8, False): ql.unpack64 + } + + def __read_string(self, addr: int) -> str: ret = bytearray() c = self.read(addr, 1) @@ -344,22 +369,12 @@ class QlMemoryManager: if not size: size = self.ql.arch.pointersize - __unpack = ({ - 1: self.ql.unpack8s, - 2: self.ql.unpack16s, - 4: self.ql.unpack32s, - 8: self.ql.unpack64s - } if signed else { - 1: self.ql.unpack8, - 2: self.ql.unpack16, - 4: self.ql.unpack32, - 8: self.ql.unpack64 - }).get(size) - - if __unpack is None: + try: + _unpack = self._unpackers[(size, signed)] + except KeyError: raise QlErrorStructConversion(f"Unsupported pointer size: {size}") - return __unpack(self.read(addr, size)) + return _unpack(self.read(addr, size)) def write(self, addr: int, data: bytes) -> None: """Write bytes to a memory. @@ -385,22 +400,12 @@ class QlMemoryManager: if not size: size = self.ql.arch.pointersize - __pack = ({ - 1: self.ql.pack8s, - 2: self.ql.pack16s, - 4: self.ql.pack32s, - 8: self.ql.pack64s - } if signed else { - 1: self.ql.pack8, - 2: self.ql.pack16, - 4: self.ql.pack32, - 8: self.ql.pack64 - }).get(size) - - if __pack is None: + try: + _pack = self._packers[(size, signed)] + except KeyError: raise QlErrorStructConversion(f"Unsupported pointer size: {size}") - self.write(addr, __pack(value)) + self.write(addr, _pack(value)) def search(self, needle: Union[bytes, Pattern[bytes]], begin: Optional[int] = None, end: Optional[int] = None) -> List[int]: """Search for a sequence of bytes in memory. From 94aeb96350be281aa5611543083128fa51933752 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:38:20 +0300 Subject: [PATCH 099/138] Use disasm_lite for faster DISASM output --- qiling/arch/utils.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/qiling/arch/utils.py b/qiling/arch/utils.py index 6782c932..4d44b545 100644 --- a/qiling/arch/utils.py +++ b/qiling/arch/utils.py @@ -48,7 +48,7 @@ class QlArchUtils: return addr, '-' def disassembler(self, ql: Qiling, address: int, size: int): - data = ql.mem.read(address, size) + data = memoryview(ql.mem.read(address, size)) # knowing that all binary sections are aligned to page boundary allows # us to 'cheat' and search for the containing image using the aligned @@ -64,11 +64,14 @@ class QlArchUtils: ba, name = self.get_base_and_name(ql.mem.align(address)) anibbles = ql.arch.bits // 4 + pos = 0 - for insn in ql.arch.disassembler.disasm(data, address): - offset = insn.address - ba + for iaddr, isize, mnem, ops in ql.arch.disassembler.disasm_lite(data, address): + offset = iaddr - ba + ibytes = data[pos:pos + isize] - ql.log.info(f'{insn.address:0{anibbles}x} [{name:20s} + {offset:#08x}] {insn.bytes.hex(" "):20s} {insn.mnemonic:20s} {insn.op_str}') + ql.log.info(f'{iaddr:0{anibbles}x} [{name:20s} + {offset:#08x}] {ibytes.hex():22s} {mnem:16s} {ops}') + pos += isize if ql.verbose >= QL_VERBOSE.DUMP: for reg in ql.arch.regs.register_mapping: From d846c75e591b11c2d5b1a9c561bfd19026edd4d7 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:40:00 +0300 Subject: [PATCH 100/138] Add a CC accessor to tell the return address --- qiling/cc/__init__.py | 6 ++++++ qiling/cc/arm.py | 7 ++++++- qiling/cc/intel.py | 3 +++ qiling/cc/mips.py | 3 +++ qiling/cc/ppc.py | 3 +++ qiling/cc/riscv.py | 3 +++ 6 files changed, 24 insertions(+), 1 deletion(-) diff --git a/qiling/cc/__init__.py b/qiling/cc/__init__.py index 99c9e564..a1f35481 100644 --- a/qiling/cc/__init__.py +++ b/qiling/cc/__init__.py @@ -70,6 +70,12 @@ class QlCC: raise NotImplementedError + def getReturnAddress(self) -> int: + """Get function return address. + """ + + raise NotImplementedError + def setReturnAddress(self, addr: int) -> None: """Set function return address. diff --git a/qiling/cc/arm.py b/qiling/cc/arm.py index 51d798b2..29ac126b 100644 --- a/qiling/cc/arm.py +++ b/qiling/cc/arm.py @@ -21,17 +21,22 @@ class QlArmBaseCC(QlCommonBaseCC): def getNumSlots(argbits: int) -> int: return 1 + def getReturnAddress(self) -> int: + return self.arch.regs.lr + def setReturnAddress(self, addr: int) -> None: self.arch.regs.lr = addr def unwind(self, nslots: int) -> int: # TODO: cleanup? - return self.arch.regs.lr + return self.getReturnAddress() + class aarch64(QlArmBaseCC): _retreg = UC_ARM64_REG_X0 _argregs = make_arg_list(UC_ARM64_REG_X0, UC_ARM64_REG_X1, UC_ARM64_REG_X2, UC_ARM64_REG_X3, UC_ARM64_REG_X4, UC_ARM64_REG_X5, UC_ARM64_REG_X6, UC_ARM64_REG_X7) + class aarch32(QlArmBaseCC): _retreg = UC_ARM_REG_R0 _argregs = make_arg_list(UC_ARM_REG_R0, UC_ARM_REG_R1, UC_ARM_REG_R2, UC_ARM_REG_R3) diff --git a/qiling/cc/intel.py b/qiling/cc/intel.py index ca179603..f2e6971d 100644 --- a/qiling/cc/intel.py +++ b/qiling/cc/intel.py @@ -15,6 +15,9 @@ class QlIntelBaseCC(QlCommonBaseCC): Supports arguments passing over registers and stack. """ + def getReturnAddress(self) -> int: + return self.arch.stack_read(0) + def setReturnAddress(self, addr: int) -> None: self.arch.stack_push(addr) diff --git a/qiling/cc/mips.py b/qiling/cc/mips.py index 9ebf2337..472b2a3e 100644 --- a/qiling/cc/mips.py +++ b/qiling/cc/mips.py @@ -12,6 +12,9 @@ class mipso32(QlCommonBaseCC): _shadow = 4 _retaddr_on_stack = False + def getReturnAddress(self) -> int: + return self.arch.regs.ra + def setReturnAddress(self, addr: int): self.arch.regs.ra = addr diff --git a/qiling/cc/ppc.py b/qiling/cc/ppc.py index 2440fab1..b4a88f79 100644 --- a/qiling/cc/ppc.py +++ b/qiling/cc/ppc.py @@ -22,5 +22,8 @@ class ppc(QlCommonBaseCC): def getNumSlots(argbits: int): return 1 + def getReturnAddress(self) -> int: + return self.arch.regs.lr + def setReturnAddress(self, addr: int): self.arch.regs.lr = addr diff --git a/qiling/cc/riscv.py b/qiling/cc/riscv.py index 3a360bd8..f9f09522 100644 --- a/qiling/cc/riscv.py +++ b/qiling/cc/riscv.py @@ -22,5 +22,8 @@ class riscv(QlCommonBaseCC): def getNumSlots(argbits: int): return 1 + def getReturnAddress(self) -> int: + return self.arch.regs.ra + def setReturnAddress(self, addr: int): self.arch.regs.ra = addr From a845b5ee2297089ae4db23f779e95bc0f76be948 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:42:41 +0300 Subject: [PATCH 101/138] Avoid using mutable object as default parameter value --- qiling/os/uefi/fncc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/qiling/os/uefi/fncc.py b/qiling/os/uefi/fncc.py index 83f999bf..3e354920 100644 --- a/qiling/os/uefi/fncc.py +++ b/qiling/os/uefi/fncc.py @@ -1,14 +1,14 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from typing import Any, Mapping +from typing import Any, Mapping, Optional from qiling import Qiling from qiling.const import QL_INTERCEPT -def dxeapi(params: Mapping[str, Any] = {}): +def dxeapi(params: Optional[Mapping[str, Any]] = None): def decorator(func): def wrapper(ql: Qiling): pc = ql.arch.regs.arch_pc @@ -18,7 +18,7 @@ def dxeapi(params: Mapping[str, Any] = {}): onenter = ql.os.user_defined_api[QL_INTERCEPT.ENTER].get(fname) onexit = ql.os.user_defined_api[QL_INTERCEPT.EXIT].get(fname) - return ql.os.call(pc, f, params, onenter, onexit) + return ql.os.call(pc, f, params or {}, onenter, onexit) return wrapper From e14ccf1a8d903fe1099c075ac84e5de3a99801f6 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:43:29 +0300 Subject: [PATCH 102/138] Enable passthrough on dxeapi --- qiling/os/uefi/fncc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiling/os/uefi/fncc.py b/qiling/os/uefi/fncc.py index 3e354920..6294cbd7 100644 --- a/qiling/os/uefi/fncc.py +++ b/qiling/os/uefi/fncc.py @@ -8,7 +8,7 @@ from typing import Any, Mapping, Optional from qiling import Qiling from qiling.const import QL_INTERCEPT -def dxeapi(params: Optional[Mapping[str, Any]] = None): +def dxeapi(params: Optional[Mapping[str, Any]] = None, passthru: bool = False): def decorator(func): def wrapper(ql: Qiling): pc = ql.arch.regs.arch_pc @@ -18,7 +18,7 @@ def dxeapi(params: Optional[Mapping[str, Any]] = None): onenter = ql.os.user_defined_api[QL_INTERCEPT.ENTER].get(fname) onexit = ql.os.user_defined_api[QL_INTERCEPT.EXIT].get(fname) - return ql.os.call(pc, f, params or {}, onenter, onexit) + return ql.os.call(pc, f, params or {}, onenter, onexit, passthru) return wrapper From 18d812efc1b3e016b2979febebd5dbf8b4c59184 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:45:05 +0300 Subject: [PATCH 103/138] Add SimpleText protocols to UEFI --- qiling/os/uefi/UefiSpec.py | 14 +- .../uefi/protocols/EfiSimpleTextInProtocol.py | 56 ++++++++ .../protocols/EfiSimpleTextOutProtocol.py | 128 ++++++++++++++++++ qiling/os/uefi/st.py | 104 ++++++++------ 4 files changed, 254 insertions(+), 48 deletions(-) create mode 100644 qiling/os/uefi/protocols/EfiSimpleTextInProtocol.py create mode 100644 qiling/os/uefi/protocols/EfiSimpleTextOutProtocol.py diff --git a/qiling/os/uefi/UefiSpec.py b/qiling/os/uefi/UefiSpec.py index 2259e8c3..583ef6d8 100644 --- a/qiling/os/uefi/UefiSpec.py +++ b/qiling/os/uefi/UefiSpec.py @@ -10,6 +10,10 @@ from .ProcessorBind import * from .UefiBaseType import * from .UefiMultiPhase import * +from .protocols.EfiSimpleTextInProtocol import EFI_SIMPLE_TEXT_INPUT_PROTOCOL +from .protocols.EfiSimpleTextOutProtocol import EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL + + # definitions for EFI_TIME.Daylight EFI_TIME_ADJUST_DAYLIGHT = (1 << 1) EFI_TIME_IN_DAYLIGHT = (1 << 2) @@ -223,14 +227,6 @@ class EFI_CONFIGURATION_TABLE(STRUCT): ('VendorTable', PTR(VOID)), ] -# TODO: to be implemented -# @see: MdePkg\Include\Protocol\SimpleTextIn.h -EFI_SIMPLE_TEXT_INPUT_PROTOCOL = STRUCT - -# TODO: to be implemented -# @see: MdePkg\Include\Protocol\SimpleTextOut.h -EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL = STRUCT - class EFI_SYSTEM_TABLE(STRUCT): _pack_ = 8 @@ -264,4 +260,4 @@ __all__ = [ 'EFI_DEVICE_PATH_PROTOCOL', 'EFI_OPEN_PROTOCOL_INFORMATION_ENTRY', 'EFI_IMAGE_UNLOAD' -] \ No newline at end of file +] diff --git a/qiling/os/uefi/protocols/EfiSimpleTextInProtocol.py b/qiling/os/uefi/protocols/EfiSimpleTextInProtocol.py new file mode 100644 index 00000000..1a8e3eed --- /dev/null +++ b/qiling/os/uefi/protocols/EfiSimpleTextInProtocol.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +from qiling.os.const import * +from qiling.os.uefi.fncc import dxeapi +from qiling.os.uefi.utils import * +from qiling.os.uefi.ProcessorBind import * +from qiling.os.uefi.UefiBaseType import EFI_STATUS, EFI_EVENT + + +# @see: MdePkg/Include/Protocol/SimpleTextIn.h +class EFI_INPUT_KEY(STRUCT): + _fields_ = [ + ('ScanCode', UINT16), + ('UnicodeChar', CHAR16) + ] + +class EFI_SIMPLE_TEXT_INPUT_PROTOCOL(STRUCT): + EFI_SIMPLE_TEXT_INPUT_PROTOCOL = STRUCT + + _fields_ = [ + ('Reset', FUNCPTR(EFI_STATUS, PTR(EFI_SIMPLE_TEXT_INPUT_PROTOCOL), BOOLEAN)), + ('ReadKeyStroke', FUNCPTR(EFI_STATUS, PTR(EFI_SIMPLE_TEXT_INPUT_PROTOCOL), PTR(EFI_INPUT_KEY))), + ('WaitForKey', EFI_EVENT) + ] + + +@dxeapi(params={ + "This": POINTER, # IN PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL) + "ExtendedVerification": BOOL # IN BOOLEAN +}) +def hook_Input_Reset(ql: Qiling, address: int, params): + pass + +@dxeapi(params={ + "This": POINTER, # IN PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL) + "Key": POINTER # OUT PTR(EFI_INPUT_KEY) +}) +def hook_Read_Key_Stroke(ql: Qiling, address: int, params): + pass + + +def initialize(ql: Qiling, gIP: int): + descriptor = { + 'struct': EFI_SIMPLE_TEXT_INPUT_PROTOCOL, + 'fields': ( + ('Reset', hook_Input_Reset), + ('ReadKeyStroke', hook_Read_Key_Stroke), + ('WaitForKey', None) + ) + } + + instance = init_struct(ql, gIP, descriptor) + instance.save_to(ql.mem, gIP) diff --git a/qiling/os/uefi/protocols/EfiSimpleTextOutProtocol.py b/qiling/os/uefi/protocols/EfiSimpleTextOutProtocol.py new file mode 100644 index 00000000..d69cd3a3 --- /dev/null +++ b/qiling/os/uefi/protocols/EfiSimpleTextOutProtocol.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +from qiling.os.const import * +from qiling.os.uefi.fncc import dxeapi +from qiling.os.uefi.utils import * +from qiling.os.uefi.ProcessorBind import * +from qiling.os.uefi.UefiBaseType import EFI_STATUS + + +# @see: MdePkg/Include/Protocol/SimpleTextOut.h +class SIMPLE_TEXT_OUTPUT_MODE(STRUCT): + _fields_ = [ + ("MaxMode", INT32), + ("Mode", INT32), + ("Attribute", INT32), + ("CursorColumn", INT32), + ("CursorRow", INT32), + ("CursorVisible", BOOLEAN), + ] + + +class EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL(STRUCT): + EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL = STRUCT + + _fields_ = [ + ("Reset", FUNCPTR(EFI_STATUS, PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL), BOOLEAN)), + ("OutputString", FUNCPTR(EFI_STATUS, PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL), PTR(CHAR16))), + ("TestString", FUNCPTR(EFI_STATUS, PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL), PTR(CHAR16))), + ("QueryMode", FUNCPTR(EFI_STATUS, PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL), UINTN, PTR(UINTN), PTR(UINTN))), + ("SetMode", FUNCPTR(EFI_STATUS, PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL), UINTN)), + ("SetAttribute", FUNCPTR(EFI_STATUS, PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL), UINTN)), + ("ClearScreen", FUNCPTR(EFI_STATUS, PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL))), + ("SetCursorPosition", FUNCPTR(EFI_STATUS, PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL), UINTN, UINTN)), + ("EnableCursor", FUNCPTR(EFI_STATUS, PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL), BOOLEAN)), + ("Mode", PTR(SIMPLE_TEXT_OUTPUT_MODE)) + ] + + +@dxeapi(params={ + "This": POINTER, # IN PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL) + "ExtendedVerification": BOOL # IN BOOLEAN +}) +def hook_TextReset(ql: Qiling, address: int, params): + pass + +@dxeapi(params={ + "This": POINTER, # IN PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL) + "String": WSTRING # IN PTR(CHAR16) +}) +def hook_OutputString(ql: Qiling, address: int, params): + print(params['String']) + + return EFI_SUCCESS + +@dxeapi(params={ + "This": POINTER, # IN PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL) + "String": WSTRING # IN PTR(CHAR16) +}) +def hook_TestString(ql: Qiling, address: int, params): + pass + +@dxeapi(params={ + "This": POINTER, # IN PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL) + "ModeNumber": ULONGLONG, # IN UINTN + "Columns": POINTER, # OUT PTR(UINTN) + "Rows": POINTER # OUT PTR(UINTN) +}) +def hook_QueryMode(ql: Qiling, address: int, params): + pass + +@dxeapi(params={ + "This": POINTER, # IN PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL) + "ModeNumber": ULONGLONG # IN UINTN +}) +def hook_SetMode(ql: Qiling, address: int, params): + pass + +@dxeapi(params={ + "This": POINTER, # IN PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL) + "Attribute": ULONGLONG # IN UINTN +}) +def hook_SetAttribute(ql: Qiling, address: int, params): + pass + +@dxeapi(params={ + "This": POINTER # IN PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL) +}) +def hook_ClearScreen(ql: Qiling, address: int, params): + pass + +@dxeapi(params={ + "This": POINTER, # IN PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL) + "Column": ULONGLONG, # IN UINTN + "Row": ULONGLONG # IN UINTN +}) +def hook_SetCursorPosition(ql: Qiling, address: int, params): + pass + +@dxeapi(params={ + "This": POINTER, # IN PTR(EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL) + "Visible": BOOL # IN BOOLEAN +}) +def hook_EnableCursor(ql: Qiling, address: int, params): + pass + + +def initialize(ql: Qiling, base: int): + descriptor = { + 'struct': EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL, + 'fields': ( + ('Reset', hook_TextReset), + ('OutputString', hook_OutputString), + ('TestString', hook_TestString), + ('QueryMode', hook_QueryMode), + ('SetMode', hook_SetMode), + ('SetAttribute', hook_SetAttribute), + ('ClearScreen', hook_ClearScreen), + ('SetCursorPosition', hook_SetCursorPosition), + ('EnableCursor', hook_EnableCursor), + ('Mode', None) + ) + } + + instance = init_struct(ql, base, descriptor) + instance.save_to(ql.mem, base) diff --git a/qiling/os/uefi/st.py b/qiling/os/uefi/st.py index b5fca922..6163b6c7 100644 --- a/qiling/os/uefi/st.py +++ b/qiling/os/uefi/st.py @@ -3,58 +3,79 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -from qiling import Qiling +from typing import TYPE_CHECKING + from qiling.os.uefi import bs, rt, ds from qiling.os.uefi.context import UefiContext from qiling.os.uefi.utils import install_configuration_table -from qiling.os.uefi.UefiSpec import EFI_SYSTEM_TABLE, EFI_BOOT_SERVICES, EFI_RUNTIME_SERVICES +from qiling.os.uefi.UefiSpec import EFI_SYSTEM_TABLE, EFI_SIMPLE_TEXT_INPUT_PROTOCOL, EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL, EFI_BOOT_SERVICES, EFI_RUNTIME_SERVICES + +import qiling.os.uefi.protocols.EfiSimpleTextInProtocol as txt_in +import qiling.os.uefi.protocols.EfiSimpleTextOutProtocol as txt_out + + +if TYPE_CHECKING: + from qiling import Qiling # static mem layout: # -# +-- EFI_SYSTEM_TABLE ---------+ -# | | -# | ... | -# | RuntimeServices* -> (1) | -# | BootServices* -> (2) | -# | NumberOfTableEntries | -# | ConfigurationTable* -> (4) | -# +-----------------------------+ -# (1) +-- EFI_RUNTIME_SERVICES -----+ -# | | -# | ... | -# +-----------------------------+ -# (2) +-- EFI_BOOT_SERVICES --------+ -# | | -# | ... | -# +-----------------------------+ -# (3) +-- EFI_DXE_SERVICES ---------+ -# | | -# | ... | -# +-----------------------------+ -# (4) +-- EFI_CONFIGURATION_TABLE --+ of HOB_LIST -# | VendorGuid | -# | VendorTable* -> (5) | -# +-----------------------------+ -# +-- EFI_CONFIGURATION_TABLE --+ of DXE_SERVICE_TABLE -# | VendorGuid | -# | VendorTable* -> (3) | -# +-----------------------------+ +# +-- EFI_SYSTEM_TABLE -----------------+ +# | | +# | ... | +# | ConIn* -> (1) | +# | ConOut* -> (2) | +# | RuntimeServices* -> (3) | +# | BootServices* -> (4) | +# | NumberOfTableEntries | +# | ConfigurationTable* -> (6) | +# +-------------------------------------+ +# (1) +-- EFI_SIMPLE_TEXT_INPUT_PROTOCOL ---+ +# | | +# | ... | +# +-------------------------------------+ +# (2) +-- EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL --+ +# | | +# | ... | +# +-------------------------------------+ +# (3) +-- EFI_RUNTIME_SERVICES -------------+ +# | | +# | ... | +# +-------------------------------------+ +# (4) +-- EFI_BOOT_SERVICES ----------------+ +# | | +# | ... | +# +-------------------------------------+ +# (5) +-- EFI_DXE_SERVICES -----------------+ +# | | +# | ... | +# +-------------------------------------+ +# (6) +-- EFI_CONFIGURATION_TABLE ----------+ of HOB_LIST +# | VendorGuid | +# | VendorTable* -> (7) | +# +-------------------------------------+ +# +-- EFI_CONFIGURATION_TABLE ----------+ of DXE_SERVICE_TABLE +# | VendorGuid | +# | VendorTable* -> (5) | +# +-------------------------------------+ # # ... the remainder of the chunk may be used for additional EFI_CONFIGURATION_TABLE entries - +# # dynamically allocated (context.conf_table_data_ptr): # -# (5) +-- VOID* --------------------+ -# | ... | -# +-----------------------------+ +# (7) +-- VOID* ----------------------------+ +# | ... | +# +-------------------------------------+ + def initialize(ql: Qiling, context: UefiContext, gST: int): ql.loader.gST = gST - gBS = gST + EFI_SYSTEM_TABLE.sizeof() # boot services - gRT = gBS + EFI_BOOT_SERVICES.sizeof() # runtime services - gDS = gRT + EFI_RUNTIME_SERVICES.sizeof() # dxe services - cfg = gDS + ds.EFI_DXE_SERVICES.sizeof() # configuration tables array + sti = gST + EFI_SYSTEM_TABLE.sizeof() # input protocols + sto = sti + EFI_SIMPLE_TEXT_INPUT_PROTOCOL.sizeof() # output protocols + gRT = sto + EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL.sizeof() # runtime services + gBS = gRT + EFI_RUNTIME_SERVICES.sizeof() # boot services + gDS = gBS + EFI_BOOT_SERVICES.sizeof() # dxe services + cfg = gDS + ds.EFI_DXE_SERVICES.sizeof() # configuration tables array ql.log.info(f'Global tables:') ql.log.info(f' | gST {gST:#010x}') @@ -63,11 +84,16 @@ def initialize(ql: Qiling, context: UefiContext, gST: int): ql.log.info(f' | gDS {gDS:#010x}') ql.log.info(f'') + txt_in.initialize(ql, sti) + txt_out.initialize(ql, sto) + bs.initialize(ql, gBS) rt.initialize(ql, gRT) ds.initialize(ql, gDS) EFI_SYSTEM_TABLE( + ConIn = sti, + ConOut = sto, RuntimeServices = gRT, BootServices = gBS, NumberOfTableEntries = 0, @@ -79,4 +105,4 @@ def initialize(ql: Qiling, context: UefiContext, gST: int): __all__ = [ 'initialize' -] \ No newline at end of file +] From 0af477cde5b1aad310332f1d37880cbac5d4a4eb Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:48:15 +0300 Subject: [PATCH 104/138] Fix the way input btyes are passed on AFL crash validation --- qiling/extensions/afl/afl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qiling/extensions/afl/afl.py b/qiling/extensions/afl/afl.py index 4aef943e..4128af5a 100644 --- a/qiling/extensions/afl/afl.py +++ b/qiling/extensions/afl/afl.py @@ -96,8 +96,8 @@ def ql_afl_fuzz_custom(ql: Qiling, def __place_input_wrapper(uc: Uc, input_bytes: Array[c_char], iters: int, context: Any) -> bool: return place_input_callback(ql, input_bytes.raw, iters) - def __validate_crash_wrapper(uc: Uc, result: int, input_bytes: bytes, iters: int, context: Any) -> bool: - return validate_crash_callback(ql, result, input_bytes, iters) + def __validate_crash_wrapper(uc: Uc, result: int, input_bytes: Array[c_char], iters: int, context: Any) -> bool: + return validate_crash_callback(ql, result, input_bytes.raw, iters) def __fuzzing_wrapper(uc: Uc, context: Any) -> int: return fuzzing_callback(ql) From 6b136dda1dca825519d465c543c8762cbd4b4e52 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:50:25 +0300 Subject: [PATCH 105/138] Annotate QlDisk methods --- qiling/os/disk.py | 77 ++++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/qiling/os/disk.py b/qiling/os/disk.py index 765712ac..ddc68e4a 100644 --- a/qiling/os/disk.py +++ b/qiling/os/disk.py @@ -1,23 +1,33 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from typing import AnyStr, Optional, Union from .mapper import QlFsMappedObject +ReadableBuffer = Union[bytes, bytearray, memoryview] + + # Open a file as a Disk # host_path: The file path on the host machine. # drive_path: The drive path on the emulated system. e.g. /dev/sda \\.\PHYSICALDRIVE0 0x80 -# +# # Note: CHS and LBA support is very limited since a raw file doesn't contain enough information. # We simply assume that it is a disk with 1 head, 1 cylinder and (filesize/512) sectors. +# # See: https://en.wikipedia.org/wiki/Cylinder-head-sector # https://en.wikipedia.org/wiki/Logical_block_addressing # http://www.uruk.org/orig-grub/PC_partitioning.txt + class QlDisk(QlFsMappedObject): - def __init__(self, host_path, drive_path, n_heads=1, n_cylinders=1, sector_size=512): - self._host_path = host_path + # 512 bytes/sector + # 63 sectors/track + # 255 heads (tracks/cylinder) + # 1024 cylinders + + def __init__(self, host_path: AnyStr, drive_path, n_cylinders: int = 1, n_heads: int = 1, sector_size: int = 512): self._drive_path = drive_path self._fp = open(host_path, "rb+") self._n_heads = n_heads @@ -25,7 +35,7 @@ class QlDisk(QlFsMappedObject): self._sector_size = sector_size self.lseek(0, 2) self._filesize = self.tell() - self._n_sectors = (self._filesize - 1)// self.sector_size + 1 + self._n_sectors = (self._filesize - 1) // self.sector_size + 1 def __del__(self): if not self.fp.closed: @@ -51,50 +61,43 @@ class QlDisk(QlFsMappedObject): def sector_size(self): return self._sector_size - @property - def host_path(self): - return self._host_path - - @property - def drive_path(self): - return self._drive_path - @property def fp(self): return self._fp # Methods from FsMappedObject - def read(self, l): - return self.fp.read(l) - - def write(self, bs): - return self.fp.write(bs) + def read(self, size: Optional[int]) -> bytes: + return self.fp.read(size) - def lseek(self, offset, origin): + def write(self, buffer: ReadableBuffer) -> int: + return self.fp.write(buffer) + + def lseek(self, offset: int, origin: int) -> int: return self.fp.seek(offset, origin) - - def tell(self): + + def tell(self) -> int: return self.fp.tell() - def close(self): - return self.fp.close() - + def close(self) -> None: + self.fp.close() + # Methods for QlDisk - def lba(self, cylinder, head, sector): + def lba(self, cylinder: int, head: int, sector: int) -> int: return (cylinder * self.n_heads + head) * self._n_sectors + sector - 1 - - def read_sectors(self, lba, cnt): + + def read_sectors(self, lba: int, cnt: int) -> bytes: self.lseek(self.sector_size * lba, 0) - return self.read(self.sector_size*cnt) - - def read_chs(self, cylinder, head, sector, cnt): + + return self.read(self.sector_size * cnt) + + def read_chs(self, cylinder: int, head: int, sector: int, cnt: int) -> bytes: return self.read_sectors(self.lba(cylinder, head, sector), cnt) - def write_sectors(self, lba, cnt, buffer): - if len(buffer) > self.sector_size * cnt: - buffer = buffer[:self.sector_size*cnt] + def write_sectors(self, lba: int, cnt: int, buffer: ReadableBuffer) -> int: + buffer = memoryview(buffer) self.lseek(self.sector_size * lba, 0) - return self.write(buffer) - - def write_chs(self, cylinder, head, sector, cnt, buffer): - return self.write_sectors(self.lba(cylinder, head, sector), cnt, buffer) \ No newline at end of file + + return self.write(buffer[:self.sector_size * cnt]) + + def write_chs(self, cylinder: int, head: int, sector: int, cnt: int, buffer: ReadableBuffer): + return self.write_sectors(self.lba(cylinder, head, sector), cnt, buffer) From 7531c330aace8b3ed034608fa8fafc25b278f947 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:50:59 +0300 Subject: [PATCH 106/138] Misc DOS additions --- qiling/os/dos/interrupts/int21.py | 44 +++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/qiling/os/dos/interrupts/int21.py b/qiling/os/dos/interrupts/int21.py index da9ea64e..0b3dc02f 100644 --- a/qiling/os/dos/interrupts/int21.py +++ b/qiling/os/dos/interrupts/int21.py @@ -9,11 +9,6 @@ from qiling import Qiling from .. import utils -# exit -def __leaf_4c(ql: Qiling): - ql.log.info("Program terminated gracefully") - ql.emu_stop() - # write a character to screen def __leaf_02(ql: Qiling): ch = ql.arch.regs.dl @@ -131,6 +126,45 @@ def __leaf_43(ql: Qiling): ql.arch.regs.cx = 0xffff ql.os.clear_cf() + +def __leaf_48(ql: Qiling): + """Allocate memory. + """ + + size = ql.arch.regs.bx * 0x10 + + # announce it but do not do anything really + ql.log.debug(f'allocating memory block at {addr:#06x} to {size:#x} bytes') + + # success + ql.os.clear_cf() + + +def __leaf_49(ql: Qiling): + """Deallocate memory. + """ + ... + + +def __leaf_4a(ql: Qiling): + """Modify memory allocation. + """ + + addr = ql.arch.regs.es + size = ql.arch.regs.bx * 0x10 + + # announce it but do not do anything really + ql.log.debug(f'resizing memory block at {addr:#06x} to {size:#x} bytes') + + # success + ql.os.clear_cf() + + +def __leaf_4c(ql: Qiling): + ql.log.info("Program terminated gracefully") + ql.emu_stop() + + def handler(ql: Qiling): ah = ql.arch.regs.ah From 0d9c722aa7f434a4af9ef94205f7e9e1457cedc7 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:51:22 +0300 Subject: [PATCH 107/138] Misc Linux kernel API additions --- qiling/os/linux/kernel_api/kernel_api.py | 78 +++++++++++++++++------- 1 file changed, 55 insertions(+), 23 deletions(-) diff --git a/qiling/os/linux/kernel_api/kernel_api.py b/qiling/os/linux/kernel_api/kernel_api.py index c16ed325..43510e3e 100644 --- a/qiling/os/linux/kernel_api/kernel_api.py +++ b/qiling/os/linux/kernel_api/kernel_api.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -# +# # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # @@ -53,31 +53,63 @@ def hook_mcount(ql, address, params): return 0 -@linux_kernel_api(params={ - "Ptr": POINTER -}) -def hook___x86_indirect_thunk_rax(ql, address, params): - return 0 +def __x86_indirect_thunk(ql: Qiling, dest: int): + ql.log.debug('retpoline to %#010x', dest) + + ql.arch.regs.arch_pc = dest + +# using passthru as a hack to avoid syscall handler overwrite instruction pointer +@linux_kernel_api(passthru=True) +def hook___x86_indirect_thunk_rax(ql: Qiling, address: int, params): + __x86_indirect_thunk(ql, ql.arch.regs.rax) -@linux_kernel_api(params={ - "Ptr": POINTER -}) -def hook__copy_to_user(ql, address, params): - return 0 - - -@linux_kernel_api(params={ - "Ptr": POINTER -}) -def hook__copy_from_user(ql, address, params): - return 0 - - -@linux_kernel_api(params={ - "Ptr": POINTER -}) +@linux_kernel_api(passthru=True) def hook___x86_indirect_thunk_r14(ql, address, params): + __x86_indirect_thunk(ql, ql.arch.regs.r14) + + +@linux_kernel_api(params={ + "ubuf": POINTER, + "kbuf": POINTER, + "count": SIZE_T +}) +def hook__copy_to_user(ql: Qiling, address: int, params) -> int: + ubuf = params['ubuf'] + kbuf = params['kbuf'] + count = params['count'] + + # if user-mode buffer is not available, fail + # TODO: also fail if destination is not writeable + if not ql.mem.is_mapped(ubuf, count): + return count + + data = ql.mem.read(kbuf, count) + + ql.mem.write(ubuf, data) + + return 0 + + +@linux_kernel_api(params={ + "kbuf": POINTER, + "ubuf": POINTER, + "count": SIZE_T +}) +def hook__copy_from_user(ql: Qiling, address: int, params) -> int: + ubuf = params['ubuf'] + kbuf = params['kbuf'] + count = params['count'] + + # if user-mode buffer is not available, fail + # TODO: also fail if source is not readable + if not ql.mem.is_mapped(ubuf, count): + return count + + data = ql.mem.read(ubuf, count) + + ql.mem.write(kbuf, data) + return 0 From b0d6384468c110e45e5b8f22667767d22ec97770 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 12:51:59 +0300 Subject: [PATCH 108/138] Misc examples fixes --- .../dlink_dir815/dir815_mips32el_linux.py | 46 +++++++++---------- examples/sality.py | 2 +- examples/tendaac1518_httpd.py | 2 + 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/examples/fuzzing/dlink_dir815/dir815_mips32el_linux.py b/examples/fuzzing/dlink_dir815/dir815_mips32el_linux.py index 28afce92..22ea02ff 100644 --- a/examples/fuzzing/dlink_dir815/dir815_mips32el_linux.py +++ b/examples/fuzzing/dlink_dir815/dir815_mips32el_linux.py @@ -5,7 +5,7 @@ # Everything about the bug and firmware https://www.exploit-db.com/exploits/33863 -import os,sys +import sys sys.path.append("../../..") from qiling import Qiling @@ -13,7 +13,7 @@ from qiling.const import QL_VERBOSE from qiling.extensions.afl import ql_afl_fuzz -def main(input_file, enable_trace=False): +def main(input_file: str): env_vars = { "REQUEST_METHOD": "POST", @@ -24,40 +24,36 @@ def main(input_file, enable_trace=False): # "CONTENT_LENGTH": "8", # no needed } - ql = Qiling(["./rootfs/htdocs/web/hedwig.cgi"], "./rootfs", - verbose=QL_VERBOSE.DEBUG, env=env_vars, console=enable_trace) + ql = Qiling(["./rootfs/htdocs/web/hedwig.cgi"], "./rootfs", verbose=QL_VERBOSE.DISABLED, env=env_vars) - def place_input_callback(ql: Qiling, input: bytes, _: int): - env_var = ("HTTP_COOKIE=uid=1234&password=").encode() - env_vars = env_var + input + b"\x00" + (ql.path).encode() + b"\x00" - ql.mem.write(ql.target_addr, env_vars) + def place_input_callback(ql: Qiling, data: bytes, _: int) -> bool: + # construct the payload + payload = b''.join((b"HTTP_COOKIE=uid=1234&password=", bytes(data), b"\x00", ql_path, b"\x00")) + + # patch the value of 'HTTP_COOKIE' in memory + ql.mem.write(target_addr, payload) + + # payload is in place, we are good to go + return True def start_afl(_ql: Qiling): - """ Callback from inside """ + ql_afl_fuzz(_ql, input_file=input_file, place_input_callback=place_input_callback, exits=[ql.os.exit_point]) - addr = ql.mem.search("HTTP_COOKIE=uid=1234&password=".encode()) - ql.target_addr = addr[0] + addr = ql.mem.search(b"HTTP_COOKIE=uid=1234&password=") + target_addr = addr[0] + ql_path = ql.path.encode() - main_addr = ql.loader.elf_entry - ql.hook_address(callback=start_afl, address=main_addr) + ql.hook_address(start_afl, ql.loader.elf_entry) - try: - ql.run() - os._exit(0) - except: - if enable_trace: - print("\nFuzzer Went Shit") - os._exit(0) + ql.run() if __name__ == "__main__": - if len(sys.argv) == 1: + if len(sys.argv) < 2: raise ValueError("No input file provided.") - if len(sys.argv) > 2 and sys.argv[1] == "-t": - main(sys.argv[2], enable_trace=True) - else: - main(sys.argv[1]) + + main(sys.argv[1]) diff --git a/examples/sality.py b/examples/sality.py index 22d6f651..be05753b 100644 --- a/examples/sality.py +++ b/examples/sality.py @@ -159,7 +159,7 @@ def hook_StartServiceA(ql: Qiling, address: int, params): init_unseen_symbols(ql.amsint32_driver, ntoskrnl.base+0xb7695, b"NtTerminateProcess", 0, "ntoskrnl.exe") #ql.amsint32_driver.debugger= ":9999" try: - ql.amsint32_driver.load() + ql.amsint32_driver.run() return 1 except UcError as e: print("Load driver error: ", e) diff --git a/examples/tendaac1518_httpd.py b/examples/tendaac1518_httpd.py index 0a32fd27..165aff1f 100644 --- a/examples/tendaac1518_httpd.py +++ b/examples/tendaac1518_httpd.py @@ -78,6 +78,8 @@ def my_sandbox(path, rootfs): ql.os.set_syscall('vfork', __vfork) + os.unlink(fr'{ROOTFS}/proc/sys/kernel/core_pattern') + ql.run() From 44087e16b0e9362f7078c77b6d96cdc4a9aaee65 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 10 Oct 2025 13:03:40 +0300 Subject: [PATCH 109/138] Add missing import --- qiling/os/uefi/st.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qiling/os/uefi/st.py b/qiling/os/uefi/st.py index 6163b6c7..305c4664 100644 --- a/qiling/os/uefi/st.py +++ b/qiling/os/uefi/st.py @@ -3,6 +3,8 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +from __future__ import annotations + from typing import TYPE_CHECKING from qiling.os.uefi import bs, rt, ds From a8edbe1323f7b2f0c2d98c711d66cddd34b46760 Mon Sep 17 00:00:00 2001 From: juliangrtz Date: Mon, 13 Oct 2025 21:42:38 +0200 Subject: [PATCH 110/138] Support IDA 9.x --- qiling/extensions/idaplugin/qilingida.py | 28 ++++++++++-------------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/qiling/extensions/idaplugin/qilingida.py b/qiling/extensions/idaplugin/qilingida.py index 9aaebc35..045d8009 100644 --- a/qiling/extensions/idaplugin/qilingida.py +++ b/qiling/extensions/idaplugin/qilingida.py @@ -38,8 +38,8 @@ import ida_netnode import ida_hexrays import ida_range # PyQt -from PyQt5 import QtCore, QtWidgets -from PyQt5.QtWidgets import (QPushButton, QHBoxLayout) +from PySide6 import QtCore, QtWidgets +from PySide6.QtWidgets import (QPushButton, QHBoxLayout) # Qiling from qiling import Qiling @@ -293,25 +293,21 @@ class IDA: def get_input_file_path(): return ida_nalt.get_input_file_path() - @staticmethod - def get_info_structure(): - return ida_idaapi.get_inf_structure() - @staticmethod def get_main_address(): - return IDA.get_info_structure().main + return ida_ida.inf_get_main() @staticmethod def get_max_address(): - return IDA.get_info_structure().max_ea + return ida_ida.inf_get_max_ea() @staticmethod def get_min_address(): - return IDA.get_info_structure().min_ea + return ida_ida.inf_get_min_ea() @staticmethod def is_big_endian(): - return IDA.get_info_structure().is_be() + return ida_ida.inf_is_be() @staticmethod def is_little_endian(): @@ -319,8 +315,7 @@ class IDA: @staticmethod def get_filetype(): - info = IDA.get_info_structure() - ftype = info.filetype + ftype = ida_ida.inf_get_filetype() if ftype == ida_ida.f_MACHO: return "macho" elif ftype == ida_ida.f_PE or ftype == ida_ida.f_EXE or ftype == ida_ida.f_EXE_old: # is this correct? @@ -332,18 +327,17 @@ class IDA: @staticmethod def get_ql_arch_string(): - info = IDA.get_info_structure() - proc = info.procname.lower() + proc = ida_ida.inf_get_procname().lower() result = None if proc == "metapc": result = "x86" - if info.is_64bit(): + if ida_ida.inf_is_64bit(): result = "x8664" elif "mips" in proc: result = "mips" elif "arm" in proc: result = "arm32" - if info.is_64bit(): + if ida_ida.inf_is_64bit(): result = "arm64" # That's all we support :( return result @@ -1006,7 +1000,7 @@ class QlEmuPlugin(plugin_t, UI_Hooks): def init(self): # init data logging.info('---------------------------------------------------------------------------------------') - logging.info('Qiling Emulator Plugin For IDA, by Qiling Team. Version {0}, 2020'.format(QLVERSION)) + logging.info('Qiling Emulator Plugin For IDA, by Qiling Team. Version {0}, 2025'.format(QLVERSION)) logging.info('Based on Qiling v{0}'.format(QLVERSION)) logging.info('Find more information about Qiling at https://qiling.io') logging.info('---------------------------------------------------------------------------------------') From bfbc1c83b0a14428371c52e9d8e49ab526ec8f44 Mon Sep 17 00:00:00 2001 From: juliangrtz Date: Tue, 14 Oct 2025 10:11:17 +0200 Subject: [PATCH 111/138] =?UTF-8?q?Implement=20backwards=20compatibility?= =?UTF-8?q?=20for=20IDA=207=E2=80=938?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- qiling/extensions/idaplugin/qilingida.py | 31 ++++++++++++++++-------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/qiling/extensions/idaplugin/qilingida.py b/qiling/extensions/idaplugin/qilingida.py index 045d8009..c77ba75a 100644 --- a/qiling/extensions/idaplugin/qilingida.py +++ b/qiling/extensions/idaplugin/qilingida.py @@ -15,6 +15,8 @@ from elftools.elf.elffile import ELFFile from json import load # IDA Python SDK +IDA_VERSION = IDAPYTHON_VERSION[0] + from idaapi import * from idc import * from idautils import * @@ -38,8 +40,12 @@ import ida_netnode import ida_hexrays import ida_range # PyQt -from PySide6 import QtCore, QtWidgets -from PySide6.QtWidgets import (QPushButton, QHBoxLayout) +if IDA_VERSION >= 9: + from PySide6 import QtCore, QtWidgets + from PySide6.QtWidgets import (QPushButton, QHBoxLayout) +else: + from PyQt5 import QtCore, QtWidgets + from PyQt5.QtWidgets import (QPushButton, QHBoxLayout) # Qiling from qiling import Qiling @@ -293,21 +299,25 @@ class IDA: def get_input_file_path(): return ida_nalt.get_input_file_path() + @staticmethod + def get_info_structure(): + return ida_idaapi.get_inf_structure() + @staticmethod def get_main_address(): - return ida_ida.inf_get_main() + return ida_ida.inf_get_main() if IDA_VERSION >= 9 else IDA.get_info_structure().main @staticmethod def get_max_address(): - return ida_ida.inf_get_max_ea() + return ida_ida.inf_get_max_ea() if IDA_VERSION >= 9 else IDA.get_info_structure().max_ea @staticmethod def get_min_address(): - return ida_ida.inf_get_min_ea() + return ida_ida.inf_get_min_ea() if IDA_VERSION >= 9 else IDA.get_info_structure().min_ea @staticmethod def is_big_endian(): - return ida_ida.inf_is_be() + return ida_ida.inf_is_be() if IDA_VERSION >= 9 else IDA.get_info_structure().is_be @staticmethod def is_little_endian(): @@ -315,7 +325,7 @@ class IDA: @staticmethod def get_filetype(): - ftype = ida_ida.inf_get_filetype() + ftype = ida_ida.inf_get_filetype() if IDA_VERSION >= 9 else IDA.get_info_structure().filetype if ftype == ida_ida.f_MACHO: return "macho" elif ftype == ida_ida.f_PE or ftype == ida_ida.f_EXE or ftype == ida_ida.f_EXE_old: # is this correct? @@ -327,17 +337,18 @@ class IDA: @staticmethod def get_ql_arch_string(): - proc = ida_ida.inf_get_procname().lower() + proc = (ida_ida.inf_get_procname() if IDA_VERSION >= 9 else IDA.get_info_structure().procname).lower() result = None + is_64_bit = ida_ida.inf_is_64bit() if IDA_VERSION >= 9 else IDA.get_info_structure().is_64bit() if proc == "metapc": result = "x86" - if ida_ida.inf_is_64bit(): + if is_64_bit: result = "x8664" elif "mips" in proc: result = "mips" elif "arm" in proc: result = "arm32" - if ida_ida.inf_is_64bit(): + if is_64_bit: result = "arm64" # That's all we support :( return result From 2a4bf75ea94434c93a175574d1ed21b34c89bd51 Mon Sep 17 00:00:00 2001 From: juliangrtz Date: Wed, 15 Oct 2025 22:12:21 +0200 Subject: [PATCH 112/138] Separate IDA class into IDABase, IDA7 and IDA9 --- qiling/extensions/idaplugin/qilingida.py | 270 ++++++++++++++--------- 1 file changed, 162 insertions(+), 108 deletions(-) diff --git a/qiling/extensions/idaplugin/qilingida.py b/qiling/extensions/idaplugin/qilingida.py index c77ba75a..cdbcbf6b 100644 --- a/qiling/extensions/idaplugin/qilingida.py +++ b/qiling/extensions/idaplugin/qilingida.py @@ -5,7 +5,6 @@ UseAsScript = True import sys import collections -import time import struct import re import logging @@ -15,8 +14,6 @@ from elftools.elf.elffile import ELFFile from json import load # IDA Python SDK -IDA_VERSION = IDAPYTHON_VERSION[0] - from idaapi import * from idc import * from idautils import * @@ -39,13 +36,6 @@ import ida_auto import ida_netnode import ida_hexrays import ida_range -# PyQt -if IDA_VERSION >= 9: - from PySide6 import QtCore, QtWidgets - from PySide6.QtWidgets import (QPushButton, QHBoxLayout) -else: - from PyQt5 import QtCore, QtWidgets - from PyQt5.QtWidgets import (QPushButton, QHBoxLayout) # Qiling from qiling import Qiling @@ -61,7 +51,6 @@ from qiling import __version__ as QLVERSION from qiling.os.filestruct import ql_file from keystone import * - QilingHomePage = 'https://www.qiling.io' QilingStableVersionURL = 'https://raw.githubusercontent.com/qilingframework/qiling/master/qiling/__version__.py' logging.basicConfig(level=logging.INFO, format='[%(levelname)s][%(module)s:%(lineno)d] %(message)s') @@ -75,7 +64,27 @@ class Colors(Enum): Gray = 0xd9d9d9 Beige = 0xCCF2FF -class IDA: +def _load_qt_bindings(): + if IDA_SDK_VERSION >= 900: + try: + from PySide6 import QtCore, QtWidgets + from PySide6.QtWidgets import (QPushButton, QHBoxLayout) + logging.info("Using PySide6 for Qt bindings (IDA >= 9).") + return QtCore, QtWidgets, QPushButton, QHBoxLayout + except Exception as e: + logging.warning("Failed to import PySide6: %s. Trying PyQt5 fallback.", e) + try: + from PyQt5 import QtCore, QtWidgets + from PyQt5.QtWidgets import (QPushButton, QHBoxLayout) + logging.info("Using PyQt5 for Qt bindings (IDA < 9 or fallback).") + return QtCore, QtWidgets, QPushButton, QHBoxLayout + except Exception as e: + logging.error("Failed to import PyQt bindings: %s", e) + raise + +QtCore, QtWidgets, QPushButton, QHBoxLayout = _load_qt_bindings() + +class IDABase: def __init__(self): pass @@ -85,15 +94,15 @@ class IDA: @staticmethod def get_function_start(addr): - return IDA.get_function(addr).start_ea + return IDABase.get_function(addr).start_ea @staticmethod def get_function_end(addr): - return IDA.get_function(addr).end_ea + return IDABase.get_function(addr).end_ea @staticmethod def get_function_framesize(addr): - return IDA.get_function(addr).frsize + return IDABase.get_function(addr).frsize @staticmethod def get_function_name(addr): @@ -101,7 +110,7 @@ class IDA: @staticmethod def get_functions(): - return [IDA.get_function(func) for func in idautils.Functions()] + return [IDABase.get_function(func) for func in idautils.Functions()] @staticmethod def set_color(addr, what, color): @@ -110,7 +119,7 @@ class IDA: @staticmethod def color_block(bb, color): for i in range(bb.start_ea, bb.end_ea): - IDA.set_color(i, idc.CIC_ITEM, color) + IDABase.set_color(i, idc.CIC_ITEM, color) # note: # corresponds to IDA graph view @@ -119,8 +128,8 @@ class IDA: # arg can be a function or a (start, end) tuple or an address in the function @staticmethod def get_flowchart(arg): - if type(arg) is int: - func = IDA.get_function(arg) + if isinstance(arg, int): + func = IDABase.get_function(arg) if func is None: return None return ida_gdl.FlowChart(func) @@ -128,7 +137,9 @@ class IDA: @staticmethod def get_block(addr): - flowchart = IDA.get_flowchart(addr) + flowchart = IDABase.get_flowchart(addr) + if flowchart is None: + return None for bb in flowchart: if bb.start_ea <= addr and addr < bb.end_ea: return bb @@ -149,10 +160,10 @@ class IDA: @staticmethod def get_starting_block(addr): - flowchart = IDA.get_flowchart(addr) + flowchart = IDABase.get_flowchart(addr) if flowchart is None: return None - func = IDA.get_function(addr) + func = IDABase.get_function(addr) for bb in flowchart: if bb.start_ea == func.start_ea: return bb @@ -160,8 +171,10 @@ class IDA: @staticmethod def get_terminating_blocks(addr): - flowchart = IDA.get_flowchart(addr) - return [bb for bb in flowchart if IDA.block_is_terminating(bb)] + flowchart = IDABase.get_flowchart(addr) + if flowchart is None: + return [] + return [bb for bb in flowchart if IDABase.block_is_terminating(bb)] @staticmethod def get_prev_head(addr, minea=0): @@ -186,46 +199,45 @@ class IDA: @staticmethod def __addr_in_seg(addr): - segs = IDA.get_segments() + segs = IDABase.get_segments() for seg in segs: if addr < seg.end_ea and addr >= seg.start_ea: return seg return None - # note: accept name and address in the segment @staticmethod def get_segment(arg): - if type(arg) is int: - return IDA.__addr_in_seg(arg) - else: # str - return IDA.get_segment_by_name(arg) + if isinstance(arg, int): + return IDABase.__addr_in_seg(arg) + else: + return IDABase.get_segment_by_name(arg) @staticmethod def get_segment_start(arg): - seg = IDA.get_segment(arg) + seg = IDABase.get_segment(arg) if seg is not None: return seg.start_ea return None @staticmethod def get_segment_end(arg): - seg = IDA.get_segment(arg) + seg = IDABase.get_segment(arg) if seg is not None: return seg.end_ea return None @staticmethod def get_segment_perm(arg): - seg = IDA.get_segment(arg) + seg = IDABase.get_segment(arg) if seg is not None: - return seg.perm # RWX e.g. 0b101 = R + X + return seg.perm return None @staticmethod def get_segment_type(arg): - seg = IDA.get_segment(arg) + seg = IDABase.get_segment(arg) if seg is not None: - return seg.type # 0x1 SEG_DATA 0x2 SEG_CODE See doc for details + return seg.type return None @staticmethod @@ -235,12 +247,10 @@ class IDA: return None return r - # immidiate value @staticmethod def get_operand(addr, n): return (idc.get_operand_type(addr, n), idc.get_operand_value(addr, n)) - # eax, ecx, etc @staticmethod def print_operand(addr, n): return idc.print_operand(addr, n) @@ -254,7 +264,7 @@ class IDA: p = begin cnt = 0 while p < end: - sz = IDA.get_instruction_size(p) + sz = IDABase.get_instruction_size(p) cnt += 1 p += sz return cnt @@ -299,95 +309,34 @@ class IDA: def get_input_file_path(): return ida_nalt.get_input_file_path() - @staticmethod - def get_info_structure(): - return ida_idaapi.get_inf_structure() - - @staticmethod - def get_main_address(): - return ida_ida.inf_get_main() if IDA_VERSION >= 9 else IDA.get_info_structure().main - - @staticmethod - def get_max_address(): - return ida_ida.inf_get_max_ea() if IDA_VERSION >= 9 else IDA.get_info_structure().max_ea - - @staticmethod - def get_min_address(): - return ida_ida.inf_get_min_ea() if IDA_VERSION >= 9 else IDA.get_info_structure().min_ea - - @staticmethod - def is_big_endian(): - return ida_ida.inf_is_be() if IDA_VERSION >= 9 else IDA.get_info_structure().is_be - - @staticmethod - def is_little_endian(): - return not IDA.is_big_endian() - - @staticmethod - def get_filetype(): - ftype = ida_ida.inf_get_filetype() if IDA_VERSION >= 9 else IDA.get_info_structure().filetype - if ftype == ida_ida.f_MACHO: - return "macho" - elif ftype == ida_ida.f_PE or ftype == ida_ida.f_EXE or ftype == ida_ida.f_EXE_old: # is this correct? - return "pe" - elif ftype == ida_ida.f_ELF: - return "elf" - else: - return None - - @staticmethod - def get_ql_arch_string(): - proc = (ida_ida.inf_get_procname() if IDA_VERSION >= 9 else IDA.get_info_structure().procname).lower() - result = None - is_64_bit = ida_ida.inf_is_64bit() if IDA_VERSION >= 9 else IDA.get_info_structure().is_64bit() - if proc == "metapc": - result = "x86" - if is_64_bit: - result = "x8664" - elif "mips" in proc: - result = "mips" - elif "arm" in proc: - result = "arm32" - if is_64_bit: - result = "arm64" - # That's all we support :( - return result - @staticmethod def get_current_address(): return ida_kernwin.get_screen_ea() - # return (?, start, end) @staticmethod def get_last_selection(): return ida_kernwin.read_range_selection(None) - # Use with skipcalls - # note that the address is the end of target instruction - # e.g.: - # 0x1 push eax - # 0x4 mov eax, 0 - # call get_frame_sp_delta(0x4) and get -4. @staticmethod def get_frame_sp_delta(addr): - return ida_frame.get_sp_delta(IDA.get_function(addr), addr) + return ida_frame.get_sp_delta(IDABase.get_function(addr), addr) @staticmethod def patch_bytes(addr, bs): return ida_bytes.patch_bytes(addr, bs) @staticmethod - def fill_bytes(start, end, bs = b'\x90'): + def fill_bytes(start, end, bs=b'\x90'): return ida_bytes.patch_bytes(start, bs*(end-start)) @staticmethod def nop_selection(): - _, start, end = IDA.get_last_selection() - return IDA.fill_bytes(start, end) + _, start, end = IDABase.get_last_selection() + return IDABase.fill_bytes(start, end) @staticmethod def fill_block(bb, bs=b'\x90'): - return IDA.fill_bytes(bb.start_ea, bb.end_ea, bs) + return IDABase.fill_bytes(bb.start_ea, bb.end_ea, bs) @staticmethod def assemble(ea, cs, ip, use32, line): @@ -399,7 +348,7 @@ class IDA: @staticmethod def create_bytes_array(start, end): - return IDA.create_data(start, ida_bytes.byte_flag(), end-start) + return IDABase.create_data(start, ida_bytes.byte_flag(), end-start) @staticmethod def create_byte(ea, length, force=False): @@ -423,13 +372,12 @@ class IDA: @staticmethod def get_item(ea): - return (IDA.get_item_head(ea), IDA.get_item_end(ea)) + return (IDABase.get_item_head(ea), IDABase.get_item_end(ea)) @staticmethod def is_colored_item(ea): return ida_nalt.is_colored_item(ea) - # NOTE: The [start, end) range should include all control flows except long calls. @staticmethod def get_micro_code_mba(start, end, decomp_flags=ida_hexrays.DECOMP_WARNINGS, maturity=7): mbrgs = ida_hexrays.mba_ranges_t() @@ -449,6 +397,112 @@ class IDA: cur = cur.next return +class IDA7(IDABase): + @staticmethod + def get_info_structure(): + return ida_idaapi.get_inf_structure() + + @staticmethod + def get_main_address(): + return IDA7.get_info_structure().main + + @staticmethod + def get_max_address(): + return IDA7.get_info_structure().max_ea + + @staticmethod + def get_min_address(): + return IDA7.get_info_structure().min_ea + + @staticmethod + def is_big_endian(): + return IDA7.get_info_structure().is_be + + @staticmethod + def is_little_endian(): + return not IDA7.is_big_endian() + + @staticmethod + def get_filetype(): + ftype = IDA7.get_info_structure().filetype + if ftype in (ida_ida.f_PE, ida_ida.f_EXE, ida_ida.f_EXE_old): + return "pe" + elif ftype == ida_ida.f_MACHO: + return "macho" + elif ftype == ida_ida.f_ELF: + return "elf" + return None + + @staticmethod + def get_ql_arch_string(): + proc = IDA7.get_info_structure().procname.lower() + is_64_bit = IDA7.get_info_structure().is_64bit() + if proc == "metapc": + return "x8664" if is_64_bit else "x86" + if "mips" in proc: + return "mips" + if "arm" in proc: + return "arm64" if is_64_bit else "arm32" + return None + +class IDA9(IDABase): + @staticmethod + def get_info_structure(): + return ida_idaapi.get_inf_structure() + + @staticmethod + def get_main_address(): + return ida_ida.inf_get_main() + + @staticmethod + def get_max_address(): + return ida_ida.inf_get_max_ea() + + @staticmethod + def get_min_address(): + return ida_ida.inf_get_min_ea() + + @staticmethod + def is_big_endian(): + return ida_ida.inf_is_be() + + @staticmethod + def is_little_endian(): + return not ida_ida.inf_is_be() + + @staticmethod + def get_filetype(): + ftype = ida_ida.inf_get_filetype() + if ftype in (ida_ida.f_PE, ida_ida.f_EXE, ida_ida.f_EXE_old): + return "pe" + elif ftype == ida_ida.f_MACHO: + return "macho" + elif ftype == ida_ida.f_ELF: + return "elf" + return None + + @staticmethod + def get_ql_arch_string(): + proc = ida_ida.inf_get_procname().lower() + is_64_bit = ida_ida.inf_is_64bit() + if proc == "metapc": + return "x8664" if is_64_bit else "x86" + if "mips" in proc: + return "mips" + if "arm" in proc: + return "arm64" if is_64_bit else "arm32" + return None + +def get_ida_instance(): + if IDA_SDK_VERSION >= 900: + logging.info("Using IDA9 compatibility layer") + return IDA9() + else: + logging.info("Using IDA7 compatibility layer") + return IDA7() + +IDA = get_ida_instance() + ### View Class class QlEmuRegView(simplecustviewer_t): From 147ec7c18c81871cf465e9f2c783ff487438d33a Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 29 Oct 2025 14:43:10 +0200 Subject: [PATCH 113/138] Make Windows compatible to latest image --- qiling/loader/pe.py | 1 + tests/test_pe_sys.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/qiling/loader/pe.py b/qiling/loader/pe.py index 8ea5884b..30959d0f 100644 --- a/qiling/loader/pe.py +++ b/qiling/loader/pe.py @@ -644,6 +644,7 @@ class Process: # DLLs that seem to contain most of the requested symbols key_dlls = ( + 'kernel32.dll', 'ntdll.dll', 'kernelbase.dll', 'ucrtbase.dll' diff --git a/tests/test_pe_sys.py b/tests/test_pe_sys.py index 546c0dee..17af4290 100644 --- a/tests/test_pe_sys.py +++ b/tests/test_pe_sys.py @@ -219,8 +219,8 @@ class PETest(unittest.TestCase): fcall.writeParams(((DWORD, 0),)) # run until third stop - # TODO: Should stop at 0x10423, but for now just stop at 0x0001066a - amsint32.hook_address(hook_third_stop_address, 0x0001066a, stops) + # TODO: Should stop at 0x10423, but for now just stop at 0x10430 + amsint32.hook_address(hook_third_stop_address, 0x10430, stops) amsint32.run(begin=0x102D0) self.assertTrue(stops[0]) From c5d7a8533044ae33cb9cb4436107b1959257c644 Mon Sep 17 00:00:00 2001 From: elicn Date: Thu, 30 Oct 2025 10:50:18 +0200 Subject: [PATCH 114/138] Add rseq syscall dummy implementation --- qiling/os/posix/syscall/__init__.py | 1 + qiling/os/posix/syscall/rseq.py | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 qiling/os/posix/syscall/rseq.py diff --git a/qiling/os/posix/syscall/__init__.py b/qiling/os/posix/syscall/__init__.py index 38b10e64..1ed1125e 100644 --- a/qiling/os/posix/syscall/__init__.py +++ b/qiling/os/posix/syscall/__init__.py @@ -14,6 +14,7 @@ from .prctl import * from .ptrace import * from .random import * from .resource import * +from .rseq import * from .sched import * from .select import * from .sendfile import * diff --git a/qiling/os/posix/syscall/rseq.py b/qiling/os/posix/syscall/rseq.py new file mode 100644 index 00000000..403595a6 --- /dev/null +++ b/qiling/os/posix/syscall/rseq.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +from qiling import Qiling + + +def ql_syscall_rseq(ql: Qiling, rseq: int, rseq_len: int, flags: int, sig: int): + # indicate rseq is not supported by this kernel + # return -ENOSYS + + return 0 From 45d87c5b69a6e51f7d95152bca608f39e1815e7f Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 31 Oct 2025 14:54:19 +0200 Subject: [PATCH 115/138] Consolidate mmio bookeeping with ram --- qiling/hw/hw.py | 2 +- qiling/os/memory.py | 58 +++++++++++++++++++++------------------------ 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/qiling/hw/hw.py b/qiling/hw/hw.py index 3d869d56..15213a7b 100644 --- a/qiling/hw/hw.py +++ b/qiling/hw/hw.py @@ -182,6 +182,6 @@ class QlHwManager: # a dirty hack to rehydrate non-pickleable hwman # a proper fix would require a deeper refactoring to how peripherals are created and managed - for ph in self.ql.mem.mmio_cbs.values(): + for *_, ph in self.ql.mem.map_info: if isinstance(ph, QlPripheralHandler): setattr(ph, '_hwman', self) diff --git a/qiling/os/memory.py b/qiling/os/memory.py index 4c0f8654..5c85a4b2 100644 --- a/qiling/os/memory.py +++ b/qiling/os/memory.py @@ -13,12 +13,6 @@ from unicorn import UC_PROT_NONE, UC_PROT_READ, UC_PROT_WRITE, UC_PROT_EXEC, UC_ from qiling import Qiling from qiling.exception import * -# tuple: range start, range end, permissions mask, range label, is mmio? -MapInfoEntry = Tuple[int, int, int, str, bool] - -MmioReadCallback = Callable[[Qiling, int, int], int] -MmioWriteCallback = Callable[[Qiling, int, int, int], None] - class QlMmioHandler(Protocol): """A simple MMIO handler boilerplate that can be used to implement memory mapped devices. @@ -36,6 +30,13 @@ class QlMmioHandler(Protocol): ... +# tuple: range start, range end, permissions mask, range label, mmio hander object (if mmio range) +MapInfoEntry = Tuple[int, int, int, str, Optional[QlMmioHandler]] + +MmioReadCallback = Callable[[Qiling, int, int], int] +MmioWriteCallback = Callable[[Qiling, int, int, int], None] + + class QlMemoryManager: """ some ideas and code from: @@ -45,7 +46,6 @@ class QlMemoryManager: def __init__(self, ql: Qiling, pagesize: int = 0x1000): self.ql = ql self.map_info: List[MapInfoEntry] = [] - self.mmio_cbs: Dict[Tuple[int, int], QlMmioHandler] = {} bit_stuff = { 64: (1 << 64) - 1, @@ -121,7 +121,7 @@ class QlMemoryManager: self.__write_string(addr, value, encoding) - def add_mapinfo(self, mem_s: int, mem_e: int, mem_p: int, mem_info: str, is_mmio: bool = False): + def add_mapinfo(self, mem_s: int, mem_e: int, mem_p: int, mem_info: str, mmio_ctx: Optional[QlMmioHandler] = None): """Add a new memory range to map. Args: @@ -129,10 +129,10 @@ class QlMemoryManager: mem_e: memory range end mem_p: permissions mask mem_info: map entry label - is_mmio: memory range is mmio + mmio_ctx: mmio handler object; if specified the range will be treated as mmio """ - bisect.insort(self.map_info, (mem_s, mem_e, mem_p, mem_info, is_mmio)) + bisect.insort(self.map_info, (mem_s, mem_e, mem_p, mem_info, mmio_ctx)) def del_mapinfo(self, mem_s: int, mem_e: int): """Subtract a memory range from map. @@ -146,13 +146,13 @@ class QlMemoryManager: def __split_overlaps(): for idx in overlap_ranges: - lbound, ubound, perms, label, is_mmio = self.map_info[idx] + lbound, ubound, perms, label, mmio_ctx = self.map_info[idx] if lbound < mem_s: - yield (lbound, mem_s, perms, label, is_mmio) + yield (lbound, mem_s, perms, label, mmio_ctx) if mem_e < ubound: - yield (mem_e, ubound, perms, label, is_mmio) + yield (mem_e, ubound, perms, label, mmio_ctx) # indices of first and last overlapping ranges. since map info is always # sorted, we know that all overlapping rages are consecutive, so i1 > i0 @@ -209,18 +209,18 @@ class QlMemoryManager: return ''.join(val if idx & ps else '-' for idx, val in perms_d.items()) - def __process(lbound: int, ubound: int, perms: int, label: str, is_mmio: bool) -> Tuple[int, int, str, str, str]: - perms_str = __perms_mapping(perms) + def __process(entry: MapInfoEntry) -> Tuple[int, int, str, str, str]: + lbound, ubound, perms, label, mmio_ctx = entry if hasattr(self.ql, 'loader'): image = self.ql.loader.find_containing_image(lbound) - container = image.path if image and not is_mmio else '' + container = image.path if image and mmio_ctx is None else '' else: container = '' - return (lbound, ubound, perms_str, label, container) + return (lbound, ubound, __perms_mapping(perms), label, container) - return tuple(__process(*entry) for entry in self.map_info) + return tuple(__process(entry) for entry in self.map_info) def get_formatted_mapinfo(self) -> Sequence[str]: """Get memory map info in a nicely formatted table. @@ -311,12 +311,13 @@ class QlMemoryManager: "mmio" : [] } - for lbound, ubound, perm, label, is_mmio in self.map_info: - if is_mmio: - mem_dict['mmio'].append((lbound, ubound, perm, label, self.mmio_cbs[(lbound, ubound)])) + for lbound, ubound, perm, label, mmio_ctx in self.map_info: + if mmio_ctx is None: + key, data = 'ram', bytes(self.read(lbound, ubound - lbound)) else: - data = self.read(lbound, ubound - lbound) - mem_dict['ram'].append((lbound, ubound, perm, label, bytes(data))) + key, data = 'mmio', mmio_ctx + + mem_dict[key].append((lbound, ubound, perm, label, data)) return mem_dict @@ -429,7 +430,7 @@ class QlMemoryManager: assert begin < end, 'search arguments do not make sense' # narrow the search down to relevant ranges; mmio ranges are excluded due to potential read side effects - ranges = [(max(begin, lbound), min(ubound, end)) for lbound, ubound, _, _, is_mmio in self.map_info if not (end < lbound or ubound < begin or is_mmio)] + ranges = [(max(begin, lbound), min(ubound, end)) for lbound, ubound, _, _, mmio_ctx in self.map_info if not (end < lbound or ubound < begin or mmio_ctx is not None)] results = [] # if needle is a bytes sequence use it verbatim, not as a pattern @@ -455,9 +456,6 @@ class QlMemoryManager: self.del_mapinfo(addr, addr + size) self.ql.uc.mem_unmap(addr, size) - if (addr, addr + size) in self.mmio_cbs: - del self.mmio_cbs[(addr, addr+size)] - def unmap_between(self, mem_s: int, mem_e: int) -> None: """Reclaim any allocated memory region within the specified range. @@ -638,7 +636,7 @@ class QlMemoryManager: raise QlMemoryMappedError('Requested memory is unavailable') self.ql.uc.mem_map(addr, size, perms) - self.add_mapinfo(addr, addr + size, perms, info or '[mapped]', is_mmio=False) + self.add_mapinfo(addr, addr + size, perms, info or '[mapped]', None) def map_mmio(self, addr: int, size: int, handler: QlMmioHandler, info: str = '[mmio]'): # TODO: mmio memory overlap with ram? Is that possible? @@ -664,9 +662,7 @@ class QlMemoryManager: cb(self.ql, offset, size, value) self.ql.uc.mmio_map(addr, size, __mmio_read, handler.read, __mmio_write, handler.write) - self.add_mapinfo(addr, addr + size, prot, info, is_mmio=True) - - self.mmio_cbs[(addr, addr + size)] = handler + self.add_mapinfo(addr, addr + size, prot, info, handler) class Chunk: From b6f085c4d2148a15870ff1f99ea6be4e3344c4b3 Mon Sep 17 00:00:00 2001 From: elicn Date: Fri, 31 Oct 2025 15:28:34 +0200 Subject: [PATCH 116/138] Minor bug fix in change_mapinfo --- qiling/os/memory.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/qiling/os/memory.py b/qiling/os/memory.py index 5c85a4b2..76043895 100644 --- a/qiling/os/memory.py +++ b/qiling/os/memory.py @@ -170,27 +170,30 @@ class QlMemoryManager: for entry in new_entries: bisect.insort(self.map_info, entry) - def change_mapinfo(self, mem_s: int, mem_e: int, mem_p: Optional[int] = None, mem_info: Optional[str] = None): - tmp_map_info: Optional[MapInfoEntry] = None - info_idx: int = -1 + def change_mapinfo(self, mem_s: int, mem_e: int, *, new_perms: Optional[int] = None, new_info: Optional[str] = None) -> None: + if new_perms is None and new_info is None: + # nothing to do + return - for idx, map_info in enumerate(self.map_info): - if mem_s >= map_info[0] and mem_e <= map_info[1]: - tmp_map_info = map_info - info_idx = idx - break - - if tmp_map_info is None: + try: + # locate the map info entry to change + entry = next(entry for entry in self.map_info if mem_s >= entry[0] and mem_e <= entry[1]) + except StopIteration: self.ql.log.error(f'Cannot change mapinfo at {mem_s:#08x}-{mem_e:#08x}') return - if mem_p is not None: - self.del_mapinfo(mem_s, mem_e) - self.add_mapinfo(mem_s, mem_e, mem_p, mem_info if mem_info else tmp_map_info[3]) - return + _, _, perms, info, mmio_ctx = entry - if mem_info is not None: - self.map_info[info_idx] = (tmp_map_info[0], tmp_map_info[1], tmp_map_info[2], mem_info, tmp_map_info[4]) + # caller wants to change perms? + if new_perms is not None: + perms = new_perms + + # caller wants to change info? + if new_info is not None: + info = new_info + + self.del_mapinfo(mem_s, mem_e) + self.add_mapinfo(mem_s, mem_e, perms, info, mmio_ctx) def get_mapinfo(self) -> Sequence[Tuple[int, int, str, str, str]]: """Get memory map info. @@ -614,7 +617,7 @@ class QlMemoryManager: aligned_size = self.align_up((addr & (self.pagesize - 1)) + size) self.ql.uc.mem_protect(aligned_address, aligned_size, perms) - self.change_mapinfo(aligned_address, aligned_address + aligned_size, perms) + self.change_mapinfo(aligned_address, aligned_address + aligned_size, new_perms=perms) def map(self, addr: int, size: int, perms: int = UC_PROT_ALL, info: Optional[str] = None): """Map a new memory range. From 29c552e05d33f7a58c37a1aec7e2e45c9d10d2cd Mon Sep 17 00:00:00 2001 From: golem9247 Date: Tue, 4 Nov 2025 14:57:00 +0100 Subject: [PATCH 117/138] add ppc xml basic target , add a no yield coverage --- qiling/debugger/gdb/xml/ppc/ppc-core.xml | 46 +++++++++++++++++++++ qiling/debugger/gdb/xml/ppc/target.xml | 13 ++++++ qiling/debugger/gdb/xmlregs.py | 7 +++- qiling/extensions/coverage/formats/drcov.py | 11 +++++ qiling/extensions/coverage/utils.py | 9 ++++ 5 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 qiling/debugger/gdb/xml/ppc/ppc-core.xml create mode 100644 qiling/debugger/gdb/xml/ppc/target.xml diff --git a/qiling/debugger/gdb/xml/ppc/ppc-core.xml b/qiling/debugger/gdb/xml/ppc/ppc-core.xml new file mode 100644 index 00000000..c40c52ec --- /dev/null +++ b/qiling/debugger/gdb/xml/ppc/ppc-core.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/ppc/target.xml b/qiling/debugger/gdb/xml/ppc/target.xml new file mode 100644 index 00000000..6839b3e6 --- /dev/null +++ b/qiling/debugger/gdb/xml/ppc/target.xml @@ -0,0 +1,13 @@ + + + + + + ppc + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xmlregs.py b/qiling/debugger/gdb/xmlregs.py index 4749b211..fe200fce 100644 --- a/qiling/debugger/gdb/xmlregs.py +++ b/qiling/debugger/gdb/xmlregs.py @@ -30,6 +30,10 @@ from qiling.arch.x86_const import ( reg_map_ymm as x86_regs_ymm ) +from qiling.arch.ppc_const import ( + reg_map as ppc_regs +) + from qiling.const import QL_ARCH, QL_OS RegEntry = Tuple[Optional[int], int, int] @@ -134,7 +138,8 @@ class QlGdbFeatures: QL_ARCH.ARM: dict(**arm_regs, **arm_regs_vfp, **arm_regs_q, **arm_regs_s), QL_ARCH.CORTEX_M: arm_regs, QL_ARCH.ARM64: dict(**arm64_regs, **arm64_regs_v), - QL_ARCH.MIPS: dict(**mips_regs_gpr) + QL_ARCH.MIPS: dict(**mips_regs_gpr), + QL_ARCH.PPC: dict(**ppc_regs), }[archtype] regsinfo = sorted(QlGdbFeatures.__walk_xml_regs(xmltree)) diff --git a/qiling/extensions/coverage/formats/drcov.py b/qiling/extensions/coverage/formats/drcov.py index 51a42194..fa9a3017 100644 --- a/qiling/extensions/coverage/formats/drcov.py +++ b/qiling/extensions/coverage/formats/drcov.py @@ -17,6 +17,17 @@ class bb_entry(Structure): ("mod_id", c_uint16) ] + def __init__(self, start, size, module_id=None): + self.start = start + self.size = size + self.module_id = module_id + + def __eq__(self, other): + return (self.start, self.size, self.module_id) == (other.start, other.size, other.module_id) + + def __hash__(self): + return hash((self.start, self.size, self.module_id)) + class QlDrCoverage(QlBaseCoverage): """ diff --git a/qiling/extensions/coverage/utils.py b/qiling/extensions/coverage/utils.py index 4293689b..fa1f4494 100644 --- a/qiling/extensions/coverage/utils.py +++ b/qiling/extensions/coverage/utils.py @@ -61,3 +61,12 @@ def collect_coverage(ql: Qiling, name: str, coverage_file: str): finally: cov.deactivate() cov.dump_coverage(coverage_file) + +def collect_coverage_no_yield(ql: Qiling, name: str, coverage_folder: str): + cov = factory.get_coverage_collector(ql, name) + cov.coverage_folder = coverage_folder + cov.activate() + return cov + +def get_coverage_instance(ql, name): + return factory.get_coverage_collector(ql, name) From 4810ad7cfb17d7cd26213cd3fde74caed51f6e9f Mon Sep 17 00:00:00 2001 From: 0xMirasio Date: Tue, 4 Nov 2025 16:23:35 +0100 Subject: [PATCH 118/138] ppc xml fix --- qiling/debugger/gdb/gdb.py | 4 +++- qiling/debugger/gdb/xml/ppc/ppc-core.xml | 11 ++++++++--- qiling/debugger/gdb/xml/ppc/target.xml | 3 +-- qiling/extensions/coverage/utils.py | 9 --------- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/qiling/debugger/gdb/gdb.py b/qiling/debugger/gdb/gdb.py index a26bf6d9..f6d6498d 100644 --- a/qiling/debugger/gdb/gdb.py +++ b/qiling/debugger/gdb/gdb.py @@ -183,6 +183,7 @@ class QlGdb(QlDebugger): from unicorn.arm_const import UC_ARM_REG_R11 from unicorn.arm64_const import UC_ARM64_REG_X29 from unicorn.mips_const import UC_MIPS_REG_INVALID + from unicorn.ppc_const import UC_PPC_REG_31 arch_uc_bp = { QL_ARCH.X86 : UC_X86_REG_EBP, @@ -191,7 +192,8 @@ class QlGdb(QlDebugger): QL_ARCH.ARM64 : UC_ARM64_REG_X29, QL_ARCH.MIPS : UC_MIPS_REG_INVALID, # skipped QL_ARCH.A8086 : UC_X86_REG_EBP, - QL_ARCH.CORTEX_M : UC_ARM_REG_R11 + QL_ARCH.CORTEX_M : UC_ARM_REG_R11, + QL_ARCH.PPC : UC_PPC_REG_31 }[self.ql.arch.type] def __get_reg_idx(ucreg: int) -> int: diff --git a/qiling/debugger/gdb/xml/ppc/ppc-core.xml b/qiling/debugger/gdb/xml/ppc/ppc-core.xml index c40c52ec..d695132a 100644 --- a/qiling/debugger/gdb/xml/ppc/ppc-core.xml +++ b/qiling/debugger/gdb/xml/ppc/ppc-core.xml @@ -6,7 +6,7 @@ notice and this notice are preserved. --> - + @@ -39,8 +39,13 @@ - - + + + + + + + \ No newline at end of file diff --git a/qiling/debugger/gdb/xml/ppc/target.xml b/qiling/debugger/gdb/xml/ppc/target.xml index 6839b3e6..977416a3 100644 --- a/qiling/debugger/gdb/xml/ppc/target.xml +++ b/qiling/debugger/gdb/xml/ppc/target.xml @@ -7,7 +7,6 @@ - ppc + powerpc:common - \ No newline at end of file diff --git a/qiling/extensions/coverage/utils.py b/qiling/extensions/coverage/utils.py index fa1f4494..4293689b 100644 --- a/qiling/extensions/coverage/utils.py +++ b/qiling/extensions/coverage/utils.py @@ -61,12 +61,3 @@ def collect_coverage(ql: Qiling, name: str, coverage_file: str): finally: cov.deactivate() cov.dump_coverage(coverage_file) - -def collect_coverage_no_yield(ql: Qiling, name: str, coverage_folder: str): - cov = factory.get_coverage_collector(ql, name) - cov.coverage_folder = coverage_folder - cov.activate() - return cov - -def get_coverage_instance(ql, name): - return factory.get_coverage_collector(ql, name) From 08650becba38daca5e0a04e2851a739863f65647 Mon Sep 17 00:00:00 2001 From: 0xMirasio Date: Wed, 5 Nov 2025 11:23:56 +0100 Subject: [PATCH 119/138] remove hashable drcov bb_entries --- qiling/extensions/coverage/formats/drcov.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/qiling/extensions/coverage/formats/drcov.py b/qiling/extensions/coverage/formats/drcov.py index d224d98a..bed0f870 100644 --- a/qiling/extensions/coverage/formats/drcov.py +++ b/qiling/extensions/coverage/formats/drcov.py @@ -25,17 +25,6 @@ class bb_entry(Structure): ("mod_id", c_uint16) ] - def __init__(self, start, size, module_id=None): - self.start = start - self.size = size - self.module_id = module_id - - def __eq__(self, other): - return (self.start, self.size, self.module_id) == (other.start, other.size, other.module_id) - - def __hash__(self): - return hash((self.start, self.size, self.module_id)) - class QlDrCoverage(QlBaseCoverage): """ From a76b858332a05c3c987d23dafa7218dba4775755 Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Fri, 14 Nov 2025 04:39:15 +0000 Subject: [PATCH 120/138] Tentative fix for epoll server test case --- qiling/os/posix/syscall/epoll.py | 2 +- tests/test_elf.py | 9 +++------ tests/test_onlinux.sh | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 46911678..9afb57f2 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -118,7 +118,7 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if op not in (EPOLL_CTL_ADD, EPOLL_CTL_DEL, EPOLL_CTL_MOD): return -EINVAL - if epfd == fd: + if epfd == fd or fd == 0xffffffff: # latter condition was seen in testing, but should not happen in the real world return -EINVAL if epfd not in range(NR_OPEN): diff --git a/tests/test_elf.py b/tests/test_elf.py index 52db6719..afaca75c 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -217,13 +217,13 @@ class ELFTest(unittest.TestCase): ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_hello_static"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG) ql.run() del ql - + #@unittest.skip('Experiment to avoid FD issue') def test_elf_linux_x86(self): filename = 'test.qlog' ql = Qiling(["../examples/rootfs/x86_linux/bin/x86_hello"], "../examples/rootfs/x86_linux", verbose=QL_VERBOSE.DEBUG, log_devices=[filename]) ql.run() - + ql._log_file_fd.handlers[0].close() # prevent FD leak that causes downstream issues os.remove(filename) del ql @@ -789,8 +789,7 @@ class ELFTest(unittest.TestCase): ql.run() self.assertIn(b'echo\n', ql.os.stdout.read()) - del ql - @unittest.skip("See comment in https://github.com/qilingframework/qiling/pull/1558") + del ql def test_elf_linux_x8664_epoll_server(self): # This tests a simple server that uses epoll to wait for data, then prints it out. It has # been modified to exit after data has been received; instead of a typical server operation @@ -810,7 +809,6 @@ class ELFTest(unittest.TestCase): def client(): # give time for the server to listen time.sleep(3) - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect(("127.0.0.1", 8000)) s.send(b"hello world") @@ -826,7 +824,6 @@ class ELFTest(unittest.TestCase): client_thread = threading.Thread(target=client, daemon=True) client_thread.start() - ql.run() self.assertIn(b'hello world', ql.os.stdout.read(200)) # 200 is arbitrary--"good enough" for this task diff --git a/tests/test_onlinux.sh b/tests/test_onlinux.sh index 557b2e77..9a333a68 100755 --- a/tests/test_onlinux.sh +++ b/tests/test_onlinux.sh @@ -4,7 +4,7 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -python3 ./test_posix.py && +python3 -m pdb ./test_posix.py && python3 ./test_elf_multithread.py && python3 ./test_elf_ko.py && python3 ./test_debugger.py && From 4448d4c12982c632afd29c482b6c90a4515d4d0e Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Sat, 15 Nov 2025 01:57:40 +0000 Subject: [PATCH 121/138] Remove stray PDB invocation --- tests/test_onlinux.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_onlinux.sh b/tests/test_onlinux.sh index 9a333a68..557b2e77 100755 --- a/tests/test_onlinux.sh +++ b/tests/test_onlinux.sh @@ -4,7 +4,7 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -python3 -m pdb ./test_posix.py && +python3 ./test_posix.py && python3 ./test_elf_multithread.py && python3 ./test_elf_ko.py && python3 ./test_debugger.py && From 3675f557e90d9a2e83d3baeeaa42f9507afb62ac Mon Sep 17 00:00:00 2001 From: libumem <163767094+libumem@users.noreply.github.com> Date: Sun, 16 Nov 2025 18:19:20 +0000 Subject: [PATCH 122/138] Address @elicn feedback --- qiling/os/posix/syscall/epoll.py | 2 +- tests/test_elf.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 9afb57f2..669ce17b 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -118,7 +118,7 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if op not in (EPOLL_CTL_ADD, EPOLL_CTL_DEL, EPOLL_CTL_MOD): return -EINVAL - if epfd == fd or fd == 0xffffffff: # latter condition was seen in testing, but should not happen in the real world + if epfd == fd: return -EINVAL if epfd not in range(NR_OPEN): diff --git a/tests/test_elf.py b/tests/test_elf.py index afaca75c..2798028b 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -217,7 +217,7 @@ class ELFTest(unittest.TestCase): ql = Qiling(["../examples/rootfs/x8664_linux/bin/x8664_hello_static"], "../examples/rootfs/x8664_linux", verbose=QL_VERBOSE.DEBUG) ql.run() del ql - #@unittest.skip('Experiment to avoid FD issue') + def test_elf_linux_x86(self): filename = 'test.qlog' @@ -789,7 +789,9 @@ class ELFTest(unittest.TestCase): ql.run() self.assertIn(b'echo\n', ql.os.stdout.read()) + del ql + def test_elf_linux_x8664_epoll_server(self): # This tests a simple server that uses epoll to wait for data, then prints it out. It has # been modified to exit after data has been received; instead of a typical server operation From 30eba376fced05c84ed148dec552f5b84e74dba9 Mon Sep 17 00:00:00 2001 From: xwings Date: Sat, 11 Apr 2026 22:48:18 +0800 Subject: [PATCH 123/138] Add TODO.md, ARCHITECTURE.md. Update README.md. and add kernel proxy --- .gitignore | 4 + ARCHITECTURE.md | 427 ++++++++++++++ README.md | 198 +++---- TODO | 1 - TODO.md | 693 +++++++++++++++++++++++ examples/rootfs | 2 +- qiling/os/linux/map_syscall.py | 60 ++ qiling/os/posix/kernel_proxy/__init__.py | 234 ++++++++ qiling/os/posix/kernel_proxy/ipc.py | 176 ++++++ qiling/os/posix/kernel_proxy/proxy.py | 116 ++++ qiling/os/posix/kernel_proxy/proxy_fd.py | 47 ++ tests/test_kernel_proxy.py | 355 ++++++++++++ 12 files changed, 2214 insertions(+), 99 deletions(-) create mode 100644 ARCHITECTURE.md delete mode 100644 TODO create mode 100644 TODO.md create mode 100644 qiling/os/posix/kernel_proxy/__init__.py create mode 100644 qiling/os/posix/kernel_proxy/ipc.py create mode 100644 qiling/os/posix/kernel_proxy/proxy.py create mode 100644 qiling/os/posix/kernel_proxy/proxy_fd.py create mode 100644 tests/test_kernel_proxy.py diff --git a/.gitignore b/.gitignore index 766e541d..b0cd99ad 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,7 @@ poetry.toml # LSP config files pyrightconfig.json + +# AI +CLAUDE.md +.claude diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 00000000..ba924a09 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,427 @@ +# Qiling Framework Architecture + +Qiling is a cross-platform, multi-architecture binary emulation framework built on top of the [Unicorn](https://www.unicorn-engine.org/) CPU emulation engine. It adds OS-level abstractions (syscalls, file systems, loaders) on top of raw CPU emulation, enabling full binary execution without native hardware. + +## High-Level Overview + +``` +┌──────────────────────────────────────────────────┐ +│ User Script / qltool │ +├──────────────────────────────────────────────────┤ +│ Qiling Core (core.py) │ +│ hooks · state snapshots · patches │ +├────────────┬─────────────┬───────────────────────┤ +│ OS Layer │ Loader │ Memory Manager │ +│ (QlOs) │ (QlLoader) │ (QlMemoryManager) │ +│ syscalls │ ELF/PE/ │ map · read · write │ +│ APIs │ MachO/etc │ MMIO callbacks │ +├────────────┴─────────────┴───────────────────────┤ +│ Architecture Layer (QlArch) │ +│ registers · disassembly · calling conventions │ +├──────────────────────────────────────────────────┤ +│ Unicorn Engine (CPU) │ +│ instruction-level emulation │ +└──────────────────────────────────────────────────┘ +``` + +## Directory Structure + +``` +qiling/ +├── qiling/ # Core framework package +│ ├── core.py # Qiling class — main entry point and orchestrator +│ ├── core_hooks.py # Hook system (code, memory, interrupt, address hooks) +│ ├── core_hooks_types.py # Hook type definitions and dispatch +│ ├── core_struct.py # Endian-aware struct packing utilities +│ ├── const.py # Enumerations: QL_ARCH, QL_OS, QL_ENDIAN, etc. +│ ├── exception.py # Custom exception hierarchy +│ ├── utils.py # Component selection (select_arch, select_os, etc.) +│ ├── host.py # Host platform interface +│ ├── log.py # Logging configuration +│ │ +│ ├── arch/ # Architecture implementations +│ │ ├── arch.py # QlArch — abstract base class +│ │ ├── x86.py # x86 / x86-64 / 8086 +│ │ ├── arm.py # ARMv7 (32-bit) +│ │ ├── arm64.py # ARMv8 (64-bit) +│ │ ├── mips.py # MIPS32 +│ │ ├── riscv.py # RISC-V 32-bit +│ │ ├── riscv64.py # RISC-V 64-bit +│ │ ├── ppc.py # PowerPC 32-bit +│ │ ├── cortex_m.py # ARM Cortex-M (MCU) +│ │ ├── register.py # Register management +│ │ └── models.py # CPU model definitions +│ │ +│ ├── os/ # Operating system implementations +│ │ ├── os.py # QlOs — abstract base class +│ │ ├── memory.py # QlMemoryManager +│ │ ├── fcall.py # Function call interface (read params, set return) +│ │ ├── mapper.py # Syscall/API mapping +│ │ ├── path.py # Virtual filesystem path resolution +│ │ ├── filestruct.py # File descriptor abstraction +│ │ ├── thread.py # Threading primitives +│ │ ├── posix/ # POSIX shared layer (syscall handlers) +│ │ ├── linux/ # Linux-specific OS +│ │ ├── freebsd/ # FreeBSD-specific OS +│ │ ├── macos/ # macOS-specific OS +│ │ ├── qnx/ # QNX RTOS +│ │ ├── windows/ # Windows (Win32/Win64 API emulation) +│ │ ├── uefi/ # UEFI firmware services +│ │ ├── dos/ # DOS (8086 interrupts) +│ │ ├── mcu/ # Bare-metal microcontroller +│ │ └── blob/ # Raw binary blob execution +│ │ +│ ├── loader/ # Binary format loaders +│ │ ├── loader.py # QlLoader — abstract base class +│ │ ├── elf.py # ELF (Linux, FreeBSD, QNX) +│ │ ├── pe.py # PE (Windows) +│ │ ├── pe_uefi.py # PE for UEFI +│ │ ├── macho.py # Mach-O (macOS) +│ │ ├── dos.py # DOS COM/EXE +│ │ ├── mcu.py # MCU firmware images +│ │ └── blob.py # Raw binary blobs +│ │ +│ ├── cc/ # Calling conventions +│ │ ├── intel.py # cdecl, stdcall, ms64 +│ │ ├── arm.py # aarch32, aarch64 +│ │ ├── mips.py # MIPS o32 +│ │ ├── riscv.py # RISC-V ABI +│ │ └── ppc.py # PowerPC ABI +│ │ +│ ├── hw/ # Hardware peripheral emulation (MCU) +│ │ ├── peripheral.py # Base peripheral class +│ │ ├── hw.py # Hardware manager +│ │ ├── gpio/ # GPIO pins and interrupts +│ │ ├── timer/ # Timers, PWM, counters +│ │ ├── char/ # UART serial +│ │ ├── spi/ # SPI bus +│ │ ├── i2c/ # I2C bus +│ │ ├── net/ # Network interfaces +│ │ ├── analog/ # ADC/DAC +│ │ ├── intc/ # Interrupt controllers +│ │ ├── flash/ # Flash memory +│ │ ├── dma/ # DMA controllers +│ │ └── ... # Power, SD, misc peripherals +│ │ +│ ├── debugger/ # Debugger subsystem +│ │ ├── gdb/ # GDB remote protocol server +│ │ └── qdb/ # Qiling native debugger (with reverse debugging) +│ │ +│ ├── extensions/ # Optional extensions +│ │ ├── multitask.py # gevent-based multithreading +│ │ ├── trace.py # Instruction tracing +│ │ ├── coverage/ # Code coverage collection +│ │ ├── sanitizers/ # Memory sanitizers +│ │ ├── afl/ # AFL fuzzer integration +│ │ ├── r2/ # Radare2 integration +│ │ └── idaplugin/ # IDA Pro plugin +│ │ +│ └── profiles/ # Default OS configuration files (.ql) +│ ├── linux.ql # Stack/heap addresses, kernel params +│ ├── windows.ql +│ ├── macos.ql +│ └── ... +│ +├── qltool # CLI tool for running binaries +├── qltui.py # TUI interface +├── examples/ # Usage examples and sample scripts +├── tests/ # Test suite +└── docs/ # Documentation +``` + +## Core Components + +### `Qiling` (core.py) + +The central class. Every emulation session creates one `Qiling` instance that owns and wires together all other components: + +```python +ql = Qiling( + argv=["/path/to/binary", "arg1"], # binary + arguments + rootfs="/path/to/rootfs", # virtual filesystem root + ostype=QL_OS.LINUX, # target OS (auto-detected if omitted) + archtype=QL_ARCH.X8664, # target arch (auto-detected if omitted) +) +``` + +`Qiling` inherits from `QlCoreHooks` (hook management) and `QlCoreStructs` (endian-aware packing). Key properties: + +| Property | Type | Description | +|----------|------|-------------| +| `ql.arch` | `QlArch` | CPU architecture — registers, disassembly | +| `ql.os` | `QlOs` | Operating system — syscalls, I/O, APIs | +| `ql.loader` | `QlLoader` | Binary loader — parses and maps the executable | +| `ql.mem` | `QlMemoryManager` | Memory — map, read, write, MMIO | +| `ql.uc` | `unicorn.Uc` | Underlying Unicorn engine instance | + +### Architecture Layer (`qiling/arch/`) + +`QlArch` is the abstract base. Each architecture subclass configures: + +- **Unicorn engine** mode and architecture constants +- **Register** access via `ql.arch.regs` (read/write by name) +- **Disassembler** (Capstone) and **assembler** (Keystone) +- **Stack operations** — push, pop, pointer-width-aware +- **Endianness** and **bit width** (16/32/64) + +Supported: x86, x86-64, 8086, ARM, ARM64, MIPS, RISC-V (32/64), PowerPC, Cortex-M. + +### OS Layer (`qiling/os/`) + +`QlOs` is the abstract base. Each OS subclass provides: + +- **Syscall/interrupt dispatch** — routes CPU interrupts to handler functions +- **I/O streams** — `stdin`, `stdout`, `stderr` (interceptable) +- **Virtual filesystem** — path mapping through `rootfs` +- **Function call interface** (`ql.os.fcall`) — read params, set return values +- **API interception** — `set_api()` for hooking library functions + +**POSIX subsystem** (`os/posix/`): Shared syscall implementation for Linux, FreeBSD, macOS, and QNX. Individual syscall handlers live under `os/posix/syscall/`. + +**Windows** (`os/windows/`): Emulates Win32/Win64 API by hooking DLL imports. Includes registry, thread, handle, and fiber support. + +**UEFI** (`os/uefi/`): Emulates UEFI Boot Services, Runtime Services, and SMM. Uses a GUID database and protocol framework. + +### Loader Layer (`qiling/loader/`) + +`QlLoader` is the abstract base. Loaders parse a binary format, map segments into memory, resolve symbols, load dependencies, and set initial CPU state (PC, SP). + +| Loader | Format | Used By | +|--------|--------|---------| +| `QlLoaderELF` | ELF | Linux, FreeBSD, QNX | +| `QlLoaderPE` | PE/COFF | Windows, UEFI | +| `QlLoaderMacho` | Mach-O | macOS | +| `QlLoaderDOS` | COM/EXE | DOS | +| `QlLoaderMCU` | Firmware | Cortex-M MCU | +| `QlLoaderBlob` | Raw bytes | Shellcode / blob | + +### Memory Manager (`qiling/os/memory.py`) + +Wraps Unicorn's memory model with higher-level operations: + +- `map(addr, size, perms)` / `unmap(addr, size)` — region management +- `read(addr, size)` / `write(addr, data)` — data access +- `read_ptr(addr)` / `write_ptr(addr, val)` — pointer-width-aware access +- `read_cstring(addr)` — null-terminated string read +- MMIO callback support for memory-mapped peripherals + +### Calling Conventions (`qiling/cc/`) + +Each architecture has calling convention classes that abstract argument passing and return values. The `QlOs.fcall` interface uses these to provide a uniform way to read function parameters regardless of platform. + +## Execution Flow + +### 1. Initialization (`Qiling.__init__`) + +``` +Qiling(argv, rootfs) + │ + ├─ Detect arch/OS from binary headers (ql_guess_emu_env) + │ ELF magic → parse e_machine, OSABI + │ PE magic → parse Machine, Subsystem + │ MachO magic → parse CPU type + │ + ├─ Create QlArch (select_arch) → initializes Unicorn engine + ├─ Create QlLoader (select_loader) + ├─ Create QlMemoryManager + ├─ Create QlOs (select_os) + │ + └─ loader.run() + ├─ Parse binary format (headers, segments, sections) + ├─ Map segments into memory + ├─ Load shared libraries / DLLs + ├─ Setup stack, heap, TLS, auxiliary vectors + └─ Set initial PC (entry point) and SP +``` + +### 2. Execution (`ql.run()`) + +``` +ql.run(begin, end, timeout, count) + │ + ├─ Apply binary patches (ql.patch) + ├─ Write exit trap (guard address) + │ + └─ os.run() + └─ uc.emu_start(entry_point, exit_point) + │ + ├─ Unicorn executes instructions + │ + ├─ Hooks fire on: + │ ├─ Every instruction (hook_code) + │ ├─ Basic blocks (hook_block) + │ ├─ Memory access (hook_mem_read/write) + │ ├─ Interrupts (hook_intno) → syscall dispatch + │ ├─ Specific addresses (hook_address) + │ └─ Specific instructions (hook_insn) + │ + └─ Stops when PC reaches exit point, timeout, + ql.emu_stop(), or unhandled exception +``` + +### 3. Syscall Handling + +When the emulated binary issues a syscall (via `int 0x80`, `syscall`, `svc`, etc.): + +``` +CPU interrupt/instruction + → Unicorn interrupt hook + → QlOs syscall dispatcher + → Look up handler by syscall number + → Handler reads args via calling convention + → Emulates syscall behavior + → Sets return value in registers +``` + +## Component Selection + +Components are selected dynamically at runtime based on `QL_ARCH` and `QL_OS` enums. The `qiling/utils.py` module provides: + +- `select_arch(archtype)` → architecture class +- `select_os(ostype)` → OS class +- `select_loader(ostype)` → loader class +- `select_debugger(options)` → debugger class + +This makes it possible to support diverse platform combinations from a unified codebase. + +## Hook System + +The hook system (`core_hooks.py`) wraps Unicorn's callback mechanism: + +| Hook Type | Trigger | +|-----------|---------| +| `hook_code` | Every instruction (optionally within address range) | +| `hook_block` | Every basic block entry | +| `hook_address` | Specific address reached | +| `hook_intno` | CPU interrupt/exception | +| `hook_insn` | Specific instruction type (e.g., `syscall`) | +| `hook_mem_read` | Memory read | +| `hook_mem_write` | Memory write | +| `hook_mem_invalid` | Invalid memory access | + +Hooks can be scoped to address ranges and return `QL_HOOK_BLOCK` to suppress further hooks in the chain. + +## Key Extension Points + +- **Custom syscall handlers** — replace or extend any syscall +- **API hooking** — `ql.os.set_api(name, callback)` to intercept library calls +- **Binary patching** — `ql.patch(offset, data)` for runtime patching +- **State snapshots** — `ql.save()` / `ql.restore()` for checkpointing +- **Debugger attachment** — GDB remote protocol or native QDB debugger +- **Coverage/tracing** — `extensions/coverage/` and `extensions/trace.py` +- **Fuzzing** — AFL integration via `extensions/afl/` +- **Hardware peripherals** — register custom MCU peripherals in `hw/` + +## Dependencies + +| Package | Role | +|---------|------| +| `unicorn` (2.1.3) | CPU emulation engine | +| `capstone` | Disassembly | +| `keystone-engine` | Assembly | +| `pyelftools` | ELF parsing | +| `pefile` | PE parsing | +| `python-registry` | Windows registry emulation | +| `gevent` | Cooperative multithreading | +| `pyyaml` | Configuration parsing | + +Optional: `unicornafl` / `fuzzercorn` (fuzzing), `r2libr` (Radare2 integration). + +## Supported Platforms + +**Architectures:** x86, x86-64, 8086, ARM, ARM64, MIPS, RISC-V (32/64), PowerPC, Cortex-M + +**Operating Systems:** Linux, FreeBSD, macOS, Windows, UEFI, DOS, QNX, MCU (bare-metal), Blob + +## Improvement: Hybrid Kernel Architecture + +> Detailed implementation plan and task tracking: [TODO.md](TODO.md) + +### The Problem + +Qiling reimplements Linux kernel behavior syscall-by-syscall in Python. This works +for simple operations (file I/O, memory management, stat) but fundamentally cannot +scale to the full kernel surface: + +- **Networking**: No epoll. Sockets are proxied to host sockets with no isolation. + No real TCP state machine, no multicast, no raw/netlink sockets. +- **Multithreading**: Gevent greenlets are cooperative and single-threaded. No + preemption, no real concurrency. Futex is a gevent Event. Programs using pthreads, + mutexes, or condition variables don't behave correctly. +- **Signals**: `signal()`, `sigaction()`, `kill()` are mostly stubbed. No delivery, + no `EINTR`, no `SA_RESTART`. +- **Long tail**: capabilities, cgroups, namespaces, io_uring, seccomp, eBPF — the + kernel API surface is vast and growing. + +### The Solution + +A **hybrid architecture** that keeps Unicorn for CPU emulation and Qiling for +instrumentation, but offloads complex kernel subsystems to a real Linux kernel via +a **kernel proxy** helper process. Simple syscalls stay emulated in Python. + +``` +Syscall interrupt + → load_syscall() [UNCHANGED — existing dispatch in posix.py] + → check posix_syscall_hooks[CALL] + → proxy hook registered? → forward to kernel proxy + → no proxy hook? → existing Python handler [UNCHANGED] +``` + +The user explicitly chooses which missing syscalls to forward. Nothing is automatic — +by default Qiling behaves exactly as today. The integration uses the **existing +`set_syscall()` CALL hook mechanism** (`posix.py:128-143`), so `load_syscall()` and +all existing dispatch code remain completely unchanged. + +```python +proxy = KernelProxy(ql) +proxy.forward_syscall("epoll_create", returns_fd=True) +proxy.forward_syscall("epoll_ctl") +proxy.forward_syscall("epoll_wait") +ql.run() +``` + +Under the hood, `forward_syscall()` registers a CALL hook that serializes the +arguments and sends them to a helper process (the kernel proxy) which executes +the real syscall and returns the result. For syscalls that return FDs, the result +is wrapped in a `ql_proxy_fd` object and stored in Qiling's FD table. Since the +FD table is already polymorphic (`ql_socket`, `ql_file`, `ql_pipe`), existing +handlers like `ql_syscall_read` and `ql_syscall_close` dispatch through the proxy +FD's `.read()`/`.close()` methods automatically — no changes needed. + +### Phases + +| Phase | Scope | Risk | Goal | +|-------|-------|------|------| +| 0 | Proof of concept | Low | User manually forwards specific syscalls — zero existing code changed | +| 1 | Networking foundation | Low-Med | Specific hooks for socket syscalls, `ql_proxy_fd`, TCP works | +| 2 | Complete networking | Medium | epoll, poll/select, network namespaces | +| 3 | Real threading | **High** | One Unicorn per thread, shared memory, real futex | +| 4 | Signals | Medium | Real signal delivery, EINTR, handler execution | +| 5 | Integration | Low | API polish, fallback, platform support, benchmarks | + +Phase 0 gives users explicit control — they identify which missing syscalls to +forward and the proxy handles them. Phases 1-2 add pointer-aware forwarding for +networking with pre-built forwarders so users don't have to wire up each syscall. +Phase 3 (threading) is the highest-risk change and is deferred until networking is +stable. Each phase preserves backward compatibility — hybrid mode is opt-in, default +behavior is unchanged. + +### Alternatives Considered + +- **Run a real kernel in Unicorn**: Unicorn doesn't emulate hardware (interrupt + controllers, MMU page tables, timers). Would require rebuilding QEMU system mode. +- **ptrace-based execution**: Run natively, intercept syscalls. Fast, but no + cross-architecture support and limited instruction-level hooks. +- **User-Mode Linux (UML)**: Run the kernel as a userspace process. x86-only, + somewhat unmaintained, complex syscall bridge. +- **Auto-forward all unimplemented syscalls**: Forward every missing syscall + automatically. Convenient but unpredictable — hard to debug, may forward syscalls + that shouldn't be (security, state leaks). Explicit user control is safer. + +The hybrid approach was chosen because it preserves Qiling's core value +(instrumentation + cross-arch emulation) while getting real kernel behavior where +it matters most — without modifying the existing dispatch path. + +## Testing + +Tests live in `tests/` and are organized by platform: `test_elf.py`, `test_pe.py`, `test_macho.py`, `test_dos.py`, `test_mcu.py`, etc. They use binaries from `examples/rootfs/` as test fixtures. diff --git a/README.md b/README.md index 34a02ef6..05d6945b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Documentation Status](https://github.com/qilingframework/qiling/wiki)](https://github.com/qilingframework/qiling/wiki) +[![Documentation Status](https://readthedocs.org/projects/qilingframework/badge/?version=latest)](https://docs.qiling.io) [![Downloads](https://pepy.tech/badge/qiling)](https://pepy.tech/project/qiling) [![Chat on Telegram](https://img.shields.io/badge/Chat%20on-Telegram-brightgreen.svg)](https://t.me/qilingframework) @@ -8,43 +8,24 @@

-# Qiling Framework +[Qiling's use case, blog and related work](https://github.com/qilingframework/qiling/issues/134) -Qiling is an advanced binary emulation framework that allows you to emulate and sandbox code in an isolated environment across multiple platforms and architectures. Built on top of Unicorn Engine, Qiling provides a higher-level framework that understands operating system contexts, executable formats, and dynamic linking. +Qiling is an advanced binary emulation framework, with the following features: -## Table of Contents - -- [Features](#features) -- [Appearance](#Appearance) -- [Use Cases](#use-cases) -- [Quick Start](#quick-start) - - [Installation](#installation) - - [Basic Usage](#basic-usage) -- [Qiling vs. Other Emulators](#qiling-vs-other-emulators) - - [Qiling vs. Unicorn Engine](#qiling-vs-unicorn-engine) - - [Qiling vs. QEMU User Mode](#qiling-vs-qemu-user-mode) -- [Examples](#examples) -- [Qltool](#qltool) -- [Contributing](#contributing) -- [License](#license) -- [Contact](#contact) -- [Core Developers & Contributors](#core-developers--contributors) - -## Features - -- **Multi-platform Emulation**: Windows, macOS, Linux, Android, BSD, UEFI, DOS, MBR. -- **Multi-architecture Emulation**: 8086, X86, X86_64, ARM, ARM64, MIPS, RISC-V, PowerPC. -- **Multiple File Format Support**: PE, Mach-O, ELF, COM, MBR. -- **Kernel Module Emulation**: Supports Windows Driver (.sys), Linux Kernel Module (.ko) & macOS Kernel (.kext) via [Demigod](https://groundx.io/demigod/). -- **Isolated Sandboxing**: Emulates & sandboxes code in an isolated environment with a fully configurable sandbox. -- **In-depth API**: Provides in-depth memory, register, OS level, and filesystem level API. -- **Fine-grain Instrumentation**: Allows hooks at various levels (instruction/basic-block/memory-access/exception/syscall/IO/etc.). -- **Virtual Machine Level API**: Supports saving and restoring the current execution state. -- **Debugging Capabilities**: Supports cross-architecture and platform debugging, including a built-in debugger with reverse debugging capability. -- **Dynamic Hot Patching**: Allows dynamic hot patching of on-the-fly running code, including loaded libraries. -- **Python Framework**: A true framework in Python, making it easy to build customized security analysis tools. - -## Appearance +- Emulate multi-platforms: Windows, macOS, Linux, Android, BSD, UEFI, DOS, MBR. +- Emulate multi-architectures: 8086, X86, X86_64, ARM, ARM64, MIPS, RISC-V, PowerPC. +- Support multiple file formats: PE, Mach-O, ELF, COM, MBR. +- Support Windows Driver (.sys), Linux Kernel Module (.ko) & macOS Kernel (.kext) via [Demigod](https://groundx.io/demigod/). +- Emulates & sandbox code in an isolated environment. +- Provides a fully configurable sandbox. +- Provides in-depth memory, register, OS level and filesystem level API. +- Fine-grain instrumentation: allows hooks at various levels + (instruction/basic-block/memory-access/exception/syscall/IO/etc.) +- Provides virtual machine level API such as saving and restoring the current execution state. +- Supports cross architecture and platform debugging capabilities. +- Built-in debugger with reverse debugging capability. +- Allows dynamic hot patch on-the-fly running code, including the loaded library. +- True framework in Python, making it easy to build customized security analysis tools on top. Qiling also made its way to various international conferences. @@ -68,37 +49,79 @@ Qiling also made its way to various international conferences. - [Nullcon](https://nullcon.net/website/goa-2020/speakers/kaijern-lau.php) 2019: + - [DEFCON, USA](https://www.defcon.org/html/defcon-27/dc-27-demolabs.html#QiLing) - [Hitcon](https://hitcon.org/2019/CMT/agenda) - [Zeronights](https://zeronights.ru/report-en/qiling-io-advanced-binary-emulation-framework/) -## Use Cases -Qiling has been presented at various international conferences, showcasing its versatility in: +Qiling is backed by [Unicorn Engine](http://www.unicorn-engine.org). -- Binary analysis and reverse engineering. -- Malware analysis and sandboxing. -- Firmware analysis and emulation. -- Security research and vulnerability discovery. -- CTF challenges and exploit development. +Visit our [website](https://www.qiling.io) for more information. -For more details on Qiling's use cases, blog posts, and related work, please refer to [Qiling's use case, blog and related work](https://github.com/qilingframework/qiling/issues/134). +--- +#### License -## Quick Start +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. -### Installation +--- -Qiling requires Python 3.8 or newer. You can install it using pip: +#### Qiling vs. other Emulators -```bash -pip install qiling -``` +There are many open-source emulators, but two projects closest to Qiling +are [Unicorn](http://www.unicorn-engine.org) & [QEMU user mode](https://qemu.org). +This section explains the main differences of Qiling against them. -For more detailed installation instructions and dependencies, please refer to the [official documentation](https://github.com/qilingframework/qiling/wiki/Installation). +##### Qiling vs. Unicorn engine -### Basic Usage +Built on top of Unicorn, but Qiling & Unicorn are two different animals. -The example below shows how to use Qiling framework in the most straightforward way to emulate a Windows executable. +- Unicorn is just a CPU emulator, so it focuses on emulating CPU instructions, + that can understand emulator memory. + Beyond that, Unicorn is not aware of higher level concepts, such as dynamic + libraries, system calls, I/O handling or executable formats like PE, Mach-O + or ELF. As a result, Unicorn can only emulate raw machine instructions, + without Operating System (OS) context. +- Qiling is designed as a higher level framework, that leverages Unicorn to + emulate CPU instructions, but can understand OS: it has executable format + loaders (for PE, Mach-O & ELF currently), dynamic linkers (so we can + load & relocate shared libraries), syscall & IO handlers. For this reason, + Qiling can run executable binary without requiring its native OS. + +##### Qiling vs. QEMU user mode + +QEMU user mode does a similar thing to our emulator, that is, to emulate whole +executable binaries in a cross-architecture way. +However, Qiling offers some important differences against QEMU user mode: + +- Qiling is a true analysis framework, + that allows you to build your own dynamic analysis tools on top (in Python). + Meanwhile, QEMU is just a tool, not a framework. +- Qiling can perform dynamic instrumentation, and can even hot patch code at + runtime. QEMU does neither. +- Not only working cross-architecture, Qiling is also cross-platform. + For example, you can run Linux ELF file on top of Windows. + In contrast, QEMU user mode only runs binary of the same OS, such as Linux + ELF on Linux, due to the way it forwards syscall from emulated code to + native OS. +- Qiling supports more platforms, including Windows, macOS, Linux & BSD. QEMU + user mode can only handle Linux & BSD. + +--- + +#### Installation + +Please see [setup guide](https://github.com/qilingframework/qiling/wiki/Installation) file for how to install Qiling Framework. + +--- + +#### Examples + +The example below shows how to use Qiling framework in the most +straightforward way to emulate a Windows executable. ```python from qiling import Qiling @@ -112,30 +135,8 @@ if __name__ == "__main__": ql.run() ``` -## Qiling vs. Other Emulators - -There are many open-source emulators, but two projects closest to Qiling are [Unicorn](http://www.unicorn-engine.org) & [QEMU user mode](https://qemu.org). This section explains the main differences of Qiling against them. - -### Qiling vs. Unicorn Engine - -Built on top of Unicorn, but Qiling & Unicorn are two different animals. - -- **Unicorn** is just a CPU emulator, so it focuses on emulating CPU instructions, that can understand emulator memory. Beyond that, Unicorn is not aware of higher level concepts, such as dynamic libraries, system calls, I/O handling or executable formats like PE, Mach-O or ELF. As a result, Unicorn can only emulate raw machine instructions, without Operating System (OS) context. -- **Qiling** is designed as a higher level framework, that leverages Unicorn to emulate CPU instructions, but can understand OS: it has executable format loaders (for PE, Mach-O & ELF currently), dynamic linkers (so we can load & relocate shared libraries), syscall & IO handlers. For this reason, Qiling can run executable binary without requiring its native OS. - -### Qiling vs. QEMU User Mode - -QEMU user mode does a similar thing to our emulator, that is, to emulate whole executable binaries in a cross-architecture way. -However, Qiling offers some important differences against QEMU user mode: - -- **Qiling is a true analysis framework**, that allows you to build your own dynamic analysis tools on top (in Python). Meanwhile, QEMU is just a tool, not a framework. -- **Qiling can perform dynamic instrumentation**, and can even hot patch code at runtime. QEMU does neither. -- Not only working cross-architecture, **Qiling is also cross-platform**. For example, you can run Linux ELF file on top of Windows. In contrast, QEMU user mode only runs binary of the same OS, such as Linux ELF on Linux, due to the way it forwards syscall from emulated code to native OS. -- **Qiling supports more platforms**, including Windows, macOS, Linux & BSD. QEMU user mode can only handle Linux & BSD. - -## Examples - -- The following example shows how a Windows crackme may be patched dynamically to make it always display the “Congratulation” dialog. +- The following example shows how a Windows crackme may be patched dynamically + to make it always display the “Congratulation” dialog. ```python from qiling import Qiling @@ -176,13 +177,15 @@ The below YouTube video shows how the above example works. #### Emulating ARM router firmware on Ubuntu x64 host -Qiling Framework hot-patches and emulates an ARM router's `/usr/bin/httpd` on an x86_64 Ubuntu host. +Qiling Framework hot-patches and emulates an ARM router's `/usr/bin/httpd` on +an x86_64 Ubuntu host. -[![Qiling Tutorial: Emulating and Fuzz ARM router firmware](https://github.com/qilingframework/theme.qiling.io/blob/master/source/img/fuzzer.jpg?raw=true)](https://www.youtube.com/watch?v=e3_T3KLhNUs) +[![Qiling Tutorial: Emulating and Fuzz ARM router firmware](https://github.com/qilingframework/theme.qiling.io/blob/master/source/img/fuzzer.jpg?raw=true)](https://www.youtube.com/watch?v=e3_T3KLh2NU) #### Qiling's IDA Pro Plugin: Instrument and Decrypt Mirai's Secret -This video demonstrates how Qiling's IDA Pro plugin can make IDA Pro run with Qiling instrumentation engine. +This video demonstrates how Qiling's IDA Pro plugin can make IDA Pro run with +Qiling instrumentation engine. [![Qiling's IDA Pro Plugin: Instrument and Decrypt Mirai's Secret](http://img.youtube.com/vi/ZWMWTq2WTXk/0.jpg)](http://www.youtube.com/watch?v=ZWMWTq2WTXk) @@ -192,62 +195,63 @@ Solving a simple CTF challenge with Qiling Framework and IDA Pro [![Solving a simple CTF challenge with Qiling Framework and IDA Pro](https://i.ytimg.com/vi/SPjVAt2FkKA/0.jpg)](https://www.youtube.com/watch?v=SPjVAt2FkKA) + #### Emulating MBR Qiling Framework emulates MBR [![Qiling DEMO: Emulating MBR](https://github.com/qilingframework/theme.qiling.io/blob/master/source/img/mbr.png?raw=true)](https://github.com/qilingframework/theme.qiling.io/blob/master/source/img/mbr.png?raw=true) -## Qltool +--- + +#### Qltool Qiling also provides a friendly tool named `qltool` to quickly emulate shellcode & executable binaries. With qltool, easy execution can be performed: + With shellcode: -```bash +``` $ ./qltool code --os linux --arch arm --format hex -f examples/shellcodes/linarm32_tcp_reverse_shell.hex ``` With binary file: -```bash +``` $ ./qltool run -f examples/rootfs/x8664_linux/bin/x8664_hello --rootfs examples/rootfs/x8664_linux/ ``` With binary and GDB debugger enabled: -```bash +``` $ ./qltool run -f examples/rootfs/x8664_linux/bin/x8664_hello --gdb 127.0.0.1:9999 --rootfs examples/rootfs/x8664_linux ``` With code coverage collection (UEFI only for now): -```bash +``` $ ./qltool run -f examples/rootfs/x8664_efi/bin/TcgPlatformSetupPolicy --rootfs examples/rootfs/x8664_efi --coverage-format drcov --coverage-file TcgPlatformSetupPolicy.cov ``` With JSON output (Windows, mainly): -```bash +``` $ ./qltool run -f examples/rootfs/x86_windows/bin/x86_hello.exe --rootfs examples/rootfs/x86_windows/ --console False --json ``` +--- -## Contributing -We welcome contributions from the community! If you're interested in contributing to Qiling Framework, please check out our [GitHub repository](https://github.com/qilingframework/qiling) and look for open issues or submit a pull request. +#### Contact -## License +Get the latest info from our website https://www.qiling.io -This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. +Contact us at email info@qiling.io, +via Twitter [@qiling_io](https://twitter.com/qiling_io). -## Contact +--- -Get the latest info from our website [https://www.qiling.io](https://www.qiling.io) +#### Core developers, Key Contributors and etc. -Contact us at email [info@qiling.io](mailto:info@qiling.io), or via Twitter [@qiling_io](https://twitter.com/qiling_io). - -## Core Developers & Contributors - -Please refer to [CREDITS.md](https://github.com/qilingframework/qiling/blob/dev/CREDITS.md). \ No newline at end of file +Please refer to [CREDITS.md](https://github.com/qilingframework/qiling/blob/dev/CREDITS.md). diff --git a/TODO b/TODO deleted file mode 100644 index b2471eb1..00000000 --- a/TODO +++ /dev/null @@ -1 +0,0 @@ -Features request and TODO please refer to issue 333 https://github.com/qilingframework/qiling/issues/333 diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..449b7df5 --- /dev/null +++ b/TODO.md @@ -0,0 +1,693 @@ +# Qiling Framework TODO + +Features request and TODO please refer to issue 333 https://github.com/qilingframework/qiling/issues/333 + +--- + +## Hybrid Kernel Architecture + +### The Problem + +Qiling reimplements Linux kernel behavior syscall-by-syscall in Python. Simple +syscalls (file I/O, memory, stat) work well. Complex subsystems do not: + +- **Networking**: No epoll. Sockets are host sockets with no isolation. No TCP + state machine, no multicast, no raw/netlink sockets. +- **Threading**: Gevent greenlets — cooperative, single-threaded. No preemption. + Futex is a gevent Event. pthreads don't behave correctly. +- **Signals**: `signal()`, `sigaction()`, `kill()` are stubs. No delivery, no EINTR. + +Reimplementing the full kernel is not realistic. Instead, offload complex subsystems +to a real Linux kernel while keeping Qiling's instrumentation intact. + +### Key Insight: Two-Layer Forwarding + +There are two integration points, each serving a different purpose: + +**Layer 1 — Generic fallback** (catches the long tail): + +The user explicitly chooses which missing syscalls to forward. Nothing is automatic. +By default Qiling behaves exactly as it does today — if a syscall is not implemented, +it fails. The user then tells the proxy which specific syscalls to forward: + +```python +ql = Qiling(argv=["/bin/myserver"], rootfs="rootfs/x8664_linux") +proxy = KernelProxy(ql) # start proxy process +proxy.forward_syscall("epoll_create") # forward this one to real kernel +proxy.forward_syscall("epoll_ctl") # and this one +proxy.forward_syscall("epoll_wait") # and this one +ql.run() +``` + +Under the hood, `forward_syscall("epoll_create")` registers a CALL hook via the +existing `set_syscall()` mechanism (`posix.py:128-143`). The hook reads args from +the emulated registers, sends them to the proxy process, and returns the real result. +**Zero changes to `load_syscall()` or any existing dispatch code.** + +For syscalls that return FDs (socket, epoll_create, eventfd, etc.), the proxy +wraps the returned FD in a `ql_proxy_fd` object and stores it in the FD table. +The FD table (`QlFileDes`) already stores polymorphic objects — `ql_socket`, +`ql_file`, `ql_pipe` — so `ql_proxy_fd` slots in naturally. When existing handlers +like `ql_syscall_read` or `ql_syscall_close` hit a proxy FD, they dispatch through +`ql_proxy_fd.read()` / `.close()` which forwards to the proxy. + +``` +Syscall interrupt + → load_syscall() [UNCHANGED] + → has CALL hook? (user hook or proxy-registered hook) + → yes: use it + → has Python handler? (existing code) + → yes: use it [unchanged — file I/O, memory, stat, etc.] + → neither? + → log warning [existing behavior, unchanged — no auto-forwarding] +``` + +### What Changes vs What Doesn't + +| Component | Changes? | Notes | +|-----------|----------|-------| +| `load_syscall()` dispatch | **NO** | Entirely unchanged | +| Existing syscall handlers | NO | Python handlers stay as-is | +| `QlFileDes` FD table | NO | Already polymorphic, new FD type slots in | +| `set_syscall()` user hooks | NO | User CALL/ENTER/EXIT hooks still work | +| `ql.run()` / `ql.emu_start()` | NO | Unicorn execution loop untouched | +| Hook system (`core_hooks.py`) | NO | Standard Unicorn hook mechanism | +| New: kernel proxy process | YES | New module, new files only | +| New: `ql_proxy_fd` FD type | YES | New class, same interface as `ql_socket` | +| New: `KernelProxy` class | YES | User-facing API, registers CALL hooks | + +--- + +## Phase 0: Proof of Concept + +**Goal**: User can explicitly forward specific unimplemented syscalls to a real Linux +kernel. No automatic behavior. No changes to existing Qiling dispatch code. + +### Usage + +```python +from qiling import Qiling +from qiling.os.posix.kernel_proxy import KernelProxy + +ql = Qiling(argv=["/bin/myserver"], rootfs="rootfs/x8664_linux") + +# Start a kernel proxy — a helper process that executes real syscalls +proxy = KernelProxy(ql) + +# Binary needs epoll but Qiling doesn't implement it. +# User identifies the 3 missing syscalls and forwards them: +proxy.forward_syscall("epoll_create") +proxy.forward_syscall("epoll_ctl") +proxy.forward_syscall("epoll_wait") + +ql.run() +# epoll_create/ctl/wait are handled by real kernel. +# Everything else (read, write, open, mmap, ...) uses existing Qiling handlers. +``` + +If the user does NOT set up a proxy, Qiling behaves exactly as it does today. +No surprises, no magic. + +### 0.1 Kernel proxy process + +A standalone Python process that executes real Linux syscalls on behalf of Qiling. +Communicates via Unix socketpair. + +- [ ] New directory: `qiling/os/posix/kernel_proxy/` +- [ ] `proxy.py` — proxy subprocess entry point + - Main loop: read request from socket → `libc.syscall(nr, *args)` → write response + - Uses `ctypes.CDLL("libc.so.6").syscall()` for raw syscall execution + - Manages its own FD table (proxy-side FDs) +- [ ] IPC protocol (binary, over socketpair): + ``` + Request: { type: SYSCALL, syscall_nr: u32, args: [u64; 6] } + Response: { return_value: i64, errno: i32 } + + Request: { type: FD_OP, op: READ|WRITE|CLOSE, proxy_fd: i32, length: u32, data?: bytes } + Response: { return_value: i64, errno: i32, data?: bytes } + ``` + Two message types: raw syscall forwarding, and FD operations (for read/write/close + on proxy-owned FDs). +- [ ] Lifecycle: started by `KernelProxy.__init__()`, killed on `ql.run()` exit + +### 0.2 `KernelProxy` class — user-facing API + +The main integration class. Lives in `qiling/os/posix/kernel_proxy/__init__.py`. + +```python +class KernelProxy: + def __init__(self, ql: Qiling): + """Start the proxy subprocess.""" + self.ql = ql + self._proxy_process = ... # start subprocess + self._ipc = ... # socketpair connection + self._forwarded = {} # syscall_name → syscall_nr + + def forward_syscall(self, name: str, returns_fd: bool = False): + """Register a CALL hook that forwards this syscall to the proxy. + + Args: + name: syscall name (e.g. "epoll_create", "eventfd") + returns_fd: if True, wrap the return value in ql_proxy_fd + and store in the FD table + """ + # Look up syscall number from the architecture's syscall table + # Register a CALL hook via ql.os.set_syscall() + ql.os.set_syscall(name, self._make_forwarder(name, returns_fd)) + + def _make_forwarder(self, name, returns_fd): + """Create a CALL hook function for this syscall.""" + syscall_nr = self._resolve_syscall_nr(name) + + def forwarder(ql, *args): + # Send all args as raw integers to proxy + retval = self._ipc.forward(syscall_nr, args) + + if returns_fd and retval >= 0: + # Wrap proxy FD and store in Qiling's FD table + proxy_fd = ql_proxy_fd(self._ipc, retval) + guest_fd = self._alloc_fd(ql, proxy_fd) + return guest_fd + + return retval + + return forwarder +``` + +Key points: +- `forward_syscall()` uses the existing `set_syscall()` mechanism — a standard + CALL hook. No special dispatch path, no changes to `load_syscall()`. +- User explicitly passes `returns_fd=True` for FD-returning syscalls. No heuristics. +- User ENTER/EXIT hooks still fire around the forwarded syscall (existing behavior + of the hook chain in `load_syscall()` lines 206-224). + +- [ ] Implement `KernelProxy` class +- [ ] Implement `_resolve_syscall_nr()` — look up syscall number from arch tables +- [ ] Implement `_make_forwarder()` — create CALL hook closure +- [ ] Implement `_alloc_fd()` — find empty slot in `ql.os.fd[]`, store `ql_proxy_fd` + +### 0.3 `ql_proxy_fd` — proxy-side FD wrapper + +When a forwarded syscall returns an FD (e.g., `epoll_create` returns 5 in the proxy), +we store a `ql_proxy_fd` in Qiling's FD table. This object forwards read/write/close +to the proxy, matching the interface of `ql_socket` (`filestruct.py:14`). + +```python +class ql_proxy_fd: + """FD whose real file/socket lives in the kernel proxy process.""" + + def __init__(self, ipc, proxy_fd: int): + self._ipc = ipc + self._proxy_fd = proxy_fd + + def read(self, length: int) -> bytes: + return self._ipc.fd_read(self._proxy_fd, length) + + def write(self, data: bytes) -> int: + return self._ipc.fd_write(self._proxy_fd, data) + + def close(self) -> None: + self._ipc.fd_close(self._proxy_fd) + + def fileno(self) -> int: + return -1 # not a real host FD +``` + +Because `ql_syscall_read` / `ql_syscall_write` / `ql_syscall_close` already dispatch +through `ql.os.fd[fd].read()` / `.write()` / `.close()`, **these existing handlers +need no changes**. When the binary calls `read(fd, buf, n)` on a proxy FD, the +existing read handler calls `ql_proxy_fd.read(n)`, gets data back, and writes it +to guest memory as usual. + +- [ ] Implement `ql_proxy_fd` class in `qiling/os/posix/kernel_proxy/proxy_fd.py` +- [ ] Verify `ql_syscall_read` works with `ql_proxy_fd` — no changes needed +- [ ] Verify `ql_syscall_write` works with `ql_proxy_fd` — no changes needed +- [ ] Verify `ql_syscall_close` works with `ql_proxy_fd` — no changes needed + +### 0.4 Pointer-bearing syscalls (Phase 0 scope: one example) + +Some forwarded syscalls take pointers to guest memory. The proxy can't read guest +memory directly. For Phase 0, implement ONE pointer-bearing forwarder as an example +to prove the pattern works. `epoll_ctl` is a good candidate: + +``` +epoll_ctl(epfd, op, fd, struct epoll_event *event) +``` + +The forwarder must: +1. Read `struct epoll_event` (8 bytes) from guest memory at the `event` pointer +2. Send the struct data along with the integer args to the proxy +3. Proxy reconstructs the struct in its own memory, calls real `epoll_ctl` + +- [ ] Extend IPC protocol for buffer-carrying requests: + ``` + Request: { type: SYSCALL_WITH_BUFS, syscall_nr, args[6], buffers: [(arg_idx, direction, data)] } + Response: { return_value, errno, buffers: [(arg_idx, data)] } + ``` + `direction` is IN (guest→proxy), OUT (proxy→guest), or INOUT. +- [ ] Implement `forward_syscall_with_buffers()` API for pointer-bearing syscalls +- [ ] Implement `epoll_ctl` forwarder as the working example + +### 0.5 Validation + +- [ ] Test: binary that uses `epoll_create` + `epoll_ctl` + `epoll_wait` on a + timerfd or eventfd. User forwards all 4 syscalls. Verify it works end-to-end. +- [ ] Test: same binary WITHOUT proxy — Qiling fails as it does today. No regression. +- [ ] Test: binary that calls `socket()` — existing Qiling handler runs (user did + NOT forward `socket`). Verify no interference. +- [ ] Test: user `set_syscall("epoll_create", my_hook)` — user hook takes priority + over proxy hook (user hook registered after proxy hook overwrites it via + `set_syscall`). Verify user control is preserved. +- [ ] Test: proxy process crash — verify Qiling reports error cleanly, doesn't hang. + +**Existing code modified**: NONE. All new files in `qiling/os/posix/kernel_proxy/`. +Integration is purely through `set_syscall()`. + +**Risk**: LOW — new code in new module. Existing behavior completely unchanged unless +user explicitly creates a `KernelProxy` and calls `forward_syscall()`. + +--- + +## Phase 1: Networking Foundation + +**Goal**: Forward all socket syscalls. Make a TCP client work end-to-end. + +### 1.1 `ql_proxy_socket` FD type + +New class in `qiling/os/posix/filestruct.py` (or new file alongside it). Must match +`ql_socket` interface so generic I/O dispatches correctly: + +```python +class ql_proxy_socket: + """Socket FD whose real socket lives in the kernel proxy process.""" + + def read(self, length: int) -> bytes: + # Forward to proxy: recv(self.proxy_fd, length) + + def write(self, data: bytes) -> int: + # Forward to proxy: send(self.proxy_fd, data) + + def close(self) -> None: + # Forward to proxy: close(self.proxy_fd) + + def fileno(self) -> int: + # Return a sentinel — not a real host FD + + # Socket-specific methods forwarded to proxy: + def connect(self, address) -> None: ... + def bind(self, address) -> None: ... + def listen(self, backlog) -> None: ... + def accept(self) -> tuple: ... + def shutdown(self, how) -> None: ... + def setsockopt(self, level, optname, value) -> None: ... + def getsockopt(self, level, optname) -> ...: ... +``` + +Because `ql_syscall_read` / `ql_syscall_write` / `ql_syscall_close` already dispatch +through `ql.os.fd[fd].read()` / `.write()` / `.close()`, these **existing handlers +need no changes** — the proxy socket object handles forwarding internally. + +- [ ] Implement `ql_proxy_socket` class +- [ ] IPC client method for each operation +- [ ] Verify generic `read(fd, ...)` and `write(fd, ...)` work on proxy sockets + without modifying `ql_syscall_read` or `ql_syscall_write` + +### 1.2 Socket syscall CALL hooks + +Register CALL hooks for socket-specific syscalls. These are needed because socket +syscalls (bind, connect, listen, accept, etc.) have special argument handling +(sockaddr structs, address lengths) that goes beyond generic read/write. + +- [ ] `socket()` — create proxy socket, store `ql_proxy_socket` in FD table +- [ ] `bind(fd, addr, addrlen)` — read sockaddr from guest memory, forward to proxy +- [ ] `connect(fd, addr, addrlen)` — same pattern +- [ ] `listen(fd, backlog)` — forward +- [ ] `accept(fd, addr, addrlen)` — forward, create new `ql_proxy_socket` for client FD +- [ ] `send/sendto/sendmsg` — read buffer from guest memory, forward +- [ ] `recv/recvfrom/recvmsg` — forward, write received data to guest memory +- [ ] `setsockopt/getsockopt` — forward with option translation +- [ ] `getpeername/getsockname` — forward, write sockaddr to guest memory +- [ ] `shutdown` — forward +- [ ] `socketpair` — forward, create two `ql_proxy_socket` objects +- [ ] `close` on proxy sockets — handled by `ql_proxy_socket.close()`, + but also register hook to detect close on proxy FDs if needed + +**Struct translation**: sockaddr family (AF_INET, AF_INET6, AF_UNIX) is the same +across architectures. Network byte order is architecture-independent. The main +concern is pointer width (32-bit guest on 64-bit host) — read the right number +of bytes from guest memory based on `ql.arch.pointersize`. + +### 1.3 `socketcall()` multiplexer (x86 32-bit) + +On x86 32-bit, all socket operations go through a single `socketcall()` syscall +(`qiling/os/posix/syscall/net.py`). The existing multiplexer dispatches to individual +handlers. Since we hook the individual handlers (bind, connect, etc.), this works +automatically. But verify: + +- [ ] Test that x86 32-bit socket operations are correctly forwarded via the + existing socketcall → individual handler → our CALL hook chain + +### 1.4 Validation + +- [ ] TCP client: connect to a server, send/receive data, close +- [ ] TCP server: bind, listen, accept, handle client, close +- [ ] UDP: sendto/recvfrom +- [ ] Unix domain sockets (path-based) +- [ ] Existing non-network tests still pass (regression check) + +**Risk**: LOW-MEDIUM — new code + new FD type, but existing handlers and dispatch +untouched. Main risk is FD lifecycle bugs (leak, double-close). + +--- + +## Phase 2: Complete Networking + +**Goal**: epoll, network namespaces, advanced operations. Real-world network +binaries work. + +### 2.1 epoll forwarding + +epoll is currently **not implemented at all** — mapped in the syscall table but +no handler. This is new functionality, not a change to existing behavior. + +- [ ] `epoll_create` / `epoll_create1` — forward, return proxy epoll FD + (new FD type or reuse `ql_proxy_socket` with a flag) +- [ ] `epoll_ctl(epfd, op, fd, event)` — forward; `fd` must be translated to + proxy FD space. Read `epoll_event` struct from guest memory. +- [ ] `epoll_wait(epfd, events, maxevents, timeout)` — forward. **This blocks** + in the proxy. For single-threaded programs this is correct (binary would be + blocked anyway). Write returned events to guest memory. +- [ ] `epoll_pwait` — same as epoll_wait + signal mask + +**Blocking concern**: when the proxy is blocked on `epoll_wait`, the Unicorn +emulation is paused. This is correct for single-threaded programs. For multithreaded +programs, we need real threading (Phase 3) where each thread has its own Unicorn +and can block independently. + +### 2.2 poll/select integration + +Currently `poll()` and `select()` use host `select.poll()`/`select.select()` directly, +which won't work for proxy FDs (no host FD to poll). + +- [ ] Hook `poll()` and `select()` — for FD sets containing proxy FDs, forward the + entire operation to the proxy +- [ ] For mixed FD sets (some proxy, some local): forward the proxy FDs to the proxy, + poll local FDs locally, merge results. This is complex — consider forwarding all + FDs to the proxy as the simpler approach. + +### 2.3 Network namespace isolation + +- [ ] Proxy process runs in its own network namespace (`unshare(CLONE_NEWNET)`) +- [ ] Configurable modes: + - `host`: proxy shares host network (default, simplest) + - `isolated`: separate namespace, no connectivity + - `bridged`: veth pair with NAT to host +- [ ] DNS: mount a resolv.conf in the proxy's mount namespace if needed + +### 2.4 Advanced operations (lower priority) + +- [ ] `sendmmsg` / `recvmmsg` — batch send/receive +- [ ] Raw sockets / packet sockets (`AF_PACKET`) +- [ ] Netlink sockets (`AF_NETLINK`) — for binaries that call `ip`, `route`, etc. +- [ ] `SCM_RIGHTS` (FD passing over Unix sockets) — requires FD translation +- [ ] IPv6 multicast + +### 2.5 Validation + +- [ ] epoll-based TCP echo server +- [ ] HTTP client (wget/curl-like binary) +- [ ] Binary that uses poll() with mixed file + socket FDs +- [ ] Network namespace: verify proxy and emulated binary are isolated from host +- [ ] Performance: measure latency overhead of IPC per syscall + +**Risk**: MEDIUM — epoll is new functionality (no regression risk), but poll/select +changes for proxy FDs touch existing handlers. The mixed-FD-set case is the main +complexity. + +--- + +## Phase 3: Real Threading + +**Goal**: Real concurrency with one Unicorn engine per thread. + +**This phase is high-risk and should only start after Phase 2 is stable.** It touches +the Unicorn integration, memory manager, thread lifecycle, and scheduler — all core +components. Needs a detailed design document before implementation begins. + +### Prerequisites + +- [ ] Phase 1-2 networking is stable and tested +- [ ] Detailed design document covering memory sharing, thread lifecycle, + and failure modes +- [ ] Prototype benchmark: measure overhead of multiple Unicorn instances + sharing memory via `mem_map_ptr` + +### 3.1 Shared memory backing for Unicorn + +Currently `QlMemoryManager.map()` calls `uc.mem_map()` which allocates internal +Unicorn memory. For shared threading, all Unicorn instances must see the same memory. + +- [ ] Change memory backing to use `mmap(MAP_SHARED)` + `uc.mem_map_ptr()` +- [ ] This affects: `QlMemoryManager.map()`, `QlMemoryManager.protect()`, + `QlMemoryManager.unmap()` +- [ ] Loader changes: ELF/PE/MachO loaders must write segments into shared-backed + memory regions +- [ ] MMIO regions stay callback-based (not shared) +- [ ] **Critical**: This must be done as a standalone change that passes ALL existing + tests before moving to 3.2. If existing tests break, the shared memory + implementation is wrong. + +Files: `qiling/os/memory.py`, `qiling/loader/elf.py`, `qiling/loader/pe.py` + +### 3.2 One Unicorn instance per thread + +Replace gevent Greenlets with real OS threads, each owning a Unicorn instance. + +- [ ] New thread class: `QlLinuxRealThread` (alongside existing `QlLinuxThread`) + - Creates a new `Uc` instance on spawn + - Maps all shared memory regions into the new Uc via `mem_map_ptr` + - Copies parent registers to child Uc + - Sets child's SP, TLS, return value + - Runs in a real `threading.Thread` +- [ ] Modify `clone()` handler: when hybrid threading is enabled, create + `QlLinuxRealThread` instead of gevent Greenlet +- [ ] Per-thread hook context: each Unicorn instance needs its own hooks registered. + User-defined hooks must be replicated to all instances. +- [ ] Remove the 32337-instruction cooperative scheduling loop — real OS scheduler + handles preemption + +**What breaks**: The current model assumes ONE `ql.uc` instance. Code that accesses +`ql.uc` directly will see only one thread's Unicorn. Need to audit all `ql.uc` +references and route to the current thread's instance. + +Risky references: +- `ql.arch.regs` reads/writes `ql.uc` registers — must route to current thread's Uc +- `ql.mem.read/write` calls `ql.uc.mem_read/write` — with shared memory, any Uc works +- `ql.hook_*` registers on `ql.uc` — must register on all Uc instances +- `ql.save()/restore()` snapshots `ql.uc` — must snapshot correct thread + +### 3.3 Synchronization primitives via real kernel + +With real OS threads sharing real memory, kernel synchronization works natively. + +- [ ] Forward `futex()` to kernel — `FUTEX_WAIT`/`FUTEX_WAKE` operate on the shared + memory addresses directly +- [ ] Remove gevent Event-based futex emulation (`qiling/os/linux/futex.py`) +- [ ] Forward `set_robust_list`, `get_robust_list` +- [ ] `pthread_mutex_*`, `pthread_cond_*` — these use futex internally, so forwarding + futex is sufficient + +### 3.4 Thread safety for shared state + +With real concurrent threads, shared mutable state needs synchronization. + +- [ ] `QlMemoryManager`: lock `map_info` list mutations (map, unmap, protect) + - read/write don't need locks if backed by shared mmap (atomic at OS level) +- [ ] `QlFileDes`: lock FD table mutations (open, close, dup) +- [ ] Hook lists: lock registration/deregistration (hooks are usually set up before + `run()`, so contention should be minimal) +- [ ] Logging: thread-safe log handler with thread ID prefix + +### 3.5 Validation + +- [ ] pthread_create / pthread_join +- [ ] Mutex: two threads incrementing a shared counter with proper locking +- [ ] Condition variables: producer-consumer +- [ ] Futex: custom futex-based synchronization +- [ ] Thread-local storage (TLS) correctness per architecture +- [ ] Stress test: 10+ threads doing concurrent work +- [ ] ALL existing single-threaded tests still pass +- [ ] ALL existing gevent-threaded tests still pass (gevent mode preserved as fallback) + +**Risk**: HIGH — changes to memory manager, Unicorn integration, and thread model. +Keep the existing gevent threading as a fallback mode. The new threading is opt-in. + +--- + +## Phase 4: Signals + +**Depends on Phase 3** (real threads required for proper signal delivery). + +### 4.1 Signal handler registration + +- [ ] Forward `sigaction(signum, act, oldact)` to kernel proxy +- [ ] Forward `sigprocmask` / `rt_sigprocmask` +- [ ] Forward `sigaltstack` + +### 4.2 Signal delivery + +When a signal is delivered to a proxy thread: + +- [ ] Proxy catches the signal and sends notification to Qiling via IPC +- [ ] Qiling calls `emu_stop()` on the target thread's Unicorn +- [ ] Save thread context (registers) +- [ ] Build signal frame on emulated stack (architecture-specific) +- [ ] Set PC to the registered signal handler +- [ ] Resume Unicorn — handler executes in emulated code +- [ ] On `sigreturn` / `rt_sigreturn`: restore saved context, resume normal execution + +### 4.3 Signal-syscall interaction + +- [ ] `EINTR` on interrupted blocking syscalls +- [ ] `SA_RESTART` flag: automatically restart interrupted syscalls +- [ ] `kill()`, `tgkill()`, `tkill()` → forward to kernel + +### 4.4 Validation + +- [ ] SIGALRM handler (timer-based) +- [ ] SIGCHLD on child exit +- [ ] SIGPIPE on broken pipe +- [ ] Signal interrupting `read()` — verify EINTR +- [ ] Custom signal handler that modifies emulated state + +**Risk**: MEDIUM — signal frame construction is architecture-specific and fiddly, +but the mechanism is well-understood. Main risk is getting the frame layout exactly +right for each architecture. + +--- + +## Phase 5: Integration and Polish + +### 5.1 User-facing API + +```python +# Opt-in to hybrid kernel +ql = Qiling(argv=[...], rootfs="...") + +# Enable kernel proxy for networking (Phase 1-2) +ql.os.kernel_proxy.enable(networking=True) + +# Enable real threading (Phase 3) — requires networking=True +ql.os.kernel_proxy.enable(networking=True, threading=True) + +# Enable signals (Phase 4) — requires threading=True +ql.os.kernel_proxy.enable(networking=True, threading=True, signals=True) + +# Configure network namespace +ql.os.kernel_proxy.network_mode = "bridged" # "host" | "isolated" | "bridged" + +# User hooks still work — they fire before/after proxy forwarding +ql.os.set_syscall("connect", my_connect_hook, QL_INTERCEPT.ENTER) +``` + +### 5.2 Backward compatibility + +- [ ] Default behavior: no proxy, existing Python handlers — zero regression +- [ ] All existing tests pass with proxy disabled +- [ ] All existing tests pass with proxy enabled (forwarded syscalls should + produce equivalent results) +- [ ] `set_syscall()` user hooks fire correctly in both modes +- [ ] Existing gevent threading preserved as fallback when real threading not enabled + +### 5.3 Fallback on failure + +- [ ] If proxy process crashes: log error, fall back to Python handlers, warn user +- [ ] If proxy not available (non-Linux host): use Python handlers, warn user +- [ ] Graceful degradation: never crash, always fall back + +### 5.4 Platform support + +- [ ] Linux host: full support (namespaces, real threading) +- [ ] macOS host: proxy via Docker/Lima (networking only, no native namespaces) +- [ ] Windows host: proxy via WSL2 (networking only) +- [ ] Document host requirements + +### 5.5 Performance + +- [ ] Benchmark: syscall latency (Python handler vs proxy round-trip) +- [ ] Optimize IPC: shared memory ring buffer for high-frequency syscalls +- [ ] Batch small syscalls where possible +- [ ] Profile and tune for common workloads (network servers, threaded computation) + +--- + +## Existing Issues (Independent of Hybrid Architecture) + +These should be fixed regardless of the hybrid work. + +### Bare except blocks swallowing errors + +10+ bare `except:` blocks silently hide failures: + +- `qiling/utils.py:242` — PE detection +- `qiling/debugger/qdb/qdb.py:128,352,598` — debugger operations +- `qiling/os/posix/filestruct.py:62,173,179` — fcntl/ioctl +- `qiling/os/posix/syscall/select.py:78` — select failures +- `qiling/os/windows/registry.py:127,185` — registry operations + +### Asserts used for validation + +Assertions disabled with `python -O`. Replace with exceptions: + +- `qiling/os/memory.py` — page alignment, size, mapping checks +- `qiling/arch/x86_utils.py` — GDT/segment validation +- `qiling/cc/__init__.py` — calling convention validation + +### Memory manager: string label parsing + +`qiling/os/memory.py:209-218` — `get_lib_base()` uses regex on info strings. +Needs a proper mapping structure. + +### ARM Thumb mode detection + +`core.py:753` — fragile `_init_thumb` flag. Needs upstream Unicorn fix. + +### x86 GDT privilege levels + +`qiling/arch/x86_utils.py:147,178` — ring 3 forced to ring 0. + +### Unbounded `read_cstring` + +`qiling/os/memory.py:51-63` — no length limit. Can hang on MMIO. + +### Incomplete save/restore + +`QlOs.save()/restore()` empty in base class. UEFI and Windows don't implement it. + +### Incomplete Windows emulation + +Fiber, registry, handle management, DLL resolution gaps. +See `qiling/os/windows/` TODO comments. + +### macOS and UEFI gaps + +- macOS kext: 5 FIXMEs in `macos.py:79-117` +- UEFI variables: `uefi/rt.py:204-205` + +### Hook system cleanup + +- `type()` vs `isinstance()` in `core_hooks.py` +- Unclear return value semantics +- Non-intuitive `begin=1, end=0` for "entire memory" + +### Hardcoded magic numbers + +- Exit points in `os/os.py:84-87` +- Guard page `0x9000000` in `core.py:525` + +### Test coverage + +- ARM test skipped (`test_elf.py:411`) +- Multithread test skipped (`test_elf_multithread.py:185`) +- Broken wchar (`test_struct.py:170,185`) +- PowerPC, QNX, DOS, MCU: minimal coverage diff --git a/examples/rootfs b/examples/rootfs index 120fb6d3..6d4d654f 160000 --- a/examples/rootfs +++ b/examples/rootfs @@ -1 +1 @@ -Subproject commit 120fb6d37700a2d4c0e35ced599aaee7a8f98723 +Subproject commit 6d4d654fdc2892490d98c433eca3efa5c6d062c7 diff --git a/qiling/os/linux/map_syscall.py b/qiling/os/linux/map_syscall.py index 0584d58d..c3a1289e 100644 --- a/qiling/os/linux/map_syscall.py +++ b/qiling/os/linux/map_syscall.py @@ -2223,6 +2223,7 @@ riscv32_syscall_table = { 432: "fsmount", 433: "fspick", 434: "pidfd_open", + 435: "clone3", 436: "close_range", 437: "openat2", 438: "pidfd_getfd", @@ -2234,7 +2235,22 @@ riscv32_syscall_table = { 444: "landlock_create_ruleset", 445: "landlock_add_rule", 446: "landlock_restrict_self", + 447: "memfd_secret", 448: "process_mrelease", + 449: "futex_waitv", + 450: "set_mempolicy_home_node", + 451: "cachestat", + 452: "fchmodat2", + 453: "map_shadow_stack", + 454: "futex_wake", + 455: "futex_wait", + 456: "futex_requeue", + 457: "statmount", + 458: "listmount", + 459: "lsm_get_self_attr", + 460: "lsm_set_self_attr", + 461: "lsm_list_modules", + 462: "mseal", 1024: "open", 1025: "link", 1026: "unlink", @@ -2549,7 +2565,22 @@ riscv64_syscall_table = { 444: "landlock_create_ruleset", 445: "landlock_add_rule", 446: "landlock_restrict_self", + 447: "memfd_secret", 448: "process_mrelease", + 449: "futex_waitv", + 450: "set_mempolicy_home_node", + 451: "cachestat", + 452: "fchmodat2", + 453: "map_shadow_stack", + 454: "futex_wake", + 455: "futex_wait", + 456: "futex_requeue", + 457: "statmount", + 458: "listmount", + 459: "lsm_get_self_attr", + 460: "lsm_set_self_attr", + 461: "lsm_list_modules", + 462: "mseal", } ppc_syscall_table = { @@ -2968,4 +2999,33 @@ ppc_syscall_table = { 431: "fsconfig", 432: "fsmount", 433: "fspick", + 434: "pidfd_open", + 435: "clone3", + 436: "close_range", + 437: "openat2", + 438: "pidfd_getfd", + 439: "faccessat2", + 440: "process_madvise", + 441: "epoll_pwait2", + 442: "mount_setattr", + 443: "quotactl_fd", + 444: "landlock_create_ruleset", + 445: "landlock_add_rule", + 446: "landlock_restrict_self", + 447: "memfd_secret", + 448: "process_mrelease", + 449: "futex_waitv", + 450: "set_mempolicy_home_node", + 451: "cachestat", + 452: "fchmodat2", + 453: "map_shadow_stack", + 454: "futex_wake", + 455: "futex_wait", + 456: "futex_requeue", + 457: "statmount", + 458: "listmount", + 459: "lsm_get_self_attr", + 460: "lsm_set_self_attr", + 461: "lsm_list_modules", + 462: "mseal", } diff --git a/qiling/os/posix/kernel_proxy/__init__.py b/qiling/os/posix/kernel_proxy/__init__.py new file mode 100644 index 00000000..b0fae205 --- /dev/null +++ b/qiling/os/posix/kernel_proxy/__init__.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +""" +Hybrid kernel proxy — forward specific syscalls to a real Linux kernel. + +Usage: + from qiling import Qiling + from qiling.os.posix.kernel_proxy import KernelProxy + + ql = Qiling(argv=["/bin/myserver"], rootfs="rootfs/x8664_linux") + proxy = KernelProxy(ql) + proxy.forward_syscall("epoll_create", returns_fd=True) + proxy.forward_syscall("epoll_ctl") + proxy.forward_syscall("epoll_wait") + ql.run() +""" + +from __future__ import annotations + +import os +import sys +import socket +import subprocess +import logging +from typing import Optional, TYPE_CHECKING + +from qiling.const import QL_INTERCEPT, QL_OS +from qiling.os.posix.kernel_proxy.ipc import ProxyClient +from qiling.os.posix.kernel_proxy.proxy_fd import ql_proxy_fd + +if TYPE_CHECKING: + from qiling import Qiling + +log = logging.getLogger(__name__) + + +class KernelProxy: + """Forward specific syscalls to a real Linux kernel via a helper process. + + The proxy process executes real syscalls and returns results. Integration + is through set_syscall() CALL hooks — no changes to Qiling's dispatch code. + """ + + def __init__(self, ql: Qiling): + if sys.platform != 'linux': + raise RuntimeError("KernelProxy requires a Linux host") + + self.ql = ql + self._process: Optional[subprocess.Popen] = None + self._client: Optional[ProxyClient] = None + self._forwarded: dict = {} # name -> syscall_nr + self._reverse_table: Optional[dict] = None # name -> nr (built on first use) + + self._start_proxy() + + def _start_proxy(self): + """Start the proxy subprocess, connected via Unix socketpair.""" + parent_sock, child_sock = socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM) + + child_fd = child_sock.fileno() + + # ensure the subprocess can find qiling even when run from a subdirectory + env = os.environ.copy() + qiling_root = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))) + python_path = env.get('PYTHONPATH', '') + env['PYTHONPATH'] = f"{qiling_root}:{python_path}" if python_path else qiling_root + + self._process = subprocess.Popen( + [sys.executable, '-m', 'qiling.os.posix.kernel_proxy.proxy', str(child_fd)], + pass_fds=(child_fd,), + close_fds=True, + env=env, + ) + child_sock.close() + + self._client = ProxyClient(parent_sock) + log.info(f"kernel proxy started (pid={self._process.pid})") + + def _build_reverse_table(self) -> dict: + """Build name -> syscall_nr mapping from the guest architecture's syscall table.""" + if self._reverse_table is not None: + return self._reverse_table + + from qiling.os.linux.map_syscall import get_syscall_mapper + from qiling.const import QL_ARCH + + # get the raw syscall table dict for this architecture + arch_tables = { + QL_ARCH.ARM64 : 'arm64_syscall_table', + QL_ARCH.ARM : 'arm_syscall_table', + QL_ARCH.X8664 : 'x8664_syscall_table', + QL_ARCH.X86 : 'x86_syscall_table', + QL_ARCH.MIPS : 'mips_syscall_table', + QL_ARCH.RISCV : 'riscv32_syscall_table', + QL_ARCH.RISCV64 : 'riscv64_syscall_table', + QL_ARCH.PPC : 'ppc_syscall_table', + } + + table_name = arch_tables.get(self.ql.arch.type) + if table_name is None: + raise RuntimeError(f"KernelProxy: unsupported architecture {self.ql.arch.type}") + + import qiling.os.linux.map_syscall as mod + table = getattr(mod, table_name) + + # reverse: name -> nr + self._reverse_table = {name: nr for nr, name in table.items()} + return self._reverse_table + + def _resolve_syscall_nr(self, name: str) -> int: + """Resolve a syscall name to its number for the guest architecture.""" + table = self._build_reverse_table() + if name not in table: + raise ValueError( + f"KernelProxy: syscall '{name}' not found in {self.ql.arch.type.name} syscall table" + ) + return table[name] + + def forward_syscall(self, name: str, returns_fd: bool = False): + """Register a CALL hook that forwards this syscall to the kernel proxy. + + Args: + name: syscall name (e.g. "epoll_create", "eventfd2") + returns_fd: if True, wrap the return value in ql_proxy_fd and store + in the Qiling FD table. Use this for syscalls that return + file descriptors (epoll_create, eventfd, timerfd_create, etc.) + """ + nr = self._resolve_syscall_nr(name) + self._forwarded[name] = nr + + forwarder = self._make_forwarder(name, nr, returns_fd) + self.ql.os.set_syscall(name, forwarder, QL_INTERCEPT.CALL) + + log.info(f"forwarding syscall '{name}' (nr={nr}) to kernel proxy" + f"{' [returns FD]' if returns_fd else ''}") + + def _make_forwarder(self, name: str, guest_nr: int, returns_fd: bool): + """Create a CALL hook closure for one syscall.""" + client = self._client + + def _forwarder(ql, *args): + # use the HOST syscall number, not the guest number. + # for now, resolve from the host's syscall table at runtime. + host_nr = self._get_host_syscall_nr(name) + + padded = args + (0,) * (6 - len(args)) + retval = client.syscall(host_nr, padded[:6]) + + if returns_fd and retval >= 0: + # the proxy created a real FD. wrap it and store in Qiling's FD table. + proxy_fd_obj = ql_proxy_fd(client, retval) + guest_fd = self._alloc_fd(ql, proxy_fd_obj) + ql.log.debug(f"kernel_proxy: {name}() -> proxy_fd={retval}, guest_fd={guest_fd}") + return guest_fd + + ql.log.debug(f"kernel_proxy: {name}({', '.join(f'{a:#x}' for a in args)}) = {retval}") + return retval + + _forwarder.__name__ = f'ql_syscall_{name}' + return _forwarder + + def _get_host_syscall_nr(self, name: str) -> int: + """Get the syscall number on the HOST architecture.""" + # we are running on Linux — read from the host's syscall table + if not hasattr(self, '_host_table'): + self._host_table = self._load_host_syscall_table() + + if name not in self._host_table: + raise RuntimeError(f"KernelProxy: syscall '{name}' not available on host") + + return self._host_table[name] + + def _load_host_syscall_table(self) -> dict: + """Load the host's syscall name->nr mapping. + + Uses the same Qiling tables, indexed by the host architecture. + """ + import platform + from qiling.const import QL_ARCH + import qiling.os.linux.map_syscall as mod + + machine = platform.machine() + host_arch_map = { + 'x86_64': 'x8664_syscall_table', + 'aarch64': 'arm64_syscall_table', + 'armv7l': 'arm_syscall_table', + 'mips': 'mips_syscall_table', + 'riscv64': 'riscv64_syscall_table', + 'ppc': 'ppc_syscall_table', + } + + table_name = host_arch_map.get(machine) + if table_name is None: + raise RuntimeError(f"KernelProxy: unsupported host architecture '{machine}'") + + table = getattr(mod, table_name) + return {name: nr for nr, name in table.items()} + + @staticmethod + def _alloc_fd(ql, fd_obj) -> int: + """Find next free slot in Qiling's FD table and store fd_obj.""" + for i in range(len(ql.os.fd)): + if ql.os.fd[i] is None: + ql.os.fd[i] = fd_obj + return i + + raise OSError("kernel_proxy: FD table full") + + def stop(self): + """Stop the proxy process.""" + if self._client: + try: + self._client.close() + except Exception: + pass + self._client = None + + if self._process: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + self._process.wait() + log.info(f"kernel proxy stopped (pid={self._process.pid})") + self._process = None + + def __del__(self): + if hasattr(self, '_client'): + self.stop() diff --git a/qiling/os/posix/kernel_proxy/ipc.py b/qiling/os/posix/kernel_proxy/ipc.py new file mode 100644 index 00000000..8bf26eaa --- /dev/null +++ b/qiling/os/posix/kernel_proxy/ipc.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +""" +IPC protocol between Qiling and the kernel proxy process. + +Two message types: + SYSCALL — forward a raw syscall (number + 6 integer args) + FD_OP — perform an operation on a proxy-side FD (read/write/close/dup/fcntl/ioctl) + +All messages are length-prefixed binary over a Unix socketpair. +""" + +import struct +import socket +from enum import IntEnum + + +class MsgType(IntEnum): + SYSCALL = 1 + FD_OP = 2 + + +class FdOp(IntEnum): + READ = 1 + WRITE = 2 + CLOSE = 3 + DUP = 4 + FCNTL = 5 + IOCTL = 6 + + +# Wire format: +# Request header: [msg_type: u8][payload_len: u32] +# SYSCALL payload: [syscall_nr: u32][args: 6 x i64] +# FD_OP payload: [op: u8][proxy_fd: i32][arg1: i64][arg2: i64][data_len: u32][data: bytes] +# +# Response header: [status: i8][payload_len: u32] +# status 0 = success, -1 = error +# SYSCALL response payload: [return_value: i64][errno: i32] +# FD_OP response payload: [return_value: i64][errno: i32][data_len: u32][data: bytes] + +HEADER_FMT = '!BI' # msg_type/status (u8) + payload_len (u32) +HEADER_SIZE = struct.calcsize(HEADER_FMT) + +SYSCALL_REQ_FMT = '!I6q' # syscall_nr (u32) + 6 args (i64) +SYSCALL_REQ_SIZE = struct.calcsize(SYSCALL_REQ_FMT) + +SYSCALL_RESP_FMT = '!qi' # return_value (i64) + errno (i32) +SYSCALL_RESP_SIZE = struct.calcsize(SYSCALL_RESP_FMT) + +FD_OP_REQ_FMT = '!BiqqI' # op (u8) + proxy_fd (i32) + arg1 (i64) + arg2 (i64) + data_len (u32) +FD_OP_REQ_SIZE = struct.calcsize(FD_OP_REQ_FMT) + +FD_OP_RESP_FMT = '!qiI' # return_value (i64) + errno (i32) + data_len (u32) +FD_OP_RESP_SIZE = struct.calcsize(FD_OP_RESP_FMT) + + +def _recvall(sock: socket.socket, n: int) -> bytes: + """Receive exactly n bytes from a socket.""" + buf = bytearray() + while len(buf) < n: + chunk = sock.recv(n - len(buf)) + if not chunk: + raise ConnectionError("kernel proxy connection closed") + buf.extend(chunk) + return bytes(buf) + + +class ProxyClient: + """Qiling-side IPC client — sends requests to the proxy process.""" + + def __init__(self, sock: socket.socket): + self._sock = sock + + def syscall(self, nr: int, args: tuple) -> int: + """Forward a raw syscall. Returns kernel-convention result (negative errno on error).""" + padded = tuple(args) + (0,) * (6 - len(args)) + payload = struct.pack(SYSCALL_REQ_FMT, nr, *padded[:6]) + + # send request + header = struct.pack(HEADER_FMT, MsgType.SYSCALL, len(payload)) + self._sock.sendall(header + payload) + + # recv response + resp_header = _recvall(self._sock, HEADER_SIZE) + _, resp_len = struct.unpack(HEADER_FMT, resp_header) + resp_payload = _recvall(self._sock, resp_len) + + retval, errno_val = struct.unpack(SYSCALL_RESP_FMT, resp_payload) + return retval + + def _fd_op(self, op: FdOp, proxy_fd: int, arg1: int = 0, arg2: int = 0, data: bytes = b'') -> tuple: + """Send an FD operation. Returns (return_value, data).""" + payload = struct.pack(FD_OP_REQ_FMT, op, proxy_fd, arg1, arg2, len(data)) + payload += data + + header = struct.pack(HEADER_FMT, MsgType.FD_OP, len(payload)) + self._sock.sendall(header + payload) + + resp_header = _recvall(self._sock, HEADER_SIZE) + _, resp_len = struct.unpack(HEADER_FMT, resp_header) + resp_payload = _recvall(self._sock, resp_len) + + retval, errno_val, data_len = struct.unpack(FD_OP_RESP_FMT, resp_payload[:FD_OP_RESP_SIZE]) + resp_data = resp_payload[FD_OP_RESP_SIZE:FD_OP_RESP_SIZE + data_len] + + return retval, resp_data + + def fd_read(self, proxy_fd: int, length: int) -> bytes: + retval, data = self._fd_op(FdOp.READ, proxy_fd, arg1=length) + if retval < 0: + return b'' + return data + + def fd_write(self, proxy_fd: int, data: bytes) -> int: + retval, _ = self._fd_op(FdOp.WRITE, proxy_fd, data=data) + return retval + + def fd_close(self, proxy_fd: int) -> None: + self._fd_op(FdOp.CLOSE, proxy_fd) + + def fd_dup(self, proxy_fd: int) -> int: + retval, _ = self._fd_op(FdOp.DUP, proxy_fd) + return retval + + def fd_fcntl(self, proxy_fd: int, cmd: int, arg: int) -> int: + retval, _ = self._fd_op(FdOp.FCNTL, proxy_fd, arg1=cmd, arg2=arg) + return retval + + def fd_ioctl(self, proxy_fd: int, cmd: int, arg: int) -> int: + retval, _ = self._fd_op(FdOp.IOCTL, proxy_fd, arg1=cmd, arg2=arg) + return retval + + def close(self): + self._sock.close() + + +class ProxyServer: + """Proxy-side IPC server — receives requests, executes real syscalls.""" + + def __init__(self, sock: socket.socket): + self._sock = sock + + def recv_request(self) -> tuple: + """Receive one request. Returns (msg_type, parsed_fields).""" + header = _recvall(self._sock, HEADER_SIZE) + msg_type, payload_len = struct.unpack(HEADER_FMT, header) + + payload = _recvall(self._sock, payload_len) + + if msg_type == MsgType.SYSCALL: + fields = struct.unpack(SYSCALL_REQ_FMT, payload) + return MsgType.SYSCALL, fields # (nr, a0, a1, a2, a3, a4, a5) + + elif msg_type == MsgType.FD_OP: + fixed = struct.unpack(FD_OP_REQ_FMT, payload[:FD_OP_REQ_SIZE]) + op, proxy_fd, arg1, arg2, data_len = fixed + data = payload[FD_OP_REQ_SIZE:FD_OP_REQ_SIZE + data_len] + return MsgType.FD_OP, (FdOp(op), proxy_fd, arg1, arg2, data) + + else: + raise ValueError(f"unknown message type: {msg_type}") + + def send_syscall_response(self, retval: int, errno_val: int): + payload = struct.pack(SYSCALL_RESP_FMT, retval, errno_val) + header = struct.pack(HEADER_FMT, 0, len(payload)) + self._sock.sendall(header + payload) + + def send_fd_op_response(self, retval: int, errno_val: int, data: bytes = b''): + payload = struct.pack(FD_OP_RESP_FMT, retval, errno_val, len(data)) + payload += data + header = struct.pack(HEADER_FMT, 0, len(payload)) + self._sock.sendall(header + payload) diff --git a/qiling/os/posix/kernel_proxy/proxy.py b/qiling/os/posix/kernel_proxy/proxy.py new file mode 100644 index 00000000..41bceac0 --- /dev/null +++ b/qiling/os/posix/kernel_proxy/proxy.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +""" +Kernel proxy process — executes real Linux syscalls on behalf of Qiling. + +This runs as a subprocess. It receives syscall requests over a Unix socketpair, +executes them via libc.syscall(), and sends back results. + +Usage (internal — started by KernelProxy.__init__): + python -m qiling.os.posix.kernel_proxy.proxy +""" + +import os +import sys +import ctypes +import ctypes.util +import errno as errno_module +import socket + +from qiling.os.posix.kernel_proxy.ipc import ( + ProxyServer, MsgType, FdOp +) + +# load libc for raw syscall() +_libc_path = ctypes.util.find_library("c") +if _libc_path is None: + print("kernel_proxy: cannot find libc", file=sys.stderr) + sys.exit(1) + +_libc = ctypes.CDLL(_libc_path, use_errno=True) +_libc.syscall.restype = ctypes.c_long +_libc.syscall.argtypes = [ctypes.c_long] + [ctypes.c_long] * 6 + + +def raw_syscall(nr: int, a0: int, a1: int, a2: int, a3: int, a4: int, a5: int) -> tuple: + """Execute a real Linux syscall. Returns (return_value, errno) in kernel convention.""" + ctypes.set_errno(0) + result = _libc.syscall(nr, a0, a1, a2, a3, a4, a5) + + if result == -1: + err = ctypes.get_errno() + if err != 0: + return -err, err # kernel convention: negative errno + + return result, 0 + + +def handle_fd_op(op: FdOp, proxy_fd: int, arg1: int, arg2: int, data: bytes) -> tuple: + """Handle an FD operation on a proxy-side FD. Returns (retval, errno, data).""" + try: + if op == FdOp.READ: + result = os.read(proxy_fd, arg1) + return len(result), 0, result + + elif op == FdOp.WRITE: + written = os.write(proxy_fd, data) + return written, 0, b'' + + elif op == FdOp.CLOSE: + os.close(proxy_fd) + return 0, 0, b'' + + elif op == FdOp.DUP: + new_fd = os.dup(proxy_fd) + return new_fd, 0, b'' + + elif op == FdOp.FCNTL: + import fcntl + result = fcntl.fcntl(proxy_fd, arg1, arg2) + return result, 0, b'' + + elif op == FdOp.IOCTL: + import fcntl + result = fcntl.ioctl(proxy_fd, arg1, arg2) + return result, 0, b'' + + else: + return -errno_module.ENOSYS, errno_module.ENOSYS, b'' + + except OSError as e: + return -e.errno, e.errno, b'' + + +def main(): + if len(sys.argv) != 2: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + sock_fd = int(sys.argv[1]) + sock = socket.socket(fileno=sock_fd) + server = ProxyServer(sock) + + while True: + try: + msg_type, fields = server.recv_request() + except ConnectionError: + break + + if msg_type == MsgType.SYSCALL: + nr, a0, a1, a2, a3, a4, a5 = fields + retval, err = raw_syscall(nr, a0, a1, a2, a3, a4, a5) + server.send_syscall_response(retval, err) + + elif msg_type == MsgType.FD_OP: + op, proxy_fd, arg1, arg2, data = fields + retval, err, resp_data = handle_fd_op(op, proxy_fd, arg1, arg2, data) + server.send_fd_op_response(retval, err, resp_data) + + sock.close() + + +if __name__ == '__main__': + main() diff --git a/qiling/os/posix/kernel_proxy/proxy_fd.py b/qiling/os/posix/kernel_proxy/proxy_fd.py new file mode 100644 index 00000000..cbd6d5c0 --- /dev/null +++ b/qiling/os/posix/kernel_proxy/proxy_fd.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +""" +Proxy file descriptor — wraps an FD that lives in the kernel proxy process. + +When a forwarded syscall returns an FD (e.g. epoll_create, eventfd), the real +FD lives in the proxy. This wrapper forwards read/write/close to the proxy via +IPC, matching the interface of ql_socket and ql_pipe so existing syscall handlers +(ql_syscall_read, ql_syscall_write, ql_syscall_close) work without modification. +""" + +from __future__ import annotations +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from qiling.os.posix.kernel_proxy.ipc import ProxyClient + + +class ql_proxy_fd: + def __init__(self, client: ProxyClient, proxy_fd: int): + self._client = client + self._proxy_fd = proxy_fd + + def read(self, length: int) -> bytes: + return self._client.fd_read(self._proxy_fd, length) + + def write(self, data: bytes) -> int: + return self._client.fd_write(self._proxy_fd, data) + + def close(self) -> None: + self._client.fd_close(self._proxy_fd) + + def fileno(self) -> int: + return -1 + + def dup(self) -> ql_proxy_fd: + new_proxy_fd = self._client.fd_dup(self._proxy_fd) + return ql_proxy_fd(self._client, new_proxy_fd) + + def fcntl(self, cmd, arg): + return self._client.fd_fcntl(self._proxy_fd, cmd, arg) + + def ioctl(self, cmd, arg): + return self._client.fd_ioctl(self._proxy_fd, cmd, arg) diff --git a/tests/test_kernel_proxy.py b/tests/test_kernel_proxy.py new file mode 100644 index 00000000..d6690ad7 --- /dev/null +++ b/tests/test_kernel_proxy.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +import sys +import struct +import unittest +import platform + +sys.path.append("..") + +from qiling import Qiling +from qiling.const import QL_INTERCEPT, QL_VERBOSE + + +@unittest.skipUnless(platform.system() == 'Linux', 'kernel proxy requires Linux host') +class KernelProxyTest(unittest.TestCase): + """Tests for the hybrid kernel proxy (Phase 0).""" + + ROOTFS = "../examples/rootfs/x8664_linux" + HELLO_BIN = "../examples/rootfs/x8664_linux/bin/x8664_hello" + + # ------------------------------------------------------------------------- + # Proxy lifecycle + # ------------------------------------------------------------------------- + + def test_proxy_start_stop(self): + """Proxy starts and stops cleanly.""" + from qiling.os.posix.kernel_proxy import KernelProxy + + ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) + proxy = KernelProxy(ql) + + self.assertIsNotNone(proxy._process) + self.assertIsNotNone(proxy._client) + self.assertTrue(proxy._process.poll() is None) # still running + + proxy.stop() + self.assertIsNone(proxy._process) + self.assertIsNone(proxy._client) + + del ql + + def test_no_proxy_no_change(self): + """Without proxy, Qiling behaves identically.""" + from qiling.extensions import pipe + + ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) + ql.os.stdout = pipe.SimpleOutStream(1) + ql.run() + + self.assertIn(b"Hello", ql.os.stdout.read(1024)) + del ql + + def test_proxy_attached_no_forwarding(self): + """Proxy attached but no syscalls forwarded — no behavior change.""" + from qiling.os.posix.kernel_proxy import KernelProxy + from qiling.extensions import pipe + + ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) + ql.os.stdout = pipe.SimpleOutStream(1) + proxy = KernelProxy(ql) + + ql.run() + proxy.stop() + + self.assertIn(b"Hello", ql.os.stdout.read(1024)) + del ql + + # ------------------------------------------------------------------------- + # Raw syscall forwarding (integer-only args) + # ------------------------------------------------------------------------- + + def test_forward_getpid(self): + """Forward getpid — proxy returns its own PID.""" + from qiling.os.posix.kernel_proxy import KernelProxy + + ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) + proxy = KernelProxy(ql) + proxy.forward_syscall('getpid') + + results = [] + + def on_getpid_exit(ql, *args): + results.append(args[-1]) + + ql.os.set_syscall('getpid', on_getpid_exit, QL_INTERCEPT.EXIT) + ql.run() + + # if getpid was called, it should return the proxy's PID + if results: + self.assertEqual(results[0], proxy._process.pid) + + proxy.stop() + del ql + + def test_forward_brk(self): + """Forward brk to real kernel — binary still runs correctly.""" + from qiling.os.posix.kernel_proxy import KernelProxy + from qiling.extensions import pipe + + ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) + ql.os.stdout = pipe.SimpleOutStream(1) + proxy = KernelProxy(ql) + proxy.forward_syscall('brk') + + brk_results = [] + + def on_brk_exit(ql, *args): + brk_results.append(args[-1]) + + ql.os.set_syscall('brk', on_brk_exit, QL_INTERCEPT.EXIT) + ql.run() + proxy.stop() + + # brk is called during libc init + self.assertGreater(len(brk_results), 0, "brk was never called") + # binary should still produce correct output + self.assertIn(b"Hello", ql.os.stdout.read(1024)) + del ql + + # ------------------------------------------------------------------------- + # FD-returning syscalls (returns_fd=True) + # ------------------------------------------------------------------------- + + def test_forward_returns_fd(self): + """forward_syscall with returns_fd=True creates ql_proxy_fd in FD table.""" + from qiling.os.posix.kernel_proxy import KernelProxy + from qiling.os.posix.kernel_proxy.proxy_fd import ql_proxy_fd + + ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) + proxy = KernelProxy(ql) + proxy.forward_syscall('eventfd2', returns_fd=True) + + # call the forwarder directly (simulates binary calling eventfd2) + hook = ql.os.posix_syscall_hooks[QL_INTERCEPT.CALL].get('ql_syscall_eventfd2') + self.assertIsNotNone(hook, "forwarder hook not registered") + + guest_fd = hook(ql, 0, 0) + self.assertGreaterEqual(guest_fd, 0) + + fd_obj = ql.os.fd[guest_fd] + self.assertIsInstance(fd_obj, ql_proxy_fd) + + # write and read through the proxy FD + fd_obj.write(struct.pack(' Date: Sun, 12 Apr 2026 16:18:34 +0800 Subject: [PATCH 124/138] Fix docker.yaml --- poetry.lock | 530 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 450 insertions(+), 80 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3933b97c..18901472 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,24 +1,58 @@ -# This file is automatically @generated by Poetry and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. + +[[package]] +name = "antlr4-python3-runtime" +version = "4.8" +description = "ANTLR 4.8 runtime for Python 3.7" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "antlr4-python3-runtime-4.8.tar.gz", hash = "sha256:15793f5d0512a372b4e7d2284058ad32ce7dd27126b105fb0b2245130445db33"}, +] [[package]] name = "antlr4-python3-runtime" version = "4.13.2" description = "ANTLR 4.13.2 runtime for Python 3" -category = "main" optional = false python-versions = "*" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "antlr4_python3_runtime-4.13.2-py3-none-any.whl", hash = "sha256:fe3835eb8d33daece0e799090eda89719dbccee7aa39ef94eed3818cafa5a7e8"}, {file = "antlr4_python3_runtime-4.13.2.tar.gz", hash = "sha256:909b647e1d2fc2b70180ac586df3933e38919c85f98ccc656a96cd3f25ef3916"}, ] +[[package]] +name = "asciimatics" +version = "1.14.0" +description = "A cross-platform package to replace curses (mouse/keyboard input & text colours/positioning) and create ASCII animations" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "asciimatics-1.14.0-py2.py3-none-any.whl", hash = "sha256:277fe925d0d7a029b35245cde01ead009b4a1336130543ace5c8821f38df1da7"}, + {file = "asciimatics-1.14.0.tar.gz", hash = "sha256:16d20ce42210b434eb05ba469ecdb8293ac7ed3c0ce0dd4f70e30d72d7602227"}, +] + +[package.dependencies] +future = "*" +Pillow = ">=2.7.0" +pyfiglet = ">=0.7.2" +pywin32 = {version = "*", markers = "sys_platform == \"win32\""} +wcwidth = "*" + [[package]] name = "asciimatics" version = "1.15.0" description = "A cross-platform package to replace curses (mouse/keyboard input & text colours/positioning) and create ASCII animations" -category = "main" optional = false python-versions = ">= 3.8" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "asciimatics-1.15.0-py3-none-any.whl", hash = "sha256:0fe068a6bed522929bd04bb5b8a2fb6ebf0aef1b7a9b3843cf71030a34bc38d5"}, {file = "asciimatics-1.15.0.tar.gz", hash = "sha256:cfdd398042727519d8b73e62b8ef82c0becfed4eb420899c3b96c98d0b96821a"}, @@ -32,26 +66,36 @@ wcwidth = "*" [[package]] name = "capstone" -version = "4.0.2" +version = "5.0.7" description = "Capstone disassembly engine" -category = "main" optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +python-versions = ">=3.8" +groups = ["main"] files = [ - {file = "capstone-4.0.2-py2.py3-none-manylinux1_i686.whl", hash = "sha256:da442f979414cf27e4621e70e835880878c858ea438c4f0e957e132593579e37"}, - {file = "capstone-4.0.2-py2.py3-none-manylinux1_x86_64.whl", hash = "sha256:9d1a9096c5f875b11290317722ed44bb6e7c52e50cc79d791f142bce968c49aa"}, - {file = "capstone-4.0.2-py2.py3-none-win32.whl", hash = "sha256:c3d9b443d1adb40ee2d9a4e7341169b76476ddcf3a54c03793b16cdc7cd35c5a"}, - {file = "capstone-4.0.2-py2.py3-none-win_amd64.whl", hash = "sha256:0d65ffe8620920976ceadedc769f22318f6f150a592368d8a735612367ac8a1a"}, - {file = "capstone-4.0.2.tar.gz", hash = "sha256:2842913092c9b69fd903744bc1b87488e1451625460baac173056e1808ec1c66"}, + {file = "capstone-5.0.7-py3-none-macosx_10_9_universal2.whl", hash = "sha256:388af4ddb9224d3b4f9269673ee575b3f94f77774d48b3f1a283ad13c29a106a"}, + {file = "capstone-5.0.7-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:a9f64e3d75d8c4d7b3d26bba153b2992aadcf6b8d57674b4ef176b4ecdd9822f"}, + {file = "capstone-5.0.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:acb89f5bf6f625745a104a3a44819d3acea173228055c1eadc60d2282ae490bb"}, + {file = "capstone-5.0.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c58546c814567c95e4b9a63bdb8624c960cb8508855c7c767d5f108d7bc09ce2"}, + {file = "capstone-5.0.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b809a9654844ce0d35099121a851ddd2ab2689df1ff6687037babcedcaae6391"}, + {file = "capstone-5.0.7-py3-none-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80b0f1b93fc703c419fda8cf84cfa017fd8909be62a4e88024273126ab16f006"}, + {file = "capstone-5.0.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:467716e6555d50cb3526b290f0dbdccb5f961839b1f1e299b484fb5d814173e6"}, + {file = "capstone-5.0.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:e551311d4b6dc344fe5518ef6decf4c2dfafe37bba9ad027a53a406930bc5c63"}, + {file = "capstone-5.0.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a13437b28b136c886600e88bee192d25adf56ba1db5597ff5a0bec758bb9c533"}, + {file = "capstone-5.0.7-py3-none-win_amd64.whl", hash = "sha256:4ab8bcb7da8f221ff45926ca168ca33e76f7237d06fbf3c10780002faa2670e1"}, + {file = "capstone-5.0.7.tar.gz", hash = "sha256:796bdd69b05fa124fc2aa2e74b9a0b3d4c4e7f3e02add5e583cf2f3bca282ede"}, ] +[package.dependencies] +importlib-resources = {version = "*", markers = "python_version < \"3.9\""} + [[package]] name = "cffi" version = "1.17.1" description = "Foreign Function Interface for Python calling C code." -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and sys_platform == \"win32\"" files = [ {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, @@ -129,9 +173,9 @@ pycparser = "*" name = "click" version = "8.1.7" description = "Composable command line interface toolkit" -category = "main" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, @@ -144,9 +188,10 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] +markers = "sys_platform == \"win32\" or platform_system == \"Windows\"" files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, @@ -156,9 +201,9 @@ files = [ name = "dacite" version = "1.8.1" description = "Simple creation of data classes from dictionaries." -category = "main" optional = false python-versions = ">=3.6" +groups = ["main"] files = [ {file = "dacite-1.8.1-py3-none-any.whl", hash = "sha256:cc31ad6fdea1f49962ea42db9421772afe01ac5442380d9a99fcf3d188c61afe"}, ] @@ -170,9 +215,9 @@ dev = ["black", "coveralls", "mypy", "pre-commit", "pylint", "pytest (>=5)", "py name = "dill" version = "0.3.9" description = "serialize all of Python" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "dill-0.3.9-py3-none-any.whl", hash = "sha256:468dff3b89520b474c0397703366b7b95eebe6303f108adf9b19da1f702be87a"}, {file = "dill-0.3.9.tar.gz", hash = "sha256:81aa267dddf68cbfe8029c42ca9ec6a4ab3b22371d1c450abc54422577b4512c"}, @@ -186,9 +231,9 @@ profile = ["gprof2dot (>=2022.7.29)"] name = "enum-compat" version = "0.0.3" description = "enum/enum34 compatibility package" -category = "main" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "enum-compat-0.0.3.tar.gz", hash = "sha256:3677daabed56a6f724451d585662253d8fb4e5569845aafa8bb0da36b1a8751e"}, {file = "enum_compat-0.0.3-py3-none-any.whl", hash = "sha256:88091b617c7fc3bbbceae50db5958023c48dc40b50520005aa3bf27f8f7ea157"}, @@ -198,21 +243,34 @@ files = [ name = "first" version = "2.0.2" description = "Return the first true value of an iterable." -category = "main" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "first-2.0.2-py2.py3-none-any.whl", hash = "sha256:8d8e46e115ea8ac652c76123c0865e3ff18372aef6f03c22809ceefcea9dec86"}, {file = "first-2.0.2.tar.gz", hash = "sha256:ff285b08c55f8c97ce4ea7012743af2495c9f1291785f163722bd36f6af6d3bf"}, ] +[[package]] +name = "future" +version = "0.18.3" +description = "Clean single-source support for Python 3 and 2" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"}, +] + [[package]] name = "fuzzercorn" version = "0.0.1" description = "Libfuzzer bindings for Unicorn.." -category = "main" optional = true python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Linux\" and extra == \"fuzz\"" files = [ {file = "fuzzercorn-0.0.1-py3-none-manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:304ddcd19803c779a8e4a6a900c51f7c3f5ba0e8c3322bd9502d59ea0825d424"}, {file = "fuzzercorn-0.0.1-py3-none-manylinux1_i686.whl", hash = "sha256:7fa9cbffbcbf45c0af5707abc86ee8fa0d3a396f0ff48ce67c355ef9d3047c4f"}, @@ -228,9 +286,9 @@ unicorn = ">=2.0.0rc5" name = "gevent" version = "24.2.1" description = "Coroutine-based network library" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "gevent-24.2.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:6f947a9abc1a129858391b3d9334c45041c08a0f23d14333d5b844b6e5c17a07"}, {file = "gevent-24.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bde283313daf0b34a8d1bab30325f5cb0f4e11b5869dbe5bc61f8fe09a8f66f3"}, @@ -285,19 +343,20 @@ greenlet = [ "zope.interface" = "*" [package.extras] -dnspython = ["dnspython (>=1.16.0,<2.0)", "idna"] +dnspython = ["dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\""] docs = ["furo", "repoze.sphinx.autointerface", "sphinx", "sphinxcontrib-programoutput", "zope.schema"] -monitor = ["psutil (>=5.7.0)"] -recommended = ["cffi (>=1.12.2)", "dnspython (>=1.16.0,<2.0)", "idna", "psutil (>=5.7.0)"] -test = ["cffi (>=1.12.2)", "coverage (>=5.0)", "dnspython (>=1.16.0,<2.0)", "idna", "objgraph", "psutil (>=5.7.0)", "requests"] +monitor = ["psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\""] +recommended = ["cffi (>=1.12.2) ; platform_python_implementation == \"CPython\"", "dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\"", "psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\""] +test = ["cffi (>=1.12.2) ; platform_python_implementation == \"CPython\"", "coverage (>=5.0) ; sys_platform != \"win32\"", "dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\"", "objgraph", "psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\"", "requests"] [[package]] name = "greenlet" version = "3.1.1" description = "Lightweight in-process concurrent programming" -category = "main" optional = false python-versions = ">=3.7" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\"" files = [ {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"}, {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"}, @@ -382,9 +441,10 @@ test = ["objgraph", "psutil"] name = "importlib-resources" version = "6.4.5" description = "Read resources from Python packages" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.8\"" files = [ {file = "importlib_resources-6.4.5-py3-none-any.whl", hash = "sha256:ac29d5f956f01d5e4bb63102a5a19957f1b9175e45649977264a1416783bb717"}, {file = "importlib_resources-6.4.5.tar.gz", hash = "sha256:980862a1d16c9e147a59603677fa2aa5fd82b87f223b6cb870695bcfce830065"}, @@ -394,7 +454,7 @@ files = [ zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] @@ -403,11 +463,28 @@ type = ["pytest-mypy"] [[package]] name = "jsonpath-ng" -version = "1.6.1" +version = "1.6.0" description = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." -category = "main" optional = false python-versions = "*" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "jsonpath-ng-1.6.0.tar.gz", hash = "sha256:5483f8e9d74c39c9abfab554c070ae783c1c8cbadf5df60d561bc705ac68a07e"}, + {file = "jsonpath_ng-1.6.0-py3-none-any.whl", hash = "sha256:6fd04833412c4b3d9299edf369542f5e67095ca84efa17cbb7f06a34958adc9f"}, +] + +[package.dependencies] +ply = "*" + +[[package]] +name = "jsonpath-ng" +version = "1.6.1" +description = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." +optional = false +python-versions = "*" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "jsonpath-ng-1.6.1.tar.gz", hash = "sha256:086c37ba4917304850bd837aeab806670224d3f038fe2833ff593a672ef0a5fa"}, {file = "jsonpath_ng-1.6.1-py3-none-any.whl", hash = "sha256:8f22cd8273d7772eea9aaa84d922e0841aa36fdb8a2c6b7f6c3791a16a9bc0be"}, @@ -420,9 +497,9 @@ ply = "*" name = "keystone-engine" version = "0.9.2" description = "Keystone assembler engine" -category = "main" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "keystone-engine-0.9.2.tar.gz", hash = "sha256:2f7af62dab0ce6c2732dbb4f31cfa2184a8a149e280b96b92ebc0db84c6e50f5"}, {file = "keystone_engine-0.9.2-py2.py3-none-macosx_10_14_x86_64.whl", hash = "sha256:dafcc3d9450c239cbc54148855b79c4b387777099c6d054005c835768cf955f2"}, @@ -436,9 +513,9 @@ files = [ name = "loguru" version = "0.7.2" description = "Python logging made (stupidly) simple" -category = "main" optional = false python-versions = ">=3.5" +groups = ["main"] files = [ {file = "loguru-0.7.2-py3-none-any.whl", hash = "sha256:003d71e3d3ed35f0f8984898359d65b79e5b21943f78af86aa5491210429b8eb"}, {file = "loguru-0.7.2.tar.gz", hash = "sha256:e671a53522515f34fd406340ee968cb9ecafbc4b36c679da03c18fd8d0bd51ac"}, @@ -449,15 +526,15 @@ colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] -dev = ["Sphinx (==7.2.5)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.2.2)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.4.1)", "mypy (==v1.5.1)", "pre-commit (==3.4.0)", "pytest (==6.1.2)", "pytest (==7.4.0)", "pytest-cov (==2.12.1)", "pytest-cov (==4.1.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.0.0)", "sphinx-autobuild (==2021.3.14)", "sphinx-rtd-theme (==1.3.0)", "tox (==3.27.1)", "tox (==4.11.0)"] +dev = ["Sphinx (==7.2.5) ; python_version >= \"3.9\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.2.2) ; python_version >= \"3.8\"", "mypy (==0.910) ; python_version < \"3.6\"", "mypy (==0.971) ; python_version == \"3.6\"", "mypy (==1.4.1) ; python_version == \"3.7\"", "mypy (==1.5.1) ; python_version >= \"3.8\"", "pre-commit (==3.4.0) ; python_version >= \"3.8\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==7.4.0) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==4.1.0) ; python_version >= \"3.8\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.0.0) ; python_version >= \"3.8\"", "sphinx-autobuild (==2021.3.14) ; python_version >= \"3.9\"", "sphinx-rtd-theme (==1.3.0) ; python_version >= \"3.9\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.11.0) ; python_version >= \"3.8\""] [[package]] name = "multiprocess" version = "0.70.17" description = "better multiprocessing and multithreading in Python" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "multiprocess-0.70.17-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7ddb24e5bcdb64e90ec5543a1f05a39463068b6d3b804aa3f2a4e16ec28562d6"}, {file = "multiprocess-0.70.17-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:d729f55198a3579f6879766a6d9b72b42d4b320c0dcb7844afb774d75b573c62"}, @@ -482,11 +559,25 @@ dill = ">=0.3.9" [[package]] name = "overrides" -version = "7.7.0" +version = "7.4.0" description = "A decorator to automatically detect mismatch when overriding a method." -category = "main" optional = false python-versions = ">=3.6" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "overrides-7.4.0-py3-none-any.whl", hash = "sha256:3ad24583f86d6d7a49049695efe9933e67ba62f0c7625d53c59fa832ce4b8b7d"}, + {file = "overrides-7.4.0.tar.gz", hash = "sha256:9502a3cca51f4fac40b5feca985b6703a5c1f6ad815588a7ca9e285b9dca6757"}, +] + +[[package]] +name = "overrides" +version = "7.7.0" +description = "A decorator to automatically detect mismatch when overriding a method." +optional = false +python-versions = ">=3.6" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49"}, {file = "overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a"}, @@ -496,9 +587,9 @@ files = [ name = "pefile" version = "2024.8.26" description = "Python PE parsing module" -category = "main" optional = false python-versions = ">=3.6.0" +groups = ["main"] files = [ {file = "pefile-2024.8.26-py3-none-any.whl", hash = "sha256:76f8b485dcd3b1bb8166f1128d395fa3d87af26360c2358fb75b80019b957c6f"}, {file = "pefile-2024.8.26.tar.gz", hash = "sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632"}, @@ -506,11 +597,81 @@ files = [ [[package]] name = "pillow" -version = "10.4.0" +version = "10.0.1" description = "Python Imaging Library (Fork)" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "Pillow-10.0.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:8f06be50669087250f319b706decf69ca71fdecd829091a37cc89398ca4dc17a"}, + {file = "Pillow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50bd5f1ebafe9362ad622072a1d2f5850ecfa44303531ff14353a4059113b12d"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6a90167bcca1216606223a05e2cf991bb25b14695c518bc65639463d7db722d"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f11c9102c56ffb9ca87134bd025a43d2aba3f1155f508eff88f694b33a9c6d19"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:186f7e04248103482ea6354af6d5bcedb62941ee08f7f788a1c7707bc720c66f"}, + {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0462b1496505a3462d0f35dc1c4d7b54069747d65d00ef48e736acda2c8cbdff"}, + {file = "Pillow-10.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d889b53ae2f030f756e61a7bff13684dcd77e9af8b10c6048fb2c559d6ed6eaf"}, + {file = "Pillow-10.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:552912dbca585b74d75279a7570dd29fa43b6d93594abb494ebb31ac19ace6bd"}, + {file = "Pillow-10.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:787bb0169d2385a798888e1122c980c6eff26bf941a8ea79747d35d8f9210ca0"}, + {file = "Pillow-10.0.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:fd2a5403a75b54661182b75ec6132437a181209b901446ee5724b589af8edef1"}, + {file = "Pillow-10.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2d7e91b4379f7a76b31c2dda84ab9e20c6220488e50f7822e59dac36b0cd92b1"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19e9adb3f22d4c416e7cd79b01375b17159d6990003633ff1d8377e21b7f1b21"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93139acd8109edcdeffd85e3af8ae7d88b258b3a1e13a038f542b79b6d255c54"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:92a23b0431941a33242b1f0ce6c88a952e09feeea9af4e8be48236a68ffe2205"}, + {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cbe68deb8580462ca0d9eb56a81912f59eb4542e1ef8f987405e35a0179f4ea2"}, + {file = "Pillow-10.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:522ff4ac3aaf839242c6f4e5b406634bfea002469656ae8358644fc6c4856a3b"}, + {file = "Pillow-10.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:84efb46e8d881bb06b35d1d541aa87f574b58e87f781cbba8d200daa835b42e1"}, + {file = "Pillow-10.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:898f1d306298ff40dc1b9ca24824f0488f6f039bc0e25cfb549d3195ffa17088"}, + {file = "Pillow-10.0.1-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:bcf1207e2f2385a576832af02702de104be71301c2696d0012b1b93fe34aaa5b"}, + {file = "Pillow-10.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5d6c9049c6274c1bb565021367431ad04481ebb54872edecfcd6088d27edd6ed"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28444cb6ad49726127d6b340217f0627abc8732f1194fd5352dec5e6a0105635"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de596695a75496deb3b499c8c4f8e60376e0516e1a774e7bc046f0f48cd620ad"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2872f2d7846cf39b3dbff64bc1104cc48c76145854256451d33c5faa55c04d1a"}, + {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4ce90f8a24e1c15465048959f1e94309dfef93af272633e8f37361b824532e91"}, + {file = "Pillow-10.0.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ee7810cf7c83fa227ba9125de6084e5e8b08c59038a7b2c9045ef4dde61663b4"}, + {file = "Pillow-10.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b1be1c872b9b5fcc229adeadbeb51422a9633abd847c0ff87dc4ef9bb184ae08"}, + {file = "Pillow-10.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:98533fd7fa764e5f85eebe56c8e4094db912ccbe6fbf3a58778d543cadd0db08"}, + {file = "Pillow-10.0.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:764d2c0daf9c4d40ad12fbc0abd5da3af7f8aa11daf87e4fa1b834000f4b6b0a"}, + {file = "Pillow-10.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fcb59711009b0168d6ee0bd8fb5eb259c4ab1717b2f538bbf36bacf207ef7a68"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:697a06bdcedd473b35e50a7e7506b1d8ceb832dc238a336bd6f4f5aa91a4b500"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f665d1e6474af9f9da5e86c2a3a2d2d6204e04d5af9c06b9d42afa6ebde3f21"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:2fa6dd2661838c66f1a5473f3b49ab610c98a128fc08afbe81b91a1f0bf8c51d"}, + {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:3a04359f308ebee571a3127fdb1bd01f88ba6f6fb6d087f8dd2e0d9bff43f2a7"}, + {file = "Pillow-10.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:723bd25051454cea9990203405fa6b74e043ea76d4968166dfd2569b0210886a"}, + {file = "Pillow-10.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:71671503e3015da1b50bd18951e2f9daf5b6ffe36d16f1eb2c45711a301521a7"}, + {file = "Pillow-10.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:44e7e4587392953e5e251190a964675f61e4dae88d1e6edbe9f36d6243547ff3"}, + {file = "Pillow-10.0.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:3855447d98cced8670aaa63683808df905e956f00348732448b5a6df67ee5849"}, + {file = "Pillow-10.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ed2d9c0704f2dc4fa980b99d565c0c9a543fe5101c25b3d60488b8ba80f0cce1"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5bb289bb835f9fe1a1e9300d011eef4d69661bb9b34d5e196e5e82c4cb09b37"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0d3e54ab1df9df51b914b2233cf779a5a10dfd1ce339d0421748232cea9876"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:2cc6b86ece42a11f16f55fe8903595eff2b25e0358dec635d0a701ac9586588f"}, + {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:ca26ba5767888c84bf5a0c1a32f069e8204ce8c21d00a49c90dabeba00ce0145"}, + {file = "Pillow-10.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f0b4b06da13275bc02adfeb82643c4a6385bd08d26f03068c2796f60d125f6f2"}, + {file = "Pillow-10.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bc2e3069569ea9dbe88d6b8ea38f439a6aad8f6e7a6283a38edf61ddefb3a9bf"}, + {file = "Pillow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8b451d6ead6e3500b6ce5c7916a43d8d8d25ad74b9102a629baccc0808c54971"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:32bec7423cdf25c9038fef614a853c9d25c07590e1a870ed471f47fb80b244db"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7cf63d2c6928b51d35dfdbda6f2c1fddbe51a6bc4a9d4ee6ea0e11670dd981e"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f6d3d4c905e26354e8f9d82548475c46d8e0889538cb0657aa9c6f0872a37aa4"}, + {file = "Pillow-10.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:847e8d1017c741c735d3cd1883fa7b03ded4f825a6e5fcb9378fd813edee995f"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:7f771e7219ff04b79e231d099c0a28ed83aa82af91fd5fa9fdb28f5b8d5addaf"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:459307cacdd4138edee3875bbe22a2492519e060660eaf378ba3b405d1c66317"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b059ac2c4c7a97daafa7dc850b43b2d3667def858a4f112d1aa082e5c3d6cf7d"}, + {file = "Pillow-10.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6caf3cd38449ec3cd8a68b375e0c6fe4b6fd04edb6c9766b55ef84a6e8ddf2d"}, + {file = "Pillow-10.0.1.tar.gz", hash = "sha256:d72967b06be9300fed5cfbc8b5bafceec48bf7cdc7dab66b1d2549035287191d"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] +tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] + +[[package]] +name = "pillow" +version = "10.4.0" +description = "Python Imaging Library (Fork)" +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "pillow-10.4.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:4d9667937cfa347525b319ae34375c37b9ee6b525440f3ef48542fcf66f2731e"}, {file = "pillow-10.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:543f3dc61c18dafb755773efc89aae60d06b6596a63914107f75459cf984164d"}, @@ -599,16 +760,16 @@ docs = ["furo", "olefile", "sphinx (>=7.3)", "sphinx-copybutton", "sphinx-inline fpx = ["olefile"] mic = ["olefile"] tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] -typing = ["typing-extensions"] +typing = ["typing-extensions ; python_version < \"3.10\""] xmp = ["defusedxml"] [[package]] name = "ply" version = "3.11" description = "Python Lex & Yacc" -category = "main" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "ply-3.11-py2.py3-none-any.whl", hash = "sha256:096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce"}, {file = "ply-3.11.tar.gz", hash = "sha256:00c7c1aaa88358b9c765b6d3000c6eec0ba42abca5351b095321aef446081da3"}, @@ -618,9 +779,9 @@ files = [ name = "prompt-toolkit" version = "3.0.50" description = "Library for building powerful interactive command lines in Python" -category = "main" optional = false python-versions = ">=3.8.0" +groups = ["main"] files = [ {file = "prompt_toolkit-3.0.50-py3-none-any.whl", hash = "sha256:9b6427eb19e479d98acff65196a307c555eb567989e6d88ebbb1b509d9779198"}, {file = "prompt_toolkit-3.0.50.tar.gz", hash = "sha256:544748f3860a2623ca5cd6d2795e7a14f3d0e1c3c9728359013f79877fc89bab"}, @@ -633,9 +794,10 @@ wcwidth = "*" name = "pycparser" version = "2.22" description = "C parser in Python" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and sys_platform == \"win32\"" files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, @@ -645,21 +807,35 @@ files = [ name = "pyelftools" version = "0.32" description = "Library for analyzing ELF files and DWARF debugging information" -category = "main" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "pyelftools-0.32-py3-none-any.whl", hash = "sha256:013df952a006db5e138b1edf6d8a68ecc50630adbd0d83a2d41e7f846163d738"}, {file = "pyelftools-0.32.tar.gz", hash = "sha256:6de90ee7b8263e740c8715a925382d4099b354f29ac48ea40d840cf7aa14ace5"}, ] +[[package]] +name = "pyfiglet" +version = "0.8.post1" +description = "Pure-python FIGlet implementation" +optional = false +python-versions = "*" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "pyfiglet-0.8.post1-py2.py3-none-any.whl", hash = "sha256:d555bcea17fbeaf70eaefa48bb119352487e629c9b56f30f383e2c62dd67a01c"}, + {file = "pyfiglet-0.8.post1.tar.gz", hash = "sha256:c6c2321755d09267b438ec7b936825a4910fec696292139e664ca8670e103639"}, +] + [[package]] name = "pyfiglet" version = "1.0.2" description = "Pure-python FIGlet implementation" -category = "main" optional = false python-versions = ">=3.9" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "pyfiglet-1.0.2-py3-none-any.whl", hash = "sha256:889b351d79c99e50a3f619c8f8e6ffdb27fd8c939fc43ecbd7559bd57d5f93ea"}, {file = "pyfiglet-1.0.2.tar.gz", hash = "sha256:758788018ab8faaddc0984e1ea05ff330d3c64be663c513cc1f105f6a3066dab"}, @@ -667,22 +843,68 @@ files = [ [[package]] name = "pyperclip" -version = "1.9.0" +version = "1.8.2" description = "A cross-platform clipboard module for Python. (Only handles plain text for now.)" -category = "main" optional = false python-versions = "*" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "pyperclip-1.8.2.tar.gz", hash = "sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57"}, +] + +[[package]] +name = "pyperclip" +version = "1.9.0" +description = "A cross-platform clipboard module for Python. (Only handles plain text for now.)" +optional = false +python-versions = "*" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "pyperclip-1.9.0.tar.gz", hash = "sha256:b7de0142ddc81bfc5c7507eea19da920b92252b548b96186caf94a5e2527d310"}, ] [[package]] name = "python-fx" -version = "0.3.2" +version = "0.3.1" description = "A python-native fx-alike terminal JSON viewer." -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "python-fx-0.3.1.tar.gz", hash = "sha256:76044ba32195b8e0ce444aa714981cb1481f9df44c3381c5ed4b43a4f6812c73"}, + {file = "python_fx-0.3.1-py3-none-any.whl", hash = "sha256:e7cfbb8421831aaff5684dd1ae6ec855b92bcd089f9f76b06a3f2baa4670447a"}, +] + +[package.dependencies] +antlr4-python3-runtime = "4.8" +asciimatics = "1.14.0" +click = {version = "8.1.7", markers = "python_version >= \"3.7\""} +dacite = {version = "1.8.1", markers = "python_version >= \"3.6\""} +first = "2.0.2" +future = {version = "0.18.3", markers = "python_version >= \"2.6\" and python_version not in \"3.0, 3.1, 3.2, 3.3\""} +jsonpath-ng = "1.6.0" +loguru = {version = "0.7.2", markers = "python_version >= \"3.5\""} +overrides = {version = "7.4.0", markers = "python_version >= \"3.6\""} +pillow = {version = "10.0.1", markers = "python_version >= \"3.8\""} +ply = "3.11" +pyfiglet = {version = "0.8.post1", markers = "python_version >= \"3.9\""} +pyperclip = "1.8.2" +pyyaml = {version = "6.0.1", markers = "python_version >= \"3.6\""} +urwid = {version = "2.2.1", markers = "python_full_version >= \"3.7.0\""} +wcwidth = "0.2.6" +yamale = {version = "4.0.4", markers = "python_version >= \"3.6\""} + +[[package]] +name = "python-fx" +version = "0.3.2" +description = "A python-native fx-alike terminal JSON viewer." +optional = false +python-versions = ">=3.8" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "python_fx-0.3.2-py3-none-any.whl", hash = "sha256:5498475b0f391b1649732328b58d188d9fc4b3f90f5bfb77d5c6e2ece2432c5f"}, {file = "python_fx-0.3.2.tar.gz", hash = "sha256:9646f58c716e2db6698bff3dfa55fa721b8b0cb741506287a87bc08055a96ceb"}, @@ -711,9 +933,9 @@ yamale = {version = "5.2.1", markers = "python_version >= \"3.8\""} name = "python-registry" version = "1.3.1" description = "Read access to Windows Registry files." -category = "main" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "python-registry-1.3.1.tar.gz", hash = "sha256:99185f67d5601be3e7843e55902d5769aea1740869b0882f34ff1bd4b43b1eb2"}, {file = "python_registry-1.3.1-py2-none-any.whl", hash = "sha256:59d3b00c04bca0c4e1a12be0404da6ccf76b87537ee3a3ad2d8fc1bccf6f63ca"}, @@ -728,9 +950,10 @@ unicodecsv = "*" name = "pywin32" version = "308" description = "Python for Window Extensions" -category = "main" optional = false python-versions = "*" +groups = ["main"] +markers = "sys_platform == \"win32\"" files = [ {file = "pywin32-308-cp310-cp310-win32.whl", hash = "sha256:796ff4426437896550d2981b9c2ac0ffd75238ad9ea2d3bfa67a1abd546d262e"}, {file = "pywin32-308-cp310-cp310-win_amd64.whl", hash = "sha256:4fc888c59b3c0bef905ce7eb7e2106a07712015ea1c8234b703a088d46110e8e"}, @@ -752,13 +975,76 @@ files = [ {file = "pywin32-308-cp39-cp39-win_amd64.whl", hash = "sha256:71b3322d949b4cc20776436a9c9ba0eeedcbc9c650daa536df63f0ff111bb920"}, ] +[[package]] +name = "pyyaml" +version = "6.0.1" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.6" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, + {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, + {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, + {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, + {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, + {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, + {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, + {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, + {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, + {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, + {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, + {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, + {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, + {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, + {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, + {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, + {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, + {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, + {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, + {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, + {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, + {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, + {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, + {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, +] + [[package]] name = "pyyaml" version = "6.0.2" description = "YAML parser and emitter for Python" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -819,9 +1105,9 @@ files = [ name = "questionary" version = "2.1.0" description = "Python library to build pretty command line user prompts ⭐️" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "questionary-2.1.0-py3-none-any.whl", hash = "sha256:44174d237b68bc828e4878c763a9ad6790ee61990e0ae72927694ead57bab8ec"}, {file = "questionary-2.1.0.tar.gz", hash = "sha256:6302cdd645b19667d8f6e6634774e9538bfcd1aad9be287e743d96cacaf95587"}, @@ -834,9 +1120,10 @@ prompt_toolkit = ">=2.0,<4.0" name = "r2libr" version = "5.7.4" description = "Yet anohter radare2 python bindings." -category = "main" optional = true python-versions = ">=3.6" +groups = ["main"] +markers = "extra == \"re\"" files = [ {file = "r2libr-5.7.4-py3-none-macosx_10_15_universal2.whl", hash = "sha256:c752e57085fed9d34527d2ff6692068c2a162c3477c0539e9e61cd7eb5acdcfc"}, {file = "r2libr-5.7.4-py3-none-manylinux1_x86_64.whl", hash = "sha256:80df7902492de77e2cb770ecb7ffbe215908192962d95f0e43b42d3c3d464096"}, @@ -847,30 +1134,30 @@ files = [ name = "setuptools" version = "75.3.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd"}, {file = "setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.5.2)"] -core = ["importlib-metadata (>=6)", "importlib-resources (>=5.10.2)", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.5.2) ; sys_platform != \"cygwin\""] +core = ["importlib-metadata (>=6) ; python_version < \"3.10\"", "importlib-resources (>=5.10.2) ; python_version < \"3.9\"", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (>=1.12.0,<1.13.0)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib-metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.12.*)", "pytest-mypy"] [[package]] name = "termcolor" version = "2.4.0" description = "ANSI color formatting for output in terminal" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "termcolor-2.4.0-py3-none-any.whl", hash = "sha256:9297c0df9c99445c2412e832e882a7884038a25617c60cea2ad69488d4040d63"}, {file = "termcolor-2.4.0.tar.gz", hash = "sha256:aab9e56047c8ac41ed798fa36d892a37aca6b3e9159f3e0c24bc64a9b3ac7b7a"}, @@ -883,9 +1170,10 @@ tests = ["pytest", "pytest-cov"] name = "typing-extensions" version = "4.12.2" description = "Backported and Experimental Type Hints for Python 3.8+" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -895,9 +1183,9 @@ files = [ name = "unicodecsv" version = "0.14.1" description = "Python2's stdlib csv module is nice, but it doesn't support unicode. This module is a drop-in replacement which *does*." -category = "main" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "unicodecsv-0.14.1.tar.gz", hash = "sha256:018c08037d48649a0412063ff4eda26eaa81eff1546dbffa51fa5293276ff7fc"}, ] @@ -906,9 +1194,9 @@ files = [ name = "unicorn" version = "2.1.3" description = "Unicorn CPU emulator engine" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] files = [ {file = "unicorn-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6cbf99c139a238ee6ccfaadea35e65a88461c0ae0dcf78058c8266ff90f8866c"}, {file = "unicorn-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8e7b9396a7b76503b1d32c4b83d35e03e8b2ee81e80a2c7aee77dac7b71f25c"}, @@ -989,15 +1277,16 @@ files = [ importlib_resources = {version = "*", markers = "python_version < \"3.9\""} [package.extras] -test = ["capstone (==5.0.1)", "capstone (==6.0.0a2)"] +test = ["capstone (==5.0.1) ; python_version <= \"3.7\"", "capstone (==6.0.0a2) ; python_version > \"3.7\""] [[package]] name = "unicornafl" version = "2.1.0" description = "Unicornafl" -category = "main" optional = true python-versions = "*" +groups = ["main"] +markers = "platform_system != \"Windows\" and extra == \"fuzz\"" files = [ {file = "unicornafl-2.1.0-py3-none-macosx_11_7_x86_64.whl", hash = "sha256:8827c010376274730776b85f884cf9037d08ca6cd866b364bcfea8eee6db6090"}, {file = "unicornafl-2.1.0-py3-none-manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa3130b3c811291719874cfd7e728ce0c8147cc63c8853069a5195e528bafb43"}, @@ -1010,13 +1299,60 @@ files = [ [package.dependencies] unicorn = ">=2.0.1" +[[package]] +name = "urwid" +version = "2.2.1" +description = "A full-featured console (xterm et al.) user interface library" +optional = false +python-versions = ">=3.7.0" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "urwid-2.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7aa95e2f8941e323f0534a301f9d8d965d869110d326b3c9dff63e1c116772cd"}, + {file = "urwid-2.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ea22e5eabace2c66e2f52cc2494308c1a0091bcb89b3ceedf72ff91733f4dbb2"}, + {file = "urwid-2.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:07b88bdc8efe95b318201bd054ae69bed68bb9f506f127b21ae7234ffb7db3a4"}, + {file = "urwid-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bba58f24fdff58975ef603ee803909d57faaaebd407fd50042652dcc9a8dd2f2"}, + {file = "urwid-2.2.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab2d5704cbd32f729a60d2b56d076e16652b3b97ebe6773c54a192cb9f49c169"}, + {file = "urwid-2.2.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bc88edae9cb34644905e3d00e9fd2dc9a2c1eaeb2e311c1aec0d36a51d77b10"}, + {file = "urwid-2.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2fa3e83730a811466d272ab68340f9f9418ee7ca5f6de3548dd7a5661eafbbee"}, + {file = "urwid-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2f87d7efbbf1d716bbcf025d453b3481aa0d9e1c91581aa8edc9ae7af64efa85"}, + {file = "urwid-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bdafd72d6a539e16e6c179dd16609601643b85edf97b1543fc208e4fb7e6c249"}, + {file = "urwid-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e17eed4190220873531e2c11c885764d2e3bcabe9e35d5a578e84056a2c58199"}, + {file = "urwid-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7949a8d4384d170e4dfa41151e6264f6238b3ba2520649c25110bc6451978568"}, + {file = "urwid-2.2.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c96c57714fd0eac79a5e8a9d38d15d68ab9b6a96c2fe282ccd61cb707dd4be2"}, + {file = "urwid-2.2.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1bee2f966063de86a093908abb5bd56910e5d5630e021b351f240ab3b972207e"}, + {file = "urwid-2.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:57e28adf4457fb50b751838836bb94122e904ccad4429d42c4f318a3287a4802"}, + {file = "urwid-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ad5ff26140b66ebb69957c51fd168a86f212adb578c83ed590ffd7e032da973f"}, + {file = "urwid-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:653b1fb9c52c4a32c326f701dd3ceb6edd1f30f32033c040fd5edc55d3d60cdb"}, + {file = "urwid-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4438be6b4b303d1012b8208cf5ff2ef71bdc19a7732771257ece36e2d1d16283"}, + {file = "urwid-2.2.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7868c2cfd3fadd6cf42a4a2dbb2cf87a92d6c12dc5ed8b991ff96e66f0ba8c38"}, + {file = "urwid-2.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c3f00ca72de0736f3df4971a01d2278065628bef179fbd4fce37aacf93bbeb2c"}, + {file = "urwid-2.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5354319e3ac4e612a4a280421e21fa4243023df73e48ee701e4e944e769d87c"}, + {file = "urwid-2.2.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f3fdbd58a3ef1f19393a5f1d8b61adcfc89d0e235a2e05927cbecbf8012120c"}, + {file = "urwid-2.2.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55d13d1c6f15bd594a49ad165837edf34678c7c2362834f0d771990821e3bb8c"}, + {file = "urwid-2.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:93f081f1c53d7d307694ae20eb07ac731b4337d517dd6ee9dd91bae78bcb67bf"}, + {file = "urwid-2.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d42a9e4939d18c77f73183593a589c9b3b5d5fa3615d94a32e15cd97b00d3536"}, + {file = "urwid-2.2.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5eeb4e1760e9356471f8b50e3296c24881e242aae57f738a6f8534438848fb2e"}, + {file = "urwid-2.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d894543b4f3f3f2ce9837782e45cc3797df0a1697264a2939a2391076d07f641"}, + {file = "urwid-2.2.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3205f486e9fa4c6193aa5b9623abe05db864465acd02825305702849572c0828"}, + {file = "urwid-2.2.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccfbbc37565f24156e2bdb37504b010fedee8f4a70cfc353c0d9782354087484"}, + {file = "urwid-2.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4add23f02ef43497c13592e1804640b8b19fe781e8d62f445000c7acca60e2e2"}, + {file = "urwid-2.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:476f03705095fed744413d8256c7da998694b2f81e7e6e665e6244d1d3159d1e"}, + {file = "urwid-2.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:85614e0436eb8c77bb21950f7d52bf93668b4ba8a11a2986bc111b48d28390f6"}, + {file = "urwid-2.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf09c272b46ec0f78b6eaf2a515ded30e952a0e77dbbb3535593d3f05354eb82"}, + {file = "urwid-2.2.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8de3025ee488d9f56db9e8842d6c7f1c290e01fd6749320d8c171fef5cbef35"}, + {file = "urwid-2.2.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2eea5fba3dab9f0977fcf17608370686da9d18a6077fd180a98eba72c59ff5d"}, + {file = "urwid-2.2.1.tar.gz", hash = "sha256:e33911ab18f2c73fddbe9bf216d021e74e20b2d5aa9be30403c58f55131bb8a1"}, +] + [[package]] name = "urwid" version = "2.6.15" description = "A full-featured console (xterm et al.) user interface library" -category = "main" optional = false python-versions = ">3.7" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "urwid-2.6.15-py3-none-any.whl", hash = "sha256:71b3171cabaa0092902f556768756bd2f2ebb24c0da287ee08f081d235340cb7"}, {file = "urwid-2.6.15.tar.gz", hash = "sha256:9ecc57330d88c8d9663ffd7092a681674c03ff794b6330ccfef479af7aa9671b"}, @@ -1027,7 +1363,7 @@ typing-extensions = "*" wcwidth = "*" [package.extras] -curses = ["windows-curses"] +curses = ["windows-curses ; sys_platform == \"win32\""] glib = ["PyGObject"] lcd = ["pyserial"] serial = ["pyserial"] @@ -1038,11 +1374,25 @@ zmq = ["zmq"] [[package]] name = "wcwidth" -version = "0.2.13" +version = "0.2.6" description = "Measures the displayed width of unicode strings in a terminal" -category = "main" optional = false python-versions = "*" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "wcwidth-0.2.6-py2.py3-none-any.whl", hash = "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e"}, + {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"}, +] + +[[package]] +name = "wcwidth" +version = "0.2.13" +description = "Measures the displayed width of unicode strings in a terminal" +optional = false +python-versions = "*" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, @@ -1052,24 +1402,26 @@ files = [ name = "win32-setctime" version = "1.2.0" description = "A small Python utility to set file creation time on Windows" -category = "main" optional = false python-versions = ">=3.5" +groups = ["main"] +markers = "sys_platform == \"win32\"" files = [ {file = "win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390"}, {file = "win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0"}, ] [package.extras] -dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] +dev = ["black (>=19.3b0) ; python_version >= \"3.6\"", "pytest (>=4.6.2)"] [[package]] name = "windows-curses" version = "2.4.1" description = "Support for the standard curses module on Windows" -category = "main" optional = false python-versions = "*" +groups = ["main"] +markers = "platform_system == \"Windows\"" files = [ {file = "windows_curses-2.4.1-cp310-cp310-win32.whl", hash = "sha256:53d711e07194d0d3ff7ceff29e0955b35479bc01465d46c3041de67b8141db2f"}, {file = "windows_curses-2.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:325439cd4f37897a1de8a9c068a5b4c432f9244bf9c855ee2fbeb3fa721a770c"}, @@ -1089,13 +1441,30 @@ files = [ {file = "windows_curses-2.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:4588213f7ef3b0c24c5cb9e309653d7a84c1792c707561e8b471d466ca79f2b8"}, ] +[[package]] +name = "yamale" +version = "4.0.4" +description = "A schema and validator for YAML." +optional = false +python-versions = ">=3.6" +groups = ["main"] +markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" +files = [ + {file = "yamale-4.0.4-py3-none-any.whl", hash = "sha256:04f914c0886bda03ac20f8468272cfd9374a634a062549490eff2beedeb30497"}, + {file = "yamale-4.0.4.tar.gz", hash = "sha256:e524caf71cbbbd15aa295e8bdda01688ac4b5edaf38dd60851ddff6baef383ba"}, +] + +[package.dependencies] +pyyaml = "*" + [[package]] name = "yamale" version = "5.2.1" description = "A schema and validator for YAML." -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "yamale-5.2.1-py3-none-any.whl", hash = "sha256:e44cd30cf3055ee4b34c1c71d6fe35490a127dcbd36f82f27859d105a9989922"}, {file = "yamale-5.2.1.tar.gz", hash = "sha256:19bbe713d588f07177bc519a46070c0793ed126ea37f425a76055b99703f835a"}, @@ -1108,29 +1477,30 @@ pyyaml = "*" name = "zipp" version = "3.20.2" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] +markers = "python_version == \"3.8\"" files = [ {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] -test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] [[package]] name = "zope-event" version = "5.0" description = "Very basic event publishing system" -category = "main" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "zope.event-5.0-py3-none-any.whl", hash = "sha256:2832e95014f4db26c47a13fdaef84cef2f4df37e66b59d8f1f4a8f319a632c26"}, {file = "zope.event-5.0.tar.gz", hash = "sha256:bac440d8d9891b4068e2b5a2c5e2c9765a9df762944bda6955f96bb9b91e67cd"}, @@ -1147,9 +1517,9 @@ test = ["zope.testrunner"] name = "zope-interface" version = "7.2" description = "Interfaces for Python" -category = "main" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "zope.interface-7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ce290e62229964715f1011c3dbeab7a4a1e4971fd6f31324c4519464473ef9f2"}, {file = "zope.interface-7.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:05b910a5afe03256b58ab2ba6288960a2892dfeef01336dc4be6f1b9ed02ab0a"}, @@ -1199,10 +1569,10 @@ test = ["coverage[toml]", "zope.event", "zope.testing"] testing = ["coverage[toml]", "zope.event", "zope.testing"] [extras] -fuzz = ["unicornafl", "fuzzercorn"] +fuzz = ["fuzzercorn", "unicornafl"] re = ["r2libr"] [metadata] -lock-version = "2.0" +lock-version = "2.1" python-versions = "^3.8" -content-hash = "f0a6ca5220bfd011fc4c9e54e151f7752bcbd5b4cf3d80698cfc4f3a53665aa4" +content-hash = "9191e91c28fe22a05ac8185ae5fd4a1fc969cecad63b41d649981dfbc50e86ae" From cb875f84889ad0883796e634d2cf4c2831d5ad3a Mon Sep 17 00:00:00 2001 From: xwings Date: Sun, 12 Apr 2026 16:48:53 +0800 Subject: [PATCH 125/138] more qiling style. update logging and exception --- qiling/exception.py | 3 ++ qiling/os/posix/kernel_proxy/__init__.py | 36 +++++++++++------------- qiling/os/posix/kernel_proxy/ipc.py | 6 ++-- qiling/os/posix/kernel_proxy/proxy.py | 7 +++-- tests/test_kernel_proxy.py | 5 ++-- 5 files changed, 31 insertions(+), 26 deletions(-) diff --git a/qiling/exception.py b/qiling/exception.py index e6d46060..a544b254 100644 --- a/qiling/exception.py +++ b/qiling/exception.py @@ -74,6 +74,9 @@ class QlMemoryMappedError(QlErrorBase): class QlGDTError(QlErrorBase): pass +class QlProxyConnectionError(QlErrorBase): + pass + class QlSyscallError(QlErrorBase): def __init__(self, errno, msg): super(QlSyscallError, self).__init__(msg) diff --git a/qiling/os/posix/kernel_proxy/__init__.py b/qiling/os/posix/kernel_proxy/__init__.py index b0fae205..f7ac75ea 100644 --- a/qiling/os/posix/kernel_proxy/__init__.py +++ b/qiling/os/posix/kernel_proxy/__init__.py @@ -24,18 +24,16 @@ import os import sys import socket import subprocess -import logging -from typing import Optional, TYPE_CHECKING +from typing import Dict, Optional, TYPE_CHECKING from qiling.const import QL_INTERCEPT, QL_OS +from qiling.exception import QlErrorArch, QlErrorSyscallError, QlErrorSyscallNotFound from qiling.os.posix.kernel_proxy.ipc import ProxyClient from qiling.os.posix.kernel_proxy.proxy_fd import ql_proxy_fd if TYPE_CHECKING: from qiling import Qiling -log = logging.getLogger(__name__) - class KernelProxy: """Forward specific syscalls to a real Linux kernel via a helper process. @@ -46,13 +44,13 @@ class KernelProxy: def __init__(self, ql: Qiling): if sys.platform != 'linux': - raise RuntimeError("KernelProxy requires a Linux host") + raise QlErrorArch("KernelProxy requires a Linux host") self.ql = ql self._process: Optional[subprocess.Popen] = None self._client: Optional[ProxyClient] = None - self._forwarded: dict = {} # name -> syscall_nr - self._reverse_table: Optional[dict] = None # name -> nr (built on first use) + self._forwarded: Dict[str, int] = {} # name -> syscall_nr + self._reverse_table: Optional[Dict[str, int]] = None # name -> nr (built on first use) self._start_proxy() @@ -78,14 +76,13 @@ class KernelProxy: child_sock.close() self._client = ProxyClient(parent_sock) - log.info(f"kernel proxy started (pid={self._process.pid})") + self.ql.log.info(f"kernel proxy started (pid={self._process.pid})") - def _build_reverse_table(self) -> dict: + def _build_reverse_table(self) -> Dict[str, int]: """Build name -> syscall_nr mapping from the guest architecture's syscall table.""" if self._reverse_table is not None: return self._reverse_table - from qiling.os.linux.map_syscall import get_syscall_mapper from qiling.const import QL_ARCH # get the raw syscall table dict for this architecture @@ -102,7 +99,7 @@ class KernelProxy: table_name = arch_tables.get(self.ql.arch.type) if table_name is None: - raise RuntimeError(f"KernelProxy: unsupported architecture {self.ql.arch.type}") + raise QlErrorArch(f"KernelProxy: unsupported architecture {self.ql.arch.type}") import qiling.os.linux.map_syscall as mod table = getattr(mod, table_name) @@ -115,7 +112,7 @@ class KernelProxy: """Resolve a syscall name to its number for the guest architecture.""" table = self._build_reverse_table() if name not in table: - raise ValueError( + raise QlErrorSyscallNotFound( f"KernelProxy: syscall '{name}' not found in {self.ql.arch.type.name} syscall table" ) return table[name] @@ -135,8 +132,8 @@ class KernelProxy: forwarder = self._make_forwarder(name, nr, returns_fd) self.ql.os.set_syscall(name, forwarder, QL_INTERCEPT.CALL) - log.info(f"forwarding syscall '{name}' (nr={nr}) to kernel proxy" - f"{' [returns FD]' if returns_fd else ''}") + self.ql.log.info(f"forwarding syscall '{name}' (nr={nr}) to kernel proxy" + f"{' [returns FD]' if returns_fd else ''}") def _make_forwarder(self, name: str, guest_nr: int, returns_fd: bool): """Create a CALL hook closure for one syscall.""" @@ -170,17 +167,16 @@ class KernelProxy: self._host_table = self._load_host_syscall_table() if name not in self._host_table: - raise RuntimeError(f"KernelProxy: syscall '{name}' not available on host") + raise QlErrorSyscallNotFound(f"KernelProxy: syscall '{name}' not available on host") return self._host_table[name] - def _load_host_syscall_table(self) -> dict: + def _load_host_syscall_table(self) -> Dict[str, int]: """Load the host's syscall name->nr mapping. Uses the same Qiling tables, indexed by the host architecture. """ import platform - from qiling.const import QL_ARCH import qiling.os.linux.map_syscall as mod machine = platform.machine() @@ -195,7 +191,7 @@ class KernelProxy: table_name = host_arch_map.get(machine) if table_name is None: - raise RuntimeError(f"KernelProxy: unsupported host architecture '{machine}'") + raise QlErrorArch(f"KernelProxy: unsupported host architecture '{machine}'") table = getattr(mod, table_name) return {name: nr for nr, name in table.items()} @@ -208,7 +204,7 @@ class KernelProxy: ql.os.fd[i] = fd_obj return i - raise OSError("kernel_proxy: FD table full") + raise QlErrorSyscallError("kernel_proxy: FD table full") def stop(self): """Stop the proxy process.""" @@ -226,7 +222,7 @@ class KernelProxy: except subprocess.TimeoutExpired: self._process.kill() self._process.wait() - log.info(f"kernel proxy stopped (pid={self._process.pid})") + self.ql.log.info(f"kernel proxy stopped (pid={self._process.pid})") self._process = None def __del__(self): diff --git a/qiling/os/posix/kernel_proxy/ipc.py b/qiling/os/posix/kernel_proxy/ipc.py index 8bf26eaa..59202c19 100644 --- a/qiling/os/posix/kernel_proxy/ipc.py +++ b/qiling/os/posix/kernel_proxy/ipc.py @@ -17,6 +17,8 @@ import struct import socket from enum import IntEnum +from qiling.exception import QlProxyConnectionError + class MsgType(IntEnum): SYSCALL = 1 @@ -64,7 +66,7 @@ def _recvall(sock: socket.socket, n: int) -> bytes: while len(buf) < n: chunk = sock.recv(n - len(buf)) if not chunk: - raise ConnectionError("kernel proxy connection closed") + raise QlProxyConnectionError("kernel proxy connection closed") buf.extend(chunk) return bytes(buf) @@ -162,7 +164,7 @@ class ProxyServer: return MsgType.FD_OP, (FdOp(op), proxy_fd, arg1, arg2, data) else: - raise ValueError(f"unknown message type: {msg_type}") + raise QlProxyConnectionError(f"unknown message type: {msg_type}") def send_syscall_response(self, retval: int, errno_val: int): payload = struct.pack(SYSCALL_RESP_FMT, retval, errno_val) diff --git a/qiling/os/posix/kernel_proxy/proxy.py b/qiling/os/posix/kernel_proxy/proxy.py index 41bceac0..af4934bd 100644 --- a/qiling/os/posix/kernel_proxy/proxy.py +++ b/qiling/os/posix/kernel_proxy/proxy.py @@ -13,6 +13,7 @@ Usage (internal — started by KernelProxy.__init__): python -m qiling.os.posix.kernel_proxy.proxy """ +import logging import os import sys import ctypes @@ -24,10 +25,12 @@ from qiling.os.posix.kernel_proxy.ipc import ( ProxyServer, MsgType, FdOp ) +log = logging.getLogger("qiling.os.posix.kernel_proxy.proxy") + # load libc for raw syscall() _libc_path = ctypes.util.find_library("c") if _libc_path is None: - print("kernel_proxy: cannot find libc", file=sys.stderr) + log.critical("kernel_proxy: cannot find libc") sys.exit(1) _libc = ctypes.CDLL(_libc_path, use_errno=True) @@ -86,7 +89,7 @@ def handle_fd_op(op: FdOp, proxy_fd: int, arg1: int, arg2: int, data: bytes) -> def main(): if len(sys.argv) != 2: - print(f"usage: {sys.argv[0]} ", file=sys.stderr) + log.error(f"usage: {sys.argv[0]} ") sys.exit(1) sock_fd = int(sys.argv[1]) diff --git a/tests/test_kernel_proxy.py b/tests/test_kernel_proxy.py index d6690ad7..52495238 100644 --- a/tests/test_kernel_proxy.py +++ b/tests/test_kernel_proxy.py @@ -283,13 +283,14 @@ class KernelProxyTest(unittest.TestCase): # ------------------------------------------------------------------------- def test_forward_invalid_syscall_name(self): - """forward_syscall with bogus name raises ValueError.""" + """forward_syscall with bogus name raises QlErrorSyscallNotFound.""" from qiling.os.posix.kernel_proxy import KernelProxy + from qiling.exception import QlErrorSyscallNotFound ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) proxy = KernelProxy(ql) - with self.assertRaises(ValueError): + with self.assertRaises(QlErrorSyscallNotFound): proxy.forward_syscall('nonexistent_syscall_xyz') proxy.stop() From 486369d37502f5f6161837270fa33b247bcca0c4 Mon Sep 17 00:00:00 2001 From: xwings Date: Mon, 27 Apr 2026 21:52:49 +0800 Subject: [PATCH 126/138] better New SYSCALL_EX IPC message, FD arg type and Forwarder closure --- .codex | 0 qiling/os/posix/kernel_proxy/__init__.py | 148 +++++++++++++++++--- qiling/os/posix/kernel_proxy/argtypes.py | 73 ++++++++++ qiling/os/posix/kernel_proxy/ipc.py | 113 +++++++++++++-- qiling/os/posix/kernel_proxy/proxy.py | 35 +++++ tests/test_kernel_proxy.py | 168 +++++++++++++++++++++++ 6 files changed, 505 insertions(+), 32 deletions(-) create mode 100644 .codex create mode 100644 qiling/os/posix/kernel_proxy/argtypes.py diff --git a/.codex b/.codex new file mode 100644 index 00000000..e69de29b diff --git a/qiling/os/posix/kernel_proxy/__init__.py b/qiling/os/posix/kernel_proxy/__init__.py index f7ac75ea..538f90b0 100644 --- a/qiling/os/posix/kernel_proxy/__init__.py +++ b/qiling/os/posix/kernel_proxy/__init__.py @@ -8,13 +8,22 @@ Hybrid kernel proxy — forward specific syscalls to a real Linux kernel. Usage: from qiling import Qiling - from qiling.os.posix.kernel_proxy import KernelProxy + from qiling.os.posix.kernel_proxy import KernelProxy, FD, PtrIn, PtrOut ql = Qiling(argv=["/bin/myserver"], rootfs="rootfs/x8664_linux") proxy = KernelProxy(ql) - proxy.forward_syscall("epoll_create", returns_fd=True) - proxy.forward_syscall("epoll_ctl") - proxy.forward_syscall("epoll_wait") + + # integer-arg syscall returning a new FD + proxy.forward_syscall("epoll_create1", returns_fd=True) + + # epoll_ctl(int epfd, int op, int fd, struct epoll_event *event) + proxy.forward_syscall("epoll_ctl", + arg_types=(FD, "int", FD, PtrIn(size=12))) + + # epoll_wait(int epfd, struct epoll_event *events, int maxevents, int timeout) + proxy.forward_syscall("epoll_wait", + arg_types=(FD, PtrOut(size=lambda a: a[2] * 12), "int", "int")) + ql.run() """ @@ -24,10 +33,14 @@ import os import sys import socket import subprocess -from typing import Dict, Optional, TYPE_CHECKING +import weakref +from typing import Dict, Optional, Sequence, Tuple, TYPE_CHECKING -from qiling.const import QL_INTERCEPT, QL_OS +from qiling.const import QL_INTERCEPT from qiling.exception import QlErrorArch, QlErrorSyscallError, QlErrorSyscallNotFound +from qiling.os.posix.kernel_proxy.argtypes import ( + INT, FD, PtrIn, PtrOut, PtrInOut, is_pointer, +) from qiling.os.posix.kernel_proxy.ipc import ProxyClient from qiling.os.posix.kernel_proxy.proxy_fd import ql_proxy_fd @@ -35,6 +48,9 @@ if TYPE_CHECKING: from qiling import Qiling +__all__ = ['KernelProxy', 'INT', 'FD', 'PtrIn', 'PtrOut', 'PtrInOut'] + + class KernelProxy: """Forward specific syscalls to a real Linux kernel via a helper process. @@ -117,40 +133,74 @@ class KernelProxy: ) return table[name] - def forward_syscall(self, name: str, returns_fd: bool = False): + def forward_syscall(self, name: str, returns_fd: bool = False, + arg_types: Optional[Sequence] = None): """Register a CALL hook that forwards this syscall to the kernel proxy. Args: - name: syscall name (e.g. "epoll_create", "eventfd2") + name: syscall name (e.g. "epoll_create1", "eventfd2"). returns_fd: if True, wrap the return value in ql_proxy_fd and store - in the Qiling FD table. Use this for syscalls that return - file descriptors (epoll_create, eventfd, timerfd_create, etc.) + it in the Qiling FD table. Use this for syscalls that + return file descriptors (epoll_create1, eventfd2, etc.). + arg_types: optional per-arg descriptors. Each entry is one of: + INT (or "int") — pass through unchanged (default). + FD (or "fd") — guest FD; translated to the proxy FD. + PtrIn(size) — pointer; bytes copied from guest to proxy. + PtrOut(size) — pointer; bytes copied back from proxy to guest. + PtrInOut(size) — pointer; both directions. + If omitted, all arguments are treated as INT. """ nr = self._resolve_syscall_nr(name) self._forwarded[name] = nr - forwarder = self._make_forwarder(name, nr, returns_fd) + forwarder = self._make_forwarder(name, nr, returns_fd, arg_types) self.ql.os.set_syscall(name, forwarder, QL_INTERCEPT.CALL) - self.ql.log.info(f"forwarding syscall '{name}' (nr={nr}) to kernel proxy" - f"{' [returns FD]' if returns_fd else ''}") + kind = [] + if returns_fd: + kind.append('returns FD') + if arg_types: + kind.append(f'arg_types={tuple(type(a).__name__ if not isinstance(a, str) else a for a in arg_types)}') - def _make_forwarder(self, name: str, guest_nr: int, returns_fd: bool): - """Create a CALL hook closure for one syscall.""" + suffix = f" [{', '.join(kind)}]" if kind else '' + self.ql.log.info(f"forwarding syscall '{name}' (nr={nr}) to kernel proxy{suffix}") + + def _make_forwarder(self, name: str, guest_nr: int, returns_fd: bool, + arg_types: Optional[Sequence]): + """Create a CALL hook closure for one syscall. + + Captures only the data the closure needs (host syscall nr, client, weakref + to self) so the registered hook does not keep the KernelProxy alive. + """ + # resolve once at registration time so the hot path stays simple + host_nr = self._get_host_syscall_nr(name) client = self._client + weak_self = weakref.ref(self) + + # normalize arg_types to a tuple, treating the string aliases as-is + spec = tuple(arg_types) if arg_types else () + has_pointers = any(is_pointer(s) for s in spec) def _forwarder(ql, *args): - # use the HOST syscall number, not the guest number. - # for now, resolve from the host's syscall table at runtime. - host_nr = self._get_host_syscall_nr(name) + self_ref = weak_self() + if self_ref is None: + ql.log.error(f"kernel_proxy: {name}() called after proxy was destroyed") + return -1 - padded = args + (0,) * (6 - len(args)) - retval = client.syscall(host_nr, padded[:6]) + translated = self_ref._translate_args(name, args, spec) + + if has_pointers: + in_bufs, out_specs, out_arg_indices = self_ref._collect_buffers( + ql, translated, spec + ) + retval, out_data = client.syscall_ex(host_nr, translated, in_bufs, out_specs) + self_ref._writeback_buffers(ql, args, out_arg_indices, out_data) + else: + retval = client.syscall(host_nr, translated) if returns_fd and retval >= 0: - # the proxy created a real FD. wrap it and store in Qiling's FD table. proxy_fd_obj = ql_proxy_fd(client, retval) - guest_fd = self._alloc_fd(ql, proxy_fd_obj) + guest_fd = self_ref._alloc_fd(ql, proxy_fd_obj) ql.log.debug(f"kernel_proxy: {name}() -> proxy_fd={retval}, guest_fd={guest_fd}") return guest_fd @@ -160,6 +210,60 @@ class KernelProxy: _forwarder.__name__ = f'ql_syscall_{name}' return _forwarder + def _translate_args(self, name: str, args: Tuple[int, ...], + spec: Tuple) -> Tuple[int, ...]: + """Translate guest FD args to proxy FD numbers; pad to 6 args. + + Pointer args are left untouched here — _collect_buffers replaces them + with the proxy-side buffer addresses just before invocation. + """ + out = list(args) + [0] * (6 - len(args)) + + for idx, kind in enumerate(spec): + if kind == FD: + guest_fd = args[idx] + fd_obj = self.ql.os.fd[guest_fd] if 0 <= guest_fd < len(self.ql.os.fd) else None + + if not isinstance(fd_obj, ql_proxy_fd): + raise QlErrorSyscallError( + f"kernel_proxy: {name}() arg{idx} guest_fd={guest_fd} " + f"does not refer to a proxy-owned FD" + ) + + out[idx] = fd_obj._proxy_fd + + return tuple(out[:6]) + + def _collect_buffers(self, ql, args: Tuple[int, ...], spec: Tuple): + """Read PtrIn/PtrInOut buffers from guest memory; collect PtrOut sizes.""" + in_bufs = [] + out_specs = [] + out_arg_indices = [] + + for idx, kind in enumerate(spec): + if isinstance(kind, (PtrIn, PtrInOut)): + size = kind.resolve(args) + if size > 0: + data = bytes(ql.mem.read(args[idx], size)) + in_bufs.append((idx, data)) + + if isinstance(kind, (PtrOut, PtrInOut)): + size = kind.resolve(args) + if size > 0: + out_specs.append((idx, size)) + out_arg_indices.append(idx) + + return in_bufs, out_specs, out_arg_indices + + @staticmethod + def _writeback_buffers(ql, args: Tuple[int, ...], + out_arg_indices: Sequence[int], + out_data: Sequence[bytes]): + """Write PtrOut/PtrInOut response buffers back into guest memory.""" + for idx, data in zip(out_arg_indices, out_data): + if data: + ql.mem.write(args[idx], data) + def _get_host_syscall_nr(self, name: str) -> int: """Get the syscall number on the HOST architecture.""" # we are running on Linux — read from the host's syscall table diff --git a/qiling/os/posix/kernel_proxy/argtypes.py b/qiling/os/posix/kernel_proxy/argtypes.py new file mode 100644 index 00000000..e9cf9a3c --- /dev/null +++ b/qiling/os/posix/kernel_proxy/argtypes.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +""" +Argument type descriptors for forwarded syscalls. + +When forwarding a syscall whose arguments are not all plain integers, the user +declares each argument's role so the forwarder knows how to marshal it: + + INT — pass through unchanged (the default). + FD — guest file descriptor; translate to the proxy-side FD before + forwarding. If the FD does not refer to a ql_proxy_fd, an error + is raised. + PtrIn(s) — pointer to a buffer of `s` bytes. Read from guest memory and + copied to the proxy. + PtrOut(s) — pointer to a buffer of `s` bytes. Allocated on the proxy and + copied back to guest memory after the syscall. + PtrInOut — both directions. + +`s` may be an integer or a callable taking the raw arg tuple and returning the +buffer length (e.g. for syscalls where the size depends on another argument). +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable, Tuple, Union + + +INT = 'int' +FD = 'fd' + +SizeSpec = Union[int, Callable[[Tuple[int, ...]], int]] + + +def _resolve_size(size: SizeSpec, args: Tuple[int, ...]) -> int: + if callable(size): + return int(size(args)) + + return int(size) + + +@dataclass(frozen=True) +class PtrIn: + """Input pointer — buffer of `size` bytes is read from guest memory.""" + size: SizeSpec + + def resolve(self, args: Tuple[int, ...]) -> int: + return _resolve_size(self.size, args) + + +@dataclass(frozen=True) +class PtrOut: + """Output pointer — buffer of `size` bytes is written to guest memory.""" + size: SizeSpec + + def resolve(self, args: Tuple[int, ...]) -> int: + return _resolve_size(self.size, args) + + +@dataclass(frozen=True) +class PtrInOut: + """In/out pointer — buffer is read from guest, then written back.""" + size: SizeSpec + + def resolve(self, args: Tuple[int, ...]) -> int: + return _resolve_size(self.size, args) + + +def is_pointer(spec) -> bool: + return isinstance(spec, (PtrIn, PtrOut, PtrInOut)) diff --git a/qiling/os/posix/kernel_proxy/ipc.py b/qiling/os/posix/kernel_proxy/ipc.py index 59202c19..20320dd1 100644 --- a/qiling/os/posix/kernel_proxy/ipc.py +++ b/qiling/os/posix/kernel_proxy/ipc.py @@ -6,9 +6,10 @@ """ IPC protocol between Qiling and the kernel proxy process. -Two message types: - SYSCALL — forward a raw syscall (number + 6 integer args) - FD_OP — perform an operation on a proxy-side FD (read/write/close/dup/fcntl/ioctl) +Three message types: + SYSCALL — forward a raw syscall (number + 6 integer args, no buffers) + SYSCALL_EX — forward a syscall with input/output buffer marshaling + FD_OP — perform an operation on a proxy-side FD (read/write/close/dup/fcntl/ioctl) All messages are length-prefixed binary over a Unix socketpair. """ @@ -16,13 +17,15 @@ All messages are length-prefixed binary over a Unix socketpair. import struct import socket from enum import IntEnum +from typing import List, Sequence, Tuple from qiling.exception import QlProxyConnectionError class MsgType(IntEnum): - SYSCALL = 1 - FD_OP = 2 + SYSCALL = 1 + FD_OP = 2 + SYSCALL_EX = 3 class FdOp(IntEnum): @@ -35,14 +38,19 @@ class FdOp(IntEnum): # Wire format: -# Request header: [msg_type: u8][payload_len: u32] -# SYSCALL payload: [syscall_nr: u32][args: 6 x i64] -# FD_OP payload: [op: u8][proxy_fd: i32][arg1: i64][arg2: i64][data_len: u32][data: bytes] +# Request header: [msg_type: u8][payload_len: u32] +# SYSCALL payload: [syscall_nr: u32][args: 6 x i64] +# SYSCALL_EX payload: [syscall_nr: u32][args: 6 x i64] +# [num_in: u8] then num_in * [arg_idx: u8][len: u32][data: bytes] +# [num_out: u8] then num_out * [arg_idx: u8][len: u32] +# FD_OP payload: [op: u8][proxy_fd: i32][arg1: i64][arg2: i64][data_len: u32][data: bytes] # # Response header: [status: i8][payload_len: u32] # status 0 = success, -1 = error -# SYSCALL response payload: [return_value: i64][errno: i32] -# FD_OP response payload: [return_value: i64][errno: i32][data_len: u32][data: bytes] +# SYSCALL response payload: [return_value: i64][errno: i32] +# SYSCALL_EX response payload: [return_value: i64][errno: i32][num_out: u8] +# then num_out * [len: u32][data: bytes] +# FD_OP response payload: [return_value: i64][errno: i32][data_len: u32][data: bytes] HEADER_FMT = '!BI' # msg_type/status (u8) + payload_len (u32) HEADER_SIZE = struct.calcsize(HEADER_FMT) @@ -53,6 +61,9 @@ SYSCALL_REQ_SIZE = struct.calcsize(SYSCALL_REQ_FMT) SYSCALL_RESP_FMT = '!qi' # return_value (i64) + errno (i32) SYSCALL_RESP_SIZE = struct.calcsize(SYSCALL_RESP_FMT) +SYSCALL_EX_RESP_HEAD_FMT = '!qiB' # return_value (i64) + errno (i32) + num_out (u8) +SYSCALL_EX_RESP_HEAD_SIZE = struct.calcsize(SYSCALL_EX_RESP_HEAD_FMT) + FD_OP_REQ_FMT = '!BiqqI' # op (u8) + proxy_fd (i32) + arg1 (i64) + arg2 (i64) + data_len (u32) FD_OP_REQ_SIZE = struct.calcsize(FD_OP_REQ_FMT) @@ -94,6 +105,54 @@ class ProxyClient: retval, errno_val = struct.unpack(SYSCALL_RESP_FMT, resp_payload) return retval + def syscall_ex(self, nr: int, args: Sequence[int], + in_bufs: Sequence[Tuple[int, bytes]], + out_specs: Sequence[Tuple[int, int]]) -> Tuple[int, List[bytes]]: + """Forward a syscall with buffer marshaling. + + Args: + nr: host syscall number. + args: 6 integer arg values; for buffer args these are placeholders — + the proxy replaces them with the buffer address before invoking. + in_bufs: list of (arg_idx, data) — buffers to copy in. + out_specs: list of (arg_idx, length) — buffers to copy out. + + Returns: + (retval, out_bufs) where out_bufs is a list aligned with out_specs. + """ + padded = tuple(args) + (0,) * (6 - len(args)) + payload = bytearray(struct.pack(SYSCALL_REQ_FMT, nr, *padded[:6])) + + payload.append(len(in_bufs)) + for arg_idx, data in in_bufs: + payload += struct.pack('!BI', arg_idx, len(data)) + payload += data + + payload.append(len(out_specs)) + for arg_idx, length in out_specs: + payload += struct.pack('!BI', arg_idx, length) + + header = struct.pack(HEADER_FMT, MsgType.SYSCALL_EX, len(payload)) + self._sock.sendall(header + bytes(payload)) + + resp_header = _recvall(self._sock, HEADER_SIZE) + _, resp_len = struct.unpack(HEADER_FMT, resp_header) + resp_payload = _recvall(self._sock, resp_len) + + retval, _errno, num_out = struct.unpack( + SYSCALL_EX_RESP_HEAD_FMT, resp_payload[:SYSCALL_EX_RESP_HEAD_SIZE] + ) + + out_bufs: List[bytes] = [] + offset = SYSCALL_EX_RESP_HEAD_SIZE + for _ in range(num_out): + (length,) = struct.unpack('!I', resp_payload[offset:offset + 4]) + offset += 4 + out_bufs.append(resp_payload[offset:offset + length]) + offset += length + + return retval, out_bufs + def _fd_op(self, op: FdOp, proxy_fd: int, arg1: int = 0, arg2: int = 0, data: bytes = b'') -> tuple: """Send an FD operation. Returns (return_value, data).""" payload = struct.pack(FD_OP_REQ_FMT, op, proxy_fd, arg1, arg2, len(data)) @@ -157,6 +216,31 @@ class ProxyServer: fields = struct.unpack(SYSCALL_REQ_FMT, payload) return MsgType.SYSCALL, fields # (nr, a0, a1, a2, a3, a4, a5) + elif msg_type == MsgType.SYSCALL_EX: + offset = SYSCALL_REQ_SIZE + fixed = struct.unpack(SYSCALL_REQ_FMT, payload[:offset]) + nr = fixed[0] + args = list(fixed[1:]) + + num_in = payload[offset] + offset += 1 + in_bufs: List[Tuple[int, bytes]] = [] + for _ in range(num_in): + arg_idx, length = struct.unpack('!BI', payload[offset:offset + 5]) + offset += 5 + in_bufs.append((arg_idx, payload[offset:offset + length])) + offset += length + + num_out = payload[offset] + offset += 1 + out_specs: List[Tuple[int, int]] = [] + for _ in range(num_out): + arg_idx, length = struct.unpack('!BI', payload[offset:offset + 5]) + offset += 5 + out_specs.append((arg_idx, length)) + + return MsgType.SYSCALL_EX, (nr, args, in_bufs, out_specs) + elif msg_type == MsgType.FD_OP: fixed = struct.unpack(FD_OP_REQ_FMT, payload[:FD_OP_REQ_SIZE]) op, proxy_fd, arg1, arg2, data_len = fixed @@ -171,6 +255,15 @@ class ProxyServer: header = struct.pack(HEADER_FMT, 0, len(payload)) self._sock.sendall(header + payload) + def send_syscall_ex_response(self, retval: int, errno_val: int, out_bufs: Sequence[bytes]): + payload = bytearray(struct.pack(SYSCALL_EX_RESP_HEAD_FMT, retval, errno_val, len(out_bufs))) + for buf in out_bufs: + payload += struct.pack('!I', len(buf)) + payload += buf + + header = struct.pack(HEADER_FMT, 0, len(payload)) + self._sock.sendall(header + bytes(payload)) + def send_fd_op_response(self, retval: int, errno_val: int, data: bytes = b''): payload = struct.pack(FD_OP_RESP_FMT, retval, errno_val, len(data)) payload += data diff --git a/qiling/os/posix/kernel_proxy/proxy.py b/qiling/os/posix/kernel_proxy/proxy.py index af4934bd..1e725611 100644 --- a/qiling/os/posix/kernel_proxy/proxy.py +++ b/qiling/os/posix/kernel_proxy/proxy.py @@ -51,6 +51,36 @@ def raw_syscall(nr: int, a0: int, a1: int, a2: int, a3: int, a4: int, a5: int) - return result, 0 +def raw_syscall_ex(nr: int, args: list, in_bufs: list, out_specs: list) -> tuple: + """Execute a syscall with buffer marshaling. + + For each in_buf (arg_idx, data): allocate a ctypes buffer initialized with + data and place its address in args[arg_idx]. For each out_spec (arg_idx, length): + allocate a zeroed buffer and place its address in args[arg_idx]. After the + syscall, return the contents of each out buffer. + """ + keepalive = [] # keep ctypes buffers alive until after the syscall + out_buffers = [] # parallel to out_specs + + args = list(args) + + for arg_idx, data in in_bufs: + buf = ctypes.create_string_buffer(data, len(data)) + keepalive.append(buf) + args[arg_idx] = ctypes.addressof(buf) + + for arg_idx, length in out_specs: + buf = ctypes.create_string_buffer(length) + keepalive.append(buf) + out_buffers.append(buf) + args[arg_idx] = ctypes.addressof(buf) + + retval, err = raw_syscall(nr, *args) + + out_data = [bytes(buf.raw) for buf in out_buffers] + return retval, err, out_data + + def handle_fd_op(op: FdOp, proxy_fd: int, arg1: int, arg2: int, data: bytes) -> tuple: """Handle an FD operation on a proxy-side FD. Returns (retval, errno, data).""" try: @@ -107,6 +137,11 @@ def main(): retval, err = raw_syscall(nr, a0, a1, a2, a3, a4, a5) server.send_syscall_response(retval, err) + elif msg_type == MsgType.SYSCALL_EX: + nr, args, in_bufs, out_specs = fields + retval, err, out_bufs = raw_syscall_ex(nr, args, in_bufs, out_specs) + server.send_syscall_ex_response(retval, err, out_bufs) + elif msg_type == MsgType.FD_OP: op, proxy_fd, arg1, arg2, data = fields retval, err, resp_data = handle_fd_op(op, proxy_fd, arg1, arg2, data) diff --git a/tests/test_kernel_proxy.py b/tests/test_kernel_proxy.py index 52495238..4f7aae4d 100644 --- a/tests/test_kernel_proxy.py +++ b/tests/test_kernel_proxy.py @@ -352,5 +352,173 @@ class KernelProxyTest(unittest.TestCase): del ql + # ------------------------------------------------------------------------- + # FD translation (#3) + # ------------------------------------------------------------------------- + + def test_fd_arg_translated_to_proxy_fd(self): + """When a forwarded syscall's arg is declared FD, the guest fd is replaced + with the underlying proxy fd before forwarding.""" + from qiling.os.posix.kernel_proxy import KernelProxy, FD + from qiling.os.posix.kernel_proxy.proxy_fd import ql_proxy_fd + + ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) + proxy = KernelProxy(ql) + proxy.forward_syscall('close', arg_types=(FD,)) + + # capture what gets sent over IPC + sent = [] + original_syscall = proxy._client.syscall + + def spy(nr, args): + sent.append((nr, tuple(args))) + return 0 # pretend the kernel accepted + + proxy._client.syscall = spy + + # plant a ql_proxy_fd at guest_fd=42 with a chosen proxy_fd value + guest_fd = 42 + fake_proxy_fd = 9999 + ql.os.fd[guest_fd] = ql_proxy_fd(proxy._client, fake_proxy_fd) + + close_hook = ql.os.posix_syscall_hooks[QL_INTERCEPT.CALL]['ql_syscall_close'] + close_hook(ql, guest_fd) + + self.assertEqual(len(sent), 1) + _nr, args = sent[0] + self.assertEqual(args[0], fake_proxy_fd, + f"expected proxy_fd={fake_proxy_fd} forwarded, got args={args}") + + proxy._client.syscall = original_syscall + ql.os.fd[guest_fd] = None + proxy.stop() + del ql + + def test_fd_arg_rejects_non_proxy_fd(self): + """Forwarding with FD arg type rejects non-proxy guest FDs.""" + from qiling.os.posix.kernel_proxy import KernelProxy, FD + from qiling.exception import QlErrorSyscallError + + ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) + proxy = KernelProxy(ql) + proxy.forward_syscall('close', arg_types=(FD,)) + + close_hook = ql.os.posix_syscall_hooks[QL_INTERCEPT.CALL]['ql_syscall_close'] + + # stdin (fd 0) is a regular ql_pipe, not a proxy fd + with self.assertRaises(QlErrorSyscallError): + close_hook(ql, 0) + + proxy.stop() + del ql + + # ------------------------------------------------------------------------- + # Pointer marshaling (#2) + # ------------------------------------------------------------------------- + + def test_ptr_out_writes_back_to_guest_memory(self): + """PtrOut buffer is written back into guest memory after the syscall. + + Uses pipe2(int pipefd[2], int flags) — pipefd is an output buffer of + 2 * sizeof(int) = 8 bytes containing the read/write FDs created by + the kernel. + """ + from qiling.os.posix.kernel_proxy import KernelProxy, PtrOut + + ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) + proxy = KernelProxy(ql) + proxy.forward_syscall('pipe2', arg_types=(PtrOut(size=8), 'int')) + + # pick a free guest address and map it + addr = 0x800000 + ql.mem.map(addr, 0x1000) + ql.mem.write(addr, b'\xff' * 8) # poison so we can detect the writeback + + hook = ql.os.posix_syscall_hooks[QL_INTERCEPT.CALL]['ql_syscall_pipe2'] + retval = hook(ql, addr, 0) + self.assertEqual(retval, 0) + + raw = bytes(ql.mem.read(addr, 8)) + rfd, wfd = struct.unpack('=0) and distinct + self.assertGreaterEqual(rfd, 0) + self.assertGreaterEqual(wfd, 0) + self.assertNotEqual(rfd, wfd) + + # clean up the proxy-side FDs the kernel just gave us + import os as _os + _os.close(rfd) + _os.close(wfd) + + proxy.stop() + del ql + + def test_ptr_in_reads_guest_memory(self): + """PtrIn buffer is copied from guest memory and the proxy sees the data. + + Uses write(int fd, const void *buf, size_t count) on stderr (fd 2). + We can't easily inspect proxy's stderr, but a successful return value + equal to count proves the data was forwarded — write would otherwise + return -EFAULT for a bad pointer or short for less data. + """ + from qiling.os.posix.kernel_proxy import KernelProxy, PtrIn + + ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) + proxy = KernelProxy(ql) + proxy.forward_syscall('write', arg_types=('int', PtrIn(size=lambda a: a[2]), 'int')) + + addr = 0x800000 + payload = b'kernel_proxy ptr_in roundtrip\n' + ql.mem.map(addr, 0x1000) + ql.mem.write(addr, payload) + + # forward stderr (fd 2 in the proxy is our subprocess's stderr, + # which goes to the test runner — harmless) + hook = ql.os.posix_syscall_hooks[QL_INTERCEPT.CALL]['ql_syscall_write'] + retval = hook(ql, 2, addr, len(payload)) + self.assertEqual(retval, len(payload)) + + proxy.stop() + del ql + + def test_ptr_size_callable(self): + """PtrIn/PtrOut accept a size callable that depends on other args.""" + from qiling.os.posix.kernel_proxy import PtrIn, PtrOut + + ptr = PtrIn(size=lambda args: args[2] * 4) + self.assertEqual(ptr.resolve((0, 0, 5)), 20) + + ptr2 = PtrOut(size=12) + self.assertEqual(ptr2.resolve((0, 0, 0)), 12) + + # ------------------------------------------------------------------------- + # Reference cycle (#4) + # ------------------------------------------------------------------------- + + def test_no_reference_cycle_via_hook(self): + """The forwarder closure holds only a weakref to KernelProxy, so the + proxy can be garbage-collected once the user drops their reference, + even though the hook is still registered on ql.os.""" + import gc + import weakref + from qiling.os.posix.kernel_proxy import KernelProxy + + ql = Qiling([self.HELLO_BIN], self.ROOTFS, verbose=QL_VERBOSE.OFF) + proxy = KernelProxy(ql) + proxy.forward_syscall('getpid') + proxy.forward_syscall('eventfd2', returns_fd=True) + + wref = weakref.ref(proxy) + proxy.stop() # tear down the subprocess but leave the hooks registered + del proxy + gc.collect() + + self.assertIsNone(wref(), + "KernelProxy survived after stop()+del — closure must hold " + "a strong ref (cycle), defeating the weakref design") + + del ql + + if __name__ == "__main__": unittest.main() From 22947e07556ab3116b8f500ffd31c27a349cc92f Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 09:24:32 +0800 Subject: [PATCH 127/138] ignore files --- .codex | 0 .gitignore | 2 ++ 2 files changed, 2 insertions(+) delete mode 100644 .codex diff --git a/.codex b/.codex deleted file mode 100644 index e69de29b..00000000 diff --git a/.gitignore b/.gitignore index b0cd99ad..c054a119 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,5 @@ pyrightconfig.json # AI CLAUDE.md .claude +.codex +.AGENT.md From 104830d52d6364d980c864c92a5470f3ffd63649 Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 09:31:39 +0800 Subject: [PATCH 128/138] fix docker python:3-slim --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index dd0f49e4..0af1fe05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3-slim AS base +FROM python:3.13-slim-bookworm AS base WORKDIR /qiling @@ -12,7 +12,7 @@ RUN apt-get update && apt-get -y upgrade && rm -rf /var/lib/apt/lists/* FROM base AS builder RUN apt-get update && apt-get install -y --no-install-recommends \ - cmake build-essential gcc git \ + cmake build-essential gcc git pkg-config \ && rm -rf /var/lib/apt/lists/* COPY pyproject.toml poetry.lock ./ From 0d17d14a155f5d19468c7db17d491eaa629e85fa Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 10:57:47 +0800 Subject: [PATCH 129/138] fix docker python:3-slim and dropped --no-deps --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0af1fe05..861fa36d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.13-slim-bookworm AS base +FROM python:3.12-slim-bookworm AS base WORKDIR /qiling @@ -33,7 +33,7 @@ WORKDIR /qiling RUN apt-get update \ && apt-get install -y --no-install-recommends unzip apt-utils \ && rm -rf /var/lib/apt/lists/* \ - && pip3 install --no-deps --no-cache-dir dist/*.whl \ + && pip3 install --no-cache-dir dist/*.whl \ && rm -rf ./dist/ CMD ["bash"] From 47c8a92b1e39aa8a6bd1d1e504472d2e42adb1d3 Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 11:47:55 +0800 Subject: [PATCH 130/138] Bump test to 3.11 and 3.13 --- .github/workflows/build-ci.yml | 4 +- Dockerfile | 6 +- poetry.lock | 637 ++++++--------------------------- pyproject.toml | 4 +- 4 files changed, 114 insertions(+), 537 deletions(-) diff --git a/.github/workflows/build-ci.yml b/.github/workflows/build-ci.yml index b8a5d3cb..b2771559 100644 --- a/.github/workflows/build-ci.yml +++ b/.github/workflows/build-ci.yml @@ -11,10 +11,10 @@ jobs: matrix: #os: [windows-2019, macos-10.15, ubuntu-18.04, ubuntu-20.04] os: [windows-latest, ubuntu-latest] - python-version: ["3.9", "3.11"] + python-version: ["3.11", "3.13"] include: - os: ubuntu-latest - python-version: 3.9 + python-version: "3.13" container: Docker steps: diff --git a/Dockerfile b/Dockerfile index dd0f49e4..0d7d6e8a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3-slim AS base +FROM python:3.13-slim-trixie AS base WORKDIR /qiling @@ -12,7 +12,7 @@ RUN apt-get update && apt-get -y upgrade && rm -rf /var/lib/apt/lists/* FROM base AS builder RUN apt-get update && apt-get install -y --no-install-recommends \ - cmake build-essential gcc git \ + cmake build-essential gcc git pkg-config \ && rm -rf /var/lib/apt/lists/* COPY pyproject.toml poetry.lock ./ @@ -33,7 +33,7 @@ WORKDIR /qiling RUN apt-get update \ && apt-get install -y --no-install-recommends unzip apt-utils \ && rm -rf /var/lib/apt/lists/* \ - && pip3 install --no-deps --no-cache-dir dist/*.whl \ + && pip3 install --no-cache-dir dist/*.whl \ && rm -rf ./dist/ CMD ["bash"] diff --git a/poetry.lock b/poetry.lock index 18901472..0a7dfe3a 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,16 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. - -[[package]] -name = "antlr4-python3-runtime" -version = "4.8" -description = "ANTLR 4.8 runtime for Python 3.7" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "antlr4-python3-runtime-4.8.tar.gz", hash = "sha256:15793f5d0512a372b4e7d2284058ad32ce7dd27126b105fb0b2245130445db33"}, -] +# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. [[package]] name = "antlr4-python3-runtime" @@ -19,32 +7,11 @@ description = "ANTLR 4.13.2 runtime for Python 3" optional = false python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "antlr4_python3_runtime-4.13.2-py3-none-any.whl", hash = "sha256:fe3835eb8d33daece0e799090eda89719dbccee7aa39ef94eed3818cafa5a7e8"}, {file = "antlr4_python3_runtime-4.13.2.tar.gz", hash = "sha256:909b647e1d2fc2b70180ac586df3933e38919c85f98ccc656a96cd3f25ef3916"}, ] -[[package]] -name = "asciimatics" -version = "1.14.0" -description = "A cross-platform package to replace curses (mouse/keyboard input & text colours/positioning) and create ASCII animations" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "asciimatics-1.14.0-py2.py3-none-any.whl", hash = "sha256:277fe925d0d7a029b35245cde01ead009b4a1336130543ace5c8821f38df1da7"}, - {file = "asciimatics-1.14.0.tar.gz", hash = "sha256:16d20ce42210b434eb05ba469ecdb8293ac7ed3c0ce0dd4f70e30d72d7602227"}, -] - -[package.dependencies] -future = "*" -Pillow = ">=2.7.0" -pyfiglet = ">=0.7.2" -pywin32 = {version = "*", markers = "sys_platform == \"win32\""} -wcwidth = "*" - [[package]] name = "asciimatics" version = "1.15.0" @@ -52,7 +19,6 @@ description = "A cross-platform package to replace curses (mouse/keyboard input optional = false python-versions = ">= 3.8" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "asciimatics-1.15.0-py3-none-any.whl", hash = "sha256:0fe068a6bed522929bd04bb5b8a2fb6ebf0aef1b7a9b3843cf71030a34bc38d5"}, {file = "asciimatics-1.15.0.tar.gz", hash = "sha256:cfdd398042727519d8b73e62b8ef82c0becfed4eb420899c3b96c98d0b96821a"}, @@ -85,9 +51,6 @@ files = [ {file = "capstone-5.0.7.tar.gz", hash = "sha256:796bdd69b05fa124fc2aa2e74b9a0b3d4c4e7f3e02add5e583cf2f3bca282ede"}, ] -[package.dependencies] -importlib-resources = {version = "*", markers = "python_version < \"3.9\""} - [[package]] name = "cffi" version = "1.17.1" @@ -251,18 +214,6 @@ files = [ {file = "first-2.0.2.tar.gz", hash = "sha256:ff285b08c55f8c97ce4ea7012743af2495c9f1291785f163722bd36f6af6d3bf"}, ] -[[package]] -name = "future" -version = "0.18.3" -description = "Clean single-source support for Python 3 and 2" -optional = false -python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "future-0.18.3.tar.gz", hash = "sha256:34a17436ed1e96697a86f9de3d15a3b0be01d8bc8de9c1dffd59fb8234ed5307"}, -] - [[package]] name = "fuzzercorn" version = "0.0.1" @@ -284,198 +235,135 @@ unicorn = ">=2.0.0rc5" [[package]] name = "gevent" -version = "24.2.1" +version = "26.4.0" description = "Coroutine-based network library" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "gevent-24.2.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:6f947a9abc1a129858391b3d9334c45041c08a0f23d14333d5b844b6e5c17a07"}, - {file = "gevent-24.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bde283313daf0b34a8d1bab30325f5cb0f4e11b5869dbe5bc61f8fe09a8f66f3"}, - {file = "gevent-24.2.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5a1df555431f5cd5cc189a6ee3544d24f8c52f2529134685f1e878c4972ab026"}, - {file = "gevent-24.2.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:14532a67f7cb29fb055a0e9b39f16b88ed22c66b96641df8c04bdc38c26b9ea5"}, - {file = "gevent-24.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd23df885318391856415e20acfd51a985cba6919f0be78ed89f5db9ff3a31cb"}, - {file = "gevent-24.2.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:ca80b121bbec76d7794fcb45e65a7eca660a76cc1a104ed439cdbd7df5f0b060"}, - {file = "gevent-24.2.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b9913c45d1be52d7a5db0c63977eebb51f68a2d5e6fd922d1d9b5e5fd758cc98"}, - {file = "gevent-24.2.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:918cdf8751b24986f915d743225ad6b702f83e1106e08a63b736e3a4c6ead789"}, - {file = "gevent-24.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:3d5325ccfadfd3dcf72ff88a92fb8fc0b56cacc7225f0f4b6dcf186c1a6eeabc"}, - {file = "gevent-24.2.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:03aa5879acd6b7076f6a2a307410fb1e0d288b84b03cdfd8c74db8b4bc882fc5"}, - {file = "gevent-24.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8bb35ce57a63c9a6896c71a285818a3922d8ca05d150fd1fe49a7f57287b836"}, - {file = "gevent-24.2.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7f87c2c02e03d99b95cfa6f7a776409083a9e4d468912e18c7680437b29222c"}, - {file = "gevent-24.2.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:968581d1717bbcf170758580f5f97a2925854943c45a19be4d47299507db2eb7"}, - {file = "gevent-24.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7899a38d0ae7e817e99adb217f586d0a4620e315e4de577444ebeeed2c5729be"}, - {file = "gevent-24.2.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:f5e8e8d60e18d5f7fd49983f0c4696deeddaf6e608fbab33397671e2fcc6cc91"}, - {file = "gevent-24.2.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fbfdce91239fe306772faab57597186710d5699213f4df099d1612da7320d682"}, - {file = "gevent-24.2.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cdf66977a976d6a3cfb006afdf825d1482f84f7b81179db33941f2fc9673bb1d"}, - {file = "gevent-24.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:1dffb395e500613e0452b9503153f8f7ba587c67dd4a85fc7cd7aa7430cb02cc"}, - {file = "gevent-24.2.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:6c47ae7d1174617b3509f5d884935e788f325eb8f1a7efc95d295c68d83cce40"}, - {file = "gevent-24.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7cac622e11b4253ac4536a654fe221249065d9a69feb6cdcd4d9af3503602e0"}, - {file = "gevent-24.2.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bf5b9c72b884c6f0c4ed26ef204ee1f768b9437330422492c319470954bc4cc7"}, - {file = "gevent-24.2.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f5de3c676e57177b38857f6e3cdfbe8f38d1cd754b63200c0615eaa31f514b4f"}, - {file = "gevent-24.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4faf846ed132fd7ebfbbf4fde588a62d21faa0faa06e6f468b7faa6f436b661"}, - {file = "gevent-24.2.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:368a277bd9278ddb0fde308e6a43f544222d76ed0c4166e0d9f6b036586819d9"}, - {file = "gevent-24.2.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f8a04cf0c5b7139bc6368b461257d4a757ea2fe89b3773e494d235b7dd51119f"}, - {file = "gevent-24.2.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9d8d0642c63d453179058abc4143e30718b19a85cbf58c2744c9a63f06a1d388"}, - {file = "gevent-24.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:94138682e68ec197db42ad7442d3cf9b328069c3ad8e4e5022e6b5cd3e7ffae5"}, - {file = "gevent-24.2.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:8f4b8e777d39013595a7740b4463e61b1cfe5f462f1b609b28fbc1e4c4ff01e5"}, - {file = "gevent-24.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:141a2b24ad14f7b9576965c0c84927fc85f824a9bb19f6ec1e61e845d87c9cd8"}, - {file = "gevent-24.2.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:9202f22ef811053077d01f43cc02b4aaf4472792f9fd0f5081b0b05c926cca19"}, - {file = "gevent-24.2.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2955eea9c44c842c626feebf4459c42ce168685aa99594e049d03bedf53c2800"}, - {file = "gevent-24.2.1-cp38-cp38-win32.whl", hash = "sha256:44098038d5e2749b0784aabb27f1fcbb3f43edebedf64d0af0d26955611be8d6"}, - {file = "gevent-24.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:117e5837bc74a1673605fb53f8bfe22feb6e5afa411f524c835b2ddf768db0de"}, - {file = "gevent-24.2.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:2ae3a25ecce0a5b0cd0808ab716bfca180230112bb4bc89b46ae0061d62d4afe"}, - {file = "gevent-24.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a7ceb59986456ce851160867ce4929edaffbd2f069ae25717150199f8e1548b8"}, - {file = "gevent-24.2.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:2e9ac06f225b696cdedbb22f9e805e2dd87bf82e8fa5e17756f94e88a9d37cf7"}, - {file = "gevent-24.2.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:90cbac1ec05b305a1b90ede61ef73126afdeb5a804ae04480d6da12c56378df1"}, - {file = "gevent-24.2.1-cp39-cp39-win32.whl", hash = "sha256:782a771424fe74bc7e75c228a1da671578c2ba4ddb2ca09b8f959abdf787331e"}, - {file = "gevent-24.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:3adfb96637f44010be8abd1b5e73b5070f851b817a0b182e601202f20fa06533"}, - {file = "gevent-24.2.1-pp310-pypy310_pp73-macosx_11_0_universal2.whl", hash = "sha256:7b00f8c9065de3ad226f7979154a7b27f3b9151c8055c162332369262fc025d8"}, - {file = "gevent-24.2.1.tar.gz", hash = "sha256:432fc76f680acf7cf188c2ee0f5d3ab73b63c1f03114c7cd8a34cebbe5aa2056"}, + {file = "gevent-26.4.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:079dca3fe5e048714e93ab5ce7cbf2ec31709c860979feb117abbbf2b8ae6ad0"}, + {file = "gevent-26.4.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:df208cd53c7382a4cc8470d39a92fc73b3cf2f0f3379d6c88bb556823a26ccb7"}, + {file = "gevent-26.4.0-cp310-cp310-win_amd64.whl", hash = "sha256:ccaa0f002cfd69c03621ac05243c1a00ed77cee97b363d0108b0e36663e4ca33"}, + {file = "gevent-26.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:201323a5fb9a0646a0c7b384395ca55d60ee83200677919229df0648c4b78e6c"}, + {file = "gevent-26.4.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:82d68a60a4207826db295b4e80a204c9d392ce78ccc15679195faeb9e29d8388"}, + {file = "gevent-26.4.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:35b037b415ed38369717800250fe5974249525953b46026bef9def20f946dfb0"}, + {file = "gevent-26.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:6d973735d2067607a32cd182893978755eee829a0dc268087592d3b715e63fad"}, + {file = "gevent-26.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4fbd3ff28a7babbfee750684c4f46ba6eedb3bce69365dd146726986b79fa6c1"}, + {file = "gevent-26.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f0a03650ca60c4c5774cbe21333905b95f2f5abd98ea5a3dbf28d93f2a7a5a84"}, + {file = "gevent-26.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:d00d8c4ca1afab90e478b79679dd53787082c6da8d2f4fdc7667a4440d1e1a7a"}, + {file = "gevent-26.4.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:318a0a73f664113e8d86d0cb0e328e7650e2d7d9c2e045418ab6fb1285831ad3"}, + {file = "gevent-26.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:ce7aa033a3f68beb6732d1450a80c1af29e63e0c2d01abad7918cf2507f72fa6"}, + {file = "gevent-26.4.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:a1b897c952baefd72232efaeb3bdb1ca2fa7ae94cbfe68ac21201b03e843190a"}, + {file = "gevent-26.4.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:7eef2ea508ce41795e20587a5fc868ae4919543097c81a40fbdfd65bc479f54f"}, + {file = "gevent-26.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:f7e12fdd28cc9f39a463d8df5172d698c64a8ed385a21d98e7092fd8308a139a"}, + {file = "gevent-26.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d48e3ee13d7678c24c22f19d441ad6bc220a79f23662d03ff36fae0d62efdb59"}, + {file = "gevent-26.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c58c8e034f94329be4dc0979fba3301005a433dbab42cea0b2c33fd736946872"}, + {file = "gevent-26.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1c737e6ac6ce1398df0e3f41c58d982e397c993cbe73ac05b7edbe39e128c9cb"}, + {file = "gevent-26.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:1fe581d41c63cd1d8b12c69561ce53a48ad0d8763b254740d7bfea997335a38c"}, + {file = "gevent-26.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:c503b0c0a681e795255a13e5bb4e41615c3b020c1db93b8dfa04cfeb8f19d5a9"}, + {file = "gevent-26.4.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:684256c29e3e5d4d0c4d06b772d00574d0dc859dfbb2fd13d318c512b16e1f89"}, + {file = "gevent-26.4.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:73eafd06b158d511f1ec6e5902a45e0ae3b48e745f35e9df97d25f809f537d88"}, + {file = "gevent-26.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1a18e543c830a1c07a2efeb33786a57ccac360af70cb42bbaf5a6f5f7ca49300"}, + {file = "gevent-26.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:74f1e3a460c43aefcb4ff9ef91aac15abc0b42e5233771e1956574d14ba9cac6"}, + {file = "gevent-26.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:954258873ae0bcc97fb41e48db25284fb73454bfefe27db8ceb89225da5502fb"}, + {file = "gevent-26.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:4a9a64c064457c1afaf93ee2815fe0f38be6ecbb92806a6a712f12afc3e26cf5"}, + {file = "gevent-26.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:7ab0f183a6fd2369eef619832eef14f1f2f69c605163c3f2dc41deb799af4a71"}, + {file = "gevent-26.4.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:7e5906860e632bf965e1966c57e6bfc19dcb79dc262f04fdb0a9d7c12147bf69"}, + {file = "gevent-26.4.0-cp314-cp314-manylinux_2_28_ppc64le.whl", hash = "sha256:297a361071dc6708115d4544859321e93b02a6cd5823ba02c0a909530a519d45"}, + {file = "gevent-26.4.0-cp314-cp314-manylinux_2_28_s390x.whl", hash = "sha256:7e74f59e5c9011afa2a9cb7106bb9a59f2a1f74c3d7b272c1b852eb0bc0b8f90"}, + {file = "gevent-26.4.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:45d6010a6a981f5a2b3411c4e38fbe305a1b46e4b12db3b4914775927dea7ba4"}, + {file = "gevent-26.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:dc38137ba2f43794c488615aafa2eefd0cc142f484a8274d4c827ed7a031a1e2"}, + {file = "gevent-26.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:29a225d2d4da37e20c7a246754a64442d0e43e4534b8cc764f89530bb22a4237"}, + {file = "gevent-26.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:d1c08bc9bb6bd79732a26710a99588b5e9b67b668e165dd609704b876f41baab"}, + {file = "gevent-26.4.0.tar.gz", hash = "sha256:288d03addfccf0d1c67268358b6759b04392bf3bc35d26f3d9a45c82899c292d"}, ] [package.dependencies] -cffi = {version = ">=1.12.2", markers = "platform_python_implementation == \"CPython\" and sys_platform == \"win32\""} -greenlet = [ - {version = ">=2.0.0", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.11\""}, - {version = ">=3.0rc3", markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\""}, -] +cffi = {version = ">=1.17.1", markers = "platform_python_implementation == \"CPython\" and sys_platform == \"win32\""} +greenlet = {version = ">=3.2.2", markers = "platform_python_implementation == \"CPython\""} "zope.event" = "*" "zope.interface" = "*" [package.extras] dnspython = ["dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\""] docs = ["furo", "repoze.sphinx.autointerface", "sphinx", "sphinxcontrib-programoutput", "zope.schema"] -monitor = ["psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\""] -recommended = ["cffi (>=1.12.2) ; platform_python_implementation == \"CPython\"", "dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\"", "psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\""] -test = ["cffi (>=1.12.2) ; platform_python_implementation == \"CPython\"", "coverage (>=5.0) ; sys_platform != \"win32\"", "dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\"", "objgraph", "psutil (>=5.7.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\"", "requests"] +monitor = ["psutil (>=6.0.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\""] +recommended = ["cffi (>=1.17.1) ; platform_python_implementation == \"CPython\"", "dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\"", "psutil (>=6.0.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\""] +test = ["cffi (>=1.17.1) ; platform_python_implementation == \"CPython\"", "coverage (>=5.0,<7.13) ; sys_platform != \"win32\"", "dnspython (>=1.16.0,<2.0) ; python_version < \"3.10\"", "idna ; python_version < \"3.10\"", "objgraph", "psutil (>=6.0.0) ; sys_platform != \"win32\" or platform_python_implementation == \"CPython\"", "requests"] [[package]] name = "greenlet" -version = "3.1.1" +version = "3.5.0" description = "Lightweight in-process concurrent programming" optional = false -python-versions = ">=3.7" +python-versions = ">=3.10" groups = ["main"] markers = "platform_python_implementation == \"CPython\"" files = [ - {file = "greenlet-3.1.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563"}, - {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83"}, - {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:36b89d13c49216cadb828db8dfa6ce86bbbc476a82d3a6c397f0efae0525bdd0"}, - {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94b6150a85e1b33b40b1464a3f9988dcc5251d6ed06842abff82e42632fac120"}, - {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93147c513fac16385d1036b7e5b102c7fbbdb163d556b791f0f11eada7ba65dc"}, - {file = "greenlet-3.1.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da7a9bff22ce038e19bf62c4dd1ec8391062878710ded0a845bcf47cc0200617"}, - {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b2795058c23988728eec1f36a4e5e4ebad22f8320c85f3587b539b9ac84128d7"}, - {file = "greenlet-3.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ed10eac5830befbdd0c32f83e8aa6288361597550ba669b04c48f0f9a2c843c6"}, - {file = "greenlet-3.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:77c386de38a60d1dfb8e55b8c1101d68c79dfdd25c7095d51fec2dd800892b80"}, - {file = "greenlet-3.1.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70"}, - {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159"}, - {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e"}, - {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1"}, - {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383"}, - {file = "greenlet-3.1.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a"}, - {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511"}, - {file = "greenlet-3.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395"}, - {file = "greenlet-3.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39"}, - {file = "greenlet-3.1.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d"}, - {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79"}, - {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c3a701fe5a9695b238503ce5bbe8218e03c3bcccf7e204e455e7462d770268aa"}, - {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2846930c65b47d70b9d178e89c7e1a69c95c1f68ea5aa0a58646b7a96df12441"}, - {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99cfaa2110534e2cf3ba31a7abcac9d328d1d9f1b95beede58294a60348fba36"}, - {file = "greenlet-3.1.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1443279c19fca463fc33e65ef2a935a5b09bb90f978beab37729e1c3c6c25fe9"}, - {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b7cede291382a78f7bb5f04a529cb18e068dd29e0fb27376074b6d0317bf4dd0"}, - {file = "greenlet-3.1.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:23f20bb60ae298d7d8656c6ec6db134bca379ecefadb0b19ce6f19d1f232a942"}, - {file = "greenlet-3.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:7124e16b4c55d417577c2077be379514321916d5790fa287c9ed6f23bd2ffd01"}, - {file = "greenlet-3.1.1-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:05175c27cb459dcfc05d026c4232f9de8913ed006d42713cb8a5137bd49375f1"}, - {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:935e943ec47c4afab8965954bf49bfa639c05d4ccf9ef6e924188f762145c0ff"}, - {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:667a9706c970cb552ede35aee17339a18e8f2a87a51fba2ed39ceeeb1004798a"}, - {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8a678974d1f3aa55f6cc34dc480169d58f2e6d8958895d68845fa4ab566509e"}, - {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:efc0f674aa41b92da8c49e0346318c6075d734994c3c4e4430b1c3f853e498e4"}, - {file = "greenlet-3.1.1-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e"}, - {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:275f72decf9932639c1c6dd1013a1bc266438eb32710016a1c742df5da6e60a1"}, - {file = "greenlet-3.1.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:c4aab7f6381f38a4b42f269057aee279ab0fc7bf2e929e3d4abfae97b682a12c"}, - {file = "greenlet-3.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:b42703b1cf69f2aa1df7d1030b9d77d3e584a70755674d60e710f0af570f3761"}, - {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f1695e76146579f8c06c1509c7ce4dfe0706f49c6831a817ac04eebb2fd02011"}, - {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7876452af029456b3f3549b696bb36a06db7c90747740c5302f74a9e9fa14b13"}, - {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4ead44c85f8ab905852d3de8d86f6f8baf77109f9da589cb4fa142bd3b57b475"}, - {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8320f64b777d00dd7ccdade271eaf0cad6636343293a25074cc5566160e4de7b"}, - {file = "greenlet-3.1.1-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6510bf84a6b643dabba74d3049ead221257603a253d0a9873f55f6a59a65f822"}, - {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:04b013dc07c96f83134b1e99888e7a79979f1a247e2a9f59697fa14b5862ed01"}, - {file = "greenlet-3.1.1-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6"}, - {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47da355d8687fd65240c364c90a31569a133b7b60de111c255ef5b606f2ae291"}, - {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:98884ecf2ffb7d7fe6bd517e8eb99d31ff7855a840fa6d0d63cd07c037f6a981"}, - {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f1d4aeb8891338e60d1ab6127af1fe45def5259def8094b9c7e34690c8858803"}, - {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db32b5348615a04b82240cc67983cb315309e88d444a288934ee6ceaebcad6cc"}, - {file = "greenlet-3.1.1-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dcc62f31eae24de7f8dce72134c8651c58000d3b1868e01392baea7c32c247de"}, - {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1d3755bcb2e02de341c55b4fca7a745a24a9e7212ac953f6b3a48d117d7257aa"}, - {file = "greenlet-3.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b8da394b34370874b4572676f36acabac172602abf054cbc4ac910219f3340af"}, - {file = "greenlet-3.1.1-cp37-cp37m-win32.whl", hash = "sha256:a0dfc6c143b519113354e780a50381508139b07d2177cb6ad6a08278ec655798"}, - {file = "greenlet-3.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:54558ea205654b50c438029505def3834e80f0869a70fb15b871c29b4575ddef"}, - {file = "greenlet-3.1.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:346bed03fe47414091be4ad44786d1bd8bef0c3fcad6ed3dee074a032ab408a9"}, - {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfc59d69fc48664bc693842bd57acfdd490acafda1ab52c7836e3fc75c90a111"}, - {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d21e10da6ec19b457b82636209cbe2331ff4306b54d06fa04b7c138ba18c8a81"}, - {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:37b9de5a96111fc15418819ab4c4432e4f3c2ede61e660b1e33971eba26ef9ba"}, - {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6ef9ea3f137e5711f0dbe5f9263e8c009b7069d8a1acea822bd5e9dae0ae49c8"}, - {file = "greenlet-3.1.1-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85f3ff71e2e60bd4b4932a043fbbe0f499e263c628390b285cb599154a3b03b1"}, - {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:95ffcf719966dd7c453f908e208e14cde192e09fde6c7186c8f1896ef778d8cd"}, - {file = "greenlet-3.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:03a088b9de532cbfe2ba2034b2b85e82df37874681e8c470d6fb2f8c04d7e4b7"}, - {file = "greenlet-3.1.1-cp38-cp38-win32.whl", hash = "sha256:8b8b36671f10ba80e159378df9c4f15c14098c4fd73a36b9ad715f057272fbef"}, - {file = "greenlet-3.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:7017b2be767b9d43cc31416aba48aab0d2309ee31b4dbf10a1d38fb7972bdf9d"}, - {file = "greenlet-3.1.1-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:396979749bd95f018296af156201d6211240e7a23090f50a8d5d18c370084dc3"}, - {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca9d0ff5ad43e785350894d97e13633a66e2b50000e8a183a50a88d834752d42"}, - {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"}, - {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:94ebba31df2aa506d7b14866fed00ac141a867e63143fe5bca82a8e503b36437"}, - {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:73aaad12ac0ff500f62cebed98d8789198ea0e6f233421059fa68a5aa7220145"}, - {file = "greenlet-3.1.1-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:63e4844797b975b9af3a3fb8f7866ff08775f5426925e1e0bbcfe7932059a12c"}, - {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7939aa3ca7d2a1593596e7ac6d59391ff30281ef280d8632fa03d81f7c5f955e"}, - {file = "greenlet-3.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e"}, - {file = "greenlet-3.1.1-cp39-cp39-win32.whl", hash = "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c"}, - {file = "greenlet-3.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22"}, - {file = "greenlet-3.1.1.tar.gz", hash = "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467"}, + {file = "greenlet-3.5.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:29ea813b2e1f45fa9649a17853b2b5465c4072fbcb072e5af6cd3a288216574a"}, + {file = "greenlet-3.5.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:804a70b328e706b785c6ef16187051c394a63dd1a906d89be24b6ad77759f13f"}, + {file = "greenlet-3.5.0-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:884f649de075b84739713d41dd4dfd41e2b910bfb769c4a3ea02ec1da52cd9bb"}, + {file = "greenlet-3.5.0-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4d0eadc7e4d9ffb2af4247b606cae307be8e448911e5a0d0b16d72fc3d224cfd"}, + {file = "greenlet-3.5.0-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4b28037cb07768933c54d81bfe47a85f9f402f57d7d69743b991a713b63954eb"}, + {file = "greenlet-3.5.0-cp310-cp310-manylinux_2_39_riscv64.whl", hash = "sha256:f8c30c2225f40dd76c50790f0eb3b5c7c18431efb299e2782083e1981feed243"}, + {file = "greenlet-3.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cda05425526240807408156b6960a17a79a0c760b813573b67027823be760977"}, + {file = "greenlet-3.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9c615f869163e14bb1ced20322d8038fb680b08236521ac3f30cd4c1288785a0"}, + {file = "greenlet-3.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:ba8f0bdc2fae6ce915dfd0c16d2d00bca7e4247c1eae4416e06430e522137858"}, + {file = "greenlet-3.5.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:8f1cc966c126639cd152fdaa52624d2655f492faa79e013fea161de3e6dda082"}, + {file = "greenlet-3.5.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:362624e6a8e5bca3b8233e45eef33903a100e9539a2b995c364d595dbc4018b3"}, + {file = "greenlet-3.5.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5ecd83806b0f4c2f53b1018e0005cd82269ea01d42befc0368730028d850ed1c"}, + {file = "greenlet-3.5.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:fa94cb2288681e3a11645958f1871d48ee9211bd2f66628fdace505927d6e564"}, + {file = "greenlet-3.5.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ff251e9a0279522e62f6176412869395a64ddf2b5c5f782ff609a8216a4e662"}, + {file = "greenlet-3.5.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:64d6ac45f7271f48e45f67c95b54ef73534c52ec041fcda8edf520c6d811f4bc"}, + {file = "greenlet-3.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6d874e79afd41a96e11ff4c5d0bc90a80973e476fda1c2c64985667397df432b"}, + {file = "greenlet-3.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0ed006e4b86c59de7467eb2601cd1b77b5a7d657d1ee55e30fe30d76451edba4"}, + {file = "greenlet-3.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:703cb211b820dbffbbc55a16bfc6e4583a6e6e990f33a119d2cc8b83211119c8"}, + {file = "greenlet-3.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:6c18dfb59c70f5a94acd271c72e90128c3c776e41e5f07767908c8c1b74ad339"}, + {file = "greenlet-3.5.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:db2910d3c809444e0a20147361f343fe2798e106af8d9d8506f5305302655a9f"}, + {file = "greenlet-3.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3ec9ea74e7268ace7f9aab1b1a4e730193fc661b39a993cd91c606c32d4a3628"}, + {file = "greenlet-3.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:54d243512da35485fc7a6bf3c178fdda6327a9d6506fcdd62b1abd1e41b2927b"}, + {file = "greenlet-3.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:41353ec2ecedf7aa8f682753a41919f8718031a6edac46b8d3dc7ed9e1ceb136"}, + {file = "greenlet-3.5.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9d280a7f5c331622c69f97eb167f33577ff2d1df282c41cd15907fc0a3ca198c"}, + {file = "greenlet-3.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:58c1c374fe2b3d852f9b6b11a7dff4c85404e51b9a596fd9e89cf904eb09866d"}, + {file = "greenlet-3.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1eb67d5adefb5bd2e182d42678a328979a209e4e82eb93575708185d31d1f588"}, + {file = "greenlet-3.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2628d6c86f6cb0cb45e0c3c54058bbec559f57eaae699447748cb3928150577e"}, + {file = "greenlet-3.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:d4d9f0624c775f2dfc56ba54d515a8c771044346852a918b405914f6b19d7fd8"}, + {file = "greenlet-3.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:83ed9f27f1680b50e89f40f6df348a290ea234b249a4003d366663a12eab94f2"}, + {file = "greenlet-3.5.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:5a5ed18de6a0f6cc7087f1563f6bd93fc7df1c19165ca01e9bde5a5dc281d106"}, + {file = "greenlet-3.5.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a717fbc46d8a354fa675f7c1e813485b6ba3885f9bef0cd56e5ba27d758ff5b"}, + {file = "greenlet-3.5.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ddc090c5c1792b10246a78e8c2163ebbe04cf877f9d785c230a7b27b39ad038e"}, + {file = "greenlet-3.5.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4964101b8585c144cbda5532b1aa644255126c08a265dae90c16e7a0e63aaa9d"}, + {file = "greenlet-3.5.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2094acd54b272cb6eae8c03dd87b3fa1820a4cef18d6889c378d503500a1dc13"}, + {file = "greenlet-3.5.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:7022615368890680e67b9965d33f5773aade330d5343bbe25560135aaa849eae"}, + {file = "greenlet-3.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5e05ba267789ea87b5a155cf0e810b1ab88bf18e9e8740813945ceb8ee4350ba"}, + {file = "greenlet-3.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:0ecec963079cd58cbd14723582384f11f166fd58883c15dcbfb342e0bc9b5846"}, + {file = "greenlet-3.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:728d9667d8f2f586644b748dbd9bb67e50d6a9381767d1357714ea6825bb3bf5"}, + {file = "greenlet-3.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:47422135b1d308c14b2c6e758beedb1acd33bb91679f5670edf77bf46244722b"}, + {file = "greenlet-3.5.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:f35807464c4c58c55f0d31dfa83c541a5615d825c2fe3d2b95360cf7c4e3c0a8"}, + {file = "greenlet-3.5.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:55fa7ea52771be44af0de27d8b80c02cd18c2c3cddde6c847ecebdf72418b6a1"}, + {file = "greenlet-3.5.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a97e4821aa710603f94de0da25f25096454d78ffdace5dc77f3a006bc01abba3"}, + {file = "greenlet-3.5.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bf2d8a80bec89ab46221ae45c5373d5ba0bd36c19aa8508e85c6cd7e5106cd37"}, + {file = "greenlet-3.5.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f52a464e4ed91780bdfbbdd2b97197f3accaa629b98c200f4dffada759f3ae7"}, + {file = "greenlet-3.5.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:1bae92a1dd94c5f9d9493c3a212dd874c202442047cf96446412c862feca83a2"}, + {file = "greenlet-3.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:762612baf1161ccb8437c0161c668a688223cba28e1bf038f4eb47b13e39ccdf"}, + {file = "greenlet-3.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:57a43c6079a89713522bc4bcb9f75070ecf5d3dbad7792bfe42239362cbf2a16"}, + {file = "greenlet-3.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:3bc59be3945ae9750b9e7d45067d01ae3fe90ea5f9ade99239dabdd6e28a5033"}, + {file = "greenlet-3.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:a96fcee45e03fe30a62669fd16ab5c9d3c172660d3085605cb1e2d1280d3c988"}, + {file = "greenlet-3.5.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:a10a732421ab4fec934783ce3e54763470d0181db6e3468f9103a275c3ed1853"}, + {file = "greenlet-3.5.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7fc391b1566f2907d17aaebe78f8855dc45675159a775fcf9e61f8ee0078e87f"}, + {file = "greenlet-3.5.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:680bd0e7ad5e8daa8a4aa89f68fd6adc834b8a8036dc256533f7e08f4a4b01f7"}, + {file = "greenlet-3.5.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1aa4ce8debcd4ea7fb2e150f3036588c41493d1d52c43538924ae1819003f4ce"}, + {file = "greenlet-3.5.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ddb36c7d6c9c0a65f18c7258634e0c416c6ab59caac8c987b96f80c2ebda0112"}, + {file = "greenlet-3.5.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:728a73687e39ae9ca34e4694cbf2f049d3fbc7174639468d0f67200a97d8f9e2"}, + {file = "greenlet-3.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e5ddf316ced87539144621453c3aef229575825fe60c604e62bedc4003f372b2"}, + {file = "greenlet-3.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4a448128607be0de65342dc9b31be7f948ef4cc0bc8832069350abefd310a8f2"}, + {file = "greenlet-3.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d60097128cb0a1cab9ea541186ea13cd7b847b8449a7787c2e2350da0cb82d86"}, + {file = "greenlet-3.5.0.tar.gz", hash = "sha256:d419647372241bc68e957bf38d5c1f98852155e4146bd1e4121adea81f4f01e4"}, ] [package.extras] docs = ["Sphinx", "furo"] -test = ["objgraph", "psutil"] - -[[package]] -name = "importlib-resources" -version = "6.4.5" -description = "Read resources from Python packages" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.8\"" -files = [ - {file = "importlib_resources-6.4.5-py3-none-any.whl", hash = "sha256:ac29d5f956f01d5e4bb63102a5a19957f1b9175e45649977264a1416783bb717"}, - {file = "importlib_resources-6.4.5.tar.gz", hash = "sha256:980862a1d16c9e147a59603677fa2aa5fd82b87f223b6cb870695bcfce830065"}, -] - -[package.dependencies] -zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} - -[package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=2.2)"] -test = ["jaraco.test (>=5.4)", "pytest (>=6,!=8.1.*)", "zipp (>=3.17)"] -type = ["pytest-mypy"] - -[[package]] -name = "jsonpath-ng" -version = "1.6.0" -description = "A final implementation of JSONPath for Python that aims to be standard compliant, including arithmetic and binary comparison operators and providing clear AST for metaprogramming." -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "jsonpath-ng-1.6.0.tar.gz", hash = "sha256:5483f8e9d74c39c9abfab554c070ae783c1c8cbadf5df60d561bc705ac68a07e"}, - {file = "jsonpath_ng-1.6.0-py3-none-any.whl", hash = "sha256:6fd04833412c4b3d9299edf369542f5e67095ca84efa17cbb7f06a34958adc9f"}, -] - -[package.dependencies] -ply = "*" +test = ["objgraph", "psutil", "setuptools"] [[package]] name = "jsonpath-ng" @@ -484,7 +372,6 @@ description = "A final implementation of JSONPath for Python that aims to be sta optional = false python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "jsonpath-ng-1.6.1.tar.gz", hash = "sha256:086c37ba4917304850bd837aeab806670224d3f038fe2833ff593a672ef0a5fa"}, {file = "jsonpath_ng-1.6.1-py3-none-any.whl", hash = "sha256:8f22cd8273d7772eea9aaa84d922e0841aa36fdb8a2c6b7f6c3791a16a9bc0be"}, @@ -557,19 +444,6 @@ files = [ [package.dependencies] dill = ">=0.3.9" -[[package]] -name = "overrides" -version = "7.4.0" -description = "A decorator to automatically detect mismatch when overriding a method." -optional = false -python-versions = ">=3.6" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "overrides-7.4.0-py3-none-any.whl", hash = "sha256:3ad24583f86d6d7a49049695efe9933e67ba62f0c7625d53c59fa832ce4b8b7d"}, - {file = "overrides-7.4.0.tar.gz", hash = "sha256:9502a3cca51f4fac40b5feca985b6703a5c1f6ad815588a7ca9e285b9dca6757"}, -] - [[package]] name = "overrides" version = "7.7.0" @@ -577,7 +451,6 @@ description = "A decorator to automatically detect mismatch when overriding a me optional = false python-versions = ">=3.6" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "overrides-7.7.0-py3-none-any.whl", hash = "sha256:c7ed9d062f78b8e4c1a7b70bd8796b35ead4d9f510227ef9c5dc7626c60d7e49"}, {file = "overrides-7.7.0.tar.gz", hash = "sha256:55158fa3d93b98cc75299b1e67078ad9003ca27945c76162c1c0766d6f91820a"}, @@ -595,75 +468,6 @@ files = [ {file = "pefile-2024.8.26.tar.gz", hash = "sha256:3ff6c5d8b43e8c37bb6e6dd5085658d658a7a0bdcd20b6a07b1fcfc1c4e9d632"}, ] -[[package]] -name = "pillow" -version = "10.0.1" -description = "Python Imaging Library (Fork)" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "Pillow-10.0.1-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:8f06be50669087250f319b706decf69ca71fdecd829091a37cc89398ca4dc17a"}, - {file = "Pillow-10.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:50bd5f1ebafe9362ad622072a1d2f5850ecfa44303531ff14353a4059113b12d"}, - {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6a90167bcca1216606223a05e2cf991bb25b14695c518bc65639463d7db722d"}, - {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f11c9102c56ffb9ca87134bd025a43d2aba3f1155f508eff88f694b33a9c6d19"}, - {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:186f7e04248103482ea6354af6d5bcedb62941ee08f7f788a1c7707bc720c66f"}, - {file = "Pillow-10.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:0462b1496505a3462d0f35dc1c4d7b54069747d65d00ef48e736acda2c8cbdff"}, - {file = "Pillow-10.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d889b53ae2f030f756e61a7bff13684dcd77e9af8b10c6048fb2c559d6ed6eaf"}, - {file = "Pillow-10.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:552912dbca585b74d75279a7570dd29fa43b6d93594abb494ebb31ac19ace6bd"}, - {file = "Pillow-10.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:787bb0169d2385a798888e1122c980c6eff26bf941a8ea79747d35d8f9210ca0"}, - {file = "Pillow-10.0.1-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:fd2a5403a75b54661182b75ec6132437a181209b901446ee5724b589af8edef1"}, - {file = "Pillow-10.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2d7e91b4379f7a76b31c2dda84ab9e20c6220488e50f7822e59dac36b0cd92b1"}, - {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:19e9adb3f22d4c416e7cd79b01375b17159d6990003633ff1d8377e21b7f1b21"}, - {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93139acd8109edcdeffd85e3af8ae7d88b258b3a1e13a038f542b79b6d255c54"}, - {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:92a23b0431941a33242b1f0ce6c88a952e09feeea9af4e8be48236a68ffe2205"}, - {file = "Pillow-10.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:cbe68deb8580462ca0d9eb56a81912f59eb4542e1ef8f987405e35a0179f4ea2"}, - {file = "Pillow-10.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:522ff4ac3aaf839242c6f4e5b406634bfea002469656ae8358644fc6c4856a3b"}, - {file = "Pillow-10.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:84efb46e8d881bb06b35d1d541aa87f574b58e87f781cbba8d200daa835b42e1"}, - {file = "Pillow-10.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:898f1d306298ff40dc1b9ca24824f0488f6f039bc0e25cfb549d3195ffa17088"}, - {file = "Pillow-10.0.1-cp312-cp312-macosx_10_10_x86_64.whl", hash = "sha256:bcf1207e2f2385a576832af02702de104be71301c2696d0012b1b93fe34aaa5b"}, - {file = "Pillow-10.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5d6c9049c6274c1bb565021367431ad04481ebb54872edecfcd6088d27edd6ed"}, - {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28444cb6ad49726127d6b340217f0627abc8732f1194fd5352dec5e6a0105635"}, - {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de596695a75496deb3b499c8c4f8e60376e0516e1a774e7bc046f0f48cd620ad"}, - {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:2872f2d7846cf39b3dbff64bc1104cc48c76145854256451d33c5faa55c04d1a"}, - {file = "Pillow-10.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:4ce90f8a24e1c15465048959f1e94309dfef93af272633e8f37361b824532e91"}, - {file = "Pillow-10.0.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ee7810cf7c83fa227ba9125de6084e5e8b08c59038a7b2c9045ef4dde61663b4"}, - {file = "Pillow-10.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:b1be1c872b9b5fcc229adeadbeb51422a9633abd847c0ff87dc4ef9bb184ae08"}, - {file = "Pillow-10.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:98533fd7fa764e5f85eebe56c8e4094db912ccbe6fbf3a58778d543cadd0db08"}, - {file = "Pillow-10.0.1-cp38-cp38-macosx_10_10_x86_64.whl", hash = "sha256:764d2c0daf9c4d40ad12fbc0abd5da3af7f8aa11daf87e4fa1b834000f4b6b0a"}, - {file = "Pillow-10.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:fcb59711009b0168d6ee0bd8fb5eb259c4ab1717b2f538bbf36bacf207ef7a68"}, - {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:697a06bdcedd473b35e50a7e7506b1d8ceb832dc238a336bd6f4f5aa91a4b500"}, - {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f665d1e6474af9f9da5e86c2a3a2d2d6204e04d5af9c06b9d42afa6ebde3f21"}, - {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:2fa6dd2661838c66f1a5473f3b49ab610c98a128fc08afbe81b91a1f0bf8c51d"}, - {file = "Pillow-10.0.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:3a04359f308ebee571a3127fdb1bd01f88ba6f6fb6d087f8dd2e0d9bff43f2a7"}, - {file = "Pillow-10.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:723bd25051454cea9990203405fa6b74e043ea76d4968166dfd2569b0210886a"}, - {file = "Pillow-10.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:71671503e3015da1b50bd18951e2f9daf5b6ffe36d16f1eb2c45711a301521a7"}, - {file = "Pillow-10.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:44e7e4587392953e5e251190a964675f61e4dae88d1e6edbe9f36d6243547ff3"}, - {file = "Pillow-10.0.1-cp39-cp39-macosx_10_10_x86_64.whl", hash = "sha256:3855447d98cced8670aaa63683808df905e956f00348732448b5a6df67ee5849"}, - {file = "Pillow-10.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ed2d9c0704f2dc4fa980b99d565c0c9a543fe5101c25b3d60488b8ba80f0cce1"}, - {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f5bb289bb835f9fe1a1e9300d011eef4d69661bb9b34d5e196e5e82c4cb09b37"}, - {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a0d3e54ab1df9df51b914b2233cf779a5a10dfd1ce339d0421748232cea9876"}, - {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:2cc6b86ece42a11f16f55fe8903595eff2b25e0358dec635d0a701ac9586588f"}, - {file = "Pillow-10.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:ca26ba5767888c84bf5a0c1a32f069e8204ce8c21d00a49c90dabeba00ce0145"}, - {file = "Pillow-10.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f0b4b06da13275bc02adfeb82643c4a6385bd08d26f03068c2796f60d125f6f2"}, - {file = "Pillow-10.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bc2e3069569ea9dbe88d6b8ea38f439a6aad8f6e7a6283a38edf61ddefb3a9bf"}, - {file = "Pillow-10.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:8b451d6ead6e3500b6ce5c7916a43d8d8d25ad74b9102a629baccc0808c54971"}, - {file = "Pillow-10.0.1-pp310-pypy310_pp73-macosx_10_10_x86_64.whl", hash = "sha256:32bec7423cdf25c9038fef614a853c9d25c07590e1a870ed471f47fb80b244db"}, - {file = "Pillow-10.0.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7cf63d2c6928b51d35dfdbda6f2c1fddbe51a6bc4a9d4ee6ea0e11670dd981e"}, - {file = "Pillow-10.0.1-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:f6d3d4c905e26354e8f9d82548475c46d8e0889538cb0657aa9c6f0872a37aa4"}, - {file = "Pillow-10.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:847e8d1017c741c735d3cd1883fa7b03ded4f825a6e5fcb9378fd813edee995f"}, - {file = "Pillow-10.0.1-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:7f771e7219ff04b79e231d099c0a28ed83aa82af91fd5fa9fdb28f5b8d5addaf"}, - {file = "Pillow-10.0.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:459307cacdd4138edee3875bbe22a2492519e060660eaf378ba3b405d1c66317"}, - {file = "Pillow-10.0.1-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:b059ac2c4c7a97daafa7dc850b43b2d3667def858a4f112d1aa082e5c3d6cf7d"}, - {file = "Pillow-10.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6caf3cd38449ec3cd8a68b375e0c6fe4b6fd04edb6c9766b55ef84a6e8ddf2d"}, - {file = "Pillow-10.0.1.tar.gz", hash = "sha256:d72967b06be9300fed5cfbc8b5bafceec48bf7cdc7dab66b1d2549035287191d"}, -] - -[package.extras] -docs = ["furo", "olefile", "sphinx (>=2.4)", "sphinx-copybutton", "sphinx-inline-tabs", "sphinx-removed-in", "sphinxext-opengraph"] -tests = ["check-manifest", "coverage", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout"] - [[package]] name = "pillow" version = "10.4.0" @@ -671,7 +475,6 @@ description = "Python Imaging Library (Fork)" optional = false python-versions = ">=3.8" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "pillow-10.4.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:4d9667937cfa347525b319ae34375c37b9ee6b525440f3ef48542fcf66f2731e"}, {file = "pillow-10.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:543f3dc61c18dafb755773efc89aae60d06b6596a63914107f75459cf984164d"}, @@ -815,19 +618,6 @@ files = [ {file = "pyelftools-0.32.tar.gz", hash = "sha256:6de90ee7b8263e740c8715a925382d4099b354f29ac48ea40d840cf7aa14ace5"}, ] -[[package]] -name = "pyfiglet" -version = "0.8.post1" -description = "Pure-python FIGlet implementation" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "pyfiglet-0.8.post1-py2.py3-none-any.whl", hash = "sha256:d555bcea17fbeaf70eaefa48bb119352487e629c9b56f30f383e2c62dd67a01c"}, - {file = "pyfiglet-0.8.post1.tar.gz", hash = "sha256:c6c2321755d09267b438ec7b936825a4910fec696292139e664ca8670e103639"}, -] - [[package]] name = "pyfiglet" version = "1.0.2" @@ -835,24 +625,11 @@ description = "Pure-python FIGlet implementation" optional = false python-versions = ">=3.9" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "pyfiglet-1.0.2-py3-none-any.whl", hash = "sha256:889b351d79c99e50a3f619c8f8e6ffdb27fd8c939fc43ecbd7559bd57d5f93ea"}, {file = "pyfiglet-1.0.2.tar.gz", hash = "sha256:758788018ab8faaddc0984e1ea05ff330d3c64be663c513cc1f105f6a3066dab"}, ] -[[package]] -name = "pyperclip" -version = "1.8.2" -description = "A cross-platform clipboard module for Python. (Only handles plain text for now.)" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "pyperclip-1.8.2.tar.gz", hash = "sha256:105254a8b04934f0bc84e9c24eb360a591aaf6535c9def5f29d92af107a9bf57"}, -] - [[package]] name = "pyperclip" version = "1.9.0" @@ -860,43 +637,10 @@ description = "A cross-platform clipboard module for Python. (Only handles plain optional = false python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "pyperclip-1.9.0.tar.gz", hash = "sha256:b7de0142ddc81bfc5c7507eea19da920b92252b548b96186caf94a5e2527d310"}, ] -[[package]] -name = "python-fx" -version = "0.3.1" -description = "A python-native fx-alike terminal JSON viewer." -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "python-fx-0.3.1.tar.gz", hash = "sha256:76044ba32195b8e0ce444aa714981cb1481f9df44c3381c5ed4b43a4f6812c73"}, - {file = "python_fx-0.3.1-py3-none-any.whl", hash = "sha256:e7cfbb8421831aaff5684dd1ae6ec855b92bcd089f9f76b06a3f2baa4670447a"}, -] - -[package.dependencies] -antlr4-python3-runtime = "4.8" -asciimatics = "1.14.0" -click = {version = "8.1.7", markers = "python_version >= \"3.7\""} -dacite = {version = "1.8.1", markers = "python_version >= \"3.6\""} -first = "2.0.2" -future = {version = "0.18.3", markers = "python_version >= \"2.6\" and python_version not in \"3.0, 3.1, 3.2, 3.3\""} -jsonpath-ng = "1.6.0" -loguru = {version = "0.7.2", markers = "python_version >= \"3.5\""} -overrides = {version = "7.4.0", markers = "python_version >= \"3.6\""} -pillow = {version = "10.0.1", markers = "python_version >= \"3.8\""} -ply = "3.11" -pyfiglet = {version = "0.8.post1", markers = "python_version >= \"3.9\""} -pyperclip = "1.8.2" -pyyaml = {version = "6.0.1", markers = "python_version >= \"3.6\""} -urwid = {version = "2.2.1", markers = "python_full_version >= \"3.7.0\""} -wcwidth = "0.2.6" -yamale = {version = "4.0.4", markers = "python_version >= \"3.6\""} - [[package]] name = "python-fx" version = "0.3.2" @@ -904,7 +648,6 @@ description = "A python-native fx-alike terminal JSON viewer." optional = false python-versions = ">=3.8" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "python_fx-0.3.2-py3-none-any.whl", hash = "sha256:5498475b0f391b1649732328b58d188d9fc4b3f90f5bfb77d5c6e2ece2432c5f"}, {file = "python_fx-0.3.2.tar.gz", hash = "sha256:9646f58c716e2db6698bff3dfa55fa721b8b0cb741506287a87bc08055a96ceb"}, @@ -975,68 +718,6 @@ files = [ {file = "pywin32-308-cp39-cp39-win_amd64.whl", hash = "sha256:71b3322d949b4cc20776436a9c9ba0eeedcbc9c650daa536df63f0ff111bb920"}, ] -[[package]] -name = "pyyaml" -version = "6.0.1" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.6" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "PyYAML-6.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a"}, - {file = "PyYAML-6.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, - {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, - {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, - {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, - {file = "PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, - {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, - {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, - {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, - {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, - {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win32.whl", hash = "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585"}, - {file = "PyYAML-6.0.1-cp36-cp36m-win_amd64.whl", hash = "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa"}, - {file = "PyYAML-6.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3"}, - {file = "PyYAML-6.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win32.whl", hash = "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba"}, - {file = "PyYAML-6.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867"}, - {file = "PyYAML-6.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, - {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, - {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, - {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, - {file = "PyYAML-6.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, - {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, - {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, - {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, - {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, -] - [[package]] name = "pyyaml" version = "6.0.2" @@ -1044,7 +725,6 @@ description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -1173,7 +853,6 @@ description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, @@ -1273,9 +952,6 @@ files = [ {file = "unicorn-2.1.3.tar.gz", hash = "sha256:0c06456cf550c228f2003cc70366afa4aece2e6e7e4c32d8f4b22c717ba6b729"}, ] -[package.dependencies] -importlib_resources = {version = "*", markers = "python_version < \"3.9\""} - [package.extras] test = ["capstone (==5.0.1) ; python_version <= \"3.7\"", "capstone (==6.0.0a2) ; python_version > \"3.7\""] @@ -1299,52 +975,6 @@ files = [ [package.dependencies] unicorn = ">=2.0.1" -[[package]] -name = "urwid" -version = "2.2.1" -description = "A full-featured console (xterm et al.) user interface library" -optional = false -python-versions = ">=3.7.0" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "urwid-2.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7aa95e2f8941e323f0534a301f9d8d965d869110d326b3c9dff63e1c116772cd"}, - {file = "urwid-2.2.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ea22e5eabace2c66e2f52cc2494308c1a0091bcb89b3ceedf72ff91733f4dbb2"}, - {file = "urwid-2.2.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:07b88bdc8efe95b318201bd054ae69bed68bb9f506f127b21ae7234ffb7db3a4"}, - {file = "urwid-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bba58f24fdff58975ef603ee803909d57faaaebd407fd50042652dcc9a8dd2f2"}, - {file = "urwid-2.2.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab2d5704cbd32f729a60d2b56d076e16652b3b97ebe6773c54a192cb9f49c169"}, - {file = "urwid-2.2.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bc88edae9cb34644905e3d00e9fd2dc9a2c1eaeb2e311c1aec0d36a51d77b10"}, - {file = "urwid-2.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2fa3e83730a811466d272ab68340f9f9418ee7ca5f6de3548dd7a5661eafbbee"}, - {file = "urwid-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2f87d7efbbf1d716bbcf025d453b3481aa0d9e1c91581aa8edc9ae7af64efa85"}, - {file = "urwid-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bdafd72d6a539e16e6c179dd16609601643b85edf97b1543fc208e4fb7e6c249"}, - {file = "urwid-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e17eed4190220873531e2c11c885764d2e3bcabe9e35d5a578e84056a2c58199"}, - {file = "urwid-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7949a8d4384d170e4dfa41151e6264f6238b3ba2520649c25110bc6451978568"}, - {file = "urwid-2.2.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3c96c57714fd0eac79a5e8a9d38d15d68ab9b6a96c2fe282ccd61cb707dd4be2"}, - {file = "urwid-2.2.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:1bee2f966063de86a093908abb5bd56910e5d5630e021b351f240ab3b972207e"}, - {file = "urwid-2.2.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:57e28adf4457fb50b751838836bb94122e904ccad4429d42c4f318a3287a4802"}, - {file = "urwid-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ad5ff26140b66ebb69957c51fd168a86f212adb578c83ed590ffd7e032da973f"}, - {file = "urwid-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:653b1fb9c52c4a32c326f701dd3ceb6edd1f30f32033c040fd5edc55d3d60cdb"}, - {file = "urwid-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4438be6b4b303d1012b8208cf5ff2ef71bdc19a7732771257ece36e2d1d16283"}, - {file = "urwid-2.2.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7868c2cfd3fadd6cf42a4a2dbb2cf87a92d6c12dc5ed8b991ff96e66f0ba8c38"}, - {file = "urwid-2.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c3f00ca72de0736f3df4971a01d2278065628bef179fbd4fce37aacf93bbeb2c"}, - {file = "urwid-2.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c5354319e3ac4e612a4a280421e21fa4243023df73e48ee701e4e944e769d87c"}, - {file = "urwid-2.2.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5f3fdbd58a3ef1f19393a5f1d8b61adcfc89d0e235a2e05927cbecbf8012120c"}, - {file = "urwid-2.2.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:55d13d1c6f15bd594a49ad165837edf34678c7c2362834f0d771990821e3bb8c"}, - {file = "urwid-2.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:93f081f1c53d7d307694ae20eb07ac731b4337d517dd6ee9dd91bae78bcb67bf"}, - {file = "urwid-2.2.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d42a9e4939d18c77f73183593a589c9b3b5d5fa3615d94a32e15cd97b00d3536"}, - {file = "urwid-2.2.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5eeb4e1760e9356471f8b50e3296c24881e242aae57f738a6f8534438848fb2e"}, - {file = "urwid-2.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d894543b4f3f3f2ce9837782e45cc3797df0a1697264a2939a2391076d07f641"}, - {file = "urwid-2.2.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3205f486e9fa4c6193aa5b9623abe05db864465acd02825305702849572c0828"}, - {file = "urwid-2.2.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ccfbbc37565f24156e2bdb37504b010fedee8f4a70cfc353c0d9782354087484"}, - {file = "urwid-2.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4add23f02ef43497c13592e1804640b8b19fe781e8d62f445000c7acca60e2e2"}, - {file = "urwid-2.2.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:476f03705095fed744413d8256c7da998694b2f81e7e6e665e6244d1d3159d1e"}, - {file = "urwid-2.2.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:85614e0436eb8c77bb21950f7d52bf93668b4ba8a11a2986bc111b48d28390f6"}, - {file = "urwid-2.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf09c272b46ec0f78b6eaf2a515ded30e952a0e77dbbb3535593d3f05354eb82"}, - {file = "urwid-2.2.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d8de3025ee488d9f56db9e8842d6c7f1c290e01fd6749320d8c171fef5cbef35"}, - {file = "urwid-2.2.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2eea5fba3dab9f0977fcf17608370686da9d18a6077fd180a98eba72c59ff5d"}, - {file = "urwid-2.2.1.tar.gz", hash = "sha256:e33911ab18f2c73fddbe9bf216d021e74e20b2d5aa9be30403c58f55131bb8a1"}, -] - [[package]] name = "urwid" version = "2.6.15" @@ -1352,7 +982,6 @@ description = "A full-featured console (xterm et al.) user interface library" optional = false python-versions = ">3.7" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "urwid-2.6.15-py3-none-any.whl", hash = "sha256:71b3171cabaa0092902f556768756bd2f2ebb24c0da287ee08f081d235340cb7"}, {file = "urwid-2.6.15.tar.gz", hash = "sha256:9ecc57330d88c8d9663ffd7092a681674c03ff794b6330ccfef479af7aa9671b"}, @@ -1372,19 +1001,6 @@ trio = ["exceptiongroup", "trio (>=0.22.0)"] twisted = ["twisted"] zmq = ["zmq"] -[[package]] -name = "wcwidth" -version = "0.2.6" -description = "Measures the displayed width of unicode strings in a terminal" -optional = false -python-versions = "*" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "wcwidth-0.2.6-py2.py3-none-any.whl", hash = "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e"}, - {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"}, -] - [[package]] name = "wcwidth" version = "0.2.13" @@ -1392,7 +1008,6 @@ description = "Measures the displayed width of unicode strings in a terminal" optional = false python-versions = "*" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, @@ -1441,22 +1056,6 @@ files = [ {file = "windows_curses-2.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:4588213f7ef3b0c24c5cb9e309653d7a84c1792c707561e8b471d466ca79f2b8"}, ] -[[package]] -name = "yamale" -version = "4.0.4" -description = "A schema and validator for YAML." -optional = false -python-versions = ">=3.6" -groups = ["main"] -markers = "python_version < \"3.11\" or platform_python_implementation != \"CPython\"" -files = [ - {file = "yamale-4.0.4-py3-none-any.whl", hash = "sha256:04f914c0886bda03ac20f8468272cfd9374a634a062549490eff2beedeb30497"}, - {file = "yamale-4.0.4.tar.gz", hash = "sha256:e524caf71cbbbd15aa295e8bdda01688ac4b5edaf38dd60851ddff6baef383ba"}, -] - -[package.dependencies] -pyyaml = "*" - [[package]] name = "yamale" version = "5.2.1" @@ -1464,7 +1063,6 @@ description = "A schema and validator for YAML." optional = false python-versions = ">=3.8" groups = ["main"] -markers = "platform_python_implementation == \"CPython\" and python_version >= \"3.11\"" files = [ {file = "yamale-5.2.1-py3-none-any.whl", hash = "sha256:e44cd30cf3055ee4b34c1c71d6fe35490a127dcbd36f82f27859d105a9989922"}, {file = "yamale-5.2.1.tar.gz", hash = "sha256:19bbe713d588f07177bc519a46070c0793ed126ea37f425a76055b99703f835a"}, @@ -1473,27 +1071,6 @@ files = [ [package.dependencies] pyyaml = "*" -[[package]] -name = "zipp" -version = "3.20.2" -description = "Backport of pathlib-compatible object wrapper for zip files" -optional = false -python-versions = ">=3.8" -groups = ["main"] -markers = "python_version == \"3.8\"" -files = [ - {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, - {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, -] - -[package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] -cover = ["pytest-cov"] -doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -enabler = ["pytest-enabler (>=2.2)"] -test = ["big-O", "importlib-resources ; python_version < \"3.9\"", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] -type = ["pytest-mypy"] - [[package]] name = "zope-event" version = "5.0" @@ -1574,5 +1151,5 @@ re = ["r2libr"] [metadata] lock-version = "2.1" -python-versions = "^3.8" -content-hash = "9191e91c28fe22a05ac8185ae5fd4a1fc969cecad63b41d649981dfbc50e86ae" +python-versions = "^3.10" +content-hash = "b65631e55580e06f60d535d41d2c2d0c4ee786609c7232988ceda8ce6b586a6d" diff --git a/pyproject.toml b/pyproject.toml index 9b2b7b75..75079b8b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,14 +27,14 @@ keywords = [ ] [tool.poetry.dependencies] -python = "^3.8" +python = "^3.10" capstone = "^5" unicorn = "2.1.3" pefile = ">=2022.5.30" python-registry = "^1.3.1" keystone-engine = "^0.9.2" pyelftools = ">=0.28" -gevent = ">=20.9.0" +gevent = ">=24.10" multiprocess = ">=0.70.12.2" windows-curses = { version = "^2.1.0", markers = "platform_system == 'Windows'" } pyyaml = "^6.0.1" From 22d109d882490ba6e74843f48c8e15dfc33f3758 Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 13:11:17 +0800 Subject: [PATCH 131/138] Fix windows collector --- examples/scripts/dllscollector.bat | 2 ++ qiling/os/path.py | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/examples/scripts/dllscollector.bat b/examples/scripts/dllscollector.bat index b0707bba..09dea2a6 100644 --- a/examples/scripts/dllscollector.bat +++ b/examples/scripts/dllscollector.bat @@ -80,6 +80,7 @@ CALL :collect_dll32 setupapi.dll CALL :collect_dll32 shell32.dll CALL :collect_dll32 shlwapi.dll CALL :collect_dll32 sspicli.dll +CALL :collect_dll32 symcryptk.dll CALL :collect_dll32 ucrtbase.dll CALL :collect_dll32 ucrtbased.dll CALL :collect_dll32 urlmon.dll @@ -125,6 +126,7 @@ CALL :collect_dll64 rpcrt4.dll CALL :collect_dll64 sechost.dll CALL :collect_dll64 shell32.dll CALL :collect_dll64 shlwapi.dll +CALL :collect_dll64 symcryptk.dll CALL :collect_dll64 user32.dll CALL :collect_dll64 vcruntime140.dll CALL :collect_dll64 vcruntime140d.dll diff --git a/qiling/os/path.py b/qiling/os/path.py index 61b3c8cd..55d5bf4a 100644 --- a/qiling/os/path.py +++ b/qiling/os/path.py @@ -3,6 +3,8 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # +import os + from typing import Optional, Union from pathlib import Path, PurePosixPath, PureWindowsPath @@ -328,7 +330,11 @@ class QlOsPath: @staticmethod def __host_casefold_path(hostpath: str) -> Optional[str]: - # assuming posix host + # NT hosts already match paths case-insensitively, so if the caller landed here + # (after a failed os.path.exists) the file really does not exist. + if os.name == 'nt': + return None + p = PurePosixPath(hostpath) norm = Path(p.anchor) From 3a8d37168d5e477d2f0c6a26bff7e1d8ad0cd451 Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 13:52:50 +0800 Subject: [PATCH 132/138] Fix MCU wait --- qiling/os/mcu/mcu.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/qiling/os/mcu/mcu.py b/qiling/os/mcu/mcu.py index 3e84dbdd..270ab0bf 100644 --- a/qiling/os/mcu/mcu.py +++ b/qiling/os/mcu/mcu.py @@ -27,9 +27,13 @@ class MCUTask(UnicornTask): def on_interrupted(self, ucerr: int): self._begin = self.pc - # And don't restore anything. + # Real silicon would vector to HardFault here; we don't model that, so log and + # request a clean stop instead of raising — otherwise fast-mode timeouts surface + # firmware faults as test errors. if ucerr != UC_ERR_OK: - raise UcError(ucerr) + self.ql.log.warning(f'fast mode halted: {UcError(ucerr)} at PC=0x{self.pc:08x}') + self._stop_request = True + return self.ql.hw.step() From ef898e68d73a17e3f7fec266309be52a9118cc2e Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 15:03:09 +0800 Subject: [PATCH 133/138] Version 1.4.8 --- pyproject.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9b2b7b75..c190e2fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,10 +7,8 @@ authors = ["KaiJern Lau (xwings) "] license = "GPLv2" readme = "README.md" classifiers = [ - "Development Status :: 3 - Beta", - + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", - "Topic :: Software Development :: Disassemblers", "Topic :: System :: Emulators", "Topic :: System :: Operating System", From d18027e7efe3bc0c35e56341906fb60609011d3d Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 15:06:26 +0800 Subject: [PATCH 134/138] update rootfs branch --- examples/rootfs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/rootfs b/examples/rootfs index 6d4d654f..df3fa4df 160000 --- a/examples/rootfs +++ b/examples/rootfs @@ -1 +1 @@ -Subproject commit 6d4d654fdc2892490d98c433eca3efa5c6d062c7 +Subproject commit df3fa4dfc0b9d4164f8678699d8923df847eb3d2 From b6ced6fa214cd64918ea90f4892d4bd7222ae5e8 Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 15:51:49 +0800 Subject: [PATCH 135/138] update rootfs branch --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c190e2fc..6bed92ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] urls = { homepage = "https://qiling.io" } name = "qiling" -version = "1.4.8" +version = "1.4.9" description = "Qiling is an advanced binary emulation framework that cross-platform-architecture" authors = ["KaiJern Lau (xwings) "] license = "GPLv2" From f5d7a745a32576a48796aa595dedf2c896295b2f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Apr 2026 07:56:51 +0000 Subject: [PATCH 136/138] Bump setuptools from 75.3.0 to 78.1.1 Bumps [setuptools](https://github.com/pypa/setuptools) from 75.3.0 to 78.1.1. - [Release notes](https://github.com/pypa/setuptools/releases) - [Changelog](https://github.com/pypa/setuptools/blob/main/NEWS.rst) - [Commits](https://github.com/pypa/setuptools/compare/v75.3.0...v78.1.1) --- updated-dependencies: - dependency-name: setuptools dependency-version: 78.1.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/poetry.lock b/poetry.lock index 0a7dfe3a..2d99b473 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "antlr4-python3-runtime" @@ -413,7 +413,7 @@ colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] -dev = ["Sphinx (==7.2.5) ; python_version >= \"3.9\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.2.2) ; python_version >= \"3.8\"", "mypy (==0.910) ; python_version < \"3.6\"", "mypy (==0.971) ; python_version == \"3.6\"", "mypy (==1.4.1) ; python_version == \"3.7\"", "mypy (==1.5.1) ; python_version >= \"3.8\"", "pre-commit (==3.4.0) ; python_version >= \"3.8\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==7.4.0) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==4.1.0) ; python_version >= \"3.8\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.0.0) ; python_version >= \"3.8\"", "sphinx-autobuild (==2021.3.14) ; python_version >= \"3.9\"", "sphinx-rtd-theme (==1.3.0) ; python_version >= \"3.9\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.11.0) ; python_version >= \"3.8\""] +dev = ["Sphinx (==7.2.5) ; python_version >= \"3.9\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.2.2) ; python_version >= \"3.8\"", "mypy (==v0.910) ; python_version < \"3.6\"", "mypy (==v0.971) ; python_version == \"3.6\"", "mypy (==v1.4.1) ; python_version == \"3.7\"", "mypy (==v1.5.1) ; python_version >= \"3.8\"", "pre-commit (==3.4.0) ; python_version >= \"3.8\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==7.4.0) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==4.1.0) ; python_version >= \"3.8\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.0.0) ; python_version >= \"3.8\"", "sphinx-autobuild (==2021.3.14) ; python_version >= \"3.9\"", "sphinx-rtd-theme (==1.3.0) ; python_version >= \"3.9\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.11.0) ; python_version >= \"3.8\""] [[package]] name = "multiprocess" @@ -812,24 +812,24 @@ files = [ [[package]] name = "setuptools" -version = "75.3.0" +version = "78.1.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "setuptools-75.3.0-py3-none-any.whl", hash = "sha256:f2504966861356aa38616760c0f66568e535562374995367b4e69c7143cf6bcd"}, - {file = "setuptools-75.3.0.tar.gz", hash = "sha256:fba5dd4d766e97be1b1681d98712680ae8f2f26d7881245f2ce9e40714f1a686"}, + {file = "setuptools-78.1.1-py3-none-any.whl", hash = "sha256:c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561"}, + {file = "setuptools-78.1.1.tar.gz", hash = "sha256:fcc17fd9cd898242f6b4adfaca46137a9edef687f43e6f78469692a5e70d851d"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.5.2) ; sys_platform != \"cygwin\""] -core = ["importlib-metadata (>=6) ; python_version < \"3.10\"", "importlib-resources (>=5.10.2) ; python_version < \"3.9\"", "jaraco.collections", "jaraco.functools", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=23.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib-metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.12.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "termcolor" From da210f0757f3581de7e607b2b826b26eaa5aef66 Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 15:58:29 +0800 Subject: [PATCH 137/138] Update pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0eafdfd2..efaa4734 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] urls = { homepage = "https://qiling.io" } name = "qiling" -version = "1.4.9" +version = "1.4.10" description = "Qiling is an advanced binary emulation framework that cross-platform-architecture" authors = ["KaiJern Lau (xwings) "] license = "GPLv2" From a42fb2857955165f1c9e789d1888293e0a2368ea Mon Sep 17 00:00:00 2001 From: xwings Date: Tue, 28 Apr 2026 16:02:08 +0800 Subject: [PATCH 138/138] Bump version to 1.4.11-dev and update classifiers --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index efaa4734..afa35a8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] urls = { homepage = "https://qiling.io" } name = "qiling" -version = "1.4.10" +version = "1.4.11-dev" description = "Qiling is an advanced binary emulation framework that cross-platform-architecture" authors = ["KaiJern Lau (xwings) "] license = "GPLv2" readme = "README.md" classifiers = [ - "Development Status :: 5 - Production/Stable", + "Development Status :: 3 - Beta", "Intended Audience :: Developers", "Topic :: Software Development :: Disassemblers", "Topic :: System :: Emulators",