diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 4b83c2ba..07cb4f0f 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -1,44 +1,73 @@ name: PyPI 📦 Distribution -on: [push] +on: [push, pull_request] + +permissions: + contents: read jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v6 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v6 with: - python-version: "^3.8" + python-version: "3.13" - name: Install dependencies - run: pipx install poetry + run: python -m pip install "poetry>=2,<3" build twine + - name: Check package metadata + run: poetry check --lock + - name: Check release tag matches package version + if: startsWith(github.ref, 'refs/tags/') + env: + RELEASE_TAG: ${{ github.ref_name }} + run: | + python - <<'PY' + import os + import tomllib + from pathlib import Path + from packaging.version import Version + + tag = os.environ['RELEASE_TAG'].removeprefix('v') + version = tomllib.loads(Path('pyproject.toml').read_text())['tool']['poetry']['version'] + if Version(tag) != Version(version): + raise SystemExit(f'Release tag {tag!r} does not match package version {version!r}') + PY - name: Build distribution 📦 - run: poetry build -n + run: python -m build + - name: Check distribution metadata + run: python -m twine check --strict dist/* + - name: Test installed wheel + run: | + python -m pip install dist/*.whl + python -I tests/test_qltool.py InstalledQltool_Test -v - uses: actions/upload-artifact@v4 with: + name: distributions path: ${{ github.workspace }}/dist/* + if-no-files-found: error publish: needs: [build] runs-on: ubuntu-latest - if: startsWith(github.ref, 'refs/tags') + if: startsWith(github.ref, 'refs/tags/') steps: - uses: actions/download-artifact@v4 with: - name: artifact + name: distributions path: dist - name: Publish distribution 📦 to test PyPI - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.testpypi_pass }} - repository_url: https://test.pypi.org/legacy/ + repository-url: https://test.pypi.org/legacy/ + skip-existing: true - name: Publish distribution 📦 to PyPI - if: ${{ success() }} - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.pypi_pass }} diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 11f4437b..dd0822d7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -31,10 +31,10 @@ with reverse debugging), and fuzzing integration (AFL++/unicornafl). - **Shape**: pure-Python library (`from qiling import Qiling`), plus the `qltool` CLI and `qltui.py` TUI. Also shipped as a Docker image (`Dockerfile`) and PyPI package. -- **Python**: 3.10+ (`pyproject.toml:27`). Packaging via Poetry; version - 1.4.11-dev (`pyproject.toml:4`), status Beta, license GPLv2. +- **Python**: 3.10+ (`pyproject.toml:37`). Packaging via Poetry; version + 1.4.11.dev0 (`pyproject.toml:4`), status Beta, license GPL-2.0-or-later. - **Core dependencies**: `unicorn ==2.1.3` (hard-pinned CPU emulator, - `pyproject.toml:29`), `capstone` (disassembly), `keystone-engine` + `pyproject.toml:39`), `capstone` (disassembly), `keystone-engine` (assembly), `pefile`, `pyelftools`, `python-registry`, `gevent` (multithread emulation), `pyyaml` (MCU profiles). Extras: `fuzz` → unicornafl/fuzzercorn, `RE` → r2libr. @@ -55,7 +55,7 @@ with reverse debugging), and fuzzing integration (AFL++/unicornafl). | `jexamples/` | Legacy example set (not covered by CI) | | `tests/` | CI test suite — standalone `unittest` files run from `tests/` (`tests/test_onlinux.sh`, `tests/test_pe.bat`, `tests/test_macho.sh`) | | `docs/` | Stubs and assets; real documentation lives at https://docs.qiling.io | -| `qltool` | CLI entry point (see [cli.md](ARCHITECTURE/cli.md)) | +| `qltool` / `qiling/cli.py` | Checkout launcher / installed CLI implementation (see [cli.md](ARCHITECTURE/cli.md)) | | `qltui.py` | Terminal UI invoked via `qltool qltui` | | `Dockerfile` | Multi-stage Poetry wheel build on `python:3-slim` | | `pyproject.toml` / `poetry.lock` | Packaging and dependency pins | @@ -65,8 +65,9 @@ with reverse debugging), and fuzzing integration (AFL++/unicornafl). From CLI to emulated instructions (details live in the module docs): -1. `qltool` parses args and builds kwargs; `run` and `code` subcommands end in - `Qiling(**ql_args)` (`qltool:276`) — see [cli.md](ARCHITECTURE/cli.md). +1. `qltool` calls `qiling.cli.run`, which parses args and builds kwargs; + `run` and `code` subcommands end in `Qiling(**ql_args)` + (`qiling/cli.py:276`) — see [cli.md](ARCHITECTURE/cli.md). 2. `Qiling.__init__` (`qiling/core.py:35`) is the composition root. Order: guess arch/OS from the binary if not given → instantiate arch → init struct/hook mixins → logger → profile → loader → memory manager → OS layer @@ -86,17 +87,32 @@ From CLI to emulated instructions (details live in the module docs): ## Roadmap Maturity-based — Qiling is a released project in maintenance/beta -(v1.4.11-dev). There are no in-repo milestones; module Status is `done` when +(v1.4.11.dev0). There are no in-repo milestones; module Status is `done` when its test suite proves it. Feature requests and the forward-looking wishlist are tracked in GitHub issue [#333](https://github.com/qilingframework/qiling/issues/333) (the `TODO` file is a pointer to it). Known cross-cutting gaps: - `ChangeLog` stops at 1.4.6 (`ChangeLog:4`) while `pyproject.toml:4` says - 1.4.11-dev. + 1.4.11.dev0. - macOS is dropped from the CI matrix and the macOS/kext job is commented out (`.github/workflows/build-ci.yml:12`, `:84`). - `jexamples/` is legacy and unexercised by CI. +## Packaging and Releases + +`.github/workflows/pythonpublish.yml` validates metadata and the lock file, +builds a source distribution and a wheel from it, checks both with Twine, +and runs `InstalledQltool_Test` against the installed wheel outside the +checkout. Builds run on pushes and pull requests; publishing runs on tags. + +Before tagging a release, set its version with `poetry version ` +and commit the change. The tag must match the package version under PEP 440 +normalization, with an optional `v` prefix. Development versions remain +prereleases; a stable release requires a stable version in `pyproject.toml`. +Publishing uploads to TestPyPI, then PyPI, using the existing +`testpypi_pass` and `pypi_pass` API-token secrets. TestPyPI skips existing +files so a production upload can be retried after TestPyPI succeeds. + ## Development Loop Coding Discipline governs writing; Review Checks govern review. This @@ -252,7 +268,7 @@ repository evidence before implementation. Adding an unrequested syscall, Win32 API, or peripheral register is out of scope; record it under the owning module's **Open Gaps / Roadmap**. -- `unicorn` is hard-pinned (`pyproject.toml:29`). Changing it, or any +- `unicorn` is hard-pinned (`pyproject.toml:39`). Changing it, or any behavior that depends on its version, is a project-wide event and never an incidental part of another change. diff --git a/ARCHITECTURE/cli.md b/ARCHITECTURE/cli.md index 40b13a98..44ed56ba 100644 --- a/ARCHITECTURE/cli.md +++ b/ARCHITECTURE/cli.md @@ -11,21 +11,24 @@ maturity-based status. ## Status `done` — covered by `tests/test_qltool.py`, which shells out to `qltool` for -run/code subcommands including gdb attach and coverage output. +run/code subcommands and coverage output. `InstalledQltool_Test` checks the +installed command, shellcode exit status, bundled profiles, and TUI import +outside the checkout. ## Code Structure | File | Role | | ---- | ---- | -| `qltool` | Executable argparse CLI; builds kwargs and drives `Qiling` | +| `qltool` | Checkout launcher for `qiling.cli.run` | +| `qiling/cli.py` | Argparse CLI; installed as `qltool`, builds kwargs and drives `Qiling` | | `qltui.py` | questionary/pyfx/termcolor TUI; collects options, returned to qltool | ## Key Types and Entry Points -- `qltool:189` - `run()` - argparse setup with subcommands `run`, `code`, `examples`, `qltui`; enum-mapping actions translate `--arch/--os/--endian/--verbose` strings to `QL_ARCH`/`QL_OS` enums (`qltool:59-75`). -- `qltool:129` - `handle_run(options)` - builds `{'argv': [file]+args, 'rootfs': ...}`. -- `qltool:78` - `handle_code(options)` - reads hex/asm/bin shellcode, assembling asm via `qiling.arch.utils.assembler` (`qltool:104`). -- `qltool:276` - `ql = Qiling(**ql_args)` - the single construction point; then optional Qdb (`:279`), gdbserver (`:285`), coverage-wrapped `ql.run()` (`:306-310`), JSON report (`:312`), exit with `ql.os.exit_code` (`:321`). +- `qiling/cli.py:189` - `run()` - argparse setup with subcommands `run`, `code`, `examples`, `qltui`; enum-mapping actions translate `--arch/--os/--endian/--verbose` strings to `QL_ARCH`/`QL_OS` enums (`qiling/cli.py:59-75`). +- `qiling/cli.py:129` - `handle_run(options)` - builds `{'argv': [file]+args, 'rootfs': ...}`. +- `qiling/cli.py:78` - `handle_code(options)` - reads hex/asm/bin shellcode, assembling asm via `qiling.arch.utils.assembler` (`qiling/cli.py:104`). +- `qiling/cli.py:276` - `ql = Qiling(**ql_args)` - the single construction point; then optional Qdb (`:279`), gdbserver (`:285`), coverage-wrapped `ql.run()` (`:306-310`), JSON report (`:312`), exit with `ql.os.exit_code` (`:321`). ## Interactions @@ -37,12 +40,16 @@ run/code subcommands including gdb attach and coverage output. ## How to Test ```sh +python3 -m pip install . cd tests && python3 test_qltool.py # pass = unittest "OK", exit 0 ``` +- Packaging regression checks, from the repository root after installing + the built wheel: `python3 -I tests/test_qltool.py InstalledQltool_Test -v`. - Manual smoke test: `./qltool run -f examples/rootfs/x8664_linux/bin/x8664_hello --rootfs examples/rootfs/x8664_linux` — pass = prints `Hello, World!`. ## Open Gaps / Roadmap -- `qltui.py` (TUI) has no automated tests — interactive only. +- `qltui.py` has an installed import smoke test; its interactive flows are + not covered by automated tests. - `qltool` predates subcommand-style config files; complex setups (fs mappers, custom hooks) still require the Python API. diff --git a/README.md b/README.md index 05d6945b..668d335e 100644 --- a/README.md +++ b/README.md @@ -208,6 +208,10 @@ Qiling Framework emulates MBR Qiling also provides a friendly tool named `qltool` to quickly emulate shellcode & executable binaries. +Installing Qiling with pip also installs the `qltool` command, including +`qltool qltui` for the terminal UI. In a source checkout, use `./qltool` as +shown below. Example binaries and rootfs files must be downloaded separately. + With qltool, easy execution can be performed: diff --git a/pyproject.toml b/pyproject.toml index afa35a8d..da3e4b52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] urls = { homepage = "https://qiling.io" } name = "qiling" -version = "1.4.11-dev" +version = "1.4.11.dev0" description = "Qiling is an advanced binary emulation framework that cross-platform-architecture" authors = ["KaiJern Lau (xwings) "] -license = "GPLv2" +license = "GPL-2.0-or-later" readme = "README.md" classifiers = [ - "Development Status :: 3 - Beta", + "Development Status :: 4 - Beta", "Intended Audience :: Developers", "Topic :: Software Development :: Disassemblers", "Topic :: System :: Emulators", @@ -24,6 +24,15 @@ keywords = [ "IoT", ] +packages = [ + { include = "qiling" }, + { include = "qltui.py" }, +] +include = [{ path = "qltool", format = "sdist" }] + +[tool.poetry.scripts] +qltool = "qiling.cli:run" + [tool.poetry.dependencies] python = "^3.10" capstone = "^5" @@ -50,5 +59,5 @@ fuzz = ["unicornafl", "fuzzercorn"] RE = ["r2libr"] [build-system] -requires = ["poetry-core"] +requires = ["poetry-core>=2.0,<3.0"] build-backend = "poetry.core.masonry.api" diff --git a/qiling/cli.py b/qiling/cli.py new file mode 100644 index 00000000..a8e0a0df --- /dev/null +++ b/qiling/cli.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +# +# Cross Platform and Multi Architecture Advanced Binary Emulation Framework +# + +import argparse +import os +import sys +import ast +import pickle + +from pprint import pprint +from typing import TYPE_CHECKING, Any, AnyStr, Dict, Literal, Mapping, Type, overload + +from unicorn import __version__ as uc_ver +from qiling import __version__ as ql_ver + +from qiling import Qiling +from qiling.arch import utils as arch_utils +from qiling.debugger.qdb import QlQdb +from qiling.const import QL_ENDIAN, QL_VERBOSE, endian_map, os_map, arch_map, verbose_map +from qiling.extensions.coverage import utils as cov_utils +from qiling.extensions import report + + +if TYPE_CHECKING: + from enum import Enum + + +@overload +def read_file(fname: AnyStr) -> str: ... + + +@overload +def read_file(fname: AnyStr, mode: Literal['b']) -> bytes: ... + + +def read_file(fname: AnyStr, mode: Literal['b', ''] = ''): + """Read file contents. + """ + + with open(fname, f'r{mode}') as f: + content = f.read() + + return content + + +class __arg_env(argparse.Action): + def __call__(self, parser, namespace, values: str, option_string): + if os.path.exists(values): + with open(values, 'rb') as f: + env = pickle.load(f) + else: + env = ast.literal_eval(values) + + setattr(namespace, self.dest, env or {}) + + +def __make_enum_arg(enum_rmap: Mapping[str, 'Enum'], aliases: Mapping[str, str] = {}) -> Type[argparse.Action]: + class __enum_arg(argparse.Action): + def __call__(self, parser, namespace, values: str, option_string): + values = values.casefold() + + if values in aliases: + values = aliases[values] + + setattr(namespace, self.dest, enum_rmap[values]) + + return __enum_arg + + +__arg_endian = __make_enum_arg(endian_map) +__arg_archtype = __make_enum_arg(arch_map, {'x86_64': 'x8664', 'riscv32': 'riscv'}) +__arg_ostype = __make_enum_arg(os_map, {'darwin': 'macos'}) +__arg_verbose = __make_enum_arg(verbose_map) + + +def handle_code(options: argparse.Namespace) -> Dict[str, Any]: + if options.format == 'hex': + if options.input is not None: + print("Loading HEX from ARGV") + payload = str(options.input) + + elif options.filename is not None: + print("Loading HEX from FILE") + payload = read_file(options.filename) + + else: + print("Error: please specify an input source for hex") + exit(1) + + code = bytes.fromhex(payload.replace(r'\x', '')) + + elif options.format == 'asm': + print("Loading ASM from FILE") + + if options.filename is None: + print("Error: please specify an input source for asm") + exit(1) + + payload = read_file(options.filename) + + # assemble the payload and turn it into opcodes bytes + assembler = arch_utils.assembler(options.arch, options.endian, options.thumb) + code, _ = assembler.asm(payload) + code = bytes(code) + + elif options.format == 'bin': + print("Loading BIN from FILE") + + if options.filename is None: + print("Error: please specify an input source for bin") + exit(1) + + code = read_file(options.filename, 'b') + + ql_args = { + 'rootfs': options.rootfs, + 'code': code, + 'ostype': options.os, + 'archtype': options.arch, + 'endian': options.endian, + 'thumb': options.thumb + } + + return ql_args + + +def handle_run(options: argparse.Namespace) -> Dict[str, Any]: + effective_argv = [] + + # with argv + if options.filename is not None and options.run_args == []: + effective_argv = [options.filename] + options.args + + # Without argv + elif options.filename is None and options.args == [] and options.run_args != []: + effective_argv = options.run_args + + else: + print("ERROR: Command error!") + + ql_args = { + 'argv': effective_argv, + 'rootfs': options.rootfs + } + + return ql_args + + +def handle_examples(parser: argparse.ArgumentParser): + prog = parser.prog + + __ql_examples = f"""Examples: + + With code: + {prog} code --os linux --arch arm --format hex -f examples/shellcodes/linarm32_tcp_reverse_shell.hex + {prog} code --os linux --arch x86 --format asm -f examples/shellcodes/lin32_execve.asm + + With binary file: + {prog} run -f examples/rootfs/x8664_linux/bin/x8664_hello --rootfs examples/rootfs/x8664_linux + {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux + + With binary file and Qdb: + {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --qdb + {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --qdb --rr + + With binary file and gdbserver: + {prog} run -f examples/rootfs/x8664_linux/bin/x8664_hello --gdb 127.0.0.1:9999 --rootfs examples/rootfs/x8664_linux + + With binary file and additional argv: + {prog} run -f examples/rootfs/x8664_linux/bin/x8664_args --rootfs examples/rootfs/x8664_linux --args test1 test2 test3 + + With binary file and various output format: + {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --verbose disasm + {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --filter ^open + + With UEFI file: + {prog} run -f examples/rootfs/x8664_efi/bin/TcgPlatformSetupPolicy --rootfs examples/rootfs/x8664_efi --env examples/rootfs/x8664_efi/rom2_nvar.pickel + + With binary file and json output: + {prog} run -f examples/rootfs/x86_windows/bin/x86_hello.exe --rootfs examples/rootfs/x86_windows --no-console --json + +""" + + parser.exit(0, __ql_examples) + + +def run(): + parser = argparse.ArgumentParser() + parser.add_argument('--version', action='version', version=f'qltool for Qiling {ql_ver}, using Unicorn {uc_ver}') + + commands = parser.add_subparsers(title='sub commands', description='select execution mode', dest='subcommand', required=True) + + # set "run" subcommand options + run_parser = commands.add_parser('run', help='run a program') + run_parser.add_argument('-f', '--filename', default=None, metavar="FILE", help="filename") + run_parser.add_argument('--rootfs', required=True, help='emulated rootfs') + run_parser.add_argument('--args', default=[], nargs=argparse.REMAINDER, dest="args", help="args") + run_parser.add_argument('run_args', default=[], nargs=argparse.REMAINDER) + + # set "code" subcommand options + code_parser = commands.add_parser('code', help='execute a shellcode') + code_parser.add_argument('-f', '--filename', metavar="FILE", help="filename") + code_parser.add_argument('-i', '--input', metavar="INPUT", dest="input", help='input hex value') + code_parser.add_argument('--arch', required=True, choices=arch_map, action=__arg_archtype) + code_parser.add_argument('--thumb', action='store_true', default=False, help='specify thumb mode for ARM') + code_parser.add_argument('--endian', choices=endian_map, default=QL_ENDIAN.EL, action=__arg_endian, help='specify endianness for bi-endian archs') + code_parser.add_argument('--os', required=True, choices=os_map, action=__arg_ostype) + code_parser.add_argument('--rootfs', default='.', help='emulated root filesystem, that is where all libraries reside') + code_parser.add_argument('--format', choices=('asm', 'hex', 'bin'), default='bin', help='input file format') + + # set "examples" subcommand + expl_parser = commands.add_parser('examples', help='show examples and exit', add_help=False) + + # set "qltui" subcommand + qltui_parser = commands.add_parser('qltui', help='show qiling Terminal User Interface', add_help=False) + + comm_parser = run_parser + + if len(sys.argv) > 1 and sys.argv[1] == 'code': + comm_parser = code_parser + + # set common options + comm_parser.add_argument('-v', '--verbose', choices=verbose_map, default=QL_VERBOSE.DEFAULT, action=__arg_verbose, help='set verbosity level') + comm_parser.add_argument('--env', metavar="FILE", action=__arg_env, default={}, help="pickle file containing an environment dictionary") + comm_parser.add_argument('-g', '--gdb', nargs='?', metavar='SERVER:PORT', const='gdb', help='enable gdb server') + comm_parser.add_argument('--qdb', action='store_true', help='attach Qdb at entry point, it\'s MIPS, ARM(THUMB) supported only for now') + comm_parser.add_argument('--rr', action='store_true', help='switch on record and replay feature in qdb, only works with --qdb') + comm_parser.add_argument('--profile', help="define a customized profile") + comm_parser.add_argument('--no-console', action='store_false', dest='console', help='do not emit output to console') + comm_parser.add_argument('-e', '--filter', metavar='REGEXP', default=None, help="apply a filtering regexp on log output") + comm_parser.add_argument('--log-file', help="write log to a file") + comm_parser.add_argument('--log-plain', action='store_true', help="do not use colors in log output") + comm_parser.add_argument('--root', action='store_true', help='enable sudo required mode') + comm_parser.add_argument('--debug-stop', action='store_true', help='stop running on error; requires verbose to be set to either "debug" or "dump"') + comm_parser.add_argument('-m', '--multithread', action='store_true', help='run in multithread mode') + comm_parser.add_argument('--timeout', type=int, default=0, help='set emulation timeout') + comm_parser.add_argument('-c', '--coverage-file', default=None, help='code coverage file name') + comm_parser.add_argument('--coverage-format', default='drcov', choices=cov_utils.factory.formats, help='code coverage file format') + comm_parser.add_argument('--json', action='store_true', help='print a json report of the emulation') + comm_parser.add_argument('--libcache', action='store_true', help='enable dll caching for windows') + options = parser.parse_args() + + qltui_enabled = False + ql_args = {} + + if options.subcommand == 'qltui': + import qltui + options = qltui.get_data() + qltui_enabled = True + + if options.subcommand == 'examples': + handle_examples(parser) + + # ql file setup + elif options.subcommand == 'run': + ql_args = handle_run(options) + + # ql code setup + elif options.subcommand == 'code': + ql_args = handle_code(options) + + ql_args.update({ + 'env': options.env, + 'verbose': options.verbose, + 'profile': options.profile, + 'console': options.console, + 'filter': options.filter, + 'log_devices': options.log_file and [options.log_file], + 'log_plain': options.log_plain, + 'multithread': options.multithread, + 'libcache': options.libcache + }) + + ql = Qiling(**ql_args) + + # attach Qdb at entry point + if options.qdb: + QlQdb(ql, rr=bool(options.rr)).run() + + parser.exit(0) + + # ql execute additional options + if options.gdb: + argval = options.gdb + + if argval != 'gdb': + argval = f'gdb:{argval}' + + ql.debugger = argval + + if options.debug_stop: + if options.verbose not in (QL_VERBOSE.DEBUG, QL_VERBOSE.DUMP): + parser.error('the debug_stop option requires verbose to be set to either "debug" or "dump"') + + ql.debug_stop = True + + if options.root: + ql.os.root = True + + if qltui_enabled: + hook_dictionary = qltui.hook(ql) + + # ql run + if options.coverage_file: + with cov_utils.collect_coverage(ql, options.coverage_format, options.coverage_file): + ql.run(timeout=options.timeout) + else: + ql.run(timeout=options.timeout) + + if options.json: + rep = report.generate_report(ql) + + if qltui_enabled: + rep["syscalls"] = qltui.transform_syscalls(ql.os.stats.syscalls) + qltui.show_report(ql, rep, hook_dictionary) + else: + pprint(rep) + + exit(ql.os.exit_code) + +if __name__ == '__main__': + run() diff --git a/qltool b/qltool index 5164468c..6d5ec489 100755 --- a/qltool +++ b/qltool @@ -3,322 +3,8 @@ # Cross Platform and Multi Architecture Advanced Binary Emulation Framework # -import argparse -import os -import sys -import ast -import pickle +from qiling.cli import run -from pprint import pprint -from typing import TYPE_CHECKING, Any, AnyStr, Dict, Literal, Mapping, Type, overload - -from unicorn import __version__ as uc_ver -from qiling import __version__ as ql_ver - -from qiling import Qiling -from qiling.arch import utils as arch_utils -from qiling.debugger.qdb import QlQdb -from qiling.const import QL_ENDIAN, QL_VERBOSE, endian_map, os_map, arch_map, verbose_map -from qiling.extensions.coverage import utils as cov_utils -from qiling.extensions import report - - -if TYPE_CHECKING: - from enum import Enum - - -@overload -def read_file(fname: AnyStr) -> str: ... - - -@overload -def read_file(fname: AnyStr, mode: Literal['b']) -> bytes: ... - - -def read_file(fname: AnyStr, mode: Literal['b', ''] = ''): - """Read file contents. - """ - - with open(fname, f'r{mode}') as f: - content = f.read() - - return content - - -class __arg_env(argparse.Action): - def __call__(self, parser, namespace, values: str, option_string): - if os.path.exists(values): - with open(values, 'rb') as f: - env = pickle.load(f) - else: - env = ast.literal_eval(values) - - setattr(namespace, self.dest, env or {}) - - -def __make_enum_arg(enum_rmap: Mapping[str, 'Enum'], aliases: Mapping[str, str] = {}) -> Type[argparse.Action]: - class __enum_arg(argparse.Action): - def __call__(self, parser, namespace, values: str, option_string): - values = values.casefold() - - if values in aliases: - values = aliases[values] - - setattr(namespace, self.dest, enum_rmap[values]) - - return __enum_arg - - -__arg_endian = __make_enum_arg(endian_map) -__arg_archtype = __make_enum_arg(arch_map, {'x86_64': 'x8664', 'riscv32': 'riscv'}) -__arg_ostype = __make_enum_arg(os_map, {'darwin': 'macos'}) -__arg_verbose = __make_enum_arg(verbose_map) - - -def handle_code(options: argparse.Namespace) -> Dict[str, Any]: - if options.format == 'hex': - if options.input is not None: - print("Loading HEX from ARGV") - payload = str(options.input) - - elif options.filename is not None: - print("Loading HEX from FILE") - payload = read_file(options.filename) - - else: - print("Error: please specify an input source for hex") - exit(1) - - code = bytes.fromhex(payload.replace(r'\x', '')) - - elif options.format == 'asm': - print("Loading ASM from FILE") - - if options.filename is None: - print("Error: please specify an input source for asm") - exit(1) - - payload = read_file(options.filename) - - # assemble the payload and turn it into opcodes bytes - assembler = arch_utils.assembler(options.arch, options.endian, options.thumb) - code, _ = assembler.asm(payload) - code = bytes(code) - - elif options.format == 'bin': - print("Loading BIN from FILE") - - if options.filename is None: - print("Error: please specify an input source for bin") - exit(1) - - code = read_file(options.filename, 'b') - - ql_args = { - 'rootfs': options.rootfs, - 'code': code, - 'ostype': options.os, - 'archtype': options.arch, - 'endian': options.endian, - 'thumb': options.thumb - } - - return ql_args - - -def handle_run(options: argparse.Namespace) -> Dict[str, Any]: - effective_argv = [] - - # with argv - if options.filename is not None and options.run_args == []: - effective_argv = [options.filename] + options.args - - # Without argv - elif options.filename is None and options.args == [] and options.run_args != []: - effective_argv = options.run_args - - else: - print("ERROR: Command error!") - - ql_args = { - 'argv': effective_argv, - 'rootfs': options.rootfs - } - - return ql_args - - -def handle_examples(parser: argparse.ArgumentParser): - prog = os.path.basename(__file__) - - __ql_examples = f"""Examples: - - With code: - {prog} code --os linux --arch arm --format hex -f examples/shellcodes/linarm32_tcp_reverse_shell.hex - {prog} code --os linux --arch x86 --format asm -f examples/shellcodes/lin32_execve.asm - - With binary file: - {prog} run -f examples/rootfs/x8664_linux/bin/x8664_hello --rootfs examples/rootfs/x8664_linux - {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux - - With binary file and Qdb: - {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --qdb - {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --qdb --rr - - With binary file and gdbserver: - {prog} run -f examples/rootfs/x8664_linux/bin/x8664_hello --gdb 127.0.0.1:9999 --rootfs examples/rootfs/x8664_linux - - With binary file and additional argv: - {prog} run -f examples/rootfs/x8664_linux/bin/x8664_args --rootfs examples/rootfs/x8664_linux --args test1 test2 test3 - - With binary file and various output format: - {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --verbose disasm - {prog} run -f examples/rootfs/mips32el_linux/bin/mips32el_hello --rootfs examples/rootfs/mips32el_linux --filter ^open - - With UEFI file: - {prog} run -f examples/rootfs/x8664_efi/bin/TcgPlatformSetupPolicy --rootfs examples/rootfs/x8664_efi --env examples/rootfs/x8664_efi/rom2_nvar.pickel - - With binary file and json output: - {prog} run -f examples/rootfs/x86_windows/bin/x86_hello.exe --rootfs examples/rootfs/x86_windows --no-console --json - -""" - - parser.exit(0, __ql_examples) - - -def run(): - parser = argparse.ArgumentParser() - parser.add_argument('--version', action='version', version=f'qltool for Qiling {ql_ver}, using Unicorn {uc_ver}') - - commands = parser.add_subparsers(title='sub commands', description='select execution mode', dest='subcommand', required=True) - - # set "run" subcommand options - run_parser = commands.add_parser('run', help='run a program') - run_parser.add_argument('-f', '--filename', default=None, metavar="FILE", help="filename") - run_parser.add_argument('--rootfs', required=True, help='emulated rootfs') - run_parser.add_argument('--args', default=[], nargs=argparse.REMAINDER, dest="args", help="args") - run_parser.add_argument('run_args', default=[], nargs=argparse.REMAINDER) - - # set "code" subcommand options - code_parser = commands.add_parser('code', help='execute a shellcode') - code_parser.add_argument('-f', '--filename', metavar="FILE", help="filename") - code_parser.add_argument('-i', '--input', metavar="INPUT", dest="input", help='input hex value') - code_parser.add_argument('--arch', required=True, choices=arch_map, action=__arg_archtype) - code_parser.add_argument('--thumb', action='store_true', default=False, help='specify thumb mode for ARM') - code_parser.add_argument('--endian', choices=endian_map, default=QL_ENDIAN.EL, action=__arg_endian, help='specify endianness for bi-endian archs') - code_parser.add_argument('--os', required=True, choices=os_map, action=__arg_ostype) - code_parser.add_argument('--rootfs', default='.', help='emulated root filesystem, that is where all libraries reside') - code_parser.add_argument('--format', choices=('asm', 'hex', 'bin'), default='bin', help='input file format') - - # set "examples" subcommand - expl_parser = commands.add_parser('examples', help='show examples and exit', add_help=False) - - # set "qltui" subcommand - qltui_parser = commands.add_parser('qltui', help='show qiling Terminal User Interface', add_help=False) - - comm_parser = run_parser - - if len(sys.argv) > 1 and sys.argv[1] == 'code': - comm_parser = code_parser - - # set common options - comm_parser.add_argument('-v', '--verbose', choices=verbose_map, default=QL_VERBOSE.DEFAULT, action=__arg_verbose, help='set verbosity level') - comm_parser.add_argument('--env', metavar="FILE", action=__arg_env, default={}, help="pickle file containing an environment dictionary") - comm_parser.add_argument('-g', '--gdb', nargs='?', metavar='SERVER:PORT', const='gdb', help='enable gdb server') - comm_parser.add_argument('--qdb', action='store_true', help='attach Qdb at entry point, it\'s MIPS, ARM(THUMB) supported only for now') - comm_parser.add_argument('--rr', action='store_true', help='switch on record and replay feature in qdb, only works with --qdb') - comm_parser.add_argument('--profile', help="define a customized profile") - comm_parser.add_argument('--no-console', action='store_false', dest='console', help='do not emit output to console') - comm_parser.add_argument('-e', '--filter', metavar='REGEXP', default=None, help="apply a filtering regexp on log output") - comm_parser.add_argument('--log-file', help="write log to a file") - comm_parser.add_argument('--log-plain', action='store_true', help="do not use colors in log output") - comm_parser.add_argument('--root', action='store_true', help='enable sudo required mode') - comm_parser.add_argument('--debug-stop', action='store_true', help='stop running on error; requires verbose to be set to either "debug" or "dump"') - comm_parser.add_argument('-m', '--multithread', action='store_true', help='run in multithread mode') - comm_parser.add_argument('--timeout', type=int, default=0, help='set emulation timeout') - comm_parser.add_argument('-c', '--coverage-file', default=None, help='code coverage file name') - comm_parser.add_argument('--coverage-format', default='drcov', choices=cov_utils.factory.formats, help='code coverage file format') - comm_parser.add_argument('--json', action='store_true', help='print a json report of the emulation') - comm_parser.add_argument('--libcache', action='store_true', help='enable dll caching for windows') - options = parser.parse_args() - - qltui_enabled = False - ql_args = {} - - if options.subcommand == 'qltui': - import qltui - options = qltui.get_data() - qltui_enabled = True - - if options.subcommand == 'examples': - handle_examples(parser) - - # ql file setup - elif options.subcommand == 'run': - ql_args = handle_run(options) - - # ql code setup - elif options.subcommand == 'code': - ql_args = handle_code(options) - - ql_args.update({ - 'env': options.env, - 'verbose': options.verbose, - 'profile': options.profile, - 'console': options.console, - 'filter': options.filter, - 'log_devices': options.log_file and [options.log_file], - 'log_plain': options.log_plain, - 'multithread': options.multithread, - 'libcache': options.libcache - }) - - ql = Qiling(**ql_args) - - # attach Qdb at entry point - if options.qdb: - QlQdb(ql, rr=bool(options.rr)).run() - - parser.exit(0) - - # ql execute additional options - if options.gdb: - argval = options.gdb - - if argval != 'gdb': - argval = f'gdb:{argval}' - - ql.debugger = argval - - if options.debug_stop: - if options.verbose not in (QL_VERBOSE.DEBUG, QL_VERBOSE.DUMP): - parser.error('the debug_stop option requires verbose to be set to either "debug" or "dump"') - - ql.debug_stop = True - - if options.root: - ql.os.root = True - - if qltui_enabled: - hook_dictionary = qltui.hook(ql) - - # ql run - if options.coverage_file: - with cov_utils.collect_coverage(ql, options.coverage_format, options.coverage_file): - ql.run(timeout=options.timeout) - else: - ql.run(timeout=options.timeout) - - if options.json: - rep = report.generate_report(ql) - - if qltui_enabled: - rep["syscalls"] = qltui.transform_syscalls(ql.os.stats.syscalls) - qltui.show_report(ql, rep, hook_dictionary) - else: - pprint(rep) - - exit(ql.os.exit_code) if __name__ == '__main__': run() diff --git a/tests/test_qltool.py b/tests/test_qltool.py index 79f25738..9949b90f 100644 --- a/tests/test_qltool.py +++ b/tests/test_qltool.py @@ -7,7 +7,11 @@ import os import subprocess import re import sys +import sysconfig +import tempfile import unittest +from importlib.metadata import version +from pathlib import Path class Qltool_Test(unittest.TestCase): @@ -48,5 +52,43 @@ class Qltool_Test(unittest.TestCase): self.assertTrue(all(p.search(entry) is not None for entry in log_entries)) +class InstalledQltool_Test(unittest.TestCase): + """Exercise the installed distribution outside the source checkout.""" + + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.cwd = directory.name + self.qltool = Path(sysconfig.get_path('scripts')) / ('qltool.exe' if os.name == 'nt' else 'qltool') + + def __run(self, *args: str, expected_exit: int = 0) -> str: + result = subprocess.run( + [str(self.qltool), *args], cwd=self.cwd, + env={**os.environ, 'PYTHONPATH': ''}, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, timeout=30 + ) + self.assertEqual(result.returncode, expected_exit, result.stdout) + return result.stdout + + def test_qltool_version_and_examples(self): + self.assertIn(f'Qiling {version("qiling")}', self.__run('--version')) + self.assertIn('qltool code --os linux', self.__run('examples')) + + def test_qltool_shellcode_exit_status(self): + # Linux x86 exit(7), using the profiles shipped in the installed wheel. + self.__run('code', '--os', 'linux', '--arch', 'x86', '--format', 'hex', + '--input', 'b801000000bb07000000cd80', '--verbose', 'off', + expected_exit=7) + + def test_qltui_import(self): + result = subprocess.run( + [sys.executable, '-I', '-c', 'import qltui; assert callable(qltui.get_data)'], + cwd=self.cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, timeout=30 + ) + self.assertEqual(result.returncode, 0, result.stdout) + + if __name__ == "__main__": unittest.main()