Pin colored_traceback < 0.4 for Python 2

The apt package appears outdated and incompatible with the latest version on pypi.

>>> ValueError: invalid message type: 18

.coveragerc

@@ -1,12 +0,0 @@
-[run]
-branch = True
-parallel = True
-concurrency = thread,multiprocessing
-omit =
-    */constants/*
-source =
-    pwn
-    pwnlib
-    ~/.cache/.pwntools-cache-2.7/
-    ~/.cache/.pwntools-cache-3.8/
-disable_warnings = module-not-imported

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every week
+      interval: "weekly"

.github/workflows/android.yml

@@ -3,7 +3,7 @@ name: Android Tests
 on: [push, pull_request]
 
 jobs:
-  test:
+  android-test:
     strategy:
       matrix:
         python-version: [3.8]
@@ -11,17 +11,17 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Cache for pip
-      uses: actions/cache@v1
+      uses: actions/cache@v4
       id: cache-pip
       with:
         path: ~/.cache/pip
         key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -33,14 +33,14 @@ jobs:
           qemu-user-static \
           binutils-aarch64-linux-gnu \
           binutils-arm-linux-gnueabihf \
-          libc6-dbg \
-          openjdk-8-jre-headless
+          libc6-dbg
 
     - name: Install Android AVD
       run: |
-        USER=travis source travis/setup_avd.sh
+        sudo usermod -aG kvm $USER
+        source travis/setup_avd_fast.sh
         sed -i 's/skip_android = True/skip_android = False/' docs/source/conf.py
-        set | egrep '^(ANDROID|PATH)' >.android.env
+        set | grep ^PATH >.android.env
 
     - name: Install dependencies
       run: |

.github/workflows/changelog.yml

@@ -1,5 +1,9 @@
 name: Changelog
-on: [pull_request]
+on:
+  pull_request:
+    paths:
+    - pwnlib/**
+    - pwn/**
 env:
   GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
 
@@ -11,7 +15,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 5
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 

.github/workflows/ci.yml

@@ -5,31 +5,51 @@ jobs:
   test:
     strategy:
       matrix:
-        python-version: [2.7, 3.8]
+        python_version: ['2.7', '3.10']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
-        fetch-depth: 0
+        fetch-depth: 20
 
     - name: Git History
       run: |
         git fetch origin
         git log --oneline --graph -10
 
+    - name: Install RPyC for gdb
+      run: |
+        # The version packaged in python3-rpyc is too old on Ubuntu 22.04
+        sudo apt-get update && sudo apt-get install -y python3-pip gdb gdbserver
+        /usr/bin/python -m pip install rpyc
+        gdb --batch --quiet --nx --nh --ex 'py import rpyc; print(rpyc.version.version)'
+
     - name: Cache for pip
-      uses: actions/cache@v1
+      uses: actions/cache@v4
       id: cache-pip
       with:
         path: ~/.cache/pip
         key: ${{ matrix.os }}-cache-pip
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+    - name: Set up Python ${{ matrix.python_version }}
+      if: matrix.python_version != '2.7'
+      uses: actions/setup-python@v5
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: ${{ matrix.python_version }}
+
+    - name: Set up Python 2.7
+      if: matrix.python_version == '2.7'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          python2.7 python2.7-dev python2-pip-whl
+        sudo ln -sf python2.7 /usr/bin/python
+        export PYTHONPATH=`echo /usr/share/python-wheels/pip-*py2*.whl`
+        sudo --preserve-env=PYTHONPATH python -m pip install --upgrade pip setuptools wheel
+        sudo chown -R $USER /usr/local/lib/python2.7
+
 
     - name: Verify tag against version
       if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
@@ -49,7 +69,7 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
           ash bash-static dash ksh mksh zsh \
-          pandoc gdb gdbserver socat sshpass \
+          gdb gdbserver socat \
           binutils-multiarch qemu-user-static \
           binutils-aarch64-linux-gnu \
           binutils-arm-linux-gnueabihf \
@@ -59,13 +79,10 @@ jobs:
           binutils-s390x-linux-gnu \
           binutils-sparc64-linux-gnu \
           gcc-multilib \
-          libc6-dbg
-
-    - name: Install RPyC for GDB
-      run: |
-        sudo apt-get install -y python3-pip
-        /usr/bin/python3 -m pip install rpyc
-
+          libc6-dbg \
+          elfutils \
+          patchelf
+      
     - name: Testing Corefiles
       run: |
         ulimit -a
@@ -81,18 +98,19 @@ jobs:
     - name: Install dependencies
       run: |
         pip install --upgrade pip
+        pip install --upgrade wheel build
         pip install --upgrade flake8 appdirs
-        python setup.py egg_info
         pip install --upgrade --editable .
 
     - name: Sanity checks
       run:  PWNLIB_NOTERM=1 python -bb -c 'from pwn import *; print(pwnlib.term.term_mode)'
 
     - name: Install documentation dependencies
-      run:  pip install -r docs/requirements.txt
+      run: pip install -r docs/requirements.txt
 
     - name: Manually install non-broken Unicorn
-      run:  pip install unicorn==1.0.2rc3
+      if:   matrix.python_version == '2.7'
+      run:  pip install unicorn==2.0.0rc7
 
     - name: Disable yama ptrace_scope
       run: |
@@ -100,6 +118,10 @@ jobs:
 
     - name: Coverage doctests
       run: |
+        # Python version installed using setup-python interferes with gdb's python
+        # by setting LD_LIBRARY_PATH and gdb's python becoming unable to load built-in modules
+        # like _socket. This is a workaround.
+        unset LD_LIBRARY_PATH
         PWNLIB_NOTERM=1 python -bb -m coverage run -m sphinx -b doctest docs/source docs/build/doctest
 
     - name: Coverage running examples
@@ -140,6 +162,11 @@ jobs:
         pwn disasm --color ff3424c3ebfe
         pwn asm -f hex nop
 
+        pwn hex ABCD
+        pwn hex ABCD --separator ' '
+        pwn hex ABCD --prefix '\x'
+        pwn hex ABCD -p '0x' -s ' '
+
         pwn hex abcd
         pwn unhex 4141 4141
 
@@ -147,6 +174,7 @@ jobs:
         pwn phd -l 0x3d --color=always /etc/os-release
 
         pwn checksec /bin/bash
+        (ulimit -v 500000 && pwn checksec /bin/bash)
 
         pwn errno 2
         pwn errno -1
@@ -155,28 +183,58 @@ jobs:
         pwn constgrep -c freebsd -m ^PROT_ '3 + 4'
         pwn constgrep ^MAP_ 0
         pwn constgrep -e O_RDWR
+        pwn constgrep C
+
+        pwn libcdb file /lib/x86_64-linux-gnu/libc.so.6
+        pwn libcdb lookup puts 5f0 __libc_start_main_ret d0a
+        pwn libcdb hash b229d1da1e161f95e839cf90cded5f719e5de308
 
     - name: Build source and wheel distributions
+      if: matrix.python_version != '2.7'
       run: |
-        python setup.py sdist
-        python setup.py bdist_wheel --universal
+        python -m build
 
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v4
+      if: matrix.python_version != '2.7'
       with:
         name: packages
         path: dist/
 
+    - uses: actions/upload-artifact@v4
+      with:
+        name: coverage-${{ matrix.python_version }}
+        path: .coverage*
+
+
+  upload-coverage:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 20
+
+    - uses: actions/download-artifact@v4
+      with:
+        pattern: coverage-*
+        merge-multiple: true
+
+    - name: Install coveralls
+      run: |
+        pip install tomli coveralls
+
     - name: Upload coverage to coveralls.io
       run: |
         coverage combine
         COVERALLS_REPO_TOKEN=PP20MEgztXIQJJTguQwe2jeCh6Bm4lkbv coveralls
 
+
   staging-merge:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'Gallopsled' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/heads/') && endsWith(github.event.ref, '-staging')
     needs: test
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 20
     - name: Push changes to protected branch
@@ -189,22 +247,22 @@ jobs:
         git push origin "$TARGET"
         git push origin --delete "$BRANCH"
 
+
   pypi:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    permissions:
+      id-token: write
     needs: test
     steps:
     - name: Download artifacts
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v4
       with:
         name: packages
         path: dist
 
     - name: Publish package
-      uses: pypa/gh-action-pypi-publish@v1.1.0
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_password }}
+      uses: pypa/gh-action-pypi-publish@release/v1
 
     - if: failure()
       run: ls -R

.github/workflows/docker.yml

@@ -0,0 +1,64 @@
+name: Publish Docker images
+on:
+  push:
+    branches:
+      - dev
+      - beta
+      - stable
+  workflow_dispatch:
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'Gallopsled'
+    steps:
+      # Required for subdirectories in Git context
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push base image
+        uses: docker/build-push-action@v5
+        if: github.event_name == 'workflow_dispatch'
+        with:
+          context: "{{defaultContext}}:extra/docker/base"
+          push: true
+          tags: pwntools/pwntools:base
+
+      - name: Build and push stable image
+        uses: docker/build-push-action@v5
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/stable')
+        with:
+          context: "{{defaultContext}}:extra/docker/stable"
+          push: true
+          tags: pwntools/pwntools:stable
+
+      - name: Build and push beta image
+        uses: docker/build-push-action@v5
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/beta')
+        with:
+          context: "{{defaultContext}}:extra/docker/beta"
+          push: true
+          tags: pwntools/pwntools:beta
+
+      - name: Build and push dev image
+        uses: docker/build-push-action@v5
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/dev')
+        with:
+          context: "{{defaultContext}}:extra/docker/dev"
+          push: true
+          tags: |
+            pwntools/pwntools:dev
+            pwntools/pwntools:latest
+
+      - name: Build and push ci image
+        uses: docker/build-push-action@v5
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/dev')
+        with:
+          context: "{{defaultContext}}:travis/docker"
+          push: true
+          tags: pwntools/pwntools:ci

.github/workflows/lint.yml

@@ -10,16 +10,16 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Cache for pip
-      uses: actions/cache@v1
+      uses: actions/cache@v4
       id: cache-pip
       with:
         path: ~/.cache/pip
         key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/merge-conflict.yml

@@ -1,5 +1,7 @@
 name: Check for merge markers
-on: [pull_request]
+on:
+  pull_request:
+    types: [synchronize]
 env:
   GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
 
@@ -11,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 5
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 

.github/workflows/pylint.yml

@@ -10,16 +10,16 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Cache for pip
-      uses: actions/cache@v1
+      uses: actions/cache@v4
       id: cache-pip
       with:
         path: ~/.cache/pip
         key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -27,11 +27,11 @@ jobs:
       run: |
         set -x
         pip install pylint
-        pip install --upgrade -r requirements.txt
-        pylint --exit-zero --errors-only  pwnlib > current.txt
+        pip install --upgrade -e .
+        pylint --exit-zero --errors-only pwnlib -f parseable | cut -d ' ' -f2- > current.txt
         git fetch origin
         git checkout origin/"$GITHUB_BASE_REF"
-        pylint --exit-zero --errors-only  pwnlib > base.txt
+        pylint --exit-zero --errors-only pwnlib -f parseable | cut -d ' ' -f2- > base.txt
         if diff base.txt current.txt | grep '>'; then
           false
         fi

.gitignore

@@ -21,3 +21,4 @@ venv
 .idea
 __pycache__
 !.github
+.DS_Store

.readthedocs.yaml

@@ -0,0 +1,15 @@
+# https://docs.readthedocs.io/en/stable/config-file/v2.html
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3"
+
+sphinx:
+  configuration: docs/source/conf.py
+
+python:
+  install:
+  - requirements: docs/requirements.txt

.travis.yml

@@ -1,66 +0,0 @@
-language: python
-addons:
-  apt:
-    packages:
-    - lib32stdc++6
-    - ash
-    - bash
-    - dash
-    - ksh
-    - mksh
-    - zsh
-    - pandoc
-    - gdb
-    - socat
-    - sshpass
-    - binutils
-    - qemu-user-static
-    - binutils-multiarch
-    - binutils-aarch64-linux-gnu
-    - binutils-arm-linux-gnueabihf
-    - binutils-mips-linux-gnu
-    - binutils-msp430
-    - binutils-powerpc-linux-gnu
-    - binutils-s390x-linux-gnu
-    - binutils-sparc64-linux-gnu
-    - bash-static
-    - python3-pip
-cache:
-    - pip
-    - directories:
-        - usr
-        - /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/
-        - /home/travis/virtualenv/python2.7.15/bin/
-        - /home/travis/virtualenv/python3.8.*/lib/python3.8/site-packages/
-        - /home/travis/virtualenv/python3.8.*/bin/
-        - docs/build/doctest/.doctrees
-python:
-  - "2.7"
-  - "3.8"
-before_install:
-  - echo $-
-  - source travis/install.sh
-  - source travis/ssh_setup.sh
-  - echo $-
-install:
-  - /usr/bin/python3 -m pip install rpyc # for GDB
-  - pip install --upgrade pip
-  - pip install --upgrade flake8 appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
-  - python setup.py egg_info      # pypa/pip#6275
-  - pip install --upgrade --editable .
-  - pip install --upgrade --requirement docs/requirements.txt
-before_script:
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - sudo sh -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
-script:
-  # stop the build if there are Python syntax errors or undefined names
-  - flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
-  # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
-  - PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
-  - coverage combine
-after_success:
-  - coveralls
-  - source travis/update_demo.sh

CHANGELOG.md

@@ -9,9 +9,17 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.9.0](#490)    | `dev`    |
-| [4.8.0](#480)    | `beta`   |
-| [4.7.0](#470)    | `stable` | Nov 15, 2021
+| [4.15.0](#4150-dev)  | `dev`    |
+| [4.14.0](#4140-beta)  | `beta`   |
+| [4.13.0](#4130-stable)  | `stable`    | Aug 12, 2024
+| [4.12.0](#4120)  |          | Feb 22, 2024
+| [4.11.1](#4111)  |          | Nov 14, 2023
+| [4.11.0](#4110)  |          | Sep 15, 2023
+| [4.10.0](#4100)  |          | May 21, 2023
+| [4.9.0](#490)    |          | Dec 29, 2022
+| [4.8.0](#480)    |          | Apr 21, 2022
+| [4.7.1](#471)    |          | Apr 20, 2022
+| [4.7.0](#470)    |          | Nov 15, 2021
 | [4.6.0](#460)    |          | Jul 12, 2021
 | [4.5.1](#451)    |          | May 30, 2021
 | [4.5.0](#450)    |          | Apr 30, 2021
@@ -62,11 +70,213 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.9.0 (`dev`)
+## 4.15.0 (`dev`)
 
 
 
-## 4.8.0 (`beta`)
+## 4.14.0 (`beta`)
+
+- [#2356][2356] Add local libc database provider for libcdb
+- [#2360][2360] Add offline parameter for `search_by_hash` series function
+- [#2388][2388] libcdb: add `offline_only` to `search_by_symbol_offsets`
+- [#2374][2374] libcdb.unstrip_libc: debug symbols are fetched only if not present
+- [#2327][2327] Add basic support to debug processes on Windows
+- [#2437][2437] Support asm/disasm on Windows
+- [#2330][2330] Change `context.newline` when setting `context.os` to `"windows"`
+- [#2322][2322] Add basic RISCV64 shellcraft support
+- [#2376][2376] Return buffered data on first EOF in tube.readline()
+- [#2371][2371] Add functions for retrieving process mappings
+- [#2398][2398] Add support for generating multiple shellcodes at a time in shellcraft
+- [#2389][2389] Fix passing bytes to `context.log_file` and `crc.BitPolynom`
+- [#2391][2391] Fix error message when passing invalid kwargs to `xor`
+- [#2387][2387] Convert apport_corefile() output from bytes-like object to string
+- [#2415][2415] Add shellcraft template for IPv6 socket
+- [#2405][2405] Add "none" ssh authentication method
+- [#2427][2427] Document behaviour of remote()'s sni argument as string.
+- [#2382][2382] added optional port, gdb_args and gdbserver_args parameters to gdb.debug()
+- [#2435][2435] Speed up gdbserver handshake in gdb.debug()
+- [#2436][2436] Add resolution_addr parameter to Ret2dlresolvePayload
+
+[2436]: https://github.com/Gallopsled/pwntools/pull/2436
+[2371]: https://github.com/Gallopsled/pwntools/pull/2371
+[2360]: https://github.com/Gallopsled/pwntools/pull/2360
+[2356]: https://github.com/Gallopsled/pwntools/pull/2356
+[2374]: https://github.com/Gallopsled/pwntools/pull/2374
+[2327]: https://github.com/Gallopsled/pwntools/pull/2327
+[2322]: https://github.com/Gallopsled/pwntools/pull/2322
+[2330]: https://github.com/Gallopsled/pwntools/pull/2330
+[2389]: https://github.com/Gallopsled/pwntools/pull/2389
+[2391]: https://github.com/Gallopsled/pwntools/pull/2391
+[2376]: https://github.com/Gallopsled/pwntools/pull/2376
+[2387]: https://github.com/Gallopsled/pwntools/pull/2387
+[2388]: https://github.com/Gallopsled/pwntools/pull/2388
+[2398]: https://github.com/Gallopsled/pwntools/pull/2398
+[2415]: https://github.com/Gallopsled/pwntools/pull/2415
+[2405]: https://github.com/Gallopsled/pwntools/pull/2405
+[2427]: https://github.com/Gallopsled/pwntools/pull/2405
+[2382]: https://github.com/Gallopsled/pwntools/pull/2382
+[2435]: https://github.com/Gallopsled/pwntools/pull/2435
+[2437]: https://github.com/Gallopsled/pwntools/pull/2437
+
+## 4.13.0 (`stable`)
+
+- [#2242][2242] Term module revamp: activating special handling of terminal only when necessary
+- [#2277][2277] elf: Resolve more relocations into GOT entries
+- [#2281][2281] FIX: Getting right amount of data for search fix
+- [#2293][2293] Add x86 CET status to checksec output
+- [#1763][1763] Allow to add to the existing environment in `process` instead of replacing it
+- [#2307][2307] Fix `pwn libcdb file` crashing if "/bin/sh" string was not found
+- [#2309][2309] Detect challenge binary and libc in `pwn template`
+- [#2308][2308] Fix WinExec shellcraft to make sure it's 16 byte aligned
+- [#2279][2279] Make `pwn template` always set context.binary
+- [#2310][2310] Add support to start a process on Windows
+- [#2335][2335] Add lookup optimizations in DynELF 
+- [#2334][2334] Speed up disasm commandline tool with colored output
+- [#2328][2328] Lookup using $PATHEXT file extensions in `which` on Windows
+- [#2189][2189] Explicitly define p64/u64 functions for IDE support
+- [#2339][2339] Fix: Allow setting attributes on gdb Breakpoints
+- [#2323][2323] Retry failed lookups after one week in libcdb
+- [#2325][2325] Match against local system libc first in libcdb
+- [#2336][2336] Add `ELF.stripped` and `ELF.debuginfo` properties
+- [#2161][2161] Add basic support for darwin shellcraft/asm/disasm/run_shellcode/run_assembly
+- [#2161][2161] Fix freebsd amd64 SyscallABI
+- [#2160][2161] Fix invalid shellcraft.mov on arm64
+- [#2284][2161] Fix invalid shellcraft.pushstr_array on arm64
+- [#2345][2345] Fix pwn constgrep when it matches a non-constant type
+- [#2338][2338] Fix: follow symlink for libs on ssh connection
+- [#2341][2341] Launch GDB correctly in iTerm on Mac
+- [#2268][2268] Add a `flatten` argument to `ssh.libs`
+- [#2347][2347] Fix/workaround Unicorn Engine 1GB limit that calls exit()
+- [#2233][2233] Fix gdb.debug: exe parameter now respected, allow empty argv
+- [#2373][2373] Fix displaying bright color variation in terminal output
+- [#2378][2378] Don't go though a shell in `gdb.debug`
+
+[2242]: https://github.com/Gallopsled/pwntools/pull/2242
+[2277]: https://github.com/Gallopsled/pwntools/pull/2277
+[2281]: https://github.com/Gallopsled/pwntools/pull/2281
+[2293]: https://github.com/Gallopsled/pwntools/pull/2293
+[1763]: https://github.com/Gallopsled/pwntools/pull/1763
+[2307]: https://github.com/Gallopsled/pwntools/pull/2307
+[2309]: https://github.com/Gallopsled/pwntools/pull/2309
+[2308]: https://github.com/Gallopsled/pwntools/pull/2308
+[2279]: https://github.com/Gallopsled/pwntools/pull/2279
+[2310]: https://github.com/Gallopsled/pwntools/pull/2310
+[2335]: https://github.com/Gallopsled/pwntools/pull/2335
+[2334]: https://github.com/Gallopsled/pwntools/pull/2334
+[2328]: https://github.com/Gallopsled/pwntools/pull/2328
+[2189]: https://github.com/Gallopsled/pwntools/pull/2189
+[2339]: https://github.com/Gallopsled/pwntools/pull/2339
+[2323]: https://github.com/Gallopsled/pwntools/pull/2323
+[2325]: https://github.com/Gallopsled/pwntools/pull/2325
+[2336]: https://github.com/Gallopsled/pwntools/pull/2336
+[2161]: https://github.com/Gallopsled/pwntools/pull/2161
+[2345]: https://github.com/Gallopsled/pwntools/pull/2345
+[2338]: https://github.com/Gallopsled/pwntools/pull/2338
+[2341]: https://github.com/Gallopsled/pwntools/pull/2341
+[2268]: https://github.com/Gallopsled/pwntools/pull/2268
+[2347]: https://github.com/Gallopsled/pwntools/pull/2347
+[2233]: https://github.com/Gallopsled/pwntools/pull/2233
+[2373]: https://github.com/Gallopsled/pwntools/pull/2373
+[2378]: https://github.com/Gallopsled/pwntools/pull/2378
+
+## 4.12.0
+
+- [#2202][2202] Fix `remote` and `listen` in sagemath
+- [#2117][2117] Add -p (--prefix) and -s (--separator) arguments to `hex` command
+- [#2221][2221] Add shellcraft.sleep template wrapping SYS_nanosleep
+- [#2219][2219] Fix passing arguments on the stack in shellcraft syscall template
+- [#2212][2212] Add `--libc libc.so` argument to `pwn template` command
+- [#2257][2257] Allow creation of custom templates for `pwn template` command
+- [#2225][2225] Allow empty argv in ssh.process()
+- [#2349][2349] Fix term.readline omitting a trailing \n
+- [#2352][2352] add `RETURN_CONST` as an allowed `_const_code` in safeeval
+
+[2202]: https://github.com/Gallopsled/pwntools/pull/2202
+[2117]: https://github.com/Gallopsled/pwntools/pull/2117
+[2221]: https://github.com/Gallopsled/pwntools/pull/2221
+[2219]: https://github.com/Gallopsled/pwntools/pull/2219
+[2212]: https://github.com/Gallopsled/pwntools/pull/2212
+[2257]: https://github.com/Gallopsled/pwntools/pull/2257
+[2225]: https://github.com/Gallopsled/pwntools/pull/2225
+[2349]: https://github.com/Gallopsled/pwntools/pull/2349
+[2352]: https://github.com/Gallopsled/pwntools/pull/2352
+
+## 4.11.1
+
+- [#2271][2271] FIX: Generated shebang with path to python invalid if path contains spaces
+- [#2272][2272] Fix `tube.clean_and_log` not logging buffered data
+- [#2281][2281] FIX: Getting right amount of data for search fix
+- [#2287][2287] Fix `_countdown_handler` not invoking `timeout_change`
+- [#2294][2294] Fix atexit SEGV in aarch64 loader
+
+[2271]: https://github.com/Gallopsled/pwntools/pull/2271
+[2272]: https://github.com/Gallopsled/pwntools/pull/2272
+[2281]: https://github.com/Gallopsled/pwntools/pull/2281
+[2287]: https://github.com/Gallopsled/pwntools/pull/2287
+[2294]: https://github.com/Gallopsled/pwntools/pull/2294
+
+## 4.11.0
+
+- [#2185][2185] make fmtstr module able to create payload without $ notation 
+- [#2103][2103] Add search for libc binary by leaked function addresses `libcdb.search_by_symbol_offsets()`
+- [#2177][2177] Support for RISC-V 64-bit architecture
+- [#2186][2186] Enhance `ELF.nx` and `ELF.execstack`
+- [#2129][2129] Handle `context.newline` correctly when typing in `tube.interactive()`
+- [#2214][2214] Fix bug at ssh.py:`download` and `download_file` with relative paths
+- [#2241][2241] Fix ssh.process not setting ssh_process.cwd attribute
+- [#2261][2261] Fix corefile module after pyelftools update
+
+[2185]: https://github.com/Gallopsled/pwntools/pull/2185
+[2103]: https://github.com/Gallopsled/pwntools/pull/2103
+[2177]: https://github.com/Gallopsled/pwntools/pull/2177
+[2186]: https://github.com/Gallopsled/pwntools/pull/2186
+[2129]: https://github.com/Gallopsled/pwntools/pull/2129
+[2214]: https://github.com/Gallopsled/pwntools/pull/2214
+[2241]: https://github.com/Gallopsled/pwntools/pull/2241
+[2261]: https://github.com/Gallopsled/pwntools/pull/2261
+
+## 4.10.0
+
+In memoriam — [Zach Riggle][zach] — long time contributor and maintainer of Pwntools.
+
+- [#2062][2062] make pwn cyclic -l work with entry larger than 4 bytes
+- [#2092][2092] shellcraft: dup() is now called dupio() consistently across all supported arches
+- [#2093][2093] setresuid() in shellcraft uses current euid by default
+- [#2125][2125] Allow tube.recvregex to return capture groups
+- [#2144][2144] Removes `p2align 2` `asm()` headers from `x86-32`, `x86-64` and `mips` architectures to avoid inconsistent instruction length when patching binaries
+
+[2062]: https://github.com/Gallopsled/pwntools/pull/2062
+[2092]: https://github.com/Gallopsled/pwntools/pull/2092
+[2093]: https://github.com/Gallopsled/pwntools/pull/2093
+[2125]: https://github.com/Gallopsled/pwntools/pull/2125
+[2144]: https://github.com/Gallopsled/pwntools/pull/2144
+[zach]: https://github.com/zachriggle
+
+## 4.9.0
+
+- [#1975][1975] Add libcdb commandline tool
+- [#1979][1979] Add `js_escape()` and `js_unescape()` to `util.fiddling`
+- [#2011][2011] Fix tube's debug output of same byte compression
+- [#2023][2023] Support KDE Konsole in run_in_new_terminal function
+- [#2027][2027] Fix ELF.libc_start_main_return with glibc 2.34
+- [#2033][2033] Quote file and core path in generated GDB script
+- [#2035][2035] Change Buffer's parent class to object
+- [#2037][2037] Allow SSH tunnel to be treated like a TCP socket (with 'raw=True')
+- [#2123][2123] Fix ROP without a writeable cache directory
+- [#2124][2124] Fix `tube.recvpred()` timeout argument
+
+[1975]: https://github.com/Gallopsled/pwntools/pull/1975
+[1979]: https://github.com/Gallopsled/pwntools/pull/1979
+[2011]: https://github.com/Gallopsled/pwntools/pull/2011
+[2023]: https://github.com/Gallopsled/pwntools/pull/2023
+[2027]: https://github.com/Gallopsled/pwntools/pull/2027
+[2033]: https://github.com/Gallopsled/pwntools/pull/2033
+[2035]: https://github.com/Gallopsled/pwntools/pull/2035
+[2037]: https://github.com/Gallopsled/pwntools/pull/2037
+[2123]: https://github.com/Gallopsled/pwntools/pull/2123
+[2124]: https://github.com/Gallopsled/pwntools/pull/2124
+
+## 4.8.0
 
 - [#1922][1922] Fix logic in `wait_for_debugger`
 - [#1828][1828] libcdb: Load debug info and unstrip libc binary
@@ -78,7 +288,24 @@ The table below shows which release corresponds to each branch, and what date th
 [1939]: https://github.com/Gallopsled/pwntools/pull/1939
 [1981]: https://github.com/Gallopsled/pwntools/pull/1981
 
-## 4.7.0 (`stable`)
+## 4.7.1
+
+- [#1784][1784] Use temporary cache directory when persistent cache cannot be used
+- [#1973][1973] ELF symbols can be looked up by bytes values
+- several bugfixes ([#2012][2012], [#2031][2031], [#1912][1912], [#1961][1961],
+[#2007][2007], [#2040][2040], [#2051][2051])
+
+[1784]: https://github.com/Gallopsled/pwntools/pull/1784
+[1912]: https://github.com/Gallopsled/pwntools/pull/1912
+[1961]: https://github.com/Gallopsled/pwntools/pull/1961
+[1973]: https://github.com/Gallopsled/pwntools/pull/1973
+[2007]: https://github.com/Gallopsled/pwntools/pull/2007
+[2012]: https://github.com/Gallopsled/pwntools/pull/2012
+[2031]: https://github.com/Gallopsled/pwntools/pull/2031
+[2040]: https://github.com/Gallopsled/pwntools/pull/2040
+[2051]: https://github.com/Gallopsled/pwntools/pull/2051
+
+## 4.7.0
 
 - [#1733][1733] Update libc headers -> more syscalls available!
 - [#1876][1876] add `self.message` and change `sys.exc_type` to `sys.exec_info()` in PwnlibException

MANIFEST.in

@@ -1,7 +1,9 @@
+graft build
 graft examples
 graft extra
+graft travis
 include *.md *.txt *.sh *.yml MANIFEST.in
 recursive-include docs   *.rst *.png Makefile *.py *.txt
 recursive-include pwnlib *.py *.asm *.rst *.md *.txt *.sh __doc__ *.mako
 recursive-include pwn    *.py *.asm *.rst *.md *.txt *.sh
-recursive-exclude *.pyc
+global-exclude *.pyc

README.md

@@ -3,10 +3,10 @@
 
 [![PyPI](https://img.shields.io/pypi/v/pwntools?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![Travis](https://img.shields.io/travis/Gallopsled/pwntools/dev?logo=Travis)](https://travis-ci.org/Gallopsled/pwntools)
-[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/Gallopsled/pwntools/Continuous%20Integration/dev?logo=GitHub)](https://github.com/Gallopsled/pwntools/actions?query=workflow%3A%22Continuous+Integration%22+branch%3Adev)
+[![GitHub Workflow Status (dev)](https://img.shields.io/github/actions/workflow/status/Gallopsled/pwntools/ci.yml?branch=dev&logo=GitHub)](https://github.com/Gallopsled/pwntools/actions/workflows/ci.yml?query=branch%3Adev)
 [![Coveralls](https://img.shields.io/coveralls/github/Gallopsled/pwntools/dev?logo=coveralls)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
+[![Packaging status](https://img.shields.io/repology/repositories/python:pwntools)](https://repology.org/project/python:pwntools/versions)
 [![Discord](https://img.shields.io/discord/809590285687980052?label=Discord&style=plastic)](https://discord.gg/96VA2zvjCB)
 [![Twitter](https://img.shields.io/twitter/follow/Pwntools)](https://twitter.com/pwntools)
 

SECURITY.md

@@ -0,0 +1,32 @@
+# Security Policy
+
+## Supported Versions
+
+| Version       | Supported          |
+| ------------- | ------------------ |
+| latest dev    | :white_check_mark: |
+| latest beta   | :white_check_mark: |
+| latest stable | :white_check_mark: |
+| anything else | :x: |
+
+## Reporting a Vulnerability
+
+The aim of pwntools is exploiting software vulnerabilities, which is an unusual position, but it nevertheless can have its own security issues.
+Especially that an attacker (=re-victim) is usually not prepared to be attacked back (by the re-attacker).
+
+The first question to ask yourself is: is this an actual vulnerability?
+- can it be triggered by a re-attacker (malicious honeypot pretending to be a vulnerable service)?
+- does it impact the attacker (=re-victim)?
+- is it serious?
+  * *availability: medium* means *at least* exhausting RAM or disk space of the attacker (=re-victim)
+  * *confidentiality: medium* means *at least* reading the filesystem of the attacker (=re-victim)
+  * *integrity: medium* means *at least* performing uncontrolled actions or data corruption on behalf of the attacker (=re-victim)
+  * if crucial for some sophisticated exploit chain, it is always serious
+  * `safe_eval` bypasses **are** serious.
+  * an example of what was **kind of** serious: [#1732](https://github.com/Gallopsled/pwntools/pull/1732)
+- can it be fixed without compromising on Pwntools' usability?
+
+If at least one of the answers is no, then this is NOT a vulnerability, so just file a bug report or feature request, without the weird confidential disclosure dance.
+
+Just e-mail the maintainers.  Arusekk is the one that is currently the most excited to fix vulnerabilities.
+Or create a CTF task!  Prove a point the good old hacker way!

docs/Makefile

@@ -6,7 +6,6 @@ SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = build
-DASHBUILD     = ./dashbuild.py
 TAR           = tar
 
 # Internal variables.
@@ -145,8 +144,8 @@ gettext:
 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
 
 dash:
-	$(DASHBUILD) source $(BUILDDIR)/dash
-	$(TAR) --exclude='.DS_Store' -cvzf $(BUILDDIR)/dash/pwntools.tgz -C $(BUILDDIR)/dash pwntools.docset
+	$(SPHINXBUILD) -b html -d $(BUILDDIR)/doctrees -t dash source $(BUILDDIR)/html
+	doc2dash $(BUILDDIR)/html -d $(BUILDDIR)/dash -n pwntools -f -I index.html
 	@echo "Build finished. The Dash docset is in $(BUILDDIR)/dash."
 
 changes:

docs/dashbuild.py

@@ -1,99 +0,0 @@
-#!/usr/bin/env python
-
-# -*- coding: utf-8 -*-
-
-#
-# Helper to build Dash docset from sphinx source files.
-#
-# Dash docsets can be read by various applications:
-#
-# Dash         OS X, iOS         https://kapeli.com
-# Zeal         Linux, Windows    https://zealdocs.org
-# Velocity     Windows           http://velocity.silverlakesoftware.com
-# LovelyDocs   Android           http://lovelydocs.io
-# dasht        POSIX             https://github.com/sunaku/dasht
-# Helm Dash    emacs             https://github.com/areina/helm-dash
-#
-
-import argparse
-import doc2dash.__main__
-import os, os.path
-import sphinx
-import sqlite3
-import sys
-
-sys.path.append(os.path.abspath(os.path.join('..', 'pwnlib')))
-import version
-
-def main(args):
-	"""Generate a Dash docset from Sphinx source files."""
-
-	srcdir = args.srcdir
-	dstdir = args.dstdir
-	name = args.name
-
-	if not os.path.exists(dstdir):
-		os.makedirs(dstdir)
-
-	# Generate HTML without indices.
-	sphinx.build_main([ "sphinx-build", "-b", "html", "-d", os.path.join(dstdir, "doctrees"), \
-		"-t", "dash", srcdir, os.path.join(dstdir, "html") ])
-
-	# Convert to docset.
-	try:
-		doc2dash.__main__.main.main( \
-			[ os.path.join(dstdir, "html"), "-d", dstdir, "-n", name, \
-					"-f", "-I", "index.html"], "doc2dash", False)
-	except SystemExit as e:
-		pass
-
-	# Insert a link to the online version.
-	online = args.online
-	if online is not None and online != "":
-		url = online.replace("@VERSION", args.version)
-		with open(os.path.join(dstdir, name+".docset", "Contents", "Info.plist"), "r+") as f_info:
-			pl = f_info.read()
-			pl = pl.replace("</dict>", \
-				"\t<key>DashDocSetFallbackURL</key>\n\t<string>%s</string>\n</dict>" % url)
-			f_info.seek(0)
-			f_info.write(pl)
-			f_info.truncate()
-
-	# Modify the CSS to hide the menu included in the HTML.
-	with open(os.path.join(dstdir, name+".docset", "Contents", "Resources", "Documents", "_static", "css", "theme.css"), "r+") as f_css:
-		css = f_css.read()
-		css = css.replace( \
-			'@media screen and (max-width: 768px){.wy-body-for-nav{background:#fcfcfc}.wy-nav-top{display:block}',\
-			'@media screen {.wy-body-for-nav{background:#fcfcfc}' )
-		css = css.replace( \
-			'@media screen and (max-width: 480px)', \
-			'@media screen ')
-		f_css.seek(0)
-		f_css.write(css)
-		f_css.truncate()
-
-	# Modify the index
-	db_conn = sqlite3.connect(os.path.join(dstdir, name+".docset", "Contents", "Resources", "docSet.dsidx"))
-	try:
-		db_conn.execute('INSERT INTO "searchIndex" ("name","type","path") VALUES ' \
-			'("1 Contents", "Guide", "index.html"), ' \
-			'("2 About pwntools", "Guide", "about.html"), ' \
-			'("3 Installation", "Guide", "install.html"), ' \
-			'("4 Getting Started", "Guide", "intro.html"), ' \
-			'("5 Globals (pwn)", "Guide", "globals.html"), ' \
-			'("6 Command Line Tools", "Guide", "commandline.html")')
-		db_conn.execute('DELETE FROM "searchIndex" WHERE "type" = "Module" AND ("name" = "pwn" OR "name" = "pwnlib")')
-		db_conn.commit()
-	finally:
-		db_conn.close()
-
-	return 0
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--name", help="docset name", default="pwntools")
-parser.add_argument("--online", help="URL for online docs", default="https://pwntools.readthedocs.org/en/@VERSION/")
-parser.add_argument("--version", help="pwntools version", default=version.__version__)
-parser.add_argument("srcdir", help="Source directory containing .rst files")
-parser.add_argument("dstdir", help="Destination and working directory")
-
-main(parser.parse_args())

docs/requirements.txt

@@ -1,7 +1,9 @@
 capstone
-coveralls
+coverage[toml]
+python-dateutil
 doc2dash
 docutils<0.18
+intervaltree
 isort
 mako>=1.0.0
 paramiko>=1.15.2
@@ -10,9 +12,11 @@ pyelftools>=0.2.3
 pygments>=2.0
 pypandoc
 pyserial>=2.7
+pysocks
+psutil
 requests>=2.5.1
 ropgadget>=5.3
-sphinx<3.4
+sphinx==1.8.6; python_version<'3'
+sphinx>=4.5.0; python_version>='3'
 sphinx_rtd_theme
-sphinxcontrib-napoleon
 sphinxcontrib-autoprogram<=0.1.5

docs/source/conf.py

@@ -42,8 +42,8 @@ extensions = [
     'sphinx.ext.coverage',
     'sphinx.ext.todo',
     'sphinx.ext.intersphinx',
+    'sphinx.ext.napoleon',
     'sphinxcontrib.autoprogram',
-    'sphinxcontrib.napoleon'
 ]
 
 # Disable "info" logging directly to stdout by Sphinx
@@ -71,6 +71,7 @@ doctest_global_setup = '''
 import sys, os
 os.environ['PWNLIB_NOTERM'] = '1'
 os.environ['PWNLIB_RANDOMIZE'] = '0'
+import six
 import pwnlib.update
 import pwnlib.util.fiddling
 import logging
@@ -97,8 +98,8 @@ pwnlib.context.ContextType.defaults['log_console'] = stdout()
 github_actions = os.environ.get('USER') == 'runner'
 travis_ci = os.environ.get('USER') == 'travis'
 local_doctest = os.environ.get('USER') == 'pwntools'
-branch_dev = os.environ.get('GITHUB_BASE_REF') == 'dev'
 skip_android = True
+is_python2 = six.PY2
 '''
 
 autoclass_content = 'both'
@@ -169,7 +170,7 @@ pygments_style = 'sphinx'
 
 # The theme to use for HTML and HTML Help pages.  See the documentation for
 # a list of builtin themes.
-html_theme = 'default'
+html_theme = 'sphinx_rtd_theme'
 
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
@@ -265,8 +266,8 @@ latex_documents = [
    u'2016, Gallopsled et al.', 'manual'),
 ]
 
-intersphinx_mapping = {'python': ('https://docs.python.org/3.8', None),
-                       'paramiko': ('https://paramiko-docs.readthedocs.org/en/2.1/', None)}
+intersphinx_mapping = {'python': ('https://docs.python.org/3/', None),
+                       'paramiko': ('https://docs.paramiko.org/en/2.1/', None)}
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
@@ -377,11 +378,10 @@ if build_dash:
     on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
 
     if not on_rtd:  # only import and set the theme if we're building docs locally
-        import sphinx_rtd_theme
-        html_theme = 'sphinx_rtd_theme'
-        html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-
-    # otherwise, readthedocs.org uses their theme by default, so no need to specify it
+        import alabaster
+        html_theme = 'alabaster'
+        html_theme_path = [alabaster.get_path()]
+        html_theme_options = { 'nosidebar' : True }
 
 
 # -- Customization to Sphinx autodoc generation --------------------------------------------

docs/source/elf/elf.rst

@@ -3,6 +3,7 @@
    from pwn import *
    from glob import glob
    from pwnlib.elf.maps import CAT_PROC_MAPS_EXIT
+   import shutil
 
 :mod:`pwnlib.elf.elf` --- ELF Files
 ===========================================================

docs/source/index.rst

@@ -13,7 +13,7 @@ readthedocs_. It comes in three primary flavors:
 - Dev_
 
 .. _readthedocs: https://readthedocs.org
-.. _docs.pwntools.com: https://docs.pwntools.com
+.. _docs.pwntools.com: https://docs.pwntools.com/en/latest
 .. _Stable: https://docs.pwntools.com/en/stable
 .. _Beta: https://docs.pwntools.com/en/beta
 .. _Dev: https://docs.pwntools.com/en/dev
@@ -82,7 +82,6 @@ Each of the ``pwntools`` modules is documented here.
    :hidden:
 
    testexample
-   rop/call
 
 .. only:: not dash
 

docs/source/install.rst

@@ -51,13 +51,11 @@ Command-Line Tools
 
 When installed with ``sudo`` the above commands will install Pwntools' command-line tools to somewhere like ``/usr/bin``.
 
-However, if you run as an unprivileged user, you may see a warning message that looks like this:
+However, if you run as an unprivileged user, you may see a warning message that looks like this::
 
-.. code-block::
-
-      WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
-      elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
-      unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
+    WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
+    elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
+    unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
 
 Follow the instructions listed and add ``~/.local/bin`` to your ``$PATH`` environment variable.
 

docs/source/install/binutils.rst

@@ -5,7 +5,7 @@ Assembly of foreign architectures (e.g. assembling Sparc shellcode on
 Mac OS X) requires cross-compiled versions of ``binutils`` to be
 installed. We've made this process as smooth as we can.
 
-In these examples, replace ``$ARCH`` with your target architecture (e.g., arm, mips64, vax, etc.).
+In these examples, replace ``$ARCH`` with your target architecture (e.g., arm, aarch64, mips64, vax, etc.).
 
 Building `binutils` from source takes about 60 seconds on a modern 8-core machine.
 
@@ -33,7 +33,7 @@ Mac OS X
 
 Mac OS X is just as easy, but requires building binutils from source.
 However, we've made ``homebrew`` recipes to make this a single command.
-After installing `brew <http://brew.sh>`__, grab the appropriate
+After installing `brew <https://brew.sh>`__, grab the appropriate
 recipe from our `binutils
 repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
@@ -51,10 +51,10 @@ OSes, ``binutils`` is simple to build by hand.
 
     #!/usr/bin/env bash
 
-    V=2.25   # Binutils Version
+    V=2.38   # Binutils Version
     ARCH=arm # Target architecture
 
-    cd /tmp
+    cd ${TMPDIR:-/tmp}
     wget -nc https://ftp.gnu.org/gnu/binutils/binutils-$V.tar.gz
     wget -nc https://ftp.gnu.org/gnu/binutils/binutils-$V.tar.gz.sig
 
@@ -70,7 +70,7 @@ OSes, ``binutils`` is simple to build by hand.
     export AS=as
 
     ../binutils-$V/configure \
-        --prefix=/usr/local \
+        --prefix=${PREFIX:-/usr/local} \
         --target=$ARCH-unknown-linux-gnu \
         --disable-static \
         --disable-multilib \

docs/source/rop/rop.rst

@@ -15,6 +15,7 @@
    from pwnlib.tubes.process import process
    from pwnlib import shellcraft
    from pwnlib.util.misc import which
+   import pwnlib.data
 
    context.clear()
 

docs/source/shellcraft/amd64.rst

@@ -17,3 +17,9 @@
 
 .. automodule:: pwnlib.shellcraft.amd64.linux
    :members:
+
+:mod:`pwnlib.shellcraft.amd64.windows`
+---------------------------------------
+
+.. automodule:: pwnlib.shellcraft.amd64.windows
+   :members:

examples/clean_and_log.py

@@ -11,18 +11,24 @@ Solution:
 """
 
 from pwn import *
+from multiprocessing import Process
 
-os.system('''((
-echo prefix sometext ;
-echo prefix someothertext ;
-echo here comes the flag ;
-echo LostInTheInterTubes
-) | nc -l 1337) &
-''')
+def submit_data():
+    with context.quiet:
+        with listen(1337) as io:
+            io.wait_for_connection()
+            io.sendline(b'prefix sometext')
+            io.sendline(b'prefix someothertext')
+            io.sendline(b'here comes the flag')
+            io.sendline(b'LostInTheInterTubes')
 
-r = remote('localhost', 1337)
-atexit.register(r.clean_and_log)
+if __name__ == '__main__':
+    p = Process(target=submit_data)
+    p.start()
 
-while True:
-    line = r.recvline()
-    print(re.findall(r'^prefix (\S+)$', line)[0])
+    r = remote('localhost', 1337)
+    atexit.register(r.clean_and_log)
+
+    while True:
+        line = r.recvline()
+        print(re.findall(br'^prefix (\S+)$', line)[0])

examples/options.py

@@ -4,5 +4,5 @@ Example showing `pwnlib.ui.options()`
 
 from pwn import *
 
-opts = [string.letters[x] for x in range(10)]
+opts = [string.ascii_letters[x] for x in range(12)]
 print('You choose "%s"' % opts[options('Pick one:', opts)])

extra/docker/Makefile

@@ -1,4 +1,4 @@
-subdirs=$(shell find . -type d -depth 1 | xargs basename)
+subdirs=$(shell find . -mindepth 1 -maxdepth 1 -type d | xargs -n1 basename)
 tags=base stable beta dev
 ROOT=$(shell git rev-parse --show-toplevel)
 CMD ?= zsh
@@ -7,7 +7,7 @@ ifneq ($(HISTFILE),)
 MOUNT_HISTFILE=--mount type=bind,source="$(HISTFILE)",target=/home/pwntools/.history
 endif
 
-all: $(subdirs) $(tags)
+all: $(tags) $(subdirs)
 
 tags: $(tags)
 
@@ -36,4 +36,4 @@ clean:
 
 FORCE:
 
-.PHONY: all $(subdirs) $(makefiles) $(tags)
+.PHONY: all $(subdirs) $(makefiles) $(tags)

extra/docker/base/Dockerfile

@@ -3,28 +3,30 @@
 # Based on Ubuntu
 ############################################################
 
-FROM ubuntu:bionic
+FROM ubuntu:jammy
 MAINTAINER Maintainer Gallopsled et al.
 
-env DEBIAN_FRONTEND=noninteractive
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
+ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update \
-    && apt-get install locales \
-    && locale-gen en_US.UTF-8 \
     && apt-get install -y \
+        sudo \
+        locales \
         build-essential \
+        elfutils \
         git \
         libssl-dev \
         libffi-dev \
-        python \
-        python-pip \
-        python-dev \
+        python2.7 \
+        python2.7-dev \
+        python2-pip-whl \
         python3 \
         python3-pip \
         python3-dev \
+        python-is-python3 \
         qemu-user-static \
         binutils-arm-linux-gnueabihf \
         binutils-aarch64-linux-gnu \
@@ -33,15 +35,17 @@ RUN apt-get update \
         binutils-powerpc-linux-gnu \
         binutils-powerpc64-linux-gnu \
         binutils-sparc64-linux-gnu \
+        binutils-riscv64-linux-gnu \
         tmux \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && pip3 install --upgrade pip \
+        patchelf \
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8 \
+    && PYTHONPATH=`echo /usr/share/python-wheels/pip-*.whl` python2.7 -m pip install --upgrade pip setuptools wheel \
+    && python2.7 -m pip install --upgrade pwntools \
+    && python3 -m pip install --upgrade pip \
     && python3 -m pip install --upgrade pwntools \
     && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
     && useradd -m pwntools \
     && passwd --delete --unlock pwntools \
     && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
 USER pwntools
-WORKDIR /home/pwntools

extra/docker/beta/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@beta
+RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta \
+    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@beta
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/dev/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@dev
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@dev \
+    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@dev
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/develop/Dockerfile

@@ -5,8 +5,8 @@ ENV HISTFILE=/home/pwntools/.history
 
 # Uninstall existing versions of pwntools
 USER root
-RUN python -m pip uninstall -q -y pwntools \
- && python3 -m pip uninstall -q -y pwntools
+RUN python2.7 -m pip uninstall -q -y pwntools \
+    && python3 -m pip uninstall -q -y pwntools
 
 # Switch back to the pwntools user from here forward
 USER pwntools
@@ -18,20 +18,23 @@ ENV PATH="/home/pwntools/.local/bin:${PATH}"
 
 # Install Pwntools to the home directory, make it an editable install
 RUN git clone https://github.com/Gallopsled/pwntools \
- && python  -m pip install --upgrade --editable pwntools \
+ && python2.7 -m pip install --upgrade --editable pwntools \
  && python3 -m pip install --upgrade --editable pwntools \
  && PWNLIB_NOTERM=1 pwn version
 
 # Requirements for running the tests
-RUN python  -m pip install --upgrade --requirement pwntools/docs/requirements.txt \
- && python3 -m pip install --upgrade --requirement pwntools/docs/requirements.txt
+RUN python2.7 -m pip install --upgrade --requirement pwntools/docs/requirements.txt \
+    && python3 -m pip install --upgrade --requirement pwntools/docs/requirements.txt
 
 # Python niceties for debugging
-RUN python  -m pip install -U ipython ipdb \
- && python3 -m pip install -U ipython ipdb
+RUN python2.7 -m pip install -U ipython ipdb \
+    && python3 -m pip install -U ipython ipdb
 
 # Dependencies from .travis.yml addons -> apt -> packages
-RUN sudo apt-get install -y \
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ="UTC"
+RUN sudo apt-get update && sudo -E apt-get install -y \
+	tzdata \
 	ash \
 	bash \
 	bash-static \
@@ -41,7 +44,7 @@ RUN sudo apt-get install -y \
 	dash \
 	gcc \
 	gcc-multilib \
-	gdb \ 
+	gdb \
 	ksh \
 	lib32stdc++6 \
 	libc6-dev-i386 \
@@ -51,12 +54,9 @@ RUN sudo apt-get install -y \
 	socat \
 	sshpass \
 	vim \
-	zsh
-
+	zsh \
 # Misc useful things when developing
-RUN sudo apt-get install -y  \
 	curl \
-	ipython \
 	ipython3 \
 	lsb-release \
 	ssh \
@@ -64,7 +64,7 @@ RUN sudo apt-get install -y  \
 	wget
 
 # Use zsh by default
-RUN sudo chsh -s /bin/zsh pwntools
+RUN sudo -E chsh -s /bin/zsh pwntools
 
 # Get and install prezto
 RUN git clone --recursive https://github.com/sorin-ionescu/prezto.git .zprezto
@@ -83,4 +83,4 @@ ADD 10-import.py       /home/pwntools/.ipython/profile_default/startup
 ADD ipython_config.py  /home/pwntools/.ipython/profile_default
 
 # Do not require password for sudo
-RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/travis
+RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/travis

extra/docker/stable/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:base
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@stable
+RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable \
+    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

pwn/toplevel.py

@@ -16,7 +16,6 @@ import tempfile
 import threading
 import time
 
-import colored_traceback
 from pprint import pprint
 
 import pwnlib
@@ -85,7 +84,12 @@ info    = log.info
 debug   = log.debug
 success = log.success
 
-colored_traceback.add_hook()
+try:
+    import colored_traceback
+except ImportError:
+    pass
+else:
+    colored_traceback.add_hook()
 
 # Equivalence with the default behavior of "from import *"
 # __all__ = [x for x in tuple(globals()) if not x.startswith('_')]

pwnlib/abi.py

@@ -48,6 +48,8 @@ class ABI(object):
         (32, 'mips', 'linux'):   linux_mips,
         (32, 'powerpc', 'linux'): linux_ppc,
         (64, 'powerpc', 'linux'): linux_ppc64,
+        (32, 'riscv32', 'linux'): linux_riscv32,
+        (64, 'riscv64', 'linux'): linux_riscv64,
         (32, 'i386', 'freebsd'):  freebsd_i386,
         (64, 'aarch64', 'freebsd'): freebsd_aarch64,
         (64, 'amd64', 'freebsd'): freebsd_amd64,
@@ -58,6 +60,8 @@ class ABI(object):
         (64, 'powerpc', 'freebsd'): freebsd_ppc64,
         (32, 'i386', 'windows'):  windows_i386,
         (64, 'amd64', 'windows'): windows_amd64,
+        (64, 'amd64', 'darwin'): darwin_amd64,
+        (64, 'aarch64', 'darwin'): darwin_aarch64,
         }[(context.bits, context.arch, context.os)]
 
     @staticmethod
@@ -76,6 +80,8 @@ class ABI(object):
         (64, 'aarch64', 'linux'):   linux_aarch64_syscall,
         (32, 'powerpc', 'linux'): linux_ppc_syscall,
         (64, 'powerpc', 'linux'): linux_ppc64_syscall,
+        (32, 'riscv32', 'linux'): linux_riscv32_syscall,
+        (64, 'riscv64', 'linux'): linux_riscv64_syscall,
         (32, 'i386', 'freebsd'):  freebsd_i386_syscall,
         (64, 'amd64', 'freebsd'): freebsd_amd64_syscall,
         (64, 'aarch64', 'freebsd'): freebsd_aarch64_syscall,
@@ -85,6 +91,8 @@ class ABI(object):
         (64, 'aarch64', 'freebsd'):   freebsd_aarch64_syscall,
         (32, 'powerpc', 'freebsd'): freebsd_ppc_syscall,
         (64, 'powerpc', 'freebsd'): freebsd_ppc64_syscall,
+        (64, 'amd64', 'darwin'): darwin_amd64_syscall,
+        (64, 'aarch64', 'darwin'): darwin_aarch64_syscall,
         }[(context.bits, context.arch, context.os)]
 
     @staticmethod
@@ -99,13 +107,18 @@ class ABI(object):
         (32, 'arm', 'linux'):   linux_arm_sigreturn,
         (32, 'thumb', 'linux'):   linux_arm_sigreturn,
         (64, 'aarch64', 'linux'):   linux_aarch64_sigreturn,
+        (32, 'riscv32', 'linux'):   linux_riscv32_sigreturn,
+        (64, 'riscv64', 'linux'):   linux_riscv64_sigreturn,
         (32, 'i386', 'freebsd'):  freebsd_i386_sigreturn,
         (64, 'amd64', 'freebsd'): freebsd_amd64_sigreturn,
         (32, 'arm', 'freebsd'):   freebsd_arm_sigreturn,
         (32, 'thumb', 'freebsd'):   freebsd_arm_sigreturn,
         (64, 'aarch64', 'freebsd'):   freebsd_aarch64_sigreturn,
+        (64, 'amd64', 'darwin'):   darwin_amd64_sigreturn,
+        (64, 'aarch64', 'darwin'):   darwin_aarch64_sigreturn,
         }[(context.bits, context.arch, context.os)]
 
+
 class SyscallABI(ABI):
     """
     The syscall ABI treats the syscall number as the zeroth argument,
@@ -115,6 +128,7 @@ class SyscallABI(ABI):
         super(SyscallABI, self).__init__(*a, **kw)
         self.syscall_register = self.register_arguments[0]
 
+
 class SigreturnABI(SyscallABI):
     """
     The sigreturn ABI is similar to the syscall ABI, except that
@@ -132,6 +146,8 @@ linux_aarch64 = ABI('sp', ['x0', 'x1', 'x2', 'x3'], 16, 0)
 linux_mips  = ABI('$sp', ['$a0','$a1','$a2','$a3'], 4, 0)
 linux_ppc = ABI('sp', ['r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10'], 4, 0)
 linux_ppc64 = ABI('sp', ['r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10'], 8, 0)
+linux_riscv32 = ABI('sp', ['a0', 'a1', 'a2', 'a3', 'a4', 'a5', 'a6', 'a7'], 8, 0)
+linux_riscv64 = ABI('sp', ['a0', 'a1', 'a2', 'a3', 'a4', 'a5', 'a6', 'a7'], 8, 0)
 
 sysv_i386 = linux_i386
 sysv_amd64 = linux_amd64
@@ -140,24 +156,33 @@ sysv_aarch64 = linux_aarch64
 sysv_mips = linux_mips
 sysv_ppc = linux_ppc
 sysv_ppc64 = linux_ppc64
+sysv_riscv32 = linux_riscv32
+sysv_riscv64 = linux_riscv64
 
+# Docs: https://man7.org/linux/man-pages/man2/syscall.2.html
 linux_i386_syscall = SyscallABI('esp', ['eax', 'ebx', 'ecx', 'edx', 'esi', 'edi', 'ebp'], 4, 0)
 linux_amd64_syscall = SyscallABI('rsp', ['rax', 'rdi', 'rsi', 'rdx', 'r10', 'r8', 'r9'],   8, 0)
-linux_arm_syscall   = SyscallABI('sp', ['r7', 'r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6'], 4, 0)
-linux_aarch64_syscall   = SyscallABI('sp', ['x8', 'x0', 'x1', 'x2', 'x3', 'x4', 'x5', 'x6'], 16, 0)
+linux_arm_syscall = SyscallABI('sp', ['r7', 'r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6'], 4, 0)
+linux_aarch64_syscall = SyscallABI('sp', ['x8', 'x0', 'x1', 'x2', 'x3', 'x4', 'x5'], 16, 0)
 linux_mips_syscall  = SyscallABI('$sp', ['$v0','$a0','$a1','$a2','$a3'], 4, 0)
-linux_ppc_syscall = ABI('sp', ['r0', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9'], 4, 0)
-linux_ppc64_syscall = ABI('sp', ['r0', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9'], 8, 0)
+linux_ppc_syscall = SyscallABI('sp', ['r0', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9'], 4, 0)
+linux_ppc64_syscall = SyscallABI('sp', ['r0', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8'], 8, 0)
+linux_riscv32_syscall = SyscallABI('sp', ['a7', 'a0', 'a1', 'a2', 'a3', 'a4', 'a5'], 4, 0)
+linux_riscv64_syscall = SyscallABI('sp', ['a7', 'a0', 'a1', 'a2', 'a3', 'a4', 'a5'], 8, 0)
 
 linux_i386_sigreturn = SigreturnABI('esp', ['eax'], 4, 0)
-linux_amd64_sigreturn = SigreturnABI('rsp', ['rax'], 4, 0)
+linux_amd64_sigreturn = SigreturnABI('rsp', ['rax'], 8, 0)
 linux_arm_sigreturn = SigreturnABI('sp', ['r7'], 4, 0)
 linux_aarch64_sigreturn = SigreturnABI('sp', ['x8'], 16, 0)
+linux_riscv32_sigreturn = SigreturnABI('sp', ['a7'], 4, 0)
+linux_riscv64_sigreturn = SigreturnABI('sp', ['a7'], 8, 0)
 
 sysv_i386_sigreturn = linux_i386_sigreturn
 sysv_amd64_sigreturn = linux_amd64_sigreturn
 sysv_arm_sigreturn = linux_arm_sigreturn
 sysv_aarch64_sigreturn = linux_aarch64_sigreturn
+sysv_riscv32_sigreturn = linux_riscv32_sigreturn
+sysv_riscv64_sigreturn = linux_riscv64_sigreturn
 
 freebsd_i386 = sysv_i386
 freebsd_amd64 = sysv_amd64
@@ -168,7 +193,7 @@ freebsd_ppc = sysv_ppc
 freebsd_ppc64 = sysv_ppc64
 
 freebsd_i386_syscall   = SyscallABI('esp', ['eax'], 4, 0)
-freebsd_amd64_syscall  = SyscallABI('rsp', ['rax','rdi','rsi','rdx','rcx','r8','r9'], 8, 0)
+freebsd_amd64_syscall  = SyscallABI('rsp', ['rax','rdi','rsi','rdx','r10','r8','r9'], 8, 0)
 freebsd_arm_syscall    = SyscallABI('sp', ['r7', 'r0', 'r1', 'r2', 'r3'], 8, 0)
 freebsd_aarch64_syscall = SyscallABI('sp', ['x8', 'x0', 'x1', 'x2', 'x3'], 16, 0)
 freebsd_mips_syscall  = SyscallABI('$sp', ['$v0','$a0','$a1','$a2','$a3'], 4, 0)
@@ -182,3 +207,11 @@ freebsd_aarch64_sigreturn = sysv_aarch64_sigreturn
 
 windows_i386  = ABI('esp', [], 4, 0)
 windows_amd64 = ABI('rsp', ['rcx','rdx','r8','r9'], 32, 32)
+
+darwin_aarch64 = sysv_aarch64
+darwin_aarch64_syscall = SyscallABI('sp', ['x16', 'x0', 'x1', 'x2', 'x3', 'x4', 'x5'], 16, 0)
+darwin_aarch64_sigreturn = SigreturnABI('sp', ['x16'], 16, 0)
+
+darwin_amd64 = sysv_amd64
+darwin_amd64_syscall = SyscallABI('rsp', ['rax', 'rdi', 'rsi', 'rdx', 'r10', 'r8', 'r9'], 8, 0)
+darwin_amd64_sigreturn = SigreturnABI('rsp', ['rax'], 8, 0)

pwnlib/adb/adb.py

@@ -66,6 +66,7 @@ from pwnlib import tubes
 from pwnlib.context import LocalContext
 from pwnlib.context import context
 from pwnlib.device import Device
+from pwnlib.exception import PwnlibException
 from pwnlib.log import getLogger
 from pwnlib.protocols.adb import AdbClient
 from pwnlib.util.packing import _decode
@@ -122,7 +123,7 @@ def current_device(any=False):
 
         >>> device = adb.current_device(any=True)
         >>> device  # doctest: +ELLIPSIS
-        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone_armv7', model='sdk ...phone armv7', device='generic')
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone_...', model='...', device='generic...')
         >>> device.port
         'emulator'
     """
@@ -252,13 +253,13 @@ class AdbDevice(Device):
 
         >>> device = adb.wait_for_device()
         >>> device.arch
-        'arm'
+        'amd64'
         >>> device.bits
-        32
+        64
         >>> device.os
         'android'
         >>> device.product  # doctest: +ELLIPSIS
-        'sdk_...phone_armv7'
+        'sdk_...phone_...'
         >>> device.serial
         'emulator-5554'
     """
@@ -384,6 +385,9 @@ class AdbDevice(Device):
             >>> adb.getprop(property) == device.getprop(property)
             True
         """
+        if name.startswith('_'):
+            raise AttributeError(name)
+
         with context.local(device=self):
             g = globals()
 
@@ -1361,14 +1365,18 @@ def compile(source):
         >>> filename = adb.compile(temp)
         >>> sent = adb.push(filename, "/data/local/tmp")
         >>> adb.process(sent).recvall() # doctest: +ELLIPSIS
-        b'... /system/bin/linker\n...'
+        b'... /system/lib64/libc.so\n...'
     """
 
     ndk_build = misc.which('ndk-build')
     if not ndk_build:
         # Ensure that we can find the NDK.
-        ndk = os.environ.get('NDK', None)
-        if ndk is None:
+        for envvar in ('NDK', 'ANDROID_NDK', 'ANDROID_NDK_ROOT',
+                       'ANDROID_NDK_HOME', 'ANDROID_NDK_LATEST_HOME'):
+            ndk = os.environ.get(envvar)
+            if ndk is not None:
+                break
+        else:
             log.error('$NDK must be set to the Android NDK directory')
         ndk_build = os.path.join(ndk, 'ndk-build')
 
@@ -1483,8 +1491,9 @@ class Partitions(object):
     @context.quietfunc
     def by_name_dir(self):
         try:
-            return next(find('/dev/block/platform','by-name'))
-        except StopIteration:
+            with context.local(log_level=logging.FATAL):
+                return next(find('/dev/block/platform','by-name'))
+        except (StopIteration, PwnlibException):
             return '/dev/block'
 
     @context.quietfunc
@@ -1513,7 +1522,7 @@ class Partitions(object):
         return iter(names)
 
     def __getattr__(self, attr):
-        if attr.startswith("_"):
+        if attr.startswith('_'):
             raise AttributeError(attr)
 
         for name in self:
@@ -1584,7 +1593,10 @@ def uninstall(package, *arguments):
 @context.quietfunc
 def packages():
     """Returns a list of packages installed on the system"""
-    packages = process(['pm', 'list', 'packages']).recvall()
+    # Decodes the received bytes as UTF-8 per:
+    # https://developer.android.com/reference/java/nio/charset/Charset#defaultCharset()
+    # where it is specified that UTF-8 is the default charset for Android.
+    packages = process(['pm', 'list', 'packages']).recvall().decode('utf-8')
     return [line.split('package:', 1)[-1] for line in packages.splitlines()]
 
 @context.quietfunc

pwnlib/adb/bootimg.py

@@ -46,7 +46,7 @@ class BootImage(object):
         self.kernel = self.data[PAGE:PAGE+self.kernel_size]
 
     def __getattr__(self, name):
-        value = getattr(self.header, name, None)
-        if value is not None:
-            return value
-        return getattr(super(BootImage, self), name)
+        if name.startswith('_'):
+            raise AttributeError(name)
+
+        return getattr(self.header, name)

pwnlib/adb/bootloader.py

@@ -114,10 +114,11 @@ class BootloaderImage(object):
         return '\n'.join(rv)
 
     def __getattr__(self, name):
-        value = getattr(self.header, name, None)
-        if value is not None:
-            return value
-        return getattr(super(BootImage, self), name)
+        if name.startswith('_'):
+            raise AttributeError(name)
+
+        return getattr(self.header, name)
+
 
 if __name__ == '__main__':
     # Easy sanity checking

pwnlib/args.py

@@ -62,6 +62,8 @@ from pwnlib.context import context
 
 class PwnlibArgs(collections.defaultdict):
     def __getattr__(self, attr):
+        if attr.startswith('_'):
+            raise AttributeError(attr)
         return self[attr]
 
 args = PwnlibArgs(str)

pwnlib/asm.py

@@ -136,8 +136,23 @@ Try installing binutils for this architecture:
 %(instructions)s
 """.strip() % locals())
 
+
+def check_binutils_version(util):
+    if util_versions[util]:
+        return util_versions[util]
+    result = subprocess.check_output([util, '--version','/dev/null'],
+                                     stderr=subprocess.STDOUT, universal_newlines=True)
+    if 'clang' in result:
+        log.warn_once('Your binutils is clang-based and may not work!\n'
+            'Try installing with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+            'Reported version: %r', result.strip())
+    version = re.search(r' (\d+\.\d+)', result).group(1)
+    util_versions[util] = version = tuple(map(int, version.split('.')))
+    return version
+
+
 @LocalContext
-def which_binutils(util):
+def which_binutils(util, check_version=False):
     """
     Finds a binutils in the PATH somewhere.
     Expects that the utility is prefixed with the architecture name.
@@ -170,6 +185,8 @@ def which_binutils(util):
         'mips64': ['mips'],
         'powerpc64': ['powerpc'],
         'sparc64': ['sparc'],
+        'riscv32': ['riscv32', 'riscv64', 'riscv'],
+        'riscv64': ['riscv64', 'riscv32', 'riscv'],
     }.get(arch, [])
 
     # If one of the candidate architectures matches the native
@@ -204,17 +221,23 @@ def which_binutils(util):
 
             for pattern in patterns:
                 for dir in environ['PATH'].split(':'):
-                    res = sorted(glob(path.join(dir, pattern)))
-                    if res:
-                        return res[0]
+                    for res in sorted(glob(path.join(dir, pattern))):
+                        if check_version:
+                            ver = check_binutils_version(res)
+                            return res, ver
+                        return res
 
     # No dice!
     print_binutils_instructions(util, context)
 
-checked_assembler_version = defaultdict(lambda: False)
+util_versions = defaultdict(tuple)
 
 def _assembler():
-    gas = which_binutils('as')
+    gas, version = which_binutils('as', check_version=True)
+    if version < (2, 19):
+        log.warn_once('Your binutils version is too old and may not work!\n'
+            'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+            'Reported version: %r', version)
 
     E = {
         'big':    '-EB',
@@ -241,30 +264,19 @@ def _assembler():
         'powerpc64': [gas, '-m%s' % context.endianness, '-mppc%s' % context.bits],
 
         # ia64 only accepts -mbe or -mle
-        'ia64':    [gas, '-m%ce' % context.endianness[0]]
+        'ia64':    [gas, '-m%ce' % context.endianness[0]],
+
+        # riscv64-unknown-elf-as supports riscv32 as well as riscv64
+        'riscv32': [gas, '-march=rv32gc', '-mabi=ilp32'],
+        'riscv64': [gas, '-march=rv64gc', '-mabi=lp64'],
     }
 
     assembler = assemblers.get(context.arch, [gas])
 
-    if not checked_assembler_version[gas]:
-        checked_assembler_version[gas] = True
-        result = subprocess.check_output([gas, '--version','/dev/null'],
-                                         stderr=subprocess.STDOUT, universal_newlines=True)
-        version = re.search(r' (\d+\.\d+)', result).group(1)
-        if 'clang' in result:
-            log.warn_once('Your binutils is clang version and may not work!\n'
-                'Try install with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
-                'Reported Version: %r', result.strip())
-        elif version < '2.19':
-            log.warn_once('Your binutils version is too old and may not work!\n'
-                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
-                'Reported Version: %r', result.strip())
-
-
     return assembler
 
 def _linker():
-    ld  = [which_binutils('ld')]
+    ld, _ = which_binutils('ld', check_version=True)
     bfd = ['--oformat=' + _bfdname()]
 
     E = {
@@ -276,7 +288,16 @@ def _linker():
         'i386': ['-m', 'elf_i386'],
     }.get(context.arch, [])
 
-    return ld + bfd + [E] + arguments
+    return [ld] + bfd + [E] + arguments
+
+
+def _execstack(linker):
+    ldflags = ['-z', 'execstack']
+    version = util_versions[linker[0]]
+    if version >= (2, 39):
+        return ldflags + ['--no-warn-execstack', '--no-warn-rwx-segments']
+    return ldflags
+
 
 def _objcopy():
     return [which_binutils('objcopy')]
@@ -305,20 +326,23 @@ def _arch_header():
     prefix  = ['.section .shellcode,"awx"',
                 '.global _start',
                 '.global __start',
-                '.p2align 2',
                 '_start:',
                 '__start:']
     headers = {
-        'i386'  :  ['.intel_syntax noprefix'],
-        'amd64' :  ['.intel_syntax noprefix'],
+        'i386'  :  ['.intel_syntax noprefix', '.p2align 0'],
+        'amd64' :  ['.intel_syntax noprefix', '.p2align 0'],
         'arm'   : ['.syntax unified',
                    '.arch armv7-a',
-                   '.arm'],
+                   '.arm',
+                   '.p2align 2'],
         'thumb' : ['.syntax unified',
                    '.arch armv7-a',
-                   '.thumb'],
+                   '.thumb',
+                   '.p2align 2'
+                   ],
         'mips'  : ['.set mips2',
                    '.set noreorder',
+                   '.p2align 2'
                    ],
     }
 
@@ -344,7 +368,8 @@ def _bfdname():
         'msp430'  : 'elf32-msp430',
         'powerpc' : 'elf32-powerpc',
         'powerpc64' : 'elf64-powerpc',
-        'riscv'   : 'elf%d-%sriscv' % (context.bits, E),
+        'riscv32' : 'elf%d-%sriscv' % (context.bits, E),
+        'riscv64' : 'elf%d-%sriscv' % (context.bits, E),
         'vax'     : 'elf32-vax',
         's390'    : 'elf%d-s390' % context.bits,
         'sparc'   : 'elf32-sparc',
@@ -367,6 +392,8 @@ def _bfdarch():
         'powerpc64': 'powerpc',
         'sparc64':   'sparc',
         'thumb':     'arm',
+        'riscv32':   'riscv',
+        'riscv64':   'riscv',
     }
 
     if arch in convert:
@@ -442,6 +469,7 @@ def cpp(shellcode):
     ]
     return _run(cmd, code).strip('\n').rstrip() + '\n'
 
+
 @LocalContext
 def make_elf_from_assembly(assembly,
                            vma=None,
@@ -595,7 +623,7 @@ def make_elf(data,
 
         _run(assembler + ['-o', step2, step1])
 
-        linker_options = ['-z', 'execstack']
+        linker_options = _execstack(linker)
         if vma is not None:
             linker_options += ['--section-start=.shellcode=%#x' % vma,
                                '--entry=%#x' % vma]
@@ -624,6 +652,68 @@ def make_elf(data,
 
     return retval
 
+
+@LocalContext
+def make_macho_from_assembly(shellcode):
+    return make_macho(shellcode, is_shellcode=True)
+
+
+@LocalContext
+def make_macho(data, is_shellcode=False):
+    prefix = []
+    if context.arch == 'amd64':
+        prefix = [
+            '.intel_syntax noprefix',
+        ]
+    prefix.extend([
+        '.text',
+        '.global _start',
+        '_start:',
+        '.p2align 2',
+    ])
+    code = ''
+    code += '\n'.join(prefix) + '\n'
+    if is_shellcode:
+        code += cpp(data)
+    else:
+        code += '.string "%s"' % ''.join('\\x%02x' % c for c in bytearray(data))
+
+    log.debug('Assembling\n%s' % code)
+
+    tmpdir = tempfile.mkdtemp(prefix = 'pwn-asm-')
+    step1 = path.join(tmpdir, 'step1')
+    step2 = path.join(tmpdir, 'step2')
+    step3 = path.join(tmpdir, 'step3')
+
+    with open(step1, 'w') as fd:
+        fd.write(code)
+
+    assembler = [
+        '/usr/bin/as',
+    ]
+    asflags = [
+        '-mmacosx-version-min=11.0',
+        '-o', step2, step1,
+    ]
+    _run(assembler + asflags)
+
+    linker = [
+        '/usr/bin/ld',
+    ]
+    ldflags = [
+        '-macos_version_min', '11.0',
+        '-l', 'System',
+        '-e', '_start',
+        '-L', '/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk/usr/lib',
+        '-o', step3, step2,
+    ]
+    _run(linker + ldflags)
+
+    os.chmod(step3, 0o755)
+
+    return step3
+
+
 @LocalContext
 def asm(shellcode, vma = 0, extract = True, shared = False):
     r"""asm(code, vma = 0, extract = True, shared = False, ...) -> str
@@ -689,7 +779,7 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
             shutil.copy(step2, step3)
 
         if vma or not extract:
-            ldflags = ['-z', 'execstack', '-o', step3, step2]
+            ldflags = _execstack(linker) + ['-o', step3, step2]
             if vma:
                 ldflags += ['--section-start=.shellcode=%#x' % vma,
                             '--entry=%#x' % vma]
@@ -771,7 +861,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         >>> print(disasm(unhex('4ff00500'), arch = 'thumb', bits=32))
            0:   f04f 0005       mov.w   r0, #5
         >>> print(disasm(unhex('656664676665400F18A4000000000051'), byte=0, arch='amd64'))
-           0:   gs data16 fs data16 rex nop/reserved BYTE PTR gs:[eax+eax*1+0x0]
+           0:   gs data16 fs rex nop WORD PTR gs:[eax+eax*1+0x0]
            f:   push   rcx
         >>> print(disasm(unhex('01000000'), arch='sparc64'))
            0:   01 00 00 00     nop
@@ -779,6 +869,11 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
            0:   60 00 00 00     nop
         >>> print(disasm(unhex('00000000'), arch='mips64'))
            0:   00000000        nop
+        >>> print(disasm(unhex('48b84141414141414100c3'), arch='amd64'))
+           0:   48 b8 41 41 41 41 41 41 41 00   movabs rax, 0x41414141414141
+           a:   c3                      ret
+        >>> print(disasm(unhex('00000000'), vma=0x80000000, arch='mips'))
+        80000000:       00000000        nop
     """
     result = ''
 
@@ -790,7 +885,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
 
     bfdarch = _bfdarch()
     bfdname = _bfdname()
-    objdump = _objdump() + ['-d', '--adjust-vma', str(vma), '-b', bfdname]
+    objdump = _objdump() + ['-w', '-d', '--adjust-vma', str(vma), '-b', bfdname]
     objcopy = _objcopy() + [
         '-I', 'binary',
         '-O', bfdname,
@@ -815,7 +910,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         _run(objcopy + [step1, step2])
 
         output0 = _run(objdump + [step2])
-        output1 = output0.split('<.text>:\n')
+        output1 = re.split(r'<\.text(?:\+0x0)?>:\n', output0, flags=re.MULTILINE)
 
         if len(output1) != 2:
             log.error('Could not find .text in objdump output:\n%s' % output0)
@@ -828,6 +923,8 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
 
 
     lines = []
+
+    # Note: those patterns are also used in pwnlib/commandline/disasm.py
     pattern = '^( *[0-9a-f]+: *)', '((?:[0-9a-f]+ )+ *)', '(.*)'
     if not byte:
         pattern = pattern[::2]

pwnlib/commandline/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 __all__ = [
     'asm',
     'checksec',

pwnlib/commandline/asm.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/commandline/checksec.py

@@ -1,11 +1,10 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
 import argparse
 import sys
 
-from pwn import *
+from pwnlib.elf import ELF
 from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
@@ -39,4 +38,4 @@ def main(args):
         e = ELF(f.name)
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/constgrep.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -92,9 +91,13 @@ def main(args):
             if not matcher.search(k):
                 continue
 
+            # Check if the value has proper type
+            val = getattr(mod, k)
+            if not isinstance(val, pwnlib.constants.constant.Constant):
+                continue
+
             # Check the constant
             if constant is not None:
-                val = getattr(mod, k)
                 if args.mask_mode:
                     if constant & val != val:
                         continue
@@ -103,7 +106,7 @@ def main(args):
                         continue
 
             # Append it
-            out.append((getattr(mod, k), k))
+            out.append((val, k))
             maxlen = max(len(k), maxlen)
 
         # Output all matching constants

pwnlib/commandline/cyclic.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -72,13 +71,16 @@ def main(args):
 
         try:
             pat = int(pat, 0)
+            pat = pack(pat, 'all')
         except ValueError:
             pass
         pat = flat(pat, bytes=args.length)
 
-        if len(pat) != subsize:
-            log.critical('Subpattern must be %d bytes' % subsize)
+        if len(pat) < subsize:
+            log.critical('Subpattern must be at least %d bytes' % subsize)
             sys.exit(1)
+        else:
+            pat = pat[:subsize]
 
         if not all(c in alphabet for c in pat):
             log.critical('Pattern contains characters not present in the alphabet')

pwnlib/commandline/debug.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/commandline/disablenx.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/commandline/disasm.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
 import argparse
+import re
 import string
 import sys
 
@@ -77,18 +77,34 @@ def main(args):
         from pygments.formatters import TerminalFormatter
         from pwnlib.lexer import PwntoolsLexer
 
-        offsets = disasm(dat, vma=safeeval.const(args.address), instructions=False, byte=False)
-        bytes   = disasm(dat, vma=safeeval.const(args.address), instructions=False, offset=False)
-        instrs  = disasm(dat, vma=safeeval.const(args.address), byte=False, offset=False)
-        # instrs  = highlight(instrs, PwntoolsLexer(), TerminalFormatter())
+        dis = disasm(dat, vma=safeeval.const(args.address))
 
-        for o,b,i in zip(*map(str.splitlines, (offsets, bytes, instrs))):
-            b = b.replace('00', text.red('00'))
-            b = b.replace('0a', text.red('0a'))
-            i = highlight(i.strip(), PwntoolsLexer(), TerminalFormatter()).strip()
-            i = i.replace(',',', ')
+        # Note: those patterns are copied from disasm function
+        pattern = '^( *[0-9a-f]+: *)((?:[0-9a-f]+ )+ *)(.*)'
+        lines = []
+        for line in dis.splitlines():
+            match = re.search(pattern, line)
+            if not match:
+                # Append as one element tuple
+                lines.append((line,))
+                continue
 
-            print(o,b,i)
+            groups = match.groups()
+            o, b, i = groups
+
+            lines.append((o, b, i))
+
+
+        highlight_bytes = lambda t: ''.join(map(lambda x: x.replace('00', text.red('00')).replace('0a', text.red('0a')), group(2, t)))
+        for line in lines:
+            if len(line) == 3:
+                o, b, i = line
+                b = ' '.join(highlight_bytes(bb) for bb in b.split(' '))
+                i = highlight(i.strip(), PwntoolsLexer(), TerminalFormatter()).strip()
+                i = i.replace(',',', ')
+                print(o,b,i)
+            else:
+                print(line[0])
         return
 
     print(disasm(dat, vma=safeeval.const(args.address)))

pwnlib/commandline/elfdiff.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/commandline/elfpatch.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/commandline/hex.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
@@ -7,6 +6,7 @@ import sys
 
 from pwnlib.commandline import common
 from pwnlib.util.fiddling import enhex
+from pwnlib.util.lists import group
 
 parser = common.parser_commands.add_parser(
     'hex',
@@ -16,14 +16,38 @@ parser = common.parser_commands.add_parser(
 parser.add_argument('data', nargs='*',
     help='Data to convert into hex')
 
+parser.add_argument(
+    '-p', '--prefix',
+    metavar = 'prefix',
+    type = str,
+    default = '',
+    help = 'Insert a prefix before each byte',
+)
+
+parser.add_argument(
+    '-s', '--separator',
+    metavar = 'separator',
+    type = str,
+    default = '',
+    help = 'Add a separator between each byte',
+)
+
+def format_hex(hex_string, prefix, separator):
+    return separator.join([prefix + x for x in group(2, hex_string)])
+
 def main(args):
     if not args.data:
-        print(enhex(getattr(sys.stdin, 'buffer', sys.stdin).read()))
+        encoded = enhex(getattr(sys.stdin, 'buffer', sys.stdin).read())
     else:
         data = ' '.join(args.data)
         if not hasattr(data, 'decode'):
             data = data.encode('utf-8', 'surrogateescape')
-        print(enhex(data))
+        encoded = enhex(data)
+
+    if args.prefix or args.separator:
+        encoded = format_hex(encoded, args.prefix, args.separator)
+
+    print(encoded)
 
 if __name__ == '__main__':
     common.main(__file__)

pwnlib/commandline/libcdb.py

@@ -0,0 +1,251 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import re
+import shutil
+import sys
+
+import pwnlib.args
+pwnlib.args.free_form = False
+
+from pwn import *
+from pwnlib.commandline import common
+
+parser = common.parser_commands.add_parser(
+    'libcdb',
+    help = 'Print various information about a libc binary',
+    description = 'Print various information about a libc binary'
+)
+
+libc_commands = parser.add_subparsers(
+    dest = 'libc_command'
+)
+
+lookup_parser = libc_commands.add_parser(
+    'lookup',
+    help = 'Lookup a libc version by function offsets',
+    description = 'Lookup a libc version by function offsets'
+)
+
+lookup_parser.add_argument(
+    'symbol_offset_pairs',
+    metavar = 'symbol_offset_pairs',
+    nargs = '+',
+    help = 'Symbol and offset pairs to lookup matching libc version. Can be any number of pairs to narrow the search. Example: "read 3e0 write 520"'
+)
+
+lookup_parser.add_argument(
+    '--download-libc',
+    action = 'store_true',
+    default = False,
+    help = 'Attempt to download the matching libc.so'
+)
+
+lookup_parser.add_argument(
+    '--unstrip',
+    action = 'store_true',
+    default = True,
+    help = 'Attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+)
+
+lookup_parser.add_argument(
+    '--no-unstrip',
+    action = 'store_false',
+    dest = 'unstrip',
+    help = 'Do NOT attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+)
+
+hash_parser = libc_commands.add_parser(
+    'hash',
+    help = 'Display information of a libc version given an unique hash',
+    description = 'Display information of a libc version given an unique hash'
+)
+
+hash_parser.add_argument(
+    'hash_value',
+    metavar = 'hash_value',
+    nargs = '+',
+    help = 'Hex encoded hash value'
+)
+
+hash_parser.add_argument(
+    '-t', '--hash_type',
+    nargs = '?',
+    type = str,
+    choices = ['id', 'buildid', 'md5', 'sha1', 'sha256'],
+    default = 'buildid',
+    help = 'The type of the provided hash value. Supported hashtypes: id, buildid, md5, sha1, sha256'
+)
+
+hash_parser.add_argument(
+    '--download-libc',
+    action = 'store_true',
+    default = False,
+    help = 'Attempt to download the matching libc.so'
+)
+
+hash_parser.add_argument(
+    '--unstrip',
+    action = 'store_true',
+    default = True,
+    help = 'Attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+)
+
+hash_parser.add_argument(
+    '--no-unstrip',
+    action = 'store_false',
+    dest = 'unstrip',
+    help = 'Do NOT attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+)
+
+file_parser = libc_commands.add_parser(
+    'file',
+    help = 'Dump information about a libc binary',
+    description = 'Dump information about a libc binary'
+)
+
+file_parser.add_argument(
+    'files',
+    metavar = 'files',
+    nargs = '+',
+    help = 'Libc binary to dump'
+)
+
+file_parser.add_argument(
+    '-s', '--symbols',
+    metavar = 'symbols',
+    nargs = '*',
+    help = 'List of symbol offsets to dump in addition to the common ones'
+)
+
+file_parser.add_argument(
+    '-o', '--offset',
+    metavar = 'offset',
+    type = str,
+    help = 'Display all offsets relative to this symbol'
+)
+
+file_parser.add_argument(
+    '--unstrip',
+    action = 'store_true',
+    default = False,
+    help = 'Attempt to unstrip the libc binary inplace with debug symbols from a debuginfod server'
+)
+
+common_symbols = ['dup2', 'printf', 'puts', 'read', 'system', 'write']
+
+def find_libc(params):
+    import requests
+    url    = "https://libc.rip/api/find"
+    result = requests.post(url, json=params, timeout=20)
+    log.debug('Request: %s', params)
+    log.debug('Result: %s', result.json())
+    if result.status_code != 200 or len(result.json()) == 0:
+        log.failure("Could not find libc for %s on libc.rip", params)
+        return []
+
+    return result.json()
+
+def print_libc(libc):
+    log.info('%s', text.red(libc['id']))
+    log.indented('\t%-20s %s', text.green('BuildID:'), libc['buildid'])
+    log.indented('\t%-20s %s', text.green('MD5:'), libc['md5'])
+    log.indented('\t%-20s %s', text.green('SHA1:'), libc['sha1'])
+    log.indented('\t%-20s %s', text.green('SHA256:'), libc['sha256'])
+    log.indented('\t%s', text.green('Symbols:'))
+    for symbol in libc['symbols'].items():
+        log.indented('\t%25s = %s', symbol[0], symbol[1])
+
+def handle_remote_libc(args, libc):
+    print_libc(libc)
+    if args.download_libc:
+        path = libcdb.search_by_build_id(libc['buildid'], args.unstrip)
+        if path:
+            if args.unstrip:
+                libcdb.unstrip_libc(path)
+            shutil.copy(path, './{}.so'.format(libc['id']))
+
+def translate_offset(offs, args, exe):
+    if args.offset:
+        if args.offset not in exe.symbols:
+            log.info_once('offset symbol %s not found. ignoring.', args.offset)
+            return offs
+        return offs - exe.symbols[args.offset]
+    return offs
+
+def collect_synthetic_symbols(exe):
+    available_symbols = []
+    try:
+        exe.symbols['str_bin_sh'] = next(exe.search(b'/bin/sh\x00'))
+        available_symbols.append('str_bin_sh')
+    except StopIteration:
+        pass
+        
+    libc_start_main_return = exe.libc_start_main_return
+    if libc_start_main_return > 0:
+        exe.symbols['__libc_start_main_ret'] = libc_start_main_return
+        available_symbols.append('__libc_start_main_ret')
+
+    return available_symbols
+
+def main(args):
+    if len(sys.argv) < 3:
+        parser.print_usage()
+        sys.exit()
+
+    if args.libc_command == 'lookup':
+        pairs = args.symbol_offset_pairs
+        if len(pairs) % 2 != 0:
+            log.failure('Uneven number of arguments. Please provide "symbol offset" pairs')
+            return
+        
+        symbols = {pairs[i]:pairs[i+1] for i in range(0, len(pairs), 2)}
+        matched_libcs = find_libc({'symbols': symbols})
+        for libc in matched_libcs:
+            handle_remote_libc(args, libc)
+
+    elif args.libc_command == 'hash':
+        for hash_value in args.hash_value:
+            matched_libcs = find_libc({args.hash_type: hash_value})
+            for libc in matched_libcs:
+                handle_remote_libc(args, libc)
+
+    elif args.libc_command == 'file':
+        from hashlib import md5, sha1, sha256
+        for file in args.files:
+            if not os.path.exists(file) or not os.path.isfile(file):
+                log.failure('File does not exist %s', args.file)
+                continue
+            
+            if args.unstrip:
+                libcdb.unstrip_libc(file)
+
+            exe = ELF(file, checksec=False)
+            log.info('%s', text.red(os.path.basename(file)))
+
+            libc_version = re.search(br'libc[ -](\d+\.\d+)', exe.data)
+            if libc_version:
+                log.indented('%-20s %s', text.green('Version:'), libc_version.group(1).decode())
+
+            if exe.buildid:
+                log.indented('%-20s %s', text.green('BuildID:'), enhex(exe.buildid))
+            log.indented('%-20s %s', text.green('MD5:'), md5(exe.data).hexdigest())
+            log.indented('%-20s %s', text.green('SHA1:'), sha1(exe.data).hexdigest())
+            log.indented('%-20s %s', text.green('SHA256:'), sha256(exe.data).hexdigest())
+
+            # Always dump the basic list of common symbols
+            log.indented('%s', text.green('Symbols:'))
+            synthetic_symbols = collect_synthetic_symbols(exe)
+
+            symbols = common_symbols + (args.symbols or []) + synthetic_symbols
+            symbols.sort()
+            for symbol in symbols:
+                if symbol not in exe.symbols:
+                    log.indented('%25s = %s', symbol, text.red('not found'))
+                else:
+                    log.indented('%25s = %#x', symbol, translate_offset(exe.symbols[symbol], args, exe))
+
+if __name__ == '__main__':
+    pwnlib.commandline.common.main(__file__)

pwnlib/commandline/main.py

@@ -14,6 +14,7 @@ from pwnlib.commandline import elfdiff
 from pwnlib.commandline import elfpatch
 from pwnlib.commandline import errno
 from pwnlib.commandline import hex
+from pwnlib.commandline import libcdb
 from pwnlib.commandline import phd
 from pwnlib.commandline import pwnstrip
 from pwnlib.commandline import scramble
@@ -37,6 +38,7 @@ commands = {
     'elfpatch': elfpatch.main,
     'errno': errno.main,
     'hex': hex.main,
+    'libcdb': libcdb.main,
     'phd': phd.main,
     'pwnstrip': pwnstrip.main,
     'scramble': scramble.main,

pwnlib/commandline/phd.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/commandline/shellcraft.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/commandline/template.py

@@ -1,30 +1,71 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 
 from pwn import *
 from pwnlib.commandline import common
 
-from mako.lookup import TemplateLookup
+from mako.lookup import TemplateLookup, Template
 
 parser = common.parser_commands.add_parser(
     'template',
     help = 'Generate an exploit template',
-    description = 'Generate an exploit template'
+    description = 'Generate an exploit template. If no arguments are given, '
+                    'the current directory is searched for an executable binary and ' 
+                    'libc. If only one binary is found, it is assumed to be the '
+                    'challenge binary.'
 )
 
-parser.add_argument('exe', nargs='?', help='Target binary')
+# change path to hardcoded one when building the documentation
+printable_data_path = "pwnlib/data" if 'sphinx' in sys.modules else pwnlib.data.path
+
+parser.add_argument('exe', nargs='?', help='Target binary. If not given, the current directory is searched for an executable binary.')
 parser.add_argument('--host', help='Remote host / SSH server')
 parser.add_argument('--port', help='Remote port / SSH port', type=int)
 parser.add_argument('--user', help='SSH Username')
 parser.add_argument('--pass', '--password', help='SSH Password', dest='password')
+parser.add_argument('--libc', help='Path to libc binary to use. If not given, the current directory is searched for a libc binary.')
 parser.add_argument('--path', help='Remote path of file on SSH server')
 parser.add_argument('--quiet', help='Less verbose template comments', action='store_true')
 parser.add_argument('--color', help='Print the output in color', choices=['never', 'always', 'auto'], default='auto')
+parser.add_argument('--template', help='Path to a custom template. Tries to use \'~/.config/pwntools/templates/pwnup.mako\', if it exists. '
+                                   'Check \'%s\' for the default template shipped with pwntools.' % 
+                                        os.path.join(printable_data_path, "templates", "pwnup.mako"))
+parser.add_argument('--no-auto', help='Do not automatically detect missing binaries', action='store_false', dest='auto')
+
+def detect_missing_binaries(args):
+    log.info("Automatically detecting challenge binaries...")
+    # look for challenge binary, libc, and ld in current directory
+    exe, libc, ld = args.exe, args.libc, None
+    other_files = []
+    for filename in os.listdir():
+        if not os.path.isfile(filename):
+            continue
+        if not libc and ('libc-' in filename or 'libc.' in filename):
+            libc = filename
+        elif not ld and 'ld-' in filename:
+            ld = filename
+        else:
+            if os.access(filename, os.X_OK):
+                other_files.append(filename)
+    if not exe:
+        if len(other_files) == 1:
+            exe = other_files[0]
+        elif len(other_files) > 1:
+            log.warning("Failed to find challenge binary. There are multiple binaries in the current directory: %s", other_files)
+
+    if exe != args.exe:
+        log.success("Found challenge binary %r", exe)
+    if libc != args.libc:
+        log.success("Found libc binary %r", libc)
+    return exe, libc
 
 def main(args):
+
     lookup = TemplateLookup(
-        directories      = [os.path.join(pwnlib.data.path, 'templates')],
+        directories      = [
+            os.path.expanduser('~/.config/pwntools/templates/'),
+            os.path.join(pwnlib.data.path, 'templates')
+        ],
         module_directory = None
     )
 
@@ -34,25 +75,32 @@ def main(args):
         if not (args.path or args.exe):
             log.error("Must specify --path or a exe")
 
-        s = ssh(args.user, args.host, args.port or 22, args.password or None)
-
-        try:
-            remote_file = args.path or args.exe
-            s.download(remote_file)
-        except Exception:
-            log.warning("Could not download file %r, opening a shell", remote_file)
-            s.interactive()
-            return
+        with ssh(args.user, args.host, args.port or 22, args.password or None) as s:
+            try:
+                remote_file = args.path or args.exe
+                s.download(remote_file)
+            except Exception:
+                log.warning("Could not download file %r, opening a shell", remote_file)
+                s.interactive()
+                return
 
         if not args.exe:
             args.exe = os.path.basename(args.path)
 
-    template = lookup.get_template('pwnup.mako')
+    if args.auto and (args.exe is None or args.libc is None):
+        args.exe, args.libc = detect_missing_binaries(args)
+    
+    if args.template:
+        template = Template(filename=args.template) # Failing on invalid file is ok
+    else:
+        template = lookup.get_template('pwnup.mako')
+    
     output = template.render(args.exe,
                              args.host,
                              args.port,
                              args.user,
                              args.password,
+                             args.libc,
                              args.path,
                              args.quiet)
 
@@ -75,3 +123,4 @@ def main(args):
 
 if __name__ == '__main__':
     pwnlib.commandline.common.main(__file__)
+    

pwnlib/commandline/unhex.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/config.py

@@ -55,8 +55,8 @@ def initialize():
     from pwnlib.log import getLogger
     log = getLogger(__name__)
 
-    xdg_config_home = (os.environ.get('XDG_CONFIG_HOME') or
-                       os.path.expanduser("~/.config"))
+    xdg_config_home = os.environ.get('XDG_CONFIG_HOME',
+                                     os.path.expanduser("~/.config"))
 
     c = configparser.ConfigParser()
     c.read(['/etc/pwn.conf',
@@ -65,7 +65,7 @@ def initialize():
 
     for section in c.sections():
         if section not in registered_configs:
-            log.warn("Unknown configuration section %r" % section)
+            log.warn("Unknown configuration section %r", section)
             continue
         settings = dict(c.items(section))
         registered_configs[section](settings)

pwnlib/context/__init__.py

@@ -6,17 +6,20 @@ contexts work properly and as expected.
 from __future__ import absolute_import
 from __future__ import division
 
+import atexit
 import collections
+import errno
 import functools
 import logging
 import os
+import os.path
 import platform
+import shutil
 import six
 import socket
-import stat
 import string
-import subprocess
 import sys
+import tempfile
 import threading
 import time
 
@@ -345,6 +348,10 @@ class ContextType(object):
         'binary': None,
         'bits': 32,
         'buffer_size': 4096,
+        'cache_dir_base': os.environ.get(
+            'XDG_CACHE_HOME',
+            os.path.join(os.path.expanduser('~'), '.cache')
+        ),
         'cyclic_alphabet': string.ascii_lowercase.encode(),
         'cyclic_size': 4,
         'delete_corefiles': False,
@@ -369,7 +376,7 @@ class ContextType(object):
     }
 
     #: Valid values for :meth:`pwnlib.context.ContextType.os`
-    oses = sorted(('linux','freebsd','windows','cgc','android','baremetal'))
+    oses = sorted(('linux','freebsd','windows','cgc','android','baremetal','darwin'))
 
     big_32    = {'endian': 'big', 'bits': 32}
     big_64    = {'endian': 'big', 'bits': 64}
@@ -397,7 +404,8 @@ class ContextType(object):
         'msp430':    little_16,
         'powerpc':   big_32,
         'powerpc64': big_64,
-        'riscv':     little_32,
+        'riscv32':   little_32,
+        'riscv64':   little_64,
         's390':      big_32,
         'sparc':     big_32,
         'sparc64':   big_64,
@@ -762,12 +770,15 @@ class ContextType(object):
         # We have to make sure that x86_64 appears before x86 for this to work correctly.
         transform = [('ppc64', 'powerpc64'),
                      ('ppc', 'powerpc'),
+                     ('x86-64', 'amd64'),
                      ('x86_64', 'amd64'),
                      ('x86', 'i386'),
                      ('i686', 'i386'),
                      ('armv7l', 'arm'),
                      ('armeabi', 'arm'),
-                     ('arm64', 'aarch64')]
+                     ('arm64', 'aarch64'),
+                     ('rv32', 'riscv32'),
+                     ('rv64', 'riscv64')]
         for k, v in transform:
             if arch.startswith(k):
                 arch = v
@@ -776,7 +787,7 @@ class ContextType(object):
         try:
             defaults = self.architectures[arch]
         except KeyError:
-            raise AttributeError('AttributeError: arch must be one of %r' % sorted(self.architectures))
+            raise AttributeError('AttributeError: arch (%r) must be one of %r' % (arch, sorted(self.architectures)))
 
         for k,v in defaults.items():
             if k not in self._tls:
@@ -819,6 +830,7 @@ class ContextType(object):
         The default value is ``32``, but changes according to :attr:`arch`.
 
         Examples:
+
             >>> context.clear()
             >>> context.bits == 32
             True
@@ -972,7 +984,10 @@ class ContextType(object):
         except AttributeError:  pass
 
         # Otherwise, fail
-        level_names = filter(lambda x: isinstance(x,str), logging._levelNames)
+        try:
+            level_names = logging._levelToName.values()
+        except AttributeError:
+            level_names = filter(lambda x: isinstance(x,str), logging._levelNames)
         permitted = sorted(level_names)
         raise AttributeError('log_level must be an integer or one of %r' % permitted)
 
@@ -1269,6 +1284,21 @@ class ContextType(object):
         """
         return int(size)
 
+    @_validator
+    def cache_dir_base(self, new_base):
+        """Base directory to use for caching content.
+
+        Changing this to a different value will clear the `cache_dir` path
+        stored in TLS since a new path will need to be generated to respect the
+        new `cache_dir_base` value.
+        """
+
+        if new_base != self.cache_dir_base:
+            del self._tls["cache_dir"]
+        if os.access(new_base, os.F_OK) and not os.access(new_base, os.W_OK):
+            raise OSError(errno.EPERM, "Cache base dir is not writable")
+        return new_base
+
     @property
     def cache_dir(self):
         """Directory used for caching data.
@@ -1282,32 +1312,59 @@ class ContextType(object):
             >>> cache_dir is not None
             True
             >>> os.chmod(cache_dir, 0o000)
+            >>> del context._tls['cache_dir']
             >>> context.cache_dir is None
             True
             >>> os.chmod(cache_dir, 0o755)
             >>> cache_dir == context.cache_dir
             True
         """
-        xdg_cache_home = os.environ.get('XDG_CACHE_HOME') or \
-                         os.path.join(os.path.expanduser('~'), '.cache')
+        try:
+            # If the TLS already has a cache directory path, we return it
+            # without any futher checks since it must have been valid when it
+            # was set and if that has changed, hiding the TOCTOU here would be
+            # potentially confusing
+            return self._tls["cache_dir"]
+        except KeyError:
+            pass
 
-        if not os.access(xdg_cache_home, os.W_OK):
+        # Attempt to create a Python version specific cache dir and its parents
+        cache_dirname = '.pwntools-cache-%d.%d' % sys.version_info[:2]
+        cache_dirpath = os.path.join(self.cache_dir_base, cache_dirname)
+        try:
+            os.makedirs(cache_dirpath)
+        except OSError as exc:
+            # If we failed for any reason other than the cache directory
+            # already existing then we'll fall back to a temporary directory
+            # object which doesn't respect the `cache_dir_base`
+            if exc.errno != errno.EEXIST:
+                try:
+                    cache_dirpath = tempfile.mkdtemp(prefix=".pwntools-tmp")
+                except IOError:
+                    # This implies no good candidates for temporary files so we
+                    # have to return `None`
+                    return None
+                else:
+                    # Ensure the temporary cache dir is cleaned up on exit. A
+                    # `TemporaryDirectory` would do this better upon garbage
+                    # collection but this is necessary for Python 2 support.
+                    atexit.register(shutil.rmtree, cache_dirpath)
+        # By this time we have a cache directory which exists but we don't know
+        # if it is actually writable. Some wargames e.g. pwnable.kr have
+        # created dummy directories which cannot be modified by the user
+        # account (owned by root).
+        if os.access(cache_dirpath, os.W_OK):
+            # Stash this in TLS for later reuse
+            self._tls["cache_dir"] = cache_dirpath
+            return cache_dirpath
+        else:
             return None
 
-        cache = os.path.join(xdg_cache_home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
-
-        if not os.path.exists(cache):
-            try:
-                os.mkdir(cache)
-            except OSError:
-                return None
-
-        # Some wargames e.g. pwnable.kr have created dummy directories
-        # which cannot be modified by the user account (owned by root).
-        if not os.access(cache, os.W_OK):
-            return None
-
-        return cache
+    @cache_dir.setter
+    def cache_dir(self, v):
+        if os.access(v, os.W_OK):
+            # Stash this in TLS for later reuse
+            self._tls["cache_dir"] = v
 
     @_validator
     def delete_corefiles(self, v):
@@ -1394,7 +1451,7 @@ class ContextType(object):
         from pwnlib.tubes.ssh import ssh
 
         if not isinstance(shell, ssh):
-            raise AttributeError("context.ssh_session must be an ssh tube") 
+            raise AttributeError("context.ssh_session must be an ssh tube")
 
         return shell
 

pwnlib/data/crcsums.txt

@@ -1,58 +1,59 @@
 # This file was generated using the command:
-# curl -s 'http://reveng.sourceforge.net/crc-catalogue/all.htm#crc.legend' | sed -nr -e 's@<H3><A NAME="(crc\.cat[^"]*).*@\1@p' -e 's@.*<CODE>(width.*)</CODE>.*@\1@p' | sed '$!N;s/\n/ /' > crcsums.txt
+# curl -s 'https://reveng.sourceforge.io/crc-catalogue/all.htm' | sed -nr -e 's@.*?<A NAME="(crc\.cat[^"]*)">C.*@\1@p' -e 's@.*<CODE>(width=.*)</CODE>.*@\1@p' | sed '$!N;s/\n/ /'
 #
 # At the time of download, that page was available under the GPLv3 license, which can found here:
 # https://www.gnu.org/licenses/gpl.html
 #
+crc.cat.crc-3-gsm width=3  poly=0x3  init=0x0  refin=false  refout=false  xorout=0x7  check=0x4  residue=0x2  name="CRC-3/GSM"
 crc.cat.crc-3-rohc width=3  poly=0x3  init=0x7  refin=true  refout=true  xorout=0x0  check=0x6  residue=0x0  name="CRC-3/ROHC"
-crc.cat-bits.3 width=3  poly=0x3  init=0x0  refin=false  refout=false  xorout=0x7  check=0x4  residue=0x2  name="CRC-3/GSM"
-crc.cat-bits.4 width=4  poly=0x3  init=0xf  refin=false  refout=false  xorout=0xf  check=0xb  residue=0x2  name="CRC-4/INTERLAKEN"
-crc.cat.crc-4-itu width=4  poly=0x3  init=0x0  refin=true  refout=true  xorout=0x0  check=0x7  residue=0x0  name="CRC-4/ITU"
-crc.cat-bits.5 width=5  poly=0x09  init=0x09  refin=false  refout=false  xorout=0x00  check=0x00  residue=0x00  name="CRC-5/EPC"
-crc.cat.crc-5-itu width=5  poly=0x15  init=0x00  refin=true  refout=true  xorout=0x00  check=0x07  residue=0x00  name="CRC-5/ITU"
+crc.cat.crc-4-g-704 width=4  poly=0x3  init=0x0  refin=true  refout=true  xorout=0x0  check=0x7  residue=0x0  name="CRC-4/G-704"
+crc.cat.crc-4-interlaken width=4  poly=0x3  init=0xf  refin=false  refout=false  xorout=0xf  check=0xb  residue=0x2  name="CRC-4/INTERLAKEN"
+crc.cat.crc-5-epc-c1g2 width=5  poly=0x09  init=0x09  refin=false  refout=false  xorout=0x00  check=0x00  residue=0x00  name="CRC-5/EPC-C1G2"
+crc.cat.crc-5-g-704 width=5  poly=0x15  init=0x00  refin=true  refout=true  xorout=0x00  check=0x07  residue=0x00  name="CRC-5/G-704"
 crc.cat.crc-5-usb width=5  poly=0x05  init=0x1f  refin=true  refout=true  xorout=0x1f  check=0x19  residue=0x06  name="CRC-5/USB"
-crc.cat-bits.6 width=6  poly=0x27  init=0x3f  refin=false  refout=false  xorout=0x00  check=0x0d  residue=0x00  name="CRC-6/CDMA2000-A"
+crc.cat.crc-6-cdma2000-a width=6  poly=0x27  init=0x3f  refin=false  refout=false  xorout=0x00  check=0x0d  residue=0x00  name="CRC-6/CDMA2000-A"
 crc.cat.crc-6-cdma2000-b width=6  poly=0x07  init=0x3f  refin=false  refout=false  xorout=0x00  check=0x3b  residue=0x00  name="CRC-6/CDMA2000-B"
 crc.cat.crc-6-darc width=6  poly=0x19  init=0x00  refin=true  refout=true  xorout=0x00  check=0x26  residue=0x00  name="CRC-6/DARC"
+crc.cat.crc-6-g-704 width=6  poly=0x03  init=0x00  refin=true  refout=true  xorout=0x00  check=0x06  residue=0x00  name="CRC-6/G-704"
 crc.cat.crc-6-gsm width=6  poly=0x2f  init=0x00  refin=false  refout=false  xorout=0x3f  check=0x13  residue=0x3a  name="CRC-6/GSM"
-crc.cat.crc-6-itu width=6  poly=0x03  init=0x00  refin=true  refout=true  xorout=0x00  check=0x06  residue=0x00  name="CRC-6/ITU"
-crc.cat-bits.7 width=7  poly=0x09  init=0x00  refin=false  refout=false  xorout=0x00  check=0x75  residue=0x00  name="CRC-7"
+crc.cat.crc-7-mmc width=7  poly=0x09  init=0x00  refin=false  refout=false  xorout=0x00  check=0x75  residue=0x00  name="CRC-7/MMC"
 crc.cat.crc-7-rohc width=7  poly=0x4f  init=0x7f  refin=true  refout=true  xorout=0x00  check=0x53  residue=0x00  name="CRC-7/ROHC"
 crc.cat.crc-7-umts width=7  poly=0x45  init=0x00  refin=false  refout=false  xorout=0x00  check=0x61  residue=0x00  name="CRC-7/UMTS"
-crc.cat-bits.8 width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x00  check=0xf4  residue=0x00  name="CRC-8"
 crc.cat.crc-8-autosar width=8  poly=0x2f  init=0xff  refin=false  refout=false  xorout=0xff  check=0xdf  residue=0x42  name="CRC-8/AUTOSAR"
+crc.cat.crc-8-bluetooth width=8  poly=0xa7  init=0x00  refin=true  refout=true  xorout=0x00  check=0x26  residue=0x00  name="CRC-8/BLUETOOTH"
 crc.cat.crc-8-cdma2000 width=8  poly=0x9b  init=0xff  refin=false  refout=false  xorout=0x00  check=0xda  residue=0x00  name="CRC-8/CDMA2000"
 crc.cat.crc-8-darc width=8  poly=0x39  init=0x00  refin=true  refout=true  xorout=0x00  check=0x15  residue=0x00  name="CRC-8/DARC"
 crc.cat.crc-8-dvb-s2 width=8  poly=0xd5  init=0x00  refin=false  refout=false  xorout=0x00  check=0xbc  residue=0x00  name="CRC-8/DVB-S2"
-crc.cat.crc-8-ebu width=8  poly=0x1d  init=0xff  refin=true  refout=true  xorout=0x00  check=0x97  residue=0x00  name="CRC-8/EBU"
 crc.cat.crc-8-gsm-a width=8  poly=0x1d  init=0x00  refin=false  refout=false  xorout=0x00  check=0x37  residue=0x00  name="CRC-8/GSM-A"
 crc.cat.crc-8-gsm-b width=8  poly=0x49  init=0x00  refin=false  refout=false  xorout=0xff  check=0x94  residue=0x53  name="CRC-8/GSM-B"
+crc.cat.crc-8-hitag width=8  poly=0x1d  init=0xff  refin=false  refout=false  xorout=0x00  check=0xb4  residue=0x00  name="CRC-8/HITAG"
+crc.cat.crc-8-i-432-1 width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x55  check=0xa1  residue=0xac  name="CRC-8/I-432-1"
 crc.cat.crc-8-i-code width=8  poly=0x1d  init=0xfd  refin=false  refout=false  xorout=0x00  check=0x7e  residue=0x00  name="CRC-8/I-CODE"
-crc.cat.crc-8-itu width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x55  check=0xa1  residue=0xac  name="CRC-8/ITU"
 crc.cat.crc-8-lte width=8  poly=0x9b  init=0x00  refin=false  refout=false  xorout=0x00  check=0xea  residue=0x00  name="CRC-8/LTE"
-crc.cat.crc-8-maxim width=8  poly=0x31  init=0x00  refin=true  refout=true  xorout=0x00  check=0xa1  residue=0x00  name="CRC-8/MAXIM"
+crc.cat.crc-8-maxim-dow width=8  poly=0x31  init=0x00  refin=true  refout=true  xorout=0x00  check=0xa1  residue=0x00  name="CRC-8/MAXIM-DOW"
+crc.cat.crc-8-mifare-mad width=8  poly=0x1d  init=0xc7  refin=false  refout=false  xorout=0x00  check=0x99  residue=0x00  name="CRC-8/MIFARE-MAD"
+crc.cat.crc-8-nrsc-5 width=8  poly=0x31  init=0xff  refin=false  refout=false  xorout=0x00  check=0xf7  residue=0x00  name="CRC-8/NRSC-5"
 crc.cat.crc-8-opensafety width=8  poly=0x2f  init=0x00  refin=false  refout=false  xorout=0x00  check=0x3e  residue=0x00  name="CRC-8/OPENSAFETY"
 crc.cat.crc-8-rohc width=8  poly=0x07  init=0xff  refin=true  refout=true  xorout=0x00  check=0xd0  residue=0x00  name="CRC-8/ROHC"
 crc.cat.crc-8-sae-j1850 width=8  poly=0x1d  init=0xff  refin=false  refout=false  xorout=0xff  check=0x4b  residue=0xc4  name="CRC-8/SAE-J1850"
-crc.cat.crc-8-wdcma width=8  poly=0x9b  init=0x00  refin=true  refout=true  xorout=0x00  check=0x25  residue=0x00  name="CRC-8/WCDMA"
-crc.cat-bits.10 width=10  poly=0x233  init=0x000  refin=false  refout=false  xorout=0x000  check=0x199  residue=0x000  name="CRC-10"
+crc.cat.crc-8-smbus width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x00  check=0xf4  residue=0x00  name="CRC-8/SMBUS"
+crc.cat.crc-8-tech-3250 width=8  poly=0x1d  init=0xff  refin=true  refout=true  xorout=0x00  check=0x97  residue=0x00  name="CRC-8/TECH-3250"
+crc.cat.crc-8-wcdma width=8  poly=0x9b  init=0x00  refin=true  refout=true  xorout=0x00  check=0x25  residue=0x00  name="CRC-8/WCDMA"
+crc.cat.crc-10-atm width=10  poly=0x233  init=0x000  refin=false  refout=false  xorout=0x000  check=0x199  residue=0x000  name="CRC-10/ATM"
 crc.cat.crc-10-cdma2000 width=10  poly=0x3d9  init=0x3ff  refin=false  refout=false  xorout=0x000  check=0x233  residue=0x000  name="CRC-10/CDMA2000"
 crc.cat.crc-10-gsm width=10  poly=0x175  init=0x000  refin=false  refout=false  xorout=0x3ff  check=0x12a  residue=0x0c6  name="CRC-10/GSM"
-crc.cat-bits.11 width=11  poly=0x385  init=0x01a  refin=false  refout=false  xorout=0x000  check=0x5a3  residue=0x000  name="CRC-11"
+crc.cat.crc-11-flexray width=11  poly=0x385  init=0x01a  refin=false  refout=false  xorout=0x000  check=0x5a3  residue=0x000  name="CRC-11/FLEXRAY"
 crc.cat.crc-11-umts width=11  poly=0x307  init=0x000  refin=false  refout=false  xorout=0x000  check=0x061  residue=0x000  name="CRC-11/UMTS"
-crc.cat-bits.12 width=12  poly=0xf13  init=0xfff  refin=false  refout=false  xorout=0x000  check=0xd4d  residue=0x000  name="CRC-12/CDMA2000"
+crc.cat.crc-12-cdma2000 width=12  poly=0xf13  init=0xfff  refin=false  refout=false  xorout=0x000  check=0xd4d  residue=0x000  name="CRC-12/CDMA2000"
 crc.cat.crc-12-dect width=12  poly=0x80f  init=0x000  refin=false  refout=false  xorout=0x000  check=0xf5b  residue=0x000  name="CRC-12/DECT"
 crc.cat.crc-12-gsm width=12  poly=0xd31  init=0x000  refin=false  refout=false  xorout=0xfff  check=0xb34  residue=0x178  name="CRC-12/GSM"
 crc.cat.crc-12-umts width=12  poly=0x80f  init=0x000  refin=false  refout=true  xorout=0x000  check=0xdaf  residue=0x000  name="CRC-12/UMTS"
-crc.cat-bits.13 width=13  poly=0x1cf5  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x04fa  residue=0x0000  name="CRC-13/BBC"
-crc.cat-bits.14 width=14  poly=0x0805  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x082d  residue=0x0000  name="CRC-14/DARC"
+crc.cat.crc-13-bbc width=13  poly=0x1cf5  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x04fa  residue=0x0000  name="CRC-13/BBC"
+crc.cat.crc-14-darc width=14  poly=0x0805  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x082d  residue=0x0000  name="CRC-14/DARC"
 crc.cat.crc-14-gsm width=14  poly=0x202d  init=0x0000  refin=false  refout=false  xorout=0x3fff  check=0x30ae  residue=0x031e  name="CRC-14/GSM"
-crc.cat-bits.15 width=15  poly=0x4599  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x059e  residue=0x0000  name="CRC-15"
+crc.cat.crc-15-can width=15  poly=0x4599  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x059e  residue=0x0000  name="CRC-15/CAN"
 crc.cat.crc-15-mpt1327 width=15  poly=0x6815  init=0x0000  refin=false  refout=false  xorout=0x0001  check=0x2566  residue=0x6815  name="CRC-15/MPT1327"
-crc.cat-bits.16 width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0xbb3d  residue=0x0000  name="ARC"
-crc.cat.crc-16-aug-ccitt width=16  poly=0x1021  init=0x1d0f  refin=false  refout=false  xorout=0x0000  check=0xe5cc  residue=0x0000  name="CRC-16/AUG-CCITT"
-crc.cat.crc-16-buypass width=16  poly=0x8005  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xfee8  residue=0x0000  name="CRC-16/BUYPASS"
-crc.cat.crc-16-ccitt-false width=16  poly=0x1021  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x29b1  residue=0x0000  name="CRC-16/CCITT-FALSE"
+crc.cat.crc-16-arc width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0xbb3d  residue=0x0000  name="CRC-16/ARC"
 crc.cat.crc-16-cdma2000 width=16  poly=0xc867  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x4c06  residue=0x0000  name="CRC-16/CDMA2000"
 crc.cat.crc-16-cms width=16  poly=0x8005  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0xaee7  residue=0x0000  name="CRC-16/CMS"
 crc.cat.crc-16-dds-110 width=16  poly=0x8005  init=0x800d  refin=false  refout=false  xorout=0x0000  check=0x9ecf  residue=0x0000  name="CRC-16/DDS-110"
@@ -62,44 +63,55 @@ crc.cat.crc-16-dnp width=16  poly=0x3d65  init=0x0000  refin=true  refout=true
 crc.cat.crc-16-en-13757 width=16  poly=0x3d65  init=0x0000  refin=false  refout=false  xorout=0xffff  check=0xc2b7  residue=0xa366  name="CRC-16/EN-13757"
 crc.cat.crc-16-genibus width=16  poly=0x1021  init=0xffff  refin=false  refout=false  xorout=0xffff  check=0xd64e  residue=0x1d0f  name="CRC-16/GENIBUS"
 crc.cat.crc-16-gsm width=16  poly=0x1021  init=0x0000  refin=false  refout=false  xorout=0xffff  check=0xce3c  residue=0x1d0f  name="CRC-16/GSM"
+crc.cat.crc-16-ibm-3740 width=16  poly=0x1021  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x29b1  residue=0x0000  name="CRC-16/IBM-3740"
+crc.cat.crc-16-ibm-sdlc width=16  poly=0x1021  init=0xffff  refin=true  refout=true  xorout=0xffff  check=0x906e  residue=0xf0b8  name="CRC-16/IBM-SDLC"
+crc.cat.crc-16-iso-iec-14443-3-a width=16  poly=0x1021  init=0xc6c6  refin=true  refout=true  xorout=0x0000  check=0xbf05  residue=0x0000  name="CRC-16/ISO-IEC-14443-3-A"
+crc.cat.crc-16-kermit width=16  poly=0x1021  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x2189  residue=0x0000  name="CRC-16/KERMIT"
 crc.cat.crc-16-lj1200 width=16  poly=0x6f63  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xbdf4  residue=0x0000  name="CRC-16/LJ1200"
-crc.cat.crc-16-maxim width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0xffff  check=0x44c2  residue=0xb001  name="CRC-16/MAXIM"
+crc.cat.crc-16-m17 width=16  poly=0x5935  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x772b  residue=0x0000  name="CRC-16/M17"
+crc.cat.crc-16-maxim-dow width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0xffff  check=0x44c2  residue=0xb001  name="CRC-16/MAXIM-DOW"
 crc.cat.crc-16-mcrf4xx width=16  poly=0x1021  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0x6f91  residue=0x0000  name="CRC-16/MCRF4XX"
+crc.cat.crc-16-modbus width=16  poly=0x8005  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0x4b37  residue=0x0000  name="CRC-16/MODBUS"
+crc.cat.crc-16-nrsc-5 width=16  poly=0x080b  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0xa066  residue=0x0000  name="CRC-16/NRSC-5"
 crc.cat.crc-16-opensafety-a width=16  poly=0x5935  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x5d38  residue=0x0000  name="CRC-16/OPENSAFETY-A"
-crc.cat.crc-16-opensafety-a width=16  poly=0x755b  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x20fe  residue=0x0000  name="CRC-16/OPENSAFETY-B"
+crc.cat.crc-16-opensafety-b width=16  poly=0x755b  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x20fe  residue=0x0000  name="CRC-16/OPENSAFETY-B"
 crc.cat.crc-16-profibus width=16  poly=0x1dcf  init=0xffff  refin=false  refout=false  xorout=0xffff  check=0xa819  residue=0xe394  name="CRC-16/PROFIBUS"
 crc.cat.crc-16-riello width=16  poly=0x1021  init=0xb2aa  refin=true  refout=true  xorout=0x0000  check=0x63d0  residue=0x0000  name="CRC-16/RIELLO"
+crc.cat.crc-16-spi-fujitsu width=16  poly=0x1021  init=0x1d0f  refin=false  refout=false  xorout=0x0000  check=0xe5cc  residue=0x0000  name="CRC-16/SPI-FUJITSU"
 crc.cat.crc-16-t10-dif width=16  poly=0x8bb7  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xd0db  residue=0x0000  name="CRC-16/T10-DIF"
 crc.cat.crc-16-teledisk width=16  poly=0xa097  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x0fb3  residue=0x0000  name="CRC-16/TELEDISK"
 crc.cat.crc-16-tms37157 width=16  poly=0x1021  init=0x89ec  refin=true  refout=true  xorout=0x0000  check=0x26b1  residue=0x0000  name="CRC-16/TMS37157"
+crc.cat.crc-16-umts width=16  poly=0x8005  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xfee8  residue=0x0000  name="CRC-16/UMTS"
 crc.cat.crc-16-usb width=16  poly=0x8005  init=0xffff  refin=true  refout=true  xorout=0xffff  check=0xb4c8  residue=0xb001  name="CRC-16/USB"
-crc.cat.crc-a width=16  poly=0x1021  init=0xc6c6  refin=true  refout=true  xorout=0x0000  check=0xbf05  residue=0x0000  name="CRC-A"
-crc.cat.kermit width=16  poly=0x1021  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x2189  residue=0x0000  name="KERMIT"
-crc.cat.modbus width=16  poly=0x8005  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0x4b37  residue=0x0000  name="MODBUS"
-crc.cat.x-25 width=16  poly=0x1021  init=0xffff  refin=true  refout=true  xorout=0xffff  check=0x906e  residue=0xf0b8  name="X-25"
-crc.cat.xmodem width=16  poly=0x1021  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x31c3  residue=0x0000  name="XMODEM"
-crc.cat-bits.24 width=24  poly=0x864cfb  init=0xb704ce  refin=false  refout=false  xorout=0x000000  check=0x21cf02  residue=0x000000  name="CRC-24"
+crc.cat.crc-16-xmodem width=16  poly=0x1021  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x31c3  residue=0x0000  name="CRC-16/XMODEM"
+crc.cat.crc-17-can-fd width=17  poly=0x1685b  init=0x00000  refin=false  refout=false  xorout=0x00000  check=0x04f03  residue=0x00000  name="CRC-17/CAN-FD"
+crc.cat.crc-21-can-fd width=21  poly=0x102899  init=0x000000  refin=false  refout=false  xorout=0x000000  check=0x0ed841  residue=0x000000  name="CRC-21/CAN-FD"
 crc.cat.crc-24-ble width=24  poly=0x00065b  init=0x555555  refin=true  refout=true  xorout=0x000000  check=0xc25a56  residue=0x000000  name="CRC-24/BLE"
 crc.cat.crc-24-flexray-a width=24  poly=0x5d6dcb  init=0xfedcba  refin=false  refout=false  xorout=0x000000  check=0x7979bd  residue=0x000000  name="CRC-24/FLEXRAY-A"
 crc.cat.crc-24-flexray-b width=24  poly=0x5d6dcb  init=0xabcdef  refin=false  refout=false  xorout=0x000000  check=0x1f23b8  residue=0x000000  name="CRC-24/FLEXRAY-B"
 crc.cat.crc-24-interlaken width=24  poly=0x328b63  init=0xffffff  refin=false  refout=false  xorout=0xffffff  check=0xb4f3e6  residue=0x144e63  name="CRC-24/INTERLAKEN"
 crc.cat.crc-24-lte-a width=24  poly=0x864cfb  init=0x000000  refin=false  refout=false  xorout=0x000000  check=0xcde703  residue=0x000000  name="CRC-24/LTE-A"
 crc.cat.crc-24-lte-b width=24  poly=0x800063  init=0x000000  refin=false  refout=false  xorout=0x000000  check=0x23ef52  residue=0x000000  name="CRC-24/LTE-B"
-crc.cat-bits.30 width=30  poly=0x2030b9c7  init=0x3fffffff  refin=false  refout=false  xorout=0x3fffffff  check=0x04c34abf  residue=0x34efa55a  name="CRC-30/CDMA"
-crc.cat-bits.31 width=31  poly=0x04c11db7  init=0x7fffffff  refin=false  refout=false  xorout=0x7fffffff  check=0x0ce9e46c  residue=0x4eaf26f1  name="CRC-31/PHILIPS"
-crc.cat-bits.32 width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xcbf43926  residue=0xdebb20e3  name="CRC-32"
+crc.cat.crc-24-openpgp width=24  poly=0x864cfb  init=0xb704ce  refin=false  refout=false  xorout=0x000000  check=0x21cf02  residue=0x000000  name="CRC-24/OPENPGP"
+crc.cat.crc-24-os-9 width=24  poly=0x800063  init=0xffffff  refin=false  refout=false  xorout=0xffffff  check=0x200fa5  residue=0x800fe3  name="CRC-24/OS-9"
+crc.cat.crc-30-cdma width=30  poly=0x2030b9c7  init=0x3fffffff  refin=false  refout=false  xorout=0x3fffffff  check=0x04c34abf  residue=0x34efa55a  name="CRC-30/CDMA"
+crc.cat.crc-31-philips width=31  poly=0x04c11db7  init=0x7fffffff  refin=false  refout=false  xorout=0x7fffffff  check=0x0ce9e46c  residue=0x4eaf26f1  name="CRC-31/PHILIPS"
+crc.cat.crc-32-aixm width=32  poly=0x814141ab  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0x3010bf7f  residue=0x00000000  name="CRC-32/AIXM"
 crc.cat.crc-32-autosar width=32  poly=0xf4acfb13  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0x1697d06a  residue=0x904cddbf  name="CRC-32/AUTOSAR"
+crc.cat.crc-32-base91-d width=32  poly=0xa833982b  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0x87315576  residue=0x45270551  name="CRC-32/BASE91-D"
 crc.cat.crc-32-bzip2 width=32  poly=0x04c11db7  init=0xffffffff  refin=false  refout=false  xorout=0xffffffff  check=0xfc891918  residue=0xc704dd7b  name="CRC-32/BZIP2"
-crc.cat.crc-32c width=32  poly=0x1edc6f41  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xe3069283  residue=0xb798b438  name="CRC-32C"
-crc.cat.crc-32d width=32  poly=0xa833982b  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0x87315576  residue=0x45270551  name="CRC-32D"
+crc.cat.crc-32-cd-rom-edc width=32  poly=0x8001801b  init=0x00000000  refin=true  refout=true  xorout=0x00000000  check=0x6ec2edc4  residue=0x00000000  name="CRC-32/CD-ROM-EDC"
+crc.cat.crc-32-cksum width=32  poly=0x04c11db7  init=0x00000000  refin=false  refout=false  xorout=0xffffffff  check=0x765e7680  residue=0xc704dd7b  name="CRC-32/CKSUM"
+crc.cat.crc-32-iscsi width=32  poly=0x1edc6f41  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xe3069283  residue=0xb798b438  name="CRC-32/ISCSI"
+crc.cat.crc-32-iso-hdlc width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xcbf43926  residue=0xdebb20e3  name="CRC-32/ISO-HDLC"
+crc.cat.crc-32-jamcrc width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0x00000000  check=0x340bc6d9  residue=0x00000000  name="CRC-32/JAMCRC"
+crc.cat.crc-32-mef width=32  poly=0x741b8cd7  init=0xffffffff  refin=true  refout=true  xorout=0x00000000  check=0xd2c22f51  residue=0x00000000  name="CRC-32/MEF"
 crc.cat.crc-32-mpeg-2 width=32  poly=0x04c11db7  init=0xffffffff  refin=false  refout=false  xorout=0x00000000  check=0x0376e6e7  residue=0x00000000  name="CRC-32/MPEG-2"
-crc.cat.crc-32-posix width=32  poly=0x04c11db7  init=0x00000000  refin=false  refout=false  xorout=0xffffffff  check=0x765e7680  residue=0xc704dd7b  name="CRC-32/POSIX"
-crc.cat.crc-32q width=32  poly=0x814141ab  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0x3010bf7f  residue=0x00000000  name="CRC-32Q"
-crc.cat.jamcrc width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0x00000000  check=0x340bc6d9  residue=0x00000000  name="JAMCRC"
-crc.cat.xfer width=32  poly=0x000000af  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0xbd0be338  residue=0x00000000  name="XFER"
-crc.cat-bits.40 width=40  poly=0x0004820009  init=0x0000000000  refin=false  refout=false  xorout=0xffffffffff  check=0xd4164fc646  residue=0xc4ff8071ff  name="CRC-40/GSM"
-crc.cat-bits.64 width=64  poly=0x42f0e1eba9ea3693  init=0x0000000000000000  refin=false  refout=false  xorout=0x0000000000000000  check=0x6c40df5f0b497347  residue=0x0000000000000000  name="CRC-64"
+crc.cat.crc-32-xfer width=32  poly=0x000000af  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0xbd0be338  residue=0x00000000  name="CRC-32/XFER"
+crc.cat.crc-40-gsm width=40  poly=0x0004820009  init=0x0000000000  refin=false  refout=false  xorout=0xffffffffff  check=0xd4164fc646  residue=0xc4ff8071ff  name="CRC-40/GSM"
+crc.cat.crc-64-ecma-182 width=64  poly=0x42f0e1eba9ea3693  init=0x0000000000000000  refin=false  refout=false  xorout=0x0000000000000000  check=0x6c40df5f0b497347  residue=0x0000000000000000  name="CRC-64/ECMA-182"
 crc.cat.crc-64-go-iso width=64  poly=0x000000000000001b  init=0xffffffffffffffff  refin=true  refout=true  xorout=0xffffffffffffffff  check=0xb90956c775a41001  residue=0x5300000000000000  name="CRC-64/GO-ISO"
+crc.cat.crc-64-ms width=64  poly=0x259c84cba6426349  init=0xffffffffffffffff  refin=true  refout=true  xorout=0x0000000000000000  check=0x75d4b74f024eceea  residue=0x0000000000000000  name="CRC-64/MS"
 crc.cat.crc-64-we width=64  poly=0x42f0e1eba9ea3693  init=0xffffffffffffffff  refin=false  refout=false  xorout=0xffffffffffffffff  check=0x62ec59e3f1a4f00a  residue=0xfcacbebd5931a992  name="CRC-64/WE"
 crc.cat.crc-64-xz width=64  poly=0x42f0e1eba9ea3693  init=0xffffffffffffffff  refin=true  refout=true  xorout=0xffffffffffffffff  check=0x995dc9bbdf1939fa  residue=0x49958c9abd7d353f  name="CRC-64/XZ"
-crc.cat-bits.82 width=82  poly=0x0308c0111011401440411  init=0x000000000000000000000  refin=true  refout=true  xorout=0x000000000000000000000  check=0x09ea83f625023801fd612  residue=0x000000000000000000000  name="CRC-82/DARC"
+crc.cat.crc-82-darc width=82  poly=0x0308c0111011401440411  init=0x000000000000000000000  refin=true  refout=true  xorout=0x000000000000000000000  check=0x09ea83f625023801fd612  residue=0x000000000000000000000  name="CRC-82/DARC"

pwnlib/data/syscalls/Makefile

@@ -7,4 +7,7 @@ all: generate.py functions.py
 functions.py:
 	wget https://raw.githubusercontent.com/zachriggle/functions/master/functions.py
 
+generate_darwin:
+	python generate_darwin.py "$(ROOT)"
+
 .phony: all

pwnlib/data/syscalls/generate.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 from __future__ import division
 import argparse
 import keyword
@@ -72,7 +72,7 @@ CALL = """
                 target = regs[index]
                 register_arguments[target] = arg
             elif arg is not None:
-                stack_arguments[index] = arg
+                stack_arguments[name] = arg
 
         # The argument is not a register.  It is a string value, and we
         # are expecting a string value
@@ -100,7 +100,7 @@ CALL = """
                 target = regs[index]
                 register_arguments[target] = arg
             elif arg is not None:
-                stack_arguments[target] = arg
+                stack_arguments[name] = arg
 
     # Some syscalls have different names on various architectures.
     # Determine which syscall number to use for the current architecture.
@@ -108,7 +108,7 @@ CALL = """
         if hasattr(pwnlib.constants, syscall):
             break
     else:
-        raise Exception("Could not locate any syscalls: %r" % syscalls)
+        raise Exception("Could not locate any syscalls: %r" % {syscalls!r})
 %>
     /* {name}(${{', '.join(syscall_repr)}}) */
 %for name, arg in string_arguments.items():
@@ -137,6 +137,7 @@ def can_be_string(arg):
     if arg.type == 'void' and arg.derefcnt == 1:
         return True
 
+
 def can_be_array(arg):
     if arg.type == 'char' and arg.derefcnt == 2:
         return True
@@ -160,11 +161,13 @@ def fix_bad_arg_names(func, arg):
 def get_arg_default(arg):
     return 0
 
+
 def fix_rt_syscall_name(name):
     if name.startswith('rt_'):
         return name[3:]
     return name
 
+
 def fix_syscall_names(name):
     # Do not use old_mmap
     if name == 'SYS_mmap':
@@ -180,6 +183,7 @@ def main(target):
         with context.local(arch=arch):
             generate_one(target)
 
+
 def generate_one(target):
     SYSCALL_NAMES = [c for c in dir(constants) if c.startswith('SYS_')]
 
@@ -214,46 +218,50 @@ def generate_one(target):
 
         # Set up the argument string for Mako
         argument_names = []
+        argument_names_ = []
         argument_defaults = []
 
-        #
-
-        for arg in function.args:
-            argname = fix_bad_arg_names(function, arg)
-            default = get_arg_default(arg)
-
-            # Mako is unable to use *vararg and *kwarg, so we just stub in
-            # a whole bunch of additional arguments.
-            if argname == 'vararg':
-                for j in range(5):
-                    argname = 'vararg_%i' % j
-                    argument_names.append(argname)
-                    argument_defaults.append('%s=%s' % (argname, None))
-                break
-
-            argument_names.append(argname)
-            argument_defaults.append('%s=%s' % (argname, default))
-
-        arguments_default_values = ', '.join(argument_defaults)
-        arguments_comma_separated = ', '.join(argument_names)
-
         string_arguments = []
         array_arguments = []
         arg_docs = []
 
+        #
+
         for arg in function.args:
+            argname_ = fix_bad_arg_names(function, arg)
+            argname = argname_.rstrip('_')
+            default = get_arg_default(arg)
 
             if can_be_array(arg):
-                array_arguments.append(arg.name)
+                array_arguments.append(argname)
 
             if can_be_string(arg):
-                string_arguments.append(arg.name)
+                string_arguments.append(argname)
 
-            argname = arg.name
             argtype = str(arg.type) + ('*' * arg.derefcnt)
             arg_docs.append(
-                '    {argname}({argtype}): {argname}'.format(argname=argname,
-                                                             argtype=argtype))
+                '    {argname_}({argtype}): {argname}'.format(
+                    argname_=argname_,
+                    argname=argname,
+                    argtype=argtype,
+                ))
+
+            # Mako is unable to use *vararg and *kwarg, so we just stub in
+            # a whole bunch of additional arguments.
+            if argname == 'vararg':
+                for j in range(6):
+                    argname = 'vararg_%i' % j
+                    argument_names.append(argname)
+                    argument_names_.append(argname)
+                    argument_defaults.append('%s=%s' % (argname, None))
+                break
+
+            argument_names.append(argname)
+            argument_names_.append(argname_)
+            argument_defaults.append('%s=%s' % (argname_, default))
+
+        arguments_default_values = ', '.join(argument_defaults)
+        arguments_comma_separated = ', '.join(argument_names_)
 
         return_type = str(function.type) + ('*' * function.derefcnt)
         arg_docs = '\n'.join(arg_docs)
@@ -282,6 +290,7 @@ def generate_one(target):
         with open(os.path.join(target, name + '.asm'), 'wt') as f:
             f.write('\n'.join(map(str.strip, lines)) + '\n')
 
+
 if __name__ == '__main__':
     p = argparse.ArgumentParser()
     p.add_argument('target_directory')

pwnlib/data/syscalls/generate_darwin.py

@@ -0,0 +1,73 @@
+
+# ./pwnlib/data/includes/darwin/aarch64.h
+# ./pwnlib/constants/darwin/aarch64.py
+
+# https://github.com/nullgemm/instant_macos_sdk (old sdk here, please use real macos device)
+# /Library/Developer/CommandLineTools/SDKs/MacOSX14.sdk/System/Library/Frameworks/Kernel.framework/Versions/A/Headers/sys/
+
+from pathlib import Path
+import re
+import sys
+
+# In the future, you should change the version of `MacOSX14.sdk`
+sdk_path = Path('/Library/Developer/CommandLineTools/SDKs/MacOSX14.sdk/System/Library/Frameworks/Kernel.framework/Versions/A/Headers/sys/')
+if not sdk_path.exists():
+    print('missing MacOSX sdk')
+    exit(1)
+
+project_path = Path(sys.argv[1])
+
+regex = re.compile(r'^#define\s+([a-zA-Z0-9_-]+)\s+([0-9]+|0x[0-9a-fA-F]+)(?:\s|$)', re.DOTALL)
+
+out_data = {}
+for file in sdk_path.iterdir():
+    if not file.is_file():
+        continue
+
+    print(file.name)
+    for line in file.read_text(errors='ignore').split('\n'):
+        matched = regex.search(line)
+        if not matched:
+            continue
+
+        key, value = matched.groups()
+        if value.startswith('0') and not value.startswith('0x') and len(value) > 1:
+            value = '0o'+value
+
+        print(key, value)
+        out_data[key] = value
+
+
+outbuf1_aarch64 = ''
+outbuf1_aarch64 += "from pwnlib.constants.constant import Constant\n"
+
+outbuf1_amd64 = ''
+outbuf1_amd64 += "from pwnlib.constants.constant import Constant\n"
+
+outbuf2_aarch64 = ''
+outbuf2_amd64 = ''
+
+# https://www.idryman.org/blog/2014/12/02/writing-64-bit-assembly-on-mac-os-x/
+# on amd64 syscall offsets from 0x2000000 + syscall number
+
+for key, value in out_data.items():
+    value_octal = value
+    if value_octal.startswith('0o'):
+        value_octal = value_octal.replace('0o', '0')
+
+    outbuf1_aarch64 += "{} = Constant('{}',{})\n".format(key, key, value)
+    outbuf2_aarch64 += "#define {} {}\n".format(key, value_octal)
+
+    if key.startswith('SYS_'):
+        value = f'{value} + 0x2000000'
+        value_octal = f'{value_octal} + 0x2000000'
+
+    outbuf1_amd64 += "{} = Constant('{}',{})\n".format(key, key, value)
+    outbuf2_amd64 += "#define {} {}\n".format(key, value_octal)
+
+pp = project_path
+(pp / Path('./pwnlib/constants/darwin/aarch64.py')).write_bytes(outbuf1_aarch64.encode())
+(pp / Path('./pwnlib/data/includes/darwin/aarch64.h')).write_bytes(outbuf2_aarch64.encode())
+
+(pp / Path('./pwnlib/constants/darwin/amd64.py')).write_bytes(outbuf1_amd64.encode())
+(pp / Path('./pwnlib/data/includes/darwin/amd64.h')).write_bytes(outbuf2_amd64.encode())

pwnlib/data/templates/pwnup.mako

@@ -1,4 +1,4 @@
-<%page args="binary, host=None, port=None, user=None, password=None, remote_path=None, quiet=False"/>\
+<%page args="binary, host=None, port=None, user=None, password=None, libc=None, remote_path=None, quiet=False"/>\
 <%
 import os
 import sys
@@ -31,6 +31,7 @@ elif host and not port:
 remote_path = remote_path or exe
 password = password or 'secret1234'
 binary_repr = repr(binary)
+libc_repr = repr(libc)
 %>\
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
@@ -43,8 +44,8 @@ from pwn import *
 %if not quiet:
 # Set up pwntools for the correct architecture
 %endif
-%if ctx.binary:
-exe = context.binary = ELF(${binary_repr})
+%if ctx.binary or not host:
+exe = context.binary = ELF(args.EXE or ${binary_repr})
 <% binary_repr = 'exe.path' %>
 %else:
 context.update(arch='i386')
@@ -58,7 +59,7 @@ exe = ${binary_repr}
 # for all created processes...
 # ./exploit.py DEBUG NOASLR
 %if host or port or user:
-# ./exploit.py GDB HOST=example.com PORT=4141
+# ./exploit.py GDB HOST=example.com PORT=4141 EXE=/tmp/executable
 %endif
 %endif
 %if host:
@@ -83,6 +84,31 @@ if not args.LOCAL:
     shell.set_working_directory(symlink=True)
 %endif
 
+%if libc:
+%if not quiet:
+# Use the specified remote libc version unless explicitly told to use the
+# local system version with the `LOCAL_LIBC` argument.
+# ./exploit.py LOCAL LOCAL_LIBC
+%endif
+if args.LOCAL_LIBC:
+    libc = exe.libc
+%if host:
+elif args.LOCAL:
+%else:
+else:
+%endif
+    library_path = libcdb.download_libraries(${libc_repr})
+    if library_path:
+        exe = context.binary = ELF.patch_custom_libraries(${binary_repr}, library_path)
+        libc = exe.libc
+    else:
+        libc = ELF(${libc_repr})
+%if host:
+else:
+    libc = ELF(${libc_repr})
+%endif
+%endif
+
 %if host:
 def start_local(argv=[], *a, **kw):
     '''Execute the target binary locally'''

pwnlib/data/useragents/download-useragents.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 """Script for downloading lists of user agent strings."""
 from __future__ import division
 import os
@@ -43,7 +43,7 @@ with log.waitfor('Fetching from from http://www.user-agents.org') as l:
     l.success()
 
 with log.waitfor('Parsing list') as l:
-    for item in soup.__getattr__('user-agents'):
+    for item in getattr(soup, 'user-agents'):
         if item.name == 'user-agent':
             ua = item.select('string')[0].string.strip()
             try:

pwnlib/dynelf.py

@@ -15,7 +15,7 @@ Example
     # leaks at least one byte at that address.
     def leak(address):
         data = p.read(address, 4)
-        log.debug("%#x => %s" % (address, enhex(data or '')))
+        log.debug("%#x => %s", address, enhex(data or ''))
         return data
 
     # For the sake of this example, let's say that we
@@ -62,8 +62,7 @@ from pwnlib.elf import constants
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak
 from pwnlib.util.fiddling import enhex
-from pwnlib.util.packing import unpack, _need_bytes
-from pwnlib.util.web import wget
+from pwnlib.util.packing import _need_bytes
 
 log    = getLogger(__name__)
 sizeof = ctypes.sizeof
@@ -141,10 +140,10 @@ class DynELF(object):
     .. _.got.plt:  https://refspecs.linuxbase.org/LSB_3.1.1/LSB-Core-generic/LSB-Core-generic/specialsections.html
     .. _DYNAMIC:   http://www.sco.com/developers/gabi/latest/ch5.dynamic.html#dynamic_section
     .. _SYSV:      https://refspecs.linuxbase.org/elf/gabi4+/ch5.dynamic.html#hash
-    .. _GNU:       https://blogs.oracle.com/ali/entry/gnu_hash_elf_sections
+    .. _GNU:       https://blogs.oracle.com/solaris/post/gnu-hash-elf-sections
     .. _DT_DEBUG:  https://reverseengineering.stackexchange.com/questions/6525/elf-link-map-when-linked-as-relro
     .. _link map:  https://sourceware.org/git/?p=glibc.git;a=blob;f=elf/link.h;h=eaca8028e45a859ac280301a6e955a14eed1b887;hb=HEAD#l84
-    .. _DT_PLTGOT: http://refspecs.linuxfoundation.org/ELF/zSeries/lzsabi0_zSeries/x2251.html
+    .. _DT_PLTGOT: https://refspecs.linuxfoundation.org/ELF/zSeries/lzsabi0_zSeries/x2251.html
     '''
 
     def __init__(self, leak, pointer=None, elf=None, libcdb=True):
@@ -166,6 +165,18 @@ class DynELF(object):
         self._waitfor  = None
         self._bases    = {}
         self._dynamic  = None
+        self.elf = None
+
+        if elf:
+            path = elf
+            if isinstance(elf, ELF):
+                path = elf.path
+
+            # Load a fresh copy of the ELF
+            with context.local(log_level='error'):
+                w = self.waitfor("Loading from %r" % path)
+                self.elf = ELF(path)
+                w.success("[LOADED]")
 
         if not (pointer or (elf and elf.address)):
             log.error("Must specify either a pointer into a module and/or an ELF file with a valid base address")
@@ -178,12 +189,15 @@ class DynELF(object):
         if not elf:
             log.warn_once("No ELF provided.  Leaking is much faster if you have a copy of the ELF being leaked.")
 
-        self.elf     = elf
         self.leak    = leak
         self.libbase = self._find_base(pointer or elf.address)
 
         if elf:
-            self._find_linkmap_assisted(elf)
+            self._elftype = self.elf.elftype
+            self._elfclass = self.elf.elfclass
+            self.elf.address = self.libbase
+            self._dynamic = self.elf.get_section_by_name('.dynamic').header.sh_addr
+            self._dynamic = self._make_absolute_ptr(self._dynamic) 
 
     @classmethod
     def for_one_lib_only(cls, leak, ptr):
@@ -242,49 +256,6 @@ class DynELF(object):
             self._dynamic = self._find_dynamic_phdr()
         return self._dynamic
 
-    def _find_linkmap_assisted(self, path):
-        """Uses an ELF file to assist in finding the link_map.
-        """
-        if isinstance(path, ELF):
-            path = path.path
-
-        # Load a fresh copy of the ELF
-        with context.local(log_level='error'):
-            elf = ELF(path)
-        elf.address = self.libbase
-
-        w = self.waitfor("Loading from %r" % elf.path)
-
-        # Save our real leaker
-        real_leak = self.leak
-
-        # Create a fake leaker which just leaks out of the 'loaded' ELF
-        # However, we may load things which are outside of the ELF (e.g.
-        # the linkmap or GOT) so we need to fall back on the real leak.
-        @MemLeak
-        def fake_leak(address):
-            try:
-                return elf.read(address, 4)
-            except ValueError:
-                return real_leak.b(address)
-
-        # Save off our real leaker, use the fake leaker
-        self.leak = fake_leak
-
-        # Get useful pointers for resolving the linkmap faster
-        w.status("Searching for DT_PLTGOT")
-        pltgot = self._find_dt(constants.DT_PLTGOT)
-
-        w.status("Searching for DT_DEBUG")
-        debug  = self._find_dt(constants.DT_DEBUG)
-
-        # Restore the real leaker
-        self.leak = real_leak
-
-        # Find the linkmap using the helper pointers
-        self._find_linkmap(pltgot, debug)
-        self.success('Done')
-
     def _find_base(self, ptr):
         page_size = 0x1000
         page_mask = ~(page_size - 1)
@@ -381,6 +352,27 @@ class DynELF(object):
 
         return dynamic
 
+    def _find_dt_optimized(self, name):
+        """
+        Find an entry in the DYNAMIC array through an ELF
+
+        Arguments:
+            name(str): Name of the tag to find ('DT_DEBUG', 'DT_PLTGOT', ...)
+
+        Returns:
+            Pointer to the data described by the specified entry.
+        """
+        if not self.elf:
+            return None
+
+        ptr = self.elf.dynamic_value_by_tag(name)
+        if ptr:
+            ptr = self._make_absolute_ptr(ptr)
+            self.success("Found %s at %#x" % (name, ptr))
+            return ptr
+        return None
+
+
     def _find_dt(self, tag):
         """
         Find an entry in the DYNAMIC array.
@@ -391,11 +383,16 @@ class DynELF(object):
         Returns:
             Pointer to the data described by the specified entry.
         """
-        leak    = self.leak
         base    = self.libbase
         dynamic = self.dynamic
+        leak    = self.leak
         name    = next(k for k,v in ENUM_D_TAG.items() if v == tag)
 
+        # Read directly from the ELF if possible
+        ptr = self._find_dt_optimized(name)
+        if ptr:
+            return ptr
+
         Dyn = {32: elf.Elf32_Dyn,    64: elf.Elf64_Dyn}     [self.elfclass]
 
         # Found the _DYNAMIC program header, now find PLTGOT entry in it
@@ -408,10 +405,10 @@ class DynELF(object):
             self.failure("Could not find tag %s" % name)
             return None
 
-        self.status("Found %s at %#x" % (name, dynamic))
         ptr = leak.field(dynamic, Dyn.d_ptr)
 
         ptr = self._make_absolute_ptr(ptr)
+        self.status("Found %s at %#x" % (name, ptr))
 
         return ptr
 
@@ -562,7 +559,7 @@ class DynELF(object):
         else:   dynlib = self
 
         if dynlib is None:
-            log.failure("Could not find %r" % lib)
+            log.failure("Could not find %r", lib)
             return None
 
         #
@@ -573,7 +570,7 @@ class DynELF(object):
             self.status("Trying lookup based on Build ID")
             build_id = dynlib._lookup_build_id(lib=lib)
             if build_id:
-                log.info("Trying lookup based on Build ID: %s" % build_id)
+                log.info("Trying lookup based on Build ID: %s", build_id)
                 path = libcdb.search_by_build_id(build_id)
                 if path:
                     with context.local(log_level='error'):
@@ -600,6 +597,10 @@ class DynELF(object):
         Return a dictionary mapping library path to its base address.
         '''
         if not self._bases:
+            if self.link_map is None:
+                self.failure("Cannot determine bases without linkmap")
+                return {}
+                
             leak    = self.leak
             LinkMap = {32: elf.Elf32_Link_Map, 64: elf.Elf64_Link_Map}[self.elfclass]
 
@@ -615,7 +616,7 @@ class DynELF(object):
                 addr   = leak.field(cur, LinkMap.l_addr)
                 cur    = leak.field(cur, LinkMap.l_next)
 
-                log.debug('Found %r @ %#x' % (name, addr))
+                log.debug('Found %r @ %#x', name, addr)
 
                 self._bases[name] = addr
 
@@ -667,6 +668,62 @@ class DynELF(object):
         lib._waitfor = self._waitfor
         return lib
 
+    def _rel_lookup(self, symb, strtab=None, symtab=None, jmprel=None):
+        """Performs slower symbol lookup using DT_JMPREL(.rela.plt)"""
+        leak = self.leak
+        elf_obj = self.elf
+        symb_name = symb.decode()
+
+        # If elf is available look for the symbol in it
+        if elf_obj and symb_name in elf_obj.symbols:
+            self.success("Symbol '%s' found in ELF!" % symb_name)
+            return elf_obj.symbols[symb_name]
+
+        log.warning("Looking up symbol through DT_JMPREL. This might be slower...")
+
+
+        strtab  = strtab or self._find_dt(constants.DT_STRTAB)
+        symtab  = symtab or self._find_dt(constants.DT_SYMTAB)
+        jmprel  = jmprel or self._find_dt(constants.DT_JMPREL) # .rela.plt
+
+        strtab = self._make_absolute_ptr(strtab)
+        symtab = self._make_absolute_ptr(symtab)
+        jmprel = self._make_absolute_ptr(jmprel)
+
+        w = self.waitfor("Looking for %s in .rel.plt" % symb)
+        # We look for the symbol by iterating through each Elf64_Rel entry.
+        # For each Elf64_Rel, get the Elf64_Sym for that entry
+        # Then compare the Elf64_Sym.st_name with the symbol name
+       
+        Rel = {32: elf.Elf32_Rel, 64: elf.Elf64_Rel}[self.elfclass]
+        Sym = {32: elf.Elf32_Sym, 64: elf.Elf64_Sym}[self.elfclass]
+
+        rel_addr = jmprel
+        rel_entry = None
+        while True:
+            rel_entry = leak.struct(rel_addr, Rel)
+
+            # We ran out of entries in DT_JMPREL 
+            if rel_entry.r_offset == 0:
+                return None
+
+            sym_idx = rel_entry.r_info >> 32 # might be different for 32-bit
+            sym_entry_address = symtab + ( sym_idx * sizeof(Sym) )
+            sym_str_off = leak.field(sym_entry_address, Sym.st_name)
+            symb_str = leak.s(strtab+sym_str_off)
+
+            if symb_str == symb:
+                w.success("Found matching Elf64_Rel entry!")
+                break
+
+            rel_addr += sizeof(Rel)
+
+        symbol_address = self._make_absolute_ptr(rel_entry.r_offset)
+
+        return symbol_address
+
+
+
     def _lookup(self, symb):
         """Performs the actual symbol lookup within one ELF file."""
         leak = self.leak
@@ -699,9 +756,39 @@ class DynELF(object):
         #
         # Perform the hash lookup
         #
+
+        # Save off our real leaker in case we use the fake leaker
+        real_leak = self.leak
+        if self.elf:
+
+            # Create a fake leaker which just leaks out of the 'loaded' ELF
+            # However, we may load things which are outside of the ELF (e.g.
+            # the linkmap or GOT) so we need to fall back on the real leak.
+            @MemLeak
+            def fake_leak(address):
+                try:
+                    return self.elf.read(address, 4)
+                except ValueError:
+                    return real_leak.b(address)
+            # Use fake leaker since ELF is available
+            self.leak = fake_leak
+
         routine = {'sysv': self._resolve_symbol_sysv,
                    'gnu':  self._resolve_symbol_gnu}[hshtype]
-        return routine(self.libbase, symb, hshtab, strtab, symtab)
+        resolved_addr = routine(self.libbase, symb, hshtab, strtab, symtab)
+
+        if resolved_addr:
+            # Restore the original leaker
+            self.leak = real_leak
+            return resolved_addr
+
+        # if symbol not found in GNU_Hash, try looking in JMPREL
+        resolved_addr = self._rel_lookup(symb, strtab, symtab)
+
+        # Restore the original leaker
+        self.leak = real_leak
+
+        return resolved_addr
 
     def _resolve_symbol_sysv(self, libbase, symb, hshtab, strtab, symtab):
         """
@@ -770,7 +857,7 @@ class DynELF(object):
             structure.
 
             Again, Oracle has good documentation.
-            https://blogs.oracle.com/ali/entry/gnu_hash_elf_sections
+            https://blogs.oracle.com/solaris/post/gnu-hash-elf-sections
 
             You can force an ELF to use this type of symbol table by compiling
             with 'gcc -Wl,--hash-style=gnu'

pwnlib/elf/corefile.py

@@ -76,7 +76,6 @@ import tempfile
 from io import BytesIO, StringIO
 
 import elftools
-from elftools.common.py3compat import bytes2str
 from elftools.common.utils import roundup
 from elftools.common.utils import struct_parse
 from elftools.construct import CString
@@ -106,43 +105,11 @@ prstatus_types = {
     'aarch64': elf_prstatus_aarch64
 }
 
-prpsinfo_types = {
-    32: elf_prpsinfo_32,
-    64: elf_prpsinfo_64,
-}
-
 siginfo_types = {
     32: elf_siginfo_32,
     64: elf_siginfo_64
 }
 
-# Slightly modified copy of the pyelftools version of the same function,
-# until they fix this issue:
-# https://github.com/eliben/pyelftools/issues/93
-def iter_notes(self):
-    """ Iterates the list of notes in the segment.
-    """
-    offset = self['p_offset']
-    end = self['p_offset'] + self['p_filesz']
-    while offset < end:
-        note = struct_parse(
-            self.elffile.structs.Elf_Nhdr,
-            self.stream,
-            stream_pos=offset)
-        note['n_offset'] = offset
-        offset += self.elffile.structs.Elf_Nhdr.sizeof()
-        self.stream.seek(offset)
-        # n_namesz is 4-byte aligned.
-        disk_namesz = roundup(note['n_namesz'], 2)
-        note['n_name'] = bytes2str(
-            CString('').parse(self.stream.read(disk_namesz)))
-        offset += disk_namesz
-
-        desc_data = bytes2str(self.stream.read(note['n_descsz']))
-        note['n_desc'] = desc_data
-        offset += roundup(note['n_descsz'], 2)
-        note['n_size'] = offset - note['n_offset']
-        yield note
 
 class Mapping(object):
     """Encapsulates information about a memory mapping in a :class:`Corefile`.
@@ -445,7 +412,7 @@ class Corefile(ELF):
         will be loaded.
 
         >>> process('bash').corefile.libc # doctest: +ELLIPSIS
-        Mapping('/.../libc-....so', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
+        Mapping('.../libc...so...', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
 
         The corefile also contains a :attr:`.stack` property, which gives
         us direct access to the stack contents.  On Linux, the very top of the stack
@@ -530,7 +497,6 @@ class Corefile(ELF):
         >>> io = elf.process()
         >>> io.wait(2)
         >>> core = io.corefile
-        [!] End of the stack is corrupted, skipping stack parsing (got: 41414141)
         >>> core.argc, core.argv, core.env
         (0, [], {})
         >>> core.stack.data.endswith(b'AAAA')
@@ -605,7 +571,6 @@ class Corefile(ELF):
             log.warn_once("%s does not use a supported corefile architecture, registers are unavailable" % self.file.name)
 
         prstatus_type = prstatus_types.get(self.arch)
-        prpsinfo_type = prpsinfo_types.get(self.bits)
         siginfo_type = siginfo_types.get(self.bits)
 
         with log.waitfor("Parsing corefile...") as w:
@@ -616,39 +581,30 @@ class Corefile(ELF):
                     continue
 
 
-                # Note that older versions of pyelftools (<=0.24) are missing enum values
-                # for NT_PRSTATUS, NT_PRPSINFO, NT_AUXV, etc.
-                # For this reason, we have to check if note.n_type is any of several values.
-                for note in iter_notes(segment):
-                    if not isinstance(note.n_desc, bytes):
-                        note['n_desc'] = note.n_desc.encode('latin1')
+                for note in segment.iter_notes():
                     # Try to find NT_PRSTATUS.
-                    if prstatus_type and \
-                       note.n_descsz == ctypes.sizeof(prstatus_type) and \
-                       note.n_type in ('NT_GNU_ABI_TAG', 'NT_PRSTATUS'):
+                    if note.n_type == 'NT_PRSTATUS':
                         self.NT_PRSTATUS = note
                         self.prstatus = prstatus_type.from_buffer_copy(note.n_desc)
 
                     # Try to find NT_PRPSINFO
-                    if prpsinfo_type and \
-                       note.n_descsz == ctypes.sizeof(prpsinfo_type) and \
-                       note.n_type in ('NT_GNU_ABI_TAG', 'NT_PRPSINFO'):
+                    if note.n_type == 'NT_PRPSINFO':
                         self.NT_PRPSINFO = note
-                        self.prpsinfo = prpsinfo_type.from_buffer_copy(note.n_desc)
+                        self.prpsinfo = note.n_desc
 
                     # Try to find NT_SIGINFO so we can see the fault
-                    if note.n_type in (0x53494749, 'NT_SIGINFO'):
+                    if note.n_type == 'NT_SIGINFO':
                         self.NT_SIGINFO = note
                         self.siginfo = siginfo_type.from_buffer_copy(note.n_desc)
 
                     # Try to find the list of mapped files
-                    if note.n_type in (constants.NT_FILE, 'NT_FILE'):
+                    if note.n_type == 'NT_FILE':
                         with context.local(bytes=self.bytes):
                             self._parse_nt_file(note)
 
                     # Try to find the auxiliary vector, which will tell us
                     # where the top of the stack is.
-                    if note.n_type in (constants.NT_AUXV, 'NT_AUXV'):
+                    if note.n_type == 'NT_AUXV':
                         self.NT_AUXV = note
                         with context.local(bytes=self.bytes):
                             self._parse_auxv(note)
@@ -666,7 +622,7 @@ class Corefile(ELF):
                     log.warn('Could not find the stack!')
                     self.stack = None
 
-            with context.local(bytes=self.bytes, log_level='warn'):
+            with context.local(bytes=self.bytes):
                 try:
                     self._parse_stack()
                 except ValueError:
@@ -684,31 +640,16 @@ class Corefile(ELF):
             self._describe_core()
 
     def _parse_nt_file(self, note):
-        t = tube()
-        t.unrecv(note.n_desc)
-
-        count = t.unpack()
-        page_size = t.unpack()
-
         starts = []
         addresses = {}
 
-        for i in range(count):
-            start = t.unpack()
-            end = t.unpack()
-            offset = t.unpack()
-            starts.append((start, offset))
-
-        for i in range(count):
-            filename = t.recvuntil(b'\x00', drop=True)
+        for vma, filename in zip(note.n_desc.Elf_Nt_File_Entry, note.n_desc.filename):
             if not isinstance(filename, str):
-                filename = filename.decode('utf-8')
-            (start, offset) = starts[i]
-
+                filename = filename.decode('utf-8', 'surrogateescape')
             for mapping in self.mappings:
-                if mapping.start == start:
+                if mapping.start == vma.vm_start:
                     mapping.name = filename
-                    mapping.page_offset = offset
+                    mapping.page_offset = vma.page_offset
 
         self.mappings = sorted(self.mappings, key=lambda m: m.start)
 
@@ -759,7 +700,7 @@ class Corefile(ELF):
     @property
     def libc(self):
         """:class:`Mapping`: First mapping for ``libc.so``"""
-        expr = r'libc\b.*so$'
+        expr = r'^libc\b.*so(?:\.6)?$'
 
         for m in self.mappings:
             if not m.name:
@@ -1184,14 +1125,13 @@ class Corefile(ELF):
         pwnlib.gdb.attach(self, exe=self.exe.path)
 
     def __getattr__(self, attribute):
-        if self.prstatus:
-            if hasattr(self.prstatus, attribute):
-                return getattr(self.prstatus, attribute)
+        if attribute.startswith('_') or not self.prstatus:
+            raise AttributeError(attribute)
 
-            if hasattr(self.prstatus.pr_reg, attribute):
-                return getattr(self.prstatus.pr_reg, attribute)
+        if hasattr(self.prstatus, attribute):
+            return getattr(self.prstatus, attribute)
 
-        return super(Corefile, self).__getattribute__(attribute)
+        return getattr(self.prstatus.pr_reg, attribute)
 
     # Override routines which don't make sense for Corefiles
     def _populate_got(*a): pass
@@ -1407,9 +1347,8 @@ class CorefileFinder(object):
                     "coredumpctl",
                     "dump",
                     "--output=%s" % filename,
-                    # Filter coredump by pid and filename
+                    # Filter coredump by pid
                     str(self.pid),
-                    self.basename,
                 ],
                 stdout=open(os.devnull, 'w'),
                 stderr=subprocess.STDOUT,

pwnlib/elf/datatypes.py

@@ -399,6 +399,15 @@ class Elf64_Sym(ctypes.Structure):
                 ("st_value", Elf64_Addr),
                 ("st_size", Elf64_Xword),]
 
+class Elf64_Rel(ctypes.Structure):
+    _fields_ = [("r_offset", Elf64_Addr),
+                ("r_info", Elf64_Xword),
+                ("r_addend", Elf64_Sxword),]
+
+class Elf32_Rel(ctypes.Structure):
+    _fields_ = [("r_offset", Elf32_Addr),
+                ("r_info", Elf32_Word),]
+
 class Elf32_Link_Map(ctypes.Structure):
     _fields_ = [("l_addr", Elf32_Addr),
                 ("l_name", Elf32_Addr),
@@ -604,7 +613,8 @@ class user_regs_struct_aarch64(ctypes.Structure):
     def __getattr__(self, name):
         if name.startswith('r'):
             name = 'x' + name[1:]
-            return getattr(name) & 0xffffffff
+            return getattr(self, name) & 0xffffffff
+        raise AttributeError(name)
 
 class elf_prstatus_i386(ctypes.Structure):
     _fields_ = generate_prstatus_common(32, user_regs_struct_i386)
@@ -630,29 +640,6 @@ class Elf64_auxv_t(ctypes.Structure):
     _fields_ = [('a_type', ctypes.c_uint64),
                 ('a_val', ctypes.c_uint64),]
 
-def generate_prpsinfo(long):
-    return [
-        ('pr_state', byte),
-        ('pr_sname', char),
-        ('pr_zomb', byte),
-        ('pr_nice', byte),
-        ('pr_flag', long),
-        ('pr_uid', ctypes.c_ushort),
-        ('pr_gid', ctypes.c_ushort),
-        ('pr_pid', ctypes.c_int),
-        ('pr_ppid', ctypes.c_int),
-        ('pr_pgrp', ctypes.c_int),
-        ('pr_sid', ctypes.c_int),
-        ('pr_fname', char * 16),
-        ('pr_psargs', char * 80)
-    ]
-
-class elf_prpsinfo_32(ctypes.Structure):
-    _fields_ = generate_prpsinfo(Elf32_Addr)
-
-class elf_prpsinfo_64(ctypes.Structure):
-    _fields_ = generate_prpsinfo(Elf64_Addr)
-
 def generate_siginfo(int_t, long_t):
     class siginfo_t(ctypes.Structure):
         _fields_ = [('si_signo', int_t),

pwnlib/elf/elf.py

@@ -47,15 +47,15 @@ import tempfile
 
 from six import BytesIO
 
-from collections import namedtuple
+from collections import namedtuple, defaultdict
 
-from elftools.elf.constants import E_FLAGS
 from elftools.elf.constants import P_FLAGS
 from elftools.elf.constants import SHN_INDICES
 from elftools.elf.descriptions import describe_e_type
 from elftools.elf.elffile import ELFFile
+from elftools.elf.enums import ENUM_GNU_PROPERTY_X86_FEATURE_1_FLAGS
 from elftools.elf.gnuversions import GNUVerDefSection
-from elftools.elf.relocation import RelocationSection
+from elftools.elf.relocation import RelocationSection, RelrRelocationSection
 from elftools.elf.sections import SymbolTableSection
 from elftools.elf.segments import InterpSegment
 
@@ -83,7 +83,7 @@ from pwnlib.tubes.process import process
 from pwnlib.util import misc
 from pwnlib.util import packing
 from pwnlib.util.fiddling import unhex
-from pwnlib.util.misc import align, align_down
+from pwnlib.util.misc import align, align_down, which
 from pwnlib.util.sh_string import sh_string
 
 log = getLogger(__name__)
@@ -157,6 +157,12 @@ class dotdict(dict):
         >>> x.bar.baz
         4
     """
+    def __missing__(self, name):
+        if isinstance(name, (bytes, bytearray)):
+            name = packing._decode(name)
+            return self[name]
+        raise KeyError(name)
+
     def __getattr__(self, name):
         if name in self:
             return self[name]
@@ -167,8 +173,7 @@ class dotdict(dict):
 
         if subkeys:
             return dotdict(subkeys)
-
-        return getattr(super(dotdict, self), name)
+        raise AttributeError(name)
 
 class ELF(ELFFile):
     """Encapsulates information about an ELF file.
@@ -220,14 +225,7 @@ class ELF(ELFFile):
         super(ELF,self).__init__(self.mmap)
 
         #: :class:`str`: Path to the file
-        self.path = os.path.abspath(path)
-
-        #: :class:`str`: Architecture of the file (e.g. ``'i386'``, ``'arm'``).
-        #:
-        #: See: :attr:`.ContextType.arch`
-        self.arch = self.get_machine_arch()
-        if isinstance(self.arch, (bytes, six.text_type)):
-            self.arch = self.arch.lower()
+        self.path = packing._need_text(os.path.abspath(path))
 
         #: :class:`dotdict` of ``name`` to ``address`` for all symbols in the ELF
         self.symbols = dotdict()
@@ -235,7 +233,7 @@ class ELF(ELFFile):
         #: :class:`dotdict` of ``name`` to ``address`` for all Global Offset Table (GOT) entries
         self.got = dotdict()
 
-        #: :class:`dotdict` of ``name`` to ``address`` for all Procedure Linkate Table (PLT) entries
+        #: :class:`dotdict` of ``name`` to ``address`` for all Procedure Linkage Table (PLT) entries
         self.plt = dotdict()
 
         #: :class:`dotdict` of ``name`` to :class:`.Function` for each function in the ELF
@@ -263,17 +261,12 @@ class ELF(ELFFile):
         #: :class:`int`: Pointer width, in bytes
         self.bytes = self.bits // 8
 
-        if self.arch == 'mips':
-            mask = lambda a, b: a & b == b
-            flags = self.header['e_flags']
-
-            if mask(flags, E_FLAGS.EF_MIPS_ARCH_32) \
-            or mask(flags, E_FLAGS.EF_MIPS_ARCH_32R2):
-                pass
-            elif mask(flags, E_FLAGS.EF_MIPS_ARCH_64) \
-            or mask(flags, E_FLAGS.EF_MIPS_ARCH_64R2):
-                self.arch = 'mips64'
-                self.bits = 64
+        #: :class:`str`: Architecture of the file (e.g. ``'i386'``, ``'arm'``).
+        #:
+        #: See: :attr:`.ContextType.arch`
+        self.arch = self.get_machine_arch()
+        if isinstance(self.arch, (bytes, six.text_type)):
+            self.arch = self.arch.lower()
 
         self._sections = None
         self._segments = None
@@ -455,7 +448,7 @@ class ELF(ELFFile):
 
     def _describe(self, *a, **kw):
         log.info_once(
-            '%s\n%-10s%s-%s-%s\n%s',
+            '%s\n%-12s%s-%s-%s\n%s',
             repr(self.path),
             'Arch:',
             self.arch,
@@ -466,18 +459,22 @@ class ELF(ELFFile):
 
     def get_machine_arch(self):
         return {
-            'EM_X86_64': 'amd64',
-            'EM_386' :'i386',
-            'EM_486': 'i386',
-            'EM_ARM': 'arm',
-            'EM_AARCH64': 'aarch64',
-            'EM_MIPS': 'mips',
-            'EM_PPC': 'powerpc',
-            'EM_PPC64': 'powerpc64',
-            'EM_SPARC32PLUS': 'sparc',
-            'EM_SPARCV9': 'sparc64',
-            'EM_IA_64': 'ia64'
-        }.get(self['e_machine'], self['e_machine'])
+            ('EM_X86_64', 64): 'amd64',
+            ('EM_X86_64', 32): 'amd64', # x32 ABI
+            ('EM_386', 32): 'i386',
+            ('EM_486', 32): 'i386',
+            ('EM_ARM', 32): 'arm',
+            ('EM_AARCH64', 64): 'aarch64',
+            ('EM_MIPS', 32): 'mips',
+            ('EM_MIPS', 64): 'mips64',
+            ('EM_PPC', 32): 'powerpc',
+            ('EM_PPC64', 64): 'powerpc64',
+            ('EM_SPARC32PLUS', 32): 'sparc',
+            ('EM_SPARCV9', 64): 'sparc64',
+            ('EM_IA_64', 64): 'ia64',
+            ('EM_RISCV', 32): 'riscv32',
+            ('EM_RISCV', 64): 'riscv64',
+        }.get((self['e_machine'], self.bits), self['e_machine'])
 
     @property
     def entry(self):
@@ -515,6 +512,29 @@ class ELF(ELFFile):
             if t == seg.header.p_type or t in str(seg.header.p_type):
                 yield seg
 
+    def iter_notes(self):
+        """ 
+        Yields:
+            All the notes in the PT_NOTE segments.  Each result is a dictionary-
+            like object with ``n_name``, ``n_type``, and ``n_desc`` fields, amongst
+            others.
+        """
+        for seg in self.iter_segments_by_type('PT_NOTE'):
+            for note in seg.iter_notes():
+                yield note
+
+    def iter_properties(self):
+        """
+        Yields:
+            All the GNU properties in the PT_NOTE segments.  Each result is a dictionary-
+            like object with ``pr_type``, ``pr_datasz``, and ``pr_data`` fields.
+        """
+        for note in self.iter_notes():
+            if note.n_type != 'NT_GNU_PROPERTY_TYPE_0':
+                continue
+            for prop in note.n_desc:
+                yield prop
+                
     def get_segment_for_address(self, address, size=1):
         """get_segment_for_address(address, size=1) -> Segment
 
@@ -922,15 +942,25 @@ class ELF(ELFFile):
             self.symbols['got.' + symbol] = address
 
     def _populate_got(self):
-        """Loads the symbols for all relocations"""
+        """Loads the symbols for all relocations.
+
+            >>> libc = ELF(which('bash')).libc
+            >>> assert 'strchrnul' in libc.got
+            >>> assert 'memcpy' in libc.got
+            >>> assert libc.got.strchrnul != libc.got.memcpy
+        """
         # Statically linked implies no relocations, since there is no linker
         # Could always be self-relocating like Android's linker *shrug*
         if self.statically_linked:
             return
 
+        revsymbols = defaultdict(list)
+        for name, addr in self.symbols.items():
+            revsymbols[addr].append(name)
+
         for section in self.sections:
             # We are only interested in relocations
-            if not isinstance(section, RelocationSection):
+            if not isinstance(section, (RelocationSection, RelrRelocationSection)):
                 continue
 
             # Only get relocations which link to another section (for symbols)
@@ -942,7 +972,13 @@ class ELF(ELFFile):
             for rel in section.iter_relocations():
                 sym_idx  = rel.entry.r_info_sym
 
-                if not sym_idx:
+                if not sym_idx and rel.is_RELA():
+                    # TODO: actually resolve relocations
+                    relocated = rel.entry.r_addend  # sufficient for now
+
+                    symnames = revsymbols[relocated]
+                    for symname in symnames:
+                        self.got[symname] = rel.entry.r_offset
                     continue
 
                 symbol = symbols.get_symbol(sym_idx)
@@ -1056,10 +1092,10 @@ class ELF(ELFFile):
             return
 
         banner = self.string(self.symbols.linux_banner)
-        
+
         # convert banner into a utf-8 string since re.search does not accept bytes anymore
         banner = banner.decode('utf-8')
-        
+
         # 'Linux version 3.18.31-gd0846ecc
         regex = r'Linux version (\S+)'
         match = re.search(regex, banner)
@@ -1076,11 +1112,17 @@ class ELF(ELFFile):
 
     @property
     def libc_start_main_return(self):
-        """
-            Try to find the return address from main into __libc_start_main.
-            The heuristic to find the call to the function pointer of main is
-            to list all calls inside __libc_start_main, find the call to exit
-            after the call to main and select the previous call.
+        """:class:`int`: Address of the return address into __libc_start_main from main.
+
+        >>> bash = ELF(which('bash'))
+        >>> libc = bash.libc
+        >>> libc.libc_start_main_return > 0
+        True
+
+        Try to find the return address from main into __libc_start_main.
+        The heuristic to find the call to the function pointer of main is
+        to list all calls inside __libc_start_main, find the call to exit
+        after the call to main and select the previous call.
         """
         if '__libc_start_main' not in self.functions:
             return 0
@@ -1103,19 +1145,44 @@ class ELF(ELFFile):
         else:
             log.error('Unsupported architecture %s in ELF.libc_start_main_return', self.arch)
             return 0
-        
-        code = self.disasm(self.symbols['__libc_start_main'], self.functions['__libc_start_main'].size)
-        exit_addr = hex(self.symbols['exit'])
-        lines = code.split('\n')
-        calls = [(index, line) for index, line in enumerate(lines) if set(line.split()) & call_instructions]
-        exit_calls = [index for index, line in enumerate(calls) if exit_addr in line[1]]
-        if len(exit_calls) != 1:
-            return 0
 
-        call_to_main = calls[exit_calls[0] - 1]
-        return_from_main = lines[call_to_main[0] + call_return_offset].lstrip()
-        return_from_main = int(return_from_main[ : return_from_main.index(':') ], 16)
-        return return_from_main
+        lines = self.functions['__libc_start_main'].disasm().split('\n')
+        exit_addr = hex(self.symbols['exit'])
+        calls = [(index, line) for index, line in enumerate(lines) if set(line.split()) & call_instructions]
+
+        def find_ret_main_addr(lines, calls):
+            exit_calls = [index for index, line in enumerate(calls) if exit_addr in line[1]]
+            if len(exit_calls) != 1:
+                return 0
+
+            call_to_main = calls[exit_calls[0] - 1]
+            return_from_main = lines[call_to_main[0] + call_return_offset].lstrip()
+            return_from_main = int(return_from_main[ : return_from_main.index(':') ], 16)
+            return return_from_main
+
+        # Starting with glibc-2.34 calling `main` is split out into `__libc_start_call_main`
+        ret_addr = find_ret_main_addr(lines, calls)
+        # Pre glibc-2.34 case - `main` is called directly
+        if ret_addr:
+            return ret_addr
+
+        # `__libc_start_main` -> `__libc_start_call_main` -> `main`
+        # Find a direct call which calls `exit` once. That's probably `__libc_start_call_main`.
+        direct_call_pattern = re.compile(r'['+r'|'.join(call_instructions)+r']\s+(0x[0-9a-zA-Z]+)')
+        for line in calls:
+            match = direct_call_pattern.search(line[1])
+            if not match:
+                continue
+
+            target_addr = int(match.group(1), 0)
+            # `__libc_start_call_main` is usually smaller than `__libc_start_main`, so
+            # we might disassemble a bit too much, but it's a good dynamic estimate.
+            callee_lines = self.disasm(target_addr, self.functions['__libc_start_main'].size).split('\n')
+            callee_calls = [(index, line) for index, line in enumerate(callee_lines) if set(line.split()) & call_instructions]
+            ret_addr = find_ret_main_addr(callee_lines, callee_calls)
+            if ret_addr:
+                return ret_addr
+        return 0
 
     def search(self, needle, writable = False, executable = False):
         """search(needle, writable = False, executable = False) -> generator
@@ -1129,7 +1196,7 @@ class ELF(ELFFile):
             won't work.
 
         Arguments:
-            needle(str): String to search for.
+            needle(bytes): String to search for.
             writable(bool): Search only writable sections.
             executable(bool): Search only executable sections.
 
@@ -1165,13 +1232,14 @@ class ELF(ELFFile):
             segments = self.executable_segments
         else:
             segments = self.segments
-
+        needle = packing._need_bytes(needle)
         for seg in segments:
             addr   = seg.header.p_vaddr
             memsz  = seg.header.p_memsz
-            zeroed = memsz - seg.header.p_filesz
+            filesz = seg.header.p_filesz
+            zeroed = memsz - filesz
             offset = seg.header.p_offset
-            data   = self.mmap[offset:offset+memsz]
+            data   = self.mmap[offset:offset+filesz]
             data   += b'\x00' * zeroed
             offset = 0
             while True:
@@ -1278,6 +1346,7 @@ class ELF(ELFFile):
             or :const:`None`.
 
         Examples:
+
             >>> bash = ELF(which('bash'))
             >>> bash.vaddr_to_offset(bash.address)
             0
@@ -1310,7 +1379,7 @@ class ELF(ELFFile):
             count(int): Number of bytes to read
 
         Returns:
-            A :class:`str` object, or :const:`None`.
+            A :class:`bytes` object, or :const:`None`.
 
         Examples:
             The simplest example is just to read the ELF header.
@@ -1428,6 +1497,7 @@ class ELF(ELFFile):
             that it stays in the same segment.
 
         Examples:
+
           >>> bash = ELF(which('bash'))
           >>> bash.read(bash.address+1, 3)
           b'ELF'
@@ -1471,7 +1541,7 @@ class ELF(ELFFile):
 
     @property
     def data(self):
-        """:class:`str`: Raw data of the ELF file.
+        """:class:`bytes`: Raw data of the ELF file.
 
         See:
             :meth:`get_data`
@@ -1499,7 +1569,7 @@ class ELF(ELFFile):
         This modifies the ELF in-place.
         The resulting binary can be saved with :meth:`.ELF.save`
         """
-        binary = asm(assembly, vma=address)
+        binary = asm(assembly, vma=address, arch=self.arch, endian=self.endian, bits=self.bits)
         self.write(address, binary)
 
     def bss(self, offset=0):
@@ -1617,7 +1687,7 @@ class ELF(ELFFile):
         .. _page 81: https://refspecs.linuxbase.org/elf/elf.pdf#page=81
         .. _DT_BIND_NOW: https://refspecs.linuxbase.org/elf/elf.pdf#page=81
         .. _PT_GNU_RELRO: https://refspecs.linuxbase.org/LSB_3.1.1/LSB-Core-generic/LSB-Core-generic.html#PROGHEADER
-        .. _DF_BIND_NOW: http://refspecs.linuxbase.org/elf/gabi4+/ch5.dynamic.html#df_bind_now
+        .. _DF_BIND_NOW: https://refspecs.linuxbase.org/elf/gabi4+/ch5.dynamic.html#df_bind_now
 
         >>> path = pwnlib.data.elf.relro.path
         >>> for test in glob(os.path.join(path, 'test-*')):
@@ -1649,61 +1719,244 @@ class ELF(ELFFile):
         Specifically, we are checking for ``READ_IMPLIES_EXEC`` being set
         by the kernel, as a result of honoring ``PT_GNU_STACK`` in the kernel.
 
-        The **Linux kernel** directly honors ``PT_GNU_STACK`` to `mark the
-        stack as executable.`__
+        ``READ_IMPLIES_EXEC`` is set, according to a set of architecture specific
+        rules, that depend on the CPU features, and the presence of ``PT_GNU_STACK``.
 
-        .. __: https://github.com/torvalds/linux/blob/v4.9/fs/binfmt_elf.c#L784-L789
+        Unfortunately, :class:`ELF` is not context-aware, so it's not always possible
+        to determine whether the process of a binary that's missing ``PT_GNU_STACK``
+        will have NX or not.
+        
+        The rules are as follows:
+
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | ELF arch  | linux        | GNU_STACK                 | other                                          | NX       |
+            +===========+==============+===========================+================================================+==========+
+            | i386      | < 5.8        | non-exec                  |                                                | enabled  |
+            |           | [#x86_5.7]_  +---------------------------+------------------------------------------------+----------+
+            |           |              | exec / missing            |                                                | disabled |
+            |           +--------------+---------------------------+------------------------------------------------+----------+
+            |           | >= 5.8       | exec / non-exec           |                                                | enabled  |
+            |           | [#x86_5.8]_  +---------------------------+------------------------------------------------+----------+
+            |           |              | missing                   |                                                | disabled |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | amd64     | < 5.8        | non-exec                  |                                                | enabled  |
+            |           | [#x86_5.7]_  +---------------------------+------------------------------------------------+----------+
+            |           |              | exec / missing            |                                                | disabled |
+            |           +--------------+---------------------------+------------------------------------------------+----------+
+            |           | >= 5.8       | exec / non-exec / missing |                                                | enabled  |
+            |           | [#x86_5.8]_  |                           |                                                |          |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | arm       | < 5.8        | non-exec*                 |                                                | enabled  |
+            |           | [#arm_5.7]_  +---------------------------+------------------------------------------------+----------+
+            |           |              | exec / missing            |                                                | disabled |
+            |           +--------------+---------------------------+------------------------------------------------+----------+
+            |           | >= 5.8       | exec / non-exec*          |                                                | enabled  |
+            |           | [#arm_5.8]_  +---------------------------+------------------------------------------------+----------+
+            |           |              | missing                   |                                                | disabled |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | mips      | < 5.18       | non-exec*                 |                                                | enabled  |
+            |           | [#mips_5.17]_+---------------------------+------------------------------------------------+----------+
+            |           |              | exec / missing            |                                                | disabled |
+            |           +--------------+---------------------------+------------------------------------------------+----------+
+            |           | >= 5.18      | exec / non-exec*          |                                                | enabled  |
+            |           | [#mips_5.18]_+---------------------------+------------------------------------------------+----------+
+            |           |              | missing                   |                                                | disabled |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | powerpc   | [#powerpc]_  | non-exec / exec           |                                                | enabled  |
+            |           |              +---------------------------+------------------------------------------------+----------+
+            |           |              | missing                   |                                                | disabled |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | powerpc64 | [#powerpc]_  | exec / non-exec / missing |                                                | enabled  |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | ia64      | [#ia64]_     | non-exec                  |                                                | enabled  |
+            |           |              +---------------------------+------------------------------------------------+----------+
+            |           |              | exec / missing            | e_flags & EF_IA_64_LINUX_EXECUTABLE_STACK == 0 | enabled  |
+            |           |              +                           +------------------------------------------------+----------+
+            |           |              |                           | e_flags & EF_IA_64_LINUX_EXECUTABLE_STACK != 0 | disabled |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | the rest  | [#the_rest]_ | exec / non-exec / missing |                                                | enabled  |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+
+            \\* Hardware limitations are ignored.
+
+        If ``READ_IMPLIES_EXEC`` is set, then `all readable pages are executable`__.
+            .. __: https://github.com/torvalds/linux/blob/v6.3/fs/binfmt_elf.c#L1008-L1009
+            .. code-block:: c
+
+                if (elf_read_implies_exec(loc->elf_ex, executable_stack))
+                    current->personality |= READ_IMPLIES_EXEC;
+
+        .. [#x86_5.7]
+            `source <https://github.com/torvalds/linux/blob/v5.7/arch/x86/include/asm/elf.h#L285-L286>`__
+
+            .. code-block:: c
+
+                #define elf_read_implies_exec(ex, executable_stack)	\\
+                    (executable_stack != EXSTACK_DISABLE_X)
+
+        .. [#x86_5.8]
+            `source <https://github.com/torvalds/linux/blob/v5.8/arch/x86/include/asm/elf.h#L305-L306>`__
+
+            .. code-block:: c
+
+                #define elf_read_implies_exec(ex, executable_stack)	\\
+                    (mmap_is_ia32() && executable_stack == EXSTACK_DEFAULT)
+
+            `mmap_is_ia32()`__:
+                .. __: https://github.com/torvalds/linux/blob/v5.8/arch/x86/include/asm/elf.h#L318-L321
+                .. code-block:: c
+
+                    /*
+                     * True on X86_32 or when emulating IA32 on X86_64
+                     */
+                    static inline int mmap_is_ia32(void)
+
+        .. [#arm_5.7]
+            `source <https://github.com/torvalds/linux/blob/v5.7/arch/arm/kernel/elf.c#L85-L92>`__
+
+            .. code-block:: c
+
+                int arm_elf_read_implies_exec(int executable_stack)
+                {
+                    if (executable_stack != EXSTACK_DISABLE_X)
+                        return 1;
+                    if (cpu_architecture() < CPU_ARCH_ARMv6)
+                        return 1;
+                    return 0;
+                }
+
+        .. [#arm_5.8]
+            `source <https://github.com/torvalds/linux/blob/v5.8/arch/arm/kernel/elf.c#L104-L111>`__
+
+            .. code-block:: c
+
+                int arm_elf_read_implies_exec(int executable_stack)
+                {
+                    if (executable_stack == EXSTACK_DEFAULT)
+                        return 1;
+                    if (cpu_architecture() < CPU_ARCH_ARMv6)
+                        return 1;
+                    return 0;
+                }
+
+        .. [#mips_5.17]
+            `source <https://github.com/torvalds/linux/blob/v5.17/arch/mips/kernel/elf.c#L329-L342>`__
+
+            .. code-block:: c
+
+                int mips_elf_read_implies_exec(void *elf_ex, int exstack)
+                {
+                    if (exstack != EXSTACK_DISABLE_X) {
+                        /* The binary doesn't request a non-executable stack */
+                        return 1;
+                    }
+                    if (!cpu_has_rixi) {
+                        /* The CPU doesn't support non-executable memory */
+                        return 1;
+                    }
+                    return 0;
+                }
+
+        .. [#mips_5.18]
+            `source <https://github.com/torvalds/linux/blob/v5.18/arch/mips/kernel/elf.c#L329-L336>`__
+
+            .. code-block:: c
+
+                int mips_elf_read_implies_exec(void *elf_ex, int exstack)
+                {
+                    /*
+                     * Set READ_IMPLIES_EXEC only on non-NX systems that
+                     * do not request a specific state via PT_GNU_STACK.
+                     */
+                    return (!cpu_has_rixi && exstack == EXSTACK_DEFAULT);
+                }
+
+        .. [#powerpc]
+            `source <https://github.com/torvalds/linux/blob/v6.3/arch/powerpc/include/asm/elf.h#L82-L108>`__
+
+            .. code-block:: c
+
+                #ifdef __powerpc64__
+                /* stripped */
+                # define elf_read_implies_exec(ex, exec_stk) (is_32bit_task() ? \\
+                        (exec_stk == EXSTACK_DEFAULT) : 0)
+                #else 
+                # define elf_read_implies_exec(ex, exec_stk) (exec_stk == EXSTACK_DEFAULT)
+                #endif /* __powerpc64__ */
+
+        .. [#ia64]
+            `source <https://github.com/torvalds/linux/blob/v6.3/arch/ia64/include/asm/elf.h#L203-L204>`__
+
+            .. code-block:: c
+
+                #define elf_read_implies_exec(ex, executable_stack)					\\
+                    ((executable_stack!=EXSTACK_DISABLE_X) && ((ex).e_flags & EF_IA_64_LINUX_EXECUTABLE_STACK) != 0)
+
+            EF_IA_64_LINUX_EXECUTABLE_STACK__:
+                .. __: https://github.com/torvalds/linux/blob/v6.3/arch/ia64/include/asm/elf.h#L33
+
+                .. code-block:: c
+
+                    #define EF_IA_64_LINUX_EXECUTABLE_STACK	0x1	/* is stack (& heap) executable by default? */
+
+        .. [#the_rest]
+            `source <https://github.com/torvalds/linux/blob/v6.3/include/linux/elf.h#L13>`__
+
+            .. code-block:: c
+
+                # define elf_read_implies_exec(ex, have_pt_gnu_stack)	0
+        """
+        if not self.executable:
+            return True
+        
+        exec_bit = None
+        for seg in self.iter_segments_by_type('GNU_STACK'):
+            exec_bit = bool(seg.header.p_flags & P_FLAGS.PF_X)
+
+        non_exec = exec_bit is False
+        missing = exec_bit is None
+        EF_IA_64_LINUX_EXECUTABLE_STACK = 1
+
+        if self.arch in ['i386', 'arm', 'aarch64', 'mips', 'mips64']:
+            if non_exec:
+                return True
+            elif missing:
+                return False
+            return None
+        elif self.arch == 'amd64':
+            return True if non_exec else None
+        elif self.arch == 'powerpc':
+            return not missing
+        elif self.arch == 'powerpc64':
+            return True
+        elif self.arch == 'ia64':
+            if non_exec:
+                return True
+            return not bool(self['e_flags'] & EF_IA_64_LINUX_EXECUTABLE_STACK)
+
+        return True
+
+    @property
+    def execstack(self):
+        """:class:`bool`: Whether dynamically loading the current binary will make the stack executable.
+
+        This is based on the presence of a program header ``PT_GNU_STACK``,
+        its setting, and the default stack permissions for the architecture.
+
+        If ``PT_GNU_STACK`` is persent, the stack permissions are `set according to it`__:
+
+        .. __: https://github.com/bminor/glibc/blob/glibc-2.37/elf/dl-load.c#L1218-L1220
 
         .. code-block:: c
 
             case PT_GNU_STACK:
-                if (elf_ppnt->p_flags & PF_X)
-                    executable_stack = EXSTACK_ENABLE_X;
-                else
-                    executable_stack = EXSTACK_DISABLE_X;
-                break;
+              stack_flags = ph->p_flags;
+              break;
 
-        Additionally, it then sets ``read_implies_exec``, so that `all readable pages
-        are executable`__.
+        Else, the stack permissions are set according to the architecture defaults
+        as `defined by`__ ``DEFAULT_STACK_PERMS``:
 
-        .. __: https://github.com/torvalds/linux/blob/v4.9/fs/binfmt_elf.c#L849-L850
-
-        .. code-block:: c
-
-            if (elf_read_implies_exec(loc->elf_ex, executable_stack))
-                current->personality |= READ_IMPLIES_EXEC;
-        """
-        if not self.executable:
-            return True
-
-        for seg in self.iter_segments_by_type('GNU_STACK'):
-            return not bool(seg.header.p_flags & P_FLAGS.PF_X)
-
-        # If you NULL out the PT_GNU_STACK section via ELF.disable_nx(),
-        # everything is executable.
-        return False
-
-    @property
-    def execstack(self):
-        """:class:`bool`: Whether the current binary uses an executable stack.
-
-        This is based on the presence of a program header PT_GNU_STACK_
-        being present, and its setting.
-
-            ``PT_GNU_STACK``
-
-            The p_flags member specifies the permissions on the segment
-            containing the stack and is used to indicate whether the stack
-            should be executable. The absense of this header indicates
-            that the stack will be executable.
-
-        In particular, if the header is missing the stack is executable.
-        If the header is present, it may **explicitly** mark that the stack is
-        executable.
-
-        This is only somewhat accurate.  When using the GNU Linker, it usees
-        DEFAULT_STACK_PERMS_ to decide whether a lack of ``PT_GNU_STACK``
-        should mark the stack as executable:
+        .. __: https://github.com/bminor/glibc/blob/glibc-2.37/elf/dl-load.c#L1093-L1096
 
         .. code-block:: c
 
@@ -1718,29 +1971,25 @@ class ELF(ELFFile):
         ::
 
             $ git grep '#define DEFAULT_STACK_PERMS' | grep -v PF_X
-            sysdeps/aarch64/stackinfo.h:31:#define DEFAULT_STACK_PERMS (PF_R|PF_W)
-            sysdeps/nios2/stackinfo.h:31:#define DEFAULT_STACK_PERMS (PF_R|PF_W)
-            sysdeps/tile/stackinfo.h:31:#define DEFAULT_STACK_PERMS (PF_R|PF_W)
-
-        .. _PT_GNU_STACK: https://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/progheader.html
-        .. _DEFAULT_STACK_PERMS: https://github.com/bminor/glibc/blob/glibc-2.25/elf/dl-load.c#L1036-L1038
+            sysdeps/aarch64/stackinfo.h:    #define DEFAULT_STACK_PERMS (PF_R|PF_W)
+            sysdeps/arc/stackinfo.h:        #define DEFAULT_STACK_PERMS (PF_R|PF_W)
+            sysdeps/csky/stackinfo.h:       #define DEFAULT_STACK_PERMS (PF_R|PF_W)
+            sysdeps/ia64/stackinfo.h:       #define DEFAULT_STACK_PERMS (PF_R|PF_W)
+            sysdeps/loongarch/stackinfo.h:  #define DEFAULT_STACK_PERMS (PF_R | PF_W)
+            sysdeps/nios2/stackinfo.h:      #define DEFAULT_STACK_PERMS (PF_R|PF_W)
+            sysdeps/riscv/stackinfo.h:      #define DEFAULT_STACK_PERMS (PF_R | PF_W)
         """
-        # Dynamic objects do not have the ability to change the executable state of the stack.
         if not self.executable:
             return False
 
-        # If NX is completely off for the process, the stack is executable.
-        if not self.nx:
-            return True
-
+        # If the ``PT_GNU_STACK`` program header is preset, use it's premissions.
+        for seg in self.iter_segments_by_type('GNU_STACK'):
+            return bool(seg.header.p_flags & P_FLAGS.PF_X)
+        
         # If the ``PT_GNU_STACK`` program header is missing, then use the
-        # default rules.  Only AArch64 gets a non-executable stack by default.
-        for _ in self.iter_segments_by_type('GNU_STACK'):
-            break
-        else:
-            return self.arch != 'aarch64'
-
-        return False
+        # default rules. Out of the supported architectures, only AArch64,
+        # IA-64, and RISC-V get a non-executable stack by default.
+        return self.arch not in ['aarch64', 'ia64', 'riscv32', 'riscv64']
 
     @property
     def canary(self):
@@ -1753,7 +2002,17 @@ class ELF(ELFFile):
     @property
     def packed(self):
         """:class:`bool`: Whether the current binary is packed with UPX."""
-        return b'UPX!' in self.get_data()
+        return b'UPX!' in self.get_data()[:0xFF]
+
+    @property
+    def stripped(self):
+        """:class:`bool`: Whether the current binary has been stripped of symbols"""
+        return not any(section['sh_type'] == 'SHT_SYMTAB' for section in self.iter_sections())
+
+    @property
+    def debuginfo(self):
+        """:class:`bool`: Whether the current binary has debug information"""
+        return self.get_section_by_name('.debug_info') is not None
 
     @property
     def pie(self):
@@ -1798,61 +2057,74 @@ class ELF(ELFFile):
 
         # Kernel version?
         if self.version and self.version != (0,):
-            res.append('Version:'.ljust(10) + '.'.join(map(str, self.version)))
+            res.append('Version:'.ljust(12) + '.'.join(map(str, self.version)))
         if self.build:
-            res.append('Build:'.ljust(10) + self.build)
+            res.append('Build:'.ljust(12) + self.build)
 
         res.extend([
-            "RELRO:".ljust(10) + {
+            "RELRO:".ljust(12) + {
                 'Full':    green("Full RELRO"),
                 'Partial': yellow("Partial RELRO"),
                 None:      red("No RELRO")
             }[self.relro],
-            "Stack:".ljust(10) + {
+            "Stack:".ljust(12) + {
                 True:  green("Canary found"),
                 False: red("No canary found")
             }[self.canary],
-            "NX:".ljust(10) + {
+            "NX:".ljust(12) + {
                 True:  green("NX enabled"),
                 False: red("NX disabled"),
+                None: yellow("NX unknown - GNU_STACK missing"),
             }[self.nx],
-            "PIE:".ljust(10) + {
+            "PIE:".ljust(12) + {
                 True: green("PIE enabled"),
                 False: red("No PIE (%#x)" % self.address)
             }[self.pie],
         ])
 
         # Execstack may be a thing, even with NX enabled, because of glibc
-        if self.execstack and self.nx:
-            res.append("Stack:".ljust(10) + red("Executable"))
+        if self.execstack and self.nx is not False:
+            res.append("Stack:".ljust(12) + red("Executable"))
 
         # Are there any RWX areas in the binary?
         #
         # This will occur if NX is disabled and *any* area is
         # RW, or can expressly occur.
         if self.rwx_segments or (not self.nx and self.writable_segments):
-            res += [ "RWX:".ljust(10) + red("Has RWX segments") ]
+            res += [ "RWX:".ljust(12) + red("Has RWX segments") ]
 
         if self.rpath:
-            res += [ "RPATH:".ljust(10) + red(repr(self.rpath)) ]
+            res += [ "RPATH:".ljust(12) + red(repr(self.rpath)) ]
 
         if self.runpath:
-            res += [ "RUNPATH:".ljust(10) + red(repr(self.runpath)) ]
+            res += [ "RUNPATH:".ljust(12) + red(repr(self.runpath)) ]
 
         if self.packed:
-            res.append('Packer:'.ljust(10) + red("Packed with UPX"))
+            res.append('Packer:'.ljust(12) + red("Packed with UPX"))
 
         if self.fortify:
-            res.append("FORTIFY:".ljust(10) + green("Enabled"))
+            res.append("FORTIFY:".ljust(12) + green("Enabled"))
 
         if self.asan:
-            res.append("ASAN:".ljust(10) + green("Enabled"))
+            res.append("ASAN:".ljust(12) + green("Enabled"))
 
         if self.msan:
-            res.append("MSAN:".ljust(10) + green("Enabled"))
+            res.append("MSAN:".ljust(12) + green("Enabled"))
 
         if self.ubsan:
-            res.append("UBSAN:".ljust(10) + green("Enabled"))
+            res.append("UBSAN:".ljust(12) + green("Enabled"))
+        
+        if self.shadowstack:
+            res.append("SHSTK:".ljust(12) + green("Enabled"))
+        
+        if self.ibt:
+            res.append("IBT:".ljust(12) + green("Enabled"))
+        
+        if not self.stripped:
+            res.append("Stripped:".ljust(12) + red("No"))
+        
+        if self.debuginfo:
+            res.append("Debuginfo:".ljust(12) + red("Yes"))
 
         # Check for Linux configuration, it must contain more than
         # just the version.
@@ -1879,7 +2151,7 @@ class ELF(ELFFile):
 
     @property
     def buildid(self):
-        """:class:`str`: GNU Build ID embedded into the binary"""
+        """:class:`bytes`: GNU Build ID embedded into the binary"""
         section = self.get_section_by_name('.note.gnu.build-id')
         if section:
             return section.data()[16:]
@@ -1910,6 +2182,31 @@ class ELF(ELFFile):
         """:class:`bool`: Whether the current binary was built with
         Undefined Behavior Sanitizer (``UBSAN``)."""
         return any(s.startswith('__ubsan_') for s in self.symbols)
+    
+    @property
+    def shadowstack(self):
+        """:class:`bool`: Whether the current binary was built with	
+        Shadow Stack (``SHSTK``)"""
+        if self.arch not in ['i386', 'amd64']:
+            return False
+        for prop in self.iter_properties():
+            if prop.pr_type != 'GNU_PROPERTY_X86_FEATURE_1_AND':
+                continue
+            return prop.pr_data & ENUM_GNU_PROPERTY_X86_FEATURE_1_FLAGS['GNU_PROPERTY_X86_FEATURE_1_SHSTK'] > 0
+        return False
+
+    @property
+    def ibt(self):
+        """:class:`bool`: Whether the current binary was built with
+        Indirect Branch Tracking (``IBT``)"""
+        if self.arch not in ['i386', 'amd64']:
+            return False
+        for prop in self.iter_properties():
+            if prop.pr_type != 'GNU_PROPERTY_X86_FEATURE_1_AND':
+                continue
+            return prop.pr_data & ENUM_GNU_PROPERTY_X86_FEATURE_1_FLAGS['GNU_PROPERTY_X86_FEATURE_1_IBT'] > 0
+        return False
+
 
     def _update_args(self, kw):
         kw.setdefault('arch', self.arch)
@@ -2035,6 +2332,128 @@ class ELF(ELFFile):
             if self.mmap[offset:offset+4] == PT_GNU_STACK:
                 self.mmap[offset:offset+4] = b'\x00' * 4
                 self.save()
+                # Invalidate the cached segments, ``PT_GNU_STACK`` was removed.
+                self._segments = None
                 return
 
         log.error("Could not find PT_GNU_STACK, stack should already be executable")
+    
+    @staticmethod
+    def set_runpath(exepath, runpath):
+        r"""set_runpath(str, str) -> ELF
+
+        Patches the RUNPATH of the ELF to the given path using the `patchelf utility <https://github.com/NixOS/patchelf>`_.
+
+        The dynamic loader will look for any needed shared libraries in the given path first,
+        before trying the system library paths. This is useful to run a binary with a different
+        libc binary.
+
+        Arguments:
+            exepath(str): Path to the binary to patch.
+            runpath(str): Path containing the needed libraries.
+
+        Returns:
+            A new ELF instance is returned after patching the binary with the external ``patchelf`` tool.
+
+        Example:
+
+            >>> tmpdir = tempfile.mkdtemp()
+            >>> ls_path = os.path.join(tmpdir, 'ls')
+            >>> _ = shutil.copy(which('ls'), ls_path)
+            >>> e = ELF.set_runpath(ls_path, './libs')
+            >>> e.runpath == b'./libs'
+            True
+        """
+        if not which('patchelf'):
+            log.error('"patchelf" tool not installed. See https://github.com/NixOS/patchelf')
+            return None
+        try:
+            subprocess.check_output(['patchelf', '--set-rpath', runpath, exepath], stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as e:
+            log.failure('Patching RUNPATH failed (%d): %r', e.returncode, e.stdout)
+        return ELF(exepath, checksec=False)
+
+    @staticmethod
+    def set_interpreter(exepath, interpreter_path):
+        r"""set_interpreter(str, str) -> ELF
+
+        Patches the interpreter of the ELF to the given binary using the `patchelf utility <https://github.com/NixOS/patchelf>`_.
+
+        When running the binary, the new interpreter will be used to load the ELF.
+
+        Arguments:
+            exepath(str): Path to the binary to patch.
+            interpreter_path(str): Path to the ld.so dynamic loader.
+
+        Returns:
+            A new ELF instance is returned after patching the binary with the external ``patchelf`` tool.
+
+        Example:
+
+            >>> tmpdir = tempfile.mkdtemp()
+            >>> ls_path = os.path.join(tmpdir, 'ls')
+            >>> _ = shutil.copy(which('ls'), ls_path)
+            >>> e = ELF.set_interpreter(ls_path, '/tmp/correct_ld.so')
+            >>> e.linker == b'/tmp/correct_ld.so'
+            True
+        """
+        # patch the interpreter
+        if not which('patchelf'):
+            log.error('"patchelf" tool not installed. See https://github.com/NixOS/patchelf')
+            return None
+        try:
+            subprocess.check_output(['patchelf', '--set-interpreter', interpreter_path, exepath], stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as e:
+            log.failure('Patching interpreter failed (%d): %r', e.returncode, e.stdout)
+        return ELF(exepath, checksec=False)
+
+    @staticmethod
+    def patch_custom_libraries(exe_path, custom_library_path, create_copy=True, suffix='_remotelibc'):
+        r"""patch_custom_libraries(str, str, bool, str) -> ELF
+
+        Looks for the interpreter binary in the given path and patches the binary to use
+        it if available. Also patches the RUNPATH to the given path using the `patchelf utility <https://github.com/NixOS/patchelf>`_.
+
+        Arguments:
+            exe_path(str): Path to the binary to patch.
+            custom_library_path(str): Path to a folder containing the libraries.
+            create_copy(bool): Create a copy of the binary and apply the patches to the copy.
+            suffix(str): Suffix to append to the filename when creating the copy to patch.
+
+        Returns:
+            A new ELF instance is returned after patching the binary with the external ``patchelf`` tool.
+
+        Example:
+
+            >>> tmpdir = tempfile.mkdtemp()
+            >>> linker_path = os.path.join(tmpdir, 'ld-mock.so')
+            >>> write(linker_path, b'loader')
+            >>> ls_path = os.path.join(tmpdir, 'ls')
+            >>> _ = shutil.copy(which('ls'), ls_path)
+            >>> e = ELF.patch_custom_libraries(ls_path, tmpdir)
+            >>> e.runpath.decode() == tmpdir
+            True
+            >>> e.linker.decode() == linker_path
+            True
+        """
+        if not which('patchelf'):
+            log.error('"patchelf" tool not installed. See https://github.com/NixOS/patchelf')
+            return None
+        
+        # Create a copy of the ELF to patch instead of the original file.
+        if create_copy:
+            import shutil
+            patched_path = exe_path + suffix
+            shutil.copy2(exe_path, patched_path)
+            exe_path = patched_path
+
+        # Set interpreter in ELF to the one in the library path.
+        interpreter_name = [filename for filename in os.listdir(custom_library_path) if filename.startswith('ld-')]
+        if interpreter_name:
+            interpreter_path = os.path.realpath(os.path.join(custom_library_path, interpreter_name[0]))
+            ELF.set_interpreter(exe_path, interpreter_path)
+        else:
+            log.warn("Couldn't find ld.so in library path. Interpreter not set.")
+
+        # Set RUNPATH to library path in order to find other libraries.
+        return ELF.set_runpath(exe_path, custom_library_path)

pwnlib/elf/maps.py

@@ -22,6 +22,6 @@ CAT_PROC_MAPS_EXIT = {
         '726f093c2f702935f0ffa9af7365093c632f2935f4ffa9af2f6d093c6c662935f8ffa9af8cff193c9e8f393727482003fcffa9aff0ffbd272020a003ffff0528ffbf192427302003a50f02340c01010122e8a603fcffa2affcffa48f2028a003a30f02340c010101feff1924272020032028a003fcffa2affcffa68fa40f02340c010101'
         'ffff0428a10f02340c010101',
     'aarch64':
-        'ee058ed24eeeadf26eecc5f26eaeecf28fcd8cd2efa5adf22f0ccef26f0ee0f2ee3fbfa980f39fd2e0ffbff2e0ffdff2e0fffff2e1030091e2031faa080780d2010000d4020088d2ff6322cbe1030091e80780d2010000d4e20300aa200080d2e1030091080880d2010000d4'
+        'ee058ed24eeeadf26eecc5f26eaeecf28fcd8cd2efa5adf22f0ccef26f0ee0f2ee3fbfa980f39fd2e0ffbff2e0ffdff2e0fffff2e1633f8be2031faa080780d2010000d4222080d22e2088d2c20102caff6322cbe1633f8be80780d2010000d402041f8b4f2080d26c2080d28f010fcae0030faae1633f8b080880d2010000d4'
         'e0031faaa80b80d2010000d4',
 }

pwnlib/elf/plt.py

@@ -53,9 +53,30 @@ def emulate_plt_instructions(elf, got, address, data, targets):
 
     return rv
 
+
+def __ensure_memory_to_run_unicorn():
+    """
+    Check if there is enough memory to run Unicorn Engine.
+    Unicorn Engine requires 1GB of memory to run, if there isn't enough memory it calls exit(1).
+
+    This is a bug in Unicorn Engine, see: https://github.com/unicorn-engine/unicorn/issues/1766
+    """
+    try:
+        from mmap import mmap, MAP_ANON, MAP_PRIVATE, PROT_EXEC, PROT_READ, PROT_WRITE
+
+        mm = mmap(
+            -1, 1024 * 1024 * 1024, MAP_PRIVATE | MAP_ANON, PROT_WRITE | PROT_READ | PROT_EXEC
+        )
+        mm.close()
+    except OSError:
+        raise OSError("Cannot allocate 1GB memory to run Unicorn Engine")
+
+
 def prepare_unicorn_and_context(elf, got, address, data):
     import unicorn as U
 
+    __ensure_memory_to_run_unicorn()
+
     # Instantiate the emulator with the correct arguments for the current
     # architecutre.
     arch = {
@@ -64,8 +85,11 @@ def prepare_unicorn_and_context(elf, got, address, data):
         'arm': U.UC_ARCH_ARM,
         'i386': U.UC_ARCH_X86,
         'mips': U.UC_ARCH_MIPS,
+        'mips64': U.UC_ARCH_MIPS,
         # 'powerpc': U.UC_ARCH_PPC, <-- Not actually supported
         'thumb': U.UC_ARCH_ARM,
+        'riscv32': U.UC_ARCH_RISCV,
+        'riscv64': U.UC_ARCH_RISCV,
     }.get(elf.arch, None)
 
     if arch is None:
@@ -119,18 +143,6 @@ def prepare_unicorn_and_context(elf, got, address, data):
 
         uc.mem_write(got, p_magic)
 
-        # Separately, Unicorn is apparently unable to hook unmapped memory
-        # accesses on MIPS.  So we also have to map the page that contains
-        # the magic address.
-        start = magic_addr & (~0xfff)
-        try:
-            uc.mem_map(start, 0x1000)
-        except Exception:
-            # Ignore double-mapping
-            pass
-        trap = packing.p32(0x34000000, endian=elf.endian)
-        uc.mem_write(magic_addr, trap)
-
     return uc, uc.context_save()
 
 

pwnlib/encoders/arm/alphanumeric/ARM_Instructions.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 Pratik Kumar Sahu, Nagendra Chowdary, Anish Mathuria
+# Ported to Python by Gallopsled
 from __future__ import division
 
 # +------------------------------------------------------------------------+ 

pwnlib/encoders/arm/alphanumeric/__init__.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 Pratik Kumar Sahu, Nagendra Chowdary, Anish Mathuria
+# Ported to Python by Gallopsled
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/encoders/arm/alphanumeric/alphanum_byte.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 Pratik Kumar Sahu, Nagendra Chowdary, Anish Mathuria
+# Ported to Python by Gallopsled
 from __future__ import division
 from __future__ import absolute_import
 

pwnlib/encoders/arm/alphanumeric/builder.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 Pratik Kumar Sahu, Nagendra Chowdary, Anish Mathuria
+# Ported to Python by Gallopsled
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/encoders/arm/alphanumeric/random_funcs.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 Pratik Kumar Sahu, Nagendra Chowdary, Anish Mathuria
+# Ported to Python by Gallopsled
 from __future__ import division
 
 import os

pwnlib/encoders/i386/ascii_shellcode.py

@@ -21,10 +21,10 @@ class AsciiShellcodeEncoder(Encoder):
     executes (on the stack)
 
     The original paper this encoder is based on:
-    http://julianor.tripod.com/bc/bypass-msb.txt
+    https://julianor.tripod.com/bc/bypass-msb.txt
 
     A more visual explanation as well as an implementation in C:
-    https://github.com/VincentDary/PolyAsciiShellGen/blob/master/README.md#mechanism
+    https://vincentdary.github.io/blog-posts/polyasciishellgen-caezar-ascii-shellcode-generator/index.html#22-mechanism
     """
 
     def __init__(self, slop=20, max_subs=4):

pwnlib/filepointer.py

@@ -125,7 +125,7 @@ class FileStructure(object):
         >>> len(fileStr)
         224
 
-        The defination for __repr__ orders the structure members and displays then in a dictionary format. It's useful when viewing a structure objet in python/IPython shell
+        The definition for __repr__ orders the structure members and displays then in a dictionary format. It's useful when viewing a structure objet in python/IPython shell
 
         >>> q=FileStructure(0xdeadbeef)
         >>> q
@@ -176,24 +176,20 @@ class FileStructure(object):
     def __repr__(self):
         structure=[]
         for i in self.vars_:
-            structure.append(" %s: %s" % (i,hex(self.__getattr__(i))))
+            structure.append(" %s: %#x" % (i, getattr(self, i)))
         return "{"+ "\n".join(structure)+"}"
 
-    def __getattr__(self,item):
-        if item in FileStructure.__dict__ or item in self.vars_:
-            return object.__getattribute__(self,item)
-        log.error("Unknown variable %r" % item)
-
     def __len__(self):
         return len(bytes(self))
 
     def __bytes__(self):
         structure = b''
         for val in self.vars_:
-            if isinstance(self.__getattr__(val), bytes):
-                structure += self.__getattr__(val).ljust(context.bytes, b'\x00')
+            if isinstance(getattr(self, val), bytes):
+                structure += getattr(self, val).ljust(context.bytes, b'\x00')
             else:
-                structure += pack(self.__getattr__(val), self.length[val]*8)
+                if self.length[val] > 0:
+                    structure += pack(getattr(self, val), self.length[val]*8)
         return structure
 
     def struntil(self,v):
@@ -218,10 +214,10 @@ class FileStructure(object):
             return b''
         structure = b''
         for val in self.vars_:
-            if isinstance(self.__getattr__(val), bytes):
-                structure += self.__getattr__(val).ljust(context.bytes, b'\x00')
+            if isinstance(getattr(self, val), bytes):
+                structure += getattr(self, val).ljust(context.bytes, b'\x00')
             else:
-                structure += pack(self.__getattr__(val), self.length[val]*8)
+                structure += pack(getattr(self, val), self.length[val]*8)
             if val == v:
                 break
         return structure[:-1]
@@ -313,7 +309,7 @@ class FileStructure(object):
 
     def orange(self,io_list_all,vtable):
         r"""
-        Perform a House of Orange (https://github.com/shellphish/how2heap/blob/master/glibc_2.25/house_of_orange.c), provided you have libc leaks.
+        Perform a House of Orange (https://github.com/shellphish/how2heap/blob/master/glibc_2.23/house_of_orange.c), provided you have libc leaks.
 
         Arguments:
             io_list_all(int)

pwnlib/fmtstr.py

@@ -129,6 +129,7 @@ def normalize_writes(writes):
     such that all values are raw bytes and consecutive writes are merged to a single key.
 
     Examples:
+
         >>> context.clear(endian="little", bits=32)
         >>> normalize_writes({0x0: [p32(0xdeadbeef)], 0x4: p32(0xf00dface), 0x10: 0x41414141})
         [(0, b'\xef\xbe\xad\xde\xce\xfa\r\xf0'), (16, b'AAAA')]
@@ -215,6 +216,7 @@ class AtomWrite(object):
         given the current format string write counter (how many bytes have been written until now).
 
         Examples:
+
             >>> hex(pwnlib.fmtstr.AtomWrite(0x0, 0x2, 0x2345).compute_padding(0x1111))
             '0x1234'
             >>> hex(pwnlib.fmtstr.AtomWrite(0x0, 0x2, 0xaa00).compute_padding(0xaabb))
@@ -246,6 +248,7 @@ class AtomWrite(object):
         Combine adjacent writes into a single write.
 
         Example:
+
             >>> context.clear(endian = "little")
             >>> pwnlib.fmtstr.AtomWrite(0x0, 0x1, 0x1, 0xff).union(pwnlib.fmtstr.AtomWrite(0x1, 0x1, 0x2, 0x77))
             AtomWrite(start=0, size=2, integer=0x201, mask=0x77ff)
@@ -286,8 +289,16 @@ def make_atoms_simple(address, data, badbytes=frozenset()):
     This function is simple and does not try to minimize the number of atoms. For example, if there are no
     bad bytes, it simply returns one atom for each byte:
 
-    >>> pwnlib.fmtstr.make_atoms_simple(0x0, b"abc", set())
-    [AtomWrite(start=0, size=1, integer=0x61, mask=0xff), AtomWrite(start=1, size=1, integer=0x62, mask=0xff), AtomWrite(start=2, size=1, integer=0x63, mask=0xff)]
+        >>> pwnlib.fmtstr.make_atoms_simple(0x0, b"abc", set())
+        [AtomWrite(start=0, size=1, integer=0x61, mask=0xff), AtomWrite(start=1, size=1, integer=0x62, mask=0xff), AtomWrite(start=2, size=1, integer=0x63, mask=0xff)]
+    
+    If there are bad bytes, it will try to bypass by skipping addresses containing bad bytes, otherwise a
+    RuntimeError will be raised:
+
+        >>> pwnlib.fmtstr.make_atoms_simple(0x61, b'abc', b'\x62')
+        [AtomWrite(start=97, size=2, integer=0x6261, mask=0xffff), AtomWrite(start=99, size=1, integer=0x63, mask=0xff)]
+        >>> pwnlib.fmtstr.make_atoms_simple(0x61, b'a'*0x10, b'\x62\x63\x64\x65\x66\x67\x68')
+        [AtomWrite(start=97, size=8, integer=0x6161616161616161, mask=0xffffffffffffffff), AtomWrite(start=105, size=1, integer=0x61, mask=0xff), AtomWrite(start=106, size=1, integer=0x61, mask=0xff), AtomWrite(start=107, size=1, integer=0x61, mask=0xff), AtomWrite(start=108, size=1, integer=0x61, mask=0xff), AtomWrite(start=109, size=1, integer=0x61, mask=0xff), AtomWrite(start=110, size=1, integer=0x61, mask=0xff), AtomWrite(start=111, size=1, integer=0x61, mask=0xff), AtomWrite(start=112, size=1, integer=0x61, mask=0xff)]
     """
     data = bytearray(data)
     if not badbytes:
@@ -298,14 +309,15 @@ def make_atoms_simple(address, data, badbytes=frozenset()):
 
     i = 0
     out = []
+    end = address + len(data)
     while i < len(data):
         candidate = AtomWrite(address + i, 1, data[i])
-        while candidate.end < len(data) and any(x in badbytes for x in pack(candidate.end)):
+        while candidate.end < end and any(x in badbytes for x in pack(candidate.end)):
             candidate = candidate.union(AtomWrite(candidate.end, 1, data[i + candidate.size]))
 
         sz = min([s for s in SPECIFIER if s >= candidate.size] + [float("inf")])
-        if candidate.start + sz > len(data):
-            raise RuntimeError("impossible to avoid badbytes starting after offset %d (address %x)" % (i, i + address))
+        if candidate.start + sz > end:
+            raise RuntimeError("impossible to avoid badbytes starting after offset %d (address %#x)" % (i, i + address))
         i += candidate.size
         candidate = candidate.union(AtomWrite(candidate.end, sz - candidate.size, 0, 0))
         out.append(candidate)
@@ -318,6 +330,7 @@ def merge_atoms_writesize(atoms, maxsize):
     This function simply merges adjacent atoms as long as the merged atom's size is not larger than ``maxsize``.
 
     Examples:
+
         >>> from pwnlib.fmtstr import *
         >>> merge_atoms_writesize([AtomWrite(0, 1, 1), AtomWrite(1, 1, 1), AtomWrite(2, 1, 2)], 2)
         [AtomWrite(start=0, size=2, integer=0x101, mask=0xffff), AtomWrite(start=2, size=1, integer=0x2, mask=0xff)]
@@ -357,6 +370,7 @@ def find_min_hamming_in_range_step(prev, step, carry, strict):
         A tuple (score, value, mask) where score equals the number of matching bytes between the returned value and target.
 
     Examples:
+
         >>> initial = {(0,0): (0,0,0), (0,1): None, (1,0): None, (1,1): None}
         >>> pwnlib.fmtstr.find_min_hamming_in_range_step(initial, (0, 0xFF, 0x1), 0, 0)
         (1, 1, 255)
@@ -412,6 +426,7 @@ def find_min_hamming_in_range(maxbytes, lower, upper, target):
         target(int): the target value that should be approximated
 
     Examples:
+
         >>> pp = lambda svm: (svm[0], hex(svm[1]), hex(svm[2]))
         >>> pp(pwnlib.fmtstr.find_min_hamming_in_range(1, 0x0, 0x100, 0xaa))
         (1, '0xaa', '0xff')
@@ -463,6 +478,7 @@ def merge_atoms_overlapping(atoms, sz, szmax, numbwritten, overflows):
         overflows(int): how many extra overflows (of size sz) to tolerate to reduce the number of atoms
 
     Examples:
+
         >>> from pwnlib.fmtstr import *
         >>> merge_atoms_overlapping([AtomWrite(0, 1, 1), AtomWrite(1, 1, 1)], 2, 8, 0, 1)
         [AtomWrite(start=0, size=2, integer=0x101, mask=0xffff)]
@@ -492,7 +508,7 @@ def merge_atoms_overlapping(atoms, sz, szmax, numbwritten, overflows):
         # the best write is the one which sets the largest number of target
         # bytes correctly
         candidate = AtomWrite(atom.start, 0, 0)
-        best = (0, None)
+        best = (atom.size, idx, atom)
         for nextidx, nextatom in enumerate(atoms[idx:], idx):
             # if there is no atom immediately following the current candidate
             # that we haven't written yet, stop
@@ -521,6 +537,8 @@ def merge_atoms_overlapping(atoms, sz, szmax, numbwritten, overflows):
 
         _, nextidx, best_candidate = best
         numbwritten_here += best_candidate.compute_padding(numbwritten_here)
+        if numbwritten_here > maxwritten:
+            maxwritten = numbwritten_here
         offset = 0
 
         # for all atoms that we merged, check if all bytes are written already to update `done``
@@ -548,6 +566,7 @@ def overlapping_atoms(atoms):
     Finds pairs of atoms that write to the same address.
 
     Basic examples:
+
         >>> from pwnlib.fmtstr import *
         >>> list(overlapping_atoms([AtomWrite(0, 2, 0), AtomWrite(2, 10, 1)])) # no overlaps
         []
@@ -555,6 +574,7 @@ def overlapping_atoms(atoms):
         [(AtomWrite(start=0, size=2, integer=0x0, mask=0xffff), AtomWrite(start=1, size=2, integer=0x1, mask=0xffff))]
 
     When there are transitive overlaps, only the largest overlap is returned. For example:
+
         >>> list(overlapping_atoms([AtomWrite(0, 3, 0), AtomWrite(1, 4, 1), AtomWrite(2, 4, 1)]))
         [(AtomWrite(start=0, size=3, integer=0x0, mask=0xffffff), AtomWrite(start=1, size=4, integer=0x1, mask=0xffffffff)), (AtomWrite(start=1, size=4, integer=0x1, mask=0xffffffff), AtomWrite(start=2, size=4, integer=0x1, mask=0xffffffff))]
 
@@ -620,6 +640,7 @@ def sort_atoms(atoms, numbwritten):
         numbwritten(int): the value at which the counter starts
 
     Examples:
+
         >>> from pwnlib.fmtstr import *
         >>> sort_atoms([AtomWrite(0, 1, 0xff), AtomWrite(1, 1, 0xfe)], 0) # the example described above
         [AtomWrite(start=1, size=1, integer=0xfe, mask=0xff), AtomWrite(start=0, size=1, integer=0xff, mask=0xff)]
@@ -669,7 +690,7 @@ def sort_atoms(atoms, numbwritten):
 
     return out
 
-def make_payload_dollar(data_offset, atoms, numbwritten=0, countersize=4):
+def make_payload_dollar(data_offset, atoms, numbwritten=0, countersize=4, no_dollars=False):
     r'''
     Makes a format-string payload using glibc's dollar syntax to access the arguments.
 
@@ -682,8 +703,10 @@ def make_payload_dollar(data_offset, atoms, numbwritten=0, countersize=4):
         atoms(list): list of atoms to execute
         numbwritten(int): number of byte already written by the printf function
         countersize(int): size in bytes of the format string counter (usually 4)
+        no_dollars(bool) : flag to generete the payload with or w/o $ notation 
 
     Examples:
+
         >>> pwnlib.fmtstr.make_payload_dollar(1, [pwnlib.fmtstr.AtomWrite(0x0, 0x1, 0xff)])
         (b'%255c%1$hhn', b'\x00\x00\x00\x00')
     '''
@@ -691,6 +714,13 @@ def make_payload_dollar(data_offset, atoms, numbwritten=0, countersize=4):
     fmt = ""
 
     counter = numbwritten
+
+    if no_dollars:
+        # since we can't dynamically offset, we have to increment manually the parameter index, use %c, so the number of bytes written is predictable
+        fmt += "%c" * (data_offset - 1)
+        # every %c write a byte, so we need to keep track of that to have the right pad
+        counter += data_offset - 1
+
     for idx, atom in enumerate(atoms):
         # set format string counter to correct value
         padding = atom.compute_padding(counter)
@@ -701,9 +731,54 @@ def make_payload_dollar(data_offset, atoms, numbwritten=0, countersize=4):
             log.warn("padding is negative, this will not work on glibc")
 
         # perform write
-        if padding:
+        # if the padding is less than 3, it is more convenient to write it : [ len("cc") < len("%2c") ] , this could help save some bytes, if it is 3 it will take the same amout of bytes
+        # we also add ( context.bytes * no_dollars ) because , "%nccccccccc%n...ptr1ptr2" is more convenient than %"n%8c%n...ptr1ccccccccptr2"
+        if padding < 4 + context.bytes * no_dollars:
+                fmt += "c" * padding
+                ## if do not padded with %{n}c  do not need to add something in data to use as argument, since  we are not using a printf argument
+        else: 
             fmt += "%" + str(padding) + "c"
-        fmt += "%" + str(data_offset + idx) + "$" + SPECIFIER[atom.size]
+
+            if no_dollars:
+                data += b'c' * context.bytes
+                ''' 
+                [ @murph12F was here ]
+
+                the data += b'c' * context.bytes , is used to keey the arguments aligned when a %c is performed, so it wont use the actual address to write at
+                examplea stack and payload:
+                    
+                    fmtsr = %44c%hhn%66c%hhn
+
+                    ---------
+                    | addr2 |
+                    ---------
+                    | 0x000 |   
+                    ---------
+                    | addr1 |
+                    ---------
+                    | 0x000 | <-- (rsp)
+                    ---------
+                
+                    in this case the the first %44c will use the current arugument used pointed by rsp ( 0 ), and increment  rsp
+
+                    ---------
+                    | addr2 |
+                    ---------
+                    | 0X000 |   
+                    ---------
+                    | addr1 | <-- (rsp)
+                    ---------
+                    | 0x000 | 
+                    ---------
+
+                    now it will perform the %hhn, and it will correctly use the addr1 argument
+                '''
+            
+        if no_dollars:
+            fmt += "%" +  SPECIFIER[atom.size]
+        else:
+            fmt += "%" + str(data_offset + idx) + "$" + SPECIFIER[atom.size]
+
         data += pack(atom.start)
 
     return fmt.encode(), data
@@ -740,7 +815,7 @@ def make_atoms(writes, sz, szmax, numbwritten, overflows, strategy, badbytes):
         all_atoms += atoms
     return all_atoms
 
-def fmtstr_split(offset, writes, numbwritten=0, write_size='byte', write_size_max='long', overflows=16, strategy="small", badbytes=frozenset()):
+def fmtstr_split(offset, writes, numbwritten=0, write_size='byte', write_size_max='long', overflows=16, strategy="small", badbytes=frozenset(), no_dollars=False):
     """
     Build a format string like fmtstr_payload but return the string and data separately.
     """
@@ -754,9 +829,9 @@ def fmtstr_split(offset, writes, numbwritten=0, write_size='byte', write_size_ma
     szmax = WRITE_SIZE[write_size_max]
     atoms = make_atoms(writes, sz, szmax, numbwritten, overflows, strategy, badbytes)
 
-    return make_payload_dollar(offset, atoms, numbwritten)
+    return make_payload_dollar(offset, atoms, numbwritten, no_dollars=no_dollars)
 
-def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_max='long', overflows=16, strategy="small", badbytes=frozenset(), offset_bytes=0):
+def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_max='long', overflows=16, strategy="small", badbytes=frozenset(), offset_bytes=0, no_dollars=False):
     r"""fmtstr_payload(offset, writes, numbwritten=0, write_size='byte') -> str
 
     Makes payload with given parameter.
@@ -773,10 +848,12 @@ def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_
         write_size(str): must be ``byte``, ``short`` or ``int``. Tells if you want to write byte by byte, short by short or int by int (hhn, hn or n)
         overflows(int): how many extra overflows (at size sz) to tolerate to reduce the length of the format string
         strategy(str): either 'fast' or 'small' ('small' is default, 'fast' can be used if there are many writes)
+        no_dollars(bool) : flag to generete the payload with or w/o $ notation 
     Returns:
         The payload in order to do needed writes
 
     Examples:
+
         >>> context.clear(arch = 'amd64')
         >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')
         b'%322419390c%4$llnaaaabaa\x00\x00\x00\x00\x00\x00\x00\x00'
@@ -784,6 +861,8 @@ def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_
         b'%47806c%5$lln%22649c%6$hnaaaabaa\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00'
         >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')
         b'%190c%7$lln%85c%8$hhn%36c%9$hhn%131c%10$hhnaaaab\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'
+        >>> fmtstr_payload(6, {0x8: 0x55d15d2004a0}, badbytes=b'\n')
+        b'%1184c%14$lln%49c%15$hhn%6963c%16$hn%81c%17$hhn%8c%18$hhnaaaabaa\x08\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\t\x00\x00\x00\x00\x00\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00'
         >>> context.clear(arch = 'i386')
         >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')
         b'%322419390c%5$na\x00\x00\x00\x00'
@@ -792,9 +871,15 @@ def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_
         >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')
         b'%19c%12$hhn%36c%13$hhn%131c%14$hhn%4c%15$hhn\x03\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'
         >>> fmtstr_payload(1, {0x0: 0x00000001}, write_size='byte')
-        b'%1c%3$na\x00\x00\x00\x00'
+        b'c%3$naaa\x00\x00\x00\x00'
         >>> fmtstr_payload(1, {0x0: b"\xff\xff\x04\x11\x00\x00\x00\x00"}, write_size='short')
         b'%327679c%7$lln%18c%8$hhn\x00\x00\x00\x00\x03\x00\x00\x00'
+        >>> fmtstr_payload(10, {0x404048 : 0xbadc0ffe, 0x40403c : 0xdeadbeef}, no_dollars=True)
+        b'%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%125c%hhn%17c%hhn%32c%hhn%17c%hhn%203c%hhn%34c%hhn%3618c%hnacccc>@@\x00cccc=@@\x00cccc?@@\x00cccc<@@\x00ccccK@@\x00ccccJ@@\x00ccccH@@\x00'
+        >>> fmtstr_payload(6, {0x404048 : 0xbadbad00}, no_dollars=True)
+        b'%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%229c%hhn%173c%hhn%13c%hhn%33c%hhnccccH@@\x00ccccI@@\x00ccccK@@\x00ccccJ@@\x00'
+        >>> fmtstr_payload(6, {0x4040 : 0xbadbad00, 0x4060: 0xbadbad02}, no_dollars=True)
+        b'%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%212c%hhn%173c%hhn%13c%hhn%33c%hhn%39c%hhn%171c%hhn%13c%hhn%33c%hhnacccc@@\x00\x00ccccA@\x00\x00ccccC@\x00\x00ccccB@\x00\x00cccc`@\x00\x00cccca@\x00\x00ccccc@\x00\x00ccccb@\x00\x00'
     """
     sz = WRITE_SIZE[write_size]
     szmax = WRITE_SIZE[write_size_max]
@@ -803,7 +888,7 @@ def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_
     fmt = b""
     for _ in range(1000000):
         data_offset = (offset_bytes + len(fmt)) // context.bytes
-        fmt, data = make_payload_dollar(offset + data_offset, all_atoms, numbwritten=numbwritten)
+        fmt, data = make_payload_dollar(offset + data_offset, all_atoms, numbwritten=numbwritten, no_dollars=no_dollars)
         fmt = fmt + cyclic((-len(fmt)-offset_bytes) % context.bytes)
 
         if len(fmt) + offset_bytes == data_offset * context.bytes:
@@ -834,11 +919,12 @@ class FmtStr(object):
 
     """
 
-    def __init__(self, execute_fmt, offset=None, padlen=0, numbwritten=0):
+    def __init__(self, execute_fmt, offset=None, padlen=0, numbwritten=0, badbytes=frozenset()):
         self.execute_fmt = execute_fmt
         self.offset = offset
         self.padlen = padlen
         self.numbwritten = numbwritten
+        self.badbytes = badbytes
 
         if self.offset is None:
             self.offset, self.padlen = self.find_offset()
@@ -902,7 +988,7 @@ class FmtStr(object):
 
         """
         fmtstr = randoms(self.padlen).encode()
-        fmtstr += fmtstr_payload(self.offset, self.writes, numbwritten=self.padlen + self.numbwritten, write_size='byte')
+        fmtstr += fmtstr_payload(self.offset, self.writes, numbwritten=self.padlen + self.numbwritten, badbytes=self.badbytes, write_size='byte')
         self.execute_fmt(fmtstr)
         self.writes = {}
 

pwnlib/gdb.py

@@ -141,13 +141,11 @@ Member Documentation
 from __future__ import absolute_import
 from __future__ import division
 
-from contextlib import contextmanager
 import os
 import platform
 import psutil
 import random
 import re
-import shlex
 import six
 import six.moves
 import socket
@@ -249,7 +247,43 @@ def debug_shellcode(data, gdbscript=None, vma=None, api=False):
 
     return debug(tmp_elf, gdbscript=gdbscript, arch=context.arch, api=api)
 
-def _gdbserver_args(pid=None, path=None, args=None, which=None, env=None):
+def _execve_script(argv, executable, env, ssh):
+    """_execve_script(argv, executable, env, ssh) -> str
+
+    Returns the filename of a python script that calls 
+    execve the specified program with the specified arguments.
+    This script is suitable to call with gdbservers ``--wrapper`` option,
+    so we have more control over the environment of the debugged process.
+
+    Arguments:
+        argv(list): List of arguments to pass to the program
+        executable(bytes): Path to the program to run
+        env(dict): Environment variables to pass to the program
+        ssh(ssh): SSH connection to use if we are debugging a remote process
+
+    Returns:
+        The filename of the created script.
+    """
+    # Make sure args are bytes not bytearray.
+    argv = [bytes(arg) for arg in argv]
+    executable = packing._encode(executable)
+    if ssh:
+        # ssh.process with run=false creates the script for us
+        return ssh.process(argv, executable=executable, env=env, run=False)
+
+    script = misc._create_execve_script(argv=argv, executable=executable, env=env, log=log)
+    script = script.strip()
+    # Create a temporary file to hold the script
+    tmp = tempfile.NamedTemporaryFile(mode="w+t",prefix='pwnlib-execve-', suffix='.py', delete=False)
+    tmp.write(script)
+    # Make script executable
+    os.fchmod(tmp.fileno(), 0o755)
+    log.debug("Created execve wrapper script %s:\n%s", tmp.name, script)
+
+    return tmp.name
+    
+
+def _gdbserver_args(pid=None, path=None, args=None, which=None, env=None, python_wrapper_script=None):
     """_gdbserver_args(pid=None, path=None, args=None, which=None, env=None) -> list
 
     Sets up a listening gdbserver, to either connect to the specified
@@ -260,6 +294,8 @@ def _gdbserver_args(pid=None, path=None, args=None, which=None, env=None):
         path(str): Process to launch
         args(list): List of arguments to provide on the debugger command line
         which(callaable): Function to find the path of a binary.
+        env(dict): Environment variables to pass to the program
+        python_wrapper_script(str): Path to a python script to use with ``--wrapper``
 
     Returns:
         A list of arguments to invoke gdbserver.
@@ -296,14 +332,24 @@ def _gdbserver_args(pid=None, path=None, args=None, which=None, env=None):
     if pid:
         gdbserver_args += ['--once', '--attach']
 
+    env_args = []
     if env is not None:
-        env_args = []
         for key in tuple(env):
+            # Special case for LD_ environment variables, so gdbserver
+            # starts with the native libraries
             if key.startswith(b'LD_'): # LD_PRELOAD / LD_LIBRARY_PATH etc.
                 env_args.append(b'%s=%s' % (key, env.pop(key)))
             else:
                 env_args.append(b'%s=%s' % (key, env[key]))
-        gdbserver_args += ['--wrapper', 'env', '-i'] + env_args + ['--']
+    
+    if python_wrapper_script is not None:
+        gdbserver_args += ['--wrapper', python_wrapper_script, '--']
+    elif env is not None:
+        gdbserver_args += ['--wrapper', which('env'), '-i'] + env_args + ['--']
+    # --no-startup-with-shell is required for forking shells like SHELL=/bin/fish
+    # https://github.com/Gallopsled/pwntools/issues/2377
+    else:
+        gdbserver_args += ['--no-startup-with-shell']
 
     gdbserver_args += ['localhost:0']
     gdbserver_args += args
@@ -315,7 +361,14 @@ def _gdbserver_port(gdbserver, ssh):
 
     # Process /bin/bash created; pid = 14366
     # Listening on port 34816
-    process_created = gdbserver.recvline()
+    process_created = gdbserver.recvline(timeout=3)
+
+    if not process_created:
+        log.error(
+            'No output from gdbserver after 3 seconds. Try setting the SHELL=/bin/sh '
+            'environment variable or using the env={} argument if you are affected by '
+            'https://sourceware.org/bugzilla/show_bug.cgi?id=26116'
+        )
 
     if process_created.startswith(b'ERROR:'):
         raise ValueError(
@@ -374,8 +427,10 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
         exe(str): Path to the executable on disk
         env(dict): Environment to start the binary in
         ssh(:class:`.ssh`): Remote ssh session to use to launch the process.
-        sysroot(str): Foreign-architecture sysroot, used for QEMU-emulated binaries
-            and Android targets.
+        sysroot(str): Set an alternate system root. The system root is used to
+            load absolute shared library symbol files. This is useful to instruct
+            gdb to load a local version of binaries/libraries instead of downloading
+            them from the gdbserver, which is faster
         api(bool): Enable access to GDB Python API.
 
     Returns:
@@ -456,11 +511,33 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
 
         >>> io.interactive() # doctest: +SKIP
         >>> io.close()
+        
+        Start a new process with modified argv[0]
+
+        >>> io = gdb.debug(args=[b'\xde\xad\xbe\xef'], gdbscript='continue', exe="/bin/sh")
+        >>> io.sendline(b"echo $0")
+        >>> io.recvline()
+        b'\xde\xad\xbe\xef\n'
+        >>> io.close()
+
+        Demonstrate that LD_PRELOAD is respected
+
+        >>> io = process(["grep", "libc.so.6", "/proc/self/maps"])
+        >>> real_libc_path = io.recvline().split()[-1]
+        >>> io.close()
+        >>> import shutil
+        >>> local_path = shutil.copy(real_libc_path, "./local-libc.so") # make a copy of libc to demonstrate that it is loaded
+        >>> io = gdb.debug(["grep", "local-libc.so", "/proc/self/maps"], gdbscript="continue", env={"LD_PRELOAD": "./local-libc.so"})
+        >>> io.recvline().split()[-1] # doctest: +ELLIPSIS
+        b'.../local-libc.so'
+        >>> io.close()
+        >>> os.remove("./local-libc.so") # cleanup
+
 
     Using GDB Python API:
 
-    .. doctest
-       :skipif: six.PY2
+    .. doctest::
+       :skipif: is_python2
 
         Debug a new process
 
@@ -483,6 +560,7 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
         >>> io.gdb.continue_nowait()
         >>> io.recvline()
         b'foo\n'
+        >>> io.close()
 
 
     Using SSH:
@@ -505,8 +583,25 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
         >>> io.sendline(b"echo hello")
 
         Interact with the process
+
         >>> io.interactive() # doctest: +SKIP
         >>> io.close()
+
+        Using a modified args[0] on a remote process
+
+        >>> io = gdb.debug(args=[b'\xde\xad\xbe\xef'], gdbscript='continue', exe="/bin/sh", ssh=shell)
+        >>> io.sendline(b"echo $0")
+        >>> io.recvline()
+        b'$ \xde\xad\xbe\xef\n'
+        >>> io.close()
+
+        Using an empty args[0] on a remote process
+
+        >>> io = gdb.debug(args=[], gdbscript='continue', exe="/bin/sh", ssh=shell)
+        >>> io.sendline(b"echo $0")
+        >>> io.recvline()
+        b'$ \n'
+        >>> io.close()
     """
     if isinstance(args, six.integer_types + (tubes.process.process, tubes.ssh.ssh_channel)):
         log.error("Use gdb.attach() to debug a running process")
@@ -527,12 +622,25 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
     if env:
         env = {bytes(k): bytes(v) for k, v in env}
 
+    exe = which(packing._decode(exe or args[0]))
+    if not exe:
+        log.error("Could not find executable %r" % exe)
+
     if context.noptrace:
         log.warn_once("Skipping debugger since context.noptrace==True")
         return runner(args, executable=exe, env=env)
 
     if ssh or context.native or (context.os == 'android'):
-        args = _gdbserver_args(args=args, which=which, env=env)
+        if len(args) > 0 and which(packing._decode(args[0])) == packing._decode(exe):
+            args = _gdbserver_args(args=args, which=which, env=env)
+        
+        else:
+            # GDBServer is limited in it's ability to manipulate argv[0]
+            # but can use the ``--wrapper`` option to execute commands and catches
+            # ``execve`` calls.
+            # Therefore, we use a wrapper script to execute the target binary
+            script = _execve_script(args, executable=exe, env=env, ssh=ssh)
+            args = _gdbserver_args(args=args, which=which, env=env, python_wrapper_script=script)
     else:
         qemu_port = random.randint(1024, 65535)
         qemu_user = qemu.user_path()
@@ -555,10 +663,6 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
     if not which(args[0]):
         log.error("%s is not installed" % args[0])
 
-    if not ssh:
-        exe = exe or which(orig_args[0])
-        if not (exe and os.path.exists(exe)):
-            log.error("%s does not exist" % exe)
 
     # Start gdbserver/qemu
     # (Note: We override ASLR here for the gdbserver process itself.)
@@ -568,7 +672,7 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
     gdbserver.executable = exe
 
     # Find what port we need to connect to
-    if context.native or (context.os == 'android'):
+    if ssh or context.native or (context.os == 'android'):
         port = _gdbserver_port(gdbserver, ssh)
     else:
         port = qemu_port
@@ -586,7 +690,10 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
     garbage = gdbserver.recvline(timeout=1)
 
     # Some versions of gdbserver output an additional message
-    garbage2 = gdbserver.recvline_startswith(b"Remote debugging from host ", timeout=2)
+    try:
+        garbage2 = gdbserver.recvline_startswith(b"Remote debugging from host ", timeout=2)
+    except EOFError:
+        pass
 
     return gdbserver
 
@@ -597,7 +704,9 @@ def get_gdb_arch():
         'powerpc64': 'powerpc:common64',
         'mips64': 'mips:isa64',
         'thumb': 'arm',
-        'sparc64': 'sparc:v9'
+        'sparc64': 'sparc:v9',
+        'riscv32': 'riscv:rv32',
+        'riscv64': 'riscv:rv64',
     }.get(context.arch, context.arch)
 
 def binary():
@@ -641,7 +750,10 @@ class Breakpoint:
         """
         # Creates a real breakpoint and connects it with this mirror
         self.conn = conn
-        self.server_breakpoint = conn.root.set_breakpoint(
+        self.server_breakpoint = self._server_set_breakpoint(*args, **kwargs)
+
+    def _server_set_breakpoint(self, *args, **kwargs):
+        return self.conn.root.set_breakpoint(
             self, hasattr(self, 'stop'), *args, **kwargs)
 
     def __getattr__(self, item):
@@ -659,10 +771,66 @@ class Breakpoint:
             raise AttributeError()
         return getattr(self.server_breakpoint, item)
 
+    def __setattr__(self, name, value):
+        """Set attributes of the real breakpoint."""
+        if name in (
+            'enabled',
+            'silent',
+            'thread',
+            'task',
+            'ignore_count',
+            'hit_count'
+            'condition',
+            'commands',
+        ):
+            return setattr(self.server_breakpoint, name, value)
+        return super().__setattr__(name, value)
+
     def exposed_stop(self):
         # Handle stop() call from the server.
         return self.stop()
 
+class FinishBreakpoint(Breakpoint):
+    """Mirror of ``gdb.FinishBreakpoint`` class.
+
+    See https://sourceware.org/gdb/onlinedocs/gdb/Finish-Breakpoints-in-Python.html
+    for more information.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """Do not create instances of this class directly.
+
+        Use ``pwnlib.gdb.Gdb.FinishBreakpoint`` instead.
+        """
+        # See https://github.com/pylint-dev/pylint/issues/4228
+        # pylint: disable=useless-super-delegation
+        super().__init__(*args, **kwargs)
+
+    def _server_set_breakpoint(self, *args, **kwargs):
+        return self.conn.root.set_finish_breakpoint(
+            self, hasattr(self, 'stop'), hasattr(self, 'out_of_scope'),
+            *args, **kwargs)
+
+    def __getattr__(self, item):
+        """Return attributes of the real breakpoint."""
+        if item in (
+                '____id_pack__',
+                '__name__',
+                '____conn__',
+                'stop',
+                'out_of_scope',
+        ):
+            # Ignore RPyC netref attributes.
+            # Also, if stop() or out_of_scope() are not defined, hasattr() call
+            # in our __init__() will bring us here. Don't contact the
+            # server in this case either.
+            raise AttributeError()
+        return getattr(self.server_breakpoint, item)
+
+    def exposed_out_of_scope(self):
+        # Handle out_of_scope() call from the server.
+        return self.out_of_scope()
+
 class Gdb:
     """Mirror of ``gdb`` module.
 
@@ -680,8 +848,12 @@ class Gdb:
         class _Breakpoint(Breakpoint):
             def __init__(self, *args, **kwargs):
                 super().__init__(conn, *args, **kwargs)
+        class _FinishBreakpoint(FinishBreakpoint):
+            def __init__(self, *args, **kwargs):
+                super().__init__(conn, *args, **kwargs)
 
         self.Breakpoint = _Breakpoint
+        self.FinishBreakpoint = _FinishBreakpoint
         self.stopped = Event()
 
         def stop_handler(event):
@@ -728,8 +900,10 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
         arch(str): Architechture of the target binary.  If `exe` known GDB will
           detect the architechture automatically (if it is supported).
         gdb_args(list): List of additional arguments to pass to GDB.
-        sysroot(str): Foreign-architecture sysroot, used for QEMU-emulated binaries
-            and Android targets.
+        sysroot(str): Set an alternate system root. The system root is used to
+            load absolute shared library symbol files. This is useful to instruct
+            gdb to load a local version of binaries/libraries instead of downloading
+            them from the gdbserver, which is faster
         api(bool): Enable access to GDB Python API.
 
     Returns:
@@ -782,8 +956,8 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
 
         Using GDB Python API:
 
-        .. doctest
-           :skipif: six.PY2
+        .. doctest::
+           :skipif: is_python2
 
             >>> io = process('bash')
 
@@ -868,17 +1042,17 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
 
     # gdb script to run before `gdbscript`
     pre = ''
+    if sysroot:
+        pre += 'set sysroot %s\n' % sysroot
     if not context.native:
         pre += 'set endian %s\n' % context.endian
         pre += 'set architecture %s\n' % get_gdb_arch()
-        if sysroot:
-            pre += 'set sysroot %s\n' % sysroot
 
         if context.os == 'android':
             pre += 'set gnutarget ' + _bfdname() + '\n'
 
         if exe and context.os != 'baremetal':
-            pre += 'file %s\n' % exe
+            pre += 'file "%s"\n' % exe
 
     # let's see if we can find a pid to attach to
     pid = None
@@ -915,7 +1089,7 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
             cmd = ['sshpass', '-p', shell.password] + cmd
         if shell.keyfile:
             cmd += ['-i', shell.keyfile]
-        cmd += ['gdb', '-q', target.executable, target.pid, '-x', tmpfile]
+        cmd += ['gdb', '-q', target.executable, str(target.pid), '-x', tmpfile]
 
         misc.run_in_new_terminal(cmd)
         return
@@ -967,7 +1141,7 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
 
         exe = exe or findexe()
     elif isinstance(target, elf.corefile.Corefile):
-        pre += 'target core %s\n' % target.path
+        pre += 'target core "%s"\n' % target.path
     else:
         log.error("don't know how to attach to target: %r", target)
 
@@ -1028,13 +1202,12 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
     gdbscript = pre + (gdbscript or '')
 
     if gdbscript:
-        tmp = tempfile.NamedTemporaryFile(prefix = 'pwn', suffix = '.gdb',
-                                          delete = False, mode = 'w+')
-        log.debug('Wrote gdb script to %r\n%s', tmp.name, gdbscript)
-        gdbscript = 'shell rm %s\n%s' % (tmp.name, gdbscript)
+        with tempfile.NamedTemporaryFile(prefix = 'pwnlib-gdbscript-', suffix = '.gdb',
+                                          delete = False, mode = 'w+') as tmp:
+            log.debug('Wrote gdb script to %r\n%s', tmp.name, gdbscript)
+            gdbscript = 'shell rm %s\n%s' % (tmp.name, gdbscript)
 
-        tmp.write(gdbscript)
-        tmp.close()
+            tmp.write(gdbscript)
         cmd += ['-x', tmp.name]
 
     log.info('running in new terminal: %s', cmd)
@@ -1329,7 +1502,7 @@ def version(program='gdb'):
 
     Example:
 
-        >>> (7,0) <= gdb.version() <= (12,0)
+        >>> (7,0) <= gdb.version() <= (19,0)
         True
     """
     program = misc.which(program)

pwnlib/gdb_api_bridge.py

@@ -89,6 +89,17 @@ class GdbService(Service):
             return Breakpoint(*args, **kwargs)
         return gdb.Breakpoint(*args, **kwargs)
 
+    def exposed_set_finish_breakpoint(self, client, has_stop, has_out_of_scope, *args, **kwargs):
+        """Create a finish breakpoint and connect it with the client-side mirror."""
+        class FinishBreakpoint(gdb.FinishBreakpoint):
+            if has_stop:
+                def stop(self):
+                    return client.stop()
+            if has_out_of_scope:
+                def out_of_scope(self):
+                    client.out_of_scope()
+        return FinishBreakpoint(*args, **kwargs)
+
     def exposed_quit(self):
         """Terminate GDB."""
         gdb.post_event(lambda: gdb.execute('quit'))
@@ -99,5 +110,6 @@ spawn(ThreadedServer(
     socket_path=socket_path,
     protocol_config={
         'allow_all_attrs': True,
+        'allow_setattr': True,
     },
 ).start)

pwnlib/lexer.py

@@ -38,7 +38,7 @@ class PwntoolsLexer(RegexLexer):
     string = r'"(\\"|[^"])*"'
     char = r'[\w$.@-]'
     identifier = r'(?:[a-zA-Z$_]' + char + r'*|\.' + char + '+|or)'
-    number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
+    number = r'(?:-?0[xX][a-zA-Z0-9]+|\d+)'
     memory = r'(?:[\]\[])'
     bad = r'(?:\(bad\))'
 

pwnlib/libcdb.py

@@ -4,24 +4,40 @@ Fetch a LIBC binary based on some heuristics.
 from __future__ import absolute_import
 from __future__ import division
 
-import codecs
-import json
 import os
+import time
+import six
 import tempfile
 
 from pwnlib.context import context
 from pwnlib.elf import ELF
 from pwnlib.log import getLogger
-from pwnlib.util.fiddling import b64d
-from pwnlib.util.fiddling import hexdump
+from pwnlib.tubes.process import process
+from pwnlib.util.fiddling import enhex
+from pwnlib.util.hashes import sha1filehex, sha256filehex, md5filehex
 from pwnlib.util.misc import read
+from pwnlib.util.misc import which
 from pwnlib.util.misc import write
-from pwnlib.util.safeeval import const
 from pwnlib.util.web import wget
 
 log = getLogger(__name__)
 
-HASHES = ['build_id', 'sha1', 'sha256', 'md5']
+HASHES = {
+    'build_id': lambda path: enhex(ELF(path, checksec=False).buildid or b''),
+    'sha1': sha1filehex,
+    'sha256': sha256filehex,
+    'md5': md5filehex,
+}
+DEBUGINFOD_SERVERS = [
+    'https://debuginfod.elfutils.org/',
+]
+
+if 'DEBUGINFOD_URLS' in os.environ:
+    urls = os.environ['DEBUGINFOD_URLS'].split(' ')
+    DEBUGINFOD_SERVERS = urls + DEBUGINFOD_SERVERS
+
+# Retry failed lookups after some time
+NEGATIVE_CACHE_EXPIRY = 60 * 60 * 24 * 7 # 1 week
 
 # https://gitlab.com/libcdb/libcdb wasn't updated after 2019,
 # but still is a massive database of older libc binaries.
@@ -52,65 +68,77 @@ def provider_libcdb(hex_encoded_id, hash_type):
         log.warn_once("Failed to fetch libc for %s %s from libcdb: %s", hash_type, hex_encoded_id, e)
     return data
 
-# https://libc.rip/
-def provider_libc_rip(hex_encoded_id, hash_type):
+def query_libc_rip(params):
     # Deferred import because it's slow
     import requests
 
+    url = "https://libc.rip/api/find"
+    try:
+        result = requests.post(url, json=params, timeout=20)
+        result.raise_for_status()
+        if result.status_code != 200:
+            log.debug("Error: %s", result.text)
+            return None
+        return result.json()
+    except requests.RequestException as e:
+        log.warn_once("Failed to fetch libc info from libc.rip: %s", e)
+        return None
+
+# https://libc.rip/
+def provider_libc_rip(hex_encoded_id, hash_type):
     # Build the request for the hash type
     # https://github.com/niklasb/libc-database/blob/master/searchengine/api.yml
     if hash_type == 'build_id':
         hash_type = 'buildid'
-    url    = "https://libc.rip/api/find"
     params = {hash_type: hex_encoded_id}
 
-    data = b""
-    try:
-        result = requests.post(url, json=params, timeout=20)
-        if result.status_code != 200 or len(result.json()) == 0:
-            log.warn_once("Could not find libc for %s %s on libc.rip", hash_type, hex_encoded_id)
-            log.debug("Error: %s", result.text)
-            return None
+    libc_match = query_libc_rip(params)
+    if not libc_match:
+        log.warn_once("Could not find libc info for %s %s on libc.rip", hash_type, hex_encoded_id)
+        return None
 
-        libc_match = result.json()
-        assert len(libc_match) == 1, 'Invalid libc.rip response.'
+    if len(libc_match) > 1:
+        log.debug("Received multiple matches. Choosing the first match and discarding the others.")
+        log.debug("%r", libc_match)
 
-        url = libc_match[0]['download_url']
-        log.debug("Downloading data from libc.rip: %s", url)
-        data = wget(url, timeout=20)
+    url = libc_match[0]['download_url']
+    log.debug("Downloading data from libc.rip: %s", url)
+    data = wget(url, timeout=20)
 
-        if not data:
-            log.warn_once("Could not fetch libc for %s %s from libc.rip", hash_type, hex_encoded_id)
-            return None
-    except requests.RequestException as e:
-        log.warn_once("Failed to fetch libc for %s %s from libc.rip: %s", hash_type, hex_encoded_id, e)
+    if not data:
+        log.warn_once("Could not fetch libc binary for %s %s from libc.rip", hash_type, hex_encoded_id)
+        return None
     return data
 
-PROVIDERS = [provider_libcdb, provider_libc_rip]
+# Check if the local system libc matches the requested hash.
+def provider_local_system(hex_encoded_id, hash_type):
+    if hash_type == 'id':
+        return None
+    shell_path = os.environ.get('SHELL', None) or '/bin/sh'
+    if not os.path.exists(shell_path):
+        log.debug('Shell path %r does not exist. Skipping local system libc matching.', shell_path)
+        return None
+    local_libc = ELF(shell_path, checksec=False).libc
+    if not local_libc:
+        log.debug('Cannot lookup libc from shell %r. Skipping local system libc matching.', shell_path)
+        return None
+    if HASHES[hash_type](local_libc.path) == hex_encoded_id:
+        return local_libc.data
+    return None
 
-def search_by_hash(hex_encoded_id, hash_type='build_id'):
+PROVIDERS = [provider_local_system, provider_libcdb, provider_libc_rip]
+
+def search_by_hash(hex_encoded_id, hash_type='build_id', unstrip=True):
     assert hash_type in HASHES, hash_type
 
     # Ensure that the libcdb cache directory exists
-    cache_dir = os.path.join(context.cache_dir, 'libcdb', hash_type)
-
-    if not os.path.isdir(cache_dir):
-        os.makedirs(cache_dir)
-
-    # If we already downloaded the file, and it looks even passingly like
-    # a valid ELF file, return it.
-    cache = os.path.join(cache_dir, hex_encoded_id)
-
-    if os.path.exists(cache):
-        log.debug("Found existing cached libc at %r", cache)
-
-        data = read(cache)
-        if data.startswith(b'\x7FELF'):
-            log.info_once("Using cached data from %r", cache)
-            return cache
-        else:
-            log.info_once("Skipping unavailable libc %s", hex_encoded_id)
-            return None
+    cache, cache_valid = _check_elf_cache('libcdb', hex_encoded_id, hash_type)
+    if cache_valid:
+        return cache
+    
+    # We searched for this buildid before, but didn't find anything.
+    if cache is None:
+        return None
 
     # Run through all available libc database providers to see if we have a match.
     for provider in PROVIDERS:
@@ -128,21 +156,451 @@ def search_by_hash(hex_encoded_id, hash_type='build_id'):
     if not data or not data.startswith(b'\x7FELF'):
         return None
 
+    # Try to find debug info for this libc.
+    if unstrip:
+        unstrip_libc(cache)
+
     return cache
 
+def _search_debuginfo_by_hash(base_url, hex_encoded_id):
+    # Deferred import because it's slow
+    import requests
+    from six.moves import urllib
 
-def search_by_build_id(hex_encoded_id):
+    # Check if we tried this buildid before.
+    cache, cache_valid = _check_elf_cache('libcdb_dbg', hex_encoded_id, 'build_id')
+    if cache_valid:
+        return cache
+    
+    # We searched for this buildid before, but didn't find anything.
+    if cache is None:
+        return None
+
+    # Try to find separate debuginfo.
+    url  = '/buildid/{}/debuginfo'.format(hex_encoded_id)
+    url  = urllib.parse.urljoin(base_url, url)
+    data = b""
+    log.debug("Downloading data from debuginfod: %s", url)
+    try:
+        data = wget(url, timeout=20)
+    except requests.RequestException as e:
+        log.warn_once("Failed to fetch libc debuginfo for build_id %s from %s: %s", hex_encoded_id, base_url, e)
+    
+    # Save whatever we got to the cache
+    write(cache, data or b'')
+
+    # Return ``None`` if we did not get a valid ELF file
+    if not data or not data.startswith(b'\x7FELF'):
+        log.warn_once("Could not fetch libc debuginfo for build_id %s from %s", hex_encoded_id, base_url)
+        return None
+
+    return cache
+
+def _check_elf_cache(cache_type, hex_encoded_id, hash_type):
+    """
+    Check if there already is an ELF file for this hash in the cache.
+
+    >>> cache, _ = _check_elf_cache('libcdb', '2d1c5e0b85cb06ff47fa6fa088ec22cb6e06074e', 'build_id')
+    >>> os.unlink(cache) if os.path.exists(cache)
+    >>> filename = search_by_hash('2d1c5e0b85cb06ff47fa6fa088ec22cb6e06074e', 'build_id', unstrip=False)
+    >>> hex(ELF(filename).symbols.read)
+    '0xe56c0'
+    >>> filename == cache
+    True
+    """
+    # Ensure that the cache directory exists
+    cache_dir = os.path.join(context.cache_dir, cache_type, hash_type)
+
+    if not os.path.isdir(cache_dir):
+        os.makedirs(cache_dir)
+
+    # If we already downloaded the file, and it looks even passingly like
+    # a valid ELF file, return it.
+    cache = os.path.join(cache_dir, hex_encoded_id)
+
+    if not os.path.exists(cache):
+        return cache, False
+    
+    log.debug("Found existing cached ELF at %r", cache)
+
+    data = read(cache)
+    if not data.startswith(b'\x7FELF'):
+        # Retry failed lookups after some time
+        if time.time() > os.path.getmtime(cache) + NEGATIVE_CACHE_EXPIRY:
+            return cache, False
+        log.info_once("Skipping invalid cached ELF %s", hex_encoded_id)
+        return None, False
+
+    log.info_once("Using cached data from %r", cache)
+    return cache, True
+
+def unstrip_libc(filename):
+    """
+    Given a path to a libc binary, attempt to download matching debug info
+    and add them back to the given binary.
+
+    This modifies the given file.
+
+    Arguments:
+        filename(str):
+            Path to the libc binary to unstrip.
+
+    Returns:
+        :const:`True` if binary was unstripped, :const:`False` otherwise.
+
+    Examples:
+
+        >>> filename = search_by_build_id('69389d485a9793dbe873f0ea2c93e02efaa9aa3d', unstrip=False)
+        >>> libc = ELF(filename)
+        >>> 'main_arena' in libc.symbols
+        False
+        >>> unstrip_libc(filename)
+        True
+        >>> libc = ELF(filename)
+        >>> hex(libc.symbols.main_arena)
+        '0x219c80'
+        >>> unstrip_libc(pwnlib.data.elf.get('test-x86'))
+        False
+        >>> filename = search_by_build_id('d1704d25fbbb72fa95d517b883131828c0883fe9', unstrip=True)
+        >>> 'main_arena' in ELF(filename).symbols
+        True
+    """
+    if not which('eu-unstrip'):
+        log.warn_once('Couldn\'t find "eu-unstrip" in PATH. Install elfutils first.')
+        return False
+
+    libc = ELF(filename, checksec=False)
+    if not libc.buildid:
+        log.warn_once('Given libc does not have a buildid. Cannot look for debuginfo to unstrip.')
+        return False
+
+    log.debug('Trying debuginfod servers: %r', DEBUGINFOD_SERVERS)
+
+    for server_url in DEBUGINFOD_SERVERS:
+        libc_dbg = _search_debuginfo_by_hash(server_url, enhex(libc.buildid))
+        if libc_dbg:
+            break
+    else:
+        log.warn_once('Couldn\'t find debug info for libc with build_id %s on any debuginfod server.', enhex(libc.buildid))
+        return False
+
+    # Add debug info to given libc binary inplace.
+    p = process(['eu-unstrip', '-o', filename, filename, libc_dbg])
+    output = p.recvall()
+    p.close()
+
+    if output:
+        log.error('Failed to unstrip libc binary: %r', output)
+        return False
+
+    return True
+
+def _extract_tarfile(cache_dir, data_filename, tarball):
+    from six import BytesIO
+    import tarfile
+    # Handle zstandard compression, since tarfile only supports gz, bz2, and xz.
+    if data_filename.endswith('.zst') or data_filename.endswith('.zstd'):
+        import zstandard
+        dctx = zstandard.ZstdDecompressor()
+        decompressed_tar = BytesIO()
+        dctx.copy_stream(tarball, decompressed_tar)
+        decompressed_tar.seek(0)
+        tarball.close()
+        tarball = decompressed_tar
+
+    if six.PY2 and data_filename.endswith('.xz'):
+        # Python 2's tarfile doesn't support xz, so we need to decompress it first.
+        # Shell out to xz, since the Python 2 pylzma module is broken.
+        # (https://github.com/fancycode/pylzma/issues/67)
+        if not which('xz'):
+            log.error('Couldn\'t find "xz" in PATH. Please install xz first.')
+        import subprocess
+        try:
+            uncompressed_tarball = subprocess.check_output(['xz', '--decompress', '--stdout', tarball.name])
+            tarball = BytesIO(uncompressed_tarball)
+        except subprocess.CalledProcessError:
+            log.error('Failed to decompress xz archive.')
+
+    with tarfile.open(fileobj=tarball) as tar_file:
+        # Find the library folder in the archive (e.g. /lib/x86_64-linux-gnu/)
+        lib_dir = None
+        libc_name = None
+        for member in tar_file.getmembers():
+            if not member.isfile():
+                continue
+            libc_name = os.path.basename(member.name)
+            if libc_name == 'libc.so.6' or (libc_name.startswith('libc') and libc_name.endswith('.so')):
+                lib_dir = os.path.dirname(member.name)
+                break
+        else:
+            log.error('Couldn\'t find library folder containing the libc in the archive.')
+
+        # Extract everything in the library folder
+        for member in tar_file.getmembers():
+            if os.path.dirname(member.name) != lib_dir:
+                continue
+            if not member.isfile() and not member.issym():
+                continue
+            # Extract while keeping file permissions
+            tar_file.extract(member, cache_dir)
+
+        # Move the files up to the cache root
+        target_dir = os.path.join(cache_dir, lib_dir)
+        for file in os.listdir(target_dir):
+            os.rename(os.path.join(target_dir, file), os.path.join(cache_dir, file))
+        os.removedirs(target_dir)
+
+        return os.path.join(cache_dir, libc_name)
+
+def _extract_debfile(cache_dir, package_filename, package):
+    # Extract data.tar in the .deb archive.
+    if six.PY2:
+        if not which('ar'):
+            log.error('Missing command line tool "ar" to extract .deb archive. Please install "ar" first.')
+
+        import atexit
+        import shutil
+        import subprocess
+
+        # Use mkdtemp instead of TemporaryDirectory because the latter is not available in Python 2.
+        tempdir = tempfile.mkdtemp(prefix=".pwntools-tmp")
+        atexit.register(shutil.rmtree, tempdir)
+        with tempfile.NamedTemporaryFile(mode='wb', dir=tempdir) as debfile:
+            debfile.write(package)
+            debfile.flush()
+            try:
+                files_in_deb = subprocess.check_output(['ar', 't', debfile.name]).split(b'\n')
+            except subprocess.CalledProcessError:
+                log.error('Failed to list files in .deb archive.')
+            [data_filename] = filter(lambda f: f.startswith(b'data.tar'), files_in_deb)
+
+            try:
+                subprocess.check_call(['ar', 'x', debfile.name, data_filename], cwd=tempdir)
+            except subprocess.CalledProcessError:
+                log.error('Failed to extract data.tar from .deb archive.')
+
+            with open(os.path.join(tempdir, data_filename), 'rb') as tarball:
+                return _extract_tarfile(cache_dir, data_filename, tarball)
+    else:
+        import unix_ar
+        from six import BytesIO
+        ar_file = unix_ar.open(BytesIO(package))
+        try:
+            data_filename = next(filter(lambda f: f.name.startswith(b'data.tar'), ar_file.infolist())).name.decode()
+            tarball = ar_file.open(data_filename)
+            return _extract_tarfile(cache_dir, data_filename, tarball)
+        finally:
+            ar_file.close()
+
+def _extract_pkgfile(cache_dir, package_filename, package):
+    from six import BytesIO
+    return _extract_tarfile(cache_dir, package_filename, BytesIO(package))
+
+def _find_libc_package_lib_url(libc):
+    # Check https://libc.rip for the libc package
+    libc_match = query_libc_rip({'buildid': enhex(libc.buildid)})
+    if libc_match is not None:
+        for match in libc_match:
+            yield match['libs_url']
+    
+    # Check launchpad.net if it's an Ubuntu libc
+    # GNU C Library (Ubuntu GLIBC 2.36-0ubuntu4)
+    import re
+    version = re.search(br'GNU C Library \(Ubuntu E?GLIBC ([^\)]+)\)', libc.data)
+    if version is not None:
+        libc_version = version.group(1).decode()
+        yield 'https://launchpad.net/ubuntu/+archive/primary/+files/libc6_{}_{}.deb'.format(libc_version, libc.arch)
+
+def download_libraries(libc_path, unstrip=True):
+    """download_libraries(str, bool) -> str
+    Download the matching libraries for the given libc binary and cache
+    them in a local directory. The libraries are looked up using `libc.rip <https://libc.rip>`_
+    and fetched from the official package repositories if available.
+
+    This commonly includes the ``ld-linux-x86-64.so.2`` and ``libpthread.so.0`` binaries
+    which can be used to execute the program locally when the given libc is
+    incompatible with the local dynamic loader.
+
+    Note: Only .deb and .pkg.tar.* packages are currently supported (Debian/Ubuntu, Arch).
+
+    Arguments:
+        libc_path(str):
+            The path the libc binary.
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
+
+    Returns:
+        The path to the cached directory containing the downloaded libraries.
+
+    Example:
+
+        >>> libc_path = ELF(which('ls'), checksec=False).libc.path
+        >>> lib_path = download_libraries(libc_path)
+        >>> lib_path is not None
+        True
+        >>> os.path.exists(os.path.join(lib_path, 'libc.so.6'))
+        True
+        >>> os.path.exists(os.path.join(lib_path, 'ld-linux-x86-64.so.2'))
+        True
+    """
+
+    libc = ELF(libc_path, checksec=False)
+    if not libc.buildid:
+        log.warn_once('Given libc does not have a buildid.')
+        return None
+    
+    # Handle caching and don't redownload if it already exists.
+    cache_dir = os.path.join(context.cache_dir, 'libcdb_libs')
+    if not os.path.isdir(cache_dir):
+        os.makedirs(cache_dir)
+    
+    cache_dir = os.path.join(cache_dir, enhex(libc.buildid))
+    if os.path.exists(cache_dir):
+        return cache_dir
+
+    for package_url in _find_libc_package_lib_url(libc):
+        extension_handlers = {
+            '.deb': _extract_debfile,
+            '.pkg.tar.xz': _extract_pkgfile,
+            '.pkg.tar.zst': _extract_pkgfile,
+        }
+
+        package_filename = os.path.basename(package_url)
+        for extension, handler in extension_handlers.items():
+            if package_filename.endswith(extension):
+                break
+        else:
+            log.failure('Cannot handle %s (%s)', package_filename, package_url)
+            continue
+
+        # Download the package
+        package = wget(package_url, timeout=20)
+        if not package:
+            continue
+
+        # Create target cache directory to extract files into
+        if not os.path.isdir(cache_dir):
+            os.makedirs(cache_dir)
+
+        try:
+            # Extract the archive
+            libc_path = handler(cache_dir, package_filename, package)
+        except Exception as e:
+            os.removedirs(cache_dir)
+            log.failure('Failed to extract %s: %s', package_filename, e)
+            continue
+        # Unstrip the libc binary
+        try:
+            if unstrip:
+                unstrip_libc(libc_path)
+        except Exception:
+            pass
+
+        return cache_dir
+
+    log.warn_once('Failed to find matching libraries for provided libc.')
+    return None
+
+def _handle_multiple_matching_libcs(matching_libcs):
+    from pwnlib.term import text
+    from pwnlib.ui import options
+    log.info('Multiple matching libc libraries for requested symbols:')
+    for idx, libc in enumerate(matching_libcs):
+        log.info('%d. %s', idx+1, text.red(libc['id']))
+        log.indented('\t%-20s %s', text.green('BuildID:'), libc['buildid'])
+        log.indented('\t%-20s %s', text.green('MD5:'), libc['md5'])
+        log.indented('\t%-20s %s', text.green('SHA1:'), libc['sha1'])
+        log.indented('\t%-20s %s', text.green('SHA256:'), libc['sha256'])
+        log.indented('\t%s', text.green('Symbols:'))
+        for symbol, address in libc['symbols'].items():
+            log.indented('\t%25s = %s', symbol, address)
+
+    selected_index = options("Select the libc version to use:", [libc['id'] for libc in matching_libcs])
+    return matching_libcs[selected_index]
+
+def search_by_symbol_offsets(symbols, select_index=None, unstrip=True, return_as_list=False):
+    """
+    Lookup possible matching libc versions based on leaked function addresses.
+
+    The leaked function addresses have to be provided as a dict mapping the
+    function name to the leaked value. Only the lower 3 nibbles are relevant
+    for the lookup.
+
+    If there are multiple matches you are presented with a list to select one
+    interactively, unless the ``select_index`` or ``return_as_list`` arguments
+    are used.
+
+    Arguments:
+        symbols(dict):
+            Dictionary mapping symbol names to their addresses.
+        select_index(int):
+            The libc to select if there are multiple matches (starting at 1).
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
+        return_as_list(bool):
+            Return a list of build ids of all matching libc versions
+            instead of a path to a downloaded file.
+
+    Returns:
+        Path to the downloaded library on disk, or :const:`None`.
+        If the ``return_as_list`` argument is :const:`True`, a list of build ids
+        is returned instead.
+
+    Examples:
+
+        >>> filename = search_by_symbol_offsets({'puts': 0x420, 'printf': 0xc90}, select_index=1)
+        >>> libc = ELF(filename)
+        >>> libc.sym.system == 0x52290
+        True
+        >>> matched_libcs = search_by_symbol_offsets({'__libc_start_main_ret': '7f89ad926550'}, return_as_list=True)
+        >>> len(matched_libcs) > 1
+        True
+        >>> for buildid in matched_libcs: # doctest +SKIP
+        ...     libc = ELF(search_by_build_id(buildid)) # doctest +SKIP
+    """
+    for symbol, address in symbols.items():
+        if isinstance(address, int):
+            symbols[symbol] = hex(address)
+
+    params = {'symbols': symbols}
+    log.debug('Request: %s', params)
+    matching_libcs = query_libc_rip(params)
+    log.debug('Result: %s', matching_libcs)
+    if matching_libcs is None or len(matching_libcs) == 0:
+        log.warn_once("No matching libc for symbols %r on libc.rip", symbols)
+        return None
+
+    if return_as_list:
+        return [libc['buildid'] for libc in matching_libcs]
+
+    if len(matching_libcs) == 1:
+        return search_by_build_id(matching_libcs[0]['buildid'], unstrip=unstrip)
+
+    if select_index is not None:
+        if select_index > 0 and select_index <= len(matching_libcs):
+            return search_by_build_id(matching_libcs[select_index - 1]['buildid'], unstrip=unstrip)
+        else:
+            log.error('Invalid selected libc index. %d is not in the range of 1-%d.', select_index, len(matching_libcs))
+            return None
+
+    selected_libc = _handle_multiple_matching_libcs(matching_libcs)
+    return search_by_build_id(selected_libc['buildid'], unstrip=unstrip)
+
+def search_by_build_id(hex_encoded_id, unstrip=True):
     """
     Given a hex-encoded Build ID, attempt to download a matching libc from libcdb.
 
     Arguments:
         hex_encoded_id(str):
             Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
 
     Examples:
+
         >>> filename = search_by_build_id('fe136e485814fee2268cf19e5c124ed0f73f4400')
         >>> hex(ELF(filename).symbols.read)
         '0xda260'
@@ -152,20 +610,23 @@ def search_by_build_id(hex_encoded_id):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'build_id')
+    return search_by_hash(hex_encoded_id, 'build_id', unstrip)
 
-def search_by_md5(hex_encoded_id):
+def search_by_md5(hex_encoded_id, unstrip=True):
     """
     Given a hex-encoded md5sum, attempt to download a matching libc from libcdb.
 
     Arguments:
         hex_encoded_id(str):
             Hex-encoded md5sum (e.g. 'ABCDEF...') of the library
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
 
     Examples:
+
         >>> filename = search_by_md5('7a71dafb87606f360043dcd638e411bd')
         >>> hex(ELF(filename).symbols.read)
         '0xda260'
@@ -175,20 +636,23 @@ def search_by_md5(hex_encoded_id):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'md5')
+    return search_by_hash(hex_encoded_id, 'md5', unstrip)
 
-def search_by_sha1(hex_encoded_id):
+def search_by_sha1(hex_encoded_id, unstrip=True):
     """
     Given a hex-encoded sha1, attempt to download a matching libc from libcdb.
 
     Arguments:
         hex_encoded_id(str):
             Hex-encoded sha1sum (e.g. 'ABCDEF...') of the library
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
 
     Examples:
+
         >>> filename = search_by_sha1('34471e355a5e71400b9d65e78d2cd6ce7fc49de5')
         >>> hex(ELF(filename).symbols.read)
         '0xda260'
@@ -198,21 +662,24 @@ def search_by_sha1(hex_encoded_id):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'sha1')
+    return search_by_hash(hex_encoded_id, 'sha1', unstrip)
 
 
-def search_by_sha256(hex_encoded_id):
+def search_by_sha256(hex_encoded_id, unstrip=True):
     """
     Given a hex-encoded sha256, attempt to download a matching libc from libcdb.
 
     Arguments:
         hex_encoded_id(str):
             Hex-encoded sha256sum (e.g. 'ABCDEF...') of the library
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
 
     Examples:
+
         >>> filename = search_by_sha256('5e877a8272da934812d2d1f9ee94f73c77c790cbc5d8251f5322389fc9667f21')
         >>> hex(ELF(filename).symbols.read)
         '0xda260'
@@ -222,7 +689,7 @@ def search_by_sha256(hex_encoded_id):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'sha256')
+    return search_by_hash(hex_encoded_id, 'sha256', unstrip)
 
 
 
@@ -279,4 +746,4 @@ def get_build_id_offsets():
     }.get(context.arch, [])
 
 
-__all__ = ['get_build_id_offsets', 'search_by_build_id', 'search_by_sha1', 'search_by_sha256', 'search_by_md5']
+__all__ = ['get_build_id_offsets', 'search_by_build_id', 'search_by_sha1', 'search_by_sha256', 'search_by_md5', 'unstrip_libc', 'search_by_symbol_offsets', 'download_libraries']

pwnlib/log.py

@@ -84,7 +84,7 @@ result in a message being emitted but rather an animated progress line (with a
 spinner!) being created.  Note that other handlers will still see a meaningful
 log record.
 
-The custom handler will only handle log records whith a level of at least
+The custom handler will only handle log records with a level of at least
 :data:`context.log_level`.  Thus if e.g. the level for the
 ``'pwnlib.tubes.ssh'`` is set to ``'DEBUG'`` no additional output will show up
 unless :data:`context.log_level` is also set to ``'DEBUG'``.  Other handlers
@@ -99,6 +99,7 @@ import os
 import random
 import re
 import six
+import string
 import sys
 import threading
 import time
@@ -401,6 +402,20 @@ class Logger(object):
 
         self.info(pwnlib.util.fiddling.hexdump(message, *args, **kwargs))
 
+    def maybe_hexdump(self, message, *args, **kwargs):
+        """maybe_hexdump(self, message, *args, **kwargs)
+
+        Logs a message using indented. Repeated single byte is compressed, and
+        unprintable message is hexdumped.
+        """
+        if len(set(message)) == 1 and len(message) > 1:
+            self.indented('%r * %#x' % (message[:1], len(message)), *args, **kwargs)
+        elif len(message) == 1 or all(c in string.printable.encode() for c in message):
+            for line in message.splitlines(True):
+                self.indented(repr(line), *args, **kwargs)
+        else:
+            import pwnlib.util.fiddling
+            self.indented(pwnlib.util.fiddling.hexdump(message), *args, **kwargs)
 
     def warning(self, message, *args, **kwargs):
         """warning(message, *args, **kwargs)
@@ -545,7 +560,7 @@ class Handler(logging.StreamHandler):
 
         # we enrich the `Progress` object to keep track of the spinner
         if not hasattr(progress, '_spinner_handle'):
-            spinner_handle = term.output('')
+            spinner_handle = term.output('[x] ')
             msg_handle = term.output(msg)
             stop = threading.Event()
             def spin():

pwnlib/py2compat.py

@@ -0,0 +1,94 @@
+"""
+Compatibility layer with python 2, allowing us to write normal code.
+Beware, some monkey-patching is done.
+"""
+
+import os
+import shutil
+import sys
+try:
+    import fcntl
+    import termios
+except ImportError:
+    pass
+
+from collections import namedtuple
+from struct import Struct
+
+def py2_monkey_patch(module):
+    def decorator(f):
+        if sys.version_info < (3,):
+            f.__module__ = module.__name__
+            setattr(module, f.__name__, f)
+    return decorator
+
+# python3 -c 'import shutil,inspect; print(inspect.getsource(shutil.get_terminal_size))'
+@py2_monkey_patch(shutil)
+def get_terminal_size(fallback=(80, 24)):
+    """Get the size of the terminal window.
+
+    For each of the two dimensions, the environment variable, COLUMNS
+    and LINES respectively, is checked. If the variable is defined and
+    the value is a positive integer, it is used.
+
+    When COLUMNS or LINES is not defined, which is the common case,
+    the terminal connected to sys.__stdout__ is queried
+    by invoking os.get_terminal_size.
+
+    If the terminal size cannot be successfully queried, either because
+    the system doesn't support querying, or because we are not
+    connected to a terminal, the value given in fallback parameter
+    is used. Fallback defaults to (80, 24) which is the default
+    size used by many terminal emulators.
+
+    The value returned is a named tuple of type os.terminal_size.
+    """
+    # columns, lines are the working values
+    try:
+        columns = int(os.environ['COLUMNS'])
+    except (KeyError, ValueError):
+        columns = 0
+
+    try:
+        lines = int(os.environ['LINES'])
+    except (KeyError, ValueError):
+        lines = 0
+
+    # only query if necessary
+    if columns <= 0 or lines <= 0:
+        try:
+            size = os.get_terminal_size(sys.__stdout__.fileno())
+        except (AttributeError, ValueError, IOError):
+            # stdout is None, closed, detached, or not a terminal, or
+            # os.get_terminal_size() is unsupported
+            size = os.terminal_size(fallback)
+        if columns <= 0:
+            columns = size.columns
+        if lines <= 0:
+            lines = size.lines
+
+    return os.terminal_size((columns, lines))
+
+@py2_monkey_patch(os)
+class terminal_size(tuple):
+    @property
+    def columns(self):
+        return self[0]
+
+    @property
+    def lines(self):
+        return self[1]
+
+    def __repr__(self):
+        return 'os.terminal_size(columns=%r, lines=%r)' % self
+
+terminal_size = namedtuple('terminal_size', 'columns lines')
+
+termsize = Struct('HHHH')
+
+@py2_monkey_patch(os)
+def get_terminal_size(fd):  # pylint: disable=function-redefined
+    arr = b'\0' * termsize.size
+    arr = fcntl.ioctl(fd, termios.TIOCGWINSZ, arr)
+    lines, columns, xpixel, ypixel = termsize.unpack(arr)
+    return os.terminal_size((columns, lines))

pwnlib/rop/call.py

@@ -246,6 +246,13 @@ class Call(object):
                                     fmt % self.target,
                                     self.args)
 
+    def is_flat(self):
+        if isinstance(self, six.integer_types + (Unresolved,)):
+            return True
+        if not isinstance(self, Call):
+            return False
+        return bool(self.args or self.stack_arguments_before)
+
     @property
     def register_arguments(self):
         return dict(zip(self.abi.register_arguments, self.args))