Release 4.6.0

* For tmux, always specify -F and -P

Fixes #1898

* [4.5.1] Update CHANGELOG

.coveragerc

@@ -7,5 +7,6 @@ omit =
 source =
     pwn
     pwnlib
-    ~/.pwntools-cache-2.7/
-    ~/.pwntools-cache-3.8/
+    ~/.cache/.pwntools-cache-2.7/
+    ~/.cache/.pwntools-cache-3.8/
+disable_warnings = module-not-imported

.github/ISSUE_TEMPLATE/feature_request.md

@@ -12,7 +12,7 @@ Thanks for contributing to Pwntools!   Ideas from the community help make Pwntoo
 If you've got an idea for a new feature, please provide information about:
 
 * What the feature does
-* Why the feature should exit
+* Why the feature should exist
 * What tests should be included
 
 If you think you can write the feature yourself, please submit a Pull Request and we can review your changes!

.github/workflows/android.yml

@@ -0,0 +1,60 @@
+name: Android Tests
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Linux dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
+          gdb gdbserver socat \
+          qemu-user-static \
+          binutils-aarch64-linux-gnu \
+          binutils-arm-linux-gnueabihf \
+          libc6-dbg \
+          openjdk-8-jre-headless
+
+    - name: Install Android AVD
+      run: |
+        USER=travis source travis/setup_avd.sh
+        sed -i 's/skip_android = True/skip_android = False/' docs/source/conf.py
+        set | egrep '^(ANDROID|PATH)' >.android.env
+
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        python setup.py egg_info
+        pip install --upgrade --editable .
+
+    - name: Sanity checks
+      run:  PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+
+    - name: Install documentation dependencies
+      run:  pip install -r docs/requirements.txt
+
+    - name: Coverage Doctests (Android Only)
+      run: |
+        source .android.env
+        PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest docs/source/adb.rst

.github/workflows/changelog.yml

@@ -4,7 +4,7 @@ env:
   GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
 
 jobs:
-  build:
+  check:
     strategy:
       matrix:
         os: [ubuntu-latest]
@@ -20,7 +20,7 @@ jobs:
         if git log --stat ${GITHUB_BASE}..HEAD | grep CHANGELOG.md; then
            echo 'Changelog updated :D'
         else
-           if git log --stat ${GITHUB_BASE}..HEAD | grep '++\|--'; then
+           if git log --stat ${GITHUB_BASE}..HEAD pwnlib pwn | grep '++\|--'; then
               echo 'Major changes detected, changelog required!'
               false
            else

.github/workflows/ci.yml

@@ -14,8 +14,10 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Git History 2
-      run: git log --oneline --graph -10
+    - name: Git History
+      run: |
+        git fetch origin
+        git log --oneline --graph -10
 
     - name: Cache for pip
       uses: actions/cache@v1
@@ -42,28 +44,7 @@ jobs:
         [ "$vsetup" = "$vpwnlib" ]
         [ "$GITHUB_TAG" = "$vsetup" ]
 
-    - name: Detect whether java is needed
-      id: java-needed
-      env:
-        GITHUB_REF: ${{ github.event.ref }}
-        GITHUB_COMMIT_RANGE: ${{ github.event.before }}...${{ github.event.after }}
-      run: |
-        if echo "$GITHUB_REF" | grep -Eq 'staging|tags'; then
-          echo "Found release or important branch ($GITHUB_REF), forcing tests."
-        elif ! git show "$GITHUB_COMMIT_RANGE" >/dev/null; then
-          echo 'Incorrect commit range, forcing android tests.'
-        elif git log --stat "$GITHUB_COMMIT_RANGE" | grep -iE 'android|\<adb\>'; then
-          echo 'Found Android-related commits, forcing tests.'
-        else
-          # clear files that cause Android doctests
-          : > docs/source/adb.rst > docs/source/protocols/adb.rst
-          exit 0
-        fi
-        echo ::set-output name=need::openjdk-8-jre-headless
-
     - name: Install Linux dependencies
-      env:
-        ANDROID_JRE: ${{ steps.java-needed.outputs.need }}
       run: |
         sudo apt-get update
         sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
@@ -73,11 +54,17 @@ jobs:
           binutils-aarch64-linux-gnu \
           binutils-arm-linux-gnueabihf \
           binutils-mips-linux-gnu \
+          binutils-msp430 \
           binutils-powerpc-linux-gnu \
+          binutils-s390x-linux-gnu \
           binutils-sparc64-linux-gnu \
           gcc-multilib \
-          libc6-dbg \
-          $ANDROID_JRE
+          libc6-dbg
+
+    - name: Install RPyC for GDB
+      run: |
+        sudo apt-get install -y python3-pip
+        /usr/bin/python3 -m pip install rpyc
 
     - name: Testing Corefiles
       run: |
@@ -87,12 +74,6 @@ jobs:
         cat /proc/sys/kernel/core_uses_pid
         ( cd $(mktemp -d); sh -c 'kill -11 $$' || true; ls -la ./*core* /var/crash/*.crash;) || true
 
-    - name: Install Android AVD
-      if:  steps.java-needed.outputs.need
-      run: |
-        USER=travis source travis/install.sh
-        set | egrep '^(ANDROID|PATH)' >.android.env
-
     - name: Set up SSH
       run: |
         travis/ssh_setup.sh
@@ -105,7 +86,7 @@ jobs:
         pip install --upgrade --editable .
 
     - name: Sanity checks
-      run:  PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+      run:  PWNLIB_NOTERM=1 python -bb -c 'from pwn import *; print(pwnlib.term.term_mode)'
 
     - name: Install documentation dependencies
       run:  pip install -r docs/requirements.txt
@@ -113,30 +94,29 @@ jobs:
     - name: Manually install non-broken Unicorn
       run:  pip install unicorn==1.0.2rc3
 
-    - name: Version Check
-      run:  PWNLIB_NOTERM=1 pwn version
+    - name: Disable yama ptrace_scope
+      run: |
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope # required by some gdb doctests
 
     - name: Coverage doctests
       run: |
-        source .android.env || :
-        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope # required by some gdb doctests
-        PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+        PWNLIB_NOTERM=1 python -bb -m coverage run -m sphinx -b doctest docs/source docs/build/doctest
 
     - name: Coverage running examples
       run: |
         export TERM=linux
         set -x
-        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit.py
-        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit2.py || : # can fail randomly?
-        coverage run examples/asm.py
-        coverage run examples/asm.py
-        coverage run examples/text.py
+        python -bb travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit.py
+        python -bb travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit2.py || : # can fail randomly?
+        python -bb -m coverage run examples/asm.py
+        python -bb -m coverage run examples/asm.py
+        python -bb -m coverage run examples/text.py
         # for f in examples/sigreturn_corefile_*.py; do coverage run "$f"; done # XXX something is wrong
 
     - name: Coverage running commandline tools
       run: |
         export TERM=linux
-        pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec coverage run -m pwnlib.commandline."$cmd" "$@" ) }
+        pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec python -bb -m coverage run -m pwnlib.commandline."$cmd" "$@" ) }
         set -x
 
         pwn cyclic 32

.github/workflows/pylint.yml

@@ -0,0 +1,37 @@
+name: PyLint
+on: [pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: PyLint
+      run: |
+        set -x
+        pip install pylint
+        pip install --upgrade -r requirements.txt
+        pylint --exit-zero --errors-only  pwnlib > current.txt
+        git fetch origin
+        git checkout origin/"$GITHUB_BASE_REF"
+        pylint --exit-zero --errors-only  pwnlib > base.txt
+        if diff base.txt current.txt | grep '>'; then
+          false
+        fi

.gitignore

@@ -17,3 +17,6 @@ venv
 *.core
 .coverage
 .coverage.*
+.idea
+__pycache__
+!.github

.travis.yml

@@ -19,9 +19,12 @@ addons:
     - binutils-aarch64-linux-gnu
     - binutils-arm-linux-gnueabihf
     - binutils-mips-linux-gnu
+    - binutils-msp430
     - binutils-powerpc-linux-gnu
+    - binutils-s390x-linux-gnu
     - binutils-sparc64-linux-gnu
     - bash-static
+    - python3-pip
 cache:
     - pip
     - directories:
@@ -40,6 +43,7 @@ before_install:
   - source travis/ssh_setup.sh
   - echo $-
 install:
+  - /usr/bin/python3 -m pip install rpyc # for GDB
   - pip install --upgrade pip
   - pip install --upgrade flake8 appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
   - python setup.py egg_info      # pypa/pip#6275

CHANGELOG.md

@@ -9,9 +9,14 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.5.0](#450)    | `dev`    | Dec 20, 2020 (planned)
-| [4.4.0](#440)    | `beta`   | Nov 20, 2020 (planned)
-| [4.3.0](#430)    | `stable` | Oct 20, 2020
+| [4.8.0](#480)    | `dev`    |
+| [4.7.0](#470)    | `beta`   |
+| [4.6.0](#460)    | `stable` | Jul 12, 2021
+| [4.5.1](#451)    |          | May 30, 2021
+| [4.5.0](#450)    |          | Apr 30, 2021
+| [4.4.0](#440)    |          | Mar 29, 2021
+| [4.3.1](#431)    |          | Nov 29, 2020
+| [4.3.0](#430)    |          | Oct 20, 2020
 | [4.2.0](#420)    |          | Jul 3, 2020
 | [4.1.7](#417)    |          | Jun 30, 2020
 | [4.1.5](#415)    |          | Jun 27, 2020
@@ -56,9 +61,102 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.5.0 (`dev`)
+## 4.8.0 (`dev`)
 
-## 4.4.0 (`beta`)
+
+
+## 4.7.0 (`beta`)
+
+- [#1733][1733] Update libc headers -> more syscalls available!
+- [#1876][1876] add `self.message` and change `sys.exc_type` to `sys.exec_info()` in PwnlibException
+- [#1877][1877] encoders error message handles when `avoid` is bytes in python3
+- [#1891][1891] Keep ROP gadgets when setting registers via setattr/call
+- [#1892][1892] Silence SIGPIPE error for "pwn phd"
+- [#1893][1893] Fix bytes warning in "pwn cyclic"
+- [#1897][1897] Add basic support for RISC-V
+- [#1903][1903] Add zsh completion script
+- [#1904][1904] Add bash completion script
+- [#1906][1906] Defer import of several modules to save on startup time
+- [#1921][1921] Add basic support for the bare-metal ARM specific toolchain
+
+[1733]: https://github.com/Gallopsled/pwntools/pull/1733
+[1876]: https://github.com/Gallopsled/pwntools/pull/1876
+[1877]: https://github.com/Gallopsled/pwntools/pull/1877
+[1891]: https://github.com/Gallopsled/pwntools/pull/1891
+[1892]: https://github.com/Gallopsled/pwntools/pull/1892
+[1893]: https://github.com/Gallopsled/pwntools/pull/1893
+[1897]: https://github.com/Gallopsled/pwntools/pull/1897
+[1903]: https://github.com/Gallopsled/pwntools/pull/1903
+[1904]: https://github.com/Gallopsled/pwntools/pull/1904
+[1906]: https://github.com/Gallopsled/pwntools/pull/1906
+[1921]: https://github.com/Gallopsled/pwntools/pull/1921
+
+## 4.6.0 (`stable`)
+
+- [#1429][1429] Add a mechanism for ret2csu (originally #1138)
+- [#1566][1566] Add `ignore_config` argument to `pwnlib.tubes.ssh` and improve `allow_agent` implementation
+- [#1652][1652] Add `process.readmem` and `process.writemem`
+- [#1739][1739] Add/fix shellcraft.linux.kill() / shellcraft.linux.killparent()
+- [#1746][1746] Prefer Python3 over Python2 for spawning remote processes over SSH
+- [#1751][1751] Fix process() with executable relative to cwd
+- [#1753][1753] major change: less unconditional imports in pwnlib
+- [#1776][1776] mips: do not use $t0 temporary variable in dupio
+- [#1846][1846] support launching GDB in more different terminals
+
+[1429]: https://github.com/Gallopsled/pwntools/pull/1429
+[1566]: https://github.com/Gallopsled/pwntools/pull/1566
+[1652]: https://github.com/Gallopsled/pwntools/pull/1652
+[1739]: https://github.com/Gallopsled/pwntools/pull/1739
+[1746]: https://github.com/Gallopsled/pwntools/pull/1746
+[1751]: https://github.com/Gallopsled/pwntools/pull/1751
+[1753]: https://github.com/Gallopsled/pwntools/pull/1753
+[1776]: https://github.com/Gallopsled/pwntools/pull/1776
+[1846]: https://github.com/Gallopsled/pwntools/pull/1846
+
+## 4.5.1
+
+- [#1902][1902] Always specify -F and -P for tmux in `run_in_new_terminal`
+
+[1902]: https://github.com/Gallopsled/pwntools/pull/1902
+
+## 4.5.0
+
+- [#1261][1261] Misc `run_in_new_terminal` improvements (notably gdb terminated by default)
+- [#1695][1695] Allow using GDB Python API
+- [#1735][1735] Python 3.9 support in safeeval
+- [#1738][1738] Which function support custom search path
+  - process also looks now at `env['PATH']` to find the path for the executable
+- [#1742][1742] New `baremetal` os to debug binaries executed with qemu-system-$(arch)
+- [#1757][1757] update cache directories
+- [#1758][1758] Remove eval from cli
+- [#1780][1780] Re-add Python2 to the official Dockerfile
+- [#1941][1941] Disable all Android tests, `pwnlib.adb` is no longer supported in CI
+- [#1811][1811] Remove unnecessary `pwn.toplevel.__all__`
+- [#1827][1827] Support `$XDG_CONFIG_HOME` dir for `pwn.conf`
+- [#1841][1841] Add colored_traceback
+- [#1839][1839] run_in_new_terminal now creates a runner script if given a list or tuple
+- [#1833][1833] Add pwnlib.filesystem module
+- [#1852][1852] Fix `atexit` on Python 3
+- [#1883][1883] ROP gadget verifier accounts for 2 character registers
+
+[1261]: https://github.com/Gallopsled/pwntools/pull/1261
+[1695]: https://github.com/Gallopsled/pwntools/pull/1695
+[1735]: https://github.com/Gallopsled/pwntools/pull/1735
+[1738]: https://github.com/Gallopsled/pwntools/pull/1738
+[1742]: https://github.com/Gallopsled/pwntools/pull/1742
+[1757]: https://github.com/Gallopsled/pwntools/pull/1757
+[1758]: https://github.com/Gallopsled/pwntools/pull/1758
+[1780]: https://github.com/Gallopsled/pwntools/pull/1780
+[1941]: https://github.com/Gallopsled/pwntools/pull/1941
+[1811]: https://github.com/Gallopsled/pwntools/pull/1811
+[1827]: https://github.com/Gallopsled/pwntools/pull/1827
+[1841]: https://github.com/Gallopsled/pwntools/pull/1841
+[1839]: https://github.com/Gallopsled/pwntools/pull/1839
+[1833]:  https://github.com/Gallopsled/pwntools/pull/1833
+[1852]: https://github.com/Gallopsled/pwntools/pull/1852
+[1883]: https://github.com/Gallopsled/pwntools/pull/1883
+
+## 4.4.0
 
 - [#1541][1541] Use `context.newline` for tubes by default
 - [#1602][1602] Fix bytes handling in ssh tubes
@@ -80,6 +178,7 @@ The table below shows which release corresponds to each branch, and what date th
 - [#1688][1688] Add `__setattr__` and `__call__` interfaces to `ROP` for setting registers
 - [#1692][1692] Remove python2 shebangs where appropriate
 - [#1703][1703] Update libcdb buildid offsets for amd64 and i386
+- [#1704][1704] Try https://libc.rip/ for libcdb lookup
 
 [1541]: https://github.com/Gallopsled/pwntools/pull/1541
 [1602]: https://github.com/Gallopsled/pwntools/pull/1602
@@ -99,8 +198,15 @@ The table below shows which release corresponds to each branch, and what date th
 [1688]: https://github.com/Gallopsled/pwntools/pull/1688
 [1692]: https://github.com/Gallopsled/pwntools/pull/1692
 [1703]: https://github.com/Gallopsled/pwntools/pull/1703
+[1704]: https://github.com/Gallopsled/pwntools/pull/1704
 
-## 4.3.0 (`stable`)
+## 4.3.1
+
+- [#1732][1732] Fix shellcraft SSTI vulnerability (first major pwntools vuln!)
+
+[1732]: https://github.com/Gallopsled/pwntools/pull/1732
+
+## 4.3.0
 
 - [#1576][1576] Add `executable=` argument to `ELF.search`
 - [#1584][1584] Add `jmp_esp`/`jmp_rsp` attribute to `ROP`
@@ -198,7 +304,7 @@ The table below shows which release corresponds to each branch, and what date th
 [1001]: https://github.com/Gallopsled/pwntools/pull/1001
 [1389]: https://github.com/Gallopsled/pwntools/pull/1389
 [1241]: https://github.com/Gallopsled/pwntools/pull/1241
-[1218]: https://github.com/Gallopsled/pwntools/pull/1218  
+[1218]: https://github.com/Gallopsled/pwntools/pull/1218
 
 ## 4.0.1
 

DOCKER.md

@@ -0,0 +1,76 @@
+# Using Pwntools with Docker
+
+Sometimes it's annoying to set up Pwntools on your workstation, and you want something that Just Works (TM).
+
+[Docker](https://www.docker.com/) is here to the rescue! Using Docker means that you get a nice, standardized Linux environment and don't need to worry about pip or installing dependencies.
+
+## Quick Start
+
+First, install Docker for your OS, which you can find on their [Getting Started](https://www.docker.com/get-started) page.
+
+Next, download and run the Pwntools stable docker image.
+
+```sh
+$ docker run -it pwntools/pwntools:stable
+```
+
+## Recommended Settings
+
+In order to get the most from your docker image, we need to enable debugging of processes (`--privileged`) and expose the network ports from the guest to the host (`--net=host`).
+
+```sh
+$ docker run -it \
+    --privileged \
+    --net=host \
+    --hostname localhost \
+    --ulimit core=-1:-1 \
+    pwntools/pwntools:stable
+```
+
+## Sharing a Folder
+
+It's really nice to be able to use your preferred native editor, and have the changes show up live inside your Docker image.  This is easy to add, thanks to Docker's bind mounts (`--mount type=bind`).  
+
+With the command below, your `~/exploits` directory will magically show up inside the Docker image at `/home/pwntools/exploits` so that you can easily run them (from Docker) and edit them (from outside Docker).
+
+```sh
+$ mkdir $HOME/exploits
+
+$ vim $HOME/exploits/my_exploit.py
+
+$ docker run -it \
+    --privileged \
+    --net=host \
+    --hostname localhost \
+    --ulimit core=-1:-1 \
+    --mount type=bind,source="$HOME/exploits",target=/home/pwntools/exploits \
+    pwntools/pwntools:stable
+    
+$ python3 exploits/my_exploit.py
+```
+
+### Windows User Bind Mounts
+
+If you're a Windows user `$HOME` doesn't exist in the same way as on Linux, instead it is `%UserProfile%`.  The command from above would look like this, assuming your editor is Visual Studio Code and you have code.exe in your `%PATH%`.
+
+```sh
+C:\Users\user> mkdir Desktop\exploits
+
+C:\Users\user> code Desktop\exploits\my_exploit.py
+
+C:\Users\user> docker run -it \
+    --privileged \
+    --net=host \
+    --hostname localhost \
+    --ulimit core=-1:-1 \
+    --mount type=bind,source="%UserProfile%\Desktop\exploits",target=/home/pwntools/exploits \
+    pwntools/pwntools:stable
+    
+$ python3 exploits/my_exploit.py
+
+```
+
+
+
+
+

README.md

@@ -4,9 +4,10 @@
 [![PyPI](https://img.shields.io/pypi/v/pwntools?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
 [![Travis](https://img.shields.io/travis/Gallopsled/pwntools/dev?logo=Travis)](https://travis-ci.org/Gallopsled/pwntools)
-![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/Gallopsled/pwntools/Continuous%20Integration/dev?logo=GitHub)
+[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/Gallopsled/pwntools/Continuous%20Integration/dev?logo=GitHub)](https://github.com/Gallopsled/pwntools/actions?query=workflow%3A%22Continuous+Integration%22+branch%3Adev)
 [![Coveralls](https://img.shields.io/coveralls/github/Gallopsled/pwntools/dev?logo=coveralls)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
+[![Discord](https://img.shields.io/discord/809590285687980052?label=Discord&style=plastic)](https://discord.gg/96VA2zvjCB)
 [![Twitter](https://img.shields.io/twitter/follow/Pwntools)](https://twitter.com/pwntools)
 
 Pwntools is a CTF framework and exploit development library. Written in Python, it is designed for rapid prototyping and development, and intended to make exploit writing as simple as possible.
@@ -25,13 +26,15 @@ r.interactive()
 
 Our documentation is available at [docs.pwntools.com](https://docs.pwntools.com/)
 
+A series of tutorials is also [available online](https://github.com/Gallopsled/pwntools-tutorial#readme)
+
 To get you started, we've provided some example solutions for past CTF challenges in our [write-ups repository](https://github.com/Gallopsled/pwntools-write-ups).
 
 # Installation
 
-Pwntools is best supported on 64-bit Ubuntu LTE releases (14.04, 16.04, 18.04, and 20.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  Python >= 2.7 is required (Python 3 suggested as best).
+Pwntools is best supported on 64-bit Ubuntu LTS releases (14.04, 16.04, 18.04, and 20.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  
 
-Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
+Python3 is suggested, but Pwntools still works with Python 2.7.  Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
 
 ```sh
 apt-get update
@@ -40,6 +43,7 @@ python3 -m pip install --upgrade pip
 python3 -m pip install --upgrade pwntools
 ```
 
+
 However, some of the features (assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://docs.pwntools.com/en/stable/install.html).
 
 
@@ -47,8 +51,5 @@ However, some of the features (assembling/disassembling foreign architectures) r
 
 See [CONTRIBUTING.md](CONTRIBUTING.md)
 
-# Contact
-If you have any questions not worthy of a [bug report](https://github.com/Gallopsled/pwntools/issues), feel free to ping us
-at [`#pwntools` on Freenode](irc://irc.freenode.net/pwntools) and ask away.
-Click [here](https://kiwiirc.com/client/irc.freenode.net/pwntools) to connect.
-There is also a [mailing list](https://groups.google.com/forum/#!forum/pwntools-users) for higher latency discussion.
+# Contact and Community
+If you have any questions not worthy of a [bug report](https://github.com/Gallopsled/pwntools/issues), join the Discord server at https://discord.gg/96VA2zvjCB

docs/Makefile

@@ -45,7 +45,7 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 	-rm -rf /tmp/user/pwn*
-	-rm -rf $(HOME)/.pwntools-cache
+	-rm -rf $(HOME)/.cache/.pwntools-cache
 	-rm -rf /tmp/pwn* 2>/dev/null || true
 	-rm -rf build
 

docs/requirements.txt

@@ -11,7 +11,7 @@ pypandoc
 pyserial>=2.7
 requests>=2.5.1
 ropgadget>=5.3
-sphinx
+sphinx<3.4
 sphinx_rtd_theme
 sphinxcontrib-napoleon
-sphinxcontrib-autoprogram
+sphinxcontrib-autoprogram<=0.1.5

docs/source/asm.rst

@@ -2,12 +2,7 @@
 
    import tempfile
    import subprocess
-   from pwnlib.context import context
-   from pwnlib.asm import *
-   from pwnlib import shellcraft
-   from pwnlib.tubes.process import process
-   from pwnlib.util.fiddling import unhex
-
+   from pwn import *
 
 :mod:`pwnlib.asm` --- Assembler functions
 =========================================

docs/source/commandline.rst

@@ -12,6 +12,8 @@ Command Line Tools
 
 pwntools comes with a handful of useful command-line utilities which serve as wrappers for some of the internal functionality.
 
+If these tools do not appear to be installed, make sure that you have added ``~/.local/bin`` to your ``$PATH`` environment variable.
+
 .. toctree::
 
 .. autoprogram:: pwnlib.commandline.main:parser

docs/source/conf.py

@@ -25,7 +25,8 @@ build_dash = tags.has('dash')
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('../..'))
 
-import pwnlib
+import pwnlib.update
+pwnlib.update.disabled = True
 
 # If your documentation needs a minimal Sphinx version, state it here.
 #needs_sphinx = '1.0'
@@ -70,10 +71,14 @@ doctest_global_setup = '''
 import sys, os
 os.environ['PWNLIB_NOTERM'] = '1'
 os.environ['PWNLIB_RANDOMIZE'] = '0'
-import pwnlib, logging
+import pwnlib.update
+import pwnlib.util.fiddling
+import logging
+pwnlib.update.disabled = True
 pwnlib.context.context.reset_local()
 pwnlib.context.ContextType.defaults['log_level'] = logging.ERROR
 pwnlib.context.ContextType.defaults['randomize'] = False
+# pwnlib.context.ContextType.defaults['terminal'] = ['sh', '-c']
 pwnlib.util.fiddling.default_style = {}
 pwnlib.term.text.when = 'never'
 pwnlib.log.install_default_handler()
@@ -91,6 +96,9 @@ pwnlib.context.ContextType.defaults['log_console'] = stdout()
 
 github_actions = os.environ.get('USER') == 'runner'
 travis_ci = os.environ.get('USER') == 'travis'
+local_doctest = os.environ.get('USER') == 'pwntools'
+branch_dev = os.environ.get('GITHUB_BASE_REF') == 'dev'
+skip_android = True
 '''
 
 autoclass_content = 'both'
@@ -257,7 +265,7 @@ latex_documents = [
    u'2016, Gallopsled et al.', 'manual'),
 ]
 
-intersphinx_mapping = {'python': ('https://docs.python.org/2.7', None),
+intersphinx_mapping = {'python': ('https://docs.python.org/3.8', None),
                        'paramiko': ('https://paramiko-docs.readthedocs.org/en/2.1/', None)}
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -381,7 +389,9 @@ import sphinx.ext.autodoc
 
 # Test hidden members (e.g. def _foo(...))
 def dont_skip_any_doctests(app, what, name, obj, skip, options):
-    return False
+    return None
+
+autodoc_default_options = {'special-members': None, 'private-members': None}
 
 class _DummyClass(object): pass
 
@@ -403,7 +413,14 @@ class Py2OutputChecker(_DummyClass, doctest.OutputChecker):
         if sup(rly_want, got, optionflags):
             return True
         rly_want = ' '.join(x[:2].replace('b"',' "').replace("b'"," '")+x[2:] for x in want.replace('\n','\n ').split(' ')).replace('\n ','\n')
-        return sup(rly_want, got, optionflags)
+        if sup(rly_want, got, optionflags):
+            return True
+        for wantl, gotl in six.moves.zip_longest(want.splitlines(), got.splitlines(), fillvalue=''):
+            rly_want1 = '['.join(x[:2].replace('b"','"').replace("b'","'")+x[2:] for x in wantl.split('['))
+            rly_want2 = ' '.join(x[:2].replace('b"',' "').replace("b'"," '")+x[2:] for x in wantl.split(' '))
+            if not sup(rly_want1, gotl, optionflags) and not sup(rly_want2, gotl, optionflags):
+                return False
+        return True
 
 def py2_doctest_init(self, checker=None, verbose=None, optionflags=0):
     if checker is None:
@@ -412,8 +429,20 @@ def py2_doctest_init(self, checker=None, verbose=None, optionflags=0):
 
 if 'doctest' in sys.argv:
     def setup(app):
-        app.connect('autodoc-skip-member', dont_skip_any_doctests)
+        pass # app.connect('autodoc-skip-member', dont_skip_any_doctests)
 
     if sys.version_info[:1] < (3,):
         import sphinx.ext.doctest
         sphinx.ext.doctest.SphinxDocTestRunner.__init__ = py2_doctest_init
+    else:
+        # monkey patching paramiko due to https://github.com/paramiko/paramiko/pull/1661
+        import paramiko.client
+        import binascii
+        paramiko.client.hexlify = lambda x: binascii.hexlify(x).decode()
+        paramiko.util.safe_string = lambda x: '' # function result never *actually used*
+    class EndlessLoop(Exception): pass
+    def alrm_handler(sig, frame):
+        signal.alarm(180) # three minutes
+        raise EndlessLoop()
+    signal.signal(signal.SIGALRM, alrm_handler)
+    signal.alarm(600) # ten minutes

docs/source/elf/corefile.rst

@@ -3,6 +3,22 @@
    from glob import glob
    from pwn import *
 
+   # The Linux kernel won't overwrite an existing corefile, so in case 
+   # some other part of the doctests caused a segfault and core dump,
+   # we need to get rid of it before our tests run.
+   #
+   # We DONT need to worry about e.g. ./core existing when using Corefile()
+   # because we always move and rename the corefile to prevent this situation.
+   if os.path.exists('core'): 
+      os.unlink('core')
+
+   # bash-static is a statically linked version of bash, but if $SHELL is not
+   # set to anything, it decides to up and load ld.so and libc.so which breaks
+   # our example of showing `corefile.libc == None` for a statically linked bin.
+   # Set the environment here so it's not in the middle of our tests.
+   os.environ.setdefault('SHELL', '/bin/sh')
+
+
 :mod:`pwnlib.elf.corefile` --- Core Files
 ===========================================================
 

docs/source/encoders.rst

@@ -1,13 +1,18 @@
 .. testsetup:: *
 
    from pwn import *
-
+   
 :mod:`pwnlib.encoders` --- Encoding Shellcode
 ===============================================
 
 .. automodule:: pwnlib.encoders.encoder
    :members:
 
+.. automodule:: pwnlib.encoders.i386.ascii_shellcode
+   :members:
+   :special-members:
+   :exclude-members: __init__
+
 .. automodule:: pwnlib.encoders.i386.xor
    :members:
 

docs/source/filesystem.rst

@@ -0,0 +1,17 @@
+.. testsetup:: *
+
+    import time
+    import six
+    from pwnlib.context import context
+    from pwnlib.tubes.ssh import ssh
+    from pwnlib.filesystem import *
+
+:mod:`pwnlib.filesystem` --- Manipulating Files Locally and Over SSH
+====================================================================
+
+Provides a Python2-compatible :py:mod:`pathlib` interface for paths
+on the local filesystem (`.Path`) as well as on remote filesystems,
+via SSH (`.SSHPath`).
+
+.. automodule:: pwnlib.filesystem
+   :members:

docs/source/index.rst

@@ -55,6 +55,7 @@ Each of the ``pwntools`` modules is documented here.
    elf
    exception
    filepointer
+   filesystem
    flag
    fmtstr
    gdb
@@ -91,3 +92,12 @@ Each of the ``pwntools`` modules is documented here.
    * :ref:`genindex`
    * :ref:`modindex`
    * :ref:`search`
+
+Bytes
+-----
+
+The bytes vs text distinction is so important that it even made it to this main page.
+See the pwntools-tutorial_ repo for the latest tutorial finally
+explaining the difference once and for all (hopefully).
+
+.. _pwntools-tutorial: https://github.com/Gallopsled/pwntools-tutorial/blob/master/bytes.md

docs/source/install.rst

@@ -1,9 +1,7 @@
 Installation
 ============
 
-pwntools is best supported on Ubuntu 12.04 and 14.04, but most
-functionality should work on any Posix-like distribution (Debian, Arch,
-FreeBSD, OSX, etc.).
+Pwntools is best supported on 64-bit Ubuntu LTS releases (14.04, 16.04, 18.04, and 20.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).
 
 Prerequisites
 -------------
@@ -20,7 +18,10 @@ following system libraries installed.
 Released Version
 -----------------
 
-pwntools is available as a ``pip`` package.
+pwntools is available as a ``pip`` package for both Python2 and Python3.
+
+Python3
+^^^^^^^
 
 .. code-block:: bash
 
@@ -29,6 +30,37 @@ pwntools is available as a ``pip`` package.
     $ python3 -m pip install --upgrade pip
     $ python3 -m pip install --upgrade pwntools
 
+
+Python2 (Deprecated)
+^^^^^^^^^^^^^^^^^^^^
+
+NOTE: Pwntools maintainers STRONGLY recommend using Python3 for all future Pwntools-based scripts and projects.
+
+Additionally, due to `pip` dropping support for Python2, a specfic version of `pip` must be installed.
+
+.. code-block:: bash
+
+    $ apt-get update
+    $ apt-get install python python-pip python-dev git libssl-dev libffi-dev build-essential
+    $ python2 -m pip install --upgrade pip==20.3.4
+    $ python2 -m pip install --upgrade pwntools
+
+
+Command-Line Tools
+------------------
+
+When installed with ``sudo`` the above commands will install Pwntools' command-line tools to somewhere like ``/usr/bin``.
+
+However, if you run as an unprivileged user, you may see a warning message that looks like this:
+
+.. code-block::
+
+      WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
+      elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
+      unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
+
+Follow the instructions listed and add ``~/.local/bin`` to your ``$PATH`` environment variable.
+
 Development
 --------------
 

docs/source/install/binutils.rst

@@ -12,7 +12,7 @@ Building `binutils` from source takes about 60 seconds on a modern 8-core machin
 Ubuntu
 ^^^^^^^^^^^^^^^^
 
-For Ubuntu 12.04 through 15.10, you must first add the pwntools `Personal Package Archive repository <http://binutils.pwntools.com>`__.
+For Ubuntu 12.04 through 15.10, you must first add the pwntools `Personal Package Archive repository <https://launchpad.net/~pwntools/+archive/ubuntu/binutils>`__.
 
 Ubuntu Xenial (16.04) has official packages for most architectures, and does not require this step.
 

docs/source/intro.rst

@@ -17,6 +17,13 @@ assemble, disassemble, pack, unpack, and many other things with a single functio
 A full list of everything that is imported is available on :doc:`globals`.
 
 
+Tutorials
+---------
+
+A series of tutorials for Pwntools exists online, at
+https://github.com/Gallopsled/pwntools-tutorial#readme
+
+
 Making Connections
 ------------------
 
@@ -188,10 +195,10 @@ Never write another hexdump, thanks to :mod:`pwnlib.util.fiddling`.
 
 Find offsets in your buffer that cause a crash, thanks to :mod:`pwnlib.cyclic`.
 
-    >>> print(cyclic(20).decode())
-    aaaabaaacaaadaaaeaaa
+    >>> cyclic(20)
+    b'aaaabaaacaaadaaaeaaa'
     >>> # Assume EIP = 0x62616166 (b'faab' which is pack(0x62616166))  at crash time
-    >>> print(cyclic_find(b'faab'))
+    >>> cyclic_find(b'faab')
     120
 
 ELF Manipulation

docs/source/protocols.rst

@@ -1,15 +1 @@
-:mod:`pwnlib.protocols` --- Wire Protocols
-=============================================
-
-.. automodule:: pwnlib.protocols
-
-
-Supported Protocols
--------------------
-
-.. toctree::
-    :maxdepth: 3
-    :glob:
-
-    protocols/*
 

docs/source/protocols/adb.rst

@@ -1,11 +0,0 @@
-.. testsetup:: *
-
-   from pwn import *
-   from pwnlib.protocols.adb import AdbClient
-   AdbClient().wait_for_device()
-
-:mod:`pwnlib.protocols.adb` --- ADB Protocol Implementation
-===========================================================
-
-.. automodule:: pwnlib.protocols.adb
-   :members:

docs/source/rop.rst

@@ -1,7 +1,3 @@
-.. testsetup:: *
-
-   from pwn import *
-
 :mod:`pwnlib.rop` --- Return Oriented Programming
 =================================================
 

docs/source/rop/call.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+   from pwnlib.rop.call import *
+
+:mod:`pwnlib.rop.call` --- Return Oriented Programming Function Calling
+=======================================================================
+
+.. automodule:: pwnlib.rop.call
+   :members:

docs/source/rop/rop.rst

@@ -1,6 +1,23 @@
 .. testsetup:: *
 
-   from pwn import *
+   import time
+   from glob import glob
+
+   from pwnlib.asm import asm
+   from pwnlib import constants
+   from pwnlib.context import context
+   from pwnlib.elf import ELF
+   from pwnlib.rop import ROP
+   from pwnlib.rop.call import Call, AppendedArgument
+   from pwnlib.elf.maps import CAT_PROC_MAPS_EXIT
+   from pwnlib.util.packing import *
+   from pwnlib.util.fiddling import *
+   from pwnlib.tubes.process import process
+   from pwnlib import shellcraft
+   from pwnlib.util.misc import which
+
+   context.clear()
+
 
 :mod:`pwnlib.rop.rop` --- Return Oriented Programming
 ==========================================================

docs/source/rop/srop.rst

@@ -1,6 +1,11 @@
 .. testsetup:: *
 
-   from pwn import *
+   from pwnlib.rop.srop import *
+
+   from pwnlib import constants
+   from pwnlib import shellcraft
+   from pwnlib.elf import ELF
+   from pwnlib.tubes.process import process
 
 :mod:`pwnlib.rop.srop` --- Sigreturn Oriented Programming
 ==========================================================

docs/source/runner.rst

@@ -1,6 +1,7 @@
 .. testsetup:: *
 
-   from pwn import *
+   from pwnlib.runner import *
+   from pwnlib.asm import asm
 
 :mod:`pwnlib.runner` --- Running Shellcode
 ===========================================

docs/source/shellcraft/mips.rst

@@ -1,6 +1,15 @@
 .. testsetup:: *
 
-   from pwn import *
+   import tempfile
+
+   from pwnlib import shellcraft
+
+   from pwnlib.asm import asm
+   from pwnlib.context import context
+   from pwnlib.runner import run_assembly
+   from pwnlib.util.fiddling import enhex
+   from pwnlib.util.misc import write
+
    context.clear(arch='mips')
 
 :mod:`pwnlib.shellcraft.mips` --- Shellcode for MIPS

docs/source/tubes/ssh.rst

@@ -14,6 +14,10 @@
       :members: kill, poll, interactive
       :show-inheritance:
 
+   .. autoclass:: pwnlib.tubes.ssh.ssh_process
+      :members:
+      :show-inheritance:
+
    .. autoclass:: pwnlib.tubes.ssh.ssh_connecter()
       :show-inheritance:
 

docs/source/util/misc.rst

@@ -3,7 +3,6 @@
    from pprint import pprint
    from pwnlib.util.misc import *
    import os, subprocess
-   os.chdir("..")
 
 :mod:`pwnlib.util.misc` --- We could not fit it any other place
 ===============================================================

docs/source/util/net.rst

@@ -2,7 +2,6 @@
 
    from pwnlib.util.net import *
 
-
 :mod:`pwnlib.util.net` --- Networking interfaces
 ===================================================
 

docs/source/util/web.rst

@@ -1,5 +1,7 @@
 .. testsetup:: *
 
+   import tempfile
+
    from pwnlib.context import context
    from pwnlib.util.web import *
 

examples/gdb_api.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""An example of using GDB Python API with Pwntools."""
+from pwn import *
+
+
+def check_write(gdb, exp_buf):
+    """Check that write() was called with the expected arguments."""
+    fd = gdb.parse_and_eval('$rdi').cast(gdb.lookup_type('int'))
+    assert fd == 1, fd
+    buf_addr = gdb.parse_and_eval('$rsi').cast(gdb.lookup_type('long'))
+    count = gdb.parse_and_eval('$rdx').cast(gdb.lookup_type('long'))
+    buf = gdb.selected_inferior().read_memory(buf_addr, count).tobytes()
+    assert buf == exp_buf, buf
+
+
+def demo_sync_breakpoint(cat, gdb, txt):
+    """Demonstrate a synchronous breakpoint."""
+    # set the synchronous breakpoint on ``write``
+    gdb.Breakpoint('write', temporary=True)
+
+    # resume the program
+    gdb.continue_nowait()
+
+    # send the line
+    cat.sendline(txt)
+
+    # wait until we hit the breakpoint
+    gdb.wait()
+
+    # inspect program state
+    check_write(gdb, (txt + '\n').encode())
+
+    # resume the program
+    gdb.continue_nowait()
+
+    # expect to observe the line we just sent
+    cat.recvuntil(txt)
+
+
+def demo_async_breakpoint(cat, gdb, txt):
+    """Demonstrate asynchronous breakpoint."""
+    # set the asynchronous breakpoint on ``write``
+    class WriteBp(gdb.Breakpoint):
+        def __init__(self):
+            super().__init__('write')
+            self.count = 0
+
+        def stop(self):
+            # called in a separate thread
+            check_write(gdb, (txt + '\n').encode())
+            self.count += 1
+
+    bp = WriteBp()
+
+    # resume the program
+    gdb.continue_nowait()
+
+    # send the line and immediately expect to observe it
+    cat.sendline(txt)
+    cat.recvuntil(txt)
+
+    # check that we hit the breakpoint
+    assert bp.count == 1, bp.count
+
+    # interrupt the program
+    gdb.interrupt_and_wait()
+
+    # delete the breakpoint
+    bp.delete()
+
+    # resume the program
+    gdb.continue_nowait()
+
+
+def main():
+    # start ``cat`` under GDB
+    with gdb.debug('cat', gdbscript='''
+set logging on
+set pagination off
+''', api=True) as cat:
+
+        # the process is stopped
+        # set the synchronous breakpoint on ``read``
+        cat.gdb.Breakpoint('read', temporary=True)
+
+        # resume and wait until we hit it
+        cat.gdb.continue_and_wait()
+
+        # demonstrate a more interesting synchronous breakpoint
+        demo_sync_breakpoint(cat, cat.gdb, 'foo')
+
+        # terminate GDB
+        cat.gdb.quit()
+
+    # now start ``cat`` normally
+    with process('cat') as cat:
+        # attach GDB
+        _, cat_gdb = gdb.attach(cat, gdbscript='''
+set logging on
+set pagination off
+''', api=True)
+
+        # the process is stopped
+        # demonstrate asynchronous breakpoint
+        demo_async_breakpoint(cat, cat_gdb, 'bar')
+
+        # terminate GDB
+        cat_gdb.quit()
+
+
+if __name__ == '__main__':
+    main()

extra/docker/Makefile

@@ -1,13 +1,34 @@
-subdirs=xenial stable $(wildcard */)
-makefiles=$(wildcard */Makefile)
+subdirs=$(shell find . -type d -depth 1 | xargs basename)
 tags=base stable beta dev
 
-all: $(makefiles) $(subdirs)
+all: $(subdirs) $(tags)
 
-$(subdirs):
+tags: $(tags)
+
+base:
+	@echo "Building $@"
 	docker build -t pwntools/pwntools:$(shell basename $(@)) "$@"
 
-$(makefiles):
-	make -C $(shell dirname $@)
+stable beta dev: base
+	@echo "Building $@"
+	docker build -t pwntools/pwntools:$(shell basename $(@)) "$@"
 
-.PHONY: all $(subdirs) $(makefiles)
+develop:
+	@echo "Building $@"
+	docker build -t pwntools/pwntools:$(shell basename $(@)) "$@"
+	docker run -it --privileged --net=host --hostname localhost \
+		--ulimit core=-1:-1 \
+		--mount type=bind,source="$(shell git rev-parse --show-toplevel)",target=/home/pwntools/pwntools \
+		pwntools/pwntools:$(shell basename $(@))
+
+debug:
+	@echo "Subdirs   $(subdirs)"
+	@echo "Tags      $(tags)"
+
+clean:
+	docker container prune
+	docker
+
+FORCE:
+
+.PHONY: all $(subdirs) $(makefiles) $(tags)

extra/docker/README.md

@@ -0,0 +1,53 @@
+# Pwntools Docker Images
+
+This directory contains some Docker images with various versions of Pwntools pre-installed.
+
+## Getting Images
+
+You can either make them yourself, or pull from Docker Hub.
+
+### Docker Hub
+
+To pull an image from Docker hub, just run the command:
+
+```sh
+$ docker pull pwntools/pwntools:stable
+```
+
+Replace `stable` with `beta` or `dev` as desired.
+
+### Building Locally
+
+To build the images locally, just invoke the `Makefile` with the desired branch target.
+
+```sh
+$ make -C extra/docker dev
+```
+
+## Running Images
+
+To run the `stable`, `beta`, or `dev` image, we recommend the following command line, in this example it is the `dev` image.
+
+```sh
+$ docker run -it \
+    --privileged \
+    --net=host \
+ 		--ulimit core=-1:-1 \
+ 		pwntools/pwntools:dev
+```
+
+The `--privileged` option is needed to perform any debugging with GDB.
+
+The `--ulimit` option is needed for any corefiles to drop when a program crashes.
+
+The `--net=host` allows you to easily connect to services hosted inside the container from outside the container.
+
+## Development Dockerfile
+
+In addition to `stable`, `beta`, and `dev` Dockerfiles, there is also a `develop` dockerfile which mounts your Pwntools installation inside the VM so you can develop seamlessly while editing on your host.
+
+```sh
+$ make -C extra/docker develop
+```
+
+You will be dropped into a shell, and any changes you make to your local Pwntools checkout will be immediately reflected inside the Docker container.  This is useful for e.g. editing on a macOS host, but testing your changes on Ubuntu without needing to copy files around.

extra/docker/base

@@ -1 +0,0 @@
-bionic

extra/docker/base/Dockerfile

@@ -6,17 +6,32 @@
 FROM ubuntu:bionic
 MAINTAINER Maintainer Gallopsled et al.
 
+env DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get update \
     && apt-get install -y \
         build-essential \
         git \
         libssl-dev \
         libffi-dev \
-        python2.7 \
+        python \
         python-pip \
         python-dev \
+        python3 \
+        python3-pip \
+        python3-dev \
+        qemu-user-static \
+        binutils-arm-linux-gnueabihf \
+        binutils-aarch64-linux-gnu \
+        binutils-mips-linux-gnu \
+        binutils-mipsel-linux-gnu \
+        binutils-powerpc-linux-gnu \
+        binutils-powerpc64-linux-gnu \
+        binutils-sparc64-linux-gnu \
     && pip install --upgrade pip \
     && python -m pip install --upgrade pwntools \
+    && pip3 install --upgrade pip \
+    && python3 -m pip install --upgrade pwntools \
     && PWNLIB_NOTERM=1 pwn update \
     && apt-get install -y sudo \
     && useradd -m pwntools \

extra/docker/beta/Dockerfile

@@ -2,5 +2,6 @@ FROM pwntools/pwntools:stable
 
 USER root
 RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@beta
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/dev/Dockerfile

@@ -2,5 +2,6 @@ FROM pwntools/pwntools:stable
 
 USER root
 RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/develop/Dockerfile

@@ -0,0 +1,59 @@
+FROM pwntools/pwntools:base
+
+# Uninstall existing versions of pwntools
+USER root
+RUN python -m pip uninstall -q -y pwntools \
+ && python3 -m pip uninstall -q -y pwntools
+
+# Switch back to the pwntools user from here forward
+USER pwntools
+WORKDIR /home/pwntools
+
+# Since we are not installing Pwntools systemwide, the "pwn" binaries
+# etc will all end up in this path.
+ENV PATH="/home/pwntools/.local/bin:${PATH}"
+
+# Install Pwntools to the home directory, make it an editable install
+RUN git clone https://github.com/Gallopsled/pwntools \
+ && python  -m pip install --upgrade --editable pwntools \
+ && python3 -m pip install --upgrade --editable pwntools \
+ && PWNLIB_NOTERM=1 pwn version
+
+# Requirements for running the tests
+RUN python  -m pip install --upgrade --requirement pwntools/docs/requirements.txt \
+ && python3 -m pip install --upgrade --requirement pwntools/docs/requirements.txt
+
+# Dependencies from .travis.yml addons -> apt -> packages
+RUN sudo apt-get install -y \
+	ash \
+	bash \
+	bash-static \
+	binutils-msp430 \
+	binutils-multiarch \
+	binutils-s390x-linux-gnu \
+	dash \
+	gcc \
+	gcc-multilib \
+	gdb \ 
+	ksh \
+	lib32stdc++6 \
+	libc6-dev-i386 \
+	mksh \
+	pandoc \
+	qemu-user-static \
+	socat \
+	sshpass \
+	zsh
+
+# Misc useful things when developing
+RUN sudo apt-get install -y  \
+	curl \
+	ipython \
+	ipython3 \
+	lsb-release \
+	ssh \
+	unzip \
+	wget
+
+# Do not require password for sudo
+RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/travis

extra/docker/kali/Dockerfile

@@ -1,26 +0,0 @@
-############################################################
-# Dockerfile to build Pwntools container
-# Based on Ubuntu
-############################################################
-
-FROM kalilinux/kali-linux-docker
-MAINTAINER Maintainer Gallopsled et al.
-
-RUN apt-get update \
-    && apt-get install -y \
-        build-essential \
-        git \
-        libssl-dev \
-        libffi-dev \
-        python2.7 \
-        python-pip \
-        python-dev \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
-    && useradd -m pwntools \
-    && passwd --delete --unlock pwntools \
-    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
-USER pwntools
-WORKDIR /home/pwntools

extra/docker/kali/Makefile

@@ -1,8 +0,0 @@
-all: Dockerfile
-
-Dockerfile: ../xenial/Dockerfile FORCE
-	sed "s|ubuntu:xenial|kalilinux/kali-linux-docker|" < "$<" > "$@"
-
-FORCE:
-
-.PHONY: all

extra/docker/precise/Dockerfile

@@ -1,26 +0,0 @@
-############################################################
-# Dockerfile to build Pwntools container
-# Based on Ubuntu
-############################################################
-
-FROM ubuntu:precise
-MAINTAINER Maintainer Gallopsled et al.
-
-RUN apt-get update \
-    && apt-get install -y \
-        build-essential \
-        git \
-        libssl-dev \
-        libffi-dev \
-        python2.7 \
-        python-pip \
-        python-dev \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
-    && useradd -m pwntools \
-    && passwd --delete --unlock pwntools \
-    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
-USER pwntools
-WORKDIR /home/pwntools

extra/docker/precise/Makefile

@@ -1,8 +0,0 @@
-all: Dockerfile
-
-Dockerfile: ../xenial/Dockerfile FORCE
-	sed "s|ubuntu:xenial|ubuntu:precise|" < "$<" > "$@"
-
-FORCE:
-
-.PHONY: all

extra/docker/stable/Dockerfile

@@ -2,5 +2,6 @@ FROM pwntools/pwntools:base
 
 USER root
 RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/trusty/Dockerfile

@@ -1,26 +0,0 @@
-############################################################
-# Dockerfile to build Pwntools container
-# Based on Ubuntu
-############################################################
-
-FROM ubuntu:trusty
-MAINTAINER Maintainer Gallopsled et al.
-
-RUN apt-get update \
-    && apt-get install -y \
-        build-essential \
-        git \
-        libssl-dev \
-        libffi-dev \
-        python2.7 \
-        python-pip \
-        python-dev \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
-    && useradd -m pwntools \
-    && passwd --delete --unlock pwntools \
-    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
-USER pwntools
-WORKDIR /home/pwntools

extra/docker/trusty/Makefile

@@ -1,8 +0,0 @@
-all: Dockerfile
-
-Dockerfile: ../xenial/Dockerfile FORCE
-	sed "s|ubuntu:xenial|ubuntu:trusty|" < "$<" > "$@"
-
-FORCE:
-
-.PHONY: all

extra/docker/xenial/Dockerfile

@@ -1,26 +0,0 @@
-############################################################
-# Dockerfile to build Pwntools container
-# Based on Ubuntu
-############################################################
-
-FROM ubuntu:xenial
-MAINTAINER Maintainer Gallopsled et al.
-
-RUN apt-get update \
-    && apt-get install -y \
-        build-essential \
-        git \
-        libssl-dev \
-        libffi-dev \
-        python2.7 \
-        python-pip \
-        python-dev \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
-    && useradd -m pwntools \
-    && passwd --delete --unlock pwntools \
-    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
-USER pwntools
-WORKDIR /home/pwntools

pwn/__init__.py

@@ -7,7 +7,6 @@ pwnlib.args.initialize()
 pwnlib.log.install_default_handler()
 pwnlib.config.initialize()
 
-log = pwnlib.log.getLogger('pwnlib.exploit')
 args = pwnlib.args.args
 
 if not platform.architecture()[0].startswith('64'):

pwn/toplevel.py

@@ -4,7 +4,9 @@ import logging
 import math
 import operator
 import os
+import platform
 import re
+import requests
 import socks
 import signal
 import string
@@ -15,6 +17,7 @@ import tempfile
 import threading
 import time
 
+import colored_traceback
 from pprint import pprint
 
 import pwnlib
@@ -28,8 +31,9 @@ from pwnlib.elf.corefile import Core, Corefile, Coredump
 from pwnlib.elf.elf import ELF, load
 from pwnlib.encoders import *
 from pwnlib.exception import PwnlibException
-from pwnlib.gdb import attach, debug, debug_assembly, debug_shellcode
+from pwnlib.gdb import attach, debug_assembly, debug_shellcode
 from pwnlib.filepointer import *
+from pwnlib.filesystem import *
 from pwnlib.flag import *
 from pwnlib.fmtstr import FmtStr, fmtstr_payload, fmtstr_split
 from pwnlib.log import getLogger
@@ -37,6 +41,7 @@ from pwnlib.memleak import MemLeak, RelativeMemLeak
 from pwnlib.regsort import *
 from pwnlib.replacements import *
 from pwnlib.rop import ROP
+from pwnlib.rop.call import AppendedArgument
 from pwnlib.rop.srop import SigreturnFrame
 from pwnlib.rop.ret2dlresolve import Ret2dlresolvePayload
 from pwnlib.runner import *
@@ -73,6 +78,7 @@ from pwnlib.util.web import *
 from six.moves import cPickle as pickle, cStringIO as StringIO
 from six import BytesIO
 
+log = getLogger("pwnlib.exploit")
 error   = log.error
 warning = log.warning
 warn    = log.warning
@@ -80,4 +86,7 @@ info    = log.info
 debug   = log.debug
 success = log.success
 
-__all__ = [x for x in tuple(globals()) if x != '__name__']
+colored_traceback.add_hook()
+
+# Equivalence with the default behavior of "from import *"
+# __all__ = [x for x in tuple(globals()) if not x.startswith('_')]

pwnlib/__init__.py

@@ -1,12 +1,11 @@
 from __future__ import absolute_import
 
-import importlib
-
-from pwnlib.version import __version__
+from .version import __version__
 
 version = __version__
 
 __all__ = [
+    'adb',
     'args',
     'asm',
     'atexception',
@@ -35,9 +34,8 @@ __all__ = [
     'ui',
     'useragents',
     'util',
-    'adb',
     'update',
+    'version',
 ]
 
-for module in __all__:
-    importlib.import_module('.%s' % module, 'pwnlib')
+from . import args

pwnlib/adb/adb.py

@@ -68,6 +68,7 @@ from pwnlib.context import context
 from pwnlib.device import Device
 from pwnlib.log import getLogger
 from pwnlib.protocols.adb import AdbClient
+from pwnlib.util.packing import _decode
 from pwnlib.util import misc
 
 log = getLogger(__name__)
@@ -75,10 +76,13 @@ log = getLogger(__name__)
 def adb(argv, *a, **kw):
     r"""Returns the output of an ADB subcommand.
 
-    >>> adb.adb('get-serialno')
-    b'emulator-5554\n'
-    >>> adb.adb(['shell', 'uname']) # it is better to use adb.process
-    b'Linux\n'
+    .. doctest::
+       :skipif: skip_android
+
+        >>> adb.adb('get-serialno')
+        b'emulator-5554\n'
+        >>> adb.adb(['shell', 'uname']) # it is better to use adb.process
+        b'Linux\n'
     """
     if isinstance(argv, (bytes, six.text_type)):
         argv = [argv]
@@ -113,11 +117,12 @@ def current_device(any=False):
     """Returns an ``AdbDevice`` instance for the currently-selected device
     (via ``context.device``).
 
-    Example:
+    .. doctest::
+       :skipif: skip_android
 
         >>> device = adb.current_device(any=True)
-        >>> device
-        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_google_phone_armv7', model='sdk google phone armv7', device='generic')
+        >>> device  # doctest: +ELLIPSIS
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone_armv7', model='sdk ...phone armv7', device='generic')
         >>> device.port
         'emulator'
     """
@@ -144,7 +149,10 @@ def with_device(f):
 def root():
     """Restarts adbd as root.
 
-    >>> adb.root()
+    .. doctest::
+       :skipif: skip_android
+
+        >>> adb.root()
     """
     log.info("Enabling root on %s" % context.device)
 
@@ -202,6 +210,10 @@ def uptime():
         Uptime of the device, in seconds
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.uptime() > 3 # normally AVD takes ~7 seconds to boot
         True
     """
@@ -217,6 +229,10 @@ def boot_time():
         nearest second.
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> import time
         >>> adb.boot_time() < time.time()
         True
@@ -231,6 +247,9 @@ class AdbDevice(Device):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> device = adb.wait_for_device()
         >>> device.arch
         'arm'
@@ -238,8 +257,8 @@ class AdbDevice(Device):
         32
         >>> device.os
         'android'
-        >>> device.product
-        'sdk_google_phone_armv7'
+        >>> device.product  # doctest: +ELLIPSIS
+        'sdk_...phone_armv7'
         >>> device.serial
         'emulator-5554'
     """
@@ -357,10 +376,13 @@ class AdbDevice(Device):
         """Provides scoped access to ``adb`` module propertise, in the context
         of this device.
 
-        >>> property = 'ro.build.fingerprint'
-        >>> device = adb.wait_for_device()
-        >>> adb.getprop(property) == device.getprop(property)
-        True
+        .. doctest::
+           :skipif: skip_android
+
+            >>> property = 'ro.build.fingerprint'
+            >>> device = adb.wait_for_device()
+            >>> adb.getprop(property) == device.getprop(property)
+            True
         """
         with context.local(device=self):
             g = globals()
@@ -387,6 +409,9 @@ def wait_for_device(kick=False):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> device = adb.wait_for_device()
     """
     with log.waitfor("Waiting for device to come online") as w:
@@ -496,6 +521,9 @@ def pull(remote_path, local_path=None):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> _=adb.pull('/proc/version', './proc-version')
         >>> print(read('./proc-version').decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
@@ -527,6 +555,9 @@ def push(local_path, remote_path):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> write('./filename', 'contents')
         >>> adb.push('./filename', '/data/local/tmp')
         '/data/local/tmp/filename'
@@ -584,6 +615,9 @@ def read(path, target=None, callback=None):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> print(adb.read('/proc/version').decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
         >>> adb.read('/does/not/exist')
@@ -613,6 +647,9 @@ def write(path, data=b''):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.write('/dev/null', b'data')
         >>> adb.write('/data/local/tmp/')
     """
@@ -633,6 +670,9 @@ def mkdir(path):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.mkdir('/')
 
         >>> path = '/data/local/tmp/mkdir_test'
@@ -673,6 +713,9 @@ def makedirs(path):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.makedirs('/data/local/tmp/this/is/a/directory/hierarchy')
         >>> adb.listdir('/data/local/tmp/this/is/a/directory')
         ['hierarchy']
@@ -689,6 +732,9 @@ def exists(path):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.exists('/')
         True
         >>> adb.exists('/etc/hosts')
@@ -706,6 +752,9 @@ def isdir(path):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.isdir('/')
         True
         >>> adb.isdir('/init')
@@ -724,6 +773,9 @@ def unlink(path, recursive=False):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.unlink("/does/not/exist")
         Traceback (most recent call last):
         ...
@@ -775,6 +827,9 @@ def process(argv, *a, **kw):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.root()
         >>> print(adb.process(['cat','/proc/version']).recvall().decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
@@ -815,6 +870,9 @@ def which(name, all = False, *a, **kw):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.which('sh')
         '/system/bin/sh'
         >>> adb.which('sh', all=True)
@@ -838,7 +896,7 @@ def which(name, all = False, *a, **kw):
 
     which_cmd = which_cmd.strip()
     data = process(['sh','-c', which_cmd], *a, **kw).recvall()
-    data = context._decode(data)
+    data = _decode(data)
     result = []
 
     for path in data.split('\x00'):
@@ -864,6 +922,10 @@ def whoami():
     """Returns current shell user
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
        >>> adb.whoami()
        b'root'
     """
@@ -917,6 +979,10 @@ def proc_exe(pid):
     """Returns the full path of the executable for the provided PID.
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.proc_exe(1)
         b'/init'
     """
@@ -937,18 +1003,22 @@ def getprop(name=None):
         Otherwise, a string is returned with the contents of the named property.
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.getprop() # doctest: +ELLIPSIS
         {...}
     """
     with context.quiet:
         if name:
             result = process(['getprop', name]).recvall().strip()
-            result = context._decode(result)
+            result = _decode(result)
             return result
 
         result = process(['getprop']).recvall()
 
-    result = context._decode(result)
+    result = _decode(result)
     expr = r'\[([^\]]+)\]: \[(.*)\]'
 
     props = {}
@@ -1052,6 +1122,10 @@ class Kernel(object):
     def address(self):
         """Returns kernel address
         Example:
+
+        .. doctest::
+           :skipif: skip_android
+
             >>> hex(adb.kernel.address) # doctest: +ELLIPSIS
             '0x...000'
         """
@@ -1178,6 +1252,10 @@ class Property(object):
         """Allow simple comparison
 
         Example:
+
+        .. doctest::
+           :skipif: skip_android
+
             >>> adb.properties.ro.build.version.sdk == "24"
             True
         """
@@ -1266,6 +1344,10 @@ def compile(source):
     r"""Compile a source file or project with the Android NDK.
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> temp = tempfile.mktemp('.c')
         >>> write(temp, '''
         ... #include <stdio.h>
@@ -1390,6 +1472,10 @@ class Partitions(object):
     """Enable access to partitions
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> hex(adb.partitions.vda.size) # doctest: +ELLIPSIS
         '0x...000'
     """
@@ -1506,3 +1592,4 @@ def version():
     """Returns rthe platform version as a tuple."""
     prop = getprop('ro.build.version.release')
     return [int(v) for v in prop.split('.')]
+

pwnlib/args.py

@@ -1,5 +1,3 @@
-
-#!/usr/bin/env python2
 """
 Pwntools exposes several magic command-line arguments and environment
 variables when operating in `from pwn import *` mode.

pwnlib/asm.py

@@ -299,6 +299,7 @@ def _arch_header():
     prefix  = ['.section .shellcode,"awx"',
                 '.global _start',
                 '.global __start',
+                '.p2align 2',
                 '_start:',
                 '__start:']
     headers = {
@@ -334,9 +335,11 @@ def _bfdname():
         'cris'    : 'elf32-cris',
         'ia64'    : 'elf64-ia64-%s' % E,
         'm68k'    : 'elf32-m68k',
+        'msp430'  : 'elf32-msp430',
         'powerpc' : 'elf32-powerpc',
         'powerpc64' : 'elf64-powerpc',
         'vax'     : 'elf32-vax',
+        's390'    : 'elf%d-s390' % context.bits,
         'sparc'   : 'elf32-sparc',
         'sparc64' : 'elf64-sparc',
     }
@@ -365,7 +368,7 @@ def _bfdarch():
     return arch
 
 def _run(cmd, stdin = None):
-    log.debug(subprocess.list2cmdline(cmd))
+    log.debug('%s', subprocess.list2cmdline(cmd))
     try:
         proc = subprocess.Popen(
             cmd,
@@ -378,17 +381,20 @@ def _run(cmd, stdin = None):
         exitcode = proc.wait()
     except OSError as e:
         if e.errno == errno.ENOENT:
-            log.exception('Could not run %r the program' % cmd[0])
+            log.exception('Could not run %r the program', cmd[0])
         else:
             raise
 
     if (exitcode, stderr) != (0, ''):
-        msg = 'There was an error running %s:\n' % repr(cmd)
+        msg = 'There was an error running %r:\n'
+        args = cmd,
         if exitcode != 0:
-            msg += 'It had the exitcode %d.\n' % exitcode
+            msg += 'It had the exitcode %d.\n'
+            args += exitcode,
         if stderr != '':
-            msg += 'It had this on stdout:\n%s\n' % stderr
-        log.error(msg)
+            msg += 'It had this on stdout:\n%s\n'
+            args += stderr,
+        log.error(msg, *args)
 
     return stdout
 
@@ -490,9 +496,7 @@ def make_elf_from_assembly(assembly,
         >>> file_b = make_elf_from_assembly('nop', extract=True)
         >>> file_a[:4] == file_b[:4]
         True
-        >>> len(file_a) < 0x200
-        True
-        >>> len(file_b) > 0x1000
+        >>> len(file_a) < len(file_b)
         True
     """
     if shared and vma:
@@ -646,6 +650,10 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
         b'H\xc7\xc0\x17\x00\x00\x00'
         >>> asm("mov r0, #SYS_select", arch = 'arm', os = 'linux', bits=32)
         b'R\x00\xa0\xe3'
+        >>> asm("mov #42, r0", arch = 'msp430')
+        b'0@*\x00'
+        >>> asm("la %r0, 42", arch = 's390', bits=64)
+        b'A\x00\x00*'
     """
     result = ''
 

pwnlib/atexit.py

@@ -13,6 +13,7 @@ from __future__ import division
 import sys
 import threading
 import traceback
+import atexit as std_atexit
 
 from pwnlib.context import context
 
@@ -95,4 +96,8 @@ def _run_handlers():
 if hasattr(sys, "exitfunc"):
     register(sys.exitfunc)
 
-sys.exitfunc = _run_handlers
+if sys.version_info[0] < 3:
+    sys.exitfunc = _run_handlers
+else:
+    std_atexit.register(_run_handlers)
+

pwnlib/commandline/asm.py

@@ -5,7 +5,7 @@ from __future__ import division
 import argparse
 import sys
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -13,7 +13,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'asm',
-    help = 'Assemble shellcode into bytes'
+    help = 'Assemble shellcode into bytes',
+    description = 'Assemble shellcode into bytes',
 )
 
 parser.add_argument(

pwnlib/commandline/checksec.py

@@ -10,7 +10,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'checksec',
-    help = 'Check binary security settings'
+    help = 'Check binary security settings',
+    description = 'Check binary security settings',
 )
 parser.add_argument(
     'elf',

pwnlib/commandline/constgrep.py

@@ -6,7 +6,7 @@ import argparse
 import functools
 import re
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -15,6 +15,7 @@ from pwnlib.commandline import common
 p = common.parser_commands.add_parser(
     'constgrep',
     help = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
+    description = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
     formatter_class = argparse.RawDescriptionHelpFormatter,
 )
 

pwnlib/commandline/cyclic.py

@@ -7,7 +7,7 @@ import six
 import string
 import sys
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -15,14 +15,15 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'cyclic',
-    help = "Cyclic pattern creator/finder"
+    help = "Cyclic pattern creator/finder",
+    description = "Cyclic pattern creator/finder"
 )
 
 parser.add_argument(
     '-a', '--alphabet',
     metavar = 'alphabet',
     default = string.ascii_lowercase.encode(),
-    type = six.ensure_binary,
+    type = packing._encode,
     help = 'The alphabet to use in the cyclic pattern (defaults to all lower case letters)',
 )
 
@@ -70,7 +71,7 @@ def main(args):
             pat = int(pat, 0)
         except ValueError:
             pass
-        pat = flat(pat)
+        pat = flat(pat, bytes=args.length)
 
         if len(pat) != subsize:
             log.critical('Subpattern must be %d bytes' % subsize)

pwnlib/commandline/debug.py

@@ -10,7 +10,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'debug',
-    help = 'Debug a binary in GDB'
+    help = 'Debug a binary in GDB',
+    description = 'Debug a binary in GDB'
 )
 parser.add_argument(
     '-x', metavar='GDBSCRIPT',

pwnlib/commandline/disablenx.py

@@ -8,7 +8,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'disablenx',
-    help = 'Disable NX for an ELF binary'
+    help = 'Disable NX for an ELF binary',
+    description = 'Disable NX for an ELF binary'
 )
 parser.add_argument(
     'elf',

pwnlib/commandline/disasm.py

@@ -7,7 +7,7 @@ import argparse
 import string
 import sys
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -15,7 +15,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'disasm',
-    help = 'Disassemble bytes into text format'
+    help = 'Disassemble bytes into text format',
+    description = 'Disassemble bytes into text format'
 )
 
 parser.add_argument(

pwnlib/commandline/elfdiff.py

@@ -8,7 +8,7 @@ from subprocess import CalledProcessError
 from subprocess import check_output
 from tempfile import NamedTemporaryFile
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -29,7 +29,8 @@ def diff(a,b):
 
 p = common.parser_commands.add_parser(
     'elfdiff',
-    help = 'Compare two ELF files'
+    help = 'Compare two ELF files',
+    description = 'Compare two ELF files'
 )
 
 p.add_argument('a')

pwnlib/commandline/elfpatch.py

@@ -4,7 +4,7 @@ from __future__ import division
 
 import sys
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -12,7 +12,8 @@ from pwnlib.commandline import common
 
 p = common.parser_commands.add_parser(
     'elfpatch',
-    help = 'Patch an ELF file'
+    help = 'Patch an ELF file',
+    description = 'Patch an ELF file'
 )
 
 p.add_argument('elf',help="File to patch")

pwnlib/commandline/errno.py

@@ -8,7 +8,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'errno',
-    help = 'Prints out error messages'
+    help = 'Prints out error messages',
+    description = 'Prints out error messages'
 )
 
 parser.add_argument(

pwnlib/commandline/hex.py

@@ -10,9 +10,9 @@ from pwnlib.util.fiddling import enhex
 
 parser = common.parser_commands.add_parser(
     'hex',
-    help = '''
-Hex-encodes data provided on the command line or stdin
-''')
+    help = 'Hex-encodes data provided on the command line or stdin',
+    description = 'Hex-encodes data provided on the command line or stdin')
+
 parser.add_argument('data', nargs='*',
     help='Data to convert into hex')
 

pwnlib/commandline/phd.py

@@ -7,7 +7,7 @@ import os
 import sys
 import io
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -15,7 +15,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'phd',
-    help = 'Pwnlib HexDump'
+    help = 'Pretty hex dump',
+    description = 'Pretty hex dump'
 )
 
 parser.add_argument(

pwnlib/commandline/pwnstrip.py

@@ -3,7 +3,7 @@ from __future__ import division
 
 import argparse
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -12,6 +12,7 @@ from pwnlib.commandline import common
 p = common.parser_commands.add_parser(
     'pwnstrip',
     help = 'Strip binaries for CTF usage',
+    description = 'Strip binaries for CTF usage'
 )
 
 g = p.add_argument_group("actions")

pwnlib/commandline/scramble.py

@@ -4,7 +4,7 @@ from __future__ import division
 import argparse
 import sys
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -12,7 +12,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'scramble',
-    help = 'Shellcode encoder'
+    help = 'Shellcode encoder',
+    description = 'Shellcode encoder'
 )
 
 parser.add_argument(

pwnlib/commandline/shellcraft.py

@@ -8,7 +8,7 @@ import six
 import sys
 import types
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -35,6 +35,7 @@ def _string(s):
 p = common.parser_commands.add_parser(
     'shellcraft',
     help = 'Microwave shellcode -- Easy, fast and delicious',
+    description = 'Microwave shellcode -- Easy, fast and delicious',
 )
 
 
@@ -198,11 +199,11 @@ def main(args):
         common.parser.print_usage()
         exit()
 
-    if args.shellcode not in shellcraft.templates:
+    try:
+        func = get_template(args.shellcode)
+    except AttributeError:
         log.error("Unknown shellcraft template %r. Use --list to see available shellcodes." % args.shellcode)
 
-    func = get_template(args.shellcode)
-
     if args.show:
         # remove doctests
         doc = []
@@ -295,7 +296,7 @@ def main(args):
 
     vma = args.address
     if vma:
-        vma = eval(vma)
+        vma = pwnlib.util.safeeval.expr(vma)
 
     if args.format in ['e','elf']:
         args.format = 'default'

pwnlib/commandline/template.py

@@ -2,8 +2,6 @@
 from __future__ import absolute_import
 from __future__ import division
 
-import re
-
 from pwn import *
 from pwnlib.commandline import common
 
@@ -11,7 +9,8 @@ from mako.lookup import TemplateLookup
 
 parser = common.parser_commands.add_parser(
     'template',
-    help = 'Generate an exploit template'
+    help = 'Generate an exploit template',
+    description = 'Generate an exploit template'
 )
 
 parser.add_argument('exe', nargs='?', help='Target binary')
@@ -24,14 +23,9 @@ parser.add_argument('--quiet', help='Less verbose template comments', action='st
 parser.add_argument('--color', help='Print the output in color', choices=['never', 'always', 'auto'], default='auto')
 
 def main(args):
-    cache = None
-
-    if cache:
-        cache = os.path.join(context.cache_dir, 'mako')
-
     lookup = TemplateLookup(
         directories      = [os.path.join(pwnlib.data.path, 'templates')],
-        module_directory = cache
+        module_directory = None
     )
 
     # For the SSH scenario, check that the binary is at the
@@ -41,7 +35,14 @@ def main(args):
             log.error("Must specify --path or a exe")
 
         s = ssh(args.user, args.host, args.port or 22, args.password or None)
-        s.download(args.path or args.exe)
+
+        try:
+            remote_file = args.path or args.exe
+            s.download(remote_file)
+        except Exception:
+            log.warning("Could not download file %r, opening a shell", remote_file)
+            s.interactive()
+            return
 
         if not args.exe:
             args.exe = os.path.basename(args.path)

pwnlib/commandline/unhex.py

@@ -11,9 +11,9 @@ from pwnlib.util.fiddling import unhex
 
 parser = common.parser_commands.add_parser(
     'unhex',
-    help = '''
-Decodes hex-encoded data provided on the command line or via stdin.
-''')
+    help = 'Decodes hex-encoded data provided on the command line or via stdin.',
+    description = 'Decodes hex-encoded data provided on the command line or via stdin.'
+)
 
 parser.add_argument('hex', nargs='*',
     help='Hex bytes to decode')

pwnlib/commandline/update.py

@@ -4,7 +4,7 @@ from __future__ import division
 
 import subprocess
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -12,7 +12,8 @@ from pwnlib.commandline import common
 
 p = common.parser_commands.add_parser(
     'update',
-    help = 'Check for pwntools updates'
+    help = 'Check for pwntools updates',
+    description = 'Check for pwntools updates'
 )
 
 p.add_argument('--install', action='store_true', help='''

pwnlib/commandline/version.py

@@ -5,7 +5,7 @@ from __future__ import division
 import os
 import subprocess
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -13,7 +13,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'version',
-    help = 'Pwntools version'
+    help = 'Pwntools version',
+    description = 'Pwntools version'
 )
 
 def main(a):

pwnlib/config.py

@@ -5,7 +5,9 @@ The list of configurable options includes all of the logging symbols
 and colors, as well as all of the default values on the global context
 object.
 
-The configuration file is read from ``~/.pwn.conf`` and ``/etc/pwn.conf``.
+The configuration file is read from ``~/.pwn.conf``, ``$XDG_CONFIG_HOME/pwn.conf``
+(``$XDG_CONFIG_HOME`` defaults to ``~/.config``, per XDG Base Directory Specification),
+and ``/etc/pwn.conf``.
 
 The configuration file is only read in ``from pwn import *`` mode, and not
 when used in library mode (``import pwnlib``).  To read the configuration
@@ -53,8 +55,13 @@ def initialize():
     from pwnlib.log import getLogger
     log = getLogger(__name__)
 
+    xdg_config_home = (os.environ.get('XDG_CONFIG_HOME') or
+                       os.path.expanduser("~/.config"))
+
     c = configparser.ConfigParser()
-    c.read(['/etc/pwn.conf', os.path.expanduser('~/.pwn.conf')])
+    c.read(['/etc/pwn.conf',
+            os.path.join(xdg_config_home, 'pwn.conf'),
+            os.path.expanduser('~/.pwn.conf')])
 
     for section in c.sections():
         if section not in registered_configs:

pwnlib/constants/__init__.py

@@ -145,7 +145,13 @@ class ConstantsModule(ModuleType):
         if key not in self._env_store:
             self._env_store[key] = {key: getattr(self, key) for key in dir(self) if not key.endswith('__')}
 
-        return Constant('(%s)' % string, safeeval.values(string, self._env_store[key]))
+        val = safeeval.values(string, self._env_store[key])
+
+        # if the expression is not assembly-safe, it is not so vital to preserve it
+        if set(string) & (set(bytearray(range(32)).decode()) | set('"#$\',.;@[\\]`{}')):
+            string = val
+
+        return Constant('(%s)' % string, val)
 
 
 # To prevent garbage collection

pwnlib/context/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 """
 Implements context management so that nested/scoped contexts and threaded
@@ -359,17 +358,18 @@ class ContextType(object):
         'log_console': sys.stdout,
         'randomize': False,
         'rename_corefiles': True,
-        'newline': '\n',
+        'newline': b'\n',
         'noptrace': False,
         'os': 'linux',
         'proxy': None,
+        'ssh_session': None,
         'signed': False,
         'terminal': tuple(),
         'timeout': Timeout.maximum,
     }
 
     #: Valid values for :meth:`pwnlib.context.ContextType.os`
-    oses = sorted(('linux','freebsd','windows','cgc','android'))
+    oses = sorted(('linux','freebsd','windows','cgc','android','baremetal'))
 
     big_32    = {'endian': 'big', 'bits': 32}
     big_64    = {'endian': 'big', 'bits': 64}
@@ -549,14 +549,66 @@ class ContextType(object):
     @property
     def quiet(self, function=None):
         """Disables all non-error logging within the enclosed scope,
-        *unless* the debugging level is set to 'debug' or lower."""
+        *unless* the debugging level is set to 'debug' or lower.
+
+        Example:
+
+            Let's assume the normal situation, where log_level is INFO.
+
+            >>> context.clear(log_level='info')
+
+            Note that only the log levels below ERROR do not print anything.
+
+            >>> with context.quiet:
+            ...     log.debug("DEBUG")
+            ...     log.info("INFO")
+            ...     log.warn("WARN")
+
+            Next let's try with the debugging level set to 'debug' before we
+            enter the context handler:
+
+            >>> with context.local(log_level='debug'):
+            ...     with context.quiet:
+            ...         log.debug("DEBUG")
+            ...         log.info("INFO")
+            ...         log.warn("WARN")
+            [DEBUG] DEBUG
+            [*] INFO
+            [!] WARN
+        """
         level = 'error'
         if context.log_level <= logging.DEBUG:
             level = None
         return self.local(function, log_level=level)
 
     def quietfunc(self, function):
-        """Similar to :attr:`quiet`, but wraps a whole function."""
+        """Similar to :attr:`quiet`, but wraps a whole function.
+
+        Example:
+
+            Let's set up two functions, which are the same but one is
+            wrapped with :attr:`quietfunc`.
+
+            >>> def loud(): log.info("Loud")
+            >>> @context.quietfunc
+            ... def quiet(): log.info("Quiet")
+
+            If we set the logging level to 'info', the loud function
+            prints its contents.
+
+            >>> with context.local(log_level='info'): loud()
+            [*] Loud
+
+            However, the quiet function does not, since :attr:`quietfunc`
+            silences all output unless the log level is DEBUG.
+
+            >>> with context.local(log_level='info'): quiet()
+
+            Now let's try again with debugging enabled.
+
+            >>> with context.local(log_level='debug'): quiet()
+            [*] Quiet
+        """
         @functools.wraps(function)
         def wrapper(*a, **kw):
             level = 'error'
@@ -570,6 +622,28 @@ class ContextType(object):
     @property
     def verbose(self):
         """Enable all logging within the enclosed scope.
+
+        This is the opposite of :attr:`.quiet` and functionally equivalent to:
+
+        .. code-block:: python
+
+            with context.local(log_level='debug'):
+                ...
+
+        Example:
+
+            Note that the function does not emit any information by default
+
+            >>> context.clear()
+            >>> def func(): log.debug("Hello")
+            >>> func()
+
+            But if we put it inside a :attr:`.verbose` context manager, the
+            information is printed.
+
+            >>> with context.verbose: func()
+            [DEBUG] Hello
+
         """
         return self.local(log_level='debug')
 
@@ -603,7 +677,7 @@ class ContextType(object):
 
     @property
     def native(self):
-        if context.os in ('android', 'cgc'):
+        if context.os in ('android', 'baremetal', 'cgc'):
             return False
 
         arch = context.arch
@@ -827,27 +901,6 @@ class ContextType(object):
 
         return charset
 
-    def _encode(self, s):
-        if isinstance(s, (bytes, bytearray)):
-            return s   # already bytes
-
-        if self.encoding == 'auto':
-            try:
-                return s.encode('latin1')
-            except UnicodeEncodeError:
-                return s.encode('utf-8', 'surrogateescape')
-        return s.encode(self.encoding)
-
-    def _decode(self, b):
-        if self.encoding == 'auto':
-            try:
-                return b.decode('utf-8')
-            except UnicodeDecodeError:
-                return b.decode('latin1')
-            except AttributeError:
-                return b
-        return b.decode(self.encoding)
-
     @_validator
     def endian(self, endianness):
         """
@@ -932,20 +985,22 @@ class ContextType(object):
         Examples:
 
 
-            >>> context.log_file = 'foo.txt' #doctest: +ELLIPSIS
-            >>> log.debug('Hello!') #doctest: +ELLIPSIS
+            >>> foo_txt = tempfile.mktemp()
+            >>> bar_txt = tempfile.mktemp()
+            >>> context.log_file = foo_txt
+            >>> log.debug('Hello!')
             >>> with context.local(log_level='ERROR'): #doctest: +ELLIPSIS
             ...     log.info('Hello again!')
-            >>> with context.local(log_file='bar.txt'):
+            >>> with context.local(log_file=bar_txt):
             ...     log.debug('Hello from bar!')
             >>> log.info('Hello from foo!')
-            >>> open('foo.txt').readlines()[-3] #doctest: +ELLIPSIS
+            >>> open(foo_txt).readlines()[-3] #doctest: +ELLIPSIS
             '...:DEBUG:...:Hello!\n'
-            >>> open('foo.txt').readlines()[-2] #doctest: +ELLIPSIS
+            >>> open(foo_txt).readlines()[-2] #doctest: +ELLIPSIS
             '...:INFO:...:Hello again!\n'
-            >>> open('foo.txt').readlines()[-1] #doctest: +ELLIPSIS
+            >>> open(foo_txt).readlines()[-1] #doctest: +ELLIPSIS
             '...:INFO:...:Hello from foo!\n'
-            >>> open('bar.txt').readlines()[-1] #doctest: +ELLIPSIS
+            >>> open(bar_txt).readlines()[-1] #doctest: +ELLIPSIS
             '...:DEBUG:...:Hello from bar!\n'
         """
         if isinstance(value, (bytes, six.text_type)):
@@ -1019,7 +1074,7 @@ class ContextType(object):
             >>> context.os = 'foobar' #doctest: +ELLIPSIS
             Traceback (most recent call last):
             ...
-            AttributeError: os must be one of ['android', 'cgc', 'freebsd', 'linux', 'windows']
+            AttributeError: os must be one of ['android', 'baremetal', 'cgc', 'freebsd', 'linux', 'windows']
         """
         os = os.lower()
 
@@ -1279,6 +1334,17 @@ class ContextType(object):
         """
         return bool(v)
 
+    @_validator
+    def newline(self, v):
+        """Line ending used for Tubes by default.
+
+        This configures the newline emitted by e.g. ``sendline`` or that is used
+        as a delimiter for e.g. ``recvline``.
+        """
+        # circular imports
+        from pwnlib.util.packing import _need_bytes
+        return _need_bytes(v)
+
 
     @_validator
     def gdbinit(self, value):
@@ -1322,6 +1388,15 @@ class ContextType(object):
 
         return size
 
+    @_validator
+    def ssh_session(self, shell):
+        from pwnlib.tubes.ssh import ssh
+
+        if not isinstance(shell, ssh):
+            raise AttributeError("context.ssh_session must be an ssh tube") 
+
+        return shell
+
     #*************************************************************************
     #                               ALIASES
     #*************************************************************************

pwnlib/data/syscalls/generate.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python2
 from __future__ import division
 import argparse
+import keyword
 import os
 
 from pwnlib import constants
@@ -62,7 +63,7 @@ CALL = """
 
     for name, arg in zip(argument_names, argument_values):
         if arg is not None:
-            syscall_repr.append('%s=%r' % (name, arg))
+            syscall_repr.append('%s=%s' % (name, pwnlib.shellcraft.pretty(arg, False)))
 
         # If the argument itself (input) is a register...
         if arg in allregs:
@@ -75,8 +76,8 @@ CALL = """
 
         # The argument is not a register.  It is a string value, and we
         # are expecting a string value
-        elif name in can_pushstr and isinstance(arg, (bytes, six.text_type)):
-            if not isinstance(arg, bytes):
+        elif name in can_pushstr and isinstance(arg, (six.binary_type, six.text_type)):
+            if isinstance(arg, six.text_type):
                 arg = arg.encode('utf-8')
             string_arguments[name] = arg
 
@@ -144,12 +145,11 @@ def can_be_array(arg):
 
 
 def fix_bad_arg_names(func, arg):
-    if arg.name == 'str':
-        return 'str_'
     if arg.name == 'len':
         return 'length'
-    if arg.name == 'repr':
-        return 'repr_'
+
+    if arg.name in ('str', 'repr') or keyword.iskeyword(arg.name):
+        return arg.name + '_'
 
     if func.name == 'open' and arg.name == 'vararg':
         return 'mode'
@@ -277,8 +277,10 @@ def generate_one(target):
             CALL.format(**template_variables)
         ]
 
-        with open(os.path.join(target, name + '.asm'), 'wt+') as f:
-            f.write('\n'.join(map(str.strip, lines)))
+        if keyword.iskeyword(name):
+            name += '_'
+        with open(os.path.join(target, name + '.asm'), 'wt') as f:
+            f.write('\n'.join(map(str.strip, lines)) + '\n')
 
 if __name__ == '__main__':
     p = argparse.ArgumentParser()

pwnlib/data/templates/pwnup.mako

@@ -32,7 +32,7 @@ remote_path = remote_path or exe
 password = password or 'secret1234'
 binary_repr = repr(binary)
 %>\
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 %if not quiet:
 # This exploit template was generated via:
@@ -84,14 +84,14 @@ if not args.LOCAL:
 %endif
 
 %if host:
-def local(argv=[], *a, **kw):
+def start_local(argv=[], *a, **kw):
     '''Execute the target binary locally'''
     if args.GDB:
         return gdb.debug([${binary_repr}] + argv, gdbscript=gdbscript, *a, **kw)
     else:
         return process([${binary_repr}] + argv, *a, **kw)
 
-def remote(argv=[], *a, **kw):
+def start_remote(argv=[], *a, **kw):
   %if ssh:
     '''Execute the target binary on the remote host'''
     if args.GDB:
@@ -111,9 +111,9 @@ def remote(argv=[], *a, **kw):
 def start(argv=[], *a, **kw):
     '''Start the exploit against the target.'''
     if args.LOCAL:
-        return local(argv, *a, **kw)
+        return start_local(argv, *a, **kw)
     else:
-        return remote(argv, *a, **kw)
+        return start_remote(argv, *a, **kw)
 %else:
 def start(argv=[], *a, **kw):
     '''Start the exploit against the target.'''

pwnlib/dynelf.py

@@ -62,7 +62,7 @@ from pwnlib.elf import constants
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak
 from pwnlib.util.fiddling import enhex
-from pwnlib.util.packing import unpack
+from pwnlib.util.packing import unpack, _encode
 from pwnlib.util.web import wget
 
 log    = getLogger(__name__)
@@ -75,7 +75,7 @@ def sysv_hash(symbol):
     """
     h = 0
     g = 0
-    for c in bytearray(context._encode(symbol)):
+    for c in bytearray(_encode(symbol)):
         h = (h << 4) + c
         g = h & 0xf0000000
         h ^= (g >> 24)
@@ -87,7 +87,7 @@ def gnu_hash(s):
 
     Function used to generated GNU-style hashes for strings.
     """
-    s = bytearray(context._encode(s))
+    s = bytearray(_encode(s))
     h = 5381
     for c in s:
         h = h * 33 + c
@@ -538,6 +538,9 @@ class DynELF(object):
         if lib == 'libc':
             lib = 'libc.so'
 
+        if symb:
+            symb = _encode(symb)
+
         #
         # Get a pretty name for the symbol to show the user
         #
@@ -637,12 +640,14 @@ class DynELF(object):
         while leak.field(cur, LinkMap.l_prev):
             cur = leak.field(cur, LinkMap.l_prev)
 
+        libname = _encode(libname)
+
         while cur:
             self.status("link_map entry %#x" % cur)
             p_name = leak.field(cur, LinkMap.l_name)
             name   = leak.s(p_name)
 
-            if libname.encode('utf-8') in name:
+            if libname in name:
                 break
 
             if name:
@@ -741,7 +746,6 @@ class DynELF(object):
 
                 # Leak the name of the function from the symbol table
                 name = leak.s(strtab + leak.field(sym, Sym.st_name))
-                name = name.decode('utf-8')
 
                 # Make sure it matches the name of the symbol we were looking for.
                 if name == symb:
@@ -749,7 +753,7 @@ class DynELF(object):
                     addr = libbase + leak.field(sym, Sym.st_value)
                     return addr
 
-                self.status("%s (hash collision)" % name)
+                self.status("%r (hash collision)" % name)
 
             # The name did not match what we were looking for, or we assume
             # it did not since it was not a function.
@@ -825,7 +829,6 @@ class DynELF(object):
                 # Check for collision on hash values
                 sym  = symtab + sizeof(Sym) * (ndx + i)
                 name = leak.s(strtab + leak.field(sym, Sym.st_name))
-                name = name.decode('utf-8')
 
                 if name == symb:
                     # No collision, get offset and calculate address
@@ -833,7 +836,7 @@ class DynELF(object):
                     addr   = offset + libbase
                     return addr
 
-                self.status("%s (hash collision)" % name)
+                self.status("%r (hash collision)" % name)
 
             # Collision or no match, continue to the next item
             i += 1

pwnlib/elf/corefile.py

@@ -196,8 +196,8 @@ class Mapping(object):
                self.start,
                self.stop,
                self.size,
-               self.page_offset,
-               self.flags)
+               self.flags,
+               self.page_offset)
 
     def __int__(self):
         return self.start
@@ -264,13 +264,16 @@ class Mapping(object):
         return result + self.address
 
 class Corefile(ELF):
-    r"""Enhances the inforation available about a corefile (which is an extension
+    r"""Enhances the information available about a corefile (which is an extension
     of the ELF format) by permitting extraction of information about the mapped
     data segments, and register state.
 
     Registers can be accessed directly, e.g. via ``core_obj.eax`` and enumerated
     via :data:`Corefile.registers`.
 
+    Memory can be accessed directly via :meth:`.read` or :meth:`.write`, and also
+    via :meth:`.pack` or :meth:`.unpack` or even :meth:`.string`.
+
     Arguments:
         core: Path to the core file.  Alternately, may be a :class:`.process` instance,
               and the core file will be located automatically.
@@ -320,11 +323,7 @@ class Corefile(ELF):
          Mapping('/lib/i386-linux-gnu/ld-2.19.so', start=0xf7713000, stop=0xf7714000, size=0x1000, flags=0x6, page_offset=0x21),
          Mapping('[stack]', start=0xfff3e000, stop=0xfff61000, size=0x23000, flags=0x6, page_offset=0x0)]
 
-    Example:
-
-        The Linux kernel may not overwrite an existing core-file.
-
-        >>> if os.path.exists('core'): os.unlink('core')
+    Examples:
 
         Let's build an example binary which should eat ``R0=0xdeadbeef``
         and ``PC=0xcafebabe``.
@@ -345,18 +344,24 @@ class Corefile(ELF):
         You can specify a full path a la ``Corefile('/path/to/core')``,
         but you can also just access the :attr:`.process.corefile` attribute.
 
+        There's a lot of behind-the-scenes logic to locate the corefile for
+        a given process, but it's all handled transparently by Pwntools.
+
         >>> core = io.corefile
 
-        The core file has a :attr:`.Corefile.exe` property, which is a :class:`.Mapping`
+        The core file has a :attr:`exe` property, which is a :class:`.Mapping`
         object.  Each mapping can be accessed with virtual addresses via subscript, or
         contents can be examined via the :attr:`.Mapping.data` attribute.
 
-        >>> core.exe.address == address
-        True
+        >>> core.exe # doctest: +ELLIPSIS
+        Mapping('/.../step3', start=..., stop=..., size=0x1000, flags=0x..., page_offset=...)
+        >>> hex(core.exe.address)
+        '0x41410000'
 
         The core file also has registers which can be accessed direclty.
-        Pseudo-registers ``pc`` and ``sp`` are available on all architectures,
+        Pseudo-registers :attr:`pc` and :attr:`sp` are available on all architectures,
         to make writing architecture-agnostic code more simple.
+        If this were an amd64 corefile, we could access e.g. ``core.rax``.
 
         >>> core.pc == 0xcafebabe
         True
@@ -367,7 +372,8 @@ class Corefile(ELF):
 
         We may not always know which signal caused the core dump, or what address
         caused a segmentation fault.  Instead of accessing registers directly, we
-        can also extract this information from the core dump.
+        can also extract this information from the core dump via :attr:`fault_addr`
+        and :attr:`signal`.
 
         On QEMU-generated core dumps, this information is unavailable, so we
         substitute the value of PC.  In our example, that's correct anyway.
@@ -378,56 +384,99 @@ class Corefile(ELF):
         11
 
         Core files can also be generated from running processes.
-        This requires GDB to be installed, and can only be done with native
-        processes.  Getting a "complete" corefile requires GDB 7.11 or better.
+        This requires GDB to be installed, and can only be done with native processes.
+        Getting a "complete" corefile requires GDB 7.11 or better.
 
-        >>> elf = ELF('/bin/bash-static')
+        >>> elf = ELF(which('bash-static'))
         >>> context.clear(binary=elf)
-        >>> io = process(elf.path, env={'HELLO': 'WORLD'})
+        >>> env = dict(os.environ)
+        >>> env['HELLO'] = 'WORLD'
+        >>> io = process(elf.path, env=env)
+        >>> io.sendline(b'echo hello')
+        >>> io.recvline()
+        b'hello\n'
+
+        The process is still running, but accessing its :attr:`.process.corefile` property
+        automatically invokes GDB to attach and dump a corefile.
+
         >>> core = io.corefile
 
-        Data can also be extracted directly from the corefile.
+        The corefile can be inspected and read from, and even exposes various mappings
 
-        >>> elf.address > 0
-        True
-        >>> core.exe[elf.address:elf.address+4]
-        b'\x7fELF'
-        >>> core.exe.data[:4]
+        >>> core.exe # doctest: +ELLIPSIS
+        Mapping('.../bin/bash-static', start=..., stop=..., size=..., flags=..., page_offset=...)
+        >>> core.exe.data[0:4]
         b'\x7fELF'
 
-        Various other mappings are available by name.  On Linux, 32-bit Intel binaries
-        should have a VDSO section.  Since our ELF is statically linked, there is
-        no libc which gets mapped.
+        It also supports all of the features of :class:`ELF`, so you can :meth:`.read`
+        or :meth:`.write` or even the helpers like :meth:`.pack` or :meth:`.unpack`.
+
+        Don't forget to call :meth:`.ELF.save` to save the changes to disk.
+
+        >>> core.read(elf.address, 4)
+        b'\x7fELF'
+        >>> core.pack(core.sp, 0xdeadbeef)
+        >>> core.save()
+
+        Let's re-load it as a new :attr:`Corefile` object and have a look!
+
+        >>> core2 = Corefile(core.path)
+        >>> hex(core2.unpack(core2.sp))
+        '0xdeadbeef'
+
+        Various other mappings are available by name, for the first segment of:
+
+        * :attr:`.exe` the executable
+        * :attr:`.libc` the loaded libc, if any
+        * :attr:`.stack` the stack mapping
+        * :attr:`.vvar`
+        * :attr:`.vdso`
+        * :attr:`.vsyscall`
+
+        On Linux, 32-bit Intel binaries should have a VDSO section via :attr:`vdso`.  
+        Since our ELF is statically linked, there is no libc which gets mapped.
 
         >>> core.vdso.data[:4]
         b'\x7fELF'
-        >>> core.libc # doctest: +ELLIPSIS
-        Mapping('/lib/x86_64-linux-gnu/libc-...', ...)
+        >>> core.libc
 
-        The corefile also contains a :attr:`.Corefile.stack` property, which gives
+        But if we dump a corefile from a dynamically-linked binary, the :attr:`.libc`
+        will be loaded.
+
+        >>> process('bash').corefile.libc # doctest: +ELLIPSIS
+        Mapping('/.../libc-....so', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
+
+        The corefile also contains a :attr:`.stack` property, which gives
         us direct access to the stack contents.  On Linux, the very top of the stack
         should contain two pointer-widths of NULL bytes, preceded by the NULL-
         terminated path to the executable (as passed via the first arg to ``execve``).
 
-        >>> stack_end = core.exe.name.encode()
-        >>> stack_end += b'\x00' * (1+8)
-        >>> core.stack.data.endswith(stack_end)
-        True
-        >>> len(core.stack.data) == core.stack.size
-        True
+        >>> core.stack # doctest: +ELLIPSIS
+        Mapping('[stack]', start=0x..., stop=0x..., size=0x..., flags=0x6, page_offset=0x0)
 
-        We can also directly access the environment variables and arguments.
+        When creating a process, the kernel puts the absolute path of the binary and some
+        padding bytes at the end of the stack.  We can look at those by looking at 
+        ``core.stack.data``.
+
+        >>> size = len('/bin/bash-static') + 8
+        >>> core.stack.data[-size:]
+        b'bin/bash-static\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+
+        We can also directly access the environment variables and arguments, via
+        :attr:`.argc`, :attr:`.argv`, and :attr:`.env`.
 
         >>> 'HELLO' in core.env
         True
+        >>> core.string(core.env['HELLO'])
+        b'WORLD'
         >>> core.getenv('HELLO')
         b'WORLD'
         >>> core.argc
         1
         >>> core.argv[0] in core.stack
         True
-        >>> core.string(core.argv[0]) == core.exe.path.encode()
-        True
+        >>> core.string(core.argv[0]) # doctest: +ELLIPSIS
+        b'.../bin/bash-static'
 
         Corefiles can also be pulled from remote machines via SSH!
 
@@ -451,10 +500,6 @@ class Corefile(ELF):
         >>> io.corefile.fault_addr
         1234
 
-    Tests:
-
-        These are extra tests not meant to serve as examples.
-
         Corefile.getenv() works correctly, even if the environment variable's
         value contains embedded '='. Corefile is able to find the stack, even
         if the stack pointer doesn't point at the stack.
@@ -505,20 +550,21 @@ class Corefile(ELF):
         #: The NT_SIGINFO object
         self.siginfo = None
 
-        #: :class:`dict`: Dictionary of memory mappings from ``address`` to ``name``
+        #: :class:`list`: A list of :class:`.Mapping` objects for each loaded memory region
         self.mappings = []
 
-        #: :class:`int`: Address of the stack base
+        #: :class:`int`: A :class:`Mapping` corresponding to the stack
         self.stack    = None
 
-        #: :class:`dict`: Environment variables read from the stack.  Keys are
-        #: the environment variable name, values are the memory address of the
-        #: variable.
-        #:
-        #: Note: Use with the :meth:`.ELF.string` method to extract them.
-        #:
-        #: Note: If FOO=BAR is in the environment, self.env['FOO'] is the
-        #:       address of the string "BAR\x00".
+        """
+        Environment variables read from the stack.
+        Keys are the environment variable name, values are the memory 
+        address of the variable.
+        
+        Use :meth:`.getenv` or :meth:`.string` to retrieve the textual value.
+        
+        Note: If ``FOO=BAR`` is in the environment, ``self.env['FOO']`` is the address of the string ``"BAR\x00"``.
+        """
         self.env = {}
 
         #: :class:`int`: Pointer to envp on the stack
@@ -627,6 +673,13 @@ class Corefile(ELF):
                     # off the end of the stack.
                     pass
 
+            # Corefiles generated by QEMU do not have a name for the 
+            # main module mapping.
+            # Fetching self.exe will cause this to be auto-populated,
+            # and is a no-op in other cases.
+            self.exe
+
+            # Print out the nice display for the user
             self._describe_core()
 
     def _parse_nt_file(self, note):
@@ -719,15 +772,30 @@ class Corefile(ELF):
     @property
     def exe(self):
         """:class:`Mapping`: First mapping for the executable file."""
+
+        # Finding the executable mapping requires knowing the entry point
+        # from the auxv
+        if not self.at_entry:
+            return None
+
+        # The entry point may not be in the first segment of a given file,
+        # but we want to find the first segment of the file -- not the segment that 
+        # contains the entrypoint.
+        first_segment_for_name = {}
+
         for m in self.mappings:
-            if self.at_entry and m.start <= self.at_entry <= m.stop:
+            first_segment_for_name.setdefault(m.name, m)
+
+        # Find which segment conains the entry point
+        for m in self.mappings:
+            if m.start <= self.at_entry < m.stop:
 
                 if not m.name and self.at_execfn:
                     m.name = self.string(self.at_execfn)
                     if not isinstance(m.name, str):
                         m.name = m.name.decode('utf-8')
 
-                return m
+                return first_segment_for_name.get(m.name, m)
 
     @property
     def pid(self):
@@ -839,7 +907,7 @@ class Corefile(ELF):
 
     @property
     def sp(self):
-        """:class:`int`: The program counter for the Corefile
+        """:class:`int`: The stack pointer for the Corefile
 
         This is a cross-platform way to get e.g. ``core.esp``, ``core.rsp``, etc.
         """
@@ -1109,11 +1177,8 @@ class Corefile(ELF):
 
         return rv
 
-    def debug(self, *a, **kw):
+    def debug(self):
         """Open the corefile under a debugger."""
-        if a or kw:
-            log.error("Arguments are not supported for %s.debug()" % self.__class__.__name__)
-
         import pwnlib.gdb
         pwnlib.gdb.attach(self, exe=self.exe.path)
 
@@ -1504,7 +1569,7 @@ class CorefileFinder(object):
             path = os.path.join(binfmt_misc, entry)
 
             try:
-                data = self.read(path)
+                data = self.read(path).decode()
             except Exception:
                 continue
 

pwnlib/elf/elf.py

@@ -745,7 +745,7 @@ class ELF(ELFFile):
                     self._libs[qemu_lib] = address
 
     def _patch_elf_and_read_maps(self):
-        """patch_elf_and_read_maps(self) -> dict
+        r"""patch_elf_and_read_maps(self) -> dict
 
         Read ``/proc/self/maps`` as if the ELF were executing.
 
@@ -765,12 +765,13 @@ class ELF(ELFFile):
             These tests are just to ensure that our shellcode is correct.
 
             >>> for arch in CAT_PROC_MAPS_EXIT:
+            ...   context.clear()
             ...   with context.local(arch=arch):
             ...     sc = shellcraft.cat("/proc/self/maps")
             ...     sc += shellcraft.exit()
             ...     sc = asm(sc)
             ...     sc = enhex(sc)
-            ...     assert sc == CAT_PROC_MAPS_EXIT[arch]
+            ...     assert sc == CAT_PROC_MAPS_EXIT[arch], (arch, sc)
         """
 
         # Get our shellcode
@@ -815,7 +816,7 @@ class ELF(ELFFile):
         try:
             with context.silent:
                 io = process(path)
-                data = context._decode(io.recvall(timeout=2))
+                data = packing._decode(io.recvall(timeout=2))
         except Exception:
             log.warn_once("Injected /proc/self/maps code did not execute correctly")
             return {}
@@ -1047,8 +1048,8 @@ class ELF(ELFFile):
                 for address, target in sorted(res.items()):
                     self.plt[inv_symbols[target]] = address
 
-        for a,n in sorted({v:k for k,v in self.plt.items()}.items()):
-            log.debug('PLT %#x %s', a, n)
+        # for a,n in sorted({v:k for k,v in self.plt.items()}.items()):
+            # log.debug('PLT %#x %s', a, n)
 
     def _populate_kernel_version(self):
         if 'linux_banner' not in self.symbols:

pwnlib/elf/maps.py

@@ -17,7 +17,7 @@ CAT_PROC_MAPS_EXIT = {
         '000020e00170a0e3000000ef',
     'thumb':
         '004f01e0617073ff4fea07274fea172780b4dff8047001e06c662f6d80b4dff8047001e0632f736580b4dff8047001e02f70726f80b4684681ea010182ea02024ff0050741df05464ff00100294682ea02026ff000434ff0bb0741df'
-        '80ea00004ff0010741df',
+        '80ea00004ff0010741df00bf',
     'mips':
         '726f093c2f702935f0ffa9af7365093c632f2935f4ffa9af2f6d093c6c662935f8ffa9af8cff193c9e8f393727482003fcffa9aff0ffbd272020a003ffff0528ffff0628a50f02340c010101feff192427202003fcffa2affcffa58fffff0628ff7f073cffffe7346f1002340c010101'
         'ffff0428a10f02340c010101',

pwnlib/encoders/arm/alphanumeric/random_funcs.py

@@ -9,11 +9,11 @@ import struct
 # |                       RANDOM NUMBERS FUNCTIONS                         |
 # +------------------------------------------------------------------------+
 
-# get a random integer i (0<=i<max)
-# =================================
-def random_get_int(max):
-   return random.randint(0, max-1)
-   return struct.unpack("I", os.urandom(4))[0] % max
+# get a random integer i (0<=i<maxv)
+# ==================================
+def random_get_int(maxv):
+   return random.randrange(0, maxv)
+
 
 def randel(arr):
    return arr[random_get_int(len(arr))]

pwnlib/encoders/i386/__init__.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
 
+from pwnlib.encoders.i386 import ascii_shellcode
 from pwnlib.encoders.i386 import delta
 from pwnlib.encoders.i386 import xor

pwnlib/encoders/i386/ascii_shellcode.py

@@ -0,0 +1,300 @@
+""" Encoder to convert shellcode to shellcode that contains only ascii
+characters """
+# https://github.com/Gallopsled/pwntools/pull/1667
+
+from __future__ import absolute_import
+
+from itertools import product
+
+import six
+
+from pwnlib.context import LocalContext
+from pwnlib.context import context
+from pwnlib.encoders.encoder import Encoder
+from pwnlib.encoders.encoder import all_chars
+from pwnlib.util.iters import group
+from pwnlib.util.packing import *
+
+
+class AsciiShellcodeEncoder(Encoder):
+    """ Pack shellcode into only ascii characters that unpacks itself and
+    executes (on the stack)
+
+    The original paper this encoder is based on:
+    http://julianor.tripod.com/bc/bypass-msb.txt
+
+    A more visual explanation as well as an implementation in C:
+    https://github.com/VincentDary/PolyAsciiShellGen/blob/master/README.md#mechanism
+    """
+
+    def __init__(self, slop=20, max_subs=4):
+        """ Init
+
+        Args:
+            slop (int, optional): The amount esp will be increased by in the
+                allocation phase (In addition to the length of the packed
+                shellcode) as well as defines the size of the NOP sled (you can
+                increase/ decrease the size of the NOP sled by adding/removing
+                b'P'-s to/ from the end of the packed shellcode).
+                Defaults to 20.
+            max_subs (int, optional): The maximum amount of subtractions
+                allowed to be taken. This may be increased if you have a
+                relatively  restrictive ``avoid`` set. The more subtractions
+                there are, the bigger the packed shellcode will be.
+                Defaults to 4.
+        """
+        if six.PY2:
+            super(AsciiShellcodeEncoder, self).__init__()
+        elif six.PY3:
+            super().__init__()
+        self.slop = slop
+        self.max_subs = max_subs
+
+    @LocalContext
+    def __call__(self, raw_bytes, avoid=None, pcreg=None):
+        r""" Pack shellcode into only ascii characters that unpacks itself and
+        executes (on the stack)
+
+        Args:
+            raw_bytes (bytes): The shellcode to be packed
+            avoid (set, optional): Characters to avoid. Defaults to allow
+                printable ascii (0x21-0x7e).
+            pcreg (NoneType, optional): Ignored
+
+        Raises:
+            RuntimeError: A required character is in ``avoid`` (required
+                characters are characters which assemble into assembly
+                instructions and are used to unpack the shellcode onto the
+                stack, more details in the paper linked above ``\ - % T X P``).
+            RuntimeError: Not supported architecture
+            ArithmeticError: The allowed character set does not contain
+                two characters that when they are bitwise-anded with eachother
+                their result is 0
+            ArithmeticError: Could not find a correct subtraction sequence
+                to get to the the desired target value with the given ``avoid``
+                parameter
+
+        Returns:
+            bytes: The packed shellcode
+
+        Examples:
+
+            >>> context.update(arch='i386', os='linux')
+            >>> sc = b"\x83\xc4\x181\xc01\xdb\xb0\x06\xcd\x80Sh/ttyh/dev\x89\xe31\xc9f\xb9\x12'\xb0\x05\xcd\x80j\x17X1\xdb\xcd\x80j.XS\xcd\x801\xc0Ph//shh/bin\x89\xe3PS\x89\xe1\x99\xb0\x0b\xcd\x80"
+            >>> encoders.i386.ascii_shellcode.encode(sc)
+            b'TX-!!!!-"_``-~~~~P\\%!!!!%@@@@-!6!!-V~!!-~~<-P-!mha-a~~~P-!!L`-a^~~-~~~~P-!!if-9`~~P-!!!!-aOaf-~~~~P-!&!<-!~`~--~~~P-!!!!-!!H^-+A~~P-U!![-~A1~P-,<V!-~~~!-~~~GP-!2!8-j~O~P-!]!!-!~!r-y~w~P-c!!!-~<(+P-N!_W-~1~~P-!!]!-Mn~!-~~~<P-!<!!-r~!P-~~x~P-fe!$-~~S~-~~~~P-!!\'$-%z~~P-A!!!-~!#!-~*~=P-!7!!-T~!!-~~E^PPPPPPPPPPPPPPPPPPPPP'
+            >>> avoid = {'\x00', '\x83', '\x04', '\x87', '\x08', '\x8b', '\x0c', '\x8f', '\x10', '\x93', '\x14', '\x97', '\x18', '\x9b', '\x1c', '\x9f', ' ', '\xa3', '\xa7', '\xab', '\xaf', '\xb3', '\xb7', '\xbb', '\xbf', '\xc3', '\xc7', '\xcb', '\xcf', '\xd3', '\xd7', '\xdb', '\xdf', '\xe3', '\xe7', '\xeb', '\xef', '\xf3', '\xf7', '\xfb', '\xff', '\x80', '\x03', '\x84', '\x07', '\x88', '\x0b', '\x8c', '\x0f', '\x90', '\x13', '\x94', '\x17', '\x98', '\x1b', '\x9c', '\x1f', '\xa0', '\xa4', '\xa8', '\xac', '\xb0', '\xb4', '\xb8', '\xbc', '\xc0', '\xc4', '\xc8', '\xcc', '\xd0', '\xd4', '\xd8', '\xdc', '\xe0', '\xe4', '\xe8', '\xec', '\xf0', '\xf4', '\xf8', '\xfc', '\x7f', '\x81', '\x02', '\x85', '\x06', '\x89', '\n', '\x8d', '\x0e', '\x91', '\x12', '\x95', '\x16', '\x99', '\x1a', '\x9d', '\x1e', '\xa1', '\xa5', '\xa9', '\xad', '\xb1', '\xb5', '\xb9', '\xbd', '\xc1', '\xc5', '\xc9', '\xcd', '\xd1', '\xd5', '\xd9', '\xdd', '\xe1', '\xe5', '\xe9', '\xed', '\xf1', '\xf5', '\xf9', '\xfd', '\x01', '\x82', '\x05', '\x86', '\t', '\x8a', '\r', '\x8e', '\x11', '\x92', '\x15', '\x96', '\x19', '\x9a', '\x1d', '\x9e', '\xa2', '\xa6', '\xaa', '\xae', '\xb2', '\xb6', '\xba', '\xbe', '\xc2', '\xc6', '\xca', '\xce', '\xd2', '\xd6', '\xda', '\xde', '\xe2', '\xe6', '\xea', '\xee', '\xf2', '\xf6', '\xfa', '\xfe'}
+            >>> sc = shellcraft.echo("Hello world") + shellcraft.exit()
+            >>> ascii = encoders.i386.ascii_shellcode.encode(asm(sc), avoid)
+            >>> ascii += asm('jmp esp') # just for testing, the unpacker should also run on the stack
+            >>> ELF.from_bytes(ascii).process().recvall()
+            b'Hello world'
+        """
+        if not avoid:
+            vocab = bytearray(
+                b"!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~")
+        else:
+            required_chars = set('\\-%TXP')
+            allowed = set(all_chars)
+            if avoid.intersection(required_chars):
+                raise RuntimeError(
+                    '''These characters ({}) are required because they assemble
+                    into instructions used to unpack the shellcode'''.format(
+                        str(required_chars, 'ascii')))
+            allowed.difference_update(avoid)
+            vocab = bytearray(map(ord, allowed))
+
+        if context.arch != 'i386' or context.bits != 32:
+            raise RuntimeError('Only 32-bit i386 is currently supported')
+
+        int_size = context.bytes
+
+        # Prepend with NOPs for the NOP sled
+        shellcode = bytearray(b'\x90'*int_size + raw_bytes)
+        subtractions = self._get_subtractions(shellcode, vocab)
+        allocator = self._get_allocator(len(subtractions) + self.slop, vocab)
+        nop_sled = b'P' * self.slop  # push eax
+        return bytes(allocator + subtractions + nop_sled)
+
+    @LocalContext
+    def _get_allocator(self, size, vocab):
+        r""" Allocate enough space on the stack for the shellcode
+
+        int_size is taken from the context
+
+        Args:
+            size (int): The allocation size
+            vocab (bytearray): Allowed characters
+
+        Returns:
+            bytearray: The allocator shellcode
+
+        Examples:
+
+            >>> context.update(arch='i386', os='linux')
+            >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~')
+            >>> encoders.i386.ascii_shellcode.encode._get_allocator(300, vocab)
+            bytearray(b'TX-!!!!-!_``-t~~~P\\%!!!!%@@@@')
+        """
+        size += 0x1e  # add typical allocator size
+        int_size = context.bytes
+        # Use eax for subtractions because sub esp, X doesn't assemble to ascii
+        result = bytearray(b'TX')  # push esp; pop eax
+        # Set target to the `size` arg
+        target = bytearray(pack(size))
+        # All we are doing here is adding (subtracting) `size`
+        # to esp (to allocate space on the stack), so we don't care
+        # about esp's actual value. That's why the `last` parameter
+        # for `calc_subtractions` can just be zero
+        for subtraction in self._calc_subtractions(
+                bytearray(int_size), target, vocab):
+            # sub eax, subtraction
+            result += b'-' + subtraction
+        result += b'P\\'  # push eax, pop esp
+        # Zero out eax for the unpacking part
+        pos, neg = self._find_negatives(vocab)
+        # and eax, pos; and eax, neg ; (0b00010101 & 0b00101010 = 0b0)
+        result += flat((b'%', pos, b'%', neg))
+        return result
+
+    @LocalContext
+    def _find_negatives(self, vocab):
+        r""" Find two bitwise negatives in the vocab so that when they are
+        and-ed the result is 0.
+
+        int_size is taken from the context
+
+        Args:
+            vocab (bytearray): Allowed characters
+
+        Returns:
+            Tuple[int, int]: value A, value B
+
+        Raises:
+            ArithmeticError: The allowed character set does not contain
+                two characters that when they are bitwise-and-ed with eachother
+                the result is 0
+
+        Examples:
+
+            >>> context.update(arch='i386', os='linux')
+            >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~')
+            >>> a, b = encoders.i386.ascii_shellcode.encode._find_negatives(vocab)
+            >>> a & b
+            0
+        """
+        int_size = context.bytes
+        for products in product(vocab, vocab):
+            if products[0] & products[1] == 0:
+                return tuple(
+                    # pylint: disable=undefined-variable
+                    unpack(p8(x)*int_size)  # noqa: F405
+                    for x in bytearray(products)
+                )
+        else:
+            raise ArithmeticError(
+                'Could not find two bitwise negatives in the provided vocab')
+
+    @LocalContext
+    def _get_subtractions(self, shellcode, vocab):
+        r""" Covert the sellcode to sub eax and posh eax instructions
+
+        int_size is taken from the context
+
+        Args:
+            shellcode (bytearray): The shellcode to pack
+            vocab (bytearray): Allowed characters
+
+        Returns:
+            bytearray: packed shellcode
+
+        Examples:
+
+            >>> context.update(arch='i386', os='linux')
+            >>> sc = bytearray(b'ABCDEFGHIGKLMNOPQRSTUVXYZ')
+            >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~')
+            >>> encoders.i386.ascii_shellcode.encode._get_subtractions(sc, vocab)
+            bytearray(b'-(!!!-~NNNP-!=;:-f~~~-~~~~P-!!!!-edee-~~~~P-!!!!-eddd-~~~~P-!!!!-egdd-~~~~P-!!!!-eadd-~~~~P-!!!!-eddd-~~~~P')
+        """
+        int_size = context.bytes
+        result = bytearray()
+        last = bytearray(int_size)
+        # Group the shellcode into bytes of stack cell size, pad with NOPs
+        # if the shellcode does not divide into stack cell size and reverse.
+        # The shellcode will be reversed again back to it's original order once
+        # it's pushed onto the stack
+        sc = tuple(group(int_size, shellcode, 0x90))[::-1]
+        # Pack the shellcode to a sub/push sequence
+        for x in sc:
+            for subtraction in self._calc_subtractions(last, x, vocab):
+                result += b'-' + subtraction  # sub eax, ...
+            last = x
+            result += b'P'  # push eax
+        return result
+
+    @LocalContext
+    def _calc_subtractions(self, last, target, vocab):
+        r""" Given `target` and `last`, return a list of integers that when
+         subtracted from `last` will equal `target` while only constructing
+         integers from bytes in `vocab`
+
+        int_size is taken from the context
+
+        Args:
+            last (bytearray): Original value
+            target (bytearray): Desired value
+            vocab (bytearray): Allowed characters
+
+        Raises:
+            ArithmeticError: If a sequence of subtractions could not be found
+
+        Returns:
+            List[bytearray]: List of numbers that would need to be subtracted
+            from `last` to get to `target`
+
+        Examples:
+
+            >>> context.update(arch='i386', os='linux')
+            >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~')
+            >>> print(encoders.i386.ascii_shellcode.encode._calc_subtractions(bytearray(b'\x10'*4), bytearray(b'\x11'*4), vocab))
+            [bytearray(b'!!!!'), bytearray(b'`___'), bytearray(b'~~~~')]
+            >>> print(encoders.i386.ascii_shellcode.encode._calc_subtractions(bytearray(b'\x11\x12\x13\x14'), bytearray(b'\x15\x16\x17\x18'), vocab))
+            [bytearray(b'~}}}'), bytearray(b'~~~~')]
+        """
+        int_size = context.bytes
+        subtractions = [bytearray(int_size)]
+        for sub in range(self.max_subs):
+            carry = success_count = 0
+            for byte in range(int_size):
+                # Try all combinations of all the characters in vocab of
+                # `subtraction` characters in each combination. So if
+                # `max_subs` is 4 and we're on the second subtraction attempt,
+                # products will equal
+                # [\, ", #, %, ...], [\, ", #, %, ...], (0,), (0,)
+                for products in product(
+                    *[x <= sub and vocab or (0,) for x in range(self.max_subs)]
+                ):
+                    # Sum up all the products, carry from last byte and
+                    # the target
+                    attempt = target[byte] + carry + sum(products)
+                    # If the attempt equals last, we've found the combination
+                    if last[byte] == attempt & 0xff:
+                        carry = (attempt & 0xff00) >> 8
+                        # Update the result with the current `products`
+                        for p, i in zip(products, range(sub + 1)):
+                            subtractions[i][byte] = p
+                        success_count += 1
+                        break
+            if success_count == int_size:
+                return subtractions
+            else:
+                subtractions.append(bytearray(int_size))
+        else:
+            raise ArithmeticError(
+                str.format(
+                    '''Could not find the correct subtraction sequence
+                to get the the desired target ({}) from ({})''',
+                    target[byte], last[byte]))
+
+
+encode = AsciiShellcodeEncoder()

pwnlib/encoders/i386/xor.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # Source:
 # http://www.iodigitalsec.com/python-cascading-xor-polymorphic-shellcode-generator/
 #
@@ -66,7 +65,7 @@ end:
 
     def __call__(self, raw_bytes, avoid, pcreg=''):
         while len(raw_bytes) % context.bytes:
-            raw_bytes += '\x00'
+            raw_bytes += b'\x00'
 
         a, b = xor_pair(raw_bytes, avoid)
 

pwnlib/encoders/mips/xor.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # Source:
 # https://github.com/zcutlip/bowcaster/blob/master/src/bowcaster/encoders/mips.py
 #

pwnlib/filesystem/__init__.py

@@ -0,0 +1,8 @@
+# -*- coding: utf-8 -*-
+"""
+Handles file abstraction for local vs. remote (via ssh)
+"""
+from pwnlib.filesystem.path import Path
+from pwnlib.filesystem.ssh import SSHPath
+
+__all__ = ['SSHPath', 'Path']

pwnlib/filesystem/path.py

@@ -0,0 +1,18 @@
+import six
+import tempfile
+
+if six.PY3:
+    from pathlib import *
+else:
+    from pathlib2 import *
+
+@classmethod
+def mktemp(cls):
+    return cls(tempfile.mktemp())
+
+@classmethod
+def mkdtemp(cls):
+    return cls(tempfile.mkdtemp())
+
+Path.mktemp = mktemp
+Path.mkdtemp = mkdtemp

pwnlib/filesystem/ssh.py

@@ -0,0 +1,794 @@
+# -*- coding: utf-8 -*-
+"""
+Handles file abstraction for remote SSH files
+
+Emulates pathlib as much as possible, but does so through duck typing.
+"""
+import os
+import six
+import sys
+import tempfile
+import time
+
+from pwnlib.context import context
+from pwnlib.util.misc import read, write
+from pwnlib.util.packing import _decode
+
+if six.PY3:
+    from pathlib import *
+else:
+    from pathlib2 import *
+
+class SSHPath(PosixPath):
+    r"""Represents a file that exists on a remote filesystem.
+
+    See :class:`.ssh` for more information on how to set up an SSH connection.
+    See :py:class:`pathlib.Path` for documentation on what members and
+    properties this object has.
+
+    Arguments:
+        name(str): Name of the file
+        ssh(ssh): :class:`.ssh` object for manipulating remote files
+
+    Note:
+
+        You can avoid having to supply ``ssh=`` on every ``SSHPath`` by setting
+        :data:`.context.ssh_session`.  
+        In these examples we provide ``ssh=`` for clarity.
+
+    Examples:
+
+        First, create an SSH connection to the server.
+
+        >>> ssh_conn = ssh('travis', 'example.pwnme')
+
+        Let's use a temporary directory for our tests
+    
+        >>> _ = ssh_conn.set_working_directory()
+
+        Next, you can create SSHPath objects to represent the paths to files
+        on the remote system.
+
+        >>> f = SSHPath('filename', ssh=ssh_conn)
+        >>> f.touch()
+        >>> f.exists()
+        True
+        >>> f.resolve().path # doctests: +ELLIPSIS
+        '/tmp/.../filename'
+        >>> f.write_text('asdf ❤️')
+        >>> f.read_bytes()
+        b'asdf \xe2\x9d\xa4\xef\xb8\x8f'
+
+        ``context.ssh_session`` must be set to use the :meth:`.SSHPath.mktemp`
+        or :meth:`.SSHPath.mkdtemp` methods.
+
+        >>> context.ssh_session = ssh_conn
+        >>> SSHPath.mktemp() # doctest: +ELLIPSIS
+        SSHPath('...', ssh=ssh(user='travis', host='127.0.0.1'))
+    """
+
+    sep = '/'
+
+    def __init__(self, path, ssh=None):
+        self.path = self._s(path)
+        self.ssh = ssh or context.ssh_session
+
+        if self.ssh is None:
+            raise ValueError('SSHPath requires an ssh session.  Provide onee or set context.ssh_session.')
+
+    def _s(self, other):
+        # We want strings
+        if isinstance(other, str):
+            return other
+
+        # We don't want unicode
+        if isinstance(other, six.text_type):
+            return str(other)
+
+        # We also don't want binary
+        if isinstance(other, six.binary_type):
+            return str(_decode(other))
+
+    def _new(self, path, *a, **kw):
+        kw['ssh'] = self.ssh
+        path = self._s(path)
+        return SSHPath(path, *a, **kw)
+
+    def _run(self, *a, **kw):
+        with context.silent:
+            return self.ssh.run(*a, **kw)
+
+#---------------------------------- PUREPATH ----------------------------------
+    def __str__(self):
+        return self.path
+
+    def __fspath__(self):
+        return str(self)
+
+    def as_posix(self):
+        return self.path
+
+    def __bytes__(self):
+        return os.fsencode(self)
+
+    def __repr__(self):
+        return "{}({!r}, ssh={!r})".format(self.__class__.__name__, self.as_posix(), self.ssh)
+
+    def as_uri(self):
+        raise NotImplementedError()
+
+    def __eq__(self, other):
+        if not isinstance(other, SSHPath):
+            return str(self) == str(other)
+
+        if self.ssh.host != other.ssh.host:
+            return False
+
+        if self.path != other.path:
+            return False
+
+        return True
+
+    def __hash__(*a, **kw): ""; raise NotImplementedError
+    def __lt__(*a, **kw): ""; raise NotImplementedError
+    def __le__(*a, **kw): ""; raise NotImplementedError
+    def __gt__(*a, **kw): ""; raise NotImplementedError
+    def __ge__(*a, **kw): ""; raise NotImplementedError
+
+    @property
+    def anchor(self):
+        raise NotImplementedError()
+
+    @property
+    def name(self):
+        """Returns the name of the file.
+
+        >>> f = SSHPath('hello', ssh=ssh_conn)
+        >>> f.name
+        'hello'
+        """
+        return os.path.basename(self.path)
+
+    @property
+    def suffix(self):
+        """Returns the suffix of the file.
+
+        >>> f = SSHPath('hello.tar.gz', ssh=ssh_conn)
+        >>> f.suffix
+        '.gz'
+        """
+        if '.' not in self.name:
+            return ''
+        return self.name[self.name.rindex('.'):]
+
+    @property
+    def suffixes(self):
+        """Returns the suffixes of a file
+
+        >>> f = SSHPath('hello.tar.gz', ssh=ssh_conn)
+        >>> f.suffixes
+        '.tar.gz'
+        """
+
+        basename = self.name
+        if '.' not in basename:
+            return ''
+        return '.' + self.name.split('.', 1)[1]
+
+    @property
+    def stem(self):
+        """Returns the stem of a file without any extension
+        
+        >>> f = SSHPath('hello.tar.gz', ssh=ssh_conn)
+        >>> f.stem
+        'hello'
+        """
+        if '.' not in self.name:
+            return self.name
+        return self.name[:self.name.index('.')]
+
+    def with_name(self, name):
+        """Return a new path with the file name changed
+
+        >>> f = SSHPath('hello/world', ssh=ssh_conn)
+        >>> f.path
+        'hello/world'
+        >>> f.with_name('asdf').path
+        'hello/asdf'
+        """
+        if '/' not in self.path:
+            return name
+
+        path, _ = self.path.split(self.sep, 1)
+        path = self._new(path)
+        return path.joinpath(name)
+
+    def with_stem(self, name):
+        """Return a new path with the stem changed.
+
+        >>> f = SSHPath('hello/world.tar.gz', ssh=ssh_conn)
+        >>> f.with_stem('asdf').path
+        'hello/asdf.tar.gz'
+        """
+        return self.with_name(name + self.suffixes)
+
+    def with_suffix(self, suffix):
+        """Return a new path with the file suffix changed
+
+        >>> f = SSHPath('hello/world.tar.gz', ssh=ssh_conn)
+        >>> f.with_suffix('.tgz').path
+        'hello/world.tgz'
+        """
+        return self.with_name(self.stem + suffix)
+
+    def relative_to(self, *other):
+        raise NotImplementedError()
+
+    def is_relative_to(self, *other):
+        raise NotImplementedError()       
+
+    @property
+    def parts(self):
+        """Return the individual parts of the path
+    
+        >>> f = SSHPath('hello/world.tar.gz', ssh=ssh_conn)
+        >>> f.parts
+        ['hello', 'world.tar.gz']
+        """
+        return self.path.split(self.sep)
+
+    def joinpath(self, *args):
+        """Combine this path with one or several arguments.
+
+        >>> f = SSHPath('hello', ssh=ssh_conn)
+        >>> f.joinpath('world').path
+        'hello/world'
+        """
+        newpath = os.path.join(self.path, *args)
+        return SSHPath(newpath, ssh=self.ssh)
+    
+    # __truediv__
+    # __rtruediv__
+
+    @property
+    def parent(self):
+        """Return the parent of this path
+
+        >>> f = SSHPath('hello/world/file.txt', ssh=ssh_conn)
+        >>> f.parent.path
+        'hello/world'
+        """
+        a, b = self.path.rsplit(self.sep, 1)
+        if a:
+            return self._new(a)
+        return self
+
+    @property
+    def parents(self):
+        """Return the parents of this path, as individual parts
+
+        >>> f = SSHPath('hello/world/file.txt', ssh=ssh_conn)
+        >>> list(p.path for p in f.parents)
+        ['hello', 'world']
+        """
+        if '/' not in self.path:
+            return self._new('.')
+
+        return [self._new(p) for p in self.parent.path.split(self.sep)]
+
+    def is_absolute(self):
+        """Returns whether a path is absolute or not.
+
+        >>> f = SSHPath('hello/world/file.txt', ssh=ssh_conn)
+        >>> f.is_absolute()
+        False
+
+        >>> f = SSHPath('/hello/world/file.txt', ssh=ssh_conn)
+        >>> f.is_absolute()
+        True
+        """
+        return self.path.startswith(self.sep)
+
+    def is_reserved(self):
+        return False
+
+    def match(self, path_pattern):
+        raise NotImplementedError()
+
+#------------------------------------ PATH ------------------------------------
+
+    @property
+    def cwd(self):
+        return self._new(self.ssh.cwd)
+
+    @property
+    def home(self):
+        """Returns the home directory for the SSH connection
+
+        >>> f = SSHPath('...', ssh=ssh_conn)
+        >>> f.home # doctest: +ELLIPSIS
+        SSHPath('/home/...', ssh=ssh(user='...', host='127.0.0.1'))
+        """
+        path = self._run('echo ~').recvall().rstrip()
+        return self._new(path)
+
+    def samefile(self, other_path):
+        """Returns whether two files are the same
+
+        >>> a = SSHPath('a', ssh=ssh_conn)
+        >>> A = SSHPath('a', ssh=ssh_conn)
+        >>> x = SSHPath('x', ssh=ssh_conn)
+
+        >>> a.samefile(A)
+        True
+        >>> a.samefile(x)
+        False
+        """
+        if not isinstance(other_path, SSHPath):
+            return False
+
+        return self.absolute() == other_path.absolute()
+
+    def iterdir(self):
+        """Iterates over the contents of the directory
+
+        >>> directory = SSHPath('iterdir', ssh=ssh_conn)
+        >>> directory.mkdir()
+        >>> fileA = directory.joinpath('fileA')
+        >>> fileA.touch()
+        >>> fileB = directory.joinpath('fileB')
+        >>> fileB.touch()
+        >>> dirC = directory.joinpath('dirC')
+        >>> dirC.mkdir()
+        >>> [p.name for p in directory.iterdir()]
+        ['dirC', 'fileA', 'fileB']
+        """
+        for directory in sorted(self.ssh.sftp.listdir(self.path)):
+            yield self._new(directory)
+
+    def glob(self, pattern):
+        raise NotImplementedError()
+
+    def rglob(self, pattern):
+        raise NotImplementedError()
+
+    def absolute(self):
+        """Return the absolute path to a file, preserving e.g. "../".
+        The current working directory is determined via the :class:`.ssh`
+        member :attr:`.ssh.cwd`.
+
+        Example:
+            
+            >>> f = SSHPath('absA/../absB/file', ssh=ssh_conn)
+            >>> f.absolute().path # doctest: +ELLIPSIS
+            '/.../absB/file'
+        """
+        path = os.path.normpath(self.path)
+
+        if self.is_absolute():
+            return self._new(path)
+
+        return self._new(os.path.join(self.ssh.cwd, path))
+
+    def resolve(self, strict=False):
+        """Return the absolute path to a file, resolving any '..' or symlinks.
+        The current working directory is determined via the :class:`.ssh`
+        member :attr:`.ssh.cwd`.
+
+        Note:
+
+            The file must exist to call resolve().
+
+        Examples:
+
+            >>> f = SSHPath('resA/resB/../resB/file', ssh=ssh_conn)
+
+            >>> f.resolve().path # doctest: +ELLIPSIS
+            Traceback (most recent call last):
+            ...
+            ValueError: Could not normalize path: '/.../resA/resB/file'
+
+            >>> f.parent.absolute().mkdir(parents=True)
+            >>> list(f.parent.iterdir())
+            []
+
+            >>> f.touch()
+            >>> f.resolve() # doctest: +ELLIPSIS
+            SSHPath('/.../resA/resB/file', ssh=ssh(user='...', host='127.0.0.1'))
+        """
+        path = self.absolute().path
+        path = os.path.normpath(path)
+
+        if six.PY2:
+            error_type = IOError
+        else:
+            error_type = FileNotFoundError
+
+        try:
+            return self._new(self.ssh.sftp.normalize(path))
+        except error_type as e:
+            raise ValueError("Could not normalize path: %r" % path)
+
+    def stat(self):
+        """Returns the permissions and other information about the file
+
+        >>> f = SSHPath('filename', ssh=ssh_conn)
+        >>> f.touch()
+        >>> stat = f.stat()
+        >>> stat.st_size
+        0
+        >>> '%o' % stat.st_mode # doctest: +ELLIPSIS
+        '...664'
+        """
+        return self.ssh.sftp.stat(self.path)
+
+    def owner(self):
+        raise NotImplementedError()
+
+    def group(self):
+        raise NotImplementedError()
+
+    def open(self, *a, **kw):
+        """Return a file-like object for this path.
+
+        This currently seems to be broken in Paramiko.
+
+        >>> f = SSHPath('filename', ssh=ssh_conn)
+        >>> f.write_text('Hello')
+        >>> fo = f.open(mode='r+')
+        >>> fo                      # doctest: +ELLIPSIS
+        <paramiko.sftp_file.SFTPFile object at ...>
+        >>> fo.read('asdfasdf')     # doctest: +SKIP
+        b'Hello'
+        """
+        return self.ssh.sftp.open(self.path, *a, **kw)
+
+    def read_bytes(self):
+        """Read bytes from the file at this path
+
+        >>> f = SSHPath('/etc/passwd', ssh=ssh_conn)
+        >>> f.read_bytes()[:10]
+        b'root:x:0:0'
+        """
+        return self.ssh.read(str(self.absolute()))
+
+    def read_text(self):
+        """Read text from the file at this path
+
+        >>> f = SSHPath('/etc/passwd', ssh=ssh_conn)
+        >>> f.read_text()[:10]
+        'root:x:0:0'
+        """
+        return self._s(self.read_bytes())
+
+    def write_bytes(self, data):
+        r"""Write bytes to the file at this path
+
+        >>> f = SSHPath('somefile', ssh=ssh_conn)
+        >>> f.write_bytes(b'\x00HELLO\x00')
+        >>> f.read_bytes()
+        b'\x00HELLO\x00'
+        """
+        self.ssh.write(str(self.absolute()), data)
+
+    def write_text(self, data):
+        r"""Write text to the file at this path
+
+        >>> f = SSHPath('somefile', ssh=ssh_conn)
+        >>> f.write_text("HELLO 😭")
+        >>> f.read_bytes()
+        b'HELLO \xf0\x9f\x98\xad'
+        >>> f.read_text()
+        'HELLO 😭'
+        """
+        self.write_bytes(data)
+
+    def readlink(self):
+        data = self.ssh.readlink(self.path)
+        if data == b'':
+            data = self.path
+        return self._new(data)
+
+    def touch(self):
+        """Touch a file (i.e. make it exist)
+
+        >>> f = SSHPath('touchme', ssh=ssh_conn)
+        >>> f.exists()
+        False
+        >>> f.touch()
+        >>> f.exists()
+        True
+        """
+        self.ssh.write(self.path, b'')
+        # self.ssh.sftp.truncate(self.path, 0)
+
+    def mkdir(self, mode=0o777, parents=False, exist_ok=True):
+        r"""Make a directory at the specified path
+
+        >>> f = SSHPath('dirname', ssh=ssh_conn)
+        >>> f.mkdir()
+        >>> f.exists()
+        True
+
+        >>> f = SSHPath('dirA/dirB/dirC', ssh=ssh_conn)
+        >>> f.mkdir(parents=True)
+        >>> ssh_conn.run(['ls', '-la', f.absolute().path]).recvline()
+        b'total 8\n'
+        """
+        if exist_ok and self.is_dir():
+            return
+
+        if not parents:
+            self.ssh.sftp.mkdir(self.path, mode=mode)
+            return
+
+        if not self.is_absolute():
+            path = self._new(self.ssh.cwd)
+        else:
+            path = self._new('/')
+
+        parts = self.path.split(self.sep)
+
+        for part in parts:
+            # Catch against common case, need to normalize path
+            if part == '..':
+                raise ValueError("Cannot create directory '..'")
+
+            path = path.joinpath(part)
+
+            # Don't create directories that already exist            
+            try:
+                path.mkdir(mode=mode)
+            except OSError:
+                raise OSError("Could not create directory %r" % path)
+
+    def chmod(self, mode):
+        """Change the permissions of a file
+
+        >>> f = SSHPath('chmod_me', ssh=ssh_conn)
+        >>> f.touch() # E
+        >>> '0o%o' % f.stat().st_mode
+        '0o100664'
+        >>> f.chmod(0o777)
+        >>> '0o%o' % f.stat().st_mode
+        '0o100777'
+        """
+        self.ssh.sftp.chmod(self.path, mode)
+
+    def lchmod(*a, **kw):
+        raise NotImplementedError()
+
+    def unlink(self, missing_ok=False):
+        """Remove an existing file.
+
+        TODO:
+
+            This test fails catastrophically if the file name is unlink_me
+            (note the underscore)
+
+        Example:
+
+            >>> f = SSHPath('unlink_me', ssh=ssh_conn)
+            >>> f.exists()
+            False
+            >>> f.touch()
+            >>> f.exists()
+            True
+            >>> f.unlink()
+            >>> f.exists()
+            False
+
+            Note that unlink only works on files.
+
+            >>> f.mkdir()
+            >>> f.unlink()
+            Traceback (most recent call last):
+            ...
+            ValueError: Cannot unlink SSHPath(...)): is not a file
+        """
+        try:
+            self.ssh.sftp.remove(str(self))
+        except (IOError, OSError) as e:
+            if self.exists() and not self.is_file():
+                raise ValueError("Cannot unlink %r: is not a file" % self)
+            if not missing_ok:
+                raise e
+
+    def rmdir(self):
+        """Remove an existing directory.
+
+        Example:
+
+            >>> f = SSHPath('rmdir_me', ssh=ssh_conn)
+            >>> f.mkdir()
+            >>> f.is_dir()
+            True
+            >>> f.rmdir()
+            >>> f.exists()
+            False
+        """
+        if not self.exists():
+            return
+
+        if not self.is_dir():
+            raise ValueError("Cannot rmdir %r: not a directory" % self)
+
+        self.ssh.sftp.rmdir(self.path)
+
+    def link_to(self, target):
+        raise NotImplementedError()
+
+    def symlink_to(self, target):
+        r"""Create a symlink at this path to the provided target
+
+        Todo:
+
+            Paramiko's documentation is wrong and inverted.
+            https://github.com/paramiko/paramiko/issues/1821
+
+        Example:
+
+            >>> a = SSHPath('link_name', ssh=ssh_conn)
+            >>> b = SSHPath('link_target', ssh=ssh_conn)
+            >>> a.symlink_to(b)
+            >>> a.write_text("Hello")
+            >>> b.read_text()
+            'Hello'
+        """
+        if isinstance(target, SSHPath):
+            target = target.path
+
+        self.ssh.sftp.symlink(target, self.path)
+
+    def rename(self, target):
+        """Rename a file to the target path
+
+        Example:
+
+            >>> a = SSHPath('rename_from', ssh=ssh_conn)
+            >>> b = SSHPath('rename_to', ssh=ssh_conn)
+            >>> a.touch()
+            >>> b.exists()
+            False
+            >>> a.rename(b)
+            >>> b.exists()
+            True
+        """
+        if isinstance(target, SSHPath):
+            target = target.path
+
+        self.ssh.sftp.rename(self.path, target)
+
+    def replace(self, target):
+        """Replace target file with file at this path
+
+        Example:
+
+            >>> a = SSHPath('rename_from', ssh=ssh_conn)
+            >>> a.write_text('A')
+            >>> b = SSHPath('rename_to', ssh=ssh_conn)
+            >>> b.write_text('B')
+            >>> a.replace(b)
+            >>> b.read_text()
+            'A'
+        """
+        if isinstance(target, SSHPath):
+            target = target.path
+
+        self._new(target).unlink(missing_ok=True)
+        self.rename(target)
+
+    def exists(self):
+        """Returns True if the path exists
+
+        Example:
+
+            >>> a = SSHPath('exists', ssh=ssh_conn)
+            >>> a.exists()
+            False
+            >>> a.touch()
+            >>> a.exists()
+            True
+            >>> a.unlink()
+            >>> a.exists()
+            False
+        """
+        try:
+            self.stat()
+            return True
+        except IOError:
+            return False
+
+    def is_dir(self):
+        """Returns True if the path exists and is a directory
+        
+        Example:
+
+            >>> f = SSHPath('is_dir', ssh=ssh_conn)
+            >>> f.is_dir()
+            False
+            >>> f.touch()
+            >>> f.is_dir()
+            False
+            >>> f.unlink()
+            >>> f.mkdir()
+            >>> f.is_dir()
+            True
+        """
+        if not self.exists():
+            return False
+
+        if self.stat().st_mode & 0o040000:
+            return True
+
+        return False
+
+    def is_file(self):
+        """Returns True if the path exists and is a file
+        
+        Example:
+
+            >>> f = SSHPath('is_file', ssh=ssh_conn)
+            >>> f.is_file()
+            False
+            >>> f.touch()
+            >>> f.is_file()
+            True
+            >>> f.unlink()
+            >>> f.mkdir()
+            >>> f.is_file()
+            False
+        """
+        if not self.exists():
+            return False
+
+        if self.stat().st_mode & 0o040000:
+            return False
+
+        return True
+
+    def is_symlink(self):
+        raise NotImplementedError()
+
+    def is_block_device(self):
+        raise NotImplementedError()
+
+    def is_char_device(self):
+        raise NotImplementedError()
+
+    def is_fifo(self):
+        raise NotImplementedError()
+
+    def is_socket(self):
+        raise NotImplementedError()
+
+    def expanduser(self):
+        """Expands a path that starts with a tilde
+
+        Example:
+
+            >>> f = SSHPath('~/my-file', ssh=ssh_conn)
+            >>> f.path
+            '~/my-file'
+            >>> f.expanduser().path # doctest: +ELLIPSIS
+            '/home/.../my-file'
+        """
+        if not self.path.startswith('~/'):
+            return self
+        
+        home = self.home
+        subpath = self.path.replace('~/', '')
+        return home.joinpath(subpath)
+
+#----------------------------- PWNTOOLS ADDITIONS -----------------------------
+    @classmethod
+    def mktemp(cls):
+        temp = _decode(context.ssh_session.mktemp())
+        return SSHPath(temp, ssh=context.ssh_session)
+
+    @classmethod
+    def mkdtemp(self):
+        temp = _decode(context.ssh_session.mkdtemp())
+        return SSHPath(temp, ssh=context.ssh_session)
+
+__all__ = ['SSHPath']

pwnlib/flag/flag.py

@@ -58,7 +58,7 @@ def submit_flag(flag,
                       exploit,
                       target,
                       team,
-                      ''])
+                      '']).encode('ascii')
 
     if os.path.exists(env_file):
         write(env_file, data)
@@ -69,4 +69,4 @@ def submit_flag(flag,
             r.send(data)
             return r.recvall(timeout=1)
     except Exception:
-        log.warn("Could not submit flag %r to %s:%s" % (flag, server, port))
+        log.warn("Could not submit flag %r to %s:%s", flag, server, port)

pwnlib/fmtstr.py

@@ -851,7 +851,7 @@ class FmtStr(object):
         payload = b"START%%%d$pEND" % offset
         leak = self.execute_fmt(prefix + payload)
         try:
-            leak = re.findall(br"START(.*)END", leak, re.MULTILINE | re.DOTALL)[0]
+            leak = re.findall(br"START(.*?)END", leak, re.MULTILINE | re.DOTALL)[0]
             leak = int(leak, 16)
         except ValueError:
             leak = 0

pwnlib/gdb.py

@@ -41,6 +41,56 @@ the very first instruction (or even the start of ``main``), you instead should u
 When you use :func:`debug`, the return value is a :class:`.tube` object
 that you interact with exactly like normal.
 
+Using GDB Python API
+~~~~~~~~~~~~~~~~~~~~
+
+GDB provides Python API, which is documented at
+https://sourceware.org/gdb/onlinedocs/gdb/Python-API.html. Pwntools allows you
+to call it right from the exploit, without having to write a gdbscript. This is
+useful for inspecting program state, e.g. asserting that leaked values are
+correct, or that certain packets trigger a particular code path or put the heap
+in a desired state.
+
+Pass ``api=True`` to :func:`attach` or :func:`debug` in order to enable GDB
+Python API access. Pwntools will then connect to GDB using RPyC library:
+https://rpyc.readthedocs.io/en/latest/.
+
+At the moment this is an experimental feature with the following limitations:
+
+- Only Python 3 is supported.
+
+  Well, technically that's not quite true. The real limitation is that your
+  GDB's Python interpreter major version should be the same as that of
+  Pwntools. However, most GDBs use Python 3 nowadays.
+
+  Different minor versions are allowed as long as no incompatible values are
+  sent in either direction. See
+  https://rpyc.readthedocs.io/en/latest/install.html#cross-interpreter-compatibility
+  for more information.
+
+  Use
+
+  ::
+
+      $ gdb -batch -ex 'python import sys; print(sys.version)'
+
+  in order to check your GDB's Python version.
+- If your GDB uses a different Python interpreter than Pwntools (for example,
+  because you run Pwntools out of a virtualenv), you should install ``rpyc``
+  package into its ``sys.path``. Use
+
+  ::
+
+      $ gdb -batch -ex 'python import rpyc'
+
+  in order to check whether this is necessary.
+- Only local processes are supported.
+- It is not possible to tell whether ``gdb.execute('continue')`` will be
+  executed synchronously or asynchronously (in gdbscripts it is always
+  synchronous). Therefore it is recommended to use either the explicitly
+  synchronous :func:`pwnlib.gdb.Gdb.continue_and_wait` or the explicitly
+  asynchronous :func:`pwnlib.gdb.Gdb.continue_nowait` instead.
+
 Tips and Troubleshooting
 ------------------------
 
@@ -91,12 +141,18 @@ Member Documentation
 from __future__ import absolute_import
 from __future__ import division
 
+from contextlib import contextmanager
 import os
+import platform
+import psutil
 import random
 import re
 import shlex
 import six
+import six.moves
+import socket
 import tempfile
+from threading import Event
 import time
 
 from pwnlib import adb
@@ -112,13 +168,14 @@ from pwnlib.context import context
 from pwnlib.log import getLogger
 from pwnlib.timeout import Timeout
 from pwnlib.util import misc
+from pwnlib.util import packing
 from pwnlib.util import proc
 
 log = getLogger(__name__)
 
 @LocalContext
-def debug_assembly(asm, gdbscript=None, vma=None):
-    r"""debug_assembly(asm, gdbscript=None, vma=None) -> tube
+def debug_assembly(asm, gdbscript=None, vma=None, api=False):
+    r"""debug_assembly(asm, gdbscript=None, vma=None, api=False) -> tube
 
     Creates an ELF file, and launches it under a debugger.
 
@@ -130,6 +187,7 @@ def debug_assembly(asm, gdbscript=None, vma=None):
         asm(str): Assembly code to debug
         gdbscript(str): Script to run in GDB
         vma(int): Base address to load the shellcode at
+        api(bool): Enable access to GDB Python API
         \**kwargs: Override any :obj:`pwnlib.context.context` values.
 
     Returns:
@@ -152,17 +210,18 @@ def debug_assembly(asm, gdbscript=None, vma=None):
         adb.push(tmp_elf, android_path)
         tmp_elf = android_path
 
-    return debug(tmp_elf, gdbscript=gdbscript, arch=context.arch)
+    return debug(tmp_elf, gdbscript=gdbscript, arch=context.arch, api=api)
 
 @LocalContext
-def debug_shellcode(data, gdbscript=None, vma=None):
-    r"""debug_shellcode(data, gdbscript=None, vma=None) -> tube
+def debug_shellcode(data, gdbscript=None, vma=None, api=False):
+    r"""debug_shellcode(data, gdbscript=None, vma=None, api=False) -> tube
     Creates an ELF file, and launches it under a debugger.
 
     Arguments:
         data(str): Assembled shellcode bytes
         gdbscript(str): Script to run in GDB
         vma(int): Base address to load the shellcode at
+        api(bool): Enable access to GDB Python API
         \**kwargs: Override any :obj:`pwnlib.context.context` values.
 
     Returns:
@@ -188,7 +247,7 @@ def debug_shellcode(data, gdbscript=None, vma=None):
         adb.push(tmp_elf, android_path)
         tmp_elf = android_path
 
-    return debug(tmp_elf, gdbscript=gdbscript, arch=context.arch)
+    return debug(tmp_elf, gdbscript=gdbscript, arch=context.arch, api=api)
 
 def _gdbserver_args(pid=None, path=None, args=None, which=None, env=None):
     """_gdbserver_args(pid=None, path=None, args=None, which=None, env=None) -> list
@@ -241,9 +300,9 @@ def _gdbserver_args(pid=None, path=None, args=None, which=None, env=None):
         env_args = []
         for key in tuple(env):
             if key.startswith('LD_'): # LD_PRELOAD / LD_LIBRARY_PATH etc.
-                env_args.append('{}={}'.format(key, env.pop(key)))
+                env_args.append('{}="{}"'.format(key, env.pop(key)))
             else:
-                env_args.append('{}={}'.format(key, env[key]))
+                env_args.append('{}="{}"'.format(key, env[key]))
         gdbserver_args += ['--wrapper', 'env', '-i'] + env_args + ['--']
 
     gdbserver_args += ['localhost:0']
@@ -260,10 +319,13 @@ def _gdbserver_port(gdbserver, ssh):
 
     if process_created.startswith(b'ERROR:'):
         raise ValueError(
-            'Failed to spawn process under gdbserver. gdbserver error message: %s' % process_created
+            'Failed to spawn process under gdbserver. gdbserver error message: %r' % process_created
         )
 
-    gdbserver.pid   = int(process_created.split()[-1], 0)
+    try:
+        gdbserver.pid   = int(process_created.split()[-1], 0)
+    except ValueError:
+        log.error('gdbserver did not output its pid (maybe chmod +x?): %r', process_created)
 
     listening_on = b''
     while b'Listening' not in listening_on:
@@ -301,7 +363,7 @@ def _get_runner(ssh=None):
     else:                          return tubes.process.process
 
 @LocalContext
-def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kwargs):
+def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, api=False, **kwargs):
     r"""
     Launch a GDB server with the specified command line,
     and launches GDB to attach to it.
@@ -314,9 +376,12 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kw
         ssh(:class:`.ssh`): Remote ssh session to use to launch the process.
         sysroot(str): Foreign-architecture sysroot, used for QEMU-emulated binaries
             and Android targets.
+        api(bool): Enable access to GDB Python API.
 
     Returns:
-        :class:`.process` or :class:`.ssh_channel`: A tube connected to the target process
+        :class:`.process` or :class:`.ssh_channel`: A tube connected to the target process.
+        When ``api=True``, ``gdb`` member of the returned object contains a :class:`Gdb`
+        instance.
 
     Notes:
 
@@ -350,55 +415,98 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kw
 
     Examples:
 
-    >>> # Create a new process, and stop it at 'main'
-    >>> io = gdb.debug('bash', '''
-    ... break main
-    ... continue
-    ... ''')
-    >>> # Send a command to Bash
-    >>> io.sendline("echo hello")
-    >>> io.recvline()
-    b'hello\n'
-    >>> # Interact with the process
-    >>> io.interactive() # doctest: +SKIP
-    >>> io.close()
+        Create a new process, and stop it at 'main'
 
-    >>> # Create a new process, and stop it at '_start'
-    >>> io = gdb.debug('bash', '''
-    ... # Wait until we hit the main executable's entry point
-    ... break _start
-    ... continue
-    ...
-    ... # Now set breakpoint on shared library routines
-    ... break malloc
-    ... break free
-    ... continue
-    ... ''')
-    >>> # Send a command to Bash
-    >>> io.sendline("echo hello")
-    >>> io.recvline()
-    b'hello\n'
-    >>> # Interact with the process
-    >>> io.interactive() # doctest: +SKIP
-    >>> io.close()
+        >>> io = gdb.debug('bash', '''
+        ... break main
+        ... continue
+        ... ''')
 
-    You can use :func:`debug` to spawn new processes on remote machines as well,
-    by using the ``ssh=`` keyword to pass in your :class:`.ssh` instance.
+        Send a command to Bash
 
-    >>> # Connect to the SSH server
-    >>> # Start a process on the server
-    >>> shell = ssh('travis', 'example.pwnme', password='demopass')
-    >>> io = gdb.debug(['bash'],
-    ...                 ssh = shell,
-    ...                 gdbscript = '''
-    ... break main
-    ... continue
-    ... ''')
-    >>> # Send a command to Bash
-    >>> io.sendline("echo hello")
-    >>> # Interact with the process
-    >>> io.interactive() # doctest: +SKIP
-    >>> io.close()
+        >>> io.sendline(b"echo hello")
+        >>> io.recvline()
+        b'hello\n'
+
+        Interact with the process
+
+        >>> io.interactive() # doctest: +SKIP
+        >>> io.close()
+
+        Create a new process, and stop it at '_start'
+
+        >>> io = gdb.debug('bash', '''
+        ... # Wait until we hit the main executable's entry point
+        ... break _start
+        ... continue
+        ...
+        ... # Now set breakpoint on shared library routines
+        ... break malloc
+        ... break free
+        ... continue
+        ... ''')
+
+        Send a command to Bash
+
+        >>> io.sendline(b"echo hello")
+        >>> io.recvline()
+        b'hello\n'
+
+        Interact with the process
+
+        >>> io.interactive() # doctest: +SKIP
+        >>> io.close()
+
+    Using GDB Python API:
+
+    .. doctest
+       :skipif: six.PY2
+
+        Debug a new process
+
+        >>> io = gdb.debug(['echo', 'foo'], api=True)
+
+        Stop at 'write'
+
+        >>> bp = io.gdb.Breakpoint('write', temporary=True)
+        >>> io.gdb.continue_and_wait()
+
+        Dump 'count'
+
+        >>> count = io.gdb.parse_and_eval('$rdx')
+        >>> long = io.gdb.lookup_type('long')
+        >>> int(count.cast(long))
+        4
+
+        Resume the program
+
+        >>> io.gdb.continue_nowait()
+        >>> io.recvline()
+        b'foo\n'
+
+
+    Using SSH:
+
+        You can use :func:`debug` to spawn new processes on remote machines as well,
+        by using the ``ssh=`` keyword to pass in your :class:`.ssh` instance.
+
+        Connect to the SSH server and start a process on the server
+
+        >>> shell = ssh('travis', 'example.pwnme', password='demopass')
+        >>> io = gdb.debug(['whoami'],
+        ...                 ssh = shell,
+        ...                 gdbscript = '''
+        ... break main
+        ... continue
+        ... ''')
+
+        Send a command to Bash
+
+        >>> io.sendline(b"echo hello")
+
+        Interact with the process
+        >>> io.interactive() # doctest: +SKIP
+        >>> io.close()
     """
     if isinstance(args, six.integer_types + (tubes.process.process, tubes.ssh.ssh_channel)):
         log.error("Use gdb.attach() to debug a running process")
@@ -412,6 +520,9 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kw
     which  = _get_which(ssh)
     gdbscript = gdbscript or ''
 
+    if api and runner is not tubes.process.process:
+        raise ValueError('GDB Python API is supported only for local processes')
+
     if context.noptrace:
         log.warn_once("Skipping debugger since context.noptrace==True")
         return runner(args, executable=exe, env=env)
@@ -424,7 +535,10 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kw
         sysroot = sysroot or qemu.ld_prefix(env=env)
         if not qemu_user:
             log.error("Cannot debug %s binaries without appropriate QEMU binaries" % context.arch)
-        qemu_args = [qemu_user, '-g', str(qemu_port)]
+        if context.os == 'baremetal':
+            qemu_args = [qemu_user, '-S', '-gdb', 'tcp::' + str(qemu_port)]
+        else:
+            qemu_args = [qemu_user, '-g', str(qemu_port)]
         if sysroot:
             qemu_args += ['-L', sysroot]
         args = qemu_args + args
@@ -459,13 +573,16 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kw
     if not ssh and context.os == 'android':
         host = context.adb_host
 
-    attach((host, port), exe=exe, gdbscript=gdbscript, ssh=ssh, sysroot=sysroot)
+    tmp = attach((host, port), exe=exe, gdbscript=gdbscript, ssh=ssh, sysroot=sysroot, api=api)
+    if api:
+        _, gdb = tmp
+        gdbserver.gdb = gdb
 
     # gdbserver outputs a message when a client connects
     garbage = gdbserver.recvline(timeout=1)
 
     # Some versions of gdbserver output an additional message
-    garbage2 = gdbserver.recvline_startswith(b"Remote debugging from host ", timeout=1)
+    garbage2 = gdbserver.recvline_startswith(b"Remote debugging from host ", timeout=2)
 
     return gdbserver
 
@@ -506,8 +623,97 @@ def binary():
 
     return gdb
 
+class Breakpoint:
+    """Mirror of ``gdb.Breakpoint`` class.
+
+    See https://sourceware.org/gdb/onlinedocs/gdb/Breakpoints-In-Python.html
+    for more information.
+    """
+
+    def __init__(self, conn, *args, **kwargs):
+        """Do not create instances of this class directly.
+
+        Use ``pwnlib.gdb.Gdb.Breakpoint`` instead.
+        """
+        # Creates a real breakpoint and connects it with this mirror
+        self.conn = conn
+        self.server_breakpoint = conn.root.set_breakpoint(
+            self, hasattr(self, 'stop'), *args, **kwargs)
+
+    def __getattr__(self, item):
+        """Return attributes of the real breakpoint."""
+        if item in (
+                '____id_pack__',
+                '__name__',
+                '____conn__',
+                'stop',
+        ):
+            # Ignore RPyC netref attributes.
+            # Also, if stop() is not defined, hasattr() call in our
+            # __init__() will bring us here. Don't contact the
+            # server in this case either.
+            raise AttributeError()
+        return getattr(self.server_breakpoint, item)
+
+    def exposed_stop(self):
+        # Handle stop() call from the server.
+        return self.stop()
+
+class Gdb:
+    """Mirror of ``gdb`` module.
+
+    See https://sourceware.org/gdb/onlinedocs/gdb/Basic-Python.html for more
+    information.
+    """
+
+    def __init__(self, conn):
+        """Do not create instances of this class directly.
+
+        Use :func:`attach` or :func:`debug` with ``api=True`` instead.
+        """
+        self.conn = conn
+
+        class _Breakpoint(Breakpoint):
+            def __init__(self, *args, **kwargs):
+                super().__init__(conn, *args, **kwargs)
+
+        self.Breakpoint = _Breakpoint
+        self.stopped = Event()
+
+        def stop_handler(event):
+            self.stopped.set()
+
+        self.events.stop.connect(stop_handler)
+
+    def __getattr__(self, item):
+        """Provide access to the attributes of `gdb` module."""
+        return getattr(self.conn.root.gdb, item)
+
+    def wait(self):
+        """Wait until the program stops."""
+        self.stopped.wait()
+        self.stopped.clear()
+
+    def interrupt_and_wait(self):
+        """Interrupt the program and wait until it stops."""
+        self.execute('interrupt')
+        self.wait()
+
+    def continue_nowait(self):
+        """Continue the program. Do not wait until it stops again."""
+        self.execute('continue &')
+
+    def continue_and_wait(self):
+        """Continue the program and wait until it stops again."""
+        self.continue_nowait()
+        self.wait()
+
+    def quit(self):
+        """Terminate GDB."""
+        self.conn.root.quit()
+
 @LocalContext
-def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysroot = None):
+def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysroot = None, api = False):
     r"""
     Start GDB in a new terminal and attach to `target`.
 
@@ -520,9 +726,11 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
         gdb_args(list): List of additional arguments to pass to GDB.
         sysroot(str): Foreign-architecture sysroot, used for QEMU-emulated binaries
             and Android targets.
+        api(bool): Enable access to GDB Python API.
 
     Returns:
         PID of the GDB process (or the window which it is running in).
+        When ``api=True``, a (PID, :class:`Gdb`) tuple.
 
     Notes:
 
@@ -564,10 +772,43 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
         ... ''')
         >>> io.recvline()
         b'Hello from process debugger!\n'
-        >>> io.sendline('echo Hello from bash && exit')
+        >>> io.sendline(b'echo Hello from bash && exit')
         >>> io.recvall()
         b'Hello from bash\n'
 
+        Using GDB Python API:
+
+        .. doctest
+           :skipif: six.PY2
+
+            >>> io = process('bash')
+
+            Attach a debugger
+
+            >>> pid, io_gdb = gdb.attach(io, api=True)
+
+            Force the program to write something it normally wouldn't
+
+            >>> io_gdb.execute('call puts("Hello from process debugger!")')
+
+            Resume the program
+
+            >>> io_gdb.continue_nowait()
+
+            Observe the forced line
+
+            >>> io.recvline()
+            b'Hello from process debugger!\n'
+
+            Interact with the program in a regular way
+
+            >>> io.sendline(b'echo Hello from bash && exit')
+
+            Observe the results
+
+            >>> io.recvall()
+            b'Hello from bash\n'
+
         Attach to the remote process from a :class:`.remote` or :class:`.listen` tube,
         as long as it is running on the same machine.
 
@@ -582,12 +823,12 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
         ... ''')
         >>> io.recvline()
         b'Hello from remote debugger!\n'
-        >>> io.sendline('echo Hello from bash && exit')
+        >>> io.sendline(b'echo Hello from bash && exit')
         >>> io.recvall()
         b'Hello from bash\n'
 
         Attach to processes running on a remote machine via an SSH :class:`.ssh` process
-        
+
         >>> shell = ssh('travis', 'example.pwnme', password='demopass')
         >>> io = shell.process(['cat'])
         >>> pid = gdb.attach(io, gdbscript='''
@@ -598,7 +839,7 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
         ... ''')
         >>> io.recvline(timeout=5)  # doctest: +SKIP
         b'Hello from ssh debugger!\n'
-        >>> io.sendline('This will be echoed back')
+        >>> io.sendline(b'This will be echoed back')
         >>> io.recvline()
         b'This will be echoed back\n'
         >>> io.close()
@@ -632,7 +873,7 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
         if context.os == 'android':
             pre += 'set gnutarget ' + _bfdname() + '\n'
 
-        if exe:
+        if exe and context.os != 'baremetal':
             pre += 'file %s\n' % exe
 
     # let's see if we can find a pid to attach to
@@ -649,7 +890,7 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
 
         pids = list(pidof(target))
         if not pids:
-            log.error('No such process: %s' % target)
+            log.error('No such process: %s', target)
         pid = pids[0]
         log.info('Attaching to youngest process "%s" (PID = %d)' %
                  (target, pid))
@@ -672,12 +913,9 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
             cmd = ['sshpass', '-p', shell.password] + cmd
         if shell.keyfile:
             cmd += ['-i', shell.keyfile]
-        exefile = target.executable
-        cmd += ['gdb -q %s %s -x "%s"' % (exefile,
-                                       target.pid,
-                                       tmpfile)]
+        cmd += ['gdb', '-q', target.executable, target.pid, '-x', tmpfile]
 
-        misc.run_in_new_terminal(' '.join(cmd))
+        misc.run_in_new_terminal(cmd)
         return
 
     elif isinstance(target, tubes.sock.sock):
@@ -698,7 +936,7 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
         # before we attach the debugger.
         t = Timeout()
         with t.countdown(2):
-            while exe and os.realpath(proc.exe(pid)) != os.realpath(exe) and t.timeout:
+            while exe and os.path.realpath(proc.exe(pid)) != os.path.realpath(exe) and t.timeout:
                 time.sleep(0.1)
 
     elif isinstance(target, tubes.process.process):
@@ -729,7 +967,7 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
     elif isinstance(target, elf.corefile.Corefile):
         pre += 'target core %s\n' % target.path
     else:
-        log.error("don't know how to attach to target: %r" % target)
+        log.error("don't know how to attach to target: %r", target)
 
     # if we have a pid but no exe, just look it up in /proc/
     if pid and not exe:
@@ -741,30 +979,30 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
     if not pid and not exe and not ssh:
         log.error('could not find target process')
 
-    cmd = binary()
+    gdb_binary = binary()
+    cmd = [gdb_binary]
 
     if gdb_args:
-        cmd += ' '
-        cmd += ' '.join(gdb_args)
+        cmd += gdb_args
 
     if context.gdbinit:
-        cmd += ' -nh '                     # ignore ~/.gdbinit
-        cmd += ' -x %s ' % context.gdbinit # load custom gdbinit
+        cmd += ['-nh']                  # ignore ~/.gdbinit
+        cmd += ['-x', context.gdbinit]  # load custom gdbinit
 
-    cmd += ' -q '
+    cmd += ['-q']
 
     if exe and context.native:
         if not ssh and not os.path.isfile(exe):
-            log.error('No such file: %s' % exe)
-        cmd += ' "%s"' % exe
+            log.error('No such file: %s', exe)
+        cmd += [exe]
 
     if pid and not context.os == 'android':
-        cmd += ' %d' % pid
+        cmd += [str(pid)]
 
     if context.os == 'android' and pid:
         runner  = _get_runner()
         which   = _get_which()
-        gdb_cmd = _gdbserver_args(pid=pid, which=which, env=env)
+        gdb_cmd = _gdbserver_args(pid=pid, which=which)
         gdbserver = runner(gdb_cmd)
         port    = _gdbserver_port(gdbserver, None)
         host    = context.adb_host
@@ -774,26 +1012,96 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
         # when you're trying to debug anything that forks.
         pre += 'set detach-on-fork off\n'
 
+    if api:
+        # create a UNIX socket for talking to GDB
+        socket_dir = tempfile.mkdtemp()
+        socket_path = os.path.join(socket_dir, 'socket')
+        bridge = os.path.join(os.path.dirname(__file__), 'gdb_api_bridge.py')
+
+        # inject the socket path and the GDB Python API bridge
+        pre = 'python socket_path = ' + repr(socket_path) + '\n' + \
+              'source ' + bridge + '\n' + \
+              pre
+
     gdbscript = pre + (gdbscript or '')
 
     if gdbscript:
         tmp = tempfile.NamedTemporaryFile(prefix = 'pwn', suffix = '.gdb',
                                           delete = False, mode = 'w+')
-        log.debug('Wrote gdb script to %r\n%s' % (tmp.name, gdbscript))
+        log.debug('Wrote gdb script to %r\n%s', tmp.name, gdbscript)
         gdbscript = 'shell rm %s\n%s' % (tmp.name, gdbscript)
 
         tmp.write(gdbscript)
         tmp.close()
-        cmd += ' -x %s' % (tmp.name)
+        cmd += ['-x', tmp.name]
 
-    log.info('running in new terminal: %s' % cmd)
+    log.info('running in new terminal: %s', cmd)
 
-    gdb_pid = misc.run_in_new_terminal(cmd)
+    if api:
+        # prevent gdb_faketerminal.py from messing up api doctests
+        def preexec_fn():
+            os.environ['GDB_FAKETERMINAL'] = '0'
+    else:
+        preexec_fn = None
+    gdb_pid = misc.run_in_new_terminal(cmd, preexec_fn = preexec_fn)
 
     if pid and context.native:
         proc.wait_for_debugger(pid, gdb_pid)
 
-    return gdb_pid
+    if not api:
+        return gdb_pid
+
+    # connect to the GDB Python API bridge
+    from rpyc import BgServingThread
+    from rpyc.utils.factory import unix_connect
+    if six.PY2:
+        retriable = socket.error
+    else:
+        retriable = ConnectionRefusedError, FileNotFoundError
+
+    t = Timeout()
+    with t.countdown(10):
+        while t.timeout:
+            try:
+                conn = unix_connect(socket_path)
+                break
+            except retriable:
+                time.sleep(0.1)
+        else:
+            # Check to see if RPyC is installed at all in GDB
+            rpyc_check = [gdb_binary, '--nx', '-batch', '-ex',
+                          'python import rpyc; import sys; sys.exit(123)']
+
+            if 123 != tubes.process.process(rpyc_check).poll(block=True):
+                log.error('Failed to connect to GDB: rpyc is not installed')
+
+            # Check to see if the socket ever got created
+            if not os.path.exists(socket_path):
+                log.error('Failed to connect to GDB: Unix socket %s was never created', socket_path)
+
+            # Check to see if the remote RPyC client is a compatible version
+            version_check = [gdb_binary, '--nx', '-batch', '-ex',
+                            'python import platform; print(platform.python_version())']
+            gdb_python_version = tubes.process.process(version_check).recvall().strip()
+            python_version = str(platform.python_version())
+
+            if gdb_python_version != python_version:
+                log.error('Failed to connect to GDB: Version mismatch (%s vs %s)',
+                           gdb_python_version,
+                           python_version)
+
+            # Don't know what happened
+            log.error('Failed to connect to GDB: Unknown error')
+
+    # now that connection is up, remove the socket from the filesystem
+    os.unlink(socket_path)
+    os.rmdir(socket_dir)
+
+    # create a thread for receiving breakpoint notifications
+    BgServingThread(conn, callback=lambda: None)
+
+    return gdb_pid, Gdb(conn)
+
 
 def ssh_gdb(ssh, argv, gdbscript = None, arch = None, **kwargs):
     if not isinstance(argv, (list, tuple)):
@@ -904,23 +1212,23 @@ def find_module_addresses(binary, ssh=None, ulimit=False):
 
     with runner(cmd) as gdb:
         if context.aslr:
-            gdb.sendline('set disable-randomization off')
+            gdb.sendline(b'set disable-randomization off')
 
-        gdb.send("""
+        gdb.send(b"""\
         set prompt
         catch load
         run
         """)
-        gdb.sendline('info sharedlibrary')
-        lines = context._decode(gdb.recvrepeat(2))
+        gdb.sendline(b'info sharedlibrary')
+        lines = packing._decode(gdb.recvrepeat(2))
 
         for line in lines.splitlines():
             m = expr.match(line)
             if m:
                 libs[m.group(2)] = int(m.group(1),16)
-        gdb.sendline('kill')
-        gdb.sendline('y')
-        gdb.sendline('quit')
+        gdb.sendline(b'kill')
+        gdb.sendline(b'y')
+        gdb.sendline(b'quit')
 
     #
     # Fix up all of the addresses against the .text address
@@ -963,8 +1271,8 @@ def corefile(process):
 
         >>> io = process('bash')
         >>> core = gdb.corefile(io)
-        >>> core.exe.name
-        '/bin/bash'
+        >>> core.exe.name # doctest: +ELLIPSIS
+        '.../bin/bash'
     """
 
     if context.noptrace:
@@ -985,11 +1293,11 @@ def corefile(process):
     gdb_args = ['-batch',
                 '-q',
                 '-nx',
-                '-ex', '"set pagination off"',
-                '-ex', '"set height 0"',
-                '-ex', '"set width 0"',
-                '-ex', '"set use-coredump-filter on"',
-                '-ex', '"generate-core-file %s"' % corefile_path,
+                '-ex', 'set pagination off',
+                '-ex', 'set height 0',
+                '-ex', 'set width 0',
+                '-ex', 'set use-coredump-filter on',
+                '-ex', 'generate-core-file %s' % corefile_path,
                 '-ex', 'detach']
 
     with context.local(terminal = ['sh', '-c']):
@@ -997,8 +1305,8 @@ def corefile(process):
             pid = attach(process, gdb_args=gdb_args)
             log.debug("Got GDB pid %d", pid)
             try:
-                os.waitpid(pid, 0)
-            except Exception:
+                psutil.Process(pid).wait()
+            except psutil.Error:
                 pass
 
     if not os.path.exists(corefile_path):
@@ -1019,7 +1327,7 @@ def version(program='gdb'):
 
     Example:
 
-        >>> (7,0) <= gdb.version() <= (10,0)
+        >>> (7,0) <= gdb.version() <= (12,0)
         True
     """
     program = misc.which(program)

pwnlib/gdb_api_bridge.py

@@ -0,0 +1,103 @@
+"""GDB Python API bridge."""
+import gdb
+
+import socket
+from threading import Condition
+import time
+
+from rpyc.core.protocol import Connection
+from rpyc.core.service import Service
+from rpyc.lib import spawn
+from rpyc.lib.compat import select_error
+from rpyc.utils.server import ThreadedServer
+
+
+class ServeResult:
+    """Result of serving requests on GDB thread."""
+    def __init__(self):
+        self.cv = Condition()
+        self.done = False
+        self.exc = None
+
+    def set(self, exc):
+        with self.cv:
+            self.done = True
+            self.exc = exc
+            self.cv.notify()
+
+    def wait(self):
+        with self.cv:
+            while not self.done:
+                self.cv.wait()
+            if self.exc is not None:
+                raise self.exc
+
+
+class GdbConnection(Connection):
+    """A Connection implementation that serves requests on GDB thread.
+
+    Serving on GDB thread might not be ideal from the responsiveness
+    perspective, however, it is simple and reliable.
+    """
+    SERVE_TIME = 0.1  # Number of seconds to serve.
+    IDLE_TIME = 0.1  # Number of seconds to wait after serving.
+
+    def serve_gdb_thread(self, serve_result):
+        """Serve requests on GDB thread."""
+        try:
+            deadline = time.time() + self.SERVE_TIME
+            while True:
+                timeout = deadline - time.time()
+                if timeout < 0:
+                    break
+                super().serve(timeout=timeout)
+        except Exception as exc:
+            serve_result.set(exc)
+        else:
+            serve_result.set(None)
+
+    def serve_all(self):
+        """Modified version of rpyc.core.protocol.Connection.serve_all."""
+        try:
+            while not self.closed:
+                serve_result = ServeResult()
+                gdb.post_event(lambda: self.serve_gdb_thread(serve_result))
+                serve_result.wait()
+                time.sleep(self.IDLE_TIME)
+        except (socket.error, select_error, IOError):
+            if not self.closed:
+                raise
+        except EOFError:
+            pass
+        finally:
+            self.close()
+
+
+class GdbService(Service):
+    """A public interface for Pwntools."""
+
+    _protocol = GdbConnection  # Connection subclass.
+    exposed_gdb = gdb  # ``gdb`` module.
+
+    def exposed_set_breakpoint(self, client, has_stop, *args, **kwargs):
+        """Create a breakpoint and connect it with the client-side mirror."""
+        if has_stop:
+            class Breakpoint(gdb.Breakpoint):
+                def stop(self):
+                    return client.stop()
+
+            return Breakpoint(*args, **kwargs)
+        return gdb.Breakpoint(*args, **kwargs)
+
+    def exposed_quit(self):
+        """Terminate GDB."""
+        gdb.post_event(lambda: gdb.execute('quit'))
+
+
+spawn(ThreadedServer(
+    service=GdbService(),
+    socket_path=socket_path,
+    protocol_config={
+        'allow_all_attrs': True,
+    },
+).start)

pwnlib/gdb_faketerminal.py

@@ -2,13 +2,18 @@
 from pwnlib.tubes.process import process
 from time import sleep
 from sys import argv
+from os import environ
 sleep(1)
 if len(argv) == 2:
-	sh = process(argv[1], shell=True)
+    sh = process(argv[1], shell=True, timeout=30)
 else:
-	sh = process(argv[1:])
+    sh = process(argv[1:], timeout=30)
 sh.sendline('set prompt (gdb)')
-res = sh.sendlineafter('(gdb)', 'c')
-while b'The program is not being run.' not in res:
+if environ.get('GDB_FAKETERMINAL') == '0':
+    sh.sendline('set pagination off')
+    sh.recvall()
+else:
     res = sh.sendlineafter('(gdb)', 'c')
+    while res and b'The program is not being run.' not in res:
+        res = sh.sendlineafter('(gdb)', 'c')
 sh.close()

pwnlib/libcdb.py

@@ -7,6 +7,7 @@ from __future__ import division
 import codecs
 import json
 import os
+import requests
 import tempfile
 
 from six.moves import urllib
@@ -25,6 +26,64 @@ log = getLogger(__name__)
 
 HASHES = ['build_id', 'sha1', 'sha256', 'md5']
 
+# https://gitlab.com/libcdb/libcdb wasn't updated after 2019,
+# but still is a massive database of older libc binaries.
+def provider_libcdb(hex_encoded_id, hash_type):
+    # Build the URL using the requested hash type
+    url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/%s/" % hash_type
+    url      = urllib.parse.urljoin(url_base, hex_encoded_id)
+
+    data     = b""
+    log.debug("Downloading data from LibcDB: %s", url)
+    try:
+        while not data.startswith(b'\x7fELF'):
+            data = wget(url, timeout=20)
+
+            if not data:
+                log.warn_once("Could not fetch libc for %s %s from libcdb", hash_type, hex_encoded_id)
+                break
+            
+            # GitLab serves up symlinks with
+            if data.startswith(b'..'):
+                url = os.path.dirname(url) + '/'
+                url = urllib.parse.urljoin(url.encode('utf-8'), data)
+    except requests.RequestException as e:
+        log.warn_once("Failed to fetch libc for %s %s from libcdb: %s", hash_type, hex_encoded_id, e)
+    return data
+
+# https://libc.rip/
+def provider_libc_rip(hex_encoded_id, hash_type):
+    # Build the request for the hash type
+    # https://github.com/niklasb/libc-database/blob/master/searchengine/api.yml
+    if hash_type == 'build_id':
+        hash_type = 'buildid'
+    url    = "https://libc.rip/api/find"
+    params = {hash_type: hex_encoded_id}
+
+    data = b""
+    try:
+        result = requests.post(url, json=params, timeout=20)
+        if result.status_code != 200 or len(result.json()) == 0:
+            log.warn_once("Could not find libc for %s %s on libc.rip", hash_type, hex_encoded_id)
+            log.debug("Error: %s", result.text)
+            return None
+
+        libc_match = result.json()
+        assert len(libc_match) == 1, 'Invalid libc.rip response.'
+
+        url = libc_match[0]['download_url']
+        log.debug("Downloading data from libc.rip: %s", url)
+        data = wget(url, timeout=20)
+
+        if not data:
+            log.warn_once("Could not fetch libc for %s %s from libc.rip", hash_type, hex_encoded_id)
+            return None
+    except requests.RequestException as e:
+        log.warn_once("Failed to fetch libc for %s %s from libc.rip: %s", hash_type, hex_encoded_id, e)
+    return data
+
+PROVIDERS = [provider_libcdb, provider_libc_rip]
+
 def search_by_hash(hex_encoded_id, hash_type='build_id'):
     assert hash_type in HASHES, hash_type
 
@@ -46,26 +105,17 @@ def search_by_hash(hex_encoded_id, hash_type='build_id'):
             log.info_once("Using cached data from %r", cache)
             return cache
         else:
-            log.info_once("Skipping unavialable libc %s", hex_encoded_id)
+            log.info_once("Skipping unavailable libc %s", hex_encoded_id)
             return None
 
-    # Build the URL using the requested hash type
-    url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/%s/" % hash_type
-    url      = urllib.parse.urljoin(url_base, hex_encoded_id)
-
-    data   = b""
-    while not data.startswith(b'\x7fELF'):
-        log.debug("Downloading data from LibcDB: %s", url)
-        data = wget(url, timeout=20)
-
-        if not data:
-            log.warn_once("Could not fetch libc for build_id %s", hex_encoded_id)
+    # Run through all available libc database providers to see if we have a match.
+    for provider in PROVIDERS:
+        data = provider(hex_encoded_id, hash_type)
+        if data and data.startswith(b'\x7FELF'):
             break
 
-        # GitLab serves up symlinks with
-        if data.startswith(b'..'):
-            url = os.path.dirname(url) + '/'
-            url = urllib.parse.urljoin(url.encode('utf-8'), data)
+    if not data:
+        log.warn_once("Could not find libc for %s %s anywhere", hash_type, hex_encoded_id)
 
     # Save whatever we got to the cache
     write(cache, data or b'')
@@ -94,6 +144,9 @@ def search_by_build_id(hex_encoded_id):
         '0xda260'
         >>> None == search_by_build_id('XX')
         True
+        >>> filename = search_by_build_id('a5a3c3f65fd94f4c7f323a175707c3a79cbbd614')
+        >>> hex(ELF(filename).symbols.read)
+        '0xeef40'
     """
     return search_by_hash(hex_encoded_id, 'build_id')
 
@@ -103,7 +156,7 @@ def search_by_md5(hex_encoded_id):
 
     Arguments:
         hex_encoded_id(str):
-            Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
+            Hex-encoded md5sum (e.g. 'ABCDEF...') of the library
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -112,8 +165,11 @@ def search_by_md5(hex_encoded_id):
         >>> filename = search_by_md5('7a71dafb87606f360043dcd638e411bd')
         >>> hex(ELF(filename).symbols.read)
         '0xda260'
-        >>> None == search_by_build_id('XX')
+        >>> None == search_by_md5('XX')
         True
+        >>> filename = search_by_md5('74f2d3062180572fc8bcd964b587eeae')
+        >>> hex(ELF(filename).symbols.read)
+        '0xeef40'
     """
     return search_by_hash(hex_encoded_id, 'md5')
 
@@ -123,7 +179,7 @@ def search_by_sha1(hex_encoded_id):
 
     Arguments:
         hex_encoded_id(str):
-            Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
+            Hex-encoded sha1sum (e.g. 'ABCDEF...') of the library
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -134,6 +190,9 @@ def search_by_sha1(hex_encoded_id):
         '0xda260'
         >>> None == search_by_sha1('XX')
         True
+        >>> filename = search_by_sha1('0041d2f397bc2498f62aeb4134d522c5b2635e87')
+        >>> hex(ELF(filename).symbols.read)
+        '0xeef40'
     """
     return search_by_hash(hex_encoded_id, 'sha1')
 
@@ -144,7 +203,7 @@ def search_by_sha256(hex_encoded_id):
 
     Arguments:
         hex_encoded_id(str):
-            Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
+            Hex-encoded sha256sum (e.g. 'ABCDEF...') of the library
 
     Returns:
         Path to the downloaded library on disk, or :const:`None`.
@@ -155,6 +214,9 @@ def search_by_sha256(hex_encoded_id):
         '0xda260'
         >>> None == search_by_sha256('XX')
         True
+        >>> filename = search_by_sha256('5d78fc60054df18df20480c71f3379218790751090f452baffb62ac6b2aff7ee')
+        >>> hex(ELF(filename).symbols.read)
+        '0xeef40'
     """
     return search_by_hash(hex_encoded_id, 'sha256')
 
@@ -186,7 +248,7 @@ def get_build_id_offsets():
     return {
     # $ check_arch 80386
     #     181 Displaying notes found at file offset 0x00000174 with length 0x00000024:
-        'i386': [0x174],
+        'i386': [0x174, 0x1b4, 0x1d4],
     # $ check_arch "ARM, EABI5"
     #      69 Displaying notes found at file offset 0x00000174 with length 0x00000024:
         'arm':  [0x174],
@@ -197,7 +259,7 @@ def get_build_id_offsets():
     # $ check_arch "x86-64"
     #       6 Displaying notes found at file offset 0x00000174 with length 0x00000024:
     #      82 Displaying notes found at file offset 0x00000270 with length 0x00000024:
-        'amd64': [0x270, 0x174, 0x2e0],
+        'amd64': [0x270, 0x174, 0x2e0, 0x370],
     # $ check_arch "PowerPC or cisco"
     #      88 Displaying notes found at file offset 0x00000174 with length 0x00000024:
         'powerpc': [0x174],