Release 4.3.0

Closes #1690

.coveragerc

@@ -1,8 +1,11 @@
 [run]
 branch = True
+parallel = True
+concurrency = thread,multiprocessing
 omit =
     */constants/*
 source =
     pwn
     pwnlib
-    ~/.pwntools-cache/
+    ~/.pwntools-cache-2.7/
+    ~/.pwntools-cache-3.8/

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,54 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: backport-required, bug
+assignees: ''
+
+---
+
+Thanks for contributing to Pwntools!
+
+## Update Pwntools First
+
+When reporting an issue, be sure that you are running the latest released version of pwntools (`pip install --upgrade pwntools`).
+
+## Debug Output
+
+Having the extra debug output really helps us, and might help you diagnose the problem yourself.
+
+When submitting an issue that has output from Pwntools, make sure to run your script as shown below, to enable the extra debugging data.
+
+```sh
+$ python exploit.py DEBUG LOG_FILE=log.txt
+```
+
+You should see `[DEBUG]` statements that show what's happening behind the scenes:
+
+```
+[+] Starting local process '/bin/sh' argv=['sh'] : pid 16823
+[DEBUG] Sent 0x7 bytes:
+    'whoami\n'
+[DEBUG] Sent 0x5 bytes:
+    'exit\n'
+[+] Receiving all data: Done (11B)
+[DEBUG] Received 0xb bytes:
+    'crashheap\n'
+[*] Process '/bin/sh' stopped with exit code 0 (pid 16823)
+```
+
+## Verify on Ubuntu
+
+If possible, please verify that your issue occurs on 64-bit Ubuntu 18.04.  We provide a Dockerfile based on Ubuntu 18.04 via `docker.io` to make this super simple, no VM required!
+
+```sh
+# Download the Docker image
+$ docker pull pwntools/pwntools:stable
+
+# Boot the image
+$ docker run -it pwntools/pwntools:stable
+
+pwntools@7dc3ef409476:~$ python
+>>> from pwn import *
+>>> # Test your code here
+```

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: feature
+assignees: ''
+
+---
+
+Thanks for contributing to Pwntools!   Ideas from the community help make Pwntools an amazing tool for everybody.
+
+If you've got an idea for a new feature, please provide information about:
+
+* What the feature does
+* Why the feature should exit
+* What tests should be included
+
+If you think you can write the feature yourself, please submit a Pull Request and we can review your changes!

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,18 @@
+---
+name: Question
+about: Ask us a question about Pwntools
+title: ''
+labels: question
+assignees: ''
+
+---
+
+Thanks for using Pwntools!  If you have a question, here are some tips that will make it easier for us to help you.
+
+## Check the Documentation
+
+Pwntools has REALLY extensive documentation.  Please check there first!  https://docs.pwntools.com
+
+## Debug Output
+
+If you're having an issue with an exploit, run your script with full debugging output, i.e. `python exploit.py DEBUG`.  This may help you to see what's happening under the hood, and the debug output will help to diagnose any issues you're encountering.

.github/PULL_REQUEST_TEMPLATE.md

@@ -25,3 +25,8 @@ You can always [change the branch][change] after you create the PR if it's again
 [contributing]: https://github.com/Gallopsled/pwntools/blob/dev/CONTRIBUTING.md
 [testing]: https://github.com/Gallopsled/pwntools/blob/dev/TESTING.md
 [change]: https://github.com/blog/2224-change-the-base-branch-of-a-pull-request
+
+## Changelog
+
+After creating your Pull Request, please add and push a commit that updates the changelog for the appropriate branch.  
+You can look at the existing changelog for examples of how to do this.

.github/workflows/changelog.yml

@@ -0,0 +1,29 @@
+name: Changelog
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check changelog for updates
+      run: |
+        if git log --stat ${GITHUB_BASE}..HEAD | grep CHANGELOG.md; then
+           echo 'Changelog updated :D'
+        else
+           if git log --stat ${GITHUB_BASE}..HEAD | grep '++\|--'; then
+              echo 'Major changes detected, changelog required!'
+              false
+           else
+              echo 'Minor changes detected, no changelog required!'
+           fi
+        fi

.github/workflows/ci.yml

@@ -0,0 +1,230 @@
+name: Continuous Integration
+on: [push, pull_request]
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python-version: [2.7, 3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Git History 2
+      run: git log --oneline --graph -10
+
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Verify tag against version
+      if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+      run: |
+        set -x
+        GITHUB_TAG=${GITHUB_REF#refs/tags/}
+        echo "$GITHUB_TAG" | grep -E '^[0-9.]*(beta[0-9])?$'
+        vsetup=$(grep -o "version\\s*=\\s*[\"'].*[\"'],$" setup.py | grep -o "[0-9][^\"']*")
+        vpwnlib=$(grep -o "__version__\\s*=\\s*[\"'].*[\"']$" pwnlib/version.py | grep -o "[0-9][^\"']*")
+        [ "$vsetup" = "$vpwnlib" ]
+        [ "$GITHUB_TAG" = "$vsetup" ]
+
+    - name: Detect whether java is needed
+      id: java-needed
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+        GITHUB_COMMIT_RANGE: ${{ github.event.before }}...${{ github.event.after }}
+      run: |
+        if echo "$GITHUB_REF" | grep -Eq 'staging|tags'; then
+          echo "Found release or important branch ($GITHUB_REF), forcing tests."
+        elif ! git show "$GITHUB_COMMIT_RANGE" >/dev/null; then
+          echo 'Incorrect commit range, forcing android tests.'
+        elif git log --stat "$GITHUB_COMMIT_RANGE" | grep -iE 'android|\<adb\>'; then
+          echo 'Found Android-related commits, forcing tests.'
+        else
+          # clear files that cause Android doctests
+          : > docs/source/adb.rst > docs/source/protocols/adb.rst
+          exit 0
+        fi
+        echo ::set-output name=need::openjdk-8-jre-headless
+
+    - name: Install Linux dependencies
+      env:
+        ANDROID_JRE: ${{ steps.java-needed.outputs.need }}
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
+          ash bash-static dash ksh mksh zsh \
+          pandoc gdb gdbserver socat sshpass \
+          binutils-multiarch qemu-user-static \
+          binutils-aarch64-linux-gnu \
+          binutils-arm-linux-gnueabihf \
+          binutils-mips-linux-gnu \
+          binutils-powerpc-linux-gnu \
+          binutils-sparc64-linux-gnu \
+          gcc-multilib \
+          libc6-dbg \
+          $ANDROID_JRE
+
+    - name: Testing Corefiles
+      run: |
+        ulimit -a
+        ulimit -c unlimited
+        cat /proc/sys/kernel/core_pattern
+        cat /proc/sys/kernel/core_uses_pid
+        ( cd $(mktemp -d); sh -c 'kill -11 $$' || true; ls -la ./*core* /var/crash/*.crash;) || true
+
+    - name: Install Android AVD
+      if:  steps.java-needed.outputs.need
+      run: |
+        USER=travis source travis/install.sh
+        set | egrep '^(ANDROID|PATH)' >.android.env
+
+    - name: Set up SSH
+      run: |
+        travis/ssh_setup.sh
+
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        pip install --upgrade flake8 appdirs
+        python setup.py egg_info
+        pip install --upgrade --editable .
+
+    - name: Sanity checks
+      run:  PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+
+    - name: Install documentation dependencies
+      run:  pip install -r docs/requirements.txt
+
+    - name: Manually install non-broken Unicorn
+      run:  pip install unicorn==1.0.2rc3
+
+    - name: Version Check
+      run:  PWNLIB_NOTERM=1 pwn version
+
+    - name: Coverage doctests
+      run: |
+        source .android.env || :
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope # required by some gdb doctests
+        PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+
+    - name: Coverage running examples
+      run: |
+        export TERM=linux
+        set -x
+        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit.py
+        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit2.py || : # can fail randomly?
+        coverage run examples/asm.py
+        coverage run examples/asm.py
+        coverage run examples/text.py
+        # for f in examples/sigreturn_corefile_*.py; do coverage run "$f"; done # XXX something is wrong
+
+    - name: Coverage running commandline tools
+      run: |
+        export TERM=linux
+        pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec coverage run -m pwnlib.commandline."$cmd" "$@" ) }
+        set -x
+
+        pwn cyclic 32
+        pwn cyclic -l 0x62616161
+        pwn cyclic -a ab
+        echo
+
+        pwn shellcraft --list |tail
+        pwn shellcraft -l --syscalls |tail
+        pwn shellcraft -l execve
+        pwn shellcraft --show i386.linux.loader_append
+        pwn shellcraft -f asm --color amd64.linux.sh
+        pwn shellcraft -f elf amd64.linux.syscalls.exit 0 </dev/null |pwn hex
+        pwn shellcraft -f i   --color amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -f c   amd64.linux.syscalls.exit 0 </dev/null
+        pwn shellcraft -f str aarch64.linux.sh </dev/null
+        pwn shellcraft -abr -f elf -o /dev/null amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -nzr thumb.linux.syscalls.execve /bin/cat '["/bin/cat", "/etc/os-release"]' </dev/null
+        pwn shellcraft -fp aarch64.trap
+
+        pwn disasm --color ff3424c3ebfe
+        pwn asm -f hex nop
+
+        pwn hex abcd
+        pwn unhex 4141 4141
+
+        cat /dev/urandom | pwn phd --color -c 256 -s 2
+        pwn phd -l 0x3d --color=always /etc/os-release
+
+        pwn checksec /bin/bash
+
+        pwn errno 2
+        pwn errno -1
+        pwn errno EADDRINUSE
+
+        pwn constgrep -c freebsd -m ^PROT_ '3 + 4'
+        pwn constgrep ^MAP_ 0
+        pwn constgrep -e O_RDWR
+
+    - name: Build source and wheel distributions
+      run: |
+        python setup.py sdist
+        python setup.py bdist_wheel --universal
+
+    - uses: actions/upload-artifact@v2
+      with:
+        name: packages
+        path: dist/
+
+    - name: Upload coverage to coveralls.io
+      run: |
+        coverage combine
+        COVERALLS_REPO_TOKEN=PP20MEgztXIQJJTguQwe2jeCh6Bm4lkbv coveralls
+
+  staging-merge:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'Gallopsled' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/heads/') && endsWith(github.event.ref, '-staging')
+    needs: test
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 20
+    - name: Push changes to protected branch
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+      run: |
+        BRANCH=${GITHUB_REF#refs/heads/}
+        TARGET=${BRANCH%-staging}
+        git branch -f "$TARGET"
+        git push origin "$TARGET"
+        git push origin --delete "$BRANCH"
+
+  pypi:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    needs: test
+    steps:
+    - name: Download artifacts
+      uses: actions/download-artifact@v2
+      with:
+        name: packages
+        path: dist
+
+    - name: Publish package
+      uses: pypa/[email protected]
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_password }}
+
+    - if: failure()
+      run: ls -R

.github/workflows/lint.yml

@@ -0,0 +1,35 @@
+name: Lint
+on: [push, pull_request]
+
+jobs:
+  lint:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Critical lint
+      run: |
+        pip install flake8
+        flake8 . --count --select=E9,F63,F7,E71 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+
+    - name: Style lint
+      run: |
+        flake8 pwnlib setup.py docs travis --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        flake8 examples --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore='F403,F405'
+        flake8 pwn --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore='F401,F403,F405'

.github/workflows/merge-conflict.yml

@@ -0,0 +1,28 @@
+name: Check for merge markers
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  mergecheck:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check for merge conflict markers commited
+      run: |
+        if git show ${GITHUB_BASE}..HEAD | grep -EB15 --color=always '^\+\+(>>>>>|=====|<<<<<)'; then
+            echo 'WARNING: Merge marker commits found in history.'
+            if git diff ${GITHUB_BASE}..HEAD | grep -EB15 --color=always '^\+(>>>>>|=====|<<<<<)'; then
+                echo 'ERROR: Merge markers still present! Do something!'
+                exit 1
+            fi
+        else
+            echo 'Congrats! All the merges you did were clean or resolved correctly :)'
+        fi

.gitignore

@@ -16,4 +16,4 @@ venv
 *.egg-info
 *.core
 .coverage
-.tox
+.coverage.*

.travis.yml

@@ -1,9 +1,7 @@
-dist: xenial
 language: python
 addons:
   apt:
     packages:
-    - gcc-multilib
     - lib32stdc++6
     - ash
     - bash
@@ -13,6 +11,8 @@ addons:
     - zsh
     - pandoc
     - gdb
+    - socat
+    - sshpass
     - binutils
     - qemu-user-static
     - binutils-multiarch
@@ -20,15 +20,20 @@ addons:
     - binutils-arm-linux-gnueabihf
     - binutils-mips-linux-gnu
     - binutils-powerpc-linux-gnu
+    - binutils-sparc64-linux-gnu
+    - bash-static
 cache:
     - pip
     - directories:
         - usr
         - /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/
         - /home/travis/virtualenv/python2.7.15/bin/
+        - /home/travis/virtualenv/python3.8.*/lib/python3.8/site-packages/
+        - /home/travis/virtualenv/python3.8.*/bin/
+        - docs/build/doctest/.doctrees
 python:
-  - "2.7.15"
-  - "3.6"
+  - "2.7"
+  - "3.8"
 before_install:
   - echo $-
   - source travis/install.sh
@@ -36,7 +41,7 @@ before_install:
   - echo $-
 install:
   - pip install --upgrade pip
-  - pip install --upgrade appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
+  - pip install --upgrade flake8 appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
   - python setup.py egg_info      # pypa/pip#6275
   - pip install --upgrade --editable .
   - pip install --upgrade --requirement docs/requirements.txt
@@ -45,8 +50,13 @@ before_script:
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+  - sudo sh -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
 script:
+  # stop the build if there are Python syntax errors or undefined names
+  - flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+  # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
   - PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+  - coverage combine
 after_success:
   - coveralls
   - source travis/update_demo.sh

CHANGELOG.md

@@ -9,10 +9,21 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.2.0](#420)    | `dev`    | Feb 10, 2020 (planned)
-| [4.1.0](#410)    | `beta`   | Jan 30, 2020 (planned)
-| [4.0.0](#400)    | `stable` | Jan 09, 2020
+| [4.5.0](#450)    | `dev`    | Dec 20, 2020 (planned)
+| [4.4.0](#440)    | `beta`   | Nov 20, 2020 (planned)
+| [4.3.0](#430)    | `stable` | Oct 20, 2020
+| [4.2.0](#420)    |          | Jul 3, 2020
+| [4.1.7](#417)    |          | Jun 30, 2020
+| [4.1.5](#415)    |          | Jun 27, 2020
+| [4.1.4](#414)    |          | Jun 26, 2020
+| [4.1.3](#413)    |          | Jun 23, 2020
+| [4.1.2](#412)    |          | Jun 5, 2020
+| [4.1.1](#411)    |          | Jun 3, 2020
+| [4.1.0](#410)    |          | May 8, 2020
+| [4.0.1](#401)    |          | Jan 22, 2020
+| [4.0.0](#400)    |          | Jan 09, 2020
 | [3.13.0](#3130)  |          | Nov 5, 2019
+| [3.12.2](#3122)  |          | Jan 8, 2019
 | [3.12.1](#3121)  |          | Sept 17, 2018
 | [3.12.0](#3120)  |          | Feb 22, 2018
 | [3.11.0](#3110)  |          | Jan 3, 2018
@@ -45,13 +56,135 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.2.0 (`dev`)
+## 4.5.0 (`dev`)
 
-To be released on Feb 10, 2020.
+## 4.4.0 (`beta`)
 
-## 4.1.0 (`beta`)
+- [#1541][1541] Use `context.newline` for tubes by default
+- [#1602][1602] Fix bytes handling in ssh tubes
+- [#1606][1606] Fix `asm()` and `disasm()` for MSP430, S390
+- [#1616][1616] Fix `cyclic` cli for 64 bit integers
+- [#1632][1632] Enable usage of Pwntools in jupyter
+- [#1633][1633] Open a shell if `pwn template` cannot download the remote file
+- [#1644][1644] Enable and support SNI for SSL-wrapped tubes
+- [#1651][1651] Make `pwn shellcraft` faster
+- [#1654][1654] Docker images (`pwntools/pwntools:stable` etc) now use Python3 by default, and includes assemblers for a few common architectures
+- [#1667][1667] Add i386 encoder `ascii_shellcode` (Fixed docs in #1693)
+- Fix syscall instruction lists for SROP on `i386` and `amd64`
+- Fix migration to another ROP
+- [#1673][1673] Add `base=` argument to `ROP.chain()` and `ROP.dump()`
+- [#1675][1675] Gdbserver now correctly accepts multiple libraries in `LD_PRELOAD` and `LD_LIBRARY_PATH`
+- [#1678][1678] ROPGadget multibr
+- [#1682][1682] ROPGadget multibr fix
+- [#1687][1687] Actually import `requests` when doing `from pwn import *`
+- [#1688][1688] Add `__setattr__` and `__call__` interfaces to `ROP` for setting registers
+- [#1692][1692] Remove python2 shebangs where appropriate
+- [#1703][1703] Update libcdb buildid offsets for amd64 and i386
 
-To be released on Jan 30, 2020.
+[1541]: https://github.com/Gallopsled/pwntools/pull/1541
+[1602]: https://github.com/Gallopsled/pwntools/pull/1602
+[1606]: https://github.com/Gallopsled/pwntools/pull/1606
+[1616]: https://github.com/Gallopsled/pwntools/pull/1616
+[1632]: https://github.com/Gallopsled/pwntools/pull/1632
+[1633]: https://github.com/Gallopsled/pwntools/pull/1633
+[1644]: https://github.com/Gallopsled/pwntools/pull/1644
+[1651]: https://github.com/Gallopsled/pwntools/pull/1651
+[1654]: https://github.com/Gallopsled/pwntools/pull/1654
+[1667]: https://github.com/Gallopsled/pwntools/pull/1667
+[1673]: https://github.com/Gallopsled/pwntools/pull/1673
+[1675]: https://github.com/Gallopsled/pwntools/pull/1675
+[1678]: https://github.com/Gallopsled/pwntools/pull/1678
+[1682]: https://github.com/Gallopsled/pwntools/pull/1679
+[1687]: https://github.com/Gallopsled/pwntools/pull/1687
+[1688]: https://github.com/Gallopsled/pwntools/pull/1688
+[1692]: https://github.com/Gallopsled/pwntools/pull/1692
+[1703]: https://github.com/Gallopsled/pwntools/pull/1703
+
+## 4.3.0 (`stable`)
+
+- [#1576][1576] Add `executable=` argument to `ELF.search`
+- [#1584][1584] Add `jmp_esp`/`jmp_rsp` attribute to `ROP`
+- [#1592][1592] Fix over-verbose logging of process() environment
+- [#1593][1593] Colorize output of `pwn template`
+- [#1601][1601] Add `pwn version` command line tool
+- [#1605][1605] Add to `fiddling.hexdump` a way to suppress the total at the end
+- [#1613][1613] Permit `--password` for `pwn template`
+- [#1616][1616] Fix `cyclic` cli for 64 bit integers
+- [#1564][1564] Fix `asm()` and `disasm()` for PowerPC64, MIPS64, Sparc64
+- [#1621][1621] Permit negative values in flat() and fit()
+
+[1576]: https://github.com/Gallopsled/pwntools/pull/1576
+[1584]: https://github.com/Gallopsled/pwntools/pull/1584
+[1592]: https://github.com/Gallopsled/pwntools/pull/1592
+[1593]: https://github.com/Gallopsled/pwntools/pull/1593
+[1601]: https://github.com/Gallopsled/pwntools/pull/1601
+[1605]: https://github.com/Gallopsled/pwntools/pull/1605
+[1613]: https://github.com/Gallopsled/pwntools/pull/1613
+[1616]: https://github.com/Gallopsled/pwntools/pull/1616
+[1564]: https://github.com/Gallopsled/pwntools/pull/1564
+[1621]: https://github.com/Gallopsled/pwntools/pull/1621
+
+## 4.2.1
+
+- [#1625][1625] GDB now properly loads executables with QEMU
+- [#1663][1663] Change lookup algorithm of `adb.which`
+- [#1699][1699] Fix broken linux shellcraft templates
+
+[1625]: https://github.com/Gallopsled/pwntools/pull/1625
+[1699]: https://github.com/Gallopsled/pwntools/pull/1699
+
+## 4.2.0
+
+- [#1436][1436] Add ret2dlresolve automation
+- [fecf9f] tubes.ssh.process() no longer requires python 2 installed on remote (still requires python, though)
+- Miscellanous improvements to DynElf and fmtstr leaker (see examples/fmtstr/exploit2.py)
+- [#1454][1454] Support for windows console colors
+
+[1436]: https://github.com/Gallopsled/pwntools/pull/1436
+[fecf9f]: http://github.com/Gallopsled/pwntools/commit/fecf9f
+[1454]: https://github.com/Gallopsled/pwntools/pull/1454
+
+## 4.1.7 (`stable`)
+
+- [#1615][1615] Fix aarch64 pushstr and pushstr_array
+
+[1615]: https://github.com/Gallopsled/pwntools/pull/1454
+
+## 4.1.5
+
+- [#1517][1517] flat(..., filler=) is fixed for `str` values and Python2 `bytes`
+
+[1517]: https://github.com/Gallopsled/pwntools/pull/1517
+
+## 4.1.4
+
+- [#1698][1609] Fix issues in `packing.flat` with mis-ordred fields
+
+[1609]: https://github.com/Gallopsled/pwntools/pull/1609
+
+## 4.1.3
+
+- [#1590][1590] Fix `gdb.attach()` for `remote`, `listen`, `ssh` tubes
+  - Also fix `run_in_new_terminal` for Py2 unicode strings
+- [#1595][1595] Fix ssh.process(timeout=)
+
+[1590]: https://github.com/Gallopsled/pwntools/pull/1590
+[1595]: https://github.com/Gallopsled/pwntools/pull/1595
+
+## 4.1.2
+
+- Pwntools requires `six` v.1.12.0 or higher
+
+## 4.1.1
+
+- Fix PLT resolution by locking unicorn <1.0.2rc4 (#1538)
+- Fix wrong ELF/context unpack handling (c4c11a37)
+- Fix updating of ELF.functions addresses after changing ELF.address #1512 (#1513)
+- Update Corefile warnings and replace asserts with normal checks (#1526)
+- several py2-py3 issues (#1451)
+- Fix cyclic command
+
+## 4.1.0
 
 - [#1316][1316] Fix connect shellcraft in python 3
 - [#1323][1323] Fix issues related with debugging
@@ -60,10 +193,21 @@ To be released on Jan 30, 2020.
 - [#1241][1241] Launch QEMU with sysroot if specified
 - [#1218][1218] Support for FileStructure exploitation
 
-[1241]: https://github.com/Gallopsled/pwntools/pulls/1218
-[1218]: https://github.com/Gallopsled/pwntools/pulls/1218
+[1316]: https://github.com/Gallopsled/pwntools/pull/1316
+[1323]: https://github.com/Gallopsled/pwntools/pull/1323
+[1001]: https://github.com/Gallopsled/pwntools/pull/1001
+[1389]: https://github.com/Gallopsled/pwntools/pull/1389
+[1241]: https://github.com/Gallopsled/pwntools/pull/1241
+[1218]: https://github.com/Gallopsled/pwntools/pull/1218  
 
-## 4.0.0 (`stable`)
+## 4.0.1
+
+- [#1412][1412] `recvline_pred()` and similar do not reorder data
+- Bypass unicorn-engine/unicorn#1100 and unicorn-engine/unicorn#1170 requiring unstable package
+
+[1412]: https://github.com/Gallopsled/pwntools/pull/1412
+
+## 4.0.0
 
 - **Python 3 support! <3**
 - [#1402][1402] Fix serialtube in python 3
@@ -75,12 +219,13 @@ To be released on Jan 30, 2020.
 - [#1202][1202] Docker: Kill 14 layers in pwntools base images
 - [#1182][1182] shellcraft.dupio() for mips
 
-[1391]: https://github.com/Gallopsled/pwntools/pulls/1391
-[1317]: https://github.com/Gallopsled/pwntools/pulls/1317
-[1285]: https://github.com/Gallopsled/pwntools/pulls/1285
-[1216]: https://github.com/Gallopsled/pwntools/pulls/1216
-[1202]: https://github.com/Gallopsled/pwntools/pulls/1202
-[1182]: https://github.com/Gallopsled/pwntools/pulls/1182
+[1402]: https://github.com/Gallopsled/pwntools/pull/1402
+[1391]: https://github.com/Gallopsled/pwntools/pull/1391
+[1317]: https://github.com/Gallopsled/pwntools/pull/1317
+[1285]: https://github.com/Gallopsled/pwntools/pull/1285
+[1216]: https://github.com/Gallopsled/pwntools/pull/1216
+[1202]: https://github.com/Gallopsled/pwntools/pull/1202
+[1182]: https://github.com/Gallopsled/pwntools/pull/1182
 [76413f]: https://github.com/Gallopsled/pwntools/commit/76413f
 
 ## 3.13.0
@@ -91,32 +236,16 @@ To be released on Jan 30, 2020.
 - [#1277][1277] Fix timeout parameter passing in sendlineafter and other similar functions
 - [#1292][1292] Provide correct arch name to gdb for sparc64
 
-[1175]: https://github.com/Gallopsled/pwntools/pulls/1175
-[1204]: https://github.com/Gallopsled/pwntools/pulls/1355
-[1277]: https://github.com/Gallopsled/pwntools/pulls/1277
-[1292]: https://github.com/Gallopsled/pwntools/pulls/1292
-[1355]: https://github.com/Gallopsled/pwntools/pulls/1355
-
-## 3.12.1
-
-- [#1104][1104] Add `DynELF.dump()` for dumping remote ELF files
-- [#1101][1101] Set `context.os` via `context.binary`, useful for Android exploitation
-- [5fdc08][5fdc08] Work around broken `pidof` on Android
-- [63dfed][63dfed] Print warning when Corefile deletion fails instead of throwing an exception
-- [#1094][1094] Make hexdump output alignment more consistent
-- [#1096][1096] `flat()` and `fit()` are now the same function
-
-[1104]: https://github.com/Gallopsled/pwntools/pull/1104
-[1101]: https://github.com/Gallopsled/pwntools/pull/1101
-[1094]: https://github.com/Gallopsled/pwntools/pull/1094
-[1096]: https://github.com/Gallopsled/pwntools/pull/1096
-[5fdc08]: https://github.com/Gallopsled/pwntools/commit/5fdc08
-[63dfed]: https://github.com/Gallopsled/pwntools/commit/63dfed
+[1175]: https://github.com/Gallopsled/pwntools/pull/1175
+[1204]: https://github.com/Gallopsled/pwntools/pull/1204
+[1277]: https://github.com/Gallopsled/pwntools/pull/1277
+[1292]: https://github.com/Gallopsled/pwntools/pull/1292
+[1355]: https://github.com/Gallopsled/pwntools/pull/1355
 
 ## 3.12.2
 
-- [1242][1242] Use IntervalTree 2.xx, disallow use of 3.xx
-- [1243][1243] Fix a typo that caused an exception when executing a binary with `process()` which returns `-ENOEXEC` and the system does not have `qemu-user` binaries installed.
+- [#1242][1242] Use IntervalTree 2.xx, disallow use of 3.xx
+- [#1243][1243] Fix a typo that caused an exception when executing a binary with `process()` which returns `-ENOEXEC` and the system does not have `qemu-user` binaries installed.
 
 [1242]: https://github.com/Gallopsled/pwntools/pull/1242
 [1243]: https://github.com/Gallopsled/pwntools/pull/1243
@@ -134,7 +263,15 @@ To be released on Jan 30, 2020.
 - [#1131][1131] Add "libc-" to libc prefixes in `process` tubes
 - [#1125][1125] Fix a typo
 - [#1121][1121] Fix tests which were broken by an upstream Sphinx change
+- [#1104][1104] Add `DynELF.dump()` for dumping remote ELF files
+- [#1101][1101] Set `context.os` via `context.binary`, useful for Android exploitation
+- [5fdc08][5fdc08] Work around broken `pidof` on Android
+- [63dfed][63dfed] Print warning when Corefile deletion fails instead of throwing an exception
+- [#1094][1094] Make hexdump output alignment more consistent
+- [#1096][1096] `flat()` and `fit()` are now the same function
 
+[1198]: https://github.com/Gallopsled/pwntools/pull/1198
+[1191]: https://github.com/Gallopsled/pwntools/pull/1191
 [1159]: https://github.com/Gallopsled/pwntools/pull/1159
 [1162]: https://github.com/Gallopsled/pwntools/pull/1162
 [1150]: https://github.com/Gallopsled/pwntools/pull/1150
@@ -144,6 +281,12 @@ To be released on Jan 30, 2020.
 [1131]: https://github.com/Gallopsled/pwntools/pull/1131
 [1125]: https://github.com/Gallopsled/pwntools/pull/1125
 [1121]: https://github.com/Gallopsled/pwntools/pull/1121
+[1104]: https://github.com/Gallopsled/pwntools/pull/1104
+[1101]: https://github.com/Gallopsled/pwntools/pull/1101
+[1094]: https://github.com/Gallopsled/pwntools/pull/1094
+[1096]: https://github.com/Gallopsled/pwntools/pull/1096
+[5fdc08]: https://github.com/Gallopsled/pwntools/commit/5fdc08
+[63dfed]: https://github.com/Gallopsled/pwntools/commit/63dfed
 
 ## 3.12.0
 

README.md

@@ -1,12 +1,13 @@
 # pwntools - CTF toolkit
 ![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)
 
+[![PyPI](https://img.shields.io/pypi/v/pwntools?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![PyPI](https://img.shields.io/badge/pypi-v3.13.0-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
-[![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
-[![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
-[![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)
+[![Travis](https://img.shields.io/travis/Gallopsled/pwntools/dev?logo=Travis)](https://travis-ci.org/Gallopsled/pwntools)
+![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/Gallopsled/pwntools/Continuous%20Integration/dev?logo=GitHub)
+[![Coveralls](https://img.shields.io/coveralls/github/Gallopsled/pwntools/dev?logo=coveralls)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
+[![Twitter](https://img.shields.io/twitter/follow/Pwntools)](https://twitter.com/pwntools)
 
 Pwntools is a CTF framework and exploit development library. Written in Python, it is designed for rapid prototyping and development, and intended to make exploit writing as simple as possible.
 
@@ -20,10 +21,6 @@ r.send(asm(shellcraft.sh()))
 r.interactive()
 ```
 
-# Try It Now!
-
-You can now do a live demo of Pwntools, [right in your browser](https://demo.pwntools.com).
-
 # Documentation
 
 Our documentation is available at [docs.pwntools.com](https://docs.pwntools.com/)
@@ -32,7 +29,7 @@ To get you started, we've provided some example solutions for past CTF challenge
 
 # Installation
 
-Pwntools is best supported on 64-bit Ubuntu LTE releases (12.04, 14.04, 16.04 and 18.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  Python >= 2.7 is required (Python 3 suggested as best).
+Pwntools is best supported on 64-bit Ubuntu LTE releases (14.04, 16.04, 18.04, and 20.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  Python >= 2.7 is required (Python 3 suggested as best).
 
 Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
 
@@ -40,7 +37,7 @@ Most of the functionality of pwntools is self-contained and Python-only.  You sh
 apt-get update
 apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
 python3 -m pip install --upgrade pip
-python3 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools.git@dev3
+python3 -m pip install --upgrade pwntools
 ```
 
 However, some of the features (assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://docs.pwntools.com/en/stable/install.html).

docs/requirements.txt

@@ -7,6 +7,7 @@ paramiko>=1.15.2
 pip>=6.0.8
 pyelftools>=0.2.3
 pygments>=2.0
+pypandoc
 pyserial>=2.7
 requests>=2.5.1
 ropgadget>=5.3
@@ -14,4 +15,3 @@ sphinx
 sphinx_rtd_theme
 sphinxcontrib-napoleon
 sphinxcontrib-autoprogram
-tox>=1.8.1

docs/source/about.rst

@@ -47,7 +47,7 @@ This module is our "clean" python-code. As a rule, we do not think that
 importing :mod:`pwnlib` or any of the submodules should have any significant
 side-effects (besides e.g. caching).
 
-For the most part, you will also only get the bits you import. You for instance
+For the most part, you will also only get the bits you import. You for instance would
 not get access to :mod:`pwnlib.util.packing` simply by doing ``import
 pwnlib.util``.
 

docs/source/conf.py

@@ -13,6 +13,7 @@
 
 import os
 import doctest
+import signal
 import six
 import subprocess
 import sys
@@ -87,6 +88,9 @@ class stdout(object):
     def __setattr__(self, name, value):
         return setattr(sys.stdout, name, value)
 pwnlib.context.ContextType.defaults['log_console'] = stdout()
+
+github_actions = os.environ.get('USER') == 'runner'
+travis_ci = os.environ.get('USER') == 'travis'
 '''
 
 autoclass_content = 'both'
@@ -336,7 +340,7 @@ def linkcode_resolve(domain, info):
     val = mod
     for k in info['fullname'].split('.'):
         val = getattr(val, k, None)
-        if val == None:
+        if val is None:
             break
 
     # Special case for shellcraft
@@ -350,7 +354,7 @@ def linkcode_resolve(domain, info):
         if isinstance(val, (types.ModuleType, types.MethodType, types.FunctionType, types.TracebackType, types.FrameType, types.CodeType) + six.class_types):
             try:
                 lines, first = inspect.getsourcelines(val)
-                filename += '#L%d-%d' % (first, first + len(lines) - 1)
+                filename += '#L%d-L%d' % (first, first + len(lines) - 1)
             except (IOError, TypeError):
                 pass
 
@@ -374,42 +378,11 @@ if build_dash:
 
 # -- Customization to Sphinx autodoc generation --------------------------------------------
 import sphinx.ext.autodoc
-from sphinx.util.inspect import safe_getmembers, safe_getattr
 
 # Test hidden members (e.g. def _foo(...))
 def dont_skip_any_doctests(app, what, name, obj, skip, options):
     return False
 
-def get_object_members_all(self, want_all):
-    if want_all:
-        # if not hasattr(self.object, '__all__'):
-        #     for implicit module members, check __module__ to avoid
-        #     documenting imported objects
-        return True, safe_getmembers(self.object)
-    # else:
-    #     memberlist = self.object.__all__
-    #     # Sometimes __all__ is broken...
-    #     if not isinstance(memberlist, (list, tuple)) or not \
-        #        all(isinstance(entry, string_types) for entry in memberlist):
-    #         self.directive.warn(
-    #             '__all__ should be a list of strings, not %r '
-    #             '(in module %s) -- ignoring __all__' %
-    #             (memberlist, self.fullname))
-    #         # fall back to all members
-    #         return True, safe_getmembers(self.object)
-    else:
-        memberlist = self.options.members or []
-        ret = []
-        for mname in memberlist:
-            try:
-                ret.append((mname, safe_getattr(self.object, mname)))
-            except AttributeError:
-                self.directive.warn(
-                    'missing attribute mentioned in :members: or __all__: '
-                    'module %s, attribute %s' % (
-                        safe_getattr(self.object, '__name__', '???'), mname))
-        return False, ret
-
 class _DummyClass(object): pass
 
 class Py2OutputChecker(_DummyClass, doctest.OutputChecker):
@@ -444,4 +417,3 @@ if 'doctest' in sys.argv:
     if sys.version_info[:1] < (3,):
         import sphinx.ext.doctest
         sphinx.ext.doctest.SphinxDocTestRunner.__init__ = py2_doctest_init
-    sphinx.ext.autodoc.ModuleDocumenter.get_object_members = get_object_members_all

docs/source/encoders.rst

@@ -5,21 +5,20 @@
 :mod:`pwnlib.encoders` --- Encoding Shellcode
 ===============================================
 
-.. automodule:: pwnlib.encoders
-   :members:
-
 .. automodule:: pwnlib.encoders.encoder
    :members:
 
-
-.. automodule:: pwnlib.encoders.amd64
-   :members:
-
-.. automodule:: pwnlib.encoders.arm
-   :members:
-
-.. automodule:: pwnlib.encoders.i386
-   :members:
-
 .. automodule:: pwnlib.encoders.i386.xor
    :members:
+
+.. automodule:: pwnlib.encoders.i386.delta
+   :members:
+
+.. automodule:: pwnlib.encoders.amd64.delta
+   :members:
+
+.. automodule:: pwnlib.encoders.arm.xor
+   :members:
+
+.. automodule:: pwnlib.encoders.mips.xor
+   :members:

docs/source/filepointer.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+   from pwnlib.filepointer import *
+
+:mod:`pwnlib.filepointer` --- `FILE*` structure exploitation
+============================================================
+
+.. automodule:: pwnlib.filepointer
+   :members:

docs/source/gdb.rst

@@ -1,6 +1,8 @@
 .. testsetup:: *
 
     from pwn import *
+    context.arch = 'amd64'
+    context.terminal = [os.path.join(os.path.dirname(pwnlib.__file__), 'gdb_faketerminal.py')]
 
 :mod:`pwnlib.gdb` --- Working with GDB
 ======================================

docs/source/index.rst

@@ -54,6 +54,7 @@ Each of the ``pwntools`` modules is documented here.
    encoders
    elf
    exception
+   filepointer
    flag
    fmtstr
    gdb

docs/source/install.rst

@@ -25,9 +25,9 @@ pwntools is available as a ``pip`` package.
 .. code-block:: bash
 
     $ apt-get update
-    $ apt-get install python2.7 python-pip python-dev git libssl-dev libffi-dev build-essential
-    $ pip install --upgrade pip
-    $ pip install --upgrade pwntools
+    $ apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
+    $ python3 -m pip install --upgrade pip
+    $ python3 -m pip install --upgrade pwntools
 
 Development
 --------------

docs/source/install/binutils.rst

@@ -39,7 +39,7 @@ repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
 .. code-block:: bash
 
-    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/osx/binutils-$ARCH.rb
+    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
 
 Alternate OSes
 ^^^^^^^^^^^^^^^^

docs/source/protocols/adb.rst

@@ -2,6 +2,7 @@
 
    from pwn import *
    from pwnlib.protocols.adb import AdbClient
+   AdbClient().wait_for_device()
 
 :mod:`pwnlib.protocols.adb` --- ADB Protocol Implementation
 ===========================================================

docs/source/rop/ret2dlresolve.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+   import tempfile
+
+:mod:`pwnlib.rop.ret2dlresolve` --- Return to dl_resolve
+========================================================
+
+.. automodule:: pwnlib.rop.ret2dlresolve
+   :members:

docs/source/term/readline.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwnlib.term.readline import *
+   import io
+
+:mod:`pwnlib.term.readline` --- Terminal nice readline
+======================================================
+
+.. automodule:: pwnlib.term.readline
+   :members:

docs/source/ui.rst

@@ -1,3 +1,8 @@
+.. testsetup:: *
+
+   from pwn import *
+   import io
+
 :mod:`pwnlib.ui` --- Functions for user interaction
 ===================================================
 

docs/source/util/iters.rst

@@ -17,19 +17,19 @@
      count                                  ,
      cycle                                  ,
      dropwhile                              ,
+     filter                                 ,
+     filterfalse                            ,
      groupby                                ,
-     ifilter                                ,
-     ifilterfalse                           ,
-     imap                                   ,
      islice                                 ,
-     izip                                   ,
-     izip_longest                           ,
+     map                                    ,
      permutations                           ,
      product                                ,
      repeat                                 ,
      starmap                                ,
      takewhile                              ,
-     tee
+     tee                                    ,
+     zip                                    ,
+     zip_longest
 
 .. function:: chain
 
@@ -63,29 +63,29 @@
 
    Alias for :func:`itertools.groupby`
 
-.. function:: ifilter
+.. function:: filter
 
-   Alias for :func:`itertools.ifilter`
+   Alias for python3-style :func:`filter`
 
-.. function:: ifilterfalse
+.. function:: filterfalse
 
-   Alias for :func:`itertools.ifilterfalse`
+   Alias for :func:`itertools.filterfalse`
 
-.. function:: imap
+.. function:: map
 
-   Alias for :func:`itertools.imap`
+   Alias for python3-style :func:`map`
 
 .. function:: islice
 
    Alias for :func:`itertools.islice`
 
-.. function:: izip
+.. function:: zip
 
-   Alias for :func:`itertools.izip`
+   Alias for python3-style :func:`zip`
 
-.. function:: izip_longest
+.. function:: zip_longest
 
-   Alias for :func:`itertools.izip_longest`
+   Alias for :func:`itertools.zip_longest`
 
 .. function:: permutations
 

examples/README.md

@@ -26,6 +26,10 @@ Try running this example with::
 ```
 Example showing the interface to `pwnlib.asm.asm` and `pwnlib.shellcraft`.
 ```
+* `attach.py`
+```
+Example showing `pwnlib.gdb.attach()`
+```
 * `clean_and_log.py`
 ```
 Use case for `pwnlib.tubes.tube.clean_and_log`.
@@ -65,8 +69,7 @@ Example showing `pwnlib.ui.options()`
 ```
 * `port_forward.py`
 ```
-A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.  Notice
-that `<>` is just a shorthand.
+A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.
 ```
 * `readline_completers.py`
 ```
@@ -74,6 +77,10 @@ Example showing pwnlib's readline implementation and a few completers.  This
 part of pwnlib will probably see some major changes soon, but we wanted to show
 off some proof-of-concepts.
 ```
+* `remote.py`
+```
+Example showing how to use the remote class.
+```
 * `remote_gdb_debugging.py`
 ```
 Simple example showing how to use the remote
@@ -103,7 +110,7 @@ and::
 
   $ TERM=xterm-256color python text.py
 ```
-* `remote.py`
+* `yesno.py`
 ```
-Example showing how to use the remote class.
+Example showing `pwnlib.ui.yesno()`
 ```

examples/android.py

@@ -1,22 +1,25 @@
-from pwn import *
+from __future__ import print_function
+
+import gdb
+from pwn import adb, context
 
 # Tell pwntools that the target is an Android device
-context.os='android'
-context.arch='aarch64' # or 'arm'
+context.os = "android"
+context.arch = "aarch64"  # or 'arm'
 
 # Optionally, set the remote ADB server address
-context.adb_host='172.16.110.1'
+context.adb_host = "172.16.110.1"
 
 # Wait for a device to become available
-print adb.wait_for_device()
+print(adb.wait_for_device())
 
 # Who am I?
-print adb.process('id').recvall().strip()
+print(adb.process("id").recvall().strip())
 
 # Interactive sessions!
 io = adb.shell()
-io.sendline('echo Hello, world; exit')
-print io.recvall().replace('\r\n', '\n').strip()
+io.sendline("echo Hello, world; exit")
+print(io.recvall().replace("\r\n", "\n").strip())
 
 # Debugging!
-gdb.debug('sh').interactive()
+gdb.debug("sh").interactive()

examples/args.py

@@ -14,5 +14,5 @@ Try running this example with::
 
 from pwn import *
 
-print args['RHOST'] or 'RHOST is not set'
-print args['RPORT'] or 'RPORT is not set'
+print(args['RHOST'] or 'RHOST is not set')
+print(args['RPORT'] or 'RPORT is not set')

examples/asm.py

@@ -9,5 +9,5 @@ context(arch='i386', os='linux')
 shellcode = shellcraft.i386_to_amd64()
 shellcode_asm = asm(shellcode)
 
-print enhex(shellcode_asm)
-print disasm(shellcode_asm)
+print(enhex(shellcode_asm))
+print(disasm(shellcode_asm))

examples/attach.py

@@ -5,7 +5,7 @@ Example showing `pwnlib.gdb.attach()`
 from pwn import *
 
 bash = process('/bin/bash')
-gdb.attach(bash, execute = '''
+gdb.attach(bash, gdbscript = '''
 p "hello from pwnlib"
 c
 ''')

examples/clean_and_log.py

@@ -25,4 +25,4 @@ atexit.register(r.clean_and_log)
 
 while True:
     line = r.recvline()
-    print re.findall('^prefix (\S+)$', line)[0]
+    print(re.findall(r'^prefix (\S+)$', line)[0])

examples/fmtstr/Makefile

@@ -1,9 +1,11 @@
 CFLAGS := -Wno-unused-result -Wno-format-security
 TARGETS := printf.mips printf.mips64 printf.mipsel printf.mips64el printf.arm printf.aarch64 \
-	         printf.ppc printf.ppc64 printf.sparc64 printf.native printf.native32
+	         printf.ppc printf.ppc64 printf.sparc64 printf.native printf.native32 \
+	         printf-loop.native printf-loop.native32
 
 .PHONY: run
 run: all
+	./exploit2.py
 	./exploit.py
 
 .PHONY: all
@@ -43,7 +45,7 @@ clean:
 %.native: %.c
 	gcc $(CFLAGS) -o "$@" $^
 %.native32: %.c
-	i686-linux-gnu-gcc $(CFLAGS) -o "$@" $^
+	gcc $(CFLAGS) -m32 -o "$@" $^
 
 # install deps on ubuntu xenial
 .PHONY: install-apt

examples/fmtstr/exploit.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 from pwn import *
 
 @context.quiet
@@ -15,8 +15,8 @@ def exploit(binary):
         addr = unpack(p.recv(context.bytes))
         payload = fmtstr_payload(offset, {addr: p32(0x1337babe)})
         p.sendline(payload)
-        p.recvuntil("DONE")
-        print hex(u32(p.recv(4)))
+        p.recvuntil(b"DONE")
+        print(hex(u32(p.recv(4))))
 
 binaries = [
     "printf.mips",

examples/fmtstr/exploit2.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+An example combining fmtstr and dynelf to automatically exploit a PIE binary.
+"""
+
+from pwn import *
+
+def exploit(e):
+    p = e.process()
+
+    def executer(fmt):
+        p.sendline(fmt)
+        p.stdin.flush()
+        r = p.recvuntil(b'again=\n')
+        return r
+    f = FmtStr(executer)
+
+    # find some code pointer on stack
+    addrbits = min(context.bits, 48) # address space is no larger than 48-bit
+    min_code_ptr = 0x10 << addrbits-8
+    max_code_ptr = 0x7e << addrbits-8
+    for off in range(64):
+        random_code_ptr = f.leak_stack(off)
+        packed = bytearray(pack(random_code_ptr))
+        chars = set(packed) - {0}
+        # simple heuristic
+        if (min_code_ptr < random_code_ptr < max_code_ptr
+            and max(chars) > 0x80 and min(chars) < 0x80
+            and not hex(random_code_ptr).startswith('0x7ff')):
+            break
+
+    log.info("Code ptr: %#x"%random_code_ptr)
+
+    online_elf = DynELF(f.leaker, pointer=random_code_ptr, elf=e, libcdb=False)
+
+    print("Stack: %#x" % (online_elf.stack(),))
+    print("Heap: %#x" % (online_elf.heap(),))
+
+    info("Finding base address of the executable")
+    for ref in e.sym:
+        if not ref:
+            continue
+        # if we have the address of a symbol on remote, we know the offset
+        ref_addr = online_elf.lookup(ref)
+        if ref_addr:
+            print("%s: %r" % (ref, ref_addr))
+            break
+
+    system = online_elf.lookup('system', 'libc')
+    print("system: %#x" % system)
+
+    # fix up the address
+    e.address += ref_addr - e.sym[ref]
+
+    # actual exploit: overwrite printf with system
+    f.write(e.sym.got.printf, system)
+    f.execute_writes()
+
+    p.sendline(b'echo ::$(( 16 * 16 ))')
+    p.recvuntil(b'::')
+    line = p.recvline()
+    assert line == b'256\n'
+    p.sendline(b'exit')
+    p.close()
+
+
+for context.binary in 'printf-loop.native32', 'printf-loop.native':
+    exploit(context.binary)

examples/fmtstr/printf-loop.c

@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <unistd.h>
+
+int main() {
+  while (1) {
+    char str[256];
+    int n = read(STDIN_FILENO, str, sizeof(str) - 1);
+    str[n] = '\0';
+    printf(str);
+    printf("\n=try again=\n");
+    fflush(stdout);
+  }
+  return 0;
+}

examples/gen-README.py

@@ -24,7 +24,7 @@ def append_example(_arg, top, names):
             continue
         path = os.path.join(top, name)[2:] # strip './'
         log.info('-> %s' % path)
-        data = read(path).strip()
+        data = read(path).strip().decode()
         if data[0:3] not in ('"""', "'''"):
             log.warning('  Has no docstring!')
             continue
@@ -37,6 +37,7 @@ def append_example(_arg, top, names):
         out += '* `%s`\n' % path
         out += '```%s```\n' % doc
 
-os.path.walk('.', append_example, None)
+for path, dirs, files in os.walk('.', onerror=None):
+    append_example(dirs, path, sorted(files))
 
 write('README.md', out)

examples/options.py

@@ -5,4 +5,4 @@ Example showing `pwnlib.ui.options()`
 from pwn import *
 
 opts = [string.letters[x] for x in range(10)]
-print 'You choose "%s"' % opts[options('Pick one:', opts)]
+print('You choose "%s"' % opts[options('Pick one:', opts)])

examples/port_forward.py

@@ -1,11 +1,10 @@
 """
-A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.  Notice
-that `<>` is just a shorthand.
+A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.
 """
 
 from pwn import *
 
 while True:
-    listen(1337).wait_for_connection() <> remote('google.com', 80)
+    listen(1337).wait_for_connection().connect_both(remote('google.com', 80))
 
 # now point your browser (or curl(1)) to http://localhost:1337

examples/readline_completers.py

@@ -19,13 +19,13 @@ c1 = LongestPrefixCompleter([
 c2 = PathCompleter(mask = '*.py')
 
 with c1:
-    print 'type "exit" to exit'
+    print('type "exit" to exit')
     while True:
-        s = term.readline.readline(prompt = '> ').strip()
+        s = str_input(prompt = '> ').strip()
         if s in ('exit', 'enough!'):
             break
-        print 'You wrote', s
+        print('You wrote', s)
 with c2:
-    print 'choose a file'
-    s = term.readline.readline(prompt = text.bold_green('$ ')).strip()
-    print 'You picked', s
+    print('choose a file')
+    s = str_input(prompt = text.bold_green('$ ')).strip()
+    print('You picked', s)

examples/remote.py

@@ -6,6 +6,6 @@ from pwn import *
 
 sock = remote('127.0.0.1', 9001)
 
-print sock.recvline()
+print(sock.recvline())
 sock.send('foo')
 sock.interactive()

examples/remote_gdb_debugging.py

@@ -8,10 +8,10 @@ import getpass
 from pwn import *
 
 s = ssh(getpass.getuser(), '127.0.0.1', port = 22, keyfile = "~/.ssh/id_rsa")
-c = gdb.ssh_gdb(s, '/bin/sh', execute = '''
+c = gdb.ssh_gdb(s, '/bin/sh', gdbscript = '''
 p/x $pc
 c''')
 
 c.sendline('ls -la')
 c.sendline('exit')
-print c.recvall()
+print(c.recvall())

examples/sigreturn_corefile_aarch64.py

@@ -3,11 +3,12 @@ context.arch='aarch64'
 frame = SigreturnFrame()
 
 registers = ['x%i' % i for i in range(0, 31)]
-registers += ['sp', 'pc']
+registers += ['pc']
 
 
 for index, register in enumerate(registers):
     setattr(frame, register, index)
+frame.sp = 64
 
 assembly = '\n'.join([
     shellcraft.read(constants.STDIN_FILENO, 'sp', 1024),
@@ -23,8 +24,9 @@ assert io.poll() == -11
 
 corefile = io.corefile
 
-for index, register in enumerate(registers):
+for register in registers:
     value = getattr(corefile, register)
+    index = getattr(frame, register)
     if index != value:
         log.error("%s != %i (%i)" % (register, index, value))
     else:

examples/sigreturn_corefile_arm.py

@@ -4,11 +4,12 @@ context.arch='arm'
 frame = SigreturnFrame()
 
 registers = ['r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6', 'r7',
-             'r8', 'r9', 'r10', 'fp', 'ip', 'sp', 'pc', 'lr']
+             'r8', 'r9', 'r10', 'fp', 'ip', 'pc', 'lr']
 
 
 for index, register in enumerate(registers):
     setattr(frame, register, index)
+frame.sp = 64
 
 assembly = '\n'.join([
     shellcraft.read(constants.STDIN_FILENO, 'sp', 1024),
@@ -24,8 +25,9 @@ assert io.poll() == -11
 
 corefile = io.corefile
 
-for index, register in enumerate(registers):
+for register in registers:
     value = getattr(corefile, register)
+    index = getattr(frame, register)
     if index != value:
         log.error("%s != %i (%i)" % (register, index, value))
     else:

examples/spinners.py

@@ -10,9 +10,9 @@ n = 1
 h = log.waitfor('spinners running', status = str(n))
 
 hs = []
-print 'type "q" to quit'
+print('type "q" to quit')
 while True:
-    s = raw_input('> ')
+    s = str_input('> ').strip()
     if s == 'q':
         break
     hs.append(log.waitfor(s, status = 'running'))

examples/splash.py

@@ -9,5 +9,5 @@ splash()
 h = log.waitfor("You wrote", status = "--")
 
 while True:
-    l = raw_input('> ')
+    l = str_input('> ').strip()
     h.status(l.upper())

examples/ssh.py

@@ -14,7 +14,7 @@ log.info("pwd: %s" % shell.pwd())
 tube = shell.run('cat')
 tube.send("Hello, cat")
 tube.shutdown("out")
-print tube.recvall()
+print(tube.recvall())
 
 # Show automatic working directories
 shell.set_working_directory()
@@ -29,12 +29,21 @@ int main() {
 
 shell.gcc(['example.c','-o','example'])
 
-print shell['./example']
+print(shell['./example'])
 
 # Show the different styles of calling
-print shell.echo("single string")
-print shell.echo(["list","of","strings"])
-print shell["echo single statement"]
+print(shell.echo("single string"))
+print(shell.echo(["list","of","strings"]))
+print(shell["echo single statement"])
+
+# ssh.process() is the most flexible way to run a command)
+io = shell.process(['/bin/bash', '-c', 'echo Hello $FOO'], 
+                   env={'FOO': 'World'}, # Set environment
+                   stderr='/dev/null',   # Override file descriptors
+                   aslr=False,           # Disable ASLR on processes
+                   setuid=False,         # Disable setuid bit so processes can be debugged
+                   shell=False)          # Enable or disable shell evaluation
+print(io.recvall())
 
 # Show off the interactive shell
 shell.interactive()

examples/text.py

@@ -12,8 +12,8 @@ and::
 from pwn import *
 
 s = 'hello from pwntools'
-print text.black_on_green(s)
-print text.black_on_bright_green(s)
-print text.green_on_black(s)
-print text.bright_green_on_black(s)
-print text.bold_green_on_black(s)
+print(text.black_on_green(s))
+print(text.black_on_bright_green(s))
+print(text.green_on_black(s))
+print(text.bright_green_on_black(s))
+print(text.bold_green_on_black(s))

examples/yesno.py

@@ -5,6 +5,6 @@ Example showing `pwnlib.ui.yesno()`
 from pwn import *
 
 if ui.yesno('Do you like Pwntools?'):
-    print ':D'
+    print(':D')
 else:
-    print ':('
+    print(':(')

extra/docker/base

@@ -1 +1 @@
-xenial
+bionic

extra/docker/bionic/Dockerfile

@@ -0,0 +1,26 @@
+############################################################
+# Dockerfile to build Pwntools container
+# Based on Ubuntu
+############################################################
+
+FROM ubuntu:bionic
+MAINTAINER Maintainer Gallopsled et al.
+
+RUN apt-get update \
+    && apt-get install -y \
+        build-essential \
+        git \
+        libssl-dev \
+        libffi-dev \
+        python2.7 \
+        python-pip \
+        python-dev \
+    && pip install --upgrade pip \
+    && python -m pip install --upgrade pwntools \
+    && PWNLIB_NOTERM=1 pwn update \
+    && apt-get install -y sudo \
+    && useradd -m pwntools \
+    && passwd --delete --unlock pwntools \
+    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
+USER pwntools
+WORKDIR /home/pwntools

pwn/toplevel.py

@@ -29,15 +29,18 @@ from pwnlib.elf.elf import ELF, load
 from pwnlib.encoders import *
 from pwnlib.exception import PwnlibException
 from pwnlib.gdb import attach, debug, debug_assembly, debug_shellcode
+from pwnlib.filepointer import *
 from pwnlib.flag import *
-from pwnlib.fmtstr import FmtStr, fmtstr_payload
+from pwnlib.fmtstr import FmtStr, fmtstr_payload, fmtstr_split
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak, RelativeMemLeak
 from pwnlib.regsort import *
 from pwnlib.replacements import *
 from pwnlib.rop import ROP
 from pwnlib.rop.srop import SigreturnFrame
+from pwnlib.rop.ret2dlresolve import Ret2dlresolvePayload
 from pwnlib.runner import *
+from pwnlib.term.readline import str_input
 from pwnlib.timeout import Timeout
 from pwnlib.tubes.listen import listen
 from pwnlib.tubes.process import process, PTY, PIPE, STDOUT
@@ -77,4 +80,4 @@ info    = log.info
 debug   = log.debug
 success = log.success
 
-__all__ = [x for x in globals().keys() if x != '__name__']
+__all__ = [x for x in tuple(globals()) if x != '__name__']

pwnlib/adb/adb.py

@@ -75,8 +75,10 @@ log = getLogger(__name__)
 def adb(argv, *a, **kw):
     r"""Returns the output of an ADB subcommand.
 
-    >>> adb.adb(['get-serialno'])
+    >>> adb.adb('get-serialno')
     b'emulator-5554\n'
+    >>> adb.adb(['shell', 'uname']) # it is better to use adb.process
+    b'Linux\n'
     """
     if isinstance(argv, (bytes, six.text_type)):
         argv = [argv]
@@ -115,7 +117,7 @@ def current_device(any=False):
 
         >>> device = adb.current_device(any=True)
         >>> device
-        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_phone_armv7', model='sdk phone armv7', device='generic')
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_google_phone_armv7', model='sdk google phone armv7', device='generic')
         >>> device.port
         'emulator'
     """
@@ -198,6 +200,10 @@ def uptime():
 
     Returns:
         Uptime of the device, in seconds
+
+    Example:
+        >>> adb.uptime() > 3 # normally AVD takes ~7 seconds to boot
+        True
     """
     up, idle = map(float, read('/proc/uptime').split())
     return up
@@ -209,10 +215,15 @@ def boot_time():
     Returns:
         Boot time of the device, in Unix time, rounded to the
         nearest second.
+
+    Example:
+        >>> import time
+        >>> adb.boot_time() < time.time()
+        True
     """
     for line in read('/proc/stat').splitlines():
         name, value = line.split(None, 1)
-        if name == 'btime':
+        if name == b'btime':
             return int(value)
 
 class AdbDevice(Device):
@@ -228,7 +239,7 @@ class AdbDevice(Device):
         >>> device.os
         'android'
         >>> device.product
-        'sdk_phone_armv7'
+        'sdk_google_phone_armv7'
         >>> device.serial
         'emulator-5554'
     """
@@ -277,7 +288,7 @@ class AdbDevice(Device):
             return
 
         with context.local(device=self.serial):
-            abi = str(properties.ro.product.cpu.abi)
+            abi = getprop('ro.product.cpu.abi')
             context.clear()
             context.arch = str(abi)
             self._arch = context.arch
@@ -662,9 +673,9 @@ def makedirs(path):
 
     Examples:
 
-        >>> adb.makedirs('/data/local/tmp/this/is/a/directory/heirarchy')
+        >>> adb.makedirs('/data/local/tmp/this/is/a/directory/hierarchy')
         >>> adb.listdir('/data/local/tmp/this/is/a/directory')
-        ['heirarchy']
+        ['hierarchy']
     """
     if path != '/':
         makedirs(os.path.dirname(path))
@@ -680,7 +691,7 @@ def exists(path):
 
         >>> adb.exists('/')
         True
-        >>> adb.exists('/init')
+        >>> adb.exists('/etc/hosts')
         True
         >>> adb.exists('/does/not/exist')
         False
@@ -816,17 +827,18 @@ def which(name, all = False, *a, **kw):
     """
     # Unfortunately, there is no native 'which' on many phones.
     which_cmd = '''
-echo $PATH | while read -d: directory; do
-    [ -x "$directory/{name}" ] || continue;
-    echo -n "$directory/{name}\\x00";
-done
+(IFS=:
+  for directory in $PATH; do
+      [ -x "$directory/{name}" ] || continue;
+      echo -n "$directory/{name}\\x00";
+  done
+)
 [ -x "{name}" ] && echo -n "$PWD/{name}\\x00"
 '''.format(name=name)
 
     which_cmd = which_cmd.strip()
     data = process(['sh','-c', which_cmd], *a, **kw).recvall()
-    if not hasattr(data, 'encode'):
-        data = data.decode('utf-8')
+    data = context._decode(data)
     result = []
 
     for path in data.split('\x00'):
@@ -849,6 +861,12 @@ done
 
 @with_device
 def whoami():
+    """Returns current shell user
+
+    Example:
+       >>> adb.whoami()
+       b'root'
+    """
     return process(['sh','-ic','echo $USER']).recvall().strip()
 
 @with_device
@@ -896,7 +914,12 @@ def pidof(name):
 
 @with_device
 def proc_exe(pid):
-    """Returns the full path of the executable for the provided PID."""
+    """Returns the full path of the executable for the provided PID.
+
+    Example:
+        >>> adb.proc_exe(1)
+        b'/init'
+    """
     with context.quiet:
         io  = process(['realpath','/proc/%d/exe' % pid])
         data = io.recvall().strip()
@@ -912,24 +935,23 @@ def getprop(name=None):
     Returns:
         If ``name`` is not specified, a ``dict`` of all properties is returned.
         Otherwise, a string is returned with the contents of the named property.
+
+    Example:
+        >>> adb.getprop() # doctest: +ELLIPSIS
+        {...}
     """
     with context.quiet:
         if name:
             result = process(['getprop', name]).recvall().strip()
-            if not hasattr(result, 'encode'):
-                result = result.decode('utf-8')
+            result = context._decode(result)
             return result
 
-
         result = process(['getprop']).recvall()
 
-    if not hasattr(result, 'encode'):
-        result = result.decode('utf-8')
-
+    result = context._decode(result)
     expr = r'\[([^\]]+)\]: \[(.*)\]'
 
     props = {}
-
     for line in result.splitlines():
         if not line.startswith('['):
             continue
@@ -960,7 +982,7 @@ def listdir(directory='/'):
         Otherwise, less files may be returned due to restrictive SELinux
         policies on adbd.
     """
-    return list(sorted(AdbClient().list(directory)))
+    return sorted(AdbClient().list(directory))
 
 @with_device
 def fastboot(args, *a, **kw):
@@ -975,17 +997,17 @@ def fastboot(args, *a, **kw):
 @with_device
 def fingerprint():
     """Returns the device build fingerprint."""
-    return str(properties.ro.build.fingerprint)
+    return getprop('ro.build.fingerprint')
 
 @with_device
 def product():
     """Returns the device product identifier."""
-    return str(properties.ro.build.product)
+    return getprop('ro.build.product')
 
 @with_device
 def build():
     """Returns the Build ID of the device."""
-    return str(properties.ro.build.id)
+    return getprop('ro.build.id')
 
 @with_device
 @no_emulator
@@ -995,15 +1017,44 @@ def unlock_bootloader():
     Note:
         This requires physical interaction with the device.
     """
-    AdbClient().reboot_bootloader()
-    fastboot(['oem', 'unlock'])
-    fastboot(['continue'])
+    w = log.waitfor("Unlocking bootloader")
+    with w:
+        if getprop('ro.oem_unlock_supported') == '0':
+            log.error("Bootloader cannot be unlocked: ro.oem_unlock_supported=0")
+
+        if getprop('ro.boot.oem_unlock_support') == '0':
+            log.error("Bootloader cannot be unlocked: ro.boot.oem_unlock_support=0")
+
+        if getprop('sys.oem_unlock_allowed') == '0':
+            log.error("Bootloader cannot be unlocked: Enable OEM Unlock in developer settings first", context.device)
+
+        AdbClient().reboot_bootloader()
+
+        # Check to see if it's unlocked before attempting unlock
+        unlocked = fastboot(['getvar', 'unlocked'])
+        if 'unlocked: yes' in unlocked:
+            w.success("Already unlocked")
+            fastboot(['continue'])
+            return
+
+        fastboot(['oem', 'unlock'])
+        unlocked = fastboot(['getvar', 'unlocked'])
+
+        fastboot(['continue'])
+
+        if 'unlocked: yes' not in unlocked:
+            log.error("Unlock failed")
 
 class Kernel(object):
     _kallsyms = None
 
     @property
     def address(self):
+        """Returns kernel address
+        Example:
+            >>> hex(adb.kernel.address) # doctest: +ELLIPSIS
+            '0x...000'
+        """
         return self.symbols['_text']
 
     @property
@@ -1026,7 +1077,7 @@ class Kernel(object):
             self._kallsyms = {}
             root()
             write('/proc/sys/kernel/kptr_restrict', '1')
-            self._kallsyms = read('/proc/kallsyms')
+            self._kallsyms = read('/proc/kallsyms').decode('ascii')
         return self._kallsyms
 
     @property
@@ -1054,7 +1105,7 @@ class Kernel(object):
 
     def enable_uart(self):
         """Reboots the device with kernel logging to the UART enabled."""
-        model = str(properties.ro.product.model)
+        model = getprop('ro.product.model')
 
         known_commands = {
             'Nexus 4': None,
@@ -1102,15 +1153,15 @@ kernel = Kernel()
 
 class Property(object):
     def __init__(self, name=None):
+        # Need to avoid overloaded setattr() so we go through __dict__
         self.__dict__['_name'] = name
 
     def __str__(self):
         return str(getprop(self._name)).strip()
 
-    def __repr__(self):
-        return repr(str(self))
-
     def __getattr__(self, attr):
+        if attr.startswith('_'):
+            raise AttributeError(attr)
         if self._name:
             attr = '%s.%s' % (self._name, attr)
         return Property(attr)
@@ -1123,6 +1174,21 @@ class Property(object):
             attr = '%s.%s' % (self._name, attr)
         setprop(attr, value)
 
+    def __eq__(self, other):
+        """Allow simple comparison
+
+        Example:
+            >>> adb.properties.ro.build.version.sdk == "24"
+            True
+        """
+        if isinstance(other, six.string_types):
+            return str(self) == other
+        return super(Property, self).__eq__(other)
+
+    def __hash__(self, other):
+        # Allow hash indices matching on the property
+        return hash(self._name)
+
 properties = Property()
 
 def _build_date():
@@ -1197,7 +1263,24 @@ def _generate_ndk_project(file_list, abi='arm-v7a', platform_version=21):
     return root
 
 def compile(source):
-    """Compile a source file or project with the Android NDK."""
+    r"""Compile a source file or project with the Android NDK.
+
+    Example:
+        >>> temp = tempfile.mktemp('.c')
+        >>> write(temp, '''
+        ... #include <stdio.h>
+        ... static char buf[4096];
+        ... int main() {
+        ...   FILE *fp = fopen("/proc/self/maps", "r");
+        ...   int n = fread(buf, 1, sizeof(buf), fp);
+        ...   fwrite(buf, 1, n, stdout);
+        ...   return 0;
+        ... }''')
+        >>> filename = adb.compile(temp)
+        >>> sent = adb.push(filename, "/data/local/tmp")
+        >>> adb.process(sent).recvall() # doctest: +ELLIPSIS
+        b'... /system/bin/linker\n...'
+    """
 
     ndk_build = misc.which('ndk-build')
     if not ndk_build:
@@ -1225,8 +1308,8 @@ def compile(source):
 
         # If we have an attached device, use its settings.
         if context.device:
-            abi = str(properties.ro.product.cpu.abi)
-            sdk = str(properties.ro.build.version.sdk)
+            abi = getprop('ro.product.cpu.abi')
+            sdk = getprop('ro.build.version.sdk')
 
         if abi is None:
             log.error("Unknown CPU ABI")
@@ -1244,7 +1327,7 @@ def compile(source):
     result = io.recvall()
 
     if 0 != io.poll():
-        log.error("Build failed:\n%s" % result)
+        log.error("Build failed:\n%s", result)
 
     # Find all of the output files
     output = glob.glob(os.path.join(lib, '*', '*'))
@@ -1294,37 +1377,59 @@ def find(top, name):
 
 @with_device
 def readlink(path):
-    path = process(['readlink', path]).recvall()
+    path = process(['realpath', path]).recvall()
 
     # Readlink will emit a single newline
     # We can't use the '-n' flag since old versions don't support it
-    if path.endswith('\n'):
+    if path.endswith(b'\n'):
         path = path[:-1]
 
-    return path
+    return path.decode()
 
 class Partitions(object):
+    """Enable access to partitions
+
+    Example:
+        >>> hex(adb.partitions.vda.size) # doctest: +ELLIPSIS
+        '0x...000'
+    """
     @property
     @context.quietfunc
     def by_name_dir(self):
-        return next(find('/dev/block/platform','by-name'))
+        try:
+            return next(find('/dev/block/platform','by-name'))
+        except StopIteration:
+            return '/dev/block'
 
     @context.quietfunc
     def __dir__(self):
         return list(self)
 
+    @context.quietfunc
+    def iter_proc_partitions(self):
+        for line in read('/proc/partitions').splitlines():
+            if not line.strip():
+                continue
+            major, minor, blocks, name = line.split(None, 4)
+            yield blocks, name.decode()
+
     @context.quietfunc
     @with_device
     def __iter__(self):
         root()
 
         # Find all named partitions
-        for name in listdir(self.by_name_dir):
-            yield name
+        names = set(listdir(self.by_name_dir))
+
+        # Find all unnamed partitions
+        for _, name in self.iter_proc_partitions():
+            names.add(name)
+        return iter(names)
 
-    @context.quietfunc
-    @with_device
     def __getattr__(self, attr):
+        if attr.startswith("_"):
+            raise AttributeError(attr)
+
         for name in self:
             if name == attr:
                 break
@@ -1333,19 +1438,17 @@ class Partitions(object):
 
         path = os.path.join(self.by_name_dir, name)
 
-        # Find the actual path of the device
-        devpath = readlink(path)
-        devname = os.path.basename(devpath)
+        with context.quiet:
+            # Find the actual path of the device
+            devpath = readlink(path)
+            devname = os.path.basename(devpath)
 
-        # Get the size of the partition
-        for line in read('/proc/partitions').splitlines():
-            if not line.strip():
-                continue
-            major, minor, blocks, name = line.split(None, 4)
-            if devname == name:
-                break
-        else:
-            log.error("Could not find size of partition %r" % name)
+            # Get the size of the partition
+            for blocks, name in self.iter_proc_partitions():
+                if name in (devname, attr):
+                    break
+            else:
+                log.error("Could not find size of partition %r" % name)
 
         return Partition(devpath, attr, int(blocks))
 
@@ -1397,3 +1500,9 @@ def packages():
     """Returns a list of packages installed on the system"""
     packages = process(['pm', 'list', 'packages']).recvall()
     return [line.split('package:', 1)[-1] for line in packages.splitlines()]
+
+@context.quietfunc
+def version():
+    """Returns rthe platform version as a tuple."""
+    prop = getprop('ro.build.version.release')
+    return [int(v) for v in prop.split('.')]

pwnlib/adb/bootloader.py

@@ -4,6 +4,7 @@ from __future__ import division
 import ctypes
 import io
 import os
+import six
 import sys
 
 from pwnlib.log import getLogger
@@ -64,7 +65,7 @@ class BootloaderImage(object):
         Returns:
             Contents of the image.
         """
-        if isinstance(index_or_name, int):
+        if isinstance(index_or_name, six.integer_types):
             index = index_or_name
         else:
             for i in range(len(self.img_info)):

pwnlib/asm.py

@@ -167,6 +167,9 @@ def which_binutils(util):
         'i386':   ['x86_64', 'amd64'],
         'i686':   ['x86_64', 'amd64'],
         'amd64':  ['x86_64', 'i386'],
+        'mips64': ['mips'],
+        'powerpc64': ['powerpc'],
+        'sparc64': ['sparc'],
     }.get(arch, [])
 
     # If one of the candidate architectures matches the native
@@ -190,16 +193,18 @@ def which_binutils(util):
         for gutil in utils:
             # e.g. objdump
             if arch is None:
-                pattern = gutil
+                patterns = [gutil]
 
-            # e.g. aarch64-linux-gnu-objdump
+            # e.g. aarch64-linux-gnu-objdump, avr-objdump
             else:
-                pattern = '%s*linux*-%s' % (arch,gutil)
+                patterns = ['%s*linux*-%s' % (arch, gutil),
+                            '%s-%s' % (arch, gutil)]
 
-            for dir in environ['PATH'].split(':'):
-                res = sorted(glob(path.join(dir, pattern)))
-                if res:
-                    return res[0]
+            for pattern in patterns:
+                for dir in environ['PATH'].split(':'):
+                    res = sorted(glob(path.join(dir, pattern)))
+                    if res:
+                        return res[0]
 
     # No dice!
     print_binutils_instructions(util, context)
@@ -245,9 +250,9 @@ def _assembler():
                                          stderr=subprocess.STDOUT, universal_newlines=True)
         version = re.search(r' (\d\.\d+)', result).group(1)
         if version < '2.19':
-            log.warn_once('Your binutils version is too old and may not work!\n'  + \
-                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n' + \
-                'Reported Version: %r' % result.strip())
+            log.warn_once('Your binutils version is too old and may not work!\n'
+                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+                'Reported Version: %r', result.strip())
 
 
     return assembler
@@ -322,6 +327,7 @@ def _bfdname():
         'amd64'   : 'elf64-x86-64',
         'arm'     : 'elf32-%sarm' % E,
         'thumb'   : 'elf32-%sarm' % E,
+        'avr'     : 'elf32-avr',
         'mips'    : 'elf32-trad%smips' % E,
         'mips64'  : 'elf64-trad%smips' % E,
         'alpha'   : 'elf64-alpha',
@@ -344,10 +350,13 @@ def _bfdname():
 def _bfdarch():
     arch = context.arch
     convert = {
-    'i386': 'i386',
-    'amd64': 'i386:x86-64',
-    'thumb': 'arm',
-    'ia64': 'ia64-elf64'
+        'amd64':     'i386:x86-64',
+        'i386':      'i386',
+        'ia64':      'ia64-elf64',
+        'mips64':    'mips',
+        'powerpc64': 'powerpc',
+        'sparc64':   'sparc',
+        'thumb':     'arm',
     }
 
     if arch in convert:
@@ -576,7 +585,7 @@ def make_elf(data,
         _run(assembler + ['-o', step2, step1])
 
         linker_options = ['-z', 'execstack']
-        if vma:
+        if vma is not None:
             linker_options += ['--section-start=.shellcode=%#x' % vma,
                                '--entry=%#x' % vma]
         elif shared:
@@ -749,6 +758,12 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         >>> print(disasm(unhex('656664676665400F18A4000000000051'), byte=0, arch='amd64'))
            0:   gs data16 fs data16 rex nop/reserved BYTE PTR gs:[eax+eax*1+0x0]
            f:   push   rcx
+        >>> print(disasm(unhex('01000000'), arch='sparc64'))
+           0:   01 00 00 00     nop
+        >>> print(disasm(unhex('60000000'), arch='powerpc64'))
+           0:   60 00 00 00     nop
+        >>> print(disasm(unhex('00000000'), arch='mips64'))
+           0:   00000000        nop
     """
     result = ''
 

pwnlib/commandline/__init__.py

@@ -18,5 +18,6 @@ __all__ = [
     'scramble',
     'shellcraft',
     'unhex',
-    'update'
+    'update',
+    'version'
 ]

pwnlib/commandline/asm.py

@@ -5,6 +5,9 @@ from __future__ import division
 import argparse
 import sys
 
+import pwnlib
+pwnlib.args.free_form = False
+
 from pwn import *
 from pwnlib.commandline import common
 
@@ -130,7 +133,7 @@ def main(args):
             output = output.encode('ascii')
         args.output.write(output)
 
-    if tty and fmt is not 'raw':
+    if tty and fmt != 'raw':
         args.output.write(b'\n')
 
 if __name__ == '__main__':

pwnlib/commandline/constgrep.py

@@ -4,7 +4,6 @@ from __future__ import division
 
 import argparse
 import functools
-import os
 import re
 
 import pwnlib
@@ -19,25 +18,22 @@ p = common.parser_commands.add_parser(
     formatter_class = argparse.RawDescriptionHelpFormatter,
 )
 
-group = p.add_mutually_exclusive_group()
-group.add_argument(
+p.add_argument(
     '-e', '--exact',
-    metavar = 'constant',
-    # nargs = 1,
-    default = None,
-    help = 'Do an exact match for a constant instead of searching for a regex',
+    action='store_true',
+    help='Do an exact match for a constant instead of searching for a regex',
 )
-group.add_argument(
+
+p.add_argument(
     'regex',
-    nargs = '?',
-    default = '',
-    help = 'The regex matching constant you want to find',
+    help='The regex matching constant you want to find',
 )
 
 p.add_argument(
     'constant',
     nargs = '?',
     default = None,
+    type = safeeval.expr,
     help = 'The constant to find',
 )
 
@@ -65,7 +61,7 @@ p.add_argument(
 def main(args):
     if args.exact:
         # This is the simple case
-        print(cpp(args.exact).strip())
+        print(cpp(args.regex).strip())
     else:
         # New we search in the right module.
         # But first: We find the right module
@@ -80,20 +76,12 @@ def main(args):
         else:
             matcher = re.compile(args.regex)
 
-
-        # Evaluate the given constant
-        if args.constant:
-            try:
-                constant = safeeval.expr(args.constant)
-            except:
-                log.error("Could not evaluate constant %r" % args.constant)
-        else:
-            constant = None
-
         # The found matching constants and the length of the longest string
         out    = []
         maxlen = 0
 
+        constant = args.constant
+
         for k in dir(mod):
             # No python stuff
             if k.endswith('__') and k.startswith('__'):
@@ -104,7 +92,7 @@ def main(args):
                 continue
 
             # Check the constant
-            if constant != None:
+            if constant is not None:
                 val = getattr(mod, k)
                 if args.mask_mode:
                     if constant & val != val:
@@ -124,7 +112,7 @@ def main(args):
         # If we are in match_mode, then try to find a combination of
         # constants that yield the exact given value
         # We do not want to find combinations using the value 0.
-        if not (constant == None or constant == 0) and args.mask_mode:
+        if constant and args.mask_mode:
             mask = constant
             good = []
             out = [(v, k) for v, k in out if v != 0]

pwnlib/commandline/cyclic.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import
 from __future__ import division
 
 import argparse
+import six
 import string
 import sys
 
@@ -20,7 +21,8 @@ parser = common.parser_commands.add_parser(
 parser.add_argument(
     '-a', '--alphabet',
     metavar = 'alphabet',
-    default = string.ascii_lowercase,
+    default = string.ascii_lowercase.encode(),
+    type = six.ensure_binary,
     help = 'The alphabet to use in the cyclic pattern (defaults to all lower case letters)',
 )
 
@@ -41,7 +43,7 @@ parser.add_argument(
     help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
 )
 
-group = parser.add_mutually_exclusive_group(required = True)
+group = parser.add_mutually_exclusive_group(required=False)
 group.add_argument(
     '-l', '-o', '--offset', '--lookup',
     dest = 'lookup',
@@ -51,9 +53,10 @@ group.add_argument(
 
 group.add_argument(
     'count',
-    type = int,
-    nargs = '?',
-    help = 'Number of characters to print'
+    type=int,
+    nargs='?',
+    default=None,
+    help='Number of characters to print'
 )
 
 def main(args):
@@ -64,9 +67,10 @@ def main(args):
         pat = args.lookup
 
         try:
-            pat = packing.pack(int(pat, 0), subsize*8)
+            pat = int(pat, 0)
         except ValueError:
             pass
+        pat = flat(pat)
 
         if len(pat) != subsize:
             log.critical('Subpattern must be %d bytes' % subsize)
@@ -87,13 +91,14 @@ def main(args):
         want   = args.count
         result = cyclic(want, alphabet, subsize)
         got    = len(result)
-        if got < want:
+        if want is not None and got < want:
             log.failure("Alphabet too small (max length = %i)" % got)
 
-        sys.stdout.write(result)
+        out = getattr(sys.stdout, 'buffer', sys.stdout)
+        out.write(result)
 
-        if sys.stdout.isatty():
-            sys.stdout.write('\n')
+        if out.isatty():
+            out.write(b'\n')
 
 if __name__ == '__main__':
     pwnlib.commandline.common.main(__file__)

pwnlib/commandline/errno.py

@@ -45,4 +45,4 @@ def main(args):
   print(os.strerror(value))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/hex.py

@@ -26,4 +26,4 @@ def main(args):
         print(enhex(data))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/main.py

@@ -21,6 +21,7 @@ from pwnlib.commandline import shellcraft
 from pwnlib.commandline import template
 from pwnlib.commandline import unhex
 from pwnlib.commandline import update
+from pwnlib.commandline import version
 from pwnlib.commandline.common import parser
 from pwnlib.context import context
 
@@ -43,6 +44,7 @@ commands = {
     'template': template.main,
     'unhex': unhex.main,
     'update': update.main,
+    'version': version.main,
 }
 
 def main():

pwnlib/commandline/phd.py

@@ -85,12 +85,12 @@ def main(args):
     text.when = color
 
     if skip:
-        if infile == sys.stdin:
-            infile.read(skip)
-        else:
+        try:
             infile.seek(skip, os.SEEK_CUR)
+        except IOError:
+            infile.read(skip)
 
-    if count:
+    if count != -1:
         infile = io.BytesIO(infile.read(count))
 
     hl = []

pwnlib/commandline/scramble.py

@@ -80,8 +80,8 @@ def main(args):
         parser.print_usage()
         sys.exit(0)
 
-    data   = sys.stdin.read()
-    output = data
+    stdin_buffer = getattr(sys.stdin, 'buffer', sys.stdin)
+    output = stdin_buffer.read()
     fmt    = args.format or ('hex' if tty else 'raw')
     formatters = {'r':bytes, 'h':enhex, 's':repr}
 

pwnlib/commandline/template.py

@@ -18,9 +18,10 @@ parser.add_argument('exe', nargs='?', help='Target binary')
 parser.add_argument('--host', help='Remote host / SSH server')
 parser.add_argument('--port', help='Remote port / SSH port', type=int)
 parser.add_argument('--user', help='SSH Username')
-parser.add_argument('--pass', help='SSH Password', dest='password')
+parser.add_argument('--pass', '--password', help='SSH Password', dest='password')
 parser.add_argument('--path', help='Remote path of file on SSH server')
 parser.add_argument('--quiet', help='Less verbose template comments', action='store_true')
+parser.add_argument('--color', help='Print the output in color', choices=['never', 'always', 'auto'], default='auto')
 
 def main(args):
     cache = None
@@ -57,8 +58,16 @@ def main(args):
     # Fix Mako formatting bs
     output = re.sub('\n\n\n', '\n\n', output)
 
+    # Colorize the output if it's a TTY
+    if args.color == 'always' or (args.color == 'auto' and sys.stdout.isatty()):
+        from pygments import highlight
+        from pygments.formatters import TerminalFormatter
+        from pygments.lexers.python import PythonLexer
+        output = highlight(output, PythonLexer(), TerminalFormatter())
+
     print(output)
 
+    # If redirected to a file, make the resulting script executable
     if not sys.stdout.isatty():
         try: os.fchmod(sys.stdout.fileno(), 0o700)
         except OSError: pass

pwnlib/commandline/unhex.py

@@ -30,4 +30,4 @@ def main(args):
         sys.stderr.write(str(e) + '\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/version.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+from __future__ import division
+
+import os
+import subprocess
+
+import pwnlib
+pwnlib.args.free_form = False
+
+from pwn import *
+from pwnlib.commandline import common
+
+parser = common.parser_commands.add_parser(
+    'version',
+    help = 'Pwntools version'
+)
+
+def main(a):
+	version = pwnlib.version
+
+	git_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+	if os.path.exists(os.path.join(git_root, '.git')):
+		gitver = subprocess.check_output(['git', '-C', git_root, 'log', '-1', '--format=%h (%cr)'])
+		branch = subprocess.check_output(['git', '-C', git_root, 'rev-parse', '--abbrev-ref', 'HEAD'])
+		version = '%s-%s-%s' % (version, branch.decode().strip(), gitver.decode())
+
+	log.info("Pwntools v%s" % version)
+
+if __name__ == '__main__':
+    pwnlib.commandline.common.main(__file__)

pwnlib/config.py

@@ -26,6 +26,9 @@ supported by ``pwnlib.util.safeeval.expr``.
     randomize=1
     timeout=60
     terminal=['x-terminal-emulator', '-e']
+
+    [update]
+    interval=7
 """
 from __future__ import absolute_import
 from __future__ import division

pwnlib/constants/__init__.py

@@ -92,7 +92,7 @@ class ConstantsModule(ModuleType):
         # Special case for __all__, we want to return the contextually
         # relevant module.
         if key == '__all__':
-            return self.guess().__dict__.keys()
+            return list(self.guess().__dict__.keys())
 
         # Special case for all other special properties which aren't defined
         if key.endswith('__'):

pwnlib/context/__init__.py

@@ -27,6 +27,11 @@ from pwnlib.config import register_config
 from pwnlib.device import Device
 from pwnlib.timeout import Timeout
 
+try:
+    from collections.abc import Iterable
+except ImportError:
+    from collections import Iterable
+
 __all__ = ['context', 'ContextType', 'Thread']
 
 _original_socket = socket.socket
@@ -685,6 +690,7 @@ class ContextType(object):
                      ('x86_64', 'amd64'),
                      ('x86', 'i386'),
                      ('i686', 'i386'),
+                     ('armv7l', 'arm'),
                      ('armeabi', 'arm'),
                      ('arm64', 'aarch64')]
         for k, v in transform:
@@ -838,6 +844,8 @@ class ContextType(object):
                 return b.decode('utf-8')
             except UnicodeDecodeError:
                 return b.decode('latin1')
+            except AttributeError:
+                return b
         return b.decode(self.encoding)
 
     @_validator
@@ -1116,7 +1124,7 @@ class ContextType(object):
         if isinstance(proxy, str):
             proxy = (socks.SOCKS5, proxy)
 
-        if not isinstance(proxy, collections.Iterable):
+        if not isinstance(proxy, Iterable):
             raise AttributeError('proxy must be a string hostname, or tuple of arguments for socks.set_default_proxy')
 
         socks.set_default_proxy(*proxy)
@@ -1224,12 +1232,13 @@ class ContextType(object):
             >>> cache_dir == context.cache_dir
             True
         """
-        home = os.path.expanduser('~')
+        xdg_cache_home = os.environ.get('XDG_CACHE_HOME') or \
+                         os.path.join(os.path.expanduser('~'), '.cache')
 
-        if not os.access(home, os.W_OK):
+        if not os.access(xdg_cache_home, os.W_OK):
             return None
 
-        cache = os.path.join(home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
+        cache = os.path.join(xdg_cache_home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
 
         if not os.path.exists(cache):
             try:

pwnlib/data/elf/__init__.py

@@ -1,5 +1,7 @@
 from __future__ import absolute_import
+from pwnlib.data.elf import fmtstr
 from pwnlib.data.elf import relro
+from pwnlib.data.elf import ret2dlresolve
 
 import os
 path = os.path.dirname(__file__)

pwnlib/data/elf/fmtstr/Makefile

@@ -0,0 +1,5 @@
+
+all: i386
+
+i386: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -m32 -o $@

pwnlib/data/elf/fmtstr/__init__.py

@@ -0,0 +1,5 @@
+import os
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)

pwnlib/data/elf/fmtstr/vuln.c

@@ -0,0 +1,25 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#define MEMORY_ADDRESS ((void*)0x11111000)
+#define MEMORY_SIZE 1024
+#define TARGET ((int *) 0x11111110)
+int main(int argc, char const *argv[])
+{
+       char buff[1024];
+       void *ptr = NULL;
+       int *my_var = TARGET;
+       ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+       if(ptr != MEMORY_ADDRESS)
+       {
+               perror("mmap");
+               return EXIT_FAILURE;
+       }
+       *my_var = 0x41414141;
+       write(1, &my_var, sizeof(int *));
+       scanf("%s", buff);
+       dprintf(2, buff);
+       write(1, my_var, sizeof(int));
+       return 0;
+}

pwnlib/data/elf/ret2dlresolve/Makefile

@@ -0,0 +1,8 @@
+
+all: i386 amd64
+
+i386: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -m32 -o $@
+
+amd64: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -o $@

pwnlib/data/elf/ret2dlresolve/__init__.py

@@ -0,0 +1,5 @@
+import os
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)

pwnlib/data/elf/ret2dlresolve/vuln.c

@@ -0,0 +1,8 @@
+#include <unistd.h>
+void vuln(void){
+    char buf[64];
+    read(STDIN_FILENO, buf, 200);
+}
+int main(int argc, char** argv){
+    vuln();
+}

pwnlib/dynelf.py

@@ -75,8 +75,8 @@ def sysv_hash(symbol):
     """
     h = 0
     g = 0
-    for c in symbol:
-        h = (h << 4) + ord(c)
+    for c in bytearray(context._encode(symbol)):
+        h = (h << 4) + c
         g = h & 0xf0000000
         h ^= (g >> 24)
         h &= ~g
@@ -87,9 +87,10 @@ def gnu_hash(s):
 
     Function used to generated GNU-style hashes for strings.
     """
+    s = bytearray(context._encode(s))
     h = 5381
     for c in s:
-        h = h * 33 + ord(c)
+        h = h * 33 + c
     return h & 0xffffffff
 
 class DynELF(object):
@@ -292,7 +293,7 @@ class DynELF(object):
         w = None
 
         while True:
-            if self.leak.compare(ptr, '\x7fELF'):
+            if self.leak.compare(ptr, b'\x7fELF'):
                 break
 
             # See if we can short circuit the search
@@ -641,7 +642,7 @@ class DynELF(object):
             p_name = leak.field(cur, LinkMap.l_name)
             name   = leak.s(p_name)
 
-            if libname in name:
+            if libname.encode('utf-8') in name:
                 break
 
             if name:
@@ -740,6 +741,7 @@ class DynELF(object):
 
                 # Leak the name of the function from the symbol table
                 name = leak.s(strtab + leak.field(sym, Sym.st_name))
+                name = name.decode('utf-8')
 
                 # Make sure it matches the name of the symbol we were looking for.
                 if name == symb:
@@ -823,6 +825,7 @@ class DynELF(object):
                 # Check for collision on hash values
                 sym  = symtab + sizeof(Sym) * (ndx + i)
                 name = leak.s(strtab + leak.field(sym, Sym.st_name))
+                name = name.decode('utf-8')
 
                 if name == symb:
                     # No collision, get offset and calculate address
@@ -854,10 +857,10 @@ class DynELF(object):
 
         for offset in libcdb.get_build_id_offsets():
             address = libbase + offset
-            if self.leak.compare(address + 0xC, "GNU\x00"):
+            if self.leak.compare(address + 0xC, b"GNU\x00"):
                 return enhex(b''.join(self.leak.raw(address + 0x10, 20)))
             else:
-                self.status("Magic did not match")
+                self.status("Build ID not found at offset %#x" % offset)
                 pass
 
     def _make_absolute_ptr(self, ptr_or_offset):

pwnlib/elf/config.py

@@ -21,8 +21,8 @@ class KernelConfig(object):
         self.excludes = set(excludes)
 
         #: Kernel version that this check should be enforced on
-        self.minver = map(int, str(minver).split('.'))
-        self.maxver = map(int, str(maxver).split('.'))
+        self.minver = list(map(int, str(minver).split('.')))
+        self.maxver = list(map(int, str(maxver).split('.')))
 
     def relevant(self, config):
 

pwnlib/elf/corefile.py

@@ -219,8 +219,8 @@ class Mapping(object):
                 stop += self.stop
 
             if not (self.start <= start <= stop <= self.stop):
-                log.error("Byte range [%#x:%#x] not within range [%#x:%#x]" \
-                    % (start, stop, self.start, self.stop))
+                log.error("Byte range [%#x:%#x] not within range [%#x:%#x]",
+                          start, stop, self.start, self.stop)
 
             data = self._core.read(start, stop-start)
 
@@ -381,13 +381,15 @@ class Corefile(ELF):
         This requires GDB to be installed, and can only be done with native
         processes.  Getting a "complete" corefile requires GDB 7.11 or better.
 
-        >>> elf = ELF('/bin/bash')
+        >>> elf = ELF('/bin/bash-static')
         >>> context.clear(binary=elf)
         >>> io = process(elf.path, env={'HELLO': 'WORLD'})
         >>> core = io.corefile
 
         Data can also be extracted directly from the corefile.
 
+        >>> elf.address > 0
+        True
         >>> core.exe[elf.address:elf.address+4]
         b'\x7fELF'
         >>> core.exe.data[:4]
@@ -429,13 +431,13 @@ class Corefile(ELF):
 
         Corefiles can also be pulled from remote machines via SSH!
 
-        >>> s = ssh('travis', 'example.pwnme')
+        >>> s = ssh(user='travis', host='example.pwnme', password='demopass')
         >>> _ = s.set_working_directory()
         >>> elf = ELF.from_assembly(shellcraft.trap())
         >>> path = s.upload(elf.path)
         >>> _ =s.chmod('+x', path)
         >>> io = s.process(path)
-        >>> io.wait()
+        >>> io.wait(1)
         -1
         >>> io.corefile.signal == signal.SIGTRAP # doctest: +SKIP
         True
@@ -445,7 +447,7 @@ class Corefile(ELF):
         >>> context.clear(arch='amd64')
         >>> elf = ELF.from_assembly('push 1234; ret')
         >>> io = elf.process()
-        >>> io.wait()
+        >>> io.wait(1)
         >>> io.corefile.fault_addr
         1234
 
@@ -459,7 +461,7 @@ class Corefile(ELF):
 
         >>> elf = ELF.from_assembly(shellcraft.crash())
         >>> io = elf.process(env={'FOO': 'BAR=BAZ'})
-        >>> io.wait()
+        >>> io.wait(1)
         >>> core = io.corefile
         >>> core.getenv('FOO')
         b'BAR=BAZ'
@@ -480,9 +482,9 @@ class Corefile(ELF):
         ... '''
         >>> elf = ELF.from_assembly(assembly)
         >>> io = elf.process()
-        >>> io.wait()
+        >>> io.wait(2)
         >>> core = io.corefile
-        [!] End of the stack is corrupted, skipping stack parsing (got: 4141414141414141)
+        [!] End of the stack is corrupted, skipping stack parsing (got: 41414141)
         >>> core.argc, core.argv, core.env
         (0, [], {})
         >>> core.stack.data.endswith(b'AAAA')
@@ -549,15 +551,15 @@ class Corefile(ELF):
         self.load_addr = 0
         self._address  = 0
 
-        if not self.elftype == 'CORE':
+        if self.elftype != 'CORE':
             log.error("%s is not a valid corefile" % self.file.name)
 
-        if not self.arch in prstatus_types.keys():
+        if self.arch not in prstatus_types:
             log.warn_once("%s does not use a supported corefile architecture, registers are unavailable" % self.file.name)
 
-        prstatus_type = prstatus_types.get(self.arch, None)
-        prpsinfo_type = prpsinfo_types.get(self.bits, None)
-        siginfo_type = siginfo_types.get(self.bits, None)
+        prstatus_type = prstatus_types.get(self.arch)
+        prpsinfo_type = prpsinfo_types.get(self.bits)
+        siginfo_type = siginfo_types.get(self.bits)
 
         with log.waitfor("Parsing corefile...") as w:
             self._load_mappings()
@@ -667,8 +669,8 @@ class Corefile(ELF):
                 continue
 
             if mapping.start == self.at_sysinfo_ehdr \
-            or (not vdso and mapping.size in [0x1000, 0x2000] \
-                and mapping.flags == 5 \
+            or (not vdso and mapping.size in [0x1000, 0x2000]
+                and mapping.flags == 5
                 and self.read(mapping.start, 4) == b'\x7fELF'):
                 mapping.name = '[vdso]'
                 vdso = True
@@ -747,13 +749,13 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly(shellcraft.trap())
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.signal == signal.SIGTRAP
             True
 
             >>> elf = ELF.from_assembly(shellcraft.crash())
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.signal == signal.SIGSEGV
             True
         """
@@ -774,7 +776,7 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly('mov eax, 0xdeadbeef; jmp eax', arch='i386')
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.fault_addr == io.corefile.eax == 0xdeadbeef
             True
         """
@@ -922,9 +924,9 @@ class Corefile(ELF):
             return
 
         # If the stack does not end with zeroes, something is very wrong.
-        if not stack.data.endswith(b'\x00' * 8):
+        if not stack.data.endswith(b'\x00' * context.bytes):
             log.warn_once("End of the stack is corrupted, skipping stack parsing (got: %s)",
-                          enhex(self.data[-8:]))
+                          enhex(self.data[-context.bytes:]))
             return
 
         # AT_EXECFN is the start of the filename, e.g. '/bin/sh'
@@ -943,16 +945,11 @@ class Corefile(ELF):
 
         # Sanity check!
         try:
-            assert stack[address] == b'\x00'
-        except AssertionError:
-            # Something weird is happening.  Just don't touch it.
-            log.debug("Something is weird")
-            return
+            if stack[address] != b'\x00':
+                log.warning("Error parsing corefile stack: Could not find end of environment")
+                return
         except ValueError:
-            # If the stack is not actually present in the coredump, we can't
-            # read from the stack.  This will fail as:
-            # ValueError: 'seek out of range'
-            log.debug("ValueError")
+            log.warning("Error parsing corefile stack: Address out of bounds")
             return
 
         # address is currently set to the NULL terminator of the last
@@ -966,12 +963,14 @@ class Corefile(ELF):
         p_last_env_addr = stack.find(pack(last_env_addr), None, last_env_addr)
         if p_last_env_addr < 0:
             # Something weird is happening.  Just don't touch it.
-            log.warn_once("Found bad environment at %#x", last_env_addr)
+            log.warn_once("Error parsing corefile stack: Found bad environment at %#x", last_env_addr)
             return
 
         # Sanity check that we did correctly find the envp NULL terminator.
         envp_nullterm = p_last_env_addr+context.bytes
-        assert self.unpack(envp_nullterm) == 0
+        if self.unpack(envp_nullterm) != 0:
+            log.warning("Error parsing corefile stack: Could not find end of environment variables")
+            return
 
         # We've successfully located the end of the envp[] array.
         #
@@ -1071,7 +1070,7 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly(shellcraft.trap())
             >>> io = elf.process(env={'GREETING': 'Hello!'})
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.getenv('GREETING')
             b'Hello!'
         """
@@ -1090,7 +1089,7 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly('mov eax, 0xdeadbeef;' + shellcraft.trap(), arch='i386')
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.registers['eax'] == 0xdeadbeef
             True
         """
@@ -1533,4 +1532,3 @@ class CorefileFinder(object):
                 return keys['interpreter']
 
         return ''
-

pwnlib/elf/datatypes.py

@@ -191,7 +191,7 @@ class constants:
     NT_AUXV                 = 6
     #
     # Note to userspace developers: size of NT_SIGINFO note may increase
-    # in the future to accomodate more fields, don't assume it is fixed!
+    # in the future to accommodate more fields, don't assume it is fixed!
     #
     NT_SIGINFO              = 0x53494749
     NT_FILE                 = 0x46494c45

pwnlib/elf/elf.py

@@ -83,12 +83,19 @@ from pwnlib.tubes.process import process
 from pwnlib.util import misc
 from pwnlib.util import packing
 from pwnlib.util.fiddling import unhex
+from pwnlib.util.misc import align, align_down
 from pwnlib.util.sh_string import sh_string
 
 log = getLogger(__name__)
 
 __all__ = ['load', 'ELF']
 
+def _iter_symbols(sec):
+    # Cache result of iter_symbols.
+    if not hasattr(sec, '_symbols'):
+        sec._symbols = list(sec.iter_symbols())
+    return iter(sec._symbols)
+
 class Function(object):
     """Encapsulates information about a function in an :class:`.ELF` binary.
 
@@ -268,6 +275,9 @@ class ELF(ELFFile):
                 self.arch = 'mips64'
                 self.bits = 64
 
+        self._sections = None
+        self._segments = None
+
         #: IntervalTree which maps all of the loaded memory segments
         self.memory = intervaltree.IntervalTree()
         self._populate_memory()
@@ -309,7 +319,7 @@ class ELF(ELFFile):
                 config = gz.read()
 
             if config:
-                self.config = parse_kconfig(config)
+                self.config = parse_kconfig(config.decode())
 
         #: ``True`` if the ELF is a statically linked executable
         self.statically_linked = bool(self.elftype == 'EXEC' and self.load_addr)
@@ -454,9 +464,6 @@ class ELF(ELFFile):
             self.checksec(*a, **kw)
         )
 
-    def __repr__(self):
-        return "ELF(%r)" % self.path
-
     def get_machine_arch(self):
         return {
             'EM_X86_64': 'amd64',
@@ -484,6 +491,13 @@ class ELF(ELFFile):
         """:class:`str`: ELF type (``EXEC``, ``DYN``, etc)"""
         return describe_e_type(self.header.e_type).split()[0]
 
+    def iter_segments(self):
+        # Yield and cache all the segments in the file
+        if self._segments is None:
+            self._segments = [self.get_segment(i) for i in range(self.num_segments())]
+
+        return iter(self._segments)
+
     @property
     def segments(self):
         """
@@ -536,6 +550,13 @@ class ELF(ELFFile):
 
         return None
 
+    def iter_sections(self):
+        # Yield and cache all the sections in the file
+        if self._sections is None:
+            self._sections = [self.get_section(i) for i in range(self.num_sections())]
+
+        return iter(self._sections)
+
     @property
     def sections(self):
         """
@@ -591,6 +612,8 @@ class ELF(ELFFile):
         self.symbols = dotdict({k:update(v) for k,v in self.symbols.items()})
         self.plt     = dotdict({k:update(v) for k,v in self.plt.items()})
         self.got     = dotdict({k:update(v) for k,v in self.got.items()})
+        for f in self.functions.values():
+            f.address += delta
 
         # Update our view of memory
         memory = intervaltree.IntervalTree()
@@ -839,7 +862,7 @@ class ELF(ELFFile):
             if not isinstance(sec, SymbolTableSection):
                 continue
 
-            for sym in sec.iter_symbols():
+            for sym in _iter_symbols(sec):
                 # Avoid duplicates
                 if sym.name in self.functions:
                     continue
@@ -863,7 +886,7 @@ class ELF(ELFFile):
             if not isinstance(section, SymbolTableSection):
                 continue
 
-            for symbol in section.iter_symbols():
+            for symbol in _iter_symbols(section):
                 value = symbol.entry.st_value
                 if not value:
                     continue
@@ -904,7 +927,7 @@ class ELF(ELFFile):
         if self.statically_linked:
             return
 
-        for section in self.iter_sections():
+        for section in self.sections:
             # We are only interested in relocations
             if not isinstance(section, RelocationSection):
                 continue
@@ -958,7 +981,7 @@ class ELF(ELFFile):
 
         # Iterate over the dynamic symbol table
         dynsym = self.get_section_by_name('.dynsym')
-        symbol_iter = dynsym.iter_symbols()
+        symbol_iter = _iter_symbols(dynsym)
 
         # 'gotsym' is the index of the first GOT symbol
         gotsym = self.dynamic_value_by_tag('DT_MIPS_GOTSYM')
@@ -1000,9 +1023,10 @@ class ELF(ELFFile):
         #              In particular, this is where EBX points when it points into the GOT.
         dt_pltgot = self.dynamic_value_by_tag('DT_PLTGOT') or 0
 
-        # There are two PLTs we may need to search
+        # There are three PLTs we may need to search
         plt = self.get_section_by_name('.plt')          # <-- Functions only
         plt_got = self.get_section_by_name('.plt.got')  # <-- Functions used as data
+        plt_sec = self.get_section_by_name('.plt.sec')
         plt_mips = self.get_section_by_name('.MIPS.stubs')
 
         # Invert the GOT symbols we already have, so we can look up by address
@@ -1010,7 +1034,7 @@ class ELF(ELFFile):
         inv_symbols.update({v:k for k,v in self.symbols.items()})
 
         with context.local(arch=self.arch, bits=self.bits, endian=self.endian):
-            for section in (plt, plt_got, plt_mips):
+            for section in (plt, plt_got, plt_sec, plt_mips):
                 if not section:
                     continue
 
@@ -1020,7 +1044,7 @@ class ELF(ELFFile):
                                                 section.data(),
                                                 inv_symbols)
 
-                for address, target in reversed(sorted(res.items())):
+                for address, target in sorted(res.items()):
                     self.plt[inv_symbols[target]] = address
 
         for a,n in sorted({v:k for k,v in self.plt.items()}.items()):
@@ -1031,7 +1055,10 @@ class ELF(ELFFile):
             return
 
         banner = self.string(self.symbols.linux_banner)
-
+        
+        # convert banner into a utf-8 string since re.search does not accept bytes anymore
+        banner = banner.decode('utf-8')
+        
         # 'Linux version 3.18.31-gd0846ecc
         regex = r'Linux version (\S+)'
         match = re.search(regex, banner)
@@ -1046,8 +1073,51 @@ class ELF(ELFFile):
 
         self.config['version'] = self.version
 
-    def search(self, needle, writable = False):
-        """search(needle, writable = False) -> generator
+    @property
+    def libc_start_main_return(self):
+        """
+            Try to find the return address from main into __libc_start_main.
+            The heuristic to find the call to the function pointer of main is
+            to list all calls inside __libc_start_main, find the call to exit
+            after the call to main and select the previous call.
+        """
+        if '__libc_start_main' not in self.functions:
+            return 0
+
+        if 'exit' not in self.symbols:
+            return 0
+
+        # If there's no delay slot, execution continues on the next instruction after a call.
+        call_return_offset = 1
+        if self.arch in ['arm', 'thumb']:
+            call_instructions = set(['blx', 'bl'])
+        elif self.arch == 'aarch64':
+            call_instructions = set(['blr', 'bl'])
+        elif self.arch in ['mips', 'mips64']:
+            call_instructions = set(['bal', 'jalr'])
+            # Account for the delay slot.
+            call_return_offset = 2
+        elif self.arch in ['i386', 'amd64', 'ia64']:
+            call_instructions = set(['call'])
+        else:
+            log.error('Unsupported architecture %s in ELF.libc_start_main_return', self.arch)
+            return 0
+        
+        code = self.disasm(self.symbols['__libc_start_main'], self.functions['__libc_start_main'].size)
+        exit_addr = hex(self.symbols['exit'])
+        lines = code.split('\n')
+        calls = [(index, line) for index, line in enumerate(lines) if set(line.split()) & call_instructions]
+        exit_calls = [index for index, line in enumerate(calls) if exit_addr in line[1]]
+        if len(exit_calls) != 1:
+            return 0
+
+        call_to_main = calls[exit_calls[0] - 1]
+        return_from_main = lines[call_to_main[0] + call_return_offset].lstrip()
+        return_from_main = int(return_from_main[ : return_from_main.index(':') ], 16)
+        return return_from_main
+
+    def search(self, needle, writable = False, executable = False):
+        """search(needle, writable = False, executable = False) -> generator
 
         Search the ELF's virtual address space for the specified string.
 
@@ -1060,6 +1130,7 @@ class ELF(ELFFile):
         Arguments:
             needle(str): String to search for.
             writable(bool): Search only writable sections.
+            executable(bool): Search only executable sections.
 
         Yields:
             An iterator for each virtual address that matches.
@@ -1077,11 +1148,20 @@ class ELF(ELFFile):
 
             >>> len(list(bash.search(b'GNU bash'))) > 0
             True
+
+            It is also possible to search for instructions in executable sections.
+
+            >>> binary = ELF.from_assembly('nop; mov eax, 0; jmp esp; ret')
+            >>> jmp_addr = next(binary.search(asm('jmp esp'), executable = True))
+            >>> binary.read(jmp_addr, 2) == asm('jmp esp')
+            True
         """
         load_address_fixup = (self.address - self.load_addr)
 
         if writable:
             segments = self.writable_segments
+        elif executable:
+            segments = self.executable_segments
         else:
             segments = self.segments
 
@@ -1153,6 +1233,11 @@ class ELF(ELFFile):
             # DT_LOAD segment is **last** to load data into the region.
             self.memory.chop(start, stop_data)
 
+            # Fill the start of the segment's first page
+            page_start = align_down(0x1000, start)
+            if page_start < start and not self.memory[page_start]:
+                self.memory.addi(page_start, start, None)
+
             # Add the new segment
             if start != stop_data:
                 self.memory.addi(start, stop_data, segment)
@@ -1160,15 +1245,22 @@ class ELF(ELFFile):
             if stop_data != stop_mem:
                 self.memory.addi(stop_data, stop_mem, b'\x00')
 
+            page_end = align(0x1000, stop_mem)
+
             # Check for holes which we can fill
             if self._fill_gaps and i+1 < len(load_segments):
                 next_start = load_segments[i+1].header.p_vaddr
-                
-                if stop_mem < next_start and stop_mem>>(self.bits-1) == next_start>>(self.bits-1):
-                    self.memory.addi(stop_mem, next_start, None)
-            else:
-                page_end = (stop_mem + 0xfff) & ~(0xfff)
+                page_next = align_down(0x1000, next_start)
 
+                if stop_mem < next_start:
+                    if page_end < page_next:
+                        if stop_mem < page_end:
+                            self.memory.addi(stop_mem, page_end, None)
+                        if page_next < next_start:
+                            self.memory.addi(page_next, next_start, None)
+                    else:
+                        self.memory.addi(stop_mem, next_start, None)
+            else:
                 if stop_mem < page_end:
                     self.memory.addi(stop_mem, page_end, None)
 
@@ -1600,7 +1692,7 @@ class ELF(ELFFile):
             ``PT_GNU_STACK``
 
             The p_flags member specifies the permissions on the segment
-            containing the stack and is used to indicate wether the stack
+            containing the stack and is used to indicate whether the stack
             should be executable. The absense of this header indicates
             that the stack will be executable.
 
@@ -1764,7 +1856,7 @@ class ELF(ELFFile):
         # Check for Linux configuration, it must contain more than
         # just the version.
         if len(self.config) > 1:
-            config_opts = collections.defaultdict(lambda: [])
+            config_opts = collections.defaultdict(list)
             for checker in kernel_configuration:
                 result, message = checker(self.config)
 
@@ -1871,7 +1963,7 @@ class ELF(ELFFile):
     def unpack(self, address, *a, **kw):
         """Unpacks an integer from the specified ``address``."""
         self._update_args(kw)
-        return packing.unpack(self.read(address, context.bytes), *a, **kw)
+        return packing.unpack(self.read(address, self.bytes), *a, **kw)
 
     def string(self, address):
         """string(address) -> str

pwnlib/elf/plt.py

@@ -36,13 +36,17 @@ def emulate_plt_instructions(elf, got, address, data, targets):
         data = packing.unpack_many(data, bits=32, endian='little')
         data = packing.flat(data, bits=32, endian='big')
 
+    uc, ctx = prepare_unicorn_and_context(elf, got, address, data)
+
     # Brute force addresses, assume that PLT entry points are at 4-byte aligned
     # Do not emulate more than a handful of instructions.
     for i, pc in enumerate(range(address, address + len(data), 4)):
         if log.isEnabledFor(logging.DEBUG):
             log.debug('%s %#x', fiddling.enhex(data[i*4:(i+1) * 4]), pc)
             log.debug(elf.disasm(pc, 4))
-        target = emulate_plt_instructions_inner(elf, got, pc, data[i*4:], targets)
+
+        uc.context_restore(ctx)
+        target = emulate_plt_instructions_inner(uc, elf, got, pc, data[i*4:], targets)
 
         if target in targets:
             log.debug("%#x -> %#x", pc, target)
@@ -50,8 +54,7 @@ def emulate_plt_instructions(elf, got, address, data, targets):
 
     return rv
 
-def emulate_plt_instructions_inner(elf, got, pc, data, targets):
-    # Deferred import to not affect load time
+def prepare_unicorn_and_context(elf, got, address, data):
     import unicorn as U
 
     # Instantiate the emulator with the correct arguments for the current
@@ -88,15 +91,15 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
     uc = U.Uc(arch, mode)
 
     # Map the page of memory, and fill it with the contents
-    start = pc & (~0xfff)
-    stop  = (pc + len(data) + 0xfff) & (~0xfff)
+    start = address & (~0xfff)
+    stop  = (address + len(data) + 0xfff) & (~0xfff)
 
     if not (0 <= start <= stop <= (1 << elf.bits)):
         return None
 
     uc.mem_map(start, stop-start)
-    uc.mem_write(pc, data)
-    assert uc.mem_read(pc, len(data)) == data
+    uc.mem_write(address, data)
+    assert uc.mem_read(address, len(data)) == data
 
     # MIPS is unique in that it relies entirely on _DYNAMIC, at the beginning
     # of the GOT.  Each PLT stub loads an address stored here.
@@ -129,9 +132,17 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
         trap = packing.p32(0x34000000, endian=elf.endian)
         uc.mem_write(magic_addr, trap)
 
+    return uc, uc.context_save()
+
+def emulate_plt_instructions_inner(uc, elf, got, pc, data, targets):
+    import unicorn as U
+
     # Hook invalid addresses and any accesses out of the specified address range
     stopped_addr = []
 
+    # For MIPS. Explanation at prepare_unicorn_and_context.
+    magic_addr = 0xdbdbdbdb
+
     def hook_mem(uc, access, address, size, value, user_data):
         # Special case to allow MIPS to dereference the _DYNAMIC pointer
         # in the GOT.
@@ -142,8 +153,10 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
         uc.emu_stop()
         return False
 
-    uc.hook_add(U.UC_HOOK_MEM_READ, hook_mem, stopped_addr)
-    uc.hook_add(U.UC_HOOK_MEM_UNMAPPED, hook_mem, stopped_addr)
+    hooks = [
+        uc.hook_add(U.UC_HOOK_MEM_READ, hook_mem, stopped_addr),
+        uc.hook_add(U.UC_HOOK_MEM_UNMAPPED, hook_mem, stopped_addr)
+    ]
 
     # callback for tracing instructions
     # def hook_code(uc, address, size, user_data):
@@ -178,4 +191,7 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
     if stopped_addr:
         retval = stopped_addr.pop()
 
+    for hook in hooks:
+        uc.hook_del(hook)
+
     return retval

pwnlib/encoders/amd64/delta.py

@@ -5,19 +5,20 @@ from pwnlib.encoders.i386.delta import i386DeltaEncoder
 
 
 class amd64DeltaEncoder(i386DeltaEncoder):
-    """
+    r"""
     amd64 encoder built on delta-encoding.
 
     In addition to the loader stub, doubles the size of the shellcode.
 
     >>> context.clear(arch='amd64')
     >>> shellcode = asm(shellcraft.sh())
-    >>> avoid = '/bin/sh\x00'
+    >>> avoid = b'/bin/sh\x00'
     >>> encoded = pwnlib.encoders.amd64.delta.encode(shellcode, avoid)
     >>> assert not any(c in encoded for c in avoid)
     >>> p = run_shellcode(encoded)
     >>> p.sendline(b'echo hello; exit')
     >>> p.recvline()
+    b'hello\n'
     """
     assembly = '''
 base:
@@ -39,8 +40,8 @@ next:
 data:
 '''
     arch      = 'amd64'
-    raw       = 'H\x8d5\xf9\xff\xff\xffH\x83\xc6\x1a\xfcH\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'
+    raw       = b'H\x8d5\xf9\xff\xff\xffH\x83\xc6\x1a\xfcH\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'
     blacklist = set(raw)
 
 encode = amd64DeltaEncoder()
-__all__ = [encode]
+__all__ = ['encode']

pwnlib/encoders/arm/alphanumeric/__init__.py

@@ -32,7 +32,7 @@ class ArmEncoder(Encoder):
             enc_dec_loop = b.encDecoderLoopBuilder(dec_loop)
             dec = b.DecoderBuilder(dec_loop, self.icache_flush)
 
-            output,dec = b.buildInit(dec);
+            output, dec = b.buildInit(dec)
 
             output += dec
             output += enc_dec_loop

pwnlib/encoders/arm/alphanumeric/alphanum_byte.py

@@ -42,10 +42,10 @@ def off_gen(c):
             return x
    return 0
 
-# return an alphanumeric value ret such that c XOR ret is also alphanumeric 
-# ========================================================================= 
+# return an alphanumeric value ret such that c XOR ret is also alphanumeric
+# =========================================================================
 def alphanumeric_get_complement(c):
-   c &= 0xff;
+   c &= 0xff
    while True:
       ret = alphanumeric_get_byte()
       if alphanumeric_check(c ^ ret):

pwnlib/encoders/arm/xor.py

@@ -15,7 +15,7 @@ class ArmXorEncoder(Encoder):
 
     >>> context.clear(arch='arm')
     >>> shellcode = asm(shellcraft.sh())
-    >>> avoid = 'binsh\x00\n'
+    >>> avoid = b'binsh\x00\n'
     >>> encoded = pwnlib.encoders.arm.xor.encode(shellcode, avoid)
     >>> assert not any(c in encoded for c in avoid)
     >>> p = run_shellcode(encoded)

pwnlib/encoders/i386/delta.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 from __future__ import division
 
+import six
 import collections
 from random import choice
 from random import randint
@@ -41,7 +42,7 @@ class i386DeltaEncoder(Encoder):
 
     Example:
 
-        >>> sc = pwnlib.encoders.i386.delta.encode('\xcc', '\x00\xcc')
+        >>> sc = pwnlib.encoders.i386.delta.encode(b'\xcc', b'\x00\xcc')
         >>> e  = ELF.from_bytes(sc)
         >>> e.process().poll(True)
         -5
@@ -50,36 +51,35 @@ class i386DeltaEncoder(Encoder):
     arch       = 'i386'
     stub       = None
     terminator = 0xac
-    raw        = '\xd9\xd0\xfc\xd9t$\xf4^\x83\xc6\x18\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'
+    raw        = b'\xd9\xd0\xfc\xd9t$\xf4^\x83\xc6\x18\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'
 
     blacklist  = set(raw)
 
-    def __call__(self, bytes, avoid, pcreg=''):
+    def __call__(self, raw_bytes, avoid, pcreg=''):
         table = collections.defaultdict(lambda: [])
-        endchar = ''
+        endchar = bytearray()
 
-        not_bad = lambda x: chr(x) not in avoid
+        not_bad = lambda x: six.int2byte(x) not in avoid
         not_bad_or_term = lambda x: not_bad(x) and x != self.terminator
 
         for i in filter(not_bad_or_term, range(0, 256)):
-            endchar += chr(i)
+            endchar.append(i)
             for j in filter(not_bad, range(0, 256)):
-                table[(j - i) & 0xff].append(chr(i) + chr(j))
+                table[(j - i) & 0xff].append(bytearray([i, j]))
 
-        res = self.raw
+        res = bytearray(self.raw)
 
-        for c in bytes:
-            a = ord(c)
-            l = len(table[a])
+        for c in bytearray(raw_bytes):
+            l = len(table[c])
             if l == 0:
-                print('No encodings for character %02x' % a)
+                print('No encodings for character %02x' % c)
                 return None
 
-            res += table[a][randint(0, l - 1)]
+            res += table[c][randint(0, l - 1)]
 
-        res += chr(self.terminator)
-        res += choice(endchar)
+        res.append(self.terminator)
+        res.append(choice(endchar))
 
-        return res
+        return bytes(res)
 
 encode = i386DeltaEncoder()

pwnlib/encoders/mips/xor.py

@@ -26,6 +26,7 @@
 from __future__ import absolute_import
 from __future__ import division
 
+import six
 from pwnlib import asm
 from pwnlib import shellcraft
 from pwnlib.context import context
@@ -33,67 +34,67 @@ from pwnlib.encoders.encoder import Encoder
 from pwnlib.util.fiddling import xor_key
 
 decoders = {
-    'little': ''.join([
-    "SIZ2SIZ1\x0e\x24",    # li t6,-5
-    "\x27\x70\xc0\x01",    # nor    t6,t6,zero
-    "\xa3\xff\x0b\x24",    # li t3,-93
-    "\x26\x40\xce\x01",    # xor    t0,t6,t6
-    "\xff\xff\x08\x21",    # addi   t0,t0,-1
-    "\xff\xff\x10\x05",    # bltzal t0,14 <next>
-    "\x82\x82\x08\x28",    # slti   t0,zero,-32126
-    "\xe2\xff\xfd\x23",    # addi   sp,ra,-30
-    "\x27\x58\x60\x01",    # nor    t3,t3,zero
-    "\x21\xc8\xeb\x03",    # addu   t9,ra,t3
-    "\x82\x82\x17\x28",    # slti   s7,zero,-32126
-    "\xfc\xff\x31\x8f",    # lw s1,-4(t9)
-    "\xfb\xff\x0c\x24",    # li t4,-5
-    "\x27\x60\x80\x01",    # nor    t4,t4,zero
-    "\xfd\xff\x8f\x21",    # addi   t7,t4,-3
-    "\xfc\xff\x28\x8f",    # lw t0,-4(t9)
-    "\x21\xb8\xef\x02",    # addu   s7,s7,t7
-    "\x26\x18\x11\x01",    # xor    v1,t0,s1
-    "\x2b\xf0\xee\x02",    # sltu   s8,s7,t6
-    "\xfc\xff\x23\xaf",    # sw v1,-4(t9)
-    "\xfa\xff\x1e\x14",    # bne    zero,s8,3c <loop>
-    "\x21\xc8\x2c\x03",    # addu   t9,t9,t4
-    "\xfd\xff\x86\x21",    # addi   a2,t4,-3
-    "\xf8\xff\xa6\xaf",    # sw a2,-8(sp)
-    "\x26\x28\xce\x01",    # xor    a1,t6,t6
-    "\xfc\xff\xa5\xaf",    # sw a1,-4(sp)
-    "\xf8\xff\xa4\x27",    # addiu  a0,sp,-8
-    "\x46\x10\x02\x24",    # li v0,4166
-    "\x0c\x54\x4a\x01"     # syscall   0x52950
+    'little': b''.join([
+    b'SIZ2SIZ1\x0e\x24',    # li t6,-5
+    b'\x27\x70\xc0\x01',    # nor    t6,t6,zero
+    b'\xa3\xff\x0b\x24',    # li t3,-93
+    b'\x26\x40\xce\x01',    # xor    t0,t6,t6
+    b'\xff\xff\x08\x21',    # addi   t0,t0,-1
+    b'\xff\xff\x10\x05',    # bltzal t0,14 <next>
+    b'\x82\x82\x08\x28',    # slti   t0,zero,-32126
+    b'\xe2\xff\xfd\x23',    # addi   sp,ra,-30
+    b'\x27\x58\x60\x01',    # nor    t3,t3,zero
+    b'\x21\xc8\xeb\x03',    # addu   t9,ra,t3
+    b'\x82\x82\x17\x28',    # slti   s7,zero,-32126
+    b'\xfc\xff\x31\x8f',    # lw s1,-4(t9)
+    b'\xfb\xff\x0c\x24',    # li t4,-5
+    b'\x27\x60\x80\x01',    # nor    t4,t4,zero
+    b'\xfd\xff\x8f\x21',    # addi   t7,t4,-3
+    b'\xfc\xff\x28\x8f',    # lw t0,-4(t9)
+    b'\x21\xb8\xef\x02',    # addu   s7,s7,t7
+    b'\x26\x18\x11\x01',    # xor    v1,t0,s1
+    b'\x2b\xf0\xee\x02',    # sltu   s8,s7,t6
+    b'\xfc\xff\x23\xaf',    # sw v1,-4(t9)
+    b'\xfa\xff\x1e\x14',    # bne    zero,s8,3c <loop>
+    b'\x21\xc8\x2c\x03',    # addu   t9,t9,t4
+    b'\xfd\xff\x86\x21',    # addi   a2,t4,-3
+    b'\xf8\xff\xa6\xaf',    # sw a2,-8(sp)
+    b'\x26\x28\xce\x01',    # xor    a1,t6,t6
+    b'\xfc\xff\xa5\xaf',    # sw a1,-4(sp)
+    b'\xf8\xff\xa4\x27',    # addiu  a0,sp,-8
+    b'\x46\x10\x02\x24',    # li v0,4166
+    b'\x0c\x54\x4a\x01'     # syscall   0x52950
     ]),
-    'big': ''.join([
-    "\x24\x0eSIZ1SIZ2",    # li t6,-5
-    "\x01\xc0\x70\x27",    # nor    t6,t6,zero
-    "\x24\x0b\xff\xa3",    # li t3,-93
-    "\x01\xce\x40\x26",    # xor    t0,t6,t6
-    "\x21\x08\xff\xff",    # addi   t0,t0,-1
-    "\x05\x10\xff\xff",    # bltzal t0,14 <next>
-    "\x28\x08\x82\x82",    # slti   t0,zero,-32126
-    "\x23\xfd\xff\xe2",    # addi   sp,ra,-30
-    "\x01\x60\x58\x27",    # nor    t3,t3,zero
-    "\x03\xeb\xc8\x21",    # addu   t9,ra,t3
-    "\x28\x17\x82\x82",    # slti   s7,zero,-32126
-    "\x8f\x31\xff\xfc",    # lw s1,-4(t9)
-    "\x24\x0c\xff\xfb",    # li t4,-5
-    "\x01\x80\x60\x27",    # nor    t4,t4,zero
-    "\x21\x8f\xff\xfd",    # addi   t7,t4,-3
-    "\x8f\x28\xff\xfc",    # lw t0,-4(t9)
-    "\x02\xef\xb8\x21",    # addu   s7,s7,t7
-    "\x01\x11\x18\x26",    # xor    v1,t0,s1
-    "\x02\xee\xf0\x2b",    # sltu   s8,s7,t6
-    "\xaf\x23\xff\xfc",    # sw v1,-4(t9)
-    "\x14\x1e\xff\xfa",    # bne    zero,s8,3c <loop>
-    "\x03\x2c\xc8\x21",    # addu   t9,t9,t4
-    "\x21\x86\xff\xfd",    # addi   a2,t4,-3
-    "\xaf\xa6\xff\xf8",    # sw a2,-8(sp)
-    "\x01\xce\x28\x26",    # xor    a1,t6,t6
-    "\xaf\xa5\xff\xfc",    # sw a1,-4(sp)
-    "\x27\xa4\xff\xf8",    # addiu  a0,sp,-8
-    "\x24\x02\x10\x46",    # li v0,4166
-    "\x01\x4a\x54\x0c"    # syscall 0x52950
+    'big': b''.join([
+    b'\x24\x0eSIZ1SIZ2',    # li t6,-5
+    b'\x01\xc0\x70\x27',    # nor    t6,t6,zero
+    b'\x24\x0b\xff\xa3',    # li t3,-93
+    b'\x01\xce\x40\x26',    # xor    t0,t6,t6
+    b'\x21\x08\xff\xff',    # addi   t0,t0,-1
+    b'\x05\x10\xff\xff',    # bltzal t0,14 <next>
+    b'\x28\x08\x82\x82',    # slti   t0,zero,-32126
+    b'\x23\xfd\xff\xe2',    # addi   sp,ra,-30
+    b'\x01\x60\x58\x27',    # nor    t3,t3,zero
+    b'\x03\xeb\xc8\x21',    # addu   t9,ra,t3
+    b'\x28\x17\x82\x82',    # slti   s7,zero,-32126
+    b'\x8f\x31\xff\xfc',    # lw s1,-4(t9)
+    b'\x24\x0c\xff\xfb',    # li t4,-5
+    b'\x01\x80\x60\x27',    # nor    t4,t4,zero
+    b'\x21\x8f\xff\xfd',    # addi   t7,t4,-3
+    b'\x8f\x28\xff\xfc',    # lw t0,-4(t9)
+    b'\x02\xef\xb8\x21',    # addu   s7,s7,t7
+    b'\x01\x11\x18\x26',    # xor    v1,t0,s1
+    b'\x02\xee\xf0\x2b',    # sltu   s8,s7,t6
+    b'\xaf\x23\xff\xfc',    # sw v1,-4(t9)
+    b'\x14\x1e\xff\xfa',    # bne    zero,s8,3c <loop>
+    b'\x03\x2c\xc8\x21',    # addu   t9,t9,t4
+    b'\x21\x86\xff\xfd',    # addi   a2,t4,-3
+    b'\xaf\xa6\xff\xf8',    # sw a2,-8(sp)
+    b'\x01\xce\x28\x26',    # xor    a1,t6,t6
+    b'\xaf\xa5\xff\xfc',    # sw a1,-4(sp)
+    b'\x27\xa4\xff\xf8',    # addiu  a0,sp,-8
+    b'\x24\x02\x10\x46',    # li v0,4166
+    b'\x01\x4a\x54\x0c'    # syscall 0x52950
     ])
 }
 
@@ -104,7 +105,7 @@ class MipsXorEncoder(Encoder):
 
     >>> context.clear(arch='mips')
     >>> shellcode = asm(shellcraft.sh())
-    >>> avoid = '/bin/sh\x00'
+    >>> avoid = b'/bin/sh\x00'
     >>> encoded = pwnlib.encoders.mips.xor.encode(shellcode, avoid)
     >>> assert not any(c in encoded for c in avoid)
     >>> p = run_shellcode(encoded)
@@ -114,7 +115,7 @@ class MipsXorEncoder(Encoder):
     """
 
     arch = 'mips'
-    blacklist = cannot_avoid = set(''.join(v for v in decoders.values()))
+    blacklist = cannot_avoid = set(b''.join(v for v in decoders.values()))
 
     def __call__(self, raw_bytes, avoid, pcreg=''):
 
@@ -127,9 +128,9 @@ class MipsXorEncoder(Encoder):
         sizelo = size & 0xff
         sizehi = size >> 8
 
-        decoder = str(decoders[context.endian])
-        decoder = decoder.replace('SIZ1', chr(sizehi))
-        decoder = decoder.replace('SIZ2', chr(sizelo))
+        decoder = decoders[context.endian]
+        decoder = decoder.replace(b'SIZ1', six.int2byte(sizehi))
+        decoder = decoder.replace(b'SIZ2', six.int2byte(sizelo))
 
         key, data = xor_key(raw_bytes, avoid=avoid)
 

pwnlib/filepointer.py

@@ -0,0 +1,345 @@
+# -*- coding: utf-8 -*-
+
+r"""
+File Structure Exploitation
+
+struct FILE (_IO_FILE) is the structure for File Streams.
+This offers various targets for exploitation on an existing bug in the code.
+Examples - ``_IO_buf_base`` and ``_IO_buf_end`` for reading data to arbitrary location.
+
+Remembering the offsets of various structure members while faking a FILE structure can be difficult,
+so this python class helps you with that. Example-
+
+>>> context.clear(arch='amd64')
+>>> fileStr = FileStructure(null=0xdeadbeef)
+>>> fileStr.vtable = 0xcafebabe
+>>> payload = bytes(fileStr)
+>>> payload
+b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xbe\xad\xde\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xbe\xad\xde\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00'
+
+Now payload contains the FILE structure with its vtable pointer pointing to 0xcafebabe
+
+Currently only 'amd64' and 'i386' architectures are supported
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+
+from pwnlib.context import context
+from pwnlib.log import getLogger
+from pwnlib.util.misc import python_2_bytes_compatible
+from pwnlib.util.packing import pack
+
+log = getLogger(__name__)
+
+length=0
+size='size'
+name='name'
+
+variables={
+    0:{name:'flags',size:length},
+    1:{name:'_IO_read_ptr',size:length},
+    2:{name:'_IO_read_end',size:length},
+    3:{name:'_IO_read_base',size:length},
+    4:{name:'_IO_write_base',size:length},
+    5:{name:'_IO_write_ptr',size:length},
+    6:{name:'_IO_write_end',size:length},
+    7:{name:'_IO_buf_base',size:length},
+    8:{name:'_IO_buf_end',size:length},
+    9:{name:'_IO_save_base',size:length},
+    10:{name:'_IO_backup_base',size:length},
+    11:{name:'_IO_save_end',size:length},
+    12:{name:'markers',size:length},
+    13:{name:'chain',size:length},
+    14:{name:'fileno',size:4},
+    15:{name:'_flags2',size:4},
+    16:{name:'_old_offset',size:length},
+    17:{name:'_cur_column',size:2},
+    18:{name:'_vtable_offset',size:1},
+    19:{name:'_shortbuf',size:1},
+    20:{name:'unknown1',size:-4},
+    21:{name:'_lock',size:length},
+    22:{name:'_offset',size:8},
+    23:{name:'_codecvt',size:length},
+    24:{name:'_wide_data',size:length},
+    25:{name:'unknown2',size:length},
+    26:{name:'vtable',size:length}
+}
+
+del name, size, length
+
+
+def update_var(l):
+    r"""
+    Since different members of the file structure have different sizes, we need to keep track of the sizes. The following function is used by the FileStructure class to initialise the lengths of the various fields.
+
+    Arguments:
+        l(int)
+            l=8 for 'amd64' architecture and l=4 for 'i386' architecture
+
+    Return Value:
+        Returns a dictionary in which each field is mapped to its corresponding length according to the architecture set
+
+    Examples:
+
+        >>> update_var(8)
+        {'flags': 8, '_IO_read_ptr': 8, '_IO_read_end': 8, '_IO_read_base': 8, '_IO_write_base': 8, '_IO_write_ptr': 8, '_IO_write_end': 8, '_IO_buf_base': 8, '_IO_buf_end': 8, '_IO_save_base': 8, '_IO_backup_base': 8, '_IO_save_end': 8, 'markers': 8, 'chain': 8, 'fileno': 4, '_flags2': 4, '_old_offset': 8, '_cur_column': 2, '_vtable_offset': 1, '_shortbuf': 1, 'unknown1': 4, '_lock': 8, '_offset': 8, '_codecvt': 8, '_wide_data': 8, 'unknown2': 48, 'vtable': 8}
+    """
+    var={}
+    for i in variables:
+        var[variables[i]['name']]=variables[i]['size']
+    for i in var:
+        if var[i]<=0:
+            var[i]+=l
+    if l==4:
+        var['unknown2']=56
+    else:
+        var['unknown2']=48
+    return var
+
+
+@python_2_bytes_compatible
+class FileStructure(object):
+    r"""
+    Crafts a FILE structure, with default values for some fields, like _lock which should point to null ideally, set.
+
+    Arguments:
+        null(int)
+            A pointer to NULL value in memory. This pointer can lie in any segment (stack, heap, bss, libc etc)
+
+    Examples:
+
+        FILE structure with flags as 0xfbad1807 and _IO_buf_base and _IO_buf_end pointing to 0xcafebabe and 0xfacef00d
+
+        >>> context.clear(arch='amd64')
+        >>> fileStr = FileStructure(null=0xdeadbeeef)
+        >>> fileStr.flags = 0xfbad1807
+        >>> fileStr._IO_buf_base = 0xcafebabe
+        >>> fileStr._IO_buf_end = 0xfacef00d
+        >>> payload = bytes(fileStr)
+        >>> payload
+        b'\x07\x18\xad\xfb\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00\r\xf0\xce\xfa\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xee\xdb\xea\r\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xee\xdb\xea\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+
+        Check the length of the FileStructure
+
+        >>> len(fileStr)
+        224
+
+        The defination for __repr__ orders the structure members and displays then in a dictionary format. It's useful when viewing a structure objet in python/IPython shell
+
+        >>> q=FileStructure(0xdeadbeef)
+        >>> q
+        { flags: 0x0
+         _IO_read_ptr: 0x0
+         _IO_read_end: 0x0
+         _IO_read_base: 0x0
+         _IO_write_base: 0x0
+         _IO_write_ptr: 0x0
+         _IO_write_end: 0x0
+         _IO_buf_base: 0x0
+         _IO_buf_end: 0x0
+         _IO_save_base: 0x0
+         _IO_backup_base: 0x0
+         _IO_save_end: 0x0
+         markers: 0x0
+         chain: 0x0
+         fileno: 0x0
+         _flags2: 0x0
+         _old_offset: 0xffffffffffffffff
+         _cur_column: 0x0
+         _vtable_offset: 0x0
+         _shortbuf: 0x0
+         unknown1: 0x0
+         _lock: 0xdeadbeef
+         _offset: 0xffffffffffffffff
+         _codecvt: 0x0
+         _wide_data: 0xdeadbeef
+         unknown2: 0x0
+         vtable: 0x0}
+    """
+
+    vars_=[]
+    length={}
+
+    def __init__(self, null=0):
+            self.vars_ = [variables[i]['name'] for i in sorted(variables.keys())]
+            self.setdefault(null)
+            self.length = update_var(context.bytes)
+            self._old_offset = (1 << context.bits) - 1
+
+    def __setattr__(self,item,value):
+        if item in FileStructure.__dict__ or item in self.vars_:
+            object.__setattr__(self,item,value)
+        else:
+            log.error("Unknown variable %r" % item)
+
+    def __repr__(self):
+        structure=[]
+        for i in self.vars_:
+            structure.append(" %s: %s" % (i,hex(self.__getattr__(i))))
+        return "{"+ "\n".join(structure)+"}"
+
+    def __getattr__(self,item):
+        if item in FileStructure.__dict__ or item in self.vars_:
+            return object.__getattribute__(self,item)
+        log.error("Unknown variable %r" % item)
+
+    def __len__(self):
+        return len(bytes(self))
+
+    def __bytes__(self):
+        structure = b''
+        for val in self.vars_:
+            if isinstance(self.__getattr__(val), bytes):
+                structure += self.__getattr__(val).ljust(context.bytes, b'\x00')
+            else:
+                structure += pack(self.__getattr__(val), self.length[val]*8)
+        return structure
+
+    def struntil(self,v):
+        r"""
+        Payload for stuff till 'v' where 'v' is a structure member. This payload includes 'v' as well.
+
+        Arguments:
+            v(string)
+                The name of the field uptil which the payload should be created.
+
+        Example:
+
+            Payload for data uptil _IO_buf_end
+
+            >>> context.clear(arch='amd64')
+            >>> fileStr = FileStructure(0xdeadbeef)
+            >>> payload = fileStr.struntil("_IO_buf_end")
+            >>> payload
+            b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+        """
+        if v not in self.vars_:
+            return b''
+        structure = b''
+        for val in self.vars_:
+            if isinstance(self.__getattr__(val), bytes):
+                structure += self.__getattr__(val).ljust(context.bytes, b'\x00')
+            else:
+                structure += pack(self.__getattr__(val), self.length[val]*8)
+            if val == v:
+                break
+        return structure[:-1]
+
+    def setdefault(self,null):
+            self.flags=0
+            self._IO_read_ptr=0
+            self._IO_read_end=0
+            self._IO_read_base=0
+            self._IO_write_base=0
+            self._IO_write_ptr=0
+            self._IO_write_end=0
+            self._IO_buf_base=0
+            self._IO_buf_end=0
+            self._IO_save_base=0
+            self._IO_backup_base=0
+            self._IO_save_end=0
+            self.markers=0
+            self.chain=0
+            self.fileno=0
+            self._flags2=0
+            self._old_offset=0
+            self._cur_column=0
+            self._vtable_offset=0
+            self._shortbuf=0
+            self.unknown1=0
+            self._lock=null
+            self._offset=0xffffffffffffffff
+            self._codecvt=0
+            self._wide_data=null
+            self.unknown2=0
+            self.vtable=0
+
+    def write(self,addr=0,size=0):
+        r"""
+        Writing data out from arbitrary memory address.
+
+        Arguments:
+            addr(int)
+                The address from which data is to be printed to stdout
+            size(int)
+                The size, in bytes, of the data to be printed
+
+        Example:
+
+            Payload for writing 100 bytes to stdout from the address 0xcafebabe
+
+            >>> context.clear(arch='amd64')
+            >>> fileStr = FileStructure(0xdeadbeef)
+            >>> payload = fileStr.write(addr=0xcafebabe, size=100)
+            >>> payload
+            b'\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00"\xbb\xfe\xca\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00'
+        """
+        self.flags &=~8
+        self.flags |=0x800
+        self._IO_write_base = addr
+        self._IO_write_ptr = addr+size
+        self._IO_read_end = addr
+        self.fileno = 1
+        return self.struntil('fileno')
+
+    def read(self,addr=0,size=0):
+        r"""
+        Reading data into arbitrary memory location.
+
+        Arguments:
+            addr(int)
+                The address into which data is to be written from stdin
+            size(int)
+                The size, in bytes, of the data to be written
+
+        Example:
+
+            Payload for reading 100 bytes from stdin into the address 0xcafebabe
+
+            >>> context.clear(arch='amd64')
+            >>> fileStr = FileStructure(0xdeadbeef)
+            >>> payload = fileStr.read(addr=0xcafebabe, size=100)
+            >>> payload
+            b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00"\xbb\xfe\xca\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+        """
+        self.flags &=~4
+        self._IO_read_base = 0
+        self._IO_read_ptr = 0
+        self._IO_buf_base = addr
+        self._IO_buf_end = addr+size
+        self.fileno = 0
+        return self.struntil('fileno')
+
+    def orange(self,io_list_all,vtable):
+        r"""
+        Perform a House of Orange (https://github.com/shellphish/how2heap/blob/master/glibc_2.25/house_of_orange.c), provided you have libc leaks.
+
+        Arguments:
+            io_list_all(int)
+                Address of _IO_list_all in libc.
+            vtable(int)
+                Address of the fake vtable in memory
+
+        Example:
+
+            Example payload if address of _IO_list_all is 0xfacef00d and fake vtable is at 0xcafebabe -
+
+            >>> context.clear(arch='amd64')
+            >>> fileStr = FileStructure(0xdeadbeef)
+            >>> payload = fileStr.orange(io_list_all=0xfacef00d, vtable=0xcafebabe)
+            >>> payload
+            b'/bin/sh\x00a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfd\xef\xce\xfa\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xbe\xad\xde\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xbe\xad\xde\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00'
+        """
+        if context.bits == 64:
+            self.flags = b'/bin/sh\x00'
+            self._IO_read_ptr = 0x61
+            self._IO_read_base = io_list_all-0x10
+        elif context.bits == 32:
+            self.flags = b'sh\x00'
+            self._IO_read_ptr = 0x121
+            self._IO_read_base = io_list_all-0x8
+        self._IO_write_base = 0
+        self._IO_write_ptr = 1
+        self.vtable = vtable
+        return self.__bytes__()

pwnlib/flag/flag.py

@@ -11,7 +11,6 @@ from pwnlib.tubes.remote import remote
 
 env_server  = args.get('FLAG_HOST', 'flag-submission-server').strip()
 env_port    = args.get('FLAG_PORT', '31337').strip()
-env_proto   = args.get('FLAG_PROTO', 'tcp').strip()
 env_file    = args.get('FLAG_FILE', '/does/not/exist').strip()
 env_exploit_name = args.get('EXPLOIT_NAME', 'unnamed-exploit').strip()
 env_target_host  = args.get('TARGET_HOST', 'unknown-target').strip()
@@ -24,7 +23,6 @@ def submit_flag(flag,
                 target=env_target_host,
                 server=env_server,
                 port=env_port,
-                proto=env_proto,
                 team=env_team_name):
     """
     Submits a flag to the game server
@@ -35,7 +33,7 @@ def submit_flag(flag,
         target(str): Target identifier, optional
         server(str): Flag server host name, optional
         port(int): Flag server port, optional
-        proto(str), Flag server protocol, optional
+        team(str): Team identifier, optional
 
     Optional arguments are inferred from the environment,
     or omitted if none is set.
@@ -50,7 +48,7 @@ def submit_flag(flag,
         >>> _ = submit_flag('flag', server='localhost', port=l.lport)
         >>> c = l.wait_for_connection()
         >>> c.recvall().split()
-        ['flag', 'unnamed-exploit', 'unknown-target', 'unknown-team']
+        [b'flag', b'unnamed-exploit', b'unknown-target', b'unknown-team']
     """
     flag = flag.strip()
 

pwnlib/fmtstr.py

@@ -1,38 +1,39 @@
 r"""
 Provide some tools to exploit format string bug
 
-Examples:
+Let's use this program as an example:
 
-    >>> program = tempfile.mktemp()
-    >>> source  = program + ".c"
-    >>> write(source, '''
-    ... #include <stdio.h>
-    ... #include <stdlib.h>
-    ... #include <unistd.h>
-    ... #include <sys/mman.h>
-    ... #define MEMORY_ADDRESS ((void*)0x11111000)
-    ... #define MEMORY_SIZE 1024
-    ... #define TARGET ((int *) 0x11111110)
-    ... int main(int argc, char const *argv[])
-    ... {
-    ...        char buff[1024];
-    ...        void *ptr = NULL;
-    ...        int *my_var = TARGET;
-    ...        ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
-    ...        if(ptr != MEMORY_ADDRESS)
-    ...        {
-    ...                perror("mmap");
-    ...                return EXIT_FAILURE;
-    ...        }
-    ...        *my_var = 0x41414141;
-    ...        write(1, &my_var, sizeof(int *));
-    ...        scanf("%s", buff);
-    ...        dprintf(2, buff);
-    ...        write(1, my_var, sizeof(int));
-    ...        return 0;
-    ... }''')
-    >>> cmdline = ["gcc", source, "-Wno-format-security", "-m32", "-o", program]
-    >>> process(cmdline).wait_for_close()
+::
+
+    #include <stdio.h>
+    #include <stdlib.h>
+    #include <unistd.h>
+    #include <sys/mman.h>
+    #define MEMORY_ADDRESS ((void*)0x11111000)
+    #define MEMORY_SIZE 1024
+    #define TARGET ((int *) 0x11111110)
+    int main(int argc, char const *argv[])
+    {
+           char buff[1024];
+           void *ptr = NULL;
+           int *my_var = TARGET;
+           ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+           if(ptr != MEMORY_ADDRESS)
+           {
+                   perror("mmap");
+                   return EXIT_FAILURE;
+           }
+           *my_var = 0x41414141;
+           write(1, &my_var, sizeof(int *));
+           scanf("%s", buff);
+           dprintf(2, buff);
+           write(1, my_var, sizeof(int));
+           return 0;
+    }
+
+We can automate the exploitation of the process like so:
+
+    >>> program = pwnlib.data.elf.fmtstr.get('i386')
     >>> def exec_fmt(payload):
     ...     p = process(program)
     ...     p.sendline(payload)
@@ -245,9 +246,9 @@ class AtomWrite(object):
         Combine adjacent writes into a single write.
 
         Example:
-        >>> context.clear(endian = "little")
-        >>> pwnlib.fmtstr.AtomWrite(0x0, 0x1, 0x1, 0xff).union(pwnlib.fmtstr.AtomWrite(0x1, 0x1, 0x2, 0x77))
-        AtomWrite(start=0, size=2, integer=0x201, mask=0xff77)
+            >>> context.clear(endian = "little")
+            >>> pwnlib.fmtstr.AtomWrite(0x0, 0x1, 0x1, 0xff).union(pwnlib.fmtstr.AtomWrite(0x1, 0x1, 0x2, 0x77))
+            AtomWrite(start=0, size=2, integer=0x201, mask=0x77ff)
         """
         assert other.start == self.end, "writes to combine must be continous"
         if context.endian == "little":
@@ -777,22 +778,22 @@ def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_
 
     Examples:
         >>> context.clear(arch = 'amd64')
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')
         b'%322419390c%4$llnaaaabaa\x00\x00\x00\x00\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='short')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='short')
         b'%47806c%5$lln%22649c%6$hnaaaabaa\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')
         b'%190c%7$lln%85c%8$hhn%36c%9$hhn%131c%10$hhnaaaab\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'
         >>> context.clear(arch = 'i386')
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')
         b'%322419390c%5$na\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='short')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='short')
         b'%4919c%7$hn%42887c%8$hna\x02\x00\x00\x00\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')
         b'%19c%12$hhn%36c%13$hhn%131c%14$hhn%4c%15$hhn\x03\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x00000001}, write_size='byte')))
+        >>> fmtstr_payload(1, {0x0: 0x00000001}, write_size='byte')
         b'%1c%3$na\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: b"\xff\xff\x04\x11\x00\x00\x00\x00"}, write_size='short')))
+        >>> fmtstr_payload(1, {0x0: b"\xff\xff\x04\x11\x00\x00\x00\x00"}, write_size='short')
         b'%327679c%7$lln%18c%8$hhn\x00\x00\x00\x00\x03\x00\x00\x00'
     """
     sz = WRITE_SIZE[write_size]
@@ -833,23 +834,13 @@ class FmtStr(object):
 
     """
 
-    def __init__(self, execute_fmt, offset = None, padlen = 0, numbwritten = 0):
-        """
-        Instantiates an object which try to automating exploit the vulnerable process
-
-        Arguments:
-            execute_fmt(function): function to call for communicate with the vulnerable process
-            offset(int): the first formatter's offset you control
-            padlen(int): size of the pad you want to add before the payload
-            numbwritten(int): number of already written bytes
-        """
+    def __init__(self, execute_fmt, offset=None, padlen=0, numbwritten=0):
         self.execute_fmt = execute_fmt
         self.offset = offset
         self.padlen = padlen
         self.numbwritten = numbwritten
 
-
-        if self.offset == None:
+        if self.offset is None:
             self.offset, self.padlen = self.find_offset()
             log.info("Found format string offset: %d", self.offset)
 
@@ -857,7 +848,8 @@ class FmtStr(object):
         self.leaker = MemLeak(self._leaker)
 
     def leak_stack(self, offset, prefix=b""):
-        leak = self.execute_fmt(prefix + b"START%%%d$pEND" % offset)
+        payload = b"START%%%d$pEND" % offset
+        leak = self.execute_fmt(prefix + payload)
         try:
             leak = re.findall(br"START(.*)END", leak, re.MULTILINE | re.DOTALL)[0]
             leak = int(leak, 16)
@@ -871,7 +863,7 @@ class FmtStr(object):
             leak = self.leak_stack(off, marker)
             leak = pack(leak)
 
-            pad = cyclic_find(leak)
+            pad = cyclic_find(leak[:4])
             if pad >= 0 and pad < 20:
                 return off, pad
         else:
@@ -888,7 +880,10 @@ class FmtStr(object):
         if addr & 0xfff == 0 and self.leaker._leak(addr+1, 3, False) == b"ELF":
             return b"\x7f"
 
-        fmtstr = randoms(self.padlen).encode() + pack(addr) + b"START%%%d$sEND" % self.offset
+        fmtstr = fit({
+          self.padlen: b"START%%%d$sEND" % (self.offset + 16//context.bytes),
+          16 + self.padlen: addr
+        })
 
         leak = self.execute_fmt(fmtstr)
         leak = re.findall(br"START(.*)END", leak, re.MULTILINE | re.DOTALL)[0]
@@ -907,7 +902,7 @@ class FmtStr(object):
 
         """
         fmtstr = randoms(self.padlen).encode()
-        fmtstr += fmtstr_payload(self.offset, self.writes, numbwritten=self.padlen, write_size='byte')
+        fmtstr += fmtstr_payload(self.offset, self.writes, numbwritten=self.padlen + self.numbwritten, write_size='byte')
         self.execute_fmt(fmtstr)
         self.writes = {}