Release 4.6.0

* For tmux, always specify -F and -P

Fixes #1898

* [4.5.1] Update CHANGELOG

.coveragerc

@@ -7,5 +7,6 @@ omit =
 source =
     pwn
     pwnlib
-    ~/.pwntools-cache-2.7/
-    ~/.pwntools-cache-3.8/
+    ~/.cache/.pwntools-cache-2.7/
+    ~/.cache/.pwntools-cache-3.8/
+disable_warnings = module-not-imported

.github/ISSUE_TEMPLATE/feature_request.md

@@ -12,7 +12,7 @@ Thanks for contributing to Pwntools!   Ideas from the community help make Pwntoo
 If you've got an idea for a new feature, please provide information about:
 
 * What the feature does
-* Why the feature should exit
+* Why the feature should exist
 * What tests should be included
 
 If you think you can write the feature yourself, please submit a Pull Request and we can review your changes!

.github/workflows/android.yml

@@ -0,0 +1,60 @@
+name: Android Tests
+
+on: [push, pull_request]
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install Linux dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
+          gdb gdbserver socat \
+          qemu-user-static \
+          binutils-aarch64-linux-gnu \
+          binutils-arm-linux-gnueabihf \
+          libc6-dbg \
+          openjdk-8-jre-headless
+
+    - name: Install Android AVD
+      run: |
+        USER=travis source travis/setup_avd.sh
+        sed -i 's/skip_android = True/skip_android = False/' docs/source/conf.py
+        set | egrep '^(ANDROID|PATH)' >.android.env
+
+    - name: Install dependencies
+      run: |
+        pip install --upgrade pip
+        python setup.py egg_info
+        pip install --upgrade --editable .
+
+    - name: Sanity checks
+      run:  PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+
+    - name: Install documentation dependencies
+      run:  pip install -r docs/requirements.txt
+
+    - name: Coverage Doctests (Android Only)
+      run: |
+        source .android.env
+        PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest docs/source/adb.rst

.github/workflows/changelog.yml

@@ -4,7 +4,7 @@ env:
   GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
 
 jobs:
-  build:
+  check:
     strategy:
       matrix:
         os: [ubuntu-latest]
@@ -20,7 +20,7 @@ jobs:
         if git log --stat ${GITHUB_BASE}..HEAD | grep CHANGELOG.md; then
            echo 'Changelog updated :D'
         else
-           if git log --stat ${GITHUB_BASE}..HEAD | grep '++\|--'; then
+           if git log --stat ${GITHUB_BASE}..HEAD pwnlib pwn | grep '++\|--'; then
               echo 'Major changes detected, changelog required!'
               false
            else

.github/workflows/ci.yml

@@ -14,8 +14,10 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Git History 2
-      run: git log --oneline --graph -10
+    - name: Git History
+      run: |
+        git fetch origin
+        git log --oneline --graph -10
 
     - name: Cache for pip
       uses: actions/cache@v1
@@ -42,28 +44,7 @@ jobs:
         [ "$vsetup" = "$vpwnlib" ]
         [ "$GITHUB_TAG" = "$vsetup" ]
 
-    - name: Detect whether java is needed
-      id: java-needed
-      env:
-        GITHUB_REF: ${{ github.event.ref }}
-        GITHUB_COMMIT_RANGE: ${{ github.event.before }}...${{ github.event.after }}
-      run: |
-        if echo "$GITHUB_REF" | grep -Eq 'staging|tags'; then
-          echo "Found release or important branch ($GITHUB_REF), forcing tests."
-        elif ! git show "$GITHUB_COMMIT_RANGE" >/dev/null; then
-          echo 'Incorrect commit range, forcing android tests.'
-        elif git log --stat "$GITHUB_COMMIT_RANGE" | grep -iE 'android|\<adb\>'; then
-          echo 'Found Android-related commits, forcing tests.'
-        else
-          # clear files that cause Android doctests
-          : > docs/source/adb.rst > docs/source/protocols/adb.rst
-          exit 0
-        fi
-        echo ::set-output name=need::openjdk-8-jre-headless
-
     - name: Install Linux dependencies
-      env:
-        ANDROID_JRE: ${{ steps.java-needed.outputs.need }}
       run: |
         sudo apt-get update
         sudo apt-get install -y --no-install-recommends -o Acquire::Retries=3 \
@@ -73,16 +54,25 @@ jobs:
           binutils-aarch64-linux-gnu \
           binutils-arm-linux-gnueabihf \
           binutils-mips-linux-gnu \
+          binutils-msp430 \
           binutils-powerpc-linux-gnu \
+          binutils-s390x-linux-gnu \
+          binutils-sparc64-linux-gnu \
           gcc-multilib \
-          libc6-dbg \
-          $ANDROID_JRE
+          libc6-dbg
 
-    - name: Install android avd
-      if:  steps.java-needed.outputs.need
+    - name: Install RPyC for GDB
       run: |
-        USER=travis source travis/install.sh
-        set | egrep '^(ANDROID|PATH)' >.android.env
+        sudo apt-get install -y python3-pip
+        /usr/bin/python3 -m pip install rpyc
+
+    - name: Testing Corefiles
+      run: |
+        ulimit -a
+        ulimit -c unlimited
+        cat /proc/sys/kernel/core_pattern
+        cat /proc/sys/kernel/core_uses_pid
+        ( cd $(mktemp -d); sh -c 'kill -11 $$' || true; ls -la ./*core* /var/crash/*.crash;) || true
 
     - name: Set up SSH
       run: |
@@ -96,39 +86,37 @@ jobs:
         pip install --upgrade --editable .
 
     - name: Sanity checks
-      run:  PWNLIB_NOTERM=1 python -c 'from pwn import *; print(pwnlib.term.term_mode)'
+      run:  PWNLIB_NOTERM=1 python -bb -c 'from pwn import *; print(pwnlib.term.term_mode)'
 
     - name: Install documentation dependencies
       run:  pip install -r docs/requirements.txt
 
-    - name: Disable broken tests
-      run: |
-        rm -f docs/source/elf/corefile.rst
-
     - name: Manually install non-broken Unicorn
       run:  pip install unicorn==1.0.2rc3
 
+    - name: Disable yama ptrace_scope
+      run: |
+        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope # required by some gdb doctests
+
     - name: Coverage doctests
       run: |
-        source .android.env || :
-        echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope # required by some gdb doctests
-        PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+        PWNLIB_NOTERM=1 python -bb -m coverage run -m sphinx -b doctest docs/source docs/build/doctest
 
     - name: Coverage running examples
       run: |
         export TERM=linux
         set -x
-        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit.py
-        python travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit2.py || : # can fail randomly?
-        coverage run examples/asm.py
-        coverage run examples/asm.py
-        coverage run examples/text.py
+        python -bb travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit.py
+        python -bb travis/coverage_chdir.py examples/fmtstr examples/fmtstr/exploit2.py || : # can fail randomly?
+        python -bb -m coverage run examples/asm.py
+        python -bb -m coverage run examples/asm.py
+        python -bb -m coverage run examples/text.py
         # for f in examples/sigreturn_corefile_*.py; do coverage run "$f"; done # XXX something is wrong
 
     - name: Coverage running commandline tools
       run: |
         export TERM=linux
-        pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec coverage run -m pwnlib.commandline."$cmd" "$@" ) }
+        pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec python -bb -m coverage run -m pwnlib.commandline."$cmd" "$@" ) }
         set -x
 
         pwn cyclic 32
@@ -199,6 +187,7 @@ jobs:
         TARGET=${BRANCH%-staging}
         git branch -f "$TARGET"
         git push origin "$TARGET"
+        git push origin --delete "$BRANCH"
 
   pypi:
     runs-on: ubuntu-latest

.github/workflows/lint.yml

@@ -11,9 +11,6 @@ jobs:
     timeout-minutes: 30
     steps:
     - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
     - name: Cache for pip
       uses: actions/cache@v1
       id: cache-pip
@@ -26,8 +23,13 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Lint
+    - name: Critical lint
       run: |
         pip install flake8
-        flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=pwnlib/constants,android-?dk,.git,__pycache__
+        flake8 . --count --select=E9,F63,F7,E71 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+
+    - name: Style lint
+      run: |
+        flake8 pwnlib setup.py docs travis --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        flake8 examples --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore='F403,F405'
+        flake8 pwn --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --ignore='F401,F403,F405'

.github/workflows/pylint.yml

@@ -0,0 +1,37 @@
+name: PyLint
+on: [pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: PyLint
+      run: |
+        set -x
+        pip install pylint
+        pip install --upgrade -r requirements.txt
+        pylint --exit-zero --errors-only  pwnlib > current.txt
+        git fetch origin
+        git checkout origin/"$GITHUB_BASE_REF"
+        pylint --exit-zero --errors-only  pwnlib > base.txt
+        if diff base.txt current.txt | grep '>'; then
+          false
+        fi

.gitignore

@@ -17,3 +17,6 @@ venv
 *.core
 .coverage
 .coverage.*
+.idea
+__pycache__
+!.github

.travis.yml

@@ -19,8 +19,12 @@ addons:
     - binutils-aarch64-linux-gnu
     - binutils-arm-linux-gnueabihf
     - binutils-mips-linux-gnu
+    - binutils-msp430
     - binutils-powerpc-linux-gnu
+    - binutils-s390x-linux-gnu
+    - binutils-sparc64-linux-gnu
     - bash-static
+    - python3-pip
 cache:
     - pip
     - directories:
@@ -39,6 +43,7 @@ before_install:
   - source travis/ssh_setup.sh
   - echo $-
 install:
+  - /usr/bin/python3 -m pip install rpyc # for GDB
   - pip install --upgrade pip
   - pip install --upgrade flake8 appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
   - python setup.py egg_info      # pypa/pip#6275

CHANGELOG.md

@@ -9,9 +9,15 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.4.0](#440)    | `dev`    | Sep 1, 2020 (planned)
-| [4.3.0](#430)    | `beta`   | Aug 1, 2020 (planned)
-| [4.2.0](#420)    | `stable` | Jul 3, 2020 (planned)
+| [4.8.0](#480)    | `dev`    |
+| [4.7.0](#470)    | `beta`   |
+| [4.6.0](#460)    | `stable` | Jul 12, 2021
+| [4.5.1](#451)    |          | May 30, 2021
+| [4.5.0](#450)    |          | Apr 30, 2021
+| [4.4.0](#440)    |          | Mar 29, 2021
+| [4.3.1](#431)    |          | Nov 29, 2020
+| [4.3.0](#430)    |          | Oct 20, 2020
+| [4.2.0](#420)    |          | Jul 3, 2020
 | [4.1.7](#417)    |          | Jun 30, 2020
 | [4.1.5](#415)    |          | Jun 27, 2020
 | [4.1.4](#414)    |          | Jun 26, 2020
@@ -55,13 +61,152 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.4.0 (`dev`)
+## 4.8.0 (`dev`)
 
+
+
+## 4.7.0 (`beta`)
+
+- [#1733][1733] Update libc headers -> more syscalls available!
+- [#1876][1876] add `self.message` and change `sys.exc_type` to `sys.exec_info()` in PwnlibException
+- [#1877][1877] encoders error message handles when `avoid` is bytes in python3
+- [#1891][1891] Keep ROP gadgets when setting registers via setattr/call
+- [#1892][1892] Silence SIGPIPE error for "pwn phd"
+- [#1893][1893] Fix bytes warning in "pwn cyclic"
+- [#1897][1897] Add basic support for RISC-V
+- [#1903][1903] Add zsh completion script
+- [#1904][1904] Add bash completion script
+- [#1906][1906] Defer import of several modules to save on startup time
+- [#1921][1921] Add basic support for the bare-metal ARM specific toolchain
+
+[1733]: https://github.com/Gallopsled/pwntools/pull/1733
+[1876]: https://github.com/Gallopsled/pwntools/pull/1876
+[1877]: https://github.com/Gallopsled/pwntools/pull/1877
+[1891]: https://github.com/Gallopsled/pwntools/pull/1891
+[1892]: https://github.com/Gallopsled/pwntools/pull/1892
+[1893]: https://github.com/Gallopsled/pwntools/pull/1893
+[1897]: https://github.com/Gallopsled/pwntools/pull/1897
+[1903]: https://github.com/Gallopsled/pwntools/pull/1903
+[1904]: https://github.com/Gallopsled/pwntools/pull/1904
+[1906]: https://github.com/Gallopsled/pwntools/pull/1906
+[1921]: https://github.com/Gallopsled/pwntools/pull/1921
+
+## 4.6.0 (`stable`)
+
+- [#1429][1429] Add a mechanism for ret2csu (originally #1138)
+- [#1566][1566] Add `ignore_config` argument to `pwnlib.tubes.ssh` and improve `allow_agent` implementation
+- [#1652][1652] Add `process.readmem` and `process.writemem`
+- [#1739][1739] Add/fix shellcraft.linux.kill() / shellcraft.linux.killparent()
+- [#1746][1746] Prefer Python3 over Python2 for spawning remote processes over SSH
+- [#1751][1751] Fix process() with executable relative to cwd
+- [#1753][1753] major change: less unconditional imports in pwnlib
+- [#1776][1776] mips: do not use $t0 temporary variable in dupio
+- [#1846][1846] support launching GDB in more different terminals
+
+[1429]: https://github.com/Gallopsled/pwntools/pull/1429
+[1566]: https://github.com/Gallopsled/pwntools/pull/1566
+[1652]: https://github.com/Gallopsled/pwntools/pull/1652
+[1739]: https://github.com/Gallopsled/pwntools/pull/1739
+[1746]: https://github.com/Gallopsled/pwntools/pull/1746
+[1751]: https://github.com/Gallopsled/pwntools/pull/1751
+[1753]: https://github.com/Gallopsled/pwntools/pull/1753
+[1776]: https://github.com/Gallopsled/pwntools/pull/1776
+[1846]: https://github.com/Gallopsled/pwntools/pull/1846
+
+## 4.5.1
+
+- [#1902][1902] Always specify -F and -P for tmux in `run_in_new_terminal`
+
+[1902]: https://github.com/Gallopsled/pwntools/pull/1902
+
+## 4.5.0
+
+- [#1261][1261] Misc `run_in_new_terminal` improvements (notably gdb terminated by default)
+- [#1695][1695] Allow using GDB Python API
+- [#1735][1735] Python 3.9 support in safeeval
+- [#1738][1738] Which function support custom search path
+  - process also looks now at `env['PATH']` to find the path for the executable
+- [#1742][1742] New `baremetal` os to debug binaries executed with qemu-system-$(arch)
+- [#1757][1757] update cache directories
+- [#1758][1758] Remove eval from cli
+- [#1780][1780] Re-add Python2 to the official Dockerfile
+- [#1941][1941] Disable all Android tests, `pwnlib.adb` is no longer supported in CI
+- [#1811][1811] Remove unnecessary `pwn.toplevel.__all__`
+- [#1827][1827] Support `$XDG_CONFIG_HOME` dir for `pwn.conf`
+- [#1841][1841] Add colored_traceback
+- [#1839][1839] run_in_new_terminal now creates a runner script if given a list or tuple
+- [#1833][1833] Add pwnlib.filesystem module
+- [#1852][1852] Fix `atexit` on Python 3
+- [#1883][1883] ROP gadget verifier accounts for 2 character registers
+
+[1261]: https://github.com/Gallopsled/pwntools/pull/1261
+[1695]: https://github.com/Gallopsled/pwntools/pull/1695
+[1735]: https://github.com/Gallopsled/pwntools/pull/1735
+[1738]: https://github.com/Gallopsled/pwntools/pull/1738
+[1742]: https://github.com/Gallopsled/pwntools/pull/1742
+[1757]: https://github.com/Gallopsled/pwntools/pull/1757
+[1758]: https://github.com/Gallopsled/pwntools/pull/1758
+[1780]: https://github.com/Gallopsled/pwntools/pull/1780
+[1941]: https://github.com/Gallopsled/pwntools/pull/1941
+[1811]: https://github.com/Gallopsled/pwntools/pull/1811
+[1827]: https://github.com/Gallopsled/pwntools/pull/1827
+[1841]: https://github.com/Gallopsled/pwntools/pull/1841
+[1839]: https://github.com/Gallopsled/pwntools/pull/1839
+[1833]:  https://github.com/Gallopsled/pwntools/pull/1833
+[1852]: https://github.com/Gallopsled/pwntools/pull/1852
+[1883]: https://github.com/Gallopsled/pwntools/pull/1883
+
+## 4.4.0
+
+- [#1541][1541] Use `context.newline` for tubes by default
+- [#1602][1602] Fix bytes handling in ssh tubes
+- [#1606][1606] Fix `asm()` and `disasm()` for MSP430, S390
 - [#1616][1616] Fix `cyclic` cli for 64 bit integers
+- [#1632][1632] Enable usage of Pwntools in jupyter
+- [#1633][1633] Open a shell if `pwn template` cannot download the remote file
+- [#1644][1644] Enable and support SNI for SSL-wrapped tubes
+- [#1651][1651] Make `pwn shellcraft` faster
+- [#1654][1654] Docker images (`pwntools/pwntools:stable` etc) now use Python3 by default, and includes assemblers for a few common architectures
+- [#1667][1667] Add i386 encoder `ascii_shellcode` (Fixed docs in #1693)
+- Fix syscall instruction lists for SROP on `i386` and `amd64`
+- Fix migration to another ROP
+- [#1673][1673] Add `base=` argument to `ROP.chain()` and `ROP.dump()`
+- [#1675][1675] Gdbserver now correctly accepts multiple libraries in `LD_PRELOAD` and `LD_LIBRARY_PATH`
+- [#1678][1678] ROPGadget multibr
+- [#1682][1682] ROPGadget multibr fix
+- [#1687][1687] Actually import `requests` when doing `from pwn import *`
+- [#1688][1688] Add `__setattr__` and `__call__` interfaces to `ROP` for setting registers
+- [#1692][1692] Remove python2 shebangs where appropriate
+- [#1703][1703] Update libcdb buildid offsets for amd64 and i386
+- [#1704][1704] Try https://libc.rip/ for libcdb lookup
 
+[1541]: https://github.com/Gallopsled/pwntools/pull/1541
+[1602]: https://github.com/Gallopsled/pwntools/pull/1602
+[1606]: https://github.com/Gallopsled/pwntools/pull/1606
 [1616]: https://github.com/Gallopsled/pwntools/pull/1616
+[1632]: https://github.com/Gallopsled/pwntools/pull/1632
+[1633]: https://github.com/Gallopsled/pwntools/pull/1633
+[1644]: https://github.com/Gallopsled/pwntools/pull/1644
+[1651]: https://github.com/Gallopsled/pwntools/pull/1651
+[1654]: https://github.com/Gallopsled/pwntools/pull/1654
+[1667]: https://github.com/Gallopsled/pwntools/pull/1667
+[1673]: https://github.com/Gallopsled/pwntools/pull/1673
+[1675]: https://github.com/Gallopsled/pwntools/pull/1675
+[1678]: https://github.com/Gallopsled/pwntools/pull/1678
+[1682]: https://github.com/Gallopsled/pwntools/pull/1679
+[1687]: https://github.com/Gallopsled/pwntools/pull/1687
+[1688]: https://github.com/Gallopsled/pwntools/pull/1688
+[1692]: https://github.com/Gallopsled/pwntools/pull/1692
+[1703]: https://github.com/Gallopsled/pwntools/pull/1703
+[1704]: https://github.com/Gallopsled/pwntools/pull/1704
 
-## 4.3.0 (`beta`)
+## 4.3.1
+
+- [#1732][1732] Fix shellcraft SSTI vulnerability (first major pwntools vuln!)
+
+[1732]: https://github.com/Gallopsled/pwntools/pull/1732
+
+## 4.3.0
 
 - [#1576][1576] Add `executable=` argument to `ELF.search`
 - [#1584][1584] Add `jmp_esp`/`jmp_rsp` attribute to `ROP`
@@ -70,6 +215,7 @@ The table below shows which release corresponds to each branch, and what date th
 - [#1601][1601] Add `pwn version` command line tool
 - [#1605][1605] Add to `fiddling.hexdump` a way to suppress the total at the end
 - [#1613][1613] Permit `--password` for `pwn template`
+- [#1616][1616] Fix `cyclic` cli for 64 bit integers
 - [#1564][1564] Fix `asm()` and `disasm()` for PowerPC64, MIPS64, Sparc64
 - [#1621][1621] Permit negative values in flat() and fit()
 
@@ -80,10 +226,11 @@ The table below shows which release corresponds to each branch, and what date th
 [1601]: https://github.com/Gallopsled/pwntools/pull/1601
 [1605]: https://github.com/Gallopsled/pwntools/pull/1605
 [1613]: https://github.com/Gallopsled/pwntools/pull/1613
+[1616]: https://github.com/Gallopsled/pwntools/pull/1616
 [1564]: https://github.com/Gallopsled/pwntools/pull/1564
 [1621]: https://github.com/Gallopsled/pwntools/pull/1621
 
-## 4.2.1 (`stable`)
+## 4.2.1
 
 - [#1625][1625] GDB now properly loads executables with QEMU
 - [#1663][1663] Change lookup algorithm of `adb.which`
@@ -157,7 +304,7 @@ The table below shows which release corresponds to each branch, and what date th
 [1001]: https://github.com/Gallopsled/pwntools/pull/1001
 [1389]: https://github.com/Gallopsled/pwntools/pull/1389
 [1241]: https://github.com/Gallopsled/pwntools/pull/1241
-[1218]: https://github.com/Gallopsled/pwntools/pull/1218  
+[1218]: https://github.com/Gallopsled/pwntools/pull/1218
 
 ## 4.0.1
 

DOCKER.md

@@ -0,0 +1,76 @@
+# Using Pwntools with Docker
+
+Sometimes it's annoying to set up Pwntools on your workstation, and you want something that Just Works (TM).
+
+[Docker](https://www.docker.com/) is here to the rescue! Using Docker means that you get a nice, standardized Linux environment and don't need to worry about pip or installing dependencies.
+
+## Quick Start
+
+First, install Docker for your OS, which you can find on their [Getting Started](https://www.docker.com/get-started) page.
+
+Next, download and run the Pwntools stable docker image.
+
+```sh
+$ docker run -it pwntools/pwntools:stable
+```
+
+## Recommended Settings
+
+In order to get the most from your docker image, we need to enable debugging of processes (`--privileged`) and expose the network ports from the guest to the host (`--net=host`).
+
+```sh
+$ docker run -it \
+    --privileged \
+    --net=host \
+    --hostname localhost \
+    --ulimit core=-1:-1 \
+    pwntools/pwntools:stable
+```
+
+## Sharing a Folder
+
+It's really nice to be able to use your preferred native editor, and have the changes show up live inside your Docker image.  This is easy to add, thanks to Docker's bind mounts (`--mount type=bind`).  
+
+With the command below, your `~/exploits` directory will magically show up inside the Docker image at `/home/pwntools/exploits` so that you can easily run them (from Docker) and edit them (from outside Docker).
+
+```sh
+$ mkdir $HOME/exploits
+
+$ vim $HOME/exploits/my_exploit.py
+
+$ docker run -it \
+    --privileged \
+    --net=host \
+    --hostname localhost \
+    --ulimit core=-1:-1 \
+    --mount type=bind,source="$HOME/exploits",target=/home/pwntools/exploits \
+    pwntools/pwntools:stable
+    
+$ python3 exploits/my_exploit.py
+```
+
+### Windows User Bind Mounts
+
+If you're a Windows user `$HOME` doesn't exist in the same way as on Linux, instead it is `%UserProfile%`.  The command from above would look like this, assuming your editor is Visual Studio Code and you have code.exe in your `%PATH%`.
+
+```sh
+C:\Users\user> mkdir Desktop\exploits
+
+C:\Users\user> code Desktop\exploits\my_exploit.py
+
+C:\Users\user> docker run -it \
+    --privileged \
+    --net=host \
+    --hostname localhost \
+    --ulimit core=-1:-1 \
+    --mount type=bind,source="%UserProfile%\Desktop\exploits",target=/home/pwntools/exploits \
+    pwntools/pwntools:stable
+    
+$ python3 exploits/my_exploit.py
+
+```
+
+
+
+
+

README.md

@@ -1,12 +1,14 @@
 # pwntools - CTF toolkit
 ![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)
 
-[![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
 [![PyPI](https://img.shields.io/pypi/v/pwntools?style=flat)](https://pypi.python.org/pypi/pwntools/)
-[![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
-[![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
-[![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)
+[![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
+[![Travis](https://img.shields.io/travis/Gallopsled/pwntools/dev?logo=Travis)](https://travis-ci.org/Gallopsled/pwntools)
+[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/Gallopsled/pwntools/Continuous%20Integration/dev?logo=GitHub)](https://github.com/Gallopsled/pwntools/actions?query=workflow%3A%22Continuous+Integration%22+branch%3Adev)
+[![Coveralls](https://img.shields.io/coveralls/github/Gallopsled/pwntools/dev?logo=coveralls)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
+[![Discord](https://img.shields.io/discord/809590285687980052?label=Discord&style=plastic)](https://discord.gg/96VA2zvjCB)
+[![Twitter](https://img.shields.io/twitter/follow/Pwntools)](https://twitter.com/pwntools)
 
 Pwntools is a CTF framework and exploit development library. Written in Python, it is designed for rapid prototyping and development, and intended to make exploit writing as simple as possible.
 
@@ -20,29 +22,28 @@ r.send(asm(shellcraft.sh()))
 r.interactive()
 ```
 
-# Try It Now!
-
-You can now do a live demo of Pwntools, [right in your browser](https://demo.pwntools.com).
-
 # Documentation
 
 Our documentation is available at [docs.pwntools.com](https://docs.pwntools.com/)
 
+A series of tutorials is also [available online](https://github.com/Gallopsled/pwntools-tutorial#readme)
+
 To get you started, we've provided some example solutions for past CTF challenges in our [write-ups repository](https://github.com/Gallopsled/pwntools-write-ups).
 
 # Installation
 
-Pwntools is best supported on 64-bit Ubuntu LTE releases (12.04, 14.04, 16.04 and 18.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  Python >= 2.7 is required (Python 3 suggested as best).
+Pwntools is best supported on 64-bit Ubuntu LTS releases (14.04, 16.04, 18.04, and 20.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  
 
-Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
+Python3 is suggested, but Pwntools still works with Python 2.7.  Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
 
 ```sh
 apt-get update
 apt-get install python3 python3-pip python3-dev git libssl-dev libffi-dev build-essential
 python3 -m pip install --upgrade pip
-python3 -m pip install --upgrade git+https://github.com/Gallopsled/pwntools.git@dev
+python3 -m pip install --upgrade pwntools
 ```
 
+
 However, some of the features (assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://docs.pwntools.com/en/stable/install.html).
 
 
@@ -50,8 +51,5 @@ However, some of the features (assembling/disassembling foreign architectures) r
 
 See [CONTRIBUTING.md](CONTRIBUTING.md)
 
-# Contact
-If you have any questions not worthy of a [bug report](https://github.com/Gallopsled/pwntools/issues), feel free to ping us
-at [`#pwntools` on Freenode](irc://irc.freenode.net/pwntools) and ask away.
-Click [here](https://kiwiirc.com/client/irc.freenode.net/pwntools) to connect.
-There is also a [mailing list](https://groups.google.com/forum/#!forum/pwntools-users) for higher latency discussion.
+# Contact and Community
+If you have any questions not worthy of a [bug report](https://github.com/Gallopsled/pwntools/issues), join the Discord server at https://discord.gg/96VA2zvjCB

docs/Makefile

@@ -45,7 +45,7 @@ help:
 clean:
 	-rm -rf $(BUILDDIR)/*
 	-rm -rf /tmp/user/pwn*
-	-rm -rf $(HOME)/.pwntools-cache
+	-rm -rf $(HOME)/.cache/.pwntools-cache
 	-rm -rf /tmp/pwn* 2>/dev/null || true
 	-rm -rf build
 

docs/requirements.txt

@@ -11,7 +11,7 @@ pypandoc
 pyserial>=2.7
 requests>=2.5.1
 ropgadget>=5.3
-sphinx
+sphinx<3.4
 sphinx_rtd_theme
 sphinxcontrib-napoleon
-sphinxcontrib-autoprogram
+sphinxcontrib-autoprogram<=0.1.5

docs/source/asm.rst

@@ -2,12 +2,7 @@
 
    import tempfile
    import subprocess
-   from pwnlib.context import context
-   from pwnlib.asm import *
-   from pwnlib import shellcraft
-   from pwnlib.tubes.process import process
-   from pwnlib.util.fiddling import unhex
-
+   from pwn import *
 
 :mod:`pwnlib.asm` --- Assembler functions
 =========================================

docs/source/commandline.rst

@@ -12,6 +12,8 @@ Command Line Tools
 
 pwntools comes with a handful of useful command-line utilities which serve as wrappers for some of the internal functionality.
 
+If these tools do not appear to be installed, make sure that you have added ``~/.local/bin`` to your ``$PATH`` environment variable.
+
 .. toctree::
 
 .. autoprogram:: pwnlib.commandline.main:parser

docs/source/conf.py

@@ -25,7 +25,8 @@ build_dash = tags.has('dash')
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('../..'))
 
-import pwnlib
+import pwnlib.update
+pwnlib.update.disabled = True
 
 # If your documentation needs a minimal Sphinx version, state it here.
 #needs_sphinx = '1.0'
@@ -70,10 +71,14 @@ doctest_global_setup = '''
 import sys, os
 os.environ['PWNLIB_NOTERM'] = '1'
 os.environ['PWNLIB_RANDOMIZE'] = '0'
-import pwnlib, logging
+import pwnlib.update
+import pwnlib.util.fiddling
+import logging
+pwnlib.update.disabled = True
 pwnlib.context.context.reset_local()
 pwnlib.context.ContextType.defaults['log_level'] = logging.ERROR
 pwnlib.context.ContextType.defaults['randomize'] = False
+# pwnlib.context.ContextType.defaults['terminal'] = ['sh', '-c']
 pwnlib.util.fiddling.default_style = {}
 pwnlib.term.text.when = 'never'
 pwnlib.log.install_default_handler()
@@ -91,6 +96,9 @@ pwnlib.context.ContextType.defaults['log_console'] = stdout()
 
 github_actions = os.environ.get('USER') == 'runner'
 travis_ci = os.environ.get('USER') == 'travis'
+local_doctest = os.environ.get('USER') == 'pwntools'
+branch_dev = os.environ.get('GITHUB_BASE_REF') == 'dev'
+skip_android = True
 '''
 
 autoclass_content = 'both'
@@ -257,7 +265,7 @@ latex_documents = [
    u'2016, Gallopsled et al.', 'manual'),
 ]
 
-intersphinx_mapping = {'python': ('https://docs.python.org/2.7', None),
+intersphinx_mapping = {'python': ('https://docs.python.org/3.8', None),
                        'paramiko': ('https://paramiko-docs.readthedocs.org/en/2.1/', None)}
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -340,7 +348,7 @@ def linkcode_resolve(domain, info):
     val = mod
     for k in info['fullname'].split('.'):
         val = getattr(val, k, None)
-        if val == None:
+        if val is None:
             break
 
     # Special case for shellcraft
@@ -381,7 +389,9 @@ import sphinx.ext.autodoc
 
 # Test hidden members (e.g. def _foo(...))
 def dont_skip_any_doctests(app, what, name, obj, skip, options):
-    return False
+    return None
+
+autodoc_default_options = {'special-members': None, 'private-members': None}
 
 class _DummyClass(object): pass
 
@@ -403,24 +413,36 @@ class Py2OutputChecker(_DummyClass, doctest.OutputChecker):
         if sup(rly_want, got, optionflags):
             return True
         rly_want = ' '.join(x[:2].replace('b"',' "').replace("b'"," '")+x[2:] for x in want.replace('\n','\n ').split(' ')).replace('\n ','\n')
-        return sup(rly_want, got, optionflags)
+        if sup(rly_want, got, optionflags):
+            return True
+        for wantl, gotl in six.moves.zip_longest(want.splitlines(), got.splitlines(), fillvalue=''):
+            rly_want1 = '['.join(x[:2].replace('b"','"').replace("b'","'")+x[2:] for x in wantl.split('['))
+            rly_want2 = ' '.join(x[:2].replace('b"',' "').replace("b'"," '")+x[2:] for x in wantl.split(' '))
+            if not sup(rly_want1, gotl, optionflags) and not sup(rly_want2, gotl, optionflags):
+                return False
+        return True
 
 def py2_doctest_init(self, checker=None, verbose=None, optionflags=0):
     if checker is None:
         checker = Py2OutputChecker()
     doctest.DocTestRunner.__init__(self, checker, verbose, optionflags)
 
-class EndlessLoop(Exception): pass
-def alrm_handler(sig, frame):
-    signal.alarm(180) # three minutes
-    raise EndlessLoop()
-signal.signal(signal.SIGALRM, alrm_handler)
-signal.alarm(600) # ten minutes
-
 if 'doctest' in sys.argv:
     def setup(app):
-        app.connect('autodoc-skip-member', dont_skip_any_doctests)
+        pass # app.connect('autodoc-skip-member', dont_skip_any_doctests)
 
     if sys.version_info[:1] < (3,):
         import sphinx.ext.doctest
         sphinx.ext.doctest.SphinxDocTestRunner.__init__ = py2_doctest_init
+    else:
+        # monkey patching paramiko due to https://github.com/paramiko/paramiko/pull/1661
+        import paramiko.client
+        import binascii
+        paramiko.client.hexlify = lambda x: binascii.hexlify(x).decode()
+        paramiko.util.safe_string = lambda x: '' # function result never *actually used*
+    class EndlessLoop(Exception): pass
+    def alrm_handler(sig, frame):
+        signal.alarm(180) # three minutes
+        raise EndlessLoop()
+    signal.signal(signal.SIGALRM, alrm_handler)
+    signal.alarm(600) # ten minutes

docs/source/elf/corefile.rst

@@ -3,6 +3,22 @@
    from glob import glob
    from pwn import *
 
+   # The Linux kernel won't overwrite an existing corefile, so in case 
+   # some other part of the doctests caused a segfault and core dump,
+   # we need to get rid of it before our tests run.
+   #
+   # We DONT need to worry about e.g. ./core existing when using Corefile()
+   # because we always move and rename the corefile to prevent this situation.
+   if os.path.exists('core'): 
+      os.unlink('core')
+
+   # bash-static is a statically linked version of bash, but if $SHELL is not
+   # set to anything, it decides to up and load ld.so and libc.so which breaks
+   # our example of showing `corefile.libc == None` for a statically linked bin.
+   # Set the environment here so it's not in the middle of our tests.
+   os.environ.setdefault('SHELL', '/bin/sh')
+
+
 :mod:`pwnlib.elf.corefile` --- Core Files
 ===========================================================
 

docs/source/encoders.rst

@@ -1,13 +1,18 @@
 .. testsetup:: *
 
    from pwn import *
-
+   
 :mod:`pwnlib.encoders` --- Encoding Shellcode
 ===============================================
 
 .. automodule:: pwnlib.encoders.encoder
    :members:
 
+.. automodule:: pwnlib.encoders.i386.ascii_shellcode
+   :members:
+   :special-members:
+   :exclude-members: __init__
+
 .. automodule:: pwnlib.encoders.i386.xor
    :members:
 

docs/source/filesystem.rst

@@ -0,0 +1,17 @@
+.. testsetup:: *
+
+    import time
+    import six
+    from pwnlib.context import context
+    from pwnlib.tubes.ssh import ssh
+    from pwnlib.filesystem import *
+
+:mod:`pwnlib.filesystem` --- Manipulating Files Locally and Over SSH
+====================================================================
+
+Provides a Python2-compatible :py:mod:`pathlib` interface for paths
+on the local filesystem (`.Path`) as well as on remote filesystems,
+via SSH (`.SSHPath`).
+
+.. automodule:: pwnlib.filesystem
+   :members:

docs/source/index.rst

@@ -55,6 +55,7 @@ Each of the ``pwntools`` modules is documented here.
    elf
    exception
    filepointer
+   filesystem
    flag
    fmtstr
    gdb
@@ -91,3 +92,12 @@ Each of the ``pwntools`` modules is documented here.
    * :ref:`genindex`
    * :ref:`modindex`
    * :ref:`search`
+
+Bytes
+-----
+
+The bytes vs text distinction is so important that it even made it to this main page.
+See the pwntools-tutorial_ repo for the latest tutorial finally
+explaining the difference once and for all (hopefully).
+
+.. _pwntools-tutorial: https://github.com/Gallopsled/pwntools-tutorial/blob/master/bytes.md

docs/source/install.rst

@@ -1,9 +1,7 @@
 Installation
 ============
 
-pwntools is best supported on Ubuntu 12.04 and 14.04, but most
-functionality should work on any Posix-like distribution (Debian, Arch,
-FreeBSD, OSX, etc.).
+Pwntools is best supported on 64-bit Ubuntu LTS releases (14.04, 16.04, 18.04, and 20.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).
 
 Prerequisites
 -------------
@@ -20,7 +18,10 @@ following system libraries installed.
 Released Version
 -----------------
 
-pwntools is available as a ``pip`` package.
+pwntools is available as a ``pip`` package for both Python2 and Python3.
+
+Python3
+^^^^^^^
 
 .. code-block:: bash
 
@@ -29,6 +30,37 @@ pwntools is available as a ``pip`` package.
     $ python3 -m pip install --upgrade pip
     $ python3 -m pip install --upgrade pwntools
 
+
+Python2 (Deprecated)
+^^^^^^^^^^^^^^^^^^^^
+
+NOTE: Pwntools maintainers STRONGLY recommend using Python3 for all future Pwntools-based scripts and projects.
+
+Additionally, due to `pip` dropping support for Python2, a specfic version of `pip` must be installed.
+
+.. code-block:: bash
+
+    $ apt-get update
+    $ apt-get install python python-pip python-dev git libssl-dev libffi-dev build-essential
+    $ python2 -m pip install --upgrade pip==20.3.4
+    $ python2 -m pip install --upgrade pwntools
+
+
+Command-Line Tools
+------------------
+
+When installed with ``sudo`` the above commands will install Pwntools' command-line tools to somewhere like ``/usr/bin``.
+
+However, if you run as an unprivileged user, you may see a warning message that looks like this:
+
+.. code-block::
+
+      WARNING: The scripts asm, checksec, common, constgrep, cyclic, debug, disablenx, disasm, 
+      elfdiff, elfpatch, errno, hex, main, phd, pwn, pwnstrip, scramble, shellcraft, template, 
+      unhex, update and version are installed in '/home/user/.local/bin' which is not on PATH.
+
+Follow the instructions listed and add ``~/.local/bin`` to your ``$PATH`` environment variable.
+
 Development
 --------------
 

docs/source/install/binutils.rst

@@ -12,7 +12,7 @@ Building `binutils` from source takes about 60 seconds on a modern 8-core machin
 Ubuntu
 ^^^^^^^^^^^^^^^^
 
-For Ubuntu 12.04 through 15.10, you must first add the pwntools `Personal Package Archive repository <http://binutils.pwntools.com>`__.
+For Ubuntu 12.04 through 15.10, you must first add the pwntools `Personal Package Archive repository <https://launchpad.net/~pwntools/+archive/ubuntu/binutils>`__.
 
 Ubuntu Xenial (16.04) has official packages for most architectures, and does not require this step.
 

docs/source/intro.rst

@@ -17,6 +17,13 @@ assemble, disassemble, pack, unpack, and many other things with a single functio
 A full list of everything that is imported is available on :doc:`globals`.
 
 
+Tutorials
+---------
+
+A series of tutorials for Pwntools exists online, at
+https://github.com/Gallopsled/pwntools-tutorial#readme
+
+
 Making Connections
 ------------------
 
@@ -188,10 +195,10 @@ Never write another hexdump, thanks to :mod:`pwnlib.util.fiddling`.
 
 Find offsets in your buffer that cause a crash, thanks to :mod:`pwnlib.cyclic`.
 
-    >>> print(cyclic(20).decode())
-    aaaabaaacaaadaaaeaaa
+    >>> cyclic(20)
+    b'aaaabaaacaaadaaaeaaa'
     >>> # Assume EIP = 0x62616166 (b'faab' which is pack(0x62616166))  at crash time
-    >>> print(cyclic_find(b'faab'))
+    >>> cyclic_find(b'faab')
     120
 
 ELF Manipulation

docs/source/protocols.rst

@@ -1,15 +1 @@
-:mod:`pwnlib.protocols` --- Wire Protocols
-=============================================
-
-.. automodule:: pwnlib.protocols
-
-
-Supported Protocols
--------------------
-
-.. toctree::
-    :maxdepth: 3
-    :glob:
-
-    protocols/*
 

docs/source/protocols/adb.rst

@@ -1,11 +0,0 @@
-.. testsetup:: *
-
-   from pwn import *
-   from pwnlib.protocols.adb import AdbClient
-   AdbClient().wait_for_device()
-
-:mod:`pwnlib.protocols.adb` --- ADB Protocol Implementation
-===========================================================
-
-.. automodule:: pwnlib.protocols.adb
-   :members:

docs/source/rop.rst

@@ -1,7 +1,3 @@
-.. testsetup:: *
-
-   from pwn import *
-
 :mod:`pwnlib.rop` --- Return Oriented Programming
 =================================================
 

docs/source/rop/call.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+   from pwnlib.rop.call import *
+
+:mod:`pwnlib.rop.call` --- Return Oriented Programming Function Calling
+=======================================================================
+
+.. automodule:: pwnlib.rop.call
+   :members:

docs/source/rop/rop.rst

@@ -1,6 +1,23 @@
 .. testsetup:: *
 
-   from pwn import *
+   import time
+   from glob import glob
+
+   from pwnlib.asm import asm
+   from pwnlib import constants
+   from pwnlib.context import context
+   from pwnlib.elf import ELF
+   from pwnlib.rop import ROP
+   from pwnlib.rop.call import Call, AppendedArgument
+   from pwnlib.elf.maps import CAT_PROC_MAPS_EXIT
+   from pwnlib.util.packing import *
+   from pwnlib.util.fiddling import *
+   from pwnlib.tubes.process import process
+   from pwnlib import shellcraft
+   from pwnlib.util.misc import which
+
+   context.clear()
+
 
 :mod:`pwnlib.rop.rop` --- Return Oriented Programming
 ==========================================================

docs/source/rop/srop.rst

@@ -1,6 +1,11 @@
 .. testsetup:: *
 
-   from pwn import *
+   from pwnlib.rop.srop import *
+
+   from pwnlib import constants
+   from pwnlib import shellcraft
+   from pwnlib.elf import ELF
+   from pwnlib.tubes.process import process
 
 :mod:`pwnlib.rop.srop` --- Sigreturn Oriented Programming
 ==========================================================

docs/source/runner.rst

@@ -1,6 +1,7 @@
 .. testsetup:: *
 
-   from pwn import *
+   from pwnlib.runner import *
+   from pwnlib.asm import asm
 
 :mod:`pwnlib.runner` --- Running Shellcode
 ===========================================

docs/source/shellcraft/mips.rst

@@ -1,6 +1,15 @@
 .. testsetup:: *
 
-   from pwn import *
+   import tempfile
+
+   from pwnlib import shellcraft
+
+   from pwnlib.asm import asm
+   from pwnlib.context import context
+   from pwnlib.runner import run_assembly
+   from pwnlib.util.fiddling import enhex
+   from pwnlib.util.misc import write
+
    context.clear(arch='mips')
 
 :mod:`pwnlib.shellcraft.mips` --- Shellcode for MIPS

docs/source/tubes/ssh.rst

@@ -14,6 +14,10 @@
       :members: kill, poll, interactive
       :show-inheritance:
 
+   .. autoclass:: pwnlib.tubes.ssh.ssh_process
+      :members:
+      :show-inheritance:
+
    .. autoclass:: pwnlib.tubes.ssh.ssh_connecter()
       :show-inheritance:
 

docs/source/util/iters.rst

@@ -17,19 +17,19 @@
      count                                  ,
      cycle                                  ,
      dropwhile                              ,
+     filter                                 ,
+     filterfalse                            ,
      groupby                                ,
-     ifilter                                ,
-     ifilterfalse                           ,
-     imap                                   ,
      islice                                 ,
-     izip                                   ,
-     izip_longest                           ,
+     map                                    ,
      permutations                           ,
      product                                ,
      repeat                                 ,
      starmap                                ,
      takewhile                              ,
-     tee
+     tee                                    ,
+     zip                                    ,
+     zip_longest
 
 .. function:: chain
 
@@ -63,29 +63,29 @@
 
    Alias for :func:`itertools.groupby`
 
-.. function:: ifilter
+.. function:: filter
 
-   Alias for :func:`itertools.ifilter`
+   Alias for python3-style :func:`filter`
 
-.. function:: ifilterfalse
+.. function:: filterfalse
 
-   Alias for :func:`itertools.ifilterfalse`
+   Alias for :func:`itertools.filterfalse`
 
-.. function:: imap
+.. function:: map
 
-   Alias for :func:`itertools.imap`
+   Alias for python3-style :func:`map`
 
 .. function:: islice
 
    Alias for :func:`itertools.islice`
 
-.. function:: izip
+.. function:: zip
 
-   Alias for :func:`itertools.izip`
+   Alias for python3-style :func:`zip`
 
-.. function:: izip_longest
+.. function:: zip_longest
 
-   Alias for :func:`itertools.izip_longest`
+   Alias for :func:`itertools.zip_longest`
 
 .. function:: permutations
 

docs/source/util/misc.rst

@@ -3,7 +3,6 @@
    from pprint import pprint
    from pwnlib.util.misc import *
    import os, subprocess
-   os.chdir("..")
 
 :mod:`pwnlib.util.misc` --- We could not fit it any other place
 ===============================================================

docs/source/util/net.rst

@@ -2,7 +2,6 @@
 
    from pwnlib.util.net import *
 
-
 :mod:`pwnlib.util.net` --- Networking interfaces
 ===================================================
 

docs/source/util/web.rst

@@ -1,5 +1,7 @@
 .. testsetup:: *
 
+   import tempfile
+
    from pwnlib.context import context
    from pwnlib.util.web import *
 

examples/gdb_api.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python3
+"""An example of using GDB Python API with Pwntools."""
+from pwn import *
+
+
+def check_write(gdb, exp_buf):
+    """Check that write() was called with the expected arguments."""
+    fd = gdb.parse_and_eval('$rdi').cast(gdb.lookup_type('int'))
+    assert fd == 1, fd
+    buf_addr = gdb.parse_and_eval('$rsi').cast(gdb.lookup_type('long'))
+    count = gdb.parse_and_eval('$rdx').cast(gdb.lookup_type('long'))
+    buf = gdb.selected_inferior().read_memory(buf_addr, count).tobytes()
+    assert buf == exp_buf, buf
+
+
+def demo_sync_breakpoint(cat, gdb, txt):
+    """Demonstrate a synchronous breakpoint."""
+    # set the synchronous breakpoint on ``write``
+    gdb.Breakpoint('write', temporary=True)
+
+    # resume the program
+    gdb.continue_nowait()
+
+    # send the line
+    cat.sendline(txt)
+
+    # wait until we hit the breakpoint
+    gdb.wait()
+
+    # inspect program state
+    check_write(gdb, (txt + '\n').encode())
+
+    # resume the program
+    gdb.continue_nowait()
+
+    # expect to observe the line we just sent
+    cat.recvuntil(txt)
+
+
+def demo_async_breakpoint(cat, gdb, txt):
+    """Demonstrate asynchronous breakpoint."""
+    # set the asynchronous breakpoint on ``write``
+    class WriteBp(gdb.Breakpoint):
+        def __init__(self):
+            super().__init__('write')
+            self.count = 0
+
+        def stop(self):
+            # called in a separate thread
+            check_write(gdb, (txt + '\n').encode())
+            self.count += 1
+
+    bp = WriteBp()
+
+    # resume the program
+    gdb.continue_nowait()
+
+    # send the line and immediately expect to observe it
+    cat.sendline(txt)
+    cat.recvuntil(txt)
+
+    # check that we hit the breakpoint
+    assert bp.count == 1, bp.count
+
+    # interrupt the program
+    gdb.interrupt_and_wait()
+
+    # delete the breakpoint
+    bp.delete()
+
+    # resume the program
+    gdb.continue_nowait()
+
+
+def main():
+    # start ``cat`` under GDB
+    with gdb.debug('cat', gdbscript='''
+set logging on
+set pagination off
+''', api=True) as cat:
+
+        # the process is stopped
+        # set the synchronous breakpoint on ``read``
+        cat.gdb.Breakpoint('read', temporary=True)
+
+        # resume and wait until we hit it
+        cat.gdb.continue_and_wait()
+
+        # demonstrate a more interesting synchronous breakpoint
+        demo_sync_breakpoint(cat, cat.gdb, 'foo')
+
+        # terminate GDB
+        cat.gdb.quit()
+
+    # now start ``cat`` normally
+    with process('cat') as cat:
+        # attach GDB
+        _, cat_gdb = gdb.attach(cat, gdbscript='''
+set logging on
+set pagination off
+''', api=True)
+
+        # the process is stopped
+        # demonstrate asynchronous breakpoint
+        demo_async_breakpoint(cat, cat_gdb, 'bar')
+
+        # terminate GDB
+        cat_gdb.quit()
+
+
+if __name__ == '__main__':
+    main()

examples/ssh.py

@@ -36,5 +36,14 @@ print(shell.echo("single string"))
 print(shell.echo(["list","of","strings"]))
 print(shell["echo single statement"])
 
+# ssh.process() is the most flexible way to run a command)
+io = shell.process(['/bin/bash', '-c', 'echo Hello $FOO'], 
+                   env={'FOO': 'World'}, # Set environment
+                   stderr='/dev/null',   # Override file descriptors
+                   aslr=False,           # Disable ASLR on processes
+                   setuid=False,         # Disable setuid bit so processes can be debugged
+                   shell=False)          # Enable or disable shell evaluation
+print(io.recvall())
+
 # Show off the interactive shell
 shell.interactive()

extra/docker/Makefile

@@ -1,13 +1,34 @@
-subdirs=xenial stable $(wildcard */)
-makefiles=$(wildcard */Makefile)
+subdirs=$(shell find . -type d -depth 1 | xargs basename)
 tags=base stable beta dev
 
-all: $(makefiles) $(subdirs)
+all: $(subdirs) $(tags)
 
-$(subdirs):
+tags: $(tags)
+
+base:
+	@echo "Building $@"
 	docker build -t pwntools/pwntools:$(shell basename $(@)) "$@"
 
-$(makefiles):
-	make -C $(shell dirname $@)
+stable beta dev: base
+	@echo "Building $@"
+	docker build -t pwntools/pwntools:$(shell basename $(@)) "$@"
 
-.PHONY: all $(subdirs) $(makefiles)
+develop:
+	@echo "Building $@"
+	docker build -t pwntools/pwntools:$(shell basename $(@)) "$@"
+	docker run -it --privileged --net=host --hostname localhost \
+		--ulimit core=-1:-1 \
+		--mount type=bind,source="$(shell git rev-parse --show-toplevel)",target=/home/pwntools/pwntools \
+		pwntools/pwntools:$(shell basename $(@))
+
+debug:
+	@echo "Subdirs   $(subdirs)"
+	@echo "Tags      $(tags)"
+
+clean:
+	docker container prune
+	docker
+
+FORCE:
+
+.PHONY: all $(subdirs) $(makefiles) $(tags)

extra/docker/README.md

@@ -0,0 +1,53 @@
+# Pwntools Docker Images
+
+This directory contains some Docker images with various versions of Pwntools pre-installed.
+
+## Getting Images
+
+You can either make them yourself, or pull from Docker Hub.
+
+### Docker Hub
+
+To pull an image from Docker hub, just run the command:
+
+```sh
+$ docker pull pwntools/pwntools:stable
+```
+
+Replace `stable` with `beta` or `dev` as desired.
+
+### Building Locally
+
+To build the images locally, just invoke the `Makefile` with the desired branch target.
+
+```sh
+$ make -C extra/docker dev
+```
+
+## Running Images
+
+To run the `stable`, `beta`, or `dev` image, we recommend the following command line, in this example it is the `dev` image.
+
+```sh
+$ docker run -it \
+    --privileged \
+    --net=host \
+ 		--ulimit core=-1:-1 \
+ 		pwntools/pwntools:dev
+```
+
+The `--privileged` option is needed to perform any debugging with GDB.
+
+The `--ulimit` option is needed for any corefiles to drop when a program crashes.
+
+The `--net=host` allows you to easily connect to services hosted inside the container from outside the container.
+
+## Development Dockerfile
+
+In addition to `stable`, `beta`, and `dev` Dockerfiles, there is also a `develop` dockerfile which mounts your Pwntools installation inside the VM so you can develop seamlessly while editing on your host.
+
+```sh
+$ make -C extra/docker develop
+```
+
+You will be dropped into a shell, and any changes you make to your local Pwntools checkout will be immediately reflected inside the Docker container.  This is useful for e.g. editing on a macOS host, but testing your changes on Ubuntu without needing to copy files around.

extra/docker/base

@@ -1 +0,0 @@
-xenial

extra/docker/base/Dockerfile

@@ -3,20 +3,35 @@
 # Based on Ubuntu
 ############################################################
 
-FROM ubuntu:precise
+FROM ubuntu:bionic
 MAINTAINER Maintainer Gallopsled et al.
 
+env DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get update \
     && apt-get install -y \
         build-essential \
         git \
         libssl-dev \
         libffi-dev \
-        python2.7 \
+        python \
         python-pip \
         python-dev \
+        python3 \
+        python3-pip \
+        python3-dev \
+        qemu-user-static \
+        binutils-arm-linux-gnueabihf \
+        binutils-aarch64-linux-gnu \
+        binutils-mips-linux-gnu \
+        binutils-mipsel-linux-gnu \
+        binutils-powerpc-linux-gnu \
+        binutils-powerpc64-linux-gnu \
+        binutils-sparc64-linux-gnu \
     && pip install --upgrade pip \
     && python -m pip install --upgrade pwntools \
+    && pip3 install --upgrade pip \
+    && python3 -m pip install --upgrade pwntools \
     && PWNLIB_NOTERM=1 pwn update \
     && apt-get install -y sudo \
     && useradd -m pwntools \

extra/docker/beta/Dockerfile

@@ -2,5 +2,6 @@ FROM pwntools/pwntools:stable
 
 USER root
 RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@beta
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/dev/Dockerfile

@@ -2,5 +2,6 @@ FROM pwntools/pwntools:stable
 
 USER root
 RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/develop/Dockerfile

@@ -0,0 +1,59 @@
+FROM pwntools/pwntools:base
+
+# Uninstall existing versions of pwntools
+USER root
+RUN python -m pip uninstall -q -y pwntools \
+ && python3 -m pip uninstall -q -y pwntools
+
+# Switch back to the pwntools user from here forward
+USER pwntools
+WORKDIR /home/pwntools
+
+# Since we are not installing Pwntools systemwide, the "pwn" binaries
+# etc will all end up in this path.
+ENV PATH="/home/pwntools/.local/bin:${PATH}"
+
+# Install Pwntools to the home directory, make it an editable install
+RUN git clone https://github.com/Gallopsled/pwntools \
+ && python  -m pip install --upgrade --editable pwntools \
+ && python3 -m pip install --upgrade --editable pwntools \
+ && PWNLIB_NOTERM=1 pwn version
+
+# Requirements for running the tests
+RUN python  -m pip install --upgrade --requirement pwntools/docs/requirements.txt \
+ && python3 -m pip install --upgrade --requirement pwntools/docs/requirements.txt
+
+# Dependencies from .travis.yml addons -> apt -> packages
+RUN sudo apt-get install -y \
+	ash \
+	bash \
+	bash-static \
+	binutils-msp430 \
+	binutils-multiarch \
+	binutils-s390x-linux-gnu \
+	dash \
+	gcc \
+	gcc-multilib \
+	gdb \ 
+	ksh \
+	lib32stdc++6 \
+	libc6-dev-i386 \
+	mksh \
+	pandoc \
+	qemu-user-static \
+	socat \
+	sshpass \
+	zsh
+
+# Misc useful things when developing
+RUN sudo apt-get install -y  \
+	curl \
+	ipython \
+	ipython3 \
+	lsb-release \
+	ssh \
+	unzip \
+	wget
+
+# Do not require password for sudo
+RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee /etc/sudoers.d/travis

extra/docker/kali/Dockerfile

@@ -1,26 +0,0 @@
-############################################################
-# Dockerfile to build Pwntools container
-# Based on Ubuntu
-############################################################
-
-FROM kalilinux/kali-linux-docker
-MAINTAINER Maintainer Gallopsled et al.
-
-RUN apt-get update \
-    && apt-get install -y \
-        build-essential \
-        git \
-        libssl-dev \
-        libffi-dev \
-        python2.7 \
-        python-pip \
-        python-dev \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
-    && useradd -m pwntools \
-    && passwd --delete --unlock pwntools \
-    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
-USER pwntools
-WORKDIR /home/pwntools

extra/docker/kali/Makefile

@@ -1,8 +0,0 @@
-all: Dockerfile
-
-Dockerfile: ../xenial/Dockerfile FORCE
-	sed "s|ubuntu:xenial|kalilinux/kali-linux-docker|" < "$<" > "$@"
-
-FORCE:
-
-.PHONY: all

extra/docker/precise/Makefile

@@ -1,8 +0,0 @@
-all: Dockerfile
-
-Dockerfile: ../xenial/Dockerfile FORCE
-	sed "s|ubuntu:xenial|ubuntu:precise|" < "$<" > "$@"
-
-FORCE:
-
-.PHONY: all

extra/docker/stable/Dockerfile

@@ -2,5 +2,6 @@ FROM pwntools/pwntools:base
 
 USER root
 RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/trusty/Dockerfile

@@ -1,26 +0,0 @@
-############################################################
-# Dockerfile to build Pwntools container
-# Based on Ubuntu
-############################################################
-
-FROM ubuntu:trusty
-MAINTAINER Maintainer Gallopsled et al.
-
-RUN apt-get update \
-    && apt-get install -y \
-        build-essential \
-        git \
-        libssl-dev \
-        libffi-dev \
-        python2.7 \
-        python-pip \
-        python-dev \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
-    && useradd -m pwntools \
-    && passwd --delete --unlock pwntools \
-    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
-USER pwntools
-WORKDIR /home/pwntools

extra/docker/trusty/Makefile

@@ -1,8 +0,0 @@
-all: Dockerfile
-
-Dockerfile: ../xenial/Dockerfile FORCE
-	sed "s|ubuntu:xenial|ubuntu:trusty|" < "$<" > "$@"
-
-FORCE:
-
-.PHONY: all

extra/docker/xenial/Dockerfile

@@ -1,26 +0,0 @@
-############################################################
-# Dockerfile to build Pwntools container
-# Based on Ubuntu
-############################################################
-
-FROM ubuntu:xenial
-MAINTAINER Maintainer Gallopsled et al.
-
-RUN apt-get update \
-    && apt-get install -y \
-        build-essential \
-        git \
-        libssl-dev \
-        libffi-dev \
-        python2.7 \
-        python-pip \
-        python-dev \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
-    && PWNLIB_NOTERM=1 pwn update \
-    && apt-get install -y sudo \
-    && useradd -m pwntools \
-    && passwd --delete --unlock pwntools \
-    && echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
-USER pwntools
-WORKDIR /home/pwntools

pwn/__init__.py

@@ -7,7 +7,6 @@ pwnlib.args.initialize()
 pwnlib.log.install_default_handler()
 pwnlib.config.initialize()
 
-log = pwnlib.log.getLogger('pwnlib.exploit')
 args = pwnlib.args.args
 
 if not platform.architecture()[0].startswith('64'):

pwn/toplevel.py

@@ -4,7 +4,9 @@ import logging
 import math
 import operator
 import os
+import platform
 import re
+import requests
 import socks
 import signal
 import string
@@ -15,6 +17,7 @@ import tempfile
 import threading
 import time
 
+import colored_traceback
 from pprint import pprint
 
 import pwnlib
@@ -28,8 +31,9 @@ from pwnlib.elf.corefile import Core, Corefile, Coredump
 from pwnlib.elf.elf import ELF, load
 from pwnlib.encoders import *
 from pwnlib.exception import PwnlibException
-from pwnlib.gdb import attach, debug, debug_assembly, debug_shellcode
+from pwnlib.gdb import attach, debug_assembly, debug_shellcode
 from pwnlib.filepointer import *
+from pwnlib.filesystem import *
 from pwnlib.flag import *
 from pwnlib.fmtstr import FmtStr, fmtstr_payload, fmtstr_split
 from pwnlib.log import getLogger
@@ -37,6 +41,7 @@ from pwnlib.memleak import MemLeak, RelativeMemLeak
 from pwnlib.regsort import *
 from pwnlib.replacements import *
 from pwnlib.rop import ROP
+from pwnlib.rop.call import AppendedArgument
 from pwnlib.rop.srop import SigreturnFrame
 from pwnlib.rop.ret2dlresolve import Ret2dlresolvePayload
 from pwnlib.runner import *
@@ -73,6 +78,7 @@ from pwnlib.util.web import *
 from six.moves import cPickle as pickle, cStringIO as StringIO
 from six import BytesIO
 
+log = getLogger("pwnlib.exploit")
 error   = log.error
 warning = log.warning
 warn    = log.warning
@@ -80,4 +86,7 @@ info    = log.info
 debug   = log.debug
 success = log.success
 
-__all__ = [x for x in globals().keys() if x != '__name__']
+colored_traceback.add_hook()
+
+# Equivalence with the default behavior of "from import *"
+# __all__ = [x for x in tuple(globals()) if not x.startswith('_')]

pwnlib/__init__.py

@@ -1,12 +1,11 @@
 from __future__ import absolute_import
 
-import importlib
-
-from pwnlib.version import __version__
+from .version import __version__
 
 version = __version__
 
 __all__ = [
+    'adb',
     'args',
     'asm',
     'atexception',
@@ -35,9 +34,8 @@ __all__ = [
     'ui',
     'useragents',
     'util',
-    'adb',
     'update',
+    'version',
 ]
 
-for module in __all__:
-    importlib.import_module('.%s' % module, 'pwnlib')
+from . import args

pwnlib/adb/adb.py

@@ -68,6 +68,7 @@ from pwnlib.context import context
 from pwnlib.device import Device
 from pwnlib.log import getLogger
 from pwnlib.protocols.adb import AdbClient
+from pwnlib.util.packing import _decode
 from pwnlib.util import misc
 
 log = getLogger(__name__)
@@ -75,10 +76,13 @@ log = getLogger(__name__)
 def adb(argv, *a, **kw):
     r"""Returns the output of an ADB subcommand.
 
-    >>> adb.adb('get-serialno')
-    b'emulator-5554\n'
-    >>> adb.adb(['shell', 'uname']) # it is better to use adb.process
-    b'Linux\n'
+    .. doctest::
+       :skipif: skip_android
+
+        >>> adb.adb('get-serialno')
+        b'emulator-5554\n'
+        >>> adb.adb(['shell', 'uname']) # it is better to use adb.process
+        b'Linux\n'
     """
     if isinstance(argv, (bytes, six.text_type)):
         argv = [argv]
@@ -113,11 +117,12 @@ def current_device(any=False):
     """Returns an ``AdbDevice`` instance for the currently-selected device
     (via ``context.device``).
 
-    Example:
+    .. doctest::
+       :skipif: skip_android
 
         >>> device = adb.current_device(any=True)
-        >>> device
-        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_google_phone_armv7', model='sdk google phone armv7', device='generic')
+        >>> device  # doctest: +ELLIPSIS
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_...phone_armv7', model='sdk ...phone armv7', device='generic')
         >>> device.port
         'emulator'
     """
@@ -144,7 +149,10 @@ def with_device(f):
 def root():
     """Restarts adbd as root.
 
-    >>> adb.root()
+    .. doctest::
+       :skipif: skip_android
+
+        >>> adb.root()
     """
     log.info("Enabling root on %s" % context.device)
 
@@ -202,6 +210,10 @@ def uptime():
         Uptime of the device, in seconds
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.uptime() > 3 # normally AVD takes ~7 seconds to boot
         True
     """
@@ -217,6 +229,10 @@ def boot_time():
         nearest second.
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> import time
         >>> adb.boot_time() < time.time()
         True
@@ -231,6 +247,9 @@ class AdbDevice(Device):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> device = adb.wait_for_device()
         >>> device.arch
         'arm'
@@ -238,8 +257,8 @@ class AdbDevice(Device):
         32
         >>> device.os
         'android'
-        >>> device.product
-        'sdk_google_phone_armv7'
+        >>> device.product  # doctest: +ELLIPSIS
+        'sdk_...phone_armv7'
         >>> device.serial
         'emulator-5554'
     """
@@ -357,10 +376,13 @@ class AdbDevice(Device):
         """Provides scoped access to ``adb`` module propertise, in the context
         of this device.
 
-        >>> property = 'ro.build.fingerprint'
-        >>> device = adb.wait_for_device()
-        >>> adb.getprop(property) == device.getprop(property)
-        True
+        .. doctest::
+           :skipif: skip_android
+
+            >>> property = 'ro.build.fingerprint'
+            >>> device = adb.wait_for_device()
+            >>> adb.getprop(property) == device.getprop(property)
+            True
         """
         with context.local(device=self):
             g = globals()
@@ -387,6 +409,9 @@ def wait_for_device(kick=False):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> device = adb.wait_for_device()
     """
     with log.waitfor("Waiting for device to come online") as w:
@@ -496,6 +521,9 @@ def pull(remote_path, local_path=None):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> _=adb.pull('/proc/version', './proc-version')
         >>> print(read('./proc-version').decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
@@ -527,6 +555,9 @@ def push(local_path, remote_path):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> write('./filename', 'contents')
         >>> adb.push('./filename', '/data/local/tmp')
         '/data/local/tmp/filename'
@@ -584,6 +615,9 @@ def read(path, target=None, callback=None):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> print(adb.read('/proc/version').decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
         >>> adb.read('/does/not/exist')
@@ -613,6 +647,9 @@ def write(path, data=b''):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.write('/dev/null', b'data')
         >>> adb.write('/data/local/tmp/')
     """
@@ -633,6 +670,9 @@ def mkdir(path):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.mkdir('/')
 
         >>> path = '/data/local/tmp/mkdir_test'
@@ -673,6 +713,9 @@ def makedirs(path):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.makedirs('/data/local/tmp/this/is/a/directory/hierarchy')
         >>> adb.listdir('/data/local/tmp/this/is/a/directory')
         ['hierarchy']
@@ -689,6 +732,9 @@ def exists(path):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.exists('/')
         True
         >>> adb.exists('/etc/hosts')
@@ -706,6 +752,9 @@ def isdir(path):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.isdir('/')
         True
         >>> adb.isdir('/init')
@@ -724,6 +773,9 @@ def unlink(path, recursive=False):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.unlink("/does/not/exist")
         Traceback (most recent call last):
         ...
@@ -775,6 +827,9 @@ def process(argv, *a, **kw):
 
     Examples:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.root()
         >>> print(adb.process(['cat','/proc/version']).recvall().decode('utf-8')) # doctest: +ELLIPSIS
         Linux version ...
@@ -815,6 +870,9 @@ def which(name, all = False, *a, **kw):
 
     Example:
 
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.which('sh')
         '/system/bin/sh'
         >>> adb.which('sh', all=True)
@@ -838,7 +896,7 @@ def which(name, all = False, *a, **kw):
 
     which_cmd = which_cmd.strip()
     data = process(['sh','-c', which_cmd], *a, **kw).recvall()
-    data = context._decode(data)
+    data = _decode(data)
     result = []
 
     for path in data.split('\x00'):
@@ -864,6 +922,10 @@ def whoami():
     """Returns current shell user
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
        >>> adb.whoami()
        b'root'
     """
@@ -917,6 +979,10 @@ def proc_exe(pid):
     """Returns the full path of the executable for the provided PID.
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.proc_exe(1)
         b'/init'
     """
@@ -937,18 +1003,22 @@ def getprop(name=None):
         Otherwise, a string is returned with the contents of the named property.
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> adb.getprop() # doctest: +ELLIPSIS
         {...}
     """
     with context.quiet:
         if name:
             result = process(['getprop', name]).recvall().strip()
-            result = context._decode(result)
+            result = _decode(result)
             return result
 
         result = process(['getprop']).recvall()
 
-    result = context._decode(result)
+    result = _decode(result)
     expr = r'\[([^\]]+)\]: \[(.*)\]'
 
     props = {}
@@ -1052,6 +1122,10 @@ class Kernel(object):
     def address(self):
         """Returns kernel address
         Example:
+
+        .. doctest::
+           :skipif: skip_android
+
             >>> hex(adb.kernel.address) # doctest: +ELLIPSIS
             '0x...000'
         """
@@ -1178,6 +1252,10 @@ class Property(object):
         """Allow simple comparison
 
         Example:
+
+        .. doctest::
+           :skipif: skip_android
+
             >>> adb.properties.ro.build.version.sdk == "24"
             True
         """
@@ -1266,6 +1344,10 @@ def compile(source):
     r"""Compile a source file or project with the Android NDK.
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> temp = tempfile.mktemp('.c')
         >>> write(temp, '''
         ... #include <stdio.h>
@@ -1390,6 +1472,10 @@ class Partitions(object):
     """Enable access to partitions
 
     Example:
+
+    .. doctest::
+       :skipif: skip_android
+
         >>> hex(adb.partitions.vda.size) # doctest: +ELLIPSIS
         '0x...000'
     """
@@ -1506,3 +1592,4 @@ def version():
     """Returns rthe platform version as a tuple."""
     prop = getprop('ro.build.version.release')
     return [int(v) for v in prop.split('.')]
+

pwnlib/adb/bootloader.py

@@ -4,6 +4,7 @@ from __future__ import division
 import ctypes
 import io
 import os
+import six
 import sys
 
 from pwnlib.log import getLogger
@@ -64,7 +65,7 @@ class BootloaderImage(object):
         Returns:
             Contents of the image.
         """
-        if isinstance(index_or_name, int):
+        if isinstance(index_or_name, six.integer_types):
             index = index_or_name
         else:
             for i in range(len(self.img_info)):

pwnlib/args.py

@@ -1,5 +1,3 @@
-
-#!/usr/bin/env python2
 """
 Pwntools exposes several magic command-line arguments and environment
 variables when operating in `from pwn import *` mode.

pwnlib/asm.py

@@ -167,6 +167,9 @@ def which_binutils(util):
         'i386':   ['x86_64', 'amd64'],
         'i686':   ['x86_64', 'amd64'],
         'amd64':  ['x86_64', 'i386'],
+        'mips64': ['mips'],
+        'powerpc64': ['powerpc'],
+        'sparc64': ['sparc'],
     }.get(arch, [])
 
     # If one of the candidate architectures matches the native
@@ -190,16 +193,18 @@ def which_binutils(util):
         for gutil in utils:
             # e.g. objdump
             if arch is None:
-                pattern = gutil
+                patterns = [gutil]
 
-            # e.g. aarch64-linux-gnu-objdump
+            # e.g. aarch64-linux-gnu-objdump, avr-objdump
             else:
-                pattern = '%s*linux*-%s' % (arch,gutil)
+                patterns = ['%s*linux*-%s' % (arch, gutil),
+                            '%s-%s' % (arch, gutil)]
 
-            for dir in environ['PATH'].split(':'):
-                res = sorted(glob(path.join(dir, pattern)))
-                if res:
-                    return res[0]
+            for pattern in patterns:
+                for dir in environ['PATH'].split(':'):
+                    res = sorted(glob(path.join(dir, pattern)))
+                    if res:
+                        return res[0]
 
     # No dice!
     print_binutils_instructions(util, context)
@@ -245,9 +250,9 @@ def _assembler():
                                          stderr=subprocess.STDOUT, universal_newlines=True)
         version = re.search(r' (\d\.\d+)', result).group(1)
         if version < '2.19':
-            log.warn_once('Your binutils version is too old and may not work!\n'  + \
-                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n' + \
-                'Reported Version: %r' % result.strip())
+            log.warn_once('Your binutils version is too old and may not work!\n'
+                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n'
+                'Reported Version: %r', result.strip())
 
 
     return assembler
@@ -294,6 +299,7 @@ def _arch_header():
     prefix  = ['.section .shellcode,"awx"',
                 '.global _start',
                 '.global __start',
+                '.p2align 2',
                 '_start:',
                 '__start:']
     headers = {
@@ -322,15 +328,18 @@ def _bfdname():
         'amd64'   : 'elf64-x86-64',
         'arm'     : 'elf32-%sarm' % E,
         'thumb'   : 'elf32-%sarm' % E,
+        'avr'     : 'elf32-avr',
         'mips'    : 'elf32-trad%smips' % E,
         'mips64'  : 'elf64-trad%smips' % E,
         'alpha'   : 'elf64-alpha',
         'cris'    : 'elf32-cris',
         'ia64'    : 'elf64-ia64-%s' % E,
         'm68k'    : 'elf32-m68k',
+        'msp430'  : 'elf32-msp430',
         'powerpc' : 'elf32-powerpc',
         'powerpc64' : 'elf64-powerpc',
         'vax'     : 'elf32-vax',
+        's390'    : 'elf%d-s390' % context.bits,
         'sparc'   : 'elf32-sparc',
         'sparc64' : 'elf64-sparc',
     }
@@ -344,10 +353,13 @@ def _bfdname():
 def _bfdarch():
     arch = context.arch
     convert = {
-    'i386': 'i386',
-    'amd64': 'i386:x86-64',
-    'thumb': 'arm',
-    'ia64': 'ia64-elf64'
+        'amd64':     'i386:x86-64',
+        'i386':      'i386',
+        'ia64':      'ia64-elf64',
+        'mips64':    'mips',
+        'powerpc64': 'powerpc',
+        'sparc64':   'sparc',
+        'thumb':     'arm',
     }
 
     if arch in convert:
@@ -356,7 +368,7 @@ def _bfdarch():
     return arch
 
 def _run(cmd, stdin = None):
-    log.debug(subprocess.list2cmdline(cmd))
+    log.debug('%s', subprocess.list2cmdline(cmd))
     try:
         proc = subprocess.Popen(
             cmd,
@@ -369,17 +381,20 @@ def _run(cmd, stdin = None):
         exitcode = proc.wait()
     except OSError as e:
         if e.errno == errno.ENOENT:
-            log.exception('Could not run %r the program' % cmd[0])
+            log.exception('Could not run %r the program', cmd[0])
         else:
             raise
 
     if (exitcode, stderr) != (0, ''):
-        msg = 'There was an error running %s:\n' % repr(cmd)
+        msg = 'There was an error running %r:\n'
+        args = cmd,
         if exitcode != 0:
-            msg += 'It had the exitcode %d.\n' % exitcode
+            msg += 'It had the exitcode %d.\n'
+            args += exitcode,
         if stderr != '':
-            msg += 'It had this on stdout:\n%s\n' % stderr
-        log.error(msg)
+            msg += 'It had this on stdout:\n%s\n'
+            args += stderr,
+        log.error(msg, *args)
 
     return stdout
 
@@ -481,9 +496,7 @@ def make_elf_from_assembly(assembly,
         >>> file_b = make_elf_from_assembly('nop', extract=True)
         >>> file_a[:4] == file_b[:4]
         True
-        >>> len(file_a) < 0x200
-        True
-        >>> len(file_b) > 0x1000
+        >>> len(file_a) < len(file_b)
         True
     """
     if shared and vma:
@@ -637,6 +650,10 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
         b'H\xc7\xc0\x17\x00\x00\x00'
         >>> asm("mov r0, #SYS_select", arch = 'arm', os = 'linux', bits=32)
         b'R\x00\xa0\xe3'
+        >>> asm("mov #42, r0", arch = 'msp430')
+        b'0@*\x00'
+        >>> asm("la %r0, 42", arch = 's390', bits=64)
+        b'A\x00\x00*'
     """
     result = ''
 
@@ -749,6 +766,12 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         >>> print(disasm(unhex('656664676665400F18A4000000000051'), byte=0, arch='amd64'))
            0:   gs data16 fs data16 rex nop/reserved BYTE PTR gs:[eax+eax*1+0x0]
            f:   push   rcx
+        >>> print(disasm(unhex('01000000'), arch='sparc64'))
+           0:   01 00 00 00     nop
+        >>> print(disasm(unhex('60000000'), arch='powerpc64'))
+           0:   60 00 00 00     nop
+        >>> print(disasm(unhex('00000000'), arch='mips64'))
+           0:   00000000        nop
     """
     result = ''
 

pwnlib/atexit.py

@@ -13,6 +13,7 @@ from __future__ import division
 import sys
 import threading
 import traceback
+import atexit as std_atexit
 
 from pwnlib.context import context
 
@@ -95,4 +96,8 @@ def _run_handlers():
 if hasattr(sys, "exitfunc"):
     register(sys.exitfunc)
 
-sys.exitfunc = _run_handlers
+if sys.version_info[0] < 3:
+    sys.exitfunc = _run_handlers
+else:
+    std_atexit.register(_run_handlers)
+

pwnlib/commandline/__init__.py

@@ -18,5 +18,6 @@ __all__ = [
     'scramble',
     'shellcraft',
     'unhex',
-    'update'
+    'update',
+    'version'
 ]

pwnlib/commandline/asm.py

@@ -5,7 +5,7 @@ from __future__ import division
 import argparse
 import sys
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -13,7 +13,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'asm',
-    help = 'Assemble shellcode into bytes'
+    help = 'Assemble shellcode into bytes',
+    description = 'Assemble shellcode into bytes',
 )
 
 parser.add_argument(

pwnlib/commandline/checksec.py

@@ -10,7 +10,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'checksec',
-    help = 'Check binary security settings'
+    help = 'Check binary security settings',
+    description = 'Check binary security settings',
 )
 parser.add_argument(
     'elf',

pwnlib/commandline/constgrep.py

@@ -4,10 +4,9 @@ from __future__ import division
 
 import argparse
 import functools
-import os
 import re
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -16,28 +15,26 @@ from pwnlib.commandline import common
 p = common.parser_commands.add_parser(
     'constgrep',
     help = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
+    description = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
     formatter_class = argparse.RawDescriptionHelpFormatter,
 )
 
-group = p.add_mutually_exclusive_group()
-group.add_argument(
+p.add_argument(
     '-e', '--exact',
-    metavar = 'constant',
-    # nargs = 1,
-    default = None,
-    help = 'Do an exact match for a constant instead of searching for a regex',
+    action='store_true',
+    help='Do an exact match for a constant instead of searching for a regex',
 )
-group.add_argument(
+
+p.add_argument(
     'regex',
-    nargs = '?',
-    default = '',
-    help = 'The regex matching constant you want to find',
+    help='The regex matching constant you want to find',
 )
 
 p.add_argument(
     'constant',
     nargs = '?',
     default = None,
+    type = safeeval.expr,
     help = 'The constant to find',
 )
 
@@ -65,7 +62,7 @@ p.add_argument(
 def main(args):
     if args.exact:
         # This is the simple case
-        print(cpp(args.exact).strip())
+        print(cpp(args.regex).strip())
     else:
         # New we search in the right module.
         # But first: We find the right module
@@ -80,20 +77,12 @@ def main(args):
         else:
             matcher = re.compile(args.regex)
 
-
-        # Evaluate the given constant
-        if args.constant:
-            try:
-                constant = safeeval.expr(args.constant)
-            except:
-                log.error("Could not evaluate constant %r" % args.constant)
-        else:
-            constant = None
-
         # The found matching constants and the length of the longest string
         out    = []
         maxlen = 0
 
+        constant = args.constant
+
         for k in dir(mod):
             # No python stuff
             if k.endswith('__') and k.startswith('__'):
@@ -104,7 +93,7 @@ def main(args):
                 continue
 
             # Check the constant
-            if constant != None:
+            if constant is not None:
                 val = getattr(mod, k)
                 if args.mask_mode:
                     if constant & val != val:
@@ -124,7 +113,7 @@ def main(args):
         # If we are in match_mode, then try to find a combination of
         # constants that yield the exact given value
         # We do not want to find combinations using the value 0.
-        if not (constant == None or constant == 0) and args.mask_mode:
+        if constant and args.mask_mode:
             mask = constant
             good = []
             out = [(v, k) for v, k in out if v != 0]

pwnlib/commandline/cyclic.py

@@ -7,7 +7,7 @@ import six
 import string
 import sys
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -15,14 +15,15 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'cyclic',
-    help = "Cyclic pattern creator/finder"
+    help = "Cyclic pattern creator/finder",
+    description = "Cyclic pattern creator/finder"
 )
 
 parser.add_argument(
     '-a', '--alphabet',
     metavar = 'alphabet',
     default = string.ascii_lowercase.encode(),
-    type = six.ensure_binary,
+    type = packing._encode,
     help = 'The alphabet to use in the cyclic pattern (defaults to all lower case letters)',
 )
 
@@ -70,7 +71,7 @@ def main(args):
             pat = int(pat, 0)
         except ValueError:
             pass
-        pat = flat(pat)
+        pat = flat(pat, bytes=args.length)
 
         if len(pat) != subsize:
             log.critical('Subpattern must be %d bytes' % subsize)

pwnlib/commandline/debug.py

@@ -10,7 +10,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'debug',
-    help = 'Debug a binary in GDB'
+    help = 'Debug a binary in GDB',
+    description = 'Debug a binary in GDB'
 )
 parser.add_argument(
     '-x', metavar='GDBSCRIPT',

pwnlib/commandline/disablenx.py

@@ -8,7 +8,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'disablenx',
-    help = 'Disable NX for an ELF binary'
+    help = 'Disable NX for an ELF binary',
+    description = 'Disable NX for an ELF binary'
 )
 parser.add_argument(
     'elf',

pwnlib/commandline/disasm.py

@@ -7,7 +7,7 @@ import argparse
 import string
 import sys
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -15,7 +15,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'disasm',
-    help = 'Disassemble bytes into text format'
+    help = 'Disassemble bytes into text format',
+    description = 'Disassemble bytes into text format'
 )
 
 parser.add_argument(

pwnlib/commandline/elfdiff.py

@@ -8,7 +8,7 @@ from subprocess import CalledProcessError
 from subprocess import check_output
 from tempfile import NamedTemporaryFile
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -29,7 +29,8 @@ def diff(a,b):
 
 p = common.parser_commands.add_parser(
     'elfdiff',
-    help = 'Compare two ELF files'
+    help = 'Compare two ELF files',
+    description = 'Compare two ELF files'
 )
 
 p.add_argument('a')

pwnlib/commandline/elfpatch.py

@@ -4,7 +4,7 @@ from __future__ import division
 
 import sys
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -12,7 +12,8 @@ from pwnlib.commandline import common
 
 p = common.parser_commands.add_parser(
     'elfpatch',
-    help = 'Patch an ELF file'
+    help = 'Patch an ELF file',
+    description = 'Patch an ELF file'
 )
 
 p.add_argument('elf',help="File to patch")

pwnlib/commandline/errno.py

@@ -8,7 +8,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'errno',
-    help = 'Prints out error messages'
+    help = 'Prints out error messages',
+    description = 'Prints out error messages'
 )
 
 parser.add_argument(

pwnlib/commandline/hex.py

@@ -10,9 +10,9 @@ from pwnlib.util.fiddling import enhex
 
 parser = common.parser_commands.add_parser(
     'hex',
-    help = '''
-Hex-encodes data provided on the command line or stdin
-''')
+    help = 'Hex-encodes data provided on the command line or stdin',
+    description = 'Hex-encodes data provided on the command line or stdin')
+
 parser.add_argument('data', nargs='*',
     help='Data to convert into hex')
 

pwnlib/commandline/main.py

@@ -21,6 +21,7 @@ from pwnlib.commandline import shellcraft
 from pwnlib.commandline import template
 from pwnlib.commandline import unhex
 from pwnlib.commandline import update
+from pwnlib.commandline import version
 from pwnlib.commandline.common import parser
 from pwnlib.context import context
 
@@ -43,6 +44,7 @@ commands = {
     'template': template.main,
     'unhex': unhex.main,
     'update': update.main,
+    'version': version.main,
 }
 
 def main():

pwnlib/commandline/phd.py

@@ -7,7 +7,7 @@ import os
 import sys
 import io
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -15,7 +15,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'phd',
-    help = 'Pwnlib HexDump'
+    help = 'Pretty hex dump',
+    description = 'Pretty hex dump'
 )
 
 parser.add_argument(

pwnlib/commandline/pwnstrip.py

@@ -3,7 +3,7 @@ from __future__ import division
 
 import argparse
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -12,6 +12,7 @@ from pwnlib.commandline import common
 p = common.parser_commands.add_parser(
     'pwnstrip',
     help = 'Strip binaries for CTF usage',
+    description = 'Strip binaries for CTF usage'
 )
 
 g = p.add_argument_group("actions")

pwnlib/commandline/scramble.py

@@ -4,7 +4,7 @@ from __future__ import division
 import argparse
 import sys
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -12,7 +12,8 @@ from pwnlib.commandline import common
 
 parser = common.parser_commands.add_parser(
     'scramble',
-    help = 'Shellcode encoder'
+    help = 'Shellcode encoder',
+    description = 'Shellcode encoder'
 )
 
 parser.add_argument(

pwnlib/commandline/shellcraft.py

@@ -8,7 +8,7 @@ import six
 import sys
 import types
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -35,6 +35,7 @@ def _string(s):
 p = common.parser_commands.add_parser(
     'shellcraft',
     help = 'Microwave shellcode -- Easy, fast and delicious',
+    description = 'Microwave shellcode -- Easy, fast and delicious',
 )
 
 
@@ -198,11 +199,11 @@ def main(args):
         common.parser.print_usage()
         exit()
 
-    if args.shellcode not in shellcraft.templates:
+    try:
+        func = get_template(args.shellcode)
+    except AttributeError:
         log.error("Unknown shellcraft template %r. Use --list to see available shellcodes." % args.shellcode)
 
-    func = get_template(args.shellcode)
-
     if args.show:
         # remove doctests
         doc = []
@@ -295,7 +296,7 @@ def main(args):
 
     vma = args.address
     if vma:
-        vma = eval(vma)
+        vma = pwnlib.util.safeeval.expr(vma)
 
     if args.format in ['e','elf']:
         args.format = 'default'

pwnlib/commandline/template.py

@@ -2,8 +2,6 @@
 from __future__ import absolute_import
 from __future__ import division
 
-import re
-
 from pwn import *
 from pwnlib.commandline import common
 
@@ -11,26 +9,23 @@ from mako.lookup import TemplateLookup
 
 parser = common.parser_commands.add_parser(
     'template',
-    help = 'Generate an exploit template'
+    help = 'Generate an exploit template',
+    description = 'Generate an exploit template'
 )
 
 parser.add_argument('exe', nargs='?', help='Target binary')
 parser.add_argument('--host', help='Remote host / SSH server')
 parser.add_argument('--port', help='Remote port / SSH port', type=int)
 parser.add_argument('--user', help='SSH Username')
-parser.add_argument('--pass', help='SSH Password', dest='password')
+parser.add_argument('--pass', '--password', help='SSH Password', dest='password')
 parser.add_argument('--path', help='Remote path of file on SSH server')
 parser.add_argument('--quiet', help='Less verbose template comments', action='store_true')
+parser.add_argument('--color', help='Print the output in color', choices=['never', 'always', 'auto'], default='auto')
 
 def main(args):
-    cache = None
-
-    if cache:
-        cache = os.path.join(context.cache_dir, 'mako')
-
     lookup = TemplateLookup(
         directories      = [os.path.join(pwnlib.data.path, 'templates')],
-        module_directory = cache
+        module_directory = None
     )
 
     # For the SSH scenario, check that the binary is at the
@@ -40,7 +35,14 @@ def main(args):
             log.error("Must specify --path or a exe")
 
         s = ssh(args.user, args.host, args.port or 22, args.password or None)
-        s.download(args.path or args.exe)
+
+        try:
+            remote_file = args.path or args.exe
+            s.download(remote_file)
+        except Exception:
+            log.warning("Could not download file %r, opening a shell", remote_file)
+            s.interactive()
+            return
 
         if not args.exe:
             args.exe = os.path.basename(args.path)
@@ -57,8 +59,16 @@ def main(args):
     # Fix Mako formatting bs
     output = re.sub('\n\n\n', '\n\n', output)
 
+    # Colorize the output if it's a TTY
+    if args.color == 'always' or (args.color == 'auto' and sys.stdout.isatty()):
+        from pygments import highlight
+        from pygments.formatters import TerminalFormatter
+        from pygments.lexers.python import PythonLexer
+        output = highlight(output, PythonLexer(), TerminalFormatter())
+
     print(output)
 
+    # If redirected to a file, make the resulting script executable
     if not sys.stdout.isatty():
         try: os.fchmod(sys.stdout.fileno(), 0o700)
         except OSError: pass

pwnlib/commandline/unhex.py

@@ -11,9 +11,9 @@ from pwnlib.util.fiddling import unhex
 
 parser = common.parser_commands.add_parser(
     'unhex',
-    help = '''
-Decodes hex-encoded data provided on the command line or via stdin.
-''')
+    help = 'Decodes hex-encoded data provided on the command line or via stdin.',
+    description = 'Decodes hex-encoded data provided on the command line or via stdin.'
+)
 
 parser.add_argument('hex', nargs='*',
     help='Hex bytes to decode')

pwnlib/commandline/update.py

@@ -4,7 +4,7 @@ from __future__ import division
 
 import subprocess
 
-import pwnlib
+import pwnlib.args
 pwnlib.args.free_form = False
 
 from pwn import *
@@ -12,7 +12,8 @@ from pwnlib.commandline import common
 
 p = common.parser_commands.add_parser(
     'update',
-    help = 'Check for pwntools updates'
+    help = 'Check for pwntools updates',
+    description = 'Check for pwntools updates'
 )
 
 p.add_argument('--install', action='store_true', help='''

pwnlib/commandline/version.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+from __future__ import absolute_import
+from __future__ import division
+
+import os
+import subprocess
+
+import pwnlib.args
+pwnlib.args.free_form = False
+
+from pwn import *
+from pwnlib.commandline import common
+
+parser = common.parser_commands.add_parser(
+    'version',
+    help = 'Pwntools version',
+    description = 'Pwntools version'
+)
+
+def main(a):
+	version = pwnlib.version
+
+	git_root = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
+	if os.path.exists(os.path.join(git_root, '.git')):
+		gitver = subprocess.check_output(['git', '-C', git_root, 'log', '-1', '--format=%h (%cr)'])
+		branch = subprocess.check_output(['git', '-C', git_root, 'rev-parse', '--abbrev-ref', 'HEAD'])
+		version = '%s-%s-%s' % (version, branch.decode().strip(), gitver.decode())
+
+	log.info("Pwntools v%s" % version)
+
+if __name__ == '__main__':
+    pwnlib.commandline.common.main(__file__)

pwnlib/config.py

@@ -5,7 +5,9 @@ The list of configurable options includes all of the logging symbols
 and colors, as well as all of the default values on the global context
 object.
 
-The configuration file is read from ``~/.pwn.conf`` and ``/etc/pwn.conf``.
+The configuration file is read from ``~/.pwn.conf``, ``$XDG_CONFIG_HOME/pwn.conf``
+(``$XDG_CONFIG_HOME`` defaults to ``~/.config``, per XDG Base Directory Specification),
+and ``/etc/pwn.conf``.
 
 The configuration file is only read in ``from pwn import *`` mode, and not
 when used in library mode (``import pwnlib``).  To read the configuration
@@ -53,8 +55,13 @@ def initialize():
     from pwnlib.log import getLogger
     log = getLogger(__name__)
 
+    xdg_config_home = (os.environ.get('XDG_CONFIG_HOME') or
+                       os.path.expanduser("~/.config"))
+
     c = configparser.ConfigParser()
-    c.read(['/etc/pwn.conf', os.path.expanduser('~/.pwn.conf')])
+    c.read(['/etc/pwn.conf',
+            os.path.join(xdg_config_home, 'pwn.conf'),
+            os.path.expanduser('~/.pwn.conf')])
 
     for section in c.sections():
         if section not in registered_configs:

pwnlib/constants/__init__.py

@@ -92,7 +92,7 @@ class ConstantsModule(ModuleType):
         # Special case for __all__, we want to return the contextually
         # relevant module.
         if key == '__all__':
-            return self.guess().__dict__.keys()
+            return list(self.guess().__dict__.keys())
 
         # Special case for all other special properties which aren't defined
         if key.endswith('__'):
@@ -145,7 +145,13 @@ class ConstantsModule(ModuleType):
         if key not in self._env_store:
             self._env_store[key] = {key: getattr(self, key) for key in dir(self) if not key.endswith('__')}
 
-        return Constant('(%s)' % string, safeeval.values(string, self._env_store[key]))
+        val = safeeval.values(string, self._env_store[key])
+
+        # if the expression is not assembly-safe, it is not so vital to preserve it
+        if set(string) & (set(bytearray(range(32)).decode()) | set('"#$\',.;@[\\]`{}')):
+            string = val
+
+        return Constant('(%s)' % string, val)
 
 
 # To prevent garbage collection

pwnlib/context/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 """
 Implements context management so that nested/scoped contexts and threaded
@@ -359,17 +358,18 @@ class ContextType(object):
         'log_console': sys.stdout,
         'randomize': False,
         'rename_corefiles': True,
-        'newline': '\n',
+        'newline': b'\n',
         'noptrace': False,
         'os': 'linux',
         'proxy': None,
+        'ssh_session': None,
         'signed': False,
         'terminal': tuple(),
         'timeout': Timeout.maximum,
     }
 
     #: Valid values for :meth:`pwnlib.context.ContextType.os`
-    oses = sorted(('linux','freebsd','windows','cgc','android'))
+    oses = sorted(('linux','freebsd','windows','cgc','android','baremetal'))
 
     big_32    = {'endian': 'big', 'bits': 32}
     big_64    = {'endian': 'big', 'bits': 64}
@@ -549,14 +549,66 @@ class ContextType(object):
     @property
     def quiet(self, function=None):
         """Disables all non-error logging within the enclosed scope,
-        *unless* the debugging level is set to 'debug' or lower."""
+        *unless* the debugging level is set to 'debug' or lower.
+
+        Example:
+
+            Let's assume the normal situation, where log_level is INFO.
+
+            >>> context.clear(log_level='info')
+
+            Note that only the log levels below ERROR do not print anything.
+
+            >>> with context.quiet:
+            ...     log.debug("DEBUG")
+            ...     log.info("INFO")
+            ...     log.warn("WARN")
+
+            Next let's try with the debugging level set to 'debug' before we
+            enter the context handler:
+
+            >>> with context.local(log_level='debug'):
+            ...     with context.quiet:
+            ...         log.debug("DEBUG")
+            ...         log.info("INFO")
+            ...         log.warn("WARN")
+            [DEBUG] DEBUG
+            [*] INFO
+            [!] WARN
+        """
         level = 'error'
         if context.log_level <= logging.DEBUG:
             level = None
         return self.local(function, log_level=level)
 
     def quietfunc(self, function):
-        """Similar to :attr:`quiet`, but wraps a whole function."""
+        """Similar to :attr:`quiet`, but wraps a whole function.
+
+        Example:
+
+            Let's set up two functions, which are the same but one is
+            wrapped with :attr:`quietfunc`.
+
+            >>> def loud(): log.info("Loud")
+            >>> @context.quietfunc
+            ... def quiet(): log.info("Quiet")
+
+            If we set the logging level to 'info', the loud function
+            prints its contents.
+
+            >>> with context.local(log_level='info'): loud()
+            [*] Loud
+
+            However, the quiet function does not, since :attr:`quietfunc`
+            silences all output unless the log level is DEBUG.
+
+            >>> with context.local(log_level='info'): quiet()
+
+            Now let's try again with debugging enabled.
+
+            >>> with context.local(log_level='debug'): quiet()
+            [*] Quiet
+        """
         @functools.wraps(function)
         def wrapper(*a, **kw):
             level = 'error'
@@ -570,6 +622,28 @@ class ContextType(object):
     @property
     def verbose(self):
         """Enable all logging within the enclosed scope.
+
+        This is the opposite of :attr:`.quiet` and functionally equivalent to:
+
+        .. code-block:: python
+
+            with context.local(log_level='debug'):
+                ...
+
+        Example:
+
+            Note that the function does not emit any information by default
+
+            >>> context.clear()
+            >>> def func(): log.debug("Hello")
+            >>> func()
+
+            But if we put it inside a :attr:`.verbose` context manager, the
+            information is printed.
+
+            >>> with context.verbose: func()
+            [DEBUG] Hello
+
         """
         return self.local(log_level='debug')
 
@@ -603,7 +677,7 @@ class ContextType(object):
 
     @property
     def native(self):
-        if context.os in ('android', 'cgc'):
+        if context.os in ('android', 'baremetal', 'cgc'):
             return False
 
         arch = context.arch
@@ -827,27 +901,6 @@ class ContextType(object):
 
         return charset
 
-    def _encode(self, s):
-        if isinstance(s, (bytes, bytearray)):
-            return s   # already bytes
-
-        if self.encoding == 'auto':
-            try:
-                return s.encode('latin1')
-            except UnicodeEncodeError:
-                return s.encode('utf-8', 'surrogateescape')
-        return s.encode(self.encoding)
-
-    def _decode(self, b):
-        if self.encoding == 'auto':
-            try:
-                return b.decode('utf-8')
-            except UnicodeDecodeError:
-                return b.decode('latin1')
-            except AttributeError:
-                return b
-        return b.decode(self.encoding)
-
     @_validator
     def endian(self, endianness):
         """
@@ -932,20 +985,22 @@ class ContextType(object):
         Examples:
 
 
-            >>> context.log_file = 'foo.txt' #doctest: +ELLIPSIS
-            >>> log.debug('Hello!') #doctest: +ELLIPSIS
+            >>> foo_txt = tempfile.mktemp()
+            >>> bar_txt = tempfile.mktemp()
+            >>> context.log_file = foo_txt
+            >>> log.debug('Hello!')
             >>> with context.local(log_level='ERROR'): #doctest: +ELLIPSIS
             ...     log.info('Hello again!')
-            >>> with context.local(log_file='bar.txt'):
+            >>> with context.local(log_file=bar_txt):
             ...     log.debug('Hello from bar!')
             >>> log.info('Hello from foo!')
-            >>> open('foo.txt').readlines()[-3] #doctest: +ELLIPSIS
+            >>> open(foo_txt).readlines()[-3] #doctest: +ELLIPSIS
             '...:DEBUG:...:Hello!\n'
-            >>> open('foo.txt').readlines()[-2] #doctest: +ELLIPSIS
+            >>> open(foo_txt).readlines()[-2] #doctest: +ELLIPSIS
             '...:INFO:...:Hello again!\n'
-            >>> open('foo.txt').readlines()[-1] #doctest: +ELLIPSIS
+            >>> open(foo_txt).readlines()[-1] #doctest: +ELLIPSIS
             '...:INFO:...:Hello from foo!\n'
-            >>> open('bar.txt').readlines()[-1] #doctest: +ELLIPSIS
+            >>> open(bar_txt).readlines()[-1] #doctest: +ELLIPSIS
             '...:DEBUG:...:Hello from bar!\n'
         """
         if isinstance(value, (bytes, six.text_type)):
@@ -1019,7 +1074,7 @@ class ContextType(object):
             >>> context.os = 'foobar' #doctest: +ELLIPSIS
             Traceback (most recent call last):
             ...
-            AttributeError: os must be one of ['android', 'cgc', 'freebsd', 'linux', 'windows']
+            AttributeError: os must be one of ['android', 'baremetal', 'cgc', 'freebsd', 'linux', 'windows']
         """
         os = os.lower()
 
@@ -1279,6 +1334,17 @@ class ContextType(object):
         """
         return bool(v)
 
+    @_validator
+    def newline(self, v):
+        """Line ending used for Tubes by default.
+
+        This configures the newline emitted by e.g. ``sendline`` or that is used
+        as a delimiter for e.g. ``recvline``.
+        """
+        # circular imports
+        from pwnlib.util.packing import _need_bytes
+        return _need_bytes(v)
+
 
     @_validator
     def gdbinit(self, value):
@@ -1322,6 +1388,15 @@ class ContextType(object):
 
         return size
 
+    @_validator
+    def ssh_session(self, shell):
+        from pwnlib.tubes.ssh import ssh
+
+        if not isinstance(shell, ssh):
+            raise AttributeError("context.ssh_session must be an ssh tube") 
+
+        return shell
+
     #*************************************************************************
     #                               ALIASES
     #*************************************************************************

pwnlib/data/elf/__init__.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 from pwnlib.data.elf import fmtstr
 from pwnlib.data.elf import relro
+from pwnlib.data.elf import ret2dlresolve
 
 import os
 path = os.path.dirname(__file__)

pwnlib/data/elf/ret2dlresolve/Makefile

@@ -0,0 +1,8 @@
+
+all: i386 amd64
+
+i386: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -m32 -o $@
+
+amd64: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -o $@

pwnlib/data/elf/ret2dlresolve/__init__.py

@@ -0,0 +1,5 @@
+import os
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)

pwnlib/data/elf/ret2dlresolve/vuln.c

@@ -0,0 +1,8 @@
+#include <unistd.h>
+void vuln(void){
+    char buf[64];
+    read(STDIN_FILENO, buf, 200);
+}
+int main(int argc, char** argv){
+    vuln();
+}

pwnlib/data/syscalls/generate.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python2
 from __future__ import division
 import argparse
+import keyword
 import os
 
 from pwnlib import constants
@@ -62,7 +63,7 @@ CALL = """
 
     for name, arg in zip(argument_names, argument_values):
         if arg is not None:
-            syscall_repr.append('%s=%r' % (name, arg))
+            syscall_repr.append('%s=%s' % (name, pwnlib.shellcraft.pretty(arg, False)))
 
         # If the argument itself (input) is a register...
         if arg in allregs:
@@ -75,8 +76,8 @@ CALL = """
 
         # The argument is not a register.  It is a string value, and we
         # are expecting a string value
-        elif name in can_pushstr and isinstance(arg, (bytes, six.text_type)):
-            if not isinstance(arg, bytes):
+        elif name in can_pushstr and isinstance(arg, (six.binary_type, six.text_type)):
+            if isinstance(arg, six.text_type):
                 arg = arg.encode('utf-8')
             string_arguments[name] = arg
 
@@ -144,12 +145,11 @@ def can_be_array(arg):
 
 
 def fix_bad_arg_names(func, arg):
-    if arg.name == 'str':
-        return 'str_'
     if arg.name == 'len':
         return 'length'
-    if arg.name == 'repr':
-        return 'repr_'
+
+    if arg.name in ('str', 'repr') or keyword.iskeyword(arg.name):
+        return arg.name + '_'
 
     if func.name == 'open' and arg.name == 'vararg':
         return 'mode'
@@ -277,8 +277,10 @@ def generate_one(target):
             CALL.format(**template_variables)
         ]
 
-        with open(os.path.join(target, name + '.asm'), 'wt+') as f:
-            f.write('\n'.join(map(str.strip, lines)))
+        if keyword.iskeyword(name):
+            name += '_'
+        with open(os.path.join(target, name + '.asm'), 'wt') as f:
+            f.write('\n'.join(map(str.strip, lines)) + '\n')
 
 if __name__ == '__main__':
     p = argparse.ArgumentParser()

pwnlib/data/templates/pwnup.mako

@@ -32,7 +32,7 @@ remote_path = remote_path or exe
 password = password or 'secret1234'
 binary_repr = repr(binary)
 %>\
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 %if not quiet:
 # This exploit template was generated via:
@@ -84,14 +84,14 @@ if not args.LOCAL:
 %endif
 
 %if host:
-def local(argv=[], *a, **kw):
+def start_local(argv=[], *a, **kw):
     '''Execute the target binary locally'''
     if args.GDB:
         return gdb.debug([${binary_repr}] + argv, gdbscript=gdbscript, *a, **kw)
     else:
         return process([${binary_repr}] + argv, *a, **kw)
 
-def remote(argv=[], *a, **kw):
+def start_remote(argv=[], *a, **kw):
   %if ssh:
     '''Execute the target binary on the remote host'''
     if args.GDB:
@@ -111,9 +111,9 @@ def remote(argv=[], *a, **kw):
 def start(argv=[], *a, **kw):
     '''Start the exploit against the target.'''
     if args.LOCAL:
-        return local(argv, *a, **kw)
+        return start_local(argv, *a, **kw)
     else:
-        return remote(argv, *a, **kw)
+        return start_remote(argv, *a, **kw)
 %else:
 def start(argv=[], *a, **kw):
     '''Start the exploit against the target.'''

pwnlib/dynelf.py

@@ -62,7 +62,7 @@ from pwnlib.elf import constants
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak
 from pwnlib.util.fiddling import enhex
-from pwnlib.util.packing import unpack
+from pwnlib.util.packing import unpack, _encode
 from pwnlib.util.web import wget
 
 log    = getLogger(__name__)
@@ -75,7 +75,7 @@ def sysv_hash(symbol):
     """
     h = 0
     g = 0
-    for c in bytearray(context._encode(symbol)):
+    for c in bytearray(_encode(symbol)):
         h = (h << 4) + c
         g = h & 0xf0000000
         h ^= (g >> 24)
@@ -87,7 +87,7 @@ def gnu_hash(s):
 
     Function used to generated GNU-style hashes for strings.
     """
-    s = bytearray(context._encode(s))
+    s = bytearray(_encode(s))
     h = 5381
     for c in s:
         h = h * 33 + c
@@ -538,6 +538,9 @@ class DynELF(object):
         if lib == 'libc':
             lib = 'libc.so'
 
+        if symb:
+            symb = _encode(symb)
+
         #
         # Get a pretty name for the symbol to show the user
         #
@@ -637,12 +640,14 @@ class DynELF(object):
         while leak.field(cur, LinkMap.l_prev):
             cur = leak.field(cur, LinkMap.l_prev)
 
+        libname = _encode(libname)
+
         while cur:
             self.status("link_map entry %#x" % cur)
             p_name = leak.field(cur, LinkMap.l_name)
             name   = leak.s(p_name)
 
-            if libname.encode('utf-8') in name:
+            if libname in name:
                 break
 
             if name:
@@ -741,7 +746,6 @@ class DynELF(object):
 
                 # Leak the name of the function from the symbol table
                 name = leak.s(strtab + leak.field(sym, Sym.st_name))
-                name = name.decode('utf-8')
 
                 # Make sure it matches the name of the symbol we were looking for.
                 if name == symb:
@@ -749,7 +753,7 @@ class DynELF(object):
                     addr = libbase + leak.field(sym, Sym.st_value)
                     return addr
 
-                self.status("%s (hash collision)" % name)
+                self.status("%r (hash collision)" % name)
 
             # The name did not match what we were looking for, or we assume
             # it did not since it was not a function.
@@ -825,7 +829,6 @@ class DynELF(object):
                 # Check for collision on hash values
                 sym  = symtab + sizeof(Sym) * (ndx + i)
                 name = leak.s(strtab + leak.field(sym, Sym.st_name))
-                name = name.decode('utf-8')
 
                 if name == symb:
                     # No collision, get offset and calculate address
@@ -833,7 +836,7 @@ class DynELF(object):
                     addr   = offset + libbase
                     return addr
 
-                self.status("%s (hash collision)" % name)
+                self.status("%r (hash collision)" % name)
 
             # Collision or no match, continue to the next item
             i += 1

pwnlib/elf/corefile.py

@@ -196,8 +196,8 @@ class Mapping(object):
                self.start,
                self.stop,
                self.size,
-               self.page_offset,
-               self.flags)
+               self.flags,
+               self.page_offset)
 
     def __int__(self):
         return self.start
@@ -219,8 +219,8 @@ class Mapping(object):
                 stop += self.stop
 
             if not (self.start <= start <= stop <= self.stop):
-                log.error("Byte range [%#x:%#x] not within range [%#x:%#x]" \
-                    % (start, stop, self.start, self.stop))
+                log.error("Byte range [%#x:%#x] not within range [%#x:%#x]",
+                          start, stop, self.start, self.stop)
 
             data = self._core.read(start, stop-start)
 
@@ -264,13 +264,16 @@ class Mapping(object):
         return result + self.address
 
 class Corefile(ELF):
-    r"""Enhances the inforation available about a corefile (which is an extension
+    r"""Enhances the information available about a corefile (which is an extension
     of the ELF format) by permitting extraction of information about the mapped
     data segments, and register state.
 
     Registers can be accessed directly, e.g. via ``core_obj.eax`` and enumerated
     via :data:`Corefile.registers`.
 
+    Memory can be accessed directly via :meth:`.read` or :meth:`.write`, and also
+    via :meth:`.pack` or :meth:`.unpack` or even :meth:`.string`.
+
     Arguments:
         core: Path to the core file.  Alternately, may be a :class:`.process` instance,
               and the core file will be located automatically.
@@ -320,11 +323,7 @@ class Corefile(ELF):
          Mapping('/lib/i386-linux-gnu/ld-2.19.so', start=0xf7713000, stop=0xf7714000, size=0x1000, flags=0x6, page_offset=0x21),
          Mapping('[stack]', start=0xfff3e000, stop=0xfff61000, size=0x23000, flags=0x6, page_offset=0x0)]
 
-    Example:
-
-        The Linux kernel may not overwrite an existing core-file.
-
-        >>> if os.path.exists('core'): os.unlink('core')
+    Examples:
 
         Let's build an example binary which should eat ``R0=0xdeadbeef``
         and ``PC=0xcafebabe``.
@@ -345,18 +344,24 @@ class Corefile(ELF):
         You can specify a full path a la ``Corefile('/path/to/core')``,
         but you can also just access the :attr:`.process.corefile` attribute.
 
+        There's a lot of behind-the-scenes logic to locate the corefile for
+        a given process, but it's all handled transparently by Pwntools.
+
         >>> core = io.corefile
 
-        The core file has a :attr:`.Corefile.exe` property, which is a :class:`.Mapping`
+        The core file has a :attr:`exe` property, which is a :class:`.Mapping`
         object.  Each mapping can be accessed with virtual addresses via subscript, or
         contents can be examined via the :attr:`.Mapping.data` attribute.
 
-        >>> core.exe.address == address
-        True
+        >>> core.exe # doctest: +ELLIPSIS
+        Mapping('/.../step3', start=..., stop=..., size=0x1000, flags=0x..., page_offset=...)
+        >>> hex(core.exe.address)
+        '0x41410000'
 
         The core file also has registers which can be accessed direclty.
-        Pseudo-registers ``pc`` and ``sp`` are available on all architectures,
+        Pseudo-registers :attr:`pc` and :attr:`sp` are available on all architectures,
         to make writing architecture-agnostic code more simple.
+        If this were an amd64 corefile, we could access e.g. ``core.rax``.
 
         >>> core.pc == 0xcafebabe
         True
@@ -367,7 +372,8 @@ class Corefile(ELF):
 
         We may not always know which signal caused the core dump, or what address
         caused a segmentation fault.  Instead of accessing registers directly, we
-        can also extract this information from the core dump.
+        can also extract this information from the core dump via :attr:`fault_addr`
+        and :attr:`signal`.
 
         On QEMU-generated core dumps, this information is unavailable, so we
         substitute the value of PC.  In our example, that's correct anyway.
@@ -378,60 +384,103 @@ class Corefile(ELF):
         11
 
         Core files can also be generated from running processes.
-        This requires GDB to be installed, and can only be done with native
-        processes.  Getting a "complete" corefile requires GDB 7.11 or better.
+        This requires GDB to be installed, and can only be done with native processes.
+        Getting a "complete" corefile requires GDB 7.11 or better.
 
-        >>> elf = ELF('/bin/bash-static')
+        >>> elf = ELF(which('bash-static'))
         >>> context.clear(binary=elf)
-        >>> io = process(elf.path, env={'HELLO': 'WORLD'})
+        >>> env = dict(os.environ)
+        >>> env['HELLO'] = 'WORLD'
+        >>> io = process(elf.path, env=env)
+        >>> io.sendline(b'echo hello')
+        >>> io.recvline()
+        b'hello\n'
+
+        The process is still running, but accessing its :attr:`.process.corefile` property
+        automatically invokes GDB to attach and dump a corefile.
+
         >>> core = io.corefile
 
-        Data can also be extracted directly from the corefile.
+        The corefile can be inspected and read from, and even exposes various mappings
 
-        >>> elf.address > 0
-        True
-        >>> core.exe[elf.address:elf.address+4]
-        b'\x7fELF'
-        >>> core.exe.data[:4]
+        >>> core.exe # doctest: +ELLIPSIS
+        Mapping('.../bin/bash-static', start=..., stop=..., size=..., flags=..., page_offset=...)
+        >>> core.exe.data[0:4]
         b'\x7fELF'
 
-        Various other mappings are available by name.  On Linux, 32-bit Intel binaries
-        should have a VDSO section.  Since our ELF is statically linked, there is
-        no libc which gets mapped.
+        It also supports all of the features of :class:`ELF`, so you can :meth:`.read`
+        or :meth:`.write` or even the helpers like :meth:`.pack` or :meth:`.unpack`.
+
+        Don't forget to call :meth:`.ELF.save` to save the changes to disk.
+
+        >>> core.read(elf.address, 4)
+        b'\x7fELF'
+        >>> core.pack(core.sp, 0xdeadbeef)
+        >>> core.save()
+
+        Let's re-load it as a new :attr:`Corefile` object and have a look!
+
+        >>> core2 = Corefile(core.path)
+        >>> hex(core2.unpack(core2.sp))
+        '0xdeadbeef'
+
+        Various other mappings are available by name, for the first segment of:
+
+        * :attr:`.exe` the executable
+        * :attr:`.libc` the loaded libc, if any
+        * :attr:`.stack` the stack mapping
+        * :attr:`.vvar`
+        * :attr:`.vdso`
+        * :attr:`.vsyscall`
+
+        On Linux, 32-bit Intel binaries should have a VDSO section via :attr:`vdso`.  
+        Since our ELF is statically linked, there is no libc which gets mapped.
 
         >>> core.vdso.data[:4]
         b'\x7fELF'
-        >>> core.libc # doctest: +ELLIPSIS
-        Mapping('/lib/x86_64-linux-gnu/libc-...', ...)
+        >>> core.libc
 
-        The corefile also contains a :attr:`.Corefile.stack` property, which gives
+        But if we dump a corefile from a dynamically-linked binary, the :attr:`.libc`
+        will be loaded.
+
+        >>> process('bash').corefile.libc # doctest: +ELLIPSIS
+        Mapping('/.../libc-....so', start=0x..., stop=0x..., size=0x..., flags=..., page_offset=...)
+
+        The corefile also contains a :attr:`.stack` property, which gives
         us direct access to the stack contents.  On Linux, the very top of the stack
         should contain two pointer-widths of NULL bytes, preceded by the NULL-
         terminated path to the executable (as passed via the first arg to ``execve``).
 
-        >>> stack_end = core.exe.name.encode()
-        >>> stack_end += b'\x00' * (1+8)
-        >>> core.stack.data.endswith(stack_end)
-        True
-        >>> len(core.stack.data) == core.stack.size
-        True
+        >>> core.stack # doctest: +ELLIPSIS
+        Mapping('[stack]', start=0x..., stop=0x..., size=0x..., flags=0x6, page_offset=0x0)
 
-        We can also directly access the environment variables and arguments.
+        When creating a process, the kernel puts the absolute path of the binary and some
+        padding bytes at the end of the stack.  We can look at those by looking at 
+        ``core.stack.data``.
+
+        >>> size = len('/bin/bash-static') + 8
+        >>> core.stack.data[-size:]
+        b'bin/bash-static\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+
+        We can also directly access the environment variables and arguments, via
+        :attr:`.argc`, :attr:`.argv`, and :attr:`.env`.
 
         >>> 'HELLO' in core.env
         True
+        >>> core.string(core.env['HELLO'])
+        b'WORLD'
         >>> core.getenv('HELLO')
         b'WORLD'
         >>> core.argc
         1
         >>> core.argv[0] in core.stack
         True
-        >>> core.string(core.argv[0]) == core.exe.path.encode()
-        True
+        >>> core.string(core.argv[0]) # doctest: +ELLIPSIS
+        b'.../bin/bash-static'
 
         Corefiles can also be pulled from remote machines via SSH!
 
-        >>> s = ssh(host='example.pwnme')
+        >>> s = ssh(user='travis', host='example.pwnme', password='demopass')
         >>> _ = s.set_working_directory()
         >>> elf = ELF.from_assembly(shellcraft.trap())
         >>> path = s.upload(elf.path)
@@ -451,10 +500,6 @@ class Corefile(ELF):
         >>> io.corefile.fault_addr
         1234
 
-    Tests:
-
-        These are extra tests not meant to serve as examples.
-
         Corefile.getenv() works correctly, even if the environment variable's
         value contains embedded '='. Corefile is able to find the stack, even
         if the stack pointer doesn't point at the stack.
@@ -505,20 +550,21 @@ class Corefile(ELF):
         #: The NT_SIGINFO object
         self.siginfo = None
 
-        #: :class:`dict`: Dictionary of memory mappings from ``address`` to ``name``
+        #: :class:`list`: A list of :class:`.Mapping` objects for each loaded memory region
         self.mappings = []
 
-        #: :class:`int`: Address of the stack base
+        #: :class:`int`: A :class:`Mapping` corresponding to the stack
         self.stack    = None
 
-        #: :class:`dict`: Environment variables read from the stack.  Keys are
-        #: the environment variable name, values are the memory address of the
-        #: variable.
-        #:
-        #: Note: Use with the :meth:`.ELF.string` method to extract them.
-        #:
-        #: Note: If FOO=BAR is in the environment, self.env['FOO'] is the
-        #:       address of the string "BAR\x00".
+        """
+        Environment variables read from the stack.
+        Keys are the environment variable name, values are the memory 
+        address of the variable.
+        
+        Use :meth:`.getenv` or :meth:`.string` to retrieve the textual value.
+        
+        Note: If ``FOO=BAR`` is in the environment, ``self.env['FOO']`` is the address of the string ``"BAR\x00"``.
+        """
         self.env = {}
 
         #: :class:`int`: Pointer to envp on the stack
@@ -551,15 +597,15 @@ class Corefile(ELF):
         self.load_addr = 0
         self._address  = 0
 
-        if not self.elftype == 'CORE':
+        if self.elftype != 'CORE':
             log.error("%s is not a valid corefile" % self.file.name)
 
-        if not self.arch in prstatus_types.keys():
+        if self.arch not in prstatus_types:
             log.warn_once("%s does not use a supported corefile architecture, registers are unavailable" % self.file.name)
 
-        prstatus_type = prstatus_types.get(self.arch, None)
-        prpsinfo_type = prpsinfo_types.get(self.bits, None)
-        siginfo_type = siginfo_types.get(self.bits, None)
+        prstatus_type = prstatus_types.get(self.arch)
+        prpsinfo_type = prpsinfo_types.get(self.bits)
+        siginfo_type = siginfo_types.get(self.bits)
 
         with log.waitfor("Parsing corefile...") as w:
             self._load_mappings()
@@ -627,6 +673,13 @@ class Corefile(ELF):
                     # off the end of the stack.
                     pass
 
+            # Corefiles generated by QEMU do not have a name for the 
+            # main module mapping.
+            # Fetching self.exe will cause this to be auto-populated,
+            # and is a no-op in other cases.
+            self.exe
+
+            # Print out the nice display for the user
             self._describe_core()
 
     def _parse_nt_file(self, note):
@@ -669,8 +722,8 @@ class Corefile(ELF):
                 continue
 
             if mapping.start == self.at_sysinfo_ehdr \
-            or (not vdso and mapping.size in [0x1000, 0x2000] \
-                and mapping.flags == 5 \
+            or (not vdso and mapping.size in [0x1000, 0x2000]
+                and mapping.flags == 5
                 and self.read(mapping.start, 4) == b'\x7fELF'):
                 mapping.name = '[vdso]'
                 vdso = True
@@ -719,15 +772,30 @@ class Corefile(ELF):
     @property
     def exe(self):
         """:class:`Mapping`: First mapping for the executable file."""
+
+        # Finding the executable mapping requires knowing the entry point
+        # from the auxv
+        if not self.at_entry:
+            return None
+
+        # The entry point may not be in the first segment of a given file,
+        # but we want to find the first segment of the file -- not the segment that 
+        # contains the entrypoint.
+        first_segment_for_name = {}
+
         for m in self.mappings:
-            if self.at_entry and m.start <= self.at_entry <= m.stop:
+            first_segment_for_name.setdefault(m.name, m)
+
+        # Find which segment conains the entry point
+        for m in self.mappings:
+            if m.start <= self.at_entry < m.stop:
 
                 if not m.name and self.at_execfn:
                     m.name = self.string(self.at_execfn)
                     if not isinstance(m.name, str):
                         m.name = m.name.decode('utf-8')
 
-                return m
+                return first_segment_for_name.get(m.name, m)
 
     @property
     def pid(self):
@@ -839,7 +907,7 @@ class Corefile(ELF):
 
     @property
     def sp(self):
-        """:class:`int`: The program counter for the Corefile
+        """:class:`int`: The stack pointer for the Corefile
 
         This is a cross-platform way to get e.g. ``core.esp``, ``core.rsp``, etc.
         """
@@ -1109,11 +1177,8 @@ class Corefile(ELF):
 
         return rv
 
-    def debug(self, *a, **kw):
+    def debug(self):
         """Open the corefile under a debugger."""
-        if a or kw:
-            log.error("Arguments are not supported for %s.debug()" % self.__class__.__name__)
-
         import pwnlib.gdb
         pwnlib.gdb.attach(self, exe=self.exe.path)
 
@@ -1504,7 +1569,7 @@ class CorefileFinder(object):
             path = os.path.join(binfmt_misc, entry)
 
             try:
-                data = self.read(path)
+                data = self.read(path).decode()
             except Exception:
                 continue
 

pwnlib/elf/elf.py

@@ -90,6 +90,12 @@ log = getLogger(__name__)
 
 __all__ = ['load', 'ELF']
 
+def _iter_symbols(sec):
+    # Cache result of iter_symbols.
+    if not hasattr(sec, '_symbols'):
+        sec._symbols = list(sec.iter_symbols())
+    return iter(sec._symbols)
+
 class Function(object):
     """Encapsulates information about a function in an :class:`.ELF` binary.
 
@@ -269,6 +275,9 @@ class ELF(ELFFile):
                 self.arch = 'mips64'
                 self.bits = 64
 
+        self._sections = None
+        self._segments = None
+
         #: IntervalTree which maps all of the loaded memory segments
         self.memory = intervaltree.IntervalTree()
         self._populate_memory()
@@ -455,9 +464,6 @@ class ELF(ELFFile):
             self.checksec(*a, **kw)
         )
 
-    def __repr__(self):
-        return "ELF(%r)" % self.path
-
     def get_machine_arch(self):
         return {
             'EM_X86_64': 'amd64',
@@ -485,6 +491,13 @@ class ELF(ELFFile):
         """:class:`str`: ELF type (``EXEC``, ``DYN``, etc)"""
         return describe_e_type(self.header.e_type).split()[0]
 
+    def iter_segments(self):
+        # Yield and cache all the segments in the file
+        if self._segments is None:
+            self._segments = [self.get_segment(i) for i in range(self.num_segments())]
+
+        return iter(self._segments)
+
     @property
     def segments(self):
         """
@@ -537,6 +550,13 @@ class ELF(ELFFile):
 
         return None
 
+    def iter_sections(self):
+        # Yield and cache all the sections in the file
+        if self._sections is None:
+            self._sections = [self.get_section(i) for i in range(self.num_sections())]
+
+        return iter(self._sections)
+
     @property
     def sections(self):
         """
@@ -725,7 +745,7 @@ class ELF(ELFFile):
                     self._libs[qemu_lib] = address
 
     def _patch_elf_and_read_maps(self):
-        """patch_elf_and_read_maps(self) -> dict
+        r"""patch_elf_and_read_maps(self) -> dict
 
         Read ``/proc/self/maps`` as if the ELF were executing.
 
@@ -745,12 +765,13 @@ class ELF(ELFFile):
             These tests are just to ensure that our shellcode is correct.
 
             >>> for arch in CAT_PROC_MAPS_EXIT:
+            ...   context.clear()
             ...   with context.local(arch=arch):
             ...     sc = shellcraft.cat("/proc/self/maps")
             ...     sc += shellcraft.exit()
             ...     sc = asm(sc)
             ...     sc = enhex(sc)
-            ...     assert sc == CAT_PROC_MAPS_EXIT[arch]
+            ...     assert sc == CAT_PROC_MAPS_EXIT[arch], (arch, sc)
         """
 
         # Get our shellcode
@@ -795,7 +816,7 @@ class ELF(ELFFile):
         try:
             with context.silent:
                 io = process(path)
-                data = context._decode(io.recvall(timeout=2))
+                data = packing._decode(io.recvall(timeout=2))
         except Exception:
             log.warn_once("Injected /proc/self/maps code did not execute correctly")
             return {}
@@ -842,7 +863,7 @@ class ELF(ELFFile):
             if not isinstance(sec, SymbolTableSection):
                 continue
 
-            for sym in sec.iter_symbols():
+            for sym in _iter_symbols(sec):
                 # Avoid duplicates
                 if sym.name in self.functions:
                     continue
@@ -866,7 +887,7 @@ class ELF(ELFFile):
             if not isinstance(section, SymbolTableSection):
                 continue
 
-            for symbol in section.iter_symbols():
+            for symbol in _iter_symbols(section):
                 value = symbol.entry.st_value
                 if not value:
                     continue
@@ -907,7 +928,7 @@ class ELF(ELFFile):
         if self.statically_linked:
             return
 
-        for section in self.iter_sections():
+        for section in self.sections:
             # We are only interested in relocations
             if not isinstance(section, RelocationSection):
                 continue
@@ -961,7 +982,7 @@ class ELF(ELFFile):
 
         # Iterate over the dynamic symbol table
         dynsym = self.get_section_by_name('.dynsym')
-        symbol_iter = dynsym.iter_symbols()
+        symbol_iter = _iter_symbols(dynsym)
 
         # 'gotsym' is the index of the first GOT symbol
         gotsym = self.dynamic_value_by_tag('DT_MIPS_GOTSYM')
@@ -1003,9 +1024,10 @@ class ELF(ELFFile):
         #              In particular, this is where EBX points when it points into the GOT.
         dt_pltgot = self.dynamic_value_by_tag('DT_PLTGOT') or 0
 
-        # There are two PLTs we may need to search
+        # There are three PLTs we may need to search
         plt = self.get_section_by_name('.plt')          # <-- Functions only
         plt_got = self.get_section_by_name('.plt.got')  # <-- Functions used as data
+        plt_sec = self.get_section_by_name('.plt.sec')
         plt_mips = self.get_section_by_name('.MIPS.stubs')
 
         # Invert the GOT symbols we already have, so we can look up by address
@@ -1013,7 +1035,7 @@ class ELF(ELFFile):
         inv_symbols.update({v:k for k,v in self.symbols.items()})
 
         with context.local(arch=self.arch, bits=self.bits, endian=self.endian):
-            for section in (plt, plt_got, plt_mips):
+            for section in (plt, plt_got, plt_sec, plt_mips):
                 if not section:
                     continue
 
@@ -1026,8 +1048,8 @@ class ELF(ELFFile):
                 for address, target in sorted(res.items()):
                     self.plt[inv_symbols[target]] = address
 
-        for a,n in sorted({v:k for k,v in self.plt.items()}.items()):
-            log.debug('PLT %#x %s', a, n)
+        # for a,n in sorted({v:k for k,v in self.plt.items()}.items()):
+            # log.debug('PLT %#x %s', a, n)
 
     def _populate_kernel_version(self):
         if 'linux_banner' not in self.symbols:
@@ -1095,8 +1117,8 @@ class ELF(ELFFile):
         return_from_main = int(return_from_main[ : return_from_main.index(':') ], 16)
         return return_from_main
 
-    def search(self, needle, writable = False):
-        """search(needle, writable = False) -> generator
+    def search(self, needle, writable = False, executable = False):
+        """search(needle, writable = False, executable = False) -> generator
 
         Search the ELF's virtual address space for the specified string.
 
@@ -1109,6 +1131,7 @@ class ELF(ELFFile):
         Arguments:
             needle(str): String to search for.
             writable(bool): Search only writable sections.
+            executable(bool): Search only executable sections.
 
         Yields:
             An iterator for each virtual address that matches.
@@ -1126,11 +1149,20 @@ class ELF(ELFFile):
 
             >>> len(list(bash.search(b'GNU bash'))) > 0
             True
+
+            It is also possible to search for instructions in executable sections.
+
+            >>> binary = ELF.from_assembly('nop; mov eax, 0; jmp esp; ret')
+            >>> jmp_addr = next(binary.search(asm('jmp esp'), executable = True))
+            >>> binary.read(jmp_addr, 2) == asm('jmp esp')
+            True
         """
         load_address_fixup = (self.address - self.load_addr)
 
         if writable:
             segments = self.writable_segments
+        elif executable:
+            segments = self.executable_segments
         else:
             segments = self.segments
 
@@ -1825,7 +1857,7 @@ class ELF(ELFFile):
         # Check for Linux configuration, it must contain more than
         # just the version.
         if len(self.config) > 1:
-            config_opts = collections.defaultdict(lambda: [])
+            config_opts = collections.defaultdict(list)
             for checker in kernel_configuration:
                 result, message = checker(self.config)
 

pwnlib/elf/maps.py

@@ -17,7 +17,7 @@ CAT_PROC_MAPS_EXIT = {
         '000020e00170a0e3000000ef',
     'thumb':
         '004f01e0617073ff4fea07274fea172780b4dff8047001e06c662f6d80b4dff8047001e0632f736580b4dff8047001e02f70726f80b4684681ea010182ea02024ff0050741df05464ff00100294682ea02026ff000434ff0bb0741df'
-        '80ea00004ff0010741df',
+        '80ea00004ff0010741df00bf',
     'mips':
         '726f093c2f702935f0ffa9af7365093c632f2935f4ffa9af2f6d093c6c662935f8ffa9af8cff193c9e8f393727482003fcffa9aff0ffbd272020a003ffff0528ffff0628a50f02340c010101feff192427202003fcffa2affcffa58fffff0628ff7f073cffffe7346f1002340c010101'
         'ffff0428a10f02340c010101',

pwnlib/elf/plt.py

@@ -36,13 +36,17 @@ def emulate_plt_instructions(elf, got, address, data, targets):
         data = packing.unpack_many(data, bits=32, endian='little')
         data = packing.flat(data, bits=32, endian='big')
 
+    uc, ctx = prepare_unicorn_and_context(elf, got, address, data)
+
     # Brute force addresses, assume that PLT entry points are at 4-byte aligned
     # Do not emulate more than a handful of instructions.
     for i, pc in enumerate(range(address, address + len(data), 4)):
         if log.isEnabledFor(logging.DEBUG):
             log.debug('%s %#x', fiddling.enhex(data[i*4:(i+1) * 4]), pc)
             log.debug(elf.disasm(pc, 4))
-        target = emulate_plt_instructions_inner(elf, got, pc, data[i*4:], targets)
+
+        uc.context_restore(ctx)
+        target = emulate_plt_instructions_inner(uc, elf, got, pc, data[i*4:], targets)
 
         if target in targets:
             log.debug("%#x -> %#x", pc, target)
@@ -50,8 +54,7 @@ def emulate_plt_instructions(elf, got, address, data, targets):
 
     return rv
 
-def emulate_plt_instructions_inner(elf, got, pc, data, targets):
-    # Deferred import to not affect load time
+def prepare_unicorn_and_context(elf, got, address, data):
     import unicorn as U
 
     # Instantiate the emulator with the correct arguments for the current
@@ -88,15 +91,15 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
     uc = U.Uc(arch, mode)
 
     # Map the page of memory, and fill it with the contents
-    start = pc & (~0xfff)
-    stop  = (pc + len(data) + 0xfff) & (~0xfff)
+    start = address & (~0xfff)
+    stop  = (address + len(data) + 0xfff) & (~0xfff)
 
     if not (0 <= start <= stop <= (1 << elf.bits)):
         return None
 
     uc.mem_map(start, stop-start)
-    uc.mem_write(pc, data)
-    assert uc.mem_read(pc, len(data)) == data
+    uc.mem_write(address, data)
+    assert uc.mem_read(address, len(data)) == data
 
     # MIPS is unique in that it relies entirely on _DYNAMIC, at the beginning
     # of the GOT.  Each PLT stub loads an address stored here.
@@ -129,9 +132,17 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
         trap = packing.p32(0x34000000, endian=elf.endian)
         uc.mem_write(magic_addr, trap)
 
+    return uc, uc.context_save()
+
+def emulate_plt_instructions_inner(uc, elf, got, pc, data, targets):
+    import unicorn as U
+
     # Hook invalid addresses and any accesses out of the specified address range
     stopped_addr = []
 
+    # For MIPS. Explanation at prepare_unicorn_and_context.
+    magic_addr = 0xdbdbdbdb
+
     def hook_mem(uc, access, address, size, value, user_data):
         # Special case to allow MIPS to dereference the _DYNAMIC pointer
         # in the GOT.
@@ -142,8 +153,10 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
         uc.emu_stop()
         return False
 
-    uc.hook_add(U.UC_HOOK_MEM_READ, hook_mem, stopped_addr)
-    uc.hook_add(U.UC_HOOK_MEM_UNMAPPED, hook_mem, stopped_addr)
+    hooks = [
+        uc.hook_add(U.UC_HOOK_MEM_READ, hook_mem, stopped_addr),
+        uc.hook_add(U.UC_HOOK_MEM_UNMAPPED, hook_mem, stopped_addr)
+    ]
 
     # callback for tracing instructions
     # def hook_code(uc, address, size, user_data):
@@ -178,4 +191,7 @@ def emulate_plt_instructions_inner(elf, got, pc, data, targets):
     if stopped_addr:
         retval = stopped_addr.pop()
 
+    for hook in hooks:
+        uc.hook_del(hook)
+
     return retval