Release 3.2.0beta0 by merging branch 'stable' into beta

shellcraft.xxx.linux.sh now passes argv0

If you need shorter shellcode, use "execve" directly.

Fixes Gallopsled/pwntools#626

.gitignore

@@ -14,3 +14,4 @@ dist
 *.egg-info
 *.core
 .coverage
+.tox

.travis.yml

@@ -5,6 +5,14 @@ addons:
     packages:
     - gcc-multilib
     - gcc-4.6-arm-linux-gnueabihf
+    - lib32stdc++6
+    - ash
+    - bash
+    - dash
+    - ksh
+    - mksh
+    - zsh
+    - pandoc
 cache:
     - pip
     - directories:
@@ -13,14 +21,12 @@ cache:
         - /home/travis/virtualenv/python2.7.9/bin/
 python:
   - "2.7"
-after_success:
-  - coveralls
 before_install:
-  - source .travis_install.sh
-  - source .travis_ssh_setup.sh
+  - source travis/install.sh
+  - source travis/ssh_setup.sh
 install:
-  - pip install -e .
-  - pip install -r docs/requirements.txt
+  - pip install --upgrade --editable .
+  - pip install --requirement docs/requirements.txt
 before_script:
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
@@ -28,3 +34,6 @@ before_script:
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
 script:
   - PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+after_success:
+  - coveralls
+  - source travis/update_demo.sh

.travis_install.sh

@@ -1,88 +0,0 @@
-#!/usr/bin/env bash -e
-local_deb_extract()
-{
-    wget $1
-    ar vx *.deb
-    tar xvf data.tar.*
-    rm -f *.tar.* *deb*
-}
-
-get_binutils()
-{
-    BINUTILS_PREFIX='https://launchpad.net/~pwntools/+archive/ubuntu/binutils/+files/binutils-'
-    BINUTILS_SUFFIX='-linux-gnu_2.22-6ubuntu1.1cross0.11pwntools12~precise_amd64.deb'
-    local_deb_extract "${BINUTILS_PREFIX}${1}${BINUTILS_SUFFIX}"
-}
-
-get_qemu()
-{
-    echo "Installing qemu"
-    QEMU_URL='https://mirrors.kernel.org/ubuntu/pool/universe/q/qemu/qemu-user-static_2.6%2bdfsg-3ubuntu1_amd64.deb'
-    local_deb_extract "$QEMU_URL"
-}
-
-setup_travis()
-{
-    export PATH=$PWD/usr/bin:$PATH
-    export LD_LIBRARY_PATH=$PWD/usr/lib
-
-    if [ ! -d usr/bin ];
-    then
-        # Install our custom binutils
-        which arm-linux-as     || get_binutils arm
-        which mips-linux-as    || get_binutils mips
-        which powerpc-linux-as || get_binutils powerpc
-        which aarch64-linux-as || get_binutils aarch64
-
-        # Install the multiarch binutils
-        local_deb_extract http://mirrors.mit.edu/ubuntu/ubuntu/pool/universe/b/binutils/binutils-multiarch_2.22-6ubuntu1_amd64.deb
-    fi
-
-    if ! (which qemu-arm-static && qemu-arm-static -version | grep 2.6.0); then
-        get_qemu
-    fi
-
-    # Get rid of files we don't want cached
-    rm -rf usr/share
-
-    pushd usr/lib
-    ln -sf libbfd-2.22-multiarch.so libbfd-2.22.so
-    ln -sf libopcodes-2.22-multiarch.so libopcodes-2.22.so
-    popd
-
-    which arm-linux-gnu-as
-    which mips-linux-gnu-as
-    which powerpc-linux-gnu-as
-    which aarch64-linux-gnu-as
-    which qemu-arm-static
-}
-
-setup_linux()
-{
-    sudo apt-get install -y software-properties-common openssh-server libncurses5-dev libncursesw5-dev
-    sudo apt-add-repository --yes ppa:pwntools/binutils
-    sudo apt-get update
-    sudo apt-get install binutils-arm-linux-gnu binutils-mips-linux-gnu binutils-powerpc-linux-gnu
-}
-
-setup_osx()
-{
-    brew update
-    brew install binutils
-    brew install capstone
-}
-
-if [[ "$USER" == "travis" ]]; then
-    setup_travis
-elif [[ "$USER" == "shippable" ]]; then
-    sudo apt-get update
-    sudo apt-get install openssh-server gcc-multilib
-    sudo /usr/sbin/sshd -f /etc/ssh/sshd_config &
-    setup_travis
-elif [[ "$(uname)" == "Darwin" ]]; then
-    setup_osx
-elif [[ "$(uname)" == "Linux" ]]; then
-    setup_linux
-fi
-
-dpkg -l

CHANGELOG.md

@@ -3,17 +3,94 @@
 This changelog only includes added major features and changes. Bugfixes and
 minor changes are omitted.
 
-## 3.0.2 (September 6 2016)
+## Release History
+
+The table below shows which release corresponds to each branch, and what date the version was released.
+
+| Version          | Branch   | Release Date           |
+| ---------------- | -------- | ---------------------- |
+| [3.3.0](#330)    | `dev`    | Dec 24, 2016 (planned)
+| [3.2.0](#320)    | `beta`   | Nov 12, 2016 (planned)
+| [3.1.0](#310)    | `stable` | Oct 2, 2016
+| [3.0.4](#304)    |          | Sept 19, 2016
+| [3.0.3](#303)    |          | Sept 18, 2016
+| [3.0.2](#302)    |          | Sept 6, 2016
+| [3.0.1](#301)    |          | Aug 20, 2016
+| [3.0.0](#300)    |          | Aug 20, 2016
+| [2.2.0](#220)    |          | Jan 5, 2015
+
+## 3.2.0
+
+- [#695][695] Fixed a performance regression in `phd`.
+- [#700][700] Added missing MIPS shellcode documentation to readthedocs, and enabled unit tests
+- [#701][701] Command line tools refactored to have a common `pwn` entry point.
+    + Added an option to *not* install the traditional `asm`, `disasm`, `checksec`, etc scripts
+    + All existing tools can be accessed from the `pwn` command (e.g. `pwn asm nop`).
+- [#704][704] The `process` object has a new, optional argument `alarm` for setting a `SIGALRM` timeout for processes.
+- [#705][705] Added the Android Emulator to the test suite and Travis CI.
+    + Android Emulator is now required for the full test suite
+    + Android Emulator tests are skipped if no Android-related changes are detected
+- [#711][711] `DynELF` has a new attribute, `heap`, which leaks the current `brk` address (heap base).  This is useful for finding heap allocations with dlmalloc-derived allocators like those used by Glibc.
+- [#717][717] `sh_string` was rewritten to emit more compact and compatible strings
+    + This was achieved by embedding single-quoted non-printable literals
+    + Much more testing was added
+    + Emitted strings are no longer copy-paste compatible, but work fine with e.g. `tubes` module and the default `subprocess` module
+- [#709][709] The `adb` module now directly talks to the `adb` server process via a new module, `adb.protocol`
+    + Removes the need to shell out to `adb`
+    + Avoids version-compatibility issues with `adb` server vs. client
+- [#703][703] Added new methods to `adb`
+    + `install` - Installs an APK
+    + `uninstall` - Uninstalls a package
+    + `packages` - Lists installed packages
+- [4893819][4893819] Modified `shellcraft.sh` on all platforms to provide `argv[0]` and set `argc==1`
+    + This is needed for systems which have Busybox or other minimal shell for `/bin/sh` which does not behave well with `argc==0` or `argv[0]==NULL`.
+- [1e414af][1e414af] Added `connect()` alias for `remote()`
+    + For example, `io=connect('google.com', 80)`
+    + This also works with `tcp(...)` and `udp(...)` aliases
+- [869ec42][869ec42] Added `ssh.read()` and `ssh.write()` aiases
+- [2af55c9][2af55c9] `AdbDevice` objects exposed via e.g. `adb.devices()` now offer scoped access to all `adb` module properties
+    + It is now possible to e.g. `map(lambda d: d.process(['id']).recvall(), adb.devices())`
+
+[695]: https://github.com/Gallopsled/pwntools/pull/695
+[700]: https://github.com/Gallopsled/pwntools/pull/700
+[701]: https://github.com/Gallopsled/pwntools/pull/701
+[704]: https://github.com/Gallopsled/pwntools/pull/704
+[711]: https://github.com/Gallopsled/pwntools/pull/711
+[717]: https://github.com/Gallopsled/pwntools/pull/717
+[709]: https://github.com/Gallopsled/pwntools/pull/709
+[705]: https://github.com/Gallopsled/pwntools/pull/705
+[703]: https://github.com/Gallopsled/pwntools/pull/703
+[1e414af]: https://github.com/Gallopsled/pwntools/commit/1e414afbeb3a01242f4918f111febaa63b640eb7
+[869ec42]: https://github.com/Gallopsled/pwntools/commit/869ec42082b4b98958dfe85103da9b101dde7daa
+[4893819]: https://github.com/Gallopsled/pwntools/commit/4893819b4c23182da570e2f4ea4c14d73af2c0df
+[2af55c9]: https://github.com/Gallopsled/pwntools/commit/2af55c9bc382eca23f89bc0abc7a07c075521f94
+
+## 3.1.0
+
+A number of smaller bugfixes and documentation tweaks.
+
+## 3.0.4
+
+- Fixed a bug that made 3.0.3 uninstallable (Issue: #751, PR: #752)
+
+## 3.0.3
+
+- Fixed some performance and usability problems with the update system (Issues:
+  #723, #724, #736. PRs: #729, #738, #747).
+- Fixed a bug related to internals in pyelftools (PRs: #730, #746).
+- Fixed an issue with travis (Issue: #741, PRs: #743, #744, #745).
+
+## 3.0.2
 
 - Cherry-pick #695, as this was a regression-fix.
 - Added a fix for the update checker, as it would suggest prereleases as updates to stable releases.
 - Various documentation fixes.
 
-## 3.0.1 (August 20 2016)
+## 3.0.1
 
 A small bugfix release. There were a lot of references to the `master`-branch, however after 3.0.0 we use the names `stable`, `beta` and `dev` for our branches.
 
-## 3.0.0 (August 20 2016)
+## 3.0.0
 
 This was a large release (1305 commits since 2.2.0) with a lot of bugfixes and changes.  The Binjitsu project, a fork of Pwntools, was merged back into Pwntools.  As such, its features are now available here.
 
@@ -49,7 +126,7 @@ Android support via a new `adb` module, `context.device`, `context.adb_host`, an
 - Added `context.noptrace` setting for disabling actions which require `ptrace` support.  This is useful for turning all `gdb.debug` and `gdb.attach` options into no-ops, and can be set via the `NOPTRACE` magic command-line option.
 - Added `context.proxy` which hooks all connections and sends them to a SOCKS4/SOCKS5.  This can be set via the `PROXY` magic command-line option.
 - Added `context.randomize` to control randommization of settings like XOR keys and register ordering (default off).
-- Added `context.termianl` for setting how to launch commands in a new terminal.
+- Added `context.terminal` for setting how to launch commands in a new terminal.
 
 #### DynELF and MemLeak Module
 

ISSUE_TEMPLATE.md

@@ -0,0 +1,9 @@
+# Pwntools Issue Template
+
+Thanks for contributing to Pwntools!
+
+When reporting an issue, be sure that you are running the latest released version of pwntools (`pip install --upgrade pwntools`).
+
+Please verify that your issue occurs on 64-bit Ubuntu 14.04.  You can use the Dockerfile in `extra/docker/Dockerfile` to save time.
+
+If possible, provide a proof-of-concept which demonstrates the problem.  Include any binaries or scripts necessary to reproduce the issue.

MANIFEST.in

@@ -1 +1,7 @@
-include *.txt
+graft examples
+graft extra
+include *.md *.txt *.sh *.yml MANIFEST.in
+recursive-include docs   *.rst *.png Makefile *.py *.txt
+recursive-include pwnlib *.py *.asm *.rst *.md *.txt *.sh
+recursive-include pwn    *.py *.asm *.rst *.md *.txt *.sh
+recursive-exclude *.pyc

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,27 @@
+# Pwntools Pull Request
+
+Thanks for contributing to Pwntools!  Take a moment to look at [`CONTRIBUTING.md`][contributing] to make sure you're familiar with Pwntools development.
+
+Please provide a high-level explanation of what this pull request is for.
+
+## Testing
+
+Pull Requests that introduce new code should try to add doctests for that code.  See [`TESTING.md`][testing] for more information.
+
+## Target Branch
+
+Depending on what the PR is for, it needs to target a different branch.
+
+You can always [change the branch][change] after you create the PR if it's against the wrong branch.
+
+| Branch   | Type of PR                                                       |
+| -------- | ---------------------------------------------------------------- |
+| `dev`    | New features, and enhancements
+| `dev`    | Documentation fixes and new tests
+| `stable` | Bug fixes that affect the current `stable` branch
+| `beta`   | Bug fixes that affect the current `beta` branch, but not `stable`
+| `dev`    | Bug fixes for code that has never been released
+
+[contributing]: https://github.com/Gallopsled/pwntools/blob/dev/CONTRIBUTING.md
+[testing]: https://github.com/Gallopsled/pwntools/blob/dev/TESTING.md
+[change]: https://github.com/blog/2224-change-the-base-branch-of-a-pull-request

README.md

@@ -2,7 +2,7 @@
 ![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)
 
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![PyPI](https://img.shields.io/pypi/v/pwntools.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
+[![PyPI](https://img.shields.io/badge/pypi-v3.1.0-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
 [![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)
@@ -22,13 +22,10 @@ r.interactive()
 
 # Try It Now!
 
-You can now do a live demo of Pwntools, [right in your browser](http://demo.pwntools.com).  Alternately, you can SSH to the same host, and log in as user `zerocool` with [this private key][key] (password `i_promise_not_to_be_evil`).
-
-It will drop you into a clean, Docker-ized container.  There is nothing of value on the VPS, so please don't be evil.
-
-[key]: https://gist.githubusercontent.com/zachriggle/efa2e0080ae6de2e8344/raw/4b503e9db54f009d97477d03d4ba5678471f8ff0/id_rsa
+You can now do a live demo of Pwntools, [right in your browser](https://demo.pwntools.com).
 
 # Documentation
+
 Our documentation is available at [docs.pwntools.com](https://docs.pwntools.com/)
 
 To get you started, we've provided some example solutions for past CTF challenges in our [write-ups repository](https://github.com/Gallopsled/pwntools-write-ups).
@@ -45,9 +42,6 @@ apt-get install python2.7 python-pip python-dev git libssl-dev
 pip install --upgrade pwntools
 ```
 
-If you'd like to be an early tester of the the current beta release, add
-`--pre` to the command-line.
-
 However, some of the features (assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://docs.pwntools.com/en/stable/install.html).
 
 

TESTING.md

@@ -8,12 +8,10 @@ Pwntools makes extensive use of unit tests and integration tests to ensure every
 To run the test suite, it is beste to use Ubuntu 12.04 or 14.04, and run the following commands.  **Be aware** that this will add a user to the machine, and create a public key for SSH login!
 
 ```sh
-bash .travis_install.sh
-bash .travis_ssh_setup.sh
-pip install -e .
-pip install -r docs/requirements.txt
-cd docs
-PWNLIB_NOTERM=1 make clean doctest
+bash travis/install.sh
+bash travis/ssh_setup.sh
+pip install --upgrade --editable .
+PWNLIB_NOTERM=1 make -C docs doctest
 ```
 
 ## New Tests

docs/source/adb.rst

@@ -1,9 +1,13 @@
 .. testsetup:: *
 
    from pwn import *
+   adb = pwnlib.adb
 
 :mod:`pwnlib.adb` --- Android Debug Bridge
 =====================================================
 
-.. automodule:: pwnlib.adb
+.. automodule:: pwnlib.adb.adb
+   :members:
+
+.. automodule:: pwnlib.adb.protocol
    :members:

docs/source/commandline.rst

@@ -14,45 +14,5 @@ pwntools comes with a handful of useful command-line utilities which serve as wr
 
 .. toctree::
 
-.. autoprogram:: pwnlib.commandline.asm:parser
-   :prog: asm
-
-.. autoprogram:: pwnlib.commandline.checksec:parser
-   :prog: checksec
-
-.. autoprogram:: pwnlib.commandline.constgrep:p
-   :prog: constgrep
-
-.. autoprogram:: pwnlib.commandline.cyclic:parser
-   :prog: cyclic
-
-.. autoprogram:: pwnlib.commandline.disasm:parser
-   :prog: disasm
-
-.. autoprogram:: pwnlib.commandline.elfdiff:p
-   :prog: elfdiff
-
-.. autoprogram:: pwnlib.commandline.elfpatch:p
-   :prog: elfpatch
-
-.. autoprogram:: pwnlib.commandline.errno:parser
-   :prog: errno
-
-.. autoprogram:: pwnlib.commandline.hex:parser
-   :prog: hex
-
-.. autoprogram:: pwnlib.commandline.phd:parser
-   :prog: phd
-
-.. autoprogram:: pwnlib.commandline.pwnstrip:p
-   :prog: pwnstrip
-
-.. autoprogram:: pwnlib.commandline.scramble:parser
-   :prog: scramble
-
-.. autoprogram:: pwnlib.commandline.shellcraft:p
-   :prog: shellcraft
-
-.. autoprogram:: pwnlib.commandline.unhex:parser
-   :prog: unhex
-
+.. autoprogram:: pwnlib.commandline.main:parser
+   :prog: pwn

docs/source/flag.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+
+
+:mod:`pwnlib.flag` --- CTF Flag Management
+==========================================
+
+.. automodule:: pwnlib.flag
+   :members:

docs/source/index.rst

@@ -52,6 +52,7 @@ Each of the ``pwntools`` modules is documented here.
    encoders
    elf
    exception
+   flag
    fmtstr
    gdb
    log
@@ -67,6 +68,7 @@ Each of the ``pwntools`` modules is documented here.
    tubes
    tubes/*
    ui
+   update
    useragents
    util/*
 

docs/source/install.rst

@@ -27,14 +27,6 @@ pwntools is available as a ``pip`` package.
     $ apt-get install python2.7 python2.7-dev python-pip
     $ pip install --upgrade pwntools
 
-Alternatively you can get early access to the beta release if you want to help
-file bugs or simple get newer features:
-
-.. code-block:: bash
-
-    $ pip install --upgrade --pre pwntools
-
-
 Development
 --------------
 
@@ -43,7 +35,6 @@ If you are hacking on Pwntools locally, you'll want to do something like this:
 .. code-block:: bash
 
     $ git clone https://github.com/Gallopsled/pwntools
-    $ cd pwntools
-    $ pip install -e .
+    $ pip install --upgrade --editable ./pwntools
 
 .. _Ubuntu: https://launchpad.net/~pwntools/+archive/ubuntu/binutils

docs/source/install/binutils.rst

@@ -11,7 +11,10 @@ Building `binutils` from source takes about 60 seconds on a modern 8-core machin
 
 Ubuntu
 ^^^^^^^^^^^^^^^^
-First, add our `Personal Package Archive repository <http://binutils.pwntools.com>`__.
+
+For Ubuntu 12.04 through 15.10, you must first add the pwntools `Personal Package Archive repository <http://binutils.pwntools.com>`__.
+
+Ubuntu Xenial (16.04) has official packages for most architectures, and does not require this step.
 
 .. code-block:: bash
 

docs/source/intro.rst

@@ -176,8 +176,8 @@ loaded with useful time-saving shellcodes.
 Let's say that we want to `setreuid(getuid(), getuid())` followed by `dup`ing
 file descriptor 4 to `stdin`, `stdout`, and `stderr`, and then pop a shell!
 
-    >>> asm(shellcraft.setreuid() + shellcraft.dupsh(4)).encode('hex')
-    '6a3158cd8089c36a465889d9cd806a045b6a0359496a3f58cd8075f86a68682f2f2f73682f62696e6a0b5889e331c999cd80'
+    >>> asm(shellcraft.setreuid() + shellcraft.dupsh(4)).encode('hex') # doctest: +ELLIPSIS
+    '6a3158cd80...'
 
 
 Misc Tools

docs/source/shellcraft/mips.rst

@@ -0,0 +1,19 @@
+.. testsetup:: *
+
+   from pwn import *
+   context.clear(arch='mips')
+
+:mod:`pwnlib.shellcraft.mips` --- Shellcode for MIPS
+===========================================================
+
+:mod:`pwnlib.shellcraft.mips`
+-----------------------------
+
+.. automodule:: pwnlib.shellcraft.mips
+   :members:
+
+:mod:`pwnlib.shellcraft.mips.linux`
+-----------------------------------
+
+.. automodule:: pwnlib.shellcraft.mips.linux
+   :members:

docs/source/update.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+   from pwnlib.update import *
+
+:mod:`pwnlib.update` --- Updating Pwntools
+===================================================
+
+.. automodule:: pwnlib.update
+   :members:

docs/source/util/sh_string.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+   test = pwnlib.util.sh_string.test
+
+:mod:`pwnlib.util.sh_string` --- Shell Expansion is Hard
+===============================================================
+
+.. automodule:: pwnlib.util.sh_string
+   :members:

extra/bash_completion.d/shellcraft

@@ -43,4 +43,4 @@ _shellcraft()
     COMPREPLY=( $(compgen -W "${_shellcraft_shellcodes}" -- ${cur}) )
 }
 
-complete -F _shellcraft shellcraft
+complete -F _shellcraft pwn shellcraft

pwn/toplevel.py

@@ -6,6 +6,7 @@ import operator
 import os
 import re
 import socks
+import signal
 import string
 import struct
 import subprocess
@@ -14,6 +15,8 @@ import tempfile
 import threading
 import time
 
+from pprint import pprint
+
 import pwnlib
 from pwnlib import *
 from pwnlib.asm import *
@@ -26,6 +29,7 @@ from pwnlib.elf import ELF
 from pwnlib.elf import load
 from pwnlib.encoders import *
 from pwnlib.exception import PwnlibException
+from pwnlib.flag import *
 from pwnlib.fmtstr import FmtStr, fmtstr_payload
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak
@@ -37,7 +41,7 @@ from pwnlib.runner import *
 from pwnlib.timeout import Timeout
 from pwnlib.tubes.listen import listen
 from pwnlib.tubes.process import process
-from pwnlib.tubes.remote import remote, tcp, udp
+from pwnlib.tubes.remote import remote, tcp, udp, connect
 from pwnlib.tubes.serialtube import serialtube
 from pwnlib.tubes.ssh import ssh
 from pwnlib.tubes.tube import tube
@@ -55,6 +59,7 @@ from pwnlib.util.lists import *
 from pwnlib.util.misc import *
 from pwnlib.util.packing import *
 from pwnlib.util.proc import pidof
+from pwnlib.util.sh_string import sh_string, sh_prepare, sh_command_with
 from pwnlib.util.splash import *
 from pwnlib.util.web import *
 

pwnlib/adb/__init__.py

@@ -0,0 +1,2 @@
+from .adb import *
+from .protocol import Client

pwnlib/adb/adb.py

@@ -1,40 +1,124 @@
 """Provides utilities for interacting with Android devices via the Android Debug Bridge.
+
+# Using Android Devices with Pwntools
+
+Pwntools tries to be as easy as possible to use with Android devices.
+
+If you have only one device attached, everything "just works".
+
+If you have multiple devices, you have a handful of options to select one, or iterate
+over the devices.
+
+First and most important is the ``context.device`` property, which declares the "currently"
+seelcted device in any scope.  It can be set manually to a serial number, or to a ``Device``
+instance.
+
+.. code-block:: python
+
+    >>> context.device = ''
+
 """
+import functools
 import glob
+import logging
 import os
 import platform
 import re
 import shutil
+import stat
 import tempfile
 import time
 
 import dateutil.parser
 
-from . import atexit
-from . import tubes
-from .context import context
-from .context import LocalContext
-from .device import Device
-from .log import getLogger
-from .util import misc
+from .. import atexit
+from .. import tubes
+from ..context import LocalContext
+from ..context import context
+from ..device import Device
+from ..log import getLogger
+from ..util import misc
+from .protocol import Client
 
 log = getLogger(__name__)
 
 def adb(argv, *a, **kw):
-    """Returns the output of an ADB subcommand."""
+    r"""Returns the output of an ADB subcommand.
+
+    >>> adb.adb(['get-serialno'])
+    'emulator-5554\n'
+    """
     if isinstance(argv, (str, unicode)):
         argv = [argv]
 
     log.debug("$ " + ' '.join(context.adb + argv))
 
+    # All "adb shell" incantations should go through adb.process()
+    if argv[0] == 'shell':
+        return process(argv[1:], *a, **kw).recvall()
+
     return tubes.process.process(context.adb + argv, *a, **kw).recvall()
 
+@context.quiet
+def devices(serial=None):
+    """Returns a list of ``Device`` objects corresponding to the connected devices."""
+    with Client() as c:
+        lines = c.devices(long=True)
+    result = []
+
+    for line in lines.splitlines():
+        # Skip the first 'List of devices attached' line, and the final empty line.
+        if 'List of devices' in line or not line.strip():
+            continue
+        device = AdbDevice.from_adb_output(line)
+        if device.serial == serial:
+            return device
+        result.append(device)
+
+    return tuple(result)
+
+def current_device(any=False):
+    """Returns an ``AdbDevice`` instance for the currently-selected device
+    (via ``context.device``).
+
+    Example:
+
+        >>> device = adb.current_device(any=True)
+        >>> device
+        AdbDevice(serial='emulator-5554', type='device', port='emulator', product='sdk_phone_armv7', model='sdk phone armv7', device='generic')
+        >>> device.port
+        'emulator'
+    """
+    all_devices = devices()
+    for device in all_devices:
+        if any or device == context.device:
+            return device
+
+def with_device(f):
+    @functools.wraps(f)
+    def wrapper(*a,**kw):
+        if not context.device:
+            device = current_device(any=True)
+            if device:
+                log.warn_once('Automatically selecting device %s' % device)
+                context.device = device
+        if not context.device:
+            log.error('No devices connected, cannot invoke %s.%s' % (f.__module__, f.__name__))
+        return f(*a,**kw)
+    return wrapper
+
+
+@with_device
 def root():
-    """Restarts adbd as root."""
+    """Restarts adbd as root.
+
+    >>> adb.root()
+    """
     log.info("Enabling root on %s" % context.device)
 
     with context.quiet:
-        reply  = adb('root')
+        with Client() as c:
+            reply = c.root()
 
     if 'already running as root' in reply:
         return
@@ -46,33 +130,48 @@ def root():
     else:
         log.error("Could not run as root:\n%s" % reply)
 
+def no_emulator(f):
+    @functools.wraps(f)
+    def wrapper(*a,**kw):
+        c = current_device()
+        if c and c.port == 'emulator':
+            log.error("Cannot invoke %s.%s on an emulator." % (f.__module__, f.__name__))
+        return f(*a,**kw)
+    return wrapper
 
+@no_emulator
+@with_device
 def reboot(wait=True):
-    """Reboots the device."""
+    """Reboots the device.
+    """
     log.info('Rebooting device %s' % context.device)
 
-    with context.quiet:
-        adb('reboot')
+    with Client() as c:
+        c.reboot()
 
     if wait:
         wait_for_device()
 
+@no_emulator
+@with_device
 def reboot_bootloader():
-    """Reboots the device to the bootloader."""
+    """Reboots the device to the bootloader.
+    """
     log.info('Rebooting %s to bootloader' % context.device)
 
-    with context.quiet:
-        adb('reboot-bootloader')
+    with Client() as c:
+        c.reboot_bootloader()
 
 class AdbDevice(Device):
     """Encapsulates information about a connected device."""
-    def __init__(self, serial, type, port, product='unknown', model='unknown', device='unknown'):
+    def __init__(self, serial, type, port=None, product='unknown', model='unknown', device='unknown', features=None, **kw):
         self.serial  = serial
         self.type    = type
         self.port    = port
         self.product = product
         self.model   = model.replace('_', ' ')
         self.device  = device
+        self.os      = 'android'
 
         if product == 'unknown':
             return
@@ -85,6 +184,19 @@ class AdbDevice(Device):
             self.bits = context.bits
             self.endian = context.endian
 
+        if self.port == 'emulator':
+            emulator, port = self.serial.split('-')
+            port = int(port)
+            try:
+                with remote('localhost', port, level='error') as r:
+                    r.recvuntil('OK')
+                    r.recvline() # Rest of the line
+                    r.sendline('avd name')
+                    self.avd = r.recvline().strip()
+            except:
+                pass
+            # r = remote('localhost')
+
     def __str__(self):
         return self.serial
 
@@ -102,42 +214,83 @@ class AdbDevice(Device):
         ZX1G22LM7G             device usb:336789504X product:shamu model:Nexus_6 device:shamu features:cmd,shell_v2
         84B5T15A29020449       device usb:336855040X product:angler model:Nexus_6P device:angler
         0062741b0e54b353       unauthorized usb:337641472X
+        emulator-5554          offline
+        emulator-5554          device product:sdk_phone_armv7 model:sdk_phone_armv7 device:generic
         """
 
-        # The last few fields need to be split at colons.
-        split  = lambda x: x.split(':')[-1]
-        fields[3:] = list(map(split, fields[3:]))
+        fields = line.split()
 
-        return AdbDevice(*fields[:6])
+        serial = fields[0]
+        type   = fields[1]
+        kwargs = {}
 
-@context.quiet
-def devices(serial=None):
-    """Returns a list of ``Device`` objects corresponding to the connected devices."""
-    lines = adb(['devices', '-l'])
-    result = []
+        if serial.startswith('emulator-'):
+            kwargs['port'] = 'emulator'
 
-    for line in lines.splitlines():
-        # Skip the first 'List of devices attached' line, and the final empty line.
-        if 'List of devices' in line or not line.strip():
-            continue
-        device = AdbDevice.from_adb_output(line)
-        if device.serial == serial:
-            return device
-        result.append(device)
+        for field in fields[2:]:
+            k,v = field.split(':')
+            kwargs[k] = v
 
-    return tuple(result)
+        return AdbDevice(serial, type, **kwargs)
+
+    def __wrapped(self, function):
+        """Wrapps a callable in a scope which selects the current device."""
+        @functools.wraps(function)
+        def wrapper(*a, **kw):
+            with context.local(device=self):
+                return function(*a,**kw)
+        return wrapper
+
+    def __getattr__(self, name):
+        """Provides scoped access to ``adb`` module propertise, in the context
+        of this device.
+
+        >>> property = 'ro.build.fingerprint'
+        >>> device = adb.current_device()
+        >>> adb.getprop(property) == device.getprop(property)
+        True
+        """
+        with context.local(device=self):
+            g = globals()
+
+            if name not in g:
+                raise AttributeError('%r object has no attribute %r' % (type(self).__name__,name))
+
+            value = g[name]
+
+        if not hasattr(value, '__call__'):
+            return value
+
+        return self.__wrapped(value)
 
 @LocalContext
 def wait_for_device(kick=False):
-    """Waits for a device to be connected."""
+    """Waits for a device to be connected.
+
+    By default, waits for the currently-selected device (via ``context.device``).
+    To wait for a specific device, set ``context.device``.
+    To wait for *any* device, clear ``context.device``.
+
+    Return:
+        An ``AdbDevice`` instance for the device.
+
+    Examples:
+
+        >>> device = adb.wait_for_device()
+    """
     with log.waitfor("Waiting for device to come online") as w:
-        with context.quiet:
+        with Client() as c:
             if kick:
                 try:
-                    adb(['reconnect'])
+                    c.reconnect()
                 except Exception:
                     pass
-            adb('wait-for-device')
+
+            serial = ''
+            if context.device:
+                serial = str(context.device)
+
+            c.wait_for_device(serial)
 
         for device in devices():
             if context.device == device:
@@ -156,43 +309,66 @@ def wait_for_device(kick=False):
 
             return context.device
 
+@with_device
 def disable_verity():
     """Disables dm-verity on the device."""
     with log.waitfor("Disabling dm-verity on %s" % context.device) as w:
         root()
 
-        with context.quiet:
-            reply = adb('disable-verity')
+        with Client() as c:
+            reply = c.disable_verity()
 
         if 'Verity already disabled' in reply:
             return
         elif 'Now reboot your device' in reply:
             reboot(wait=True)
+        elif '0006closed' in reply:
+            return # Emulator doesnt support Verity?
         else:
             log.error("Could not disable verity:\n%s" % reply)
 
-
+@with_device
 def remount():
     """Remounts the filesystem as writable."""
     with log.waitfor("Remounting filesystem on %s" % context.device) as w:
         disable_verity()
         root()
 
-        with context.quiet:
-            reply = adb('remount')
+        with Client() as c:
+            reply = c.remount()
 
         if 'remount succeeded' not in reply:
             log.error("Could not remount filesystem:\n%s" % reply)
 
+@with_device
 def unroot():
     """Restarts adbd as AID_SHELL."""
     log.info("Unrooting %s" % context.device)
     with context.quiet:
-        reply  = adb('unroot')
+        with Client() as c:
+            reply  = c.unroot()
+
+    if '0006closed' == reply:
+        return # Emulator doesnt care
 
     if 'restarting adbd as non root' not in reply:
-        log.error("Could not run as root:\n%s" % reply)
+        log.error("Could not unroot:\n%s" % reply)
 
+def _create_adb_push_pull_callback(w):
+    def callback(filename, data, size, chunk, chunk_size):
+        have = len(data) + len(chunk)
+        if size == 0:
+            size = '???'
+            percent = '???'
+        else:
+            percent = int(100 * have // size)
+            size = misc.size(size)
+        have = misc.size(have)
+        w.status('%s/%s (%s%%)' % (have, size, percent))
+        return True
+    return callback
+
+@with_device
 def pull(remote_path, local_path=None):
     """Download a file from the device.
 
@@ -200,77 +376,130 @@ def pull(remote_path, local_path=None):
         remote_path(str): Path or directory of the file on the device.
         local_path(str): Path to save the file to.
             Uses the file's name by default.
+
+    Return:
+        The contents of the file.
+
+    Example:
+
+        >>> _=adb.pull('/proc/version', './proc-version')
+        >>> print read('./proc-version') # doctest: +ELLIPSIS
+        Linux version ...
     """
     if local_path is None:
         local_path = os.path.basename(remote_path)
 
-    msg = "Pulling %r from %r" % (remote_path, local_path)
+    msg = "Pulling %r to %r" % (remote_path, local_path)
 
-    if context.log_level == 'debug':
+    if log.isEnabledFor(logging.DEBUG):
         msg += ' (%s)' % context.device
 
-    result = ''
     with log.waitfor(msg) as w:
-        with context.quiet:
-            args = context.adb + ['pull', remote_path, local_path]
-            io = tubes.process.process(args)
-            result = io.recvall()
+        data = read(remote_path, callback=_create_adb_push_pull_callback(w))
+        misc.write(local_path, data)
 
-            if 0 != io.poll(block=True):
-                log.error(result)
-
-    return result
+    return data
 
+@with_device
 def push(local_path, remote_path):
     """Upload a file to the device.
 
     Arguments:
         local_path(str): Path to the local file to push.
         remote_path(str): Path or directory to store the file on the device.
+
+    Example:
+
+        >>> write('./filename', 'contents')
+        >>> _=adb.push('./filename', '/data/local/tmp')
+        >>> adb.read('/data/local/tmp/filename')
+        'contents'
+        >>> adb.push('./filename', '/does/not/exist')
+        Traceback (most recent call last):
+        ...
+        PwnlibException: Could not stat '/does/not/exist'
     """
     msg = "Pushing %r to %r" % (local_path, remote_path)
+    remote_filename = os.path.basename(local_path)
 
-    if context.log_level == 'debug':
+    if log.isEnabledFor(logging.DEBUG):
         msg += ' (%s)' % context.device
 
     with log.waitfor(msg) as w:
-        with context.quiet:
-            args = context.adb + ['push', local_path, remote_path]
-            io = tubes.process.process(args)
-            result = io.recvall()
+        with Client() as c:
 
-            if 0 != io.poll(block=True):
-                log.error(result)
+            # We need to discover whether remote_path is a directory or not.
+            # If we cannot stat the full path, assume it's a path-plus-filename,
+            # where the filename does not exist.
+            stat_ = c.stat(remote_path)
+            if not stat_:
+                remote_filename = os.path.basename(remote_path)
+                remote_path = os.path.dirname(remote_path)
+                stat_ = c.stat(remote_path)
 
-    return result
+            # If we can't find the exact path, or its parent directory, bail!
+            if not stat_:
+                log.error('Could not stat %r' % remote_path)
 
+            # If we found the parent directory, append the filename
+            mode = stat_['mode']
+            if stat.S_ISDIR(mode):
+                remote_path = os.path.join(remote_path, remote_filename)
+
+            return c.write(remote_path,
+                           misc.read(local_path),
+                           callback=_create_adb_push_pull_callback(w))
 @context.quiet
-def read(path, target=None):
+@with_device
+def read(path, target=None, callback=None):
     """Download a file from the device, and extract its contents.
 
     Arguments:
         path(str): Path to the file on the device.
         target(str): Optional, location to store the file.
             Uses a temporary file by default.
+        callback(callable): See the documentation for
+            ``adb.protocol.Client.read``.
+
+    Examples:
+
+        >>> print adb.read('/proc/version') # doctest: +ELLIPSIS
+        Linux version ...
+        >>> adb.read('/does/not/exist')
+        Traceback (most recent call last):
+        ...
+        PwnlibException: Could not stat '/does/not/exist'
     """
-    with tempfile.NamedTemporaryFile() as temp:
-        target = target or temp.name
-        pull(path, target)
-        result = misc.read(target)
-    return result
+    with Client() as c:
+        stat = c.stat(path)
+        if not stat:
+            log.error('Could not stat %r' % path)
+        data = c.read(path, stat['size'], callback=callback)
+
+    if target:
+        misc.write(target, data)
+
+    return data
 
 @context.quiet
+@with_device
 def write(path, data=''):
     """Create a file on the device with the provided contents.
 
     Arguments:
         path(str): Path to the file on the device
         data(str): Contents to store in the file
+
+    Examples:
+
+        >>> adb.write('/dev/null', 'data')
+        >>> adb.write('/data/local/tmp/')
     """
     with tempfile.NamedTemporaryFile() as temp:
         misc.write(temp.name, data)
         push(temp.name, path)
 
+@with_device
 def process(argv, *a, **kw):
     """Execute a process on the device.
 
@@ -278,35 +507,62 @@ def process(argv, *a, **kw):
 
     Returns:
         A ``process`` tube.
+
+    Examples:
+
+        >>> adb.root()
+        >>> print adb.process(['cat','/proc/version']).recvall() # doctest: +ELLIPSIS
+        Linux version ...
     """
-    argv = argv or []
     if isinstance(argv, (str, unicode)):
         argv = [argv]
 
-    display = argv
-    argv = context.adb + ['shell'] + argv
+    message = "Starting %s process %r" % ('Android', argv[0])
 
-    kw.setdefault('display', display)
-    kw.setdefault('where', 'Android')
+    if log.isEnabledFor(logging.DEBUG):
+        if argv != [argv[0]]: message += ' argv=%r ' % argv
 
-    return tubes.process.process(argv, *a, **kw)
+    with log.progress(message) as p:
+        return Client().execute(argv)
 
+@with_device
 def interactive(**kw):
     """Spawns an interactive shell."""
     return shell(**kw).interactive()
 
+@with_device
 def shell(**kw):
     """Returns an interactive shell."""
-    return process([], **kw)
+    return process(['sh', '-i'], **kw)
 
-@context.quiet
+@with_device
 def which(name):
-    """Retrieves the full path to a binary in ``PATH`` on the device"""
-    return process(['which', name]).recvall().strip()
+    """Retrieves the full path to a binary in ``PATH`` on the device
 
+    >>> adb.which('sh')
+    '/system/bin/sh'
+    """
+    # Unfortunately, there is no native 'which' on many phones.
+    which_cmd = '''
+IFS=:
+BINARY=%s
+P=($PATH)
+for path in "${P[@]}"; do \
+    if [ -e "$path/$BINARY" ]; then \
+        echo "$path/$BINARY";
+        break
+    fi
+done
+''' % name
+
+    which_cmd = which_cmd.strip()
+    return process(['sh','-c',which_cmd]).recvall().strip()
+
+@with_device
 def whoami():
-    return process(['whoami']).recvall().strip()
+    return process(['sh','-ic','echo $USER']).recvall().strip()
 
+@with_device
 def forward(port):
     """Sets up a port to forward to the device."""
     tcp_port = 'tcp:%s' % port
@@ -314,6 +570,7 @@ def forward(port):
     atexit.register(lambda: adb(['forward', '--remove', tcp_port]))
 
 @context.quiet
+@with_device
 def logcat(stream=False):
     """Reads the system log file.
 
@@ -331,8 +588,9 @@ def logcat(stream=False):
     if stream:
         return process(['logcat'])
     else:
-        return adb(['logcat', '-d'])
+        return process(['logcat', '-d']).recvall()
 
+@with_device
 def pidof(name):
     """Returns a list of PIDs for the named process."""
     with context.quiet:
@@ -340,13 +598,15 @@ def pidof(name):
         data = io.recvall().split()
     return list(map(int, data))
 
+@with_device
 def proc_exe(pid):
     """Returns the full path of the executable for the provided PID."""
     with context.quiet:
-        io  = process(['readlink','-e','/proc/%d/exe' % pid])
+        io  = process(['realpath','/proc/%d/exe' % pid])
         data = io.recvall().strip()
     return data
 
+@with_device
 def getprop(name=None):
     """Reads a properties from the system property store.
 
@@ -381,24 +641,24 @@ def getprop(name=None):
 
     return props
 
+@with_device
 def setprop(name, value):
     """Writes a property to the system property store."""
     return process(['setprop', name, value]).recvall().strip()
 
+@with_device
 def listdir(directory='/'):
     """Returns a list containing the entries in the provided directory.
 
     Note:
-        Because ``adb shell`` is used to retrieve the listing, shell
-        environment variable expansion and globbing are in effect.
+        This uses the SYNC LIST functionality, which runs in the adbd
+        SELinux context.  If adbd is running in the su domain ('adb root'),
+        this behaves as expected.
+
+        Otherwise, less files may be returned due to restrictive SELinux
+        policies on adbd.
     """
-    io = process(['find', directory, '-maxdepth', '1', '-print0'])
-    data = io.recvall()
-    paths = filter(len, data.split('\x00'))
-    relpaths = [os.path.relpath(path, directory) for path in paths]
-    if '.' in relpaths:
-        relpaths.remove('.')
-    return relpaths
+    return list(sorted(Client().list(directory)))
 
 def fastboot(args, *a, **kw):
     """Executes a fastboot command.
@@ -411,25 +671,30 @@ def fastboot(args, *a, **kw):
         log.error("Unknown device")
     return tubes.process.process(['fastboot', '-s', serial] + list(args), **kw).recvall()
 
+@with_device
 def fingerprint():
     """Returns the device build fingerprint."""
     return str(properties.ro.build.fingerprint)
 
+@with_device
 def product():
     """Returns the device product identifier."""
     return str(properties.ro.build.product)
 
+@with_device
 def build():
     """Returns the Build ID of the device."""
     return str(properties.ro.build.id)
 
+@with_device
+@no_emulator
 def unlock_bootloader():
     """Unlocks the bootloader of the device.
 
     Note:
         This requires physical interaction with the device.
     """
-    adb(['reboot-bootloader'])
+    Client().reboot_bootloader()
     fastboot(['oem', 'unlock'])
     fastboot(['continue'])
 
@@ -631,19 +896,29 @@ def _generate_ndk_project(file_list, abi='arm-v7a', platform_version=21):
 def compile(source):
     """Compile a source file or project with the Android NDK."""
 
-    # Ensure that we can find the NDK.
-    ndk = os.environ.get('NDK', None)
-    if ndk is None:
-        log.error('$NDK must be set to the Android NDK directory')
-    ndk_build = os.path.join(ndk, 'ndk-build')
+    ndk_build = misc.which('ndk-build')
+    if not ndk_build:
+        # Ensure that we can find the NDK.
+        ndk = os.environ.get('NDK', None)
+        if ndk is None:
+            log.error('$NDK must be set to the Android NDK directory')
+        ndk_build = os.path.join(ndk, 'ndk-build')
 
     # Determine whether the source is an NDK project or a single source file.
     project = find_ndk_project_root(source)
 
     if not project:
-        project = _generate_ndk_project(source,
-                                        str(properties.ro.product.cpu.abi),
-                                        str(properties.ro.build.version.sdk))
+        # Realistically this should inherit from context.arch, but
+        # this works for now.
+        abi = 'armeabi-v7a'
+        sdk = '21'
+
+        # If we have an atatched device, use its settings.
+        if context.device:
+            abi = str(properties.ro.product.cpu.abi)
+            sdk = str(properties.ro.build.version.sdk)
+
+        project = _generate_ndk_project(source, abi, sdk)
 
     # Remove any output files
     lib = os.path.join(project, 'libs')
@@ -687,6 +962,7 @@ class Partitions(object):
         return list(self)
 
     @context.quiet
+    @with_device
     def __iter__(self):
         root()
 
@@ -721,3 +997,51 @@ class Partitions(object):
         return Partition(devpath, attr, int(blocks))
 
 partitions = Partitions()
+
+def install(apk, *arguments):
+    """Install an APK onto the device.
+
+    This is a wrapper around 'pm install', which backs 'adb install'.
+
+    Arguments:
+        apk(str): Path to the APK to intall (e.g. ``'foo.apk'``)
+        arguments: Supplementary arguments to 'pm install',
+            e.g. ``'-l', '-g'``.
+    """
+    if not apk.endswith('.apk'):
+        log.error("APK must have .apk extension")
+
+    basename = os.path.basename(apk)
+    target_path = '/data/local/tmp/{}.apk'.format(basename)
+
+    with log.progress("Installing APK {}".format(basename)) as p:
+        with context.quiet:
+            p.status('Copying APK to device')
+            push(apk, target_path)
+
+            p.status('Installing')
+            result = process(['pm', 'install-create', target_path] + list(arguments)).recvall()
+
+            status = result.splitlines()[-1]
+            if 'Success' not in status:
+                log.error(status)
+
+def uninstall(package, *arguments):
+    """Uninstall an APK from the device.
+
+    This is a wrapper around 'pm uninstall', which backs 'adb uninstall'.
+
+    Arguments:
+        package(str): Name of the package to uninstall (e.g. ``'com.foo.MyPackage'``)
+        arguments: Supplementary arguments to ``'pm install'``, e.g. ``'-k'``.
+    """
+    with log.progress("Installing package {}".format(package)):
+        with context.quiet:
+            return process(['pm','uninstall',apk] + list(arguments)).recvall()
+
+@context.quiet
+def packages():
+    """Returns a list of packages installed on the system"""
+    packages = process(['pm', 'list', 'packages']).recvall()
+    return [line.split('package:', 1)[-1] for line in packages.splitlines()]
+

pwnlib/adb/protocol.py

@@ -0,0 +1,495 @@
+"""
+Implementation of the Android Debug Bridge (ADB) protocol, as far as Binjitsu needs it.
+
+Documentation is available here:
+https://android.googlesource.com/platform/system/core/+/master/adb/protocol.txt
+"""
+import functools
+import time
+
+from ..context import context
+from ..log import Logger
+from ..log import getLogger
+from ..tubes.listen import listen
+from ..tubes.process import process
+from ..tubes.remote import remote
+from ..util.lists import group
+from ..util.misc import size
+from ..util.packing import p32
+from ..util.proc import pidof
+from ..util.sh_string import sh_string
+
+log = getLogger(__name__)
+
+def pack(val):
+    return '%04x' % val
+
+def unpack(val):
+    return int(val, 16)
+
+OKAY = "OKAY"
+FAIL = "FAIL"
+
+class Message(object):
+    """An ADB hex-length-prefixed message"""
+    def __init__(self, string):
+        self.string = string
+    def __str__(self):
+        return ('%04x' % len(self.string)) + self.string
+    def __flat__(self):
+        return str(self)
+
+class Connection(remote):
+    """Connection to the ADB server"""
+    def __init__(self, host, port, level=None, *a, **kw):
+
+        # Try to make sure ADB is running if it's on the default host and port.
+        if host == context.defaults['adb_host'] \
+        and port == context.defaults['adb_port']:
+            process(context.adb + ['start-server'], level='error').wait_for_close()
+
+        with context.quiet:
+            super(Connection, self).__init__(host, port, level=level, *a, **kw)
+
+        self._executable = None
+        self._argv       = None
+        self._pid        = None
+        self._cwd        = None
+        self._env        = None
+
+    def close(self):
+        with context.quiet:
+            super(Connection, self).close()
+
+    def adb_send(self, message):
+        self.send(str(Message(message)))
+        return self.recvn(4)
+
+    def adb_unpack(self):
+        return unpack(self.recvn(4))
+
+    def flat(self, *a, **kw):
+        kw.setdefault('word_size', 32)
+        return super(Connection, self).flat(*a, **kw)
+
+class Process(Connection):
+    """Duck-typed ``tubes.remote`` object to add properties of a ``tubes.process``"""
+
+class Client(Logger):
+    """ADB Client"""
+    def __init__(self, level=None):
+        super(Client, self).__init__()
+
+        if level is not None:
+            self.setLevel(level)
+
+        self.host = context.adb_host
+        self.port = context.adb_port
+        self._c   = None
+
+    @property
+    def c(self):
+        """Client's connection to the ADB server"""
+        if not self._c:
+            self._c = Connection(self.host, self.port, level=self.level)
+        return self._c
+
+    def _autoclose(fn):
+        """Decorator which automatically closes the connection to the ADB server
+        after calling the decorated function."""
+        @functools.wraps(fn)
+        def wrapper(self, *a, **kw):
+            rv = fn(self, *a, **kw)
+            if self._c:
+                self._c.close()
+                self._c = None
+            return rv
+        return wrapper
+
+    def _with_transport(fn):
+        """Decorator which automatically selects a device transport before calling
+        the decorated function, and closes the connection afterward."""
+        @functools.wraps(fn)
+        def wrapper(self, *a, **kw):
+            self.transport()
+            rv = fn(self, *a, **kw)
+            if self._c:
+                self._c.close()
+                self._c = None
+            return rv
+        return wrapper
+
+    def send(self, *a, **kw):
+        """Sends data to the ADB server"""
+        return self.c.adb_send(*a, **kw)
+
+    def unpack(self, *a, **kw):
+        """Receives a hex-ascii packed integer from the ADB server"""
+        return self.c.adb_unpack(*a, **kw)
+
+    def recvl(self):
+        """Receives a length-prefixed data buffer from the ADB server"""
+        length = self.c.adb_unpack()
+        return self.c.recvn(length)
+
+    @_autoclose
+    def kill(self):
+        """Kills the remote ADB server"""
+        try:
+            self.send('host:kill')
+        except EOFError:
+            pass
+
+    def version(self):
+        """
+        Returns:
+            Tuple containing the ``(major, minor)`` version from the ADB server
+
+        Example:
+
+            >>> adb.protocol.Client().version() # doctest: +SKIP
+            (4, 36)
+        """
+        response = self.send('host:version')
+        if response == OKAY:
+            return (self.c.adb_unpack(), self.c.adb_unpack())
+        self.error("Could not fetch version")
+
+    @_autoclose
+    def devices(self, long=False):
+        """
+        Arguments:
+            long(bool): If ``True``, fetch the long-format listing.
+        Returns:
+            String representation of all available devices.
+        """
+        msg = 'host:devices'
+        if long:
+            msg += '-l'
+        response = self.send(msg)
+        if response == 'OKAY':
+            return self.recvl()
+        self.error("Could not enumerate devices")
+
+    @_autoclose
+    def track_devices(self):
+        """
+        Returns:
+            Generator which returns a short-format listing of available
+            devices each time a device state changes.
+        """
+        self.send('host:track-devices')
+        while True:
+            yield self.recvl()
+
+    def transport(self, serial=None):
+        """Sets the Transport on the rmeote device.
+
+        Examples:
+
+            >>> adb.protocol.Client().transport()
+        """
+        msg = 'host:transport:%s' % (serial or context.device)
+        if self.send(msg) == FAIL:
+            self.error("Could not set transport to %r" % serial)
+
+    @_autoclose
+    @_with_transport
+    def execute(self, argv):
+        r"""Executes a program on the device.
+
+        Returns:
+            A ``tube`` which is connected to the process.
+
+        Examples:
+
+            >>> adb.protocol.Client().execute(['echo','hello']).recvall()
+            'hello\n'
+        """
+        self.transport(context.device)
+        if isinstance(argv, str):
+            argv = [argv]
+        argv = list(map(sh_string, argv))
+        cmd = 'exec:%s' % (' '.join(argv))
+        if OKAY == self.send(cmd):
+            rv = self._c
+            self._c = None
+            return rv
+
+    def _basic_wrapper(string):
+        @_autoclose
+        @_with_transport
+        def wrapper(self):
+            self.send(string)
+            return self.c.recvall()
+        return wrapper
+
+    @_autoclose
+    @_with_transport
+    def remount(self):
+        self.send('remount:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def root(self):
+        self.send('root:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def unroot(self):
+        self.send('unroot:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def disable_verity(self):
+        self.send('disable-verity:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def enable_verity(self):
+        self.send('enable-verity:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def reconnect(self):
+        self.send('reconnect:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def reboot(self):
+        self.send('reboot:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def reboot_bootloader(self):
+        self.send('reboot:bootloader')
+        return self.c.recvall()
+
+    @_autoclose
+    def wait_for_device(self, serial=''):
+        response = self.send('host-serial:%s:wait-for-any-device' % serial)
+
+        # The first OKAY is that the command was understood
+        if response != 'OKAY':
+            self.error("An error occurred while waiting for device with serial %r" % serial)
+
+        # The second OKAY is that the device is available
+        response = self.c.recvn(4)
+        if response != 'OKAY':
+            self.error("An error occurred while waiting for device with serial %r" % serial)
+
+    def _sync(fn):
+        """Decorator which enters 'sync:' mode to the selected transport,
+        then invokes the decorated funciton."""
+        @functools.wraps(fn)
+        def wrapper(self, *a, **kw):
+            rv = None
+            if FAIL != self.send('sync:'):
+                rv = fn(self, *a, **kw)
+            return rv
+        return wrapper
+
+    @_with_transport
+    @_sync
+    def list(self, path):
+        """Execute the ``LIST`` command of the ``SYNC`` API.
+
+        Arguments:
+            path(str): Path of the directory to list.
+
+        Return:
+            A dictionary, where the keys are relative filenames,
+            and the values are a dictionary containing the same
+            values as ``stat()`` supplies.
+
+        Note:
+            In recent releases of Android (e.g. 7.0), the domain that
+            adbd executes from does not have access to everything that
+            the shell user does.
+
+            Because of this, while the shell user can get listings of
+            e.g. the root directory ('/'), adbd cannot.
+
+            The SYNC APIs are executed within the adbd context, not the
+            shell user context.
+
+            This issue is not a problem if the phone is rooted via
+            'adb root', since adbd then runs in the ``su`` domain.
+
+        Examples:
+
+            >>> pprint(adb.Client().list('/data/user'))
+            {'0': {'mode': 41471, 'size': 11, 'time': ...}}
+            >>> adb.Client().list('/does/not/exist')
+            {}
+        """
+        self.c.flat('LIST', len(path), path)
+        files = {}
+        while True:
+            response = self.c.recvn(4)
+
+            if response == 'DONE':
+                break
+
+            if response != 'DENT':
+                self.error('Unexpected response: %r' % response)
+
+            mode = self.c.u32()
+            size = self.c.u32()
+            time = self.c.u32()
+            name = self.c.recvn(self.c.u32())
+
+            # Ignore the current directory and parent
+            if name in ('', '.', '..'):
+                continue
+
+            files[name] = {'mode': mode,
+                           'size': size,
+                           'time': time}
+
+        return files
+
+    @_with_transport
+    @_sync
+    def stat(self, path):
+        """Execute the ``STAT`` command of the ``SYNC`` API.
+
+        Arguments:
+            path(str): Path to the file to stat.
+
+        Return:
+            On success, a dictionary mapping the values returned.
+            If the file cannot be ``stat()``ed, None is returned.
+
+        Example:
+
+            >>> expected = {'mode': 16749, 'size': 0, 'time': 0}
+            >>> adb.protocol.Client().stat('/proc')           == expected
+            True
+            >>> adb.protocol.Client().stat('/does/not/exist') == None
+            True
+        """
+        self.c.flat('STAT', len(path), path)
+        if self.c.recvn(4) != 'STAT':
+            self.error("An error occured while attempting to STAT a file.")
+
+        mode = self.c.u32()
+        size = self.c.u32()
+        time = self.c.u32()
+
+        if (mode,size,time) == (0,0,0):
+            return None
+
+        return {'mode': mode, 'size': size, 'time': time}
+
+    @_with_transport
+    @_sync
+    def write(self, path, data, mode=0o755, timestamp=None, callback=None):
+        path += ',' + str(mode)
+        self.c.flat('SEND', len(path), path)
+
+        sent = 0
+
+        # Data needs to be broken up into chunks!
+        for chunk in group(0x10000, data):
+            self.c.flat('DATA', len(chunk), chunk)
+            if callback:
+                callback(path, data[:sent], len(data), chunk, len(chunk))
+            sent += len(chunk)
+
+        # Send completion notification and timestamp
+        if timestamp is None:
+            timestamp = int(time.time())
+        self.c.flat('DONE', timestamp)
+
+        result = self.c.recvn(4)
+        if result != 'OKAY':
+            log.error("Sync write failed: %r (expected OKAY)" % result)
+
+        return
+
+    @_with_transport
+    @_sync
+    def read(self, path, filesize=0, callback=lambda *a: True):
+        """Execute the ``READ`` command of the ``SYNC`` API.
+
+        Arguments:
+            path(str): Path to the file to read
+            filesize(int): Size of the file, in bytes.  Optional.
+            callback(callable): Callback function invoked as data
+                becomes available.  Arguments provided are:
+
+                - File path
+                - All data
+                - Expected size of all data
+                - Current chunk
+                - Expected size of chunk
+
+        Return:
+            The data received as a string.
+        """
+        self.c.send('RECV' + p32(len(path)) + path)
+
+        # Accumulate all data here
+        all_data = ''
+
+        while True:
+            magic = self.c.recvn(4)
+
+            # adbd says there is no more data to send
+            if magic == 'DONE':
+                break
+
+            if magic == 'FAIL':
+                self.error('Could not read file %r: Got FAIL.' % path)
+
+            # did we expect to be done?
+            if magic != 'DATA':
+                self.error('Error after file read: %r (expected DATA)' % magic)
+
+            # receive all of the data in the chunk
+            chunk_size = self.c.u32()
+            chunk_data  = ''
+            while len(chunk_data) != chunk_size:
+                chunk_data += self.c.recv(chunk_size - len(chunk_data))
+
+                if callback:
+                    callback(path,
+                             all_data,
+                             filesize,
+                             chunk_data,
+                             chunk_size)
+
+            # add the chunk onto what we have
+            all_data += chunk_data
+
+        zero = self.c.u32()
+        if zero != 0:
+            self.error('Error after file read: %r (expected ZERO)' % zero)
+
+        return all_data
+
+    @_with_transport
+    def forward(self, device, host_proto, host_port, device_proto, device_port):
+        self.send('host:forward:%s:%s;%s:%s' % (host_proto, host_port, device_proto, device_port))
+        self.c.recvall()
+
+    def __enter__(self, *a, **kw):
+        return self
+
+    @_autoclose
+    def __exit__(self, *a, **kw): pass
+
+def proxy(port=9999):
+    """Starts an ADB proxy on the specified port, for debugging purposes."""
+    l = listen(port)
+    l.wait_for_connection()
+    r = remote(context.adb_host, context.adb_port, level='debug')
+    l <> r

pwnlib/asm.py

@@ -27,8 +27,8 @@ Assembly
     Finally, :func:`asm` is used to assemble shellcode provided by ``pwntools``
     in the :mod:`shellcraft` module.
 
-        >>> asm(shellcraft.sh())
-        'jhh///sh/binj\x0bX\x89\xe31\xc9\x99\xcd\x80'
+        >>> asm(shellcraft.nop())
+        '\x90'
 
 Disassembly
 ------------------------
@@ -66,6 +66,60 @@ __all__ = ['asm', 'cpp', 'disasm', 'make_elf', 'make_elf_from_assembly']
 _basedir = path.split(__file__)[0]
 _incdir  = path.join(_basedir, 'data', 'includes')
 
+def dpkg_search_for_binutils(arch, util):
+    """Use dpkg to search for any available assemblers which will work.
+
+    Returns:
+        A list of candidate package names.
+    """
+
+    # Example output:
+    # $ dpkg -S 'arm*linux*-as'
+    # binutils-arm-linux-gnu: /usr/bin/arm-linux-gnu-as
+    # binutils-arm-linux-gnueabihf: /usr/bin/arm-linux-gnueabihf-as
+    # binutils-arm-linux-gnueabihf: /usr/x86_64-linux-gnu/arm-linux-gnueabihf/include/dis-asm.h
+    # binutils-arm-linux-gnu: /usr/x86_64-linux-gnu/arm-linux-gnu/include/dis-asm.h
+    packages = []
+
+    try:
+        filename = 'bin/%s*linux*-%s' % (arch, util)
+        output = subprocess.check_output(['dpkg','-S',filename])
+        for line in output.strip().splitlines():
+            package, path = line.split(':', 1)
+            packages.append(package)
+    except OSError:
+        pass
+    except subprocess.CalledProcessError:
+        pass
+
+    return packages
+
+def print_binutils_instructions(util, context):
+    """On failure to find a binutils utility, inform the user of a way
+    they can get it easily.
+    """
+    # This links to our instructions on how to manually install binutils
+    # for several architectures.
+    instructions = 'https://docs.pwntools.com/en/stable/install/binutils.html'
+
+    # However, if we can directly provide a useful command, go for it.
+    binutils_arch = {
+        'amd64': 'x86_64',
+        'arm':   'armeabi',
+        'thumb': 'armeabi',
+    }.get(context.arch, context.arch)
+
+    packages = dpkg_search_for_binutils(binutils_arch, util)
+
+    if packages:
+        instructions = '$ sudo apt-get install %s' % packages[0]
+
+    log.error("""
+Could not find %(util)r installed for %(context)s
+Try installing binutils for this architecture:
+%(instructions)s
+""".strip() % locals())
+
 @LocalContext
 def which_binutils(util):
     """
@@ -129,13 +183,8 @@ def which_binutils(util):
                 if res:
                     return res[0]
 
-    locals()['context'] = context
-    log.warning("""
-Could not find %(util)r installed for %(context)s
-Try installing binutils for this architecture:
-https://docs.pwntools.com/en/stable/install/binutils.html
-""".strip() % locals())
-    raise Exception('Could not find %(util)r installed for %(context)s' % locals())
+    # No dice!
+    print_binutils_instructions(util, context)
 
 checked_assembler_version = defaultdict(lambda: False)
 

pwnlib/commandline/__init__.py

@@ -1 +1 @@
-# empty
+#!/usr/bin/env python2

pwnlib/commandline/asm.py

@@ -6,8 +6,9 @@ from pwn import *
 
 from . import common
 
-parser = argparse.ArgumentParser(
-    description = 'Assemble shellcode into bytes'
+parser = common.parser_commands.add_parser(
+    'asm',
+    help = 'Assemble shellcode into bytes'
 )
 
 parser.add_argument(
@@ -91,8 +92,7 @@ parser.add_argument(
     action='store_true'
 )
 
-def main():
-    args   = parser.parse_args()
+def main(args):
     tty    = args.output.isatty()
 
     data   = '\n'.join(args.lines) or args.infile.read()
@@ -123,4 +123,5 @@ def main():
     if tty and fmt is not 'raw':
         args.output.write('\n')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/checksec.py

@@ -6,11 +6,10 @@ from pwn import *
 
 from . import common
 
-parser = argparse.ArgumentParser(
-    description = 'Check binary security settings'
+parser = common.parser_commands.add_parser(
+    'checksec',
+    help = 'Check binary security settings'
 )
-
-
 parser.add_argument(
     'elf',
     nargs='*',
@@ -26,8 +25,7 @@ parser.add_argument(
     help='File to check (for compatibility with checksec.sh)'
 )
 
-def main():
-    args   = parser.parse_args()
+def main(args):
     files  = args.elf or args.elf2 or []
 
     if not files:
@@ -37,4 +35,5 @@ def main():
     for f in files:
         e = ELF(f.name)
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/common.py

@@ -1,3 +1,5 @@
+import argparse
+import os
 import sys
 
 import pwnlib
@@ -20,3 +22,13 @@ def context_arg(arg):
     try: context.endian = arg
     except Exception: pass
     return arg
+
+parser = argparse.ArgumentParser(description='Pwntools Command-line Interface',
+                                 prog='pwn')
+parser_commands = parser.add_subparsers(dest='command')
+
+def main(file=sys.argv[0]):
+    import pwnlib.commandline.main
+    name = os.path.splitext(os.path.basename(file))[0]
+    sys.argv.insert(1, name)
+    pwnlib.commandline.main.main()

pwnlib/commandline/constgrep.py

@@ -5,10 +5,12 @@ import os
 import re
 
 from pwn import *
+
 from . import common
 
-p = argparse.ArgumentParser(
-    description = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
+p = common.parser_commands.add_parser(
+    'constgrep',
+    help = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
     formatter_class = argparse.RawDescriptionHelpFormatter,
 )
 
@@ -55,9 +57,7 @@ p.add_argument(
     help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
 )
 
-def main():
-    args = p.parse_args()
-
+def main(args):
     if args.exact:
         # This is the simple case
         print cpp(args.exact).strip()
@@ -134,4 +134,5 @@ def main():
                 print
                 print '(%s) == %s' % (' | '.join(k for v, k in good), args.constant)
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/cyclic.py

@@ -7,8 +7,9 @@ from pwn import *
 
 from . import common
 
-parser = argparse.ArgumentParser(
-    description = "Cyclic pattern creator/finder"
+parser = common.parser_commands.add_parser(
+    'cyclic',
+    help = "Cyclic pattern creator/finder"
 )
 
 parser.add_argument(
@@ -50,8 +51,7 @@ group.add_argument(
     help = 'Number of characters to print'
 )
 
-def main():
-    args = parser.parse_args()
+def main(args):
     alphabet = args.alphabet
     subsize  = args.length
 
@@ -90,4 +90,5 @@ def main():
         if sys.stdout.isatty():
             sys.stdout.write('\n')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/disasm.py

@@ -7,8 +7,9 @@ from pwn import *
 
 from . import common
 
-parser = argparse.ArgumentParser(
-    description = 'Disassemble bytes into text format'
+parser = common.parser_commands.add_parser(
+    'disasm',
+    help = 'Disassemble bytes into text format'
 )
 
 parser.add_argument(
@@ -52,9 +53,7 @@ parser.add_argument(
 )
 
 
-def main():
-    args = parser.parse_args()
-
+def main(args):
     if len(args.hex) > 0:
         dat = ''.join(args.hex)
         dat = dat.translate(None, string.whitespace)
@@ -88,4 +87,5 @@ def main():
 
     print disasm(dat, vma=safeeval.const(args.address))
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/elfdiff.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python2
 import shutil
-
 from argparse import ArgumentParser
 from subprocess import CalledProcessError
 from subprocess import check_output
@@ -8,6 +7,9 @@ from tempfile import NamedTemporaryFile
 
 from pwn import *
 
+from . import common
+
+
 def dump(objdump, path):
     n = NamedTemporaryFile(delete=False)
     o = check_output([objdump,'-d','-x','-s',path])
@@ -20,13 +22,15 @@ def diff(a,b):
     except CalledProcessError as e:
         return e.output
 
-p = ArgumentParser()
+p = common.parser_commands.add_parser(
+    'elfdiff',
+    help = 'Compare two ELF files'
+)
+
 p.add_argument('a')
 p.add_argument('b')
 
-def main():
-    a = p.parse_args()
-
+def main(a):
     with context.silent:
         x = ELF(a.a)
         y = ELF(a.b)
@@ -49,4 +53,5 @@ def main():
 
     print diff(x, y)
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/elfpatch.py

@@ -4,15 +4,20 @@ import sys
 
 from pwn import *
 
+from . import common
+
+p = common.parser_commands.add_parser(
+    'elfpatch',
+    help = 'Patch an ELF file'
+)
+
 p = argparse.ArgumentParser()
 p.add_argument('elf',help="File to patch")
 p.add_argument('offset',help="Offset to patch in virtual address (hex encoded)")
 p.add_argument('bytes',help='Bytes to patch (hex encoded)')
 
 
-def main():
-    a = p.parse_args()
-
+def main(a):
     if not a.offset.startswith('0x'):
         a.offset = '0x' + a.offset
 
@@ -25,4 +30,5 @@ def main():
     elf.write(offset, bytes)
     sys.stdout.write(elf.get_data())
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/errno.py

@@ -1,17 +1,18 @@
 import argparse
 import os
 
-parser = argparse.ArgumentParser(
-    description = 'Prints out error messages'
+from . import common
+
+parser = common.parser_commands.add_parser(
+    'errno',
+    help = 'Prints out error messages'
 )
 
 parser.add_argument(
     'error', help='Error message or value', type=str
 )
 
-def main():
-  args, unknown = parser.parse_known_args()
-
+def main(args):
   try:
     value = int(args.error, 0)
 
@@ -40,4 +41,5 @@ def main():
   print '#define', name, value
   print os.strerror(value)
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/hex.py

@@ -2,17 +2,21 @@
 import argparse
 import sys
 
-parser = argparse.ArgumentParser(description='''
-Hex-encodes data provided on the command line or via stdin.
+from . import common
+
+parser = common.parser_commands.add_parser(
+    'hex',
+    help = '''
+Hex-encodes data provided on the command line or stdin
 ''')
 parser.add_argument('data', nargs='*',
     help='Data to convert into hex')
 
-def main():
-    args = parser.parse_args()
+def main(args):
     if not args.data:
         print sys.stdin.read().encode('hex')
     else:
         print ' '.join(args.data).encode('hex')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/main.py

@@ -0,0 +1,42 @@
+from . import asm
+from . import checksec
+from . import common
+from . import constgrep
+from . import cyclic
+from . import disasm
+from . import elfdiff
+from . import elfpatch
+from . import errno
+from . import hex
+from . import phd
+from . import pwnstrip
+from . import scramble
+from . import shellcraft
+from . import unhex
+from . import update
+from .common import parser
+
+commands = {
+    'asm': asm.main,
+    'checksec': checksec.main,
+    'constgrep': constgrep.main,
+    'cyclic': cyclic.main,
+    'disasm': disasm.main,
+    'elfdiff': elfdiff.main,
+    'elfpatch': elfpatch.main,
+    'errno': errno.main,
+    'hex': hex.main,
+    'phd': phd.main,
+    'pwnstrip': pwnstrip.main,
+    'scramble': scramble.main,
+    'shellcraft': shellcraft.main,
+    'unhex': unhex.main,
+    'update': update.main,
+}
+
+def main():
+    args = parser.parse_args()
+    commands[args.command](args)
+
+if __name__ == '__main__':
+    main()

pwnlib/commandline/phd.py

@@ -5,8 +5,11 @@ import sys
 
 from pwn import *
 
-parser = argparse.ArgumentParser(
-    description = 'Pwnlib HexDump'
+from . import common
+
+parser = common.parser_commands.add_parser(
+    'phd',
+    help = 'Pwnlib HexDump'
 )
 
 parser.add_argument(
@@ -64,9 +67,7 @@ def asint(s):
     else:
         return int(s, 10)
 
-def main():
-    args = parser.parse_args()
-
+def main(args):
     infile = args.file
     width  = asint(args.width)
     skip   = asint(args.skip)
@@ -95,4 +96,5 @@ def main():
     except (KeyboardInterrupt, IOError):
         pass
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/pwnstrip.py

@@ -4,9 +4,9 @@ from pwn import *
 
 from . import common
 
-p = argparse.ArgumentParser(
-    description = 'Strip binaries for CTF usage',
-    formatter_class = argparse.RawDescriptionHelpFormatter,
+p = common.parser_commands.add_parser(
+    'pwnstrip',
+    help = 'Strip binaries for CTF usage',
 )
 
 g = p.add_argument_group("actions")
@@ -15,9 +15,7 @@ g.add_argument('-p', '--patch', metavar='FUNCTION', help="Patch function", actio
 p.add_argument('-o', '--output', type=file, default=sys.stdout)
 p.add_argument('file', type=file)
 
-def main():
-    args = p.parse_args()
-
+def main(args):
     if not (args.patch or args.build_id):
         sys.stderr.write("Must specify at least one action\n")
         sys.stderr.write(p.format_usage())
@@ -48,6 +46,5 @@ def main():
 
     args.output.write(result)
 
-
 if __name__ == '__main__':
-    main()
+    pwnlib.common.main(__file__)

pwnlib/commandline/scramble.py

@@ -5,11 +5,11 @@ from pwn import *
 
 from . import common
 
-parser = argparse.ArgumentParser(
-    description = 'Shellcode encoder'
+parser = common.parser_commands.add_parser(
+    'scramble',
+    help = 'Shellcode encoder'
 )
 
-
 parser.add_argument(
     "-f", "--format",
     help="Output format (defaults to hex for ttys, otherwise raw)",
@@ -68,8 +68,7 @@ parser.add_argument(
     action='store_true'
 )
 
-def main():
-    args   = parser.parse_args()
+def main(args):
     tty    = args.output.isatty()
 
     if sys.stdin.isatty():
@@ -100,4 +99,6 @@ def main():
     if tty and fmt is not 'raw':
         args.output.write('\n')
 
-if __name__ == '__main__': main()
+
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/shellcraft.py

@@ -8,18 +8,6 @@ from pwn import *
 
 from . import common
 
-r = text.red
-g = text.green
-b = text.blue
-
-banner = '\n'.join(['  ' + r('____') + '  ' + g('_') + '          ' + r('_') + ' ' + g('_') + '                 ' + b('__') + ' ' + r('_'),
-                    ' ' + r('/ ___|') + g('| |__') + '   ' + b('___') + r('| |') + ' ' + g('|') + ' ' + b('___') + ' ' + r('_ __') + ' ' + g('__ _') + ' ' + b('/ _|') + ' ' + r('|_'),
-                    ' ' + r('\___ \\') + g('| \'_ \\') + ' ' + b('/ _ \\') + ' ' + r('|') + ' ' + g('|') + b('/ __|') + ' ' + r('\'__/') + ' ' + g('_` |') + ' ' + b('|_') + r('| __|'),
-                    '  ' + r('___) |') + ' ' + g('| | |') + '  ' + b('__/') + ' ' + r('|') + ' ' + g('|') + ' ' + b('(__') + r('| |') + ' ' + g('| (_| |') + '  ' + b('_|') + ' ' + r('|_'),
-                    ' ' + r('|____/') + g('|_| |_|') + b('\\___|') + r('_|') + g('_|') + b('\\___|') + r('_|') + '  ' + g('\\__,_|') + b('_|') + '  ' + r('\\__|'),
-                    '\n'
-                    ])
-
 
 #  ____  _          _ _                 __ _
 # / ___|| |__   ___| | | ___ _ __ __ _ / _| |_
@@ -37,9 +25,10 @@ def _string(s):
             out.append('\\x%02x' % co)
     return '"' + ''.join(out) + '"\n'
 
-p = argparse.ArgumentParser(
-    description = 'Microwave shellcode -- Easy, fast and delicious',
-    formatter_class = argparse.RawDescriptionHelpFormatter,
+
+p = common.parser_commands.add_parser(
+    'shellcraft',
+    help = 'Microwave shellcode -- Easy, fast and delicious',
 )
 
 
@@ -76,13 +65,10 @@ p.add_argument(
 p.add_argument(
     'shellcode',
     nargs = '?',
-    choices = shellcraft.templates,
-    metavar = 'shellcode',
     help = 'The shellcode you want',
+    type = str
 )
 
-p.epilog = 'Available shellcodes are:\n' + '\n'.join(shellcraft.templates)
-
 p.add_argument(
     'args',
     nargs = '*',
@@ -167,6 +153,12 @@ p.add_argument(
     default=None
 )
 
+p.add_argument(
+    '-l', '--list',
+    action='store_true',
+    help='List available shellcodes, optionally provide a filter'
+)
+
 def get_template(name):
     func = shellcraft
     for attr in name.split('.'):
@@ -177,21 +169,25 @@ def is_not_a_syscall_template(name):
     template_src = shellcraft._get_source(name)
     return 'man 2' not in read(template_src)
 
-def main():
-    # Banner must be added here so that it doesn't appear in the autodoc
-    # generation for command line tools
-    p.description = banner + p.description
-    args = p.parse_args()
-
-    if not args.shellcode:
+def main(args):
+    if args.list:
         templates = shellcraft.templates
 
-        if not args.syscalls:
+        if args.shellcode:
+            templates = filter(lambda a: args.shellcode in a, templates)
+        elif not args.syscalls:
             templates = filter(is_not_a_syscall_template, templates)
 
         print '\n'.join(templates)
         exit()
 
+    if not args.shellcode:
+        common.parser.print_usage()
+        exit()
+
+    if args.shellcode not in shellcraft.templates:
+        log.error("Unknown shellcraft template %r. Use --list to see available shellcodes." % args.shellcode)
+
     func = get_template(args.shellcode)
 
     if args.show:
@@ -326,7 +322,7 @@ def main():
         sys.exit(0)
 
     if args.format in ['s', 'str', 'string']:
-        code = _string(code) + '"\n'
+        code = _string(code)
     elif args.format == 'c':
         code = '{' + ', '.join(map(hex, bytearray(code))) + '}' + '\n'
     elif args.format in ['h', 'hex']:
@@ -339,4 +335,5 @@ def main():
 
     args.out.write(code)
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/unhex.py

@@ -3,14 +3,18 @@ import argparse
 import sys
 from string import whitespace
 
-parser = argparse.ArgumentParser(description='''
+from . import common
+
+parser = common.parser_commands.add_parser(
+    'unhex',
+    help = '''
 Decodes hex-encoded data provided on the command line or via stdin.
 ''')
+
 parser.add_argument('hex', nargs='*',
     help='Hex bytes to decode')
 
-def main():
-    args = parser.parse_args()
+def main(args):
     try:
         if not args.hex:
             s = sys.stdin.read().translate(None, whitespace)
@@ -20,4 +24,5 @@ def main():
     except TypeError, e:
         sys.stderr.write(str(e) + '\n')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/update.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python2
+import subprocess
+
+from pwn import *
+
+from . import common
+
+p = common.parser_commands.add_parser(
+    'update',
+    help = 'Check for pwntools updates'
+)
+
+p.add_argument('--install', action='store_true', help='''
+    Install the update automatically.
+''')
+
+p.add_argument('--pre', action='store_true', help='''
+    Check for pre-releases.
+''')
+
+def main(a):
+    result = pwnlib.update.perform_check(prerelease=a.pre)
+    if a.install:
+        subprocess.check_call(result, shell=True)
+
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/context/__init__.py

@@ -525,10 +525,21 @@ class ContextType(object):
     def quiet(self, function=None):
         """Disables all non-error logging within the enclosed scope,
         *unless* the debugging level is set to 'debug' or lower."""
-        level = 'error'
-        if context.log_level <= logging.DEBUG:
-            level = None
-        return self.local(function, log_level=level)
+        if not function:
+            level = 'error'
+            if context.log_level <= logging.DEBUG:
+                level = None
+            return self.local(function, log_level=level)
+
+        @functools.wraps(function)
+        def wrapper(*a, **kw):
+            level = 'error'
+            if context.log_level <= logging.DEBUG:
+                level = None
+            with self.local(function, log_level=level):
+                return function(*a, **kw)
+        return wrapper
+
 
     @property
     def verbose(self):
@@ -1086,6 +1097,7 @@ class ContextType(object):
             self.arch = device.arch or self.arch
             self.bits = device.bits or self.bits
             self.endian = device.endian or self.endian
+            self.os = device.os or self.os
         elif isinstance(device, str):
             device = Device(device)
         else:

pwnlib/device.py

@@ -4,12 +4,13 @@ class Device(object):
     bits = None
     endian = None
     serial = None
+    os = None
 
     def __init__(self, serial=None):
         self.serial = serial
 
     def __str__(self):
         return self.serial
- 
+
     def __eq__(self, other):
         return self.serial == other or self.serial == str(other)

pwnlib/dynelf.py

@@ -860,3 +860,14 @@ class DynELF(object):
         self.success('*environ: %#x' % stack)
 
         return stack
+
+    def heap(self):
+        """Finds the beginning of the heap via __curbrk, which is an exported
+        symbol in the linker, which points to the current brk.
+        """
+        curbrk = self.lookup('__curbrk', 'libc')
+        brk    = self.leak.p(curbrk)
+
+        self.success('*curbrk: %#x' % brk)
+
+        return brk

pwnlib/elf/corefile.py

@@ -6,7 +6,6 @@ from elftools.common.py3compat import bytes2str
 from elftools.common.utils import roundup
 from elftools.common.utils import struct_parse
 from elftools.construct import CString
-
 from ..context import context
 from ..log import getLogger
 from ..tubes.tube import tube
@@ -30,11 +29,11 @@ def iter_notes(self):
     end = self['p_offset'] + self['p_filesz']
     while offset < end:
         note = struct_parse(
-            self._elfstructs.Elf_Nhdr,
+            self.elffile.structs.Elf_Nhdr,
             self.stream,
             stream_pos=offset)
         note['n_offset'] = offset
-        offset += self._elfstructs.Elf_Nhdr.sizeof()
+        offset += self.elffile.structs.Elf_Nhdr.sizeof()
         self.stream.seek(offset)
         # n_namesz is 4-byte aligned.
         disk_namesz = roundup(note['n_namesz'], 2)

pwnlib/elf/elf.py

@@ -24,6 +24,7 @@ from ..term import text
 from ..tubes.process import process
 from ..util import misc
 from ..util import packing
+from ..util import sh_string
 
 log = getLogger(__name__)
 
@@ -312,7 +313,7 @@ class ELF(ELFFile):
             return
 
         try:
-            cmd = misc.sh_command_with('ulimit -s unlimited; LD_TRACE_LOADED_OBJECTS=1 LD_WARN=1 LD_BIND_NOW=1 %s 2>/dev/null', self.path)
+            cmd = sh_string.sh_command_with('ulimit -s unlimited; LD_TRACE_LOADED_OBJECTS=1 LD_WARN=1 LD_BIND_NOW=1 %s 2>/dev/null', self.path)
 
             data = subprocess.check_output(cmd, shell = True, stderr = subprocess.STDOUT)
             libs = misc.parse_ldd_output(data)

pwnlib/flag/__init__.py

@@ -0,0 +1,5 @@
+from .flag import submit_flag
+
+__all__ = [
+    'submit_flag',
+]

pwnlib/flag/flag.py

@@ -0,0 +1,71 @@
+"""Describes a way to submit a key to a key server.
+"""
+import os
+
+from ..args import args
+from ..tubes.remote import remote
+from ..log import getLogger
+
+env_server  = args.get('FLAG_HOST', 'flag-submission-server').strip()
+env_port    = args.get('FLAG_PORT', '31337').strip()
+env_proto   = args.get('FLAG_PROTO', 'tcp').strip()
+env_file    = args.get('FLAG_FILE', '/does/not/exist').strip()
+env_exploit_name = args.get('EXPLOIT_NAME', 'unnamed-exploit').strip()
+env_target_host  = args.get('TARGET_HOST', 'unknown-target').strip()
+env_team_name    = args.get('TEAM_NAME', 'unknown-team').strip()
+
+log = getLogger(__name__)
+
+def submit_flag(flag,
+                exploit=env_exploit_name,
+                target=env_target_host,
+                server=env_server,
+                port=env_port,
+                proto=env_proto,
+                team=env_team_name):
+    """
+    Submits a flag to the game server
+
+    Arguments:
+        flag(str): The flag to submit.
+        exploit(str): Exploit identifier, optional
+        target(str): Target identifier, optional
+        server(str): Flag server host name, optional
+        port(int): Flag server port, optional
+        proto(str), Flag server protocol, optional
+
+    Optional arguments are inferred from the environment,
+    or omitted if none is set.
+
+    Returns:
+        A string indicating the status of the key submission,
+        or an error code.
+
+    Doctest:
+
+        >>> l = listen()
+        >>> _ = submit_flag('flag', server='localhost', port=l.lport)
+        >>> c = l.wait_for_connection()
+        >>> c.recvall().split()
+        ['flag', 'unnamed-exploit', 'unknown-target', 'unknown-team']
+    """
+    flag = flag.strip()
+
+    log.success("Flag: %r" % flag)
+
+    data = "\n".join([flag,
+                      exploit,
+                      target,
+                      team,
+                      ''])
+
+    if os.path.exists(env_file):
+        write(env_file, data)
+        return
+
+    try:
+        with remote(server, int(port)) as r:
+            r.send(data)
+            return r.recvall(timeout=1)
+    except Exception:
+        log.warn("Could not submit flag %r to %s:%s" % (flag, server, port))

pwnlib/gdb.py

@@ -32,9 +32,7 @@ def debug_assembly(asm, execute=None, vma=None):
     tmp_elf = make_elf_from_assembly(asm, vma=vma, extract=False)
     os.chmod(tmp_elf, 0777)
 
-    def unlink():
-        with context.silent: os.unlink(tmp_elf)
-    atexit.register(lambda: unlink)
+    atexit.register(lambda: os.unlink(tmp_elf))
 
     if context.os == 'android':
         android_path = '/data/data/%s' % os.path.basename(tmp_elf)
@@ -60,9 +58,7 @@ def debug_shellcode(data, execute=None, vma=None):
     tmp_elf = make_elf(data, extract=False, vma=vma)
     os.chmod(tmp_elf, 0777)
 
-    def unlink():
-        with context.silent: os.unlink(tmp_elf)
-    atexit.register(lambda: unlink)
+    atexit.register(lambda: os.unlink(tmp_elf))
 
     if context.os == 'android':
         android_path = '/data/data/%s' % os.path.basename(tmp_elf)
@@ -344,6 +340,7 @@ def attach(target, execute = None, exe = None, need_ptrace_scope = True):
         shell = target.parent
 
         tmpfile = shell.mktemp()
+        execute = 'shell rm %s\n%s' % (tmpfile, execute)
         shell.upload_data(execute or '', tmpfile)
 
         cmd = ['ssh', '-C', '-t', '-p', str(shell.port), '-l', shell.user, shell.host]
@@ -351,10 +348,9 @@ def attach(target, execute = None, exe = None, need_ptrace_scope = True):
             cmd = ['sshpass', '-p', shell.password] + cmd
         if shell.keyfile:
             cmd += ['-i', shell.keyfile]
-        cmd += ['gdb %r %s -x "%s" ; rm "%s"' % (target.executable,
-                                                 target.pid,
-                                                 tmpfile,
-                                                 tmpfile)]
+        cmd += ['gdb %r %s -x "%s"' % (target.executable,
+                                       target.pid,
+                                       tmpfile)]
 
         misc.run_in_new_terminal(' '.join(cmd))
         return
@@ -427,10 +423,11 @@ def attach(target, execute = None, exe = None, need_ptrace_scope = True):
         tmp = tempfile.NamedTemporaryFile(prefix = 'pwn', suffix = '.gdb',
                                           delete = False)
         log.debug('Wrote gdb script to %r\n%s' % (tmp.name, execute))
+        execute = 'shell rm %s\n%s' % (tmp.name, execute)
+
         tmp.write(execute)
         tmp.close()
-        atexit.register(lambda: os.unlink(tmp.name))
-        cmd += ' -x "%s" ; rm "%s"' % (tmp.name, tmp.name)
+        cmd += ' -x "%s"' % (tmp.name)
 
     log.info('running in new terminal: %s' % cmd)
     misc.run_in_new_terminal(cmd)

pwnlib/shellcraft/templates/__doc__

@@ -3,14 +3,3 @@ The shellcode module.
 This module contains functions for generating shellcode.
 
 It is organized first by architecture and then by operating system.
-
-Example:
-
-    >>> print shellcraft.i386.nop().strip('\n')
-        nop
-    >>> print shellcraft.i386.linux.sh()
-        /* push '/bin///sh\x00' */
-        push 0x68
-        push 0x732f2f2f
-        push 0x6e69622f
-    ...

pwnlib/shellcraft/templates/amd64/linux/sh.asm

@@ -8,4 +8,4 @@ Execute a different process.
     'Hello\n'
 
 </%docstring>
-${amd64.linux.execve('/bin///sh', 0, 0)}
+${amd64.linux.execve('/bin///sh', ['sh'], 0)}

pwnlib/shellcraft/templates/arm/linux/sh.asm

@@ -8,4 +8,4 @@ Execute a different process.
     'Hello\n'
 
 </%docstring>
-    ${arm.linux.execve('/bin///sh', 0, 0)}
+    ${arm.linux.execve('/bin///sh', ['sh'], 0)}

pwnlib/shellcraft/templates/i386/linux/sh.asm

@@ -8,4 +8,4 @@ Execute a different process.
     'Hello\n'
 
 </%docstring>
-${i386.linux.execve('/bin///sh', 0, 0)}
+${i386.linux.execve('/bin///sh', ['sh'], 0)}

pwnlib/shellcraft/templates/mips/android/sh.asm

@@ -1,11 +1,4 @@
 <% from pwnlib.shellcraft import mips %>
 <%docstring>Execute /bin/sh</%docstring>
 
-${mips.push(0)}
-${mips.pushstr('sh')}
-${mips.mov('$a1','$sp')}}
-
-${mips.pushstr('/system/bin//sh')}
-
-${mips.syscall('SYS_execve', '$sp', '$a1', 0)}
-
+${mips.execve('//system/bin/sh', ['sh'], {})}

pwnlib/shellcraft/templates/mips/linux/cat.asm

@@ -1,6 +1,6 @@
 <%
   from pwnlib import constants
-  from pwnlib.shellcraft import mips 
+  from pwnlib.shellcraft import mips
   from pwnlib.shellcraft import common
 %>
 <%page args="filename, fd=1"/>
@@ -11,7 +11,9 @@ Example:
 
     >>> f = tempfile.mktemp()
     >>> write(f, 'FLAG')
-    >>> run_assembly(shellcraft.mips.linux.cat(f)).recvall()
+    >>> asm  = shellcraft.mips.linux.cat(f)
+    >>> asm += shellcraft.mips.linux.exit(0)
+    >>> run_assembly(asm).recvall()
     'FLAG'
 
 </%docstring>

pwnlib/shellcraft/templates/mips/linux/execve.asm

@@ -1,15 +1,47 @@
-
 <%
-    from pwnlib.shellcraft.mips.linux import syscall
+from pwnlib.shellcraft import mips, registers
+from pwnlib.abi import linux_mips_syscall as abi
 %>
-<%page args="path, argv, envp"/>
 <%docstring>
-Invokes the syscall execve.  See 'man 2 execve' for more information.
+Execute a different process.
 
-Arguments:
-    path(char): path
-    argv(char): argv
-    envp(char): envp
+Attempts to perform some automatic detection of types.
+Otherwise, the arguments behave as normal.
+
+- If ``path`` is a string that is not a known register,
+  it is pushed onto the stack.
+- If ``argv`` is an array of strings, it is pushed onto
+  the stack, and NULL-terminated.
+- If ``envp`` is an dictionary of {string:string},
+  it is pushed onto the stack, and NULL-terminated.
+
+Example:
+
+    >>> path = '/bin/sh'
+    >>> argv = ['sh', '-c', 'echo Hello, $NAME; exit $STATUS']
+    >>> envp = {'NAME': 'zerocool', 'STATUS': 3}
+    >>> sc = shellcraft.mips.linux.execve(path, argv, envp)
+    >>> io = run_assembly(sc)
+    >>> io.recvall()
+    'Hello, zerocool\n'
+    >>> io.poll(True)
+    3
 </%docstring>
-
-    ${syscall('SYS_execve', path, argv, envp)}
+<%page args="path = '/system/bin//sh', argv=0, envp=0"/>
+<%
+if isinstance(envp, dict):
+    envp = ['%s=%s' % (k,v) for (k,v) in envp.items()]
+%>
+% if isinstance(argv, (list, tuple)):
+    ${mips.pushstr_array(abi.register_arguments[3], argv)}
+    <% argv = abi.register_arguments[3] %>
+% endif
+% if isinstance(envp, (list, tuple)):
+    ${mips.pushstr_array(abi.register_arguments[2], envp)}
+    <% envp = abi.register_arguments[2] %>
+% endif
+% if isinstance(path, str) and not registers.is_register(path):
+    ${mips.pushstr(path)}
+    <% path = '$sp' %>
+%endif
+    ${mips.syscall('SYS_execve', path, argv, envp)}

pwnlib/shellcraft/templates/mips/linux/mov.asm

@@ -11,8 +11,7 @@ Thin wrapper around :func:`pwnlib.shellcraft.mips.mov`, which sets
 Example:
 
     >>> print pwnlib.shellcraft.mips.linux.mov('$a1', 'SYS_execve').rstrip()
-        .set noat
-        ori $a1, $zero, SYS_execve
+        ori $a1, $zero, (SYS_execve)
 
 </%docstring>
 

pwnlib/shellcraft/templates/mips/linux/pushstr.asm

@@ -12,26 +12,14 @@ Example:
 
     >>> print pwnlib.shellcraft.mips.linux.pushstr('Hello, World').rstrip()
         /* push 'Hello, World\x00' */
-        .set noat
-        slti $at, $zero, -1
-        sw $at, -4($sp)
-        add $sp, $sp, -4
-        .set noat
-        lui $at, 25708
-        ori $at, $at, 29295
-        sw $at, -4($sp)
-        add $sp, $sp, -4
-        .set noat
-        lui $at, 22304
-        ori $at, $at, 11375
-        sw $at, -4($sp)
-        add $sp, $sp, -4
-        .set noat
-        lui $at, 27756
-        ori $at, $at, 25928
-        sw $at, -4($sp)
-        add $sp, $sp, -4
-
+        li $t1, 0x6c6c6548
+        sw $t1, -16($sp)
+        li $t1, 0x57202c6f
+        sw $t1, -12($sp)
+        li $t1, 0x646c726f
+        sw $t1, -8($sp)
+        sw $zero, -4($sp)
+        addiu $sp, $sp, -16
 Args:
   string (str): The string to push.
   append_null (bool): Whether to append a single NULL-byte before pushing.

pwnlib/shellcraft/templates/mips/linux/sh.asm

@@ -1,7 +1,4 @@
 <% from pwnlib.shellcraft import mips %>
 <%docstring>Execute /bin/sh</%docstring>
 
-${mips.pushstr('//bin/sh')}
-
-${mips.syscall('SYS_execve', '$sp', 0, 0)}
-
+${mips.execve('//bin/sh', ['sh'], {})}

pwnlib/shellcraft/templates/mips/linux/syscall.asm

@@ -12,63 +12,55 @@ Any of the arguments can be expressions to be evaluated by :func:`pwnlib.constan
 
 Example:
 
-        >>> print pwnlib.shellcraft.i386.linux.syscall('SYS_execve', 1, 'esp', 2, 0).rstrip()
-            /* call execve(1, 'esp', 2, 0) */
-            push 0x1
-            pop ebx
-            mov ecx, esp
-            push 0x2
-            pop edx
-            xor esi, esi
-            push 0xb
-            pop eax
-            int 0x80
-        >>> print pwnlib.shellcraft.i386.linux.syscall('SYS_execve', 2, 1, 0, 20).rstrip()
-            /* call execve(2, 1, 0, 20) */
-            push 0x2
-            pop ebx
-            push 0x1
-            pop ecx
-            push 0x14
-            pop esi
-            push 0xb
-            pop eax
-            cdq /* Set edx to 0, eax is known to be positive */
-            int 0x80
-        >>> print pwnlib.shellcraft.i386.linux.syscall().rstrip()
+        >>> print pwnlib.shellcraft.mips.linux.syscall('SYS_execve', 1, '$sp', 2, 0).rstrip()
+            /* call execve(1, '$sp', 2, 0) */
+            li $t9, ~1
+            not $a0, $t9
+            add $a1, $sp, $0 /* mov $a1, $sp */
+            li $t9, ~2
+            not $a2, $t9
+            slti $a3, $zero, 0xFFFF /* $a3 = 0 */
+            ori $v0, $zero, (SYS_execve)
+            syscall 0x40404
+        >>> print pwnlib.shellcraft.mips.linux.syscall('SYS_execve', 2, 1, 0, 20).rstrip()
+            /* call execve(2, 1, 0, 0x14) */
+            li $t9, ~2
+            not $a0, $t9
+            li $t9, ~1
+            not $a1, $t9
+            slti $a2, $zero, 0xFFFF /* $a2 = 0 */
+            li $t9, ~0x14
+            not $a3, $t9
+            ori $v0, $zero, (SYS_execve)
+            syscall 0x40404
+        >>> print pwnlib.shellcraft.mips.linux.syscall().rstrip()
             /* call syscall() */
-            int 0x80
-        >>> print pwnlib.shellcraft.i386.linux.syscall('eax', 'ebx', 'ecx').rstrip()
-            /* call syscall('eax', 'ebx', 'ecx') */
-            /* moving ebx into ebx, but this is a no-op */
-            /* moving ecx into ecx, but this is a no-op */
-            /* moving eax into eax, but this is a no-op */
-            int 0x80
-        >>> print pwnlib.shellcraft.i386.linux.syscall('ebp', None, None, 1).rstrip()
-            /* call syscall('ebp', ?, ?, 1) */
-            push 0x1
-            pop edx
-            mov eax, ebp
-            int 0x80
-        >>> print pwnlib.shellcraft.i386.linux.syscall(
+            syscall 0x40404
+        >>> print pwnlib.shellcraft.mips.linux.syscall('$v0', '$a0', '$a1').rstrip()
+            /* call syscall('$v0', '$a0', '$a1') */
+            /* setregs noop */
+            syscall 0x40404
+        >>> print pwnlib.shellcraft.mips.linux.syscall('$a3', None, None, 1).rstrip()
+            /* call syscall('$a3', ?, ?, 1) */
+            li $t9, ~1
+            not $a2, $t9
+            sw $a3, -4($sp) /* mov $v0, $a3 */
+            lw $v0, -4($sp)
+            syscall 0x40404
+        >>> print pwnlib.shellcraft.mips.linux.syscall(
         ...               'SYS_mmap2', 0, 0x1000,
         ...               'PROT_READ | PROT_WRITE | PROT_EXEC',
         ...               'MAP_PRIVATE | MAP_ANONYMOUS',
         ...               -1, 0).rstrip()
-            /* call mmap2(0, 4096, 'PROT_READ | PROT_WRITE | PROT_EXEC', 'MAP_PRIVATE | MAP_ANONYMOUS', -1, 0) */
-            xor ebx, ebx
-            xor ecx, ecx
-            mov ch, 0x10
-            push 0x7
-            pop edx
-            push 0x22
-            pop esi
-            push -1
-            pop edi
-            xor ebp, ebp
-            xor eax, eax
-            mov al, 0xc0
-            int 0x80
+            /* call mmap2(0, 0x1000, 'PROT_READ | PROT_WRITE | PROT_EXEC', 'MAP_PRIVATE | MAP_ANONYMOUS', -1, 0) */
+            slti $a0, $zero, 0xFFFF /* $a0 = 0 */
+            li $t9, ~0x1000
+            not $a1, $t9
+            li $t9, ~(PROT_READ | PROT_WRITE | PROT_EXEC) /* 7 */
+            not $a2, $t9
+            ori $a3, $zero, (MAP_PRIVATE | MAP_ANONYMOUS)
+            ori $v0, $zero, (SYS_mmap2)
+            syscall 0x40404
 </%docstring>
 <%
   append_cdq = False

pwnlib/shellcraft/templates/mips/mov.asm

@@ -120,6 +120,8 @@ if src_reg == 0:
 % elif okay(src):
 ## Nice and easy
     li ${dst}, ${pretty(src)}
+% elif 0 < src <= 0xffff and okay(src, bytes=2):
+    ori ${dst}, $zero, ${src}
 % elif okay((~srcu) & 0xffffffff):
 ## Almost nice and easy
     li $t9, ~${pretty(src)}

pwnlib/shellcraft/templates/mips/pushstr.asm

@@ -21,8 +21,7 @@ Example:
         addiu $sp, $sp, -4
     >>> print shellcraft.mips.pushstr('aa').rstrip()
         /* push 'aa\x00' */
-        li $t9, ~0x6161
-        not $t1, $t9
+        ori $t1, $zero, 24929
         sw $t1, -4($sp)
         addiu $sp, $sp, -4
     >>> print shellcraft.mips.pushstr('aaa').rstrip()

pwnlib/shellcraft/templates/mips/pushstr_array.asm

@@ -0,0 +1,38 @@
+<% from pwnlib.shellcraft import mips %>
+<%docstring>
+Pushes an array/envp-style array of pointers onto the stack.
+
+Arguments:
+    reg(str):
+        Destination register to hold the pointer.
+    array(str,list):
+        Single argument or list of arguments to push.
+        NULL termination is normalized so that each argument
+        ends with exactly one NULL byte.
+</%docstring>
+<%page args="reg, array"/>
+<%
+if isinstance(array, (str)):
+    array = [array]
+
+array_str = ''
+
+# Normalize all of the arguments' endings
+array      = [arg.rstrip('\x00') + '\x00' for arg in array]
+array_str  = ''.join(array)
+
+word_size = 4
+offset = len(array_str) + word_size
+
+%>\
+    /* push argument array ${repr(array)} */
+    ${mips.pushstr(array_str)}
+    ${mips.mov(reg, 0)}
+    ${mips.push(reg)} /* null terminate */
+% for i,arg in enumerate(reversed(array)):
+    ${mips.mov(reg, offset + word_size*i - len(arg))}
+    add ${reg}, $sp
+    ${mips.push(reg)} /* ${repr(arg)} */
+    <% offset -= len(arg) %>\
+% endfor
+    ${mips.mov(reg,'$sp')}

pwnlib/shellcraft/templates/thumb/linux/sh.asm

@@ -8,4 +8,4 @@ Execute a different process.
     'Hello\n'
 
 </%docstring>
-${thumb.linux.execve('/bin///sh', 0, 0)}
+${thumb.linux.execve('/bin///sh', ['sh'], 0)}

pwnlib/term/__init__.py

@@ -7,7 +7,6 @@ from . import readline
 from . import term
 from . import termcap
 from . import text
-from . import completer
 
 # Re-exports (XXX: Are these needed?)
 output = term.output

pwnlib/tubes/process.py

@@ -6,6 +6,7 @@ import os
 import pty
 import resource
 import select
+import signal
 import subprocess
 import tty
 
@@ -97,6 +98,8 @@ class process(tube):
             Where the process is running, used for logging purposes.
         display(list):
             List of arguments to display, instead of the main executable name.
+        alarm(int):
+            Set a SIGALRM alarm timeout on the process.
 
     Attributes:
         proc(subprocess)
@@ -183,6 +186,10 @@ class process(tube):
 
         >>> process(['sh','-c','ulimit -s'], aslr=0).recvline()
         'unlimited\n'
+
+        >>> io = process(['sh','-c','sleep 10; exit 7'], alarm=2)
+        >>> io.poll(block=True) == -signal.SIGALRM
+        True
     """
 
     PTY = PTY
@@ -206,7 +213,8 @@ class process(tube):
                  aslr = None,
                  setuid = None,
                  where = 'local',
-                 display = None):
+                 display = None,
+                 alarm = None):
         super(process, self).__init__(timeout, level = level)
 
         #: `subprocess.Popen` object
@@ -253,6 +261,9 @@ class process(tube):
         #: Directory the process was created in
         self.cwd          = cwd or os.path.curdir
 
+        #: Alarm timeout of the process
+        self.alarm        = alarm
+
         self.preexec_fn = preexec_fn
         self.display    = display or self.program
 
@@ -357,6 +368,9 @@ class process(tube):
             except:
                 pass
 
+        if self.alarm is not None:
+            signal.alarm(self.alarm)
+
         self.preexec_fn()
 
     @property

pwnlib/tubes/remote.py

@@ -157,13 +157,18 @@ class remote(sock):
         return remote(host, port, fam=s.family, typ=s.type, sock=s)
 
 class tcp(remote):
+    __doc__ = remote.__doc__
     def __init__(self, host, port,
                  fam = "any", typ = "tcp",
                  timeout = Timeout.default, ssl=False, sock=None, level = None):
         return super(tcp, self).__init__(host, port, fam, typ, timeout, ssl, sock, level)
 
 class udp(remote):
+    __doc__ = remote.__doc__
     def __init__(self, host, port,
                  fam = "any", typ = "udp",
                  timeout = Timeout.default, ssl=False, sock=None, level = None):
         return super(udp, self).__init__(host, port, fam, typ, timeout, ssl, sock, level)
+
+class connect(remote):
+    __doc__ = remote.__doc__

pwnlib/tubes/ssh.py

@@ -19,6 +19,7 @@ from ..timeout import Timeout
 from ..util import hashes
 from ..util import misc
 from ..util import safeeval
+from ..util import sh_string
 from .process import process
 from .sock import sock
 
@@ -84,16 +85,16 @@ class ssh_channel(sock):
 
         if isinstance(process, (list, tuple)):
             fmt = ' '.join('%s' for s in process)
-            process = misc.sh_command_with(fmt, *process)
+            process = sh_string.sh_command_with(fmt, *process)
 
         if process and wd:
-            process = misc.sh_command_with('cd %s >/dev/null 2>&1;', wd) + process
+            process = sh_string.sh_command_with('cd %s >/dev/null 2>&1;', wd) + process
 
         if process and env:
             for name, value in env.items():
                 if not re.match('^[a-zA-Z_][a-zA-Z0-9_]*$', name):
                     self.error('run(): Invalid environment key $r' % name)
-                process = '%s;%s' % (misc.sh_prepare(name, value, export=True), process)
+                process = '%s;%s' % (sh_string.sh_prepare(name, value, export=True), process)
 
         if process and tty:
             if raw:
@@ -876,7 +877,7 @@ os.execve(exe, argv, os.environ)
             if not aslr:
                 self.warn_once("ASLR is disabled!")
 
-            script = misc.sh_command_with('for py in python2.7 python2 python; do test -x "$(which $py 2>&1)" && exec $py -c %s check; done; echo 2', script)
+            script = sh_string.sh_command_with('for py in python2.7 python2 python; do test -x "$(which $py 2>&1)" && exec $py -c %s check; done; echo 2', script)
             with context.local(log_level='error'):
                 python = self.run(script, raw=raw)
             result = safeeval.const(python.recvline())
@@ -1147,7 +1148,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
     def _libs_remote(self, remote):
         """Return a dictionary of the libraries used by a remote file."""
         cmd = '(ulimit -s unlimited; ldd %s > /dev/null && (LD_TRACE_LOADED_OBJECTS=1 %s || ldd %s)) 2>/dev/null'
-        cmd = misc.sh_command_with(lambda arg: cmd % (arg, arg, arg), remote)
+        cmd = sh_string.sh_command_with(lambda arg: cmd % (arg, arg, arg), remote)
         data, status = self.run_to_end(cmd)
         if status != 0:
             self.error('Unable to find libraries for %r' % remote)
@@ -1157,7 +1158,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
 
     def _get_fingerprint(self, remote):
         cmd = '(openssl sha256 || sha256 || sha256sum) 2>/dev/null < %s'
-        cmd = misc.sh_command_with(cmd, remote)
+        cmd = sh_string.sh_command_with(cmd, remote)
         data, status = self.run_to_end(cmd)
 
         if status != 0:
@@ -1200,7 +1201,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             self.sftp.get(remote, local, update)
             return
 
-        cmd = misc.sh_command_with('wc -c < %s', remote)
+        cmd = sh_string.sh_command_with('wc -c < %s', remote)
         total, exitcode = self.run_to_end(cmd)
 
         if exitcode != 0:
@@ -1210,7 +1211,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         total = int(total)
 
         with context.local(log_level = 'ERROR'):
-            cmd = misc.sh_command_with('cat < %s', remote)
+            cmd = sh_string.sh_command_with('cat < %s', remote)
             c = self.run(cmd)
         data = ''
 
@@ -1321,7 +1322,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             remote = str(self.sftp.normalize(remote))
         else:
             with context.local(log_level='error'):
-                remote = self.system(misc.sh_command_with('readlink -f %s', remote))
+                remote = self.system(sh_string.sh_command_with('readlink -f %s', remote))
 
         dirname  = os.path.dirname(remote)
         basename = os.path.basename(remote)
@@ -1333,7 +1334,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
 
         with context.local(log_level='error'):
             remote_tar = self.mktemp()
-            tar = self.system(misc.sh_command_with('tar -C %s -czf %s %s', dirname, remote_tar, basename))
+            tar = self.system(sh_string.sh_command_with('tar -C %s -czf %s %s', dirname, remote_tar, basename))
 
             if 0 != tar.wait():
                 self.error("Could not create remote tar")
@@ -1377,7 +1378,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
                 return
 
         with context.local(log_level = 'ERROR'):
-            cmd = misc.sh_command_with('cat>%s', remote)
+            cmd = sh_string.sh_command_with('cat>%s', remote)
             s = self.run(cmd, tty=False)
             s.send(data)
             s.shutdown('send')
@@ -1386,7 +1387,6 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             if result != 0:
                 self.error("Could not upload file %r (%r)\n%s" % (remote, result, data))
 
-
     def upload_file(self, filename, remote = None):
         """Uploads a file to the remote server. Returns the remote filename.
 
@@ -1461,7 +1461,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         if not self.sftp:
             self.error("Cannot determine remote file type without SFTP")
 
-        if 0 == self.system(misc.sh_command_with('test -d %s', file_or_directory)).wait():
+        if 0 == self.system(sh_string.sh_command_with('test -d %s', file_or_directory)).wait():
             self.download_dir(file_or_directory, remote)
         else:
             self.download_file(file_or_directory, remote)
@@ -1517,7 +1517,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         s = self.shell(shell)
 
         if self.cwd != '.':
-            cmd = misc.sh_command_with('cd %s', self.cwd)
+            cmd = sh_string.sh_command_with('cd %s', self.cwd)
             s.sendline(cmd)
 
         s.interactive()
@@ -1559,7 +1559,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
                 self.error("Could not generate a temporary directory (%i)\n%s" % (status, wd))
 
         else:
-            cmd = misc.sh_command_with('ls %s', wd)
+            cmd = sh_string.sh_command_with('ls %s', wd)
             _, status = self.run_to_end(cmd, wd = '.')
 
             if status:
@@ -1568,3 +1568,11 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         self.info("Working directory: %r" % wd)
         self.cwd = wd
         return self.cwd
+
+    def write(self, path, data):
+        """Wrapper around upload_data to match ``pwnlib.util.misc.write``"""
+        return self.upload_data(data, path)
+
+    def read(self, path):
+        """Wrapper around download_data to match ``pwnlib.util.misc.read``"""
+        return self.download_data(path)

pwnlib/update.py

@@ -1,54 +1,83 @@
+"""
+# Pwntools Update
+
+In order to ensure that Pwntools users always have the latest and
+greatest version, Pwntools automatically checks for updates.
+
+Since this update check takes a moment, it is only performed once
+every week.  It can be permanently disabled via:
+
+.. code-block:: bash
+
+    $ echo never > ~/.pwntools-cache/update
+
+"""
 import datetime
 import json
-import pip
-import pkg_resources
 import os
 import time
+import xmlrpclib
+
+import packaging.version
 
 from .context import context
 from .log import getLogger
-from .version import __version__
 from .util.misc import read
 from .util.misc import write
 from .util.web import wget
+from .version import __version__
 
 log = getLogger(__name__)
 
-current_version = pkg_resources.parse_version(__version__)
+current_version = packaging.version.Version(__version__)
 package_name    = 'pwntools'
 package_repo    = 'Gallopsled/pwntools'
 update_freq     = datetime.timedelta(days=7).total_seconds()
 
-def available_on_github(prerelease=False):
-    """Return True if an update is available on Github."""
+def available_on_github(prerelease=current_version.is_prerelease):
+    """Return True if an update is available on Github.
+
+    >>> available_on_github() # doctest: +ELLIPSIS
+    <Version('...')>
+    >>> available_on_github(prerelease=False).is_prerelease
+    False
+    """
     url = 'https://api.github.com/repos/%s/tags' % package_repo
 
     with context.quiet:
-        tags = json.loads(wget(url))
+        tags = json.loads(wget(url, timeout = 60))
 
-
-    versions = map(pkg_resources.parse_version, [t['name'] for t in tags])
+    # 'pwntools-ancient' is a tag, but not a valid version.
+    # Handle this here, and for all potential tags which cause
+    # issues.
+    versions = []
+    for tag in [t['name'] for t in tags]:
+        try:
+            versions.append(packaging.version.Version(tag))
+        except Exception:
+            pass
 
     if not prerelease:
         versions = filter(lambda v: not v.is_prerelease, versions)
 
     return max(versions)
 
-def available_on_pypi(prerelease=False):
-    """Return True if an update is available on PyPI."""
-    search_command = pip.commands.search.SearchCommand()
-    options, _ = search_command.parse_args([package_name])
-    pypi_hits = search_command.search(package_name, options)
-    for hit in pip.commands.search.transform_hits(pypi_hits):
-        if hit['name'] != 'pwntools':
-            continue
+def available_on_pypi(prerelease=current_version.is_prerelease):
+    """Return True if an update is available on PyPI.
 
-        versions = map(pkg_resources.parse_version, hit['versions'])
+    >>> available_on_pypi() # doctest: +ELLIPSIS
+    <Version('...')>
+    >>> available_on_pypi(prerelease=False).is_prerelease
+    False
+    """
+    client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
+    versions = client.package_releases('pwntools', True)
+    versions = map(packaging.version.Version, versions)
 
-        if not prerelease:
-            versions = filter(lambda v: not v.is_prerelease, versions)
+    if not prerelease:
+        versions = filter(lambda v: not v.is_prerelease, versions)
 
-        return max(versions)
+    return max(versions)
 
 def cache_file():
     """Returns the path of the file used to cache update data, and ensures that it exists."""
@@ -73,18 +102,50 @@ def should_check():
         return False
     return time.time() > (last_check() + update_freq)
 
-def perform_check(prerelease=False):
-    """Perform the update check, and report to the user."""
+def perform_check(prerelease=current_version.is_prerelease):
+    """Perform the update check, and report to the user.
+
+    Arguments:
+        prerelease(bool): Whether or not to include pre-release versions.
+
+    Returns:
+        A list of arguments to the update command.
+
+    >>> from packaging.version import Version
+    >>> pwnlib.update.current_version = Version("999.0.0")
+    >>> print perform_check()
+    None
+    >>> pwnlib.update.current_version = Version("0.0.0")
+    >>> perform_check() # doctest: +ELLIPSIS
+    ['pip', 'install', '-U', ...]
+
+    >>> def bail(*a): raise Exception()
+    >>> github = pwnlib.update.available_on_github
+    >>> pypi   = pwnlib.update.available_on_pypi
+
+    >>> pwnlib.update.available_on_github = bail
+    >>> perform_check(prerelease=False)
+    ['pip', 'install', '-U', 'pwntools']
+    >>> perform_check(prerelease=True)  # doctest: +ELLIPSIS
+    ['pip', 'install', '-U', 'pwntools...']
+    >>> pwnlib.update.available_on_github = github
+
+    >>> pwnlib.update.available_on_pypi = bail
+    >>> perform_check(prerelease=False)
+    ['pip', 'install', '-U', 'git+https://github.com/Gallopsled/pwntools.git@...']
+    >>> perform_check(prerelease=True)  # doctest: +ELLIPSIS
+    ['pip', 'install', '-U', 'git+https://github.com/Gallopsled/pwntools.git@...']
+    """
     pypi = current_version
     try:
         pypi = available_on_pypi(prerelease)
-    except:
+    except Exception:
         log.warning("An issue occurred while checking PyPI")
 
     github = current_version
     try:
         github = available_on_github(prerelease)
-    except:
+    except Exception:
         log.warning("An issue occurred while checking Github")
 
     best = max(pypi, github, current_version)
@@ -97,15 +158,26 @@ def perform_check(prerelease=False):
         log.info("You have the latest version of Pwntools (%s)" % best)
         return
 
+    command = [
+        'pip',
+        'install',
+        '-U'
+    ]
+
     if best == pypi:
         where = 'pypi'
-        command = 'pip install -U %s' % package_name
+        pypi_package = package_name
+        if best.is_prerelease:
+            pypi_package += '==%s' % (best)
+        command += [pypi_package]
     else:
         where = 'GitHub'
-        command = 'pip install -U git+https://github.com/%s.git@%s' % (package_repo, github)
+        command += ['git+https://github.com/%s.git@%s' % (package_repo, github)]
+
+    command_str = ' '.join(command)
 
     log.info("A newer version of %s is available on %s (%s --> %s).\n" % (package_name, where, current_version, best) +
-             "Update with: $ %s" % command)
+             "Update with: $ %s" % command_str)
 
     return command
 

pwnlib/util/__init__.py

@@ -15,4 +15,5 @@ from . import net
 from . import packing
 from . import proc
 from . import safeeval
+from . import sh_string
 from . import web

pwnlib/util/cyclic.py

@@ -1,8 +1,8 @@
 import string
 
+from . import packing
 from ..context import context
 from ..log import getLogger
-from . import packing
 
 log = getLogger(__name__)
 

pwnlib/util/misc.py

@@ -7,8 +7,8 @@ import socket
 import stat
 import string
 
-from . import lists
 from . import fiddling
+from . import lists
 from ..context import context
 from ..log import getLogger
 
@@ -189,7 +189,6 @@ def run_in_new_terminal(command, terminal = None, args = None):
 
     Returns:
       None
-
     """
 
     if not terminal:
@@ -214,8 +213,12 @@ def run_in_new_terminal(command, terminal = None, args = None):
     argv = [terminal_path] + args
 
     if isinstance(command, str):
+        if ';' in command:
+            log.error("Cannot use commands with semicolon.  Create a script and invoke that directly.")
         argv += [command]
     elif isinstance(command, (list, tuple)):
+        if any(';' in c for c in command):
+            log.error("Cannot use commands with semicolon.  Create a script and invoke that directly.")
         argv += list(command)
 
     log.debug("Launching a new terminal: %r" % argv)
@@ -271,163 +274,6 @@ def mkdir_p(path):
         else:
             raise
 
-def sh_string(s):
-    """Outputs a string in a format that will be understood by /bin/sh.
-
-    If the string does not contain any bad characters, it will simply be
-    returned, possibly with quotes. If it contains bad characters, it will
-    be escaped in a way which is compatible with most known systems.
-
-    Examples:
-
-        >>> print sh_string('foobar')
-        foobar
-        >>> print sh_string('foo bar')
-        'foo bar'
-        >>> print sh_string("foo'bar")
-        "foo'bar"
-        >>> print sh_string("foo\\\\bar")
-        'foo\\bar'
-        >>> print sh_string("foo\\\\'bar")
-        "foo\\\\'bar"
-        >>> print sh_string("foo\\x01'bar")
-        "$(printf 'foo\\001\\047bar')"
-        >>> print `subprocess.check_output("echo -n " + sh_string("foo\\x01'bar"), shell = True)`
-        "foo\\x01'bar"
-    """
-
-    very_good = set(string.ascii_letters + string.digits)
-    good      = (very_good | set(string.punctuation + ' ')) - set("'")
-    alt_good  = (very_good | set(string.punctuation + ' ')) - set('!')
-
-    if '\x00' in s:
-        log.error("sh_string(): Cannot create a null-byte")
-    if s.endswith('\n'):
-        log.error("sh_string(): Cannot create a newline-terminated string")
-
-    if all(c in very_good for c in s):
-        return s
-    elif all(c in good for c in s):
-        return "'%s'" % s
-    elif all(c in alt_good for c in s):
-        fixed = ''
-        for c in s:
-            if c in '"\\$`':
-                fixed += '\\' + c
-            else:
-                fixed += c
-        return '"%s"' % fixed
-    else:
-        fixed = ''
-        for c in s:
-            if c == '\\':
-                fixed += '\\\\'
-            elif c == '\n':
-                fixed += '\\n'
-            elif c in good:
-                fixed += c
-            else:
-                fixed += '\\%03o' % ord(c)
-        return '"$(printf \'%s\')"' % fixed
-
-def sh_prepare(variables, export = False):
-    """Outputs a posix compliant shell command that will put the data specified
-    by the dictionary into the environment.
-
-    It is assumed that the keys in the dictionary are valid variable names that
-    does not need any escaping.
-
-    Arguments:
-      variables(dict): The variables to set.
-      export(bool): Should the variables be exported or only stored in the shell environment?
-      output(str): A valid posix shell command that will set the given variables.
-
-    It is assumed that `var` is a valid name for a variable in the shell.
-
-    Examples:
-
-        >>> print sh_prepare({'X': 'foobar'})
-        X=foobar
-        >>> r = sh_prepare({'X': 'foobar', 'Y': 'cookies'})
-        >>> r == 'X=foobar;Y=cookies' or r == 'Y=cookies;X=foobar'
-        True
-        >>> print sh_prepare({'X': 'foo bar'})
-        X='foo bar'
-        >>> print sh_prepare({'X': "foo'bar"})
-        X="foo'bar"
-        >>> print sh_prepare({'X': "foo\\\\bar"})
-        X='foo\\bar'
-        >>> print sh_prepare({'X': "foo\\\\'bar"})
-        X="foo\\\\'bar"
-        >>> print sh_prepare({'X': "foo\\x01'bar"})
-        X="$(printf 'foo\\001\\047bar')"
-        >>> print sh_prepare({'X': "foo\\x01'bar"}, export = True)
-        export X="$(printf 'foo\\001\\047bar')"
-        >>> print sh_prepare({'X': "foo\\x01'bar\\n"})
-        X="$(printf 'foo\\001\\047bar\\nx')";X=${X%x}
-        >>> print sh_prepare({'X': "foo\\x01'bar\\n"}, export = True)
-        X="$(printf 'foo\\001\\047bar\\nx')";export X=${X%x}
-        >>> print `subprocess.check_output('%s;echo -n "$X"' % sh_prepare({'X': "foo\\x01'bar"}), shell = True)`
-        "foo\\x01'bar"
-    """
-
-    out = []
-    export = 'export ' if export else ''
-
-    for k, v in variables.items():
-        if v.endswith('\n'):
-            out.append('%s=%s;%s%s=${%s%%x}' % (k, sh_string(v + "x"), export, k, k))
-        else:
-            out.append('%s%s=%s' % (export, k, sh_string(v)))
-    return ';'.join(out)
-
-def sh_command_with(f, *args):
-    """sh_command_with(f, arg0, ..., argN) -> command
-
-    Returns a command create by evaluating `f(new_arg0, ..., new_argN)`
-    whenever `f` is a function and `f % (new_arg0, ..., new_argN)` otherwise.
-
-    If the arguments are purely alphanumeric, then they are simply passed to
-    function. If they are simple to escape, they will be escaped and passed to
-    the function.
-
-    If the arguments contain trailing newlines, then it is hard to use them
-    directly because of a limitation in the posix shell. In this case the
-    output from `f` is prepended with a bit of code to create the variables.
-
-    Examples:
-
-        >>> print sh_command_with(lambda: "echo hello")
-        echo hello
-        >>> print sh_command_with(lambda x: "echo " + x, "hello")
-        echo hello
-        >>> print sh_command_with(lambda x: "echo " + x, "\\x01")
-        echo "$(printf '\\001')"
-        >>> import random
-        >>> random.seed(1)
-        >>> print sh_command_with(lambda x: "echo " + x, "\\x01\\n")
-        dwtgmlqu="$(printf '\\001\\nx')";dwtgmlqu=${dwtgmlqu%x};echo "$dwtgmlqu"
-        >>> random.seed(1)
-        >>> print sh_command_with("echo %s", "\\x01\\n")
-        dwtgmlqu="$(printf '\\001\\nx')";dwtgmlqu=${dwtgmlqu%x};echo "$dwtgmlqu"
-    """
-
-    args = list(args)
-    out = []
-
-    for n in range(len(args)):
-        if args[n].endswith('\n'):
-            v = fiddling.randoms(8)
-            out.append(sh_prepare({v: args[n]}))
-            args[n] = '"$%s"' % v
-        else:
-            args[n] = sh_string(args[n])
-    if hasattr(f, '__call__'):
-        out.append(f(*args))
-    else:
-        out.append(f % tuple(args))
-    return ';'.join(out)
-
 def dealarm_shell(tube):
     """Given a tube which is a shell, dealarm it.
     """

pwnlib/util/net.py

@@ -2,9 +2,9 @@ import ctypes
 import ctypes.util
 import socket
 
-from packing import pack
 from packing import p16
 from packing import p32
+from packing import pack
 
 __all__ = ['getifaddrs', 'interfaces', 'interfaces4', 'interfaces6', 'sockaddr']
 

pwnlib/util/sh_string.py

@@ -0,0 +1,491 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""
+# sh_string
+
+Routines here are for getting any NULL-terminated sequence of bytes evaluated
+intact by any shell.  This includes all variants of quotes, whitespace, and
+non-printable characters.
+
+## Supported Shells
+
+The following shells have been evaluated:
+
+- Ubuntu (dash/sh)
+- MacOS (GNU Bash)
+- Zsh
+- FreeBSD (sh)
+- OpenBSD (sh)
+- NetBSD (sh)
+
+### Debian Almquist shell (Dash)
+
+Ubuntu 14.04 and 16.04 use the Dash shell, and /bin/sh is actually just a
+symlink to /bin/dash.  The feature set supported when invoked as "sh" instead
+of "dash" is different, and we focus exclusively on the "/bin/sh" implementation.
+
+From the [Ubuntu Man Pages][ubuntu], every character except for single-quote
+can be wrapped in single-quotes, and a backslash can be used to escape unquoted
+single-quotes.
+
+   Quoting
+     Quoting is used to remove the special meaning of certain characters or
+     words to the shell, such as operators, whitespace, or keywords.  There
+     are three types of quoting: matched single quotes, matched double quotes,
+     and backslash.
+
+   Backslash
+     A backslash preserves the literal meaning of the following character,
+     with the exception of ⟨newline⟩.  A backslash preceding a ⟨newline⟩ is
+     treated as a line continuation.
+
+   Single Quotes
+     Enclosing characters in single quotes preserves the literal meaning of
+     all the characters (except single quotes, making it impossible to put
+     single-quotes in a single-quoted string).
+
+   Double Quotes
+     Enclosing characters within double quotes preserves the literal meaning
+     of all characters except dollarsign ($), backquote (`), and backslash
+     (\).  The backslash inside double quotes is historically weird, and
+     serves to quote only the following characters:
+           $ ` " \ <newline>.
+     Otherwise it remains literal.
+
+### GNU Bash
+
+The Bash shell is default on many systems, though it is not generally the default
+system-wide shell (i.e., the `system` syscall does not generally invoke it).
+
+That said, its prevalence suggests that it also be addressed.
+
+From the [GNU Bash Manual][bash], every character except for single-quote
+can be wrapped in single-quotes, and a backslash can be used to escape unquoted
+single-quotes.
+
+    3.1.2.1 Escape Character
+
+    A non-quoted backslash ‘\’ is the Bash escape character. It preserves the
+    literal value of the next character that follows, with the exception of
+    newline. If a ``\\newline`` pair appears, and the backslash itself is not
+    quoted, the ``\\newline`` is treated as a line continuation (that is, it
+    is removed from the input stream and effectively ignored).
+
+    3.1.2.2 Single Quotes
+
+    Enclosing characters in single quotes (‘'’) preserves the literal value of
+    each character within the quotes. A single quote may not occur between single
+    uotes, even when preceded by a backslash.
+
+    3.1.2.3 Double Quotes
+
+    Enclosing characters in double quotes (‘"’) preserves the literal value of a
+    ll characters within the quotes, with the exception of ‘$’, ‘`’, ‘\’, and,
+    when history expansion is enabled, ‘!’. The characters ‘$’ and ‘`’ retain their
+    pecial meaning within double quotes (see Shell Expansions). The backslash retains
+    its special meaning only when followed by one of the following characters:
+    ‘$’, ‘`’, ‘"’, ‘\’, or newline. Within double quotes, backslashes that are
+    followed by one of these characters are removed. Backslashes preceding
+    characters without a special meaning are left unmodified. A double quote may
+    be quoted within double quotes by preceding it with a backslash. If enabled,
+    history expansion will be performed unless an ‘!’ appearing in double quotes
+    is escaped using a backslash. The backslash preceding the ‘!’ is not removed.
+
+    The special parameters ‘*’ and ‘@’ have special meaning when in double quotes
+    see Shell Parameter Expansion).
+
+### Z Shell
+
+The Z shell is also a relatively common user shell, even though it's not generally
+the default system-wide shell.
+
+From the [Z Shell Manual][zsh], every character except for single-quote
+can be wrapped in single-quotes, and a backslash can be used to escape unquoted
+single-quotes.
+
+    A character may be quoted (that is, made to stand for itself) by preceding
+    it with a ‘\’. ‘\’ followed by a newline is ignored.
+
+    A string enclosed between ‘$'’ and ‘'’ is processed the same way as the
+    string arguments of the print builtin, and the resulting string is considered
+    o be entirely quoted. A literal ‘'’ character can be included in the string
+    by using the ‘\\'’ escape.
+
+    All characters enclosed between a pair of single quotes ('') that is not
+    preceded by a ‘$’ are quoted. A single quote cannot appear within single
+    quotes unless the option RC_QUOTES is set, in which case a pair of single
+    quotes are turned into a single quote. For example,
+
+    print ''''
+    outputs nothing apart from a newline if RC_QUOTES is not set, but one single
+    quote if it is set.
+
+    Inside double quotes (""), parameter and command substitution occur, and
+    ‘\’ quotes the characters ‘\’, ‘`’, ‘"’, and ‘$’.
+
+### FreeBSD Shell
+
+Compatibility with the FreeBSD shell is included for completeness.
+
+From the [FreeBSD man pages][freebsd], every character except for single-quote
+can be wrapped in single-quotes, and a backslash can be used to escape unquoted
+single-quotes.
+
+     Quoting is used to remove the special meaning of certain characters or
+     words to the shell, such as operators, whitespace, keywords, or alias
+     names.
+
+     There are four types of quoting: matched single quotes, dollar-single
+     quotes, matched double quotes, and backslash.
+
+     Single Quotes
+         Enclosing characters in single quotes preserves the literal mean-
+         ing of all the characters (except single quotes, making it impos-
+         sible to put single-quotes in a single-quoted string).
+
+     Dollar-Single Quotes
+         Enclosing characters between $' and ' preserves the literal mean-
+         ing of all characters except backslashes and single quotes.  A
+         backslash introduces a C-style escape sequence:
+
+         ...
+
+     Double Quotes
+         Enclosing characters within double quotes preserves the literal
+         meaning of all characters except dollar sign (`$'), backquote
+         (``'), and backslash (`\\').  The backslash inside double quotes
+         is historically weird.  It remains literal unless it precedes the
+         following characters, which it serves to quote:
+
+           $     `     "     \     \\n
+
+     Backslash
+         A backslash preserves the literal meaning of the following char-
+         acter, with the exception of the newline character (`\n').  A
+         backslash preceding a newline is treated as a line continuation.
+
+### OpenBSD Shell
+
+From the [OpenBSD Man Pages][openbsd], every character except for single-quote
+can be wrapped in single-quotes, and a backslash can be used to escape unquoted
+single-quotes.
+
+
+    A backslash (\) can be used to quote any character except a newline.
+    If a newline follows a backslash the shell removes them both, effectively
+    making the following line part of the current one.
+
+    A group of characters can be enclosed within single quotes (') to quote
+    every character within the quotes.
+
+    A group of characters can be enclosed within double quotes (") to quote
+    every character within the quotes except a backquote (`) or a dollar
+    sign ($), both of which retain their special meaning. A backslash (\)
+    within double quotes retains its special meaning, but only when followed
+    by a backquote, dollar sign, double quote, or another backslash.
+    An at sign (@) within double quotes has a special meaning
+    (see SPECIAL PARAMETERS, below).
+
+### NetBSD Shell
+
+The NetBSD shell's documentation is identical to the Dash documentation.
+
+### Android Shells
+
+Android has gone through some number of shells.
+
+- Mksh, a Korn shell, was used with Toolbox releases (5.0 and prior)
+- Toybox, also derived from the Almquist Shell (6.0 and newer)
+
+Notably, the Toolbox implementation is not POSIX compliant
+as it lacks a "printf" builtin (e.g. Android 5.0 emulator images).
+
+### Toybox Shell
+
+Android 6.0 (and possibly other versions) use a shell based on ``toybox``.
+
+While it does not include a ``printf`` builtin, ``toybox`` itself includes
+a POSIX-compliant ``printf`` binary.
+
+The Ash shells should be feature-compatible with ``dash``.
+
+### BusyBox Shell
+
+[BusyBox's Wikipedia page][busybox] claims to use an ``ash``-compliant shell,
+and should therefore be compatible with ``dash``.
+
+
+[ubuntu]: http://manpages.ubuntu.com/manpages/trusty/man1/dash.1.html
+[bash]: https://www.gnu.org/software/bash/manual/bash.html#Quoting
+[zsh]: http://zsh.sourceforge.net/Doc/Release/Shell-Grammar.html#Quoting
+[freebsd]: https://www.freebsd.org/cgi/man.cgi?query=sh
+[openbsd]: http://man.openbsd.org/cgi-bin/man.cgi?query=sh#SHELL_GRAMMAR
+[busybox]: https://en.wikipedia.org/wiki/BusyBox#Features
+"""
+import string
+import subprocess
+
+from . import fiddling
+from ..context import context
+from ..log import getLogger
+from ..tubes.process import process
+from .misc import which
+
+log = getLogger(__name__)
+
+def test_all():
+    test('a')
+    test('ab')
+    test('a b')
+    test(r"a\'b")
+    everything_1 = ''.join(chr(c) for c in range(1,256))
+    for s in everything_1:
+        test(s)
+        test(s*4)
+        test(s * 2 + 'X')
+        test('X' + s * 2)
+        test((s*2 + 'X') * 2)
+        test(s + 'X' + s)
+        test(s*2 + 'X' + s*2)
+        test('X' + s*2 + 'X')
+    test(everything_1)
+    test(everything_1 * 2)
+    test(everything_1 * 4)
+    everything_2 = ''.join(chr(c) * 2 for c in range(1,256))
+    test(everything_2)
+
+    test(randoms(1000, everything_1))
+
+
+def test(original):
+    r"""Tests the output provided by a shell interpreting a string
+
+    >>> test('foobar')
+    >>> test('foo bar')
+    >>> test('foo bar\n')
+    >>> test("foo'bar")
+    >>> test("foo\\\\bar")
+    >>> test("foo\\\\'bar")
+    >>> test("foo\\x01'bar")
+    >>> test('\n')
+    >>> test('\xff')
+    >>> test(os.urandom(16 * 1024).replace('\x00', ''))
+    """
+    input = sh_string(original)
+
+    cmdstr = '/bin/echo %s' % input
+
+    SUPPORTED_SHELLS = [
+        ['ash', '-c', cmdstr],
+        ['bash', '-c', cmdstr],
+        ['bash', '-o', 'posix', '-c', cmdstr],
+        ['ksh', '-c', cmdstr],
+        ['busybox', 'ash', '-c', cmdstr],
+        ['busybox', 'sh', '-c', cmdstr],
+        ['zsh', '-c', cmdstr],
+        ['posh', '-c', cmdstr],
+        ['dash', '-c', cmdstr],
+        ['mksh', '-c', cmdstr],
+        ['sh', '-c', cmdstr],
+        # ['adb', 'exec-out', cmdstr]
+    ]
+
+    for shell in SUPPORTED_SHELLS:
+        binary = shell[0]
+
+        if not which(binary):
+            log.warn_once('Shell %r is not available' % binary)
+            continue
+
+        progress = log.progress('%s: %r' % (binary, original))
+
+        with context.quiet:
+            with process(shell) as p:
+                data = p.recvall(timeout=2)
+                p.kill()
+
+        # Remove exactly one trailing newline added by echo
+        # We cannot assume "echo -n" exists.
+        data = data[:-1]
+
+        if data != original:
+            for i,(a,b) in enumerate(zip(data, original)):
+                if a == b:
+                    continue
+                log.error(('Shell %r failed\n' +
+                          'Expect %r\n' +
+                          'Sent   %r\n' +
+                          'Output %r\n' +
+                          'Mismatch @ %i: %r vs %r') \
+                        % (binary, original, input, data, i, a, b))
+
+        progress.success()
+
+
+
+SINGLE_QUOTE = "'"
+ESCAPED_SINGLE_QUOTE = r"\'"
+
+ESCAPED = {
+    # The single quote itself must be escaped, outside of single quotes.
+    "'": "\\'",
+
+    # Slashes must themselves be escaped
+    #
+    # Additionally, some shells coalesce any number N>1 of '\' into
+    # a single backslash literal.
+    # '\\': '"\\\\\\\\"'
+}
+
+def sh_string(s):
+    r"""Outputs a string in a format that will be understood by /bin/sh.
+
+    If the string does not contain any bad characters, it will simply be
+    returned, possibly with quotes. If it contains bad characters, it will
+    be escaped in a way which is compatible with most known systems.
+
+    Warning:
+        This does not play along well with the shell's built-in "echo".
+        It works exactly as expected to set environment variables and
+        arguments, **unless** it's the shell-builtin echo.
+
+    Argument:
+        s(str): String to escape.
+
+    Examples:
+
+        >>> sh_string('foobar')
+        'foobar'
+        >>> sh_string('foo bar')
+        "'foo bar'"
+        >>> sh_string("foo'bar")
+        "'foo'\\''bar'"
+        >>> sh_string("foo\\\\bar")
+        "'foo\\\\bar'"
+        >>> sh_string("foo\\\\'bar")
+        "'foo\\\\'\\''bar'"
+        >>> sh_string("foo\\x01'bar")
+        "'foo\\x01'\\''bar'"
+    """
+    if '\x00' in s:
+        log.error("sh_string(): Cannot create a null-byte")
+
+    chars = set(s)
+    very_good = set(string.ascii_letters + string.digits + "_+.,/")
+
+    # Alphanumeric can always just be used verbatim.
+    if chars <= very_good:
+        return s
+
+    # If there are no single-quotes, the entire thing can be single-quoted
+    if not (chars & set(ESCAPED)):
+        return "'%s'" % s
+
+    # If there are single-quotes, we can single-quote around them, and simply
+    # escape the single-quotes.
+    quoted_string = ''
+    quoted = False
+    for char in s:
+        if char not in ESCAPED:
+            if not quoted:
+                quoted_string += SINGLE_QUOTE
+                quoted = True
+            quoted_string += char
+        else:
+            if quoted:
+                quoted = False
+                quoted_string += SINGLE_QUOTE
+            quoted_string += ESCAPED[char]
+
+    if quoted:
+        quoted_string += SINGLE_QUOTE
+
+    return quoted_string
+
+def sh_prepare(variables, export = False):
+    r"""Outputs a posix compliant shell command that will put the data specified
+    by the dictionary into the environment.
+
+    It is assumed that the keys in the dictionary are valid variable names that
+    does not need any escaping.
+
+    Arguments:
+      variables(dict): The variables to set.
+      export(bool): Should the variables be exported or only stored in the shell environment?
+      output(str): A valid posix shell command that will set the given variables.
+
+    It is assumed that `var` is a valid name for a variable in the shell.
+
+    Examples:
+
+        >>> sh_prepare({'X': 'foobar'})
+        'X=foobar'
+        >>> r = sh_prepare({'X': 'foobar', 'Y': 'cookies'})
+        >>> r == 'X=foobar;Y=cookies' or r == 'Y=cookies;X=foobar'
+        True
+        >>> sh_prepare({'X': 'foo bar'})
+        "X='foo bar'"
+        >>> sh_prepare({'X': "foo'bar"})
+        "X='foo'\\''bar'"
+        >>> sh_prepare({'X': "foo\\\\bar"})
+        "X='foo\\\\bar'"
+        >>> sh_prepare({'X': "foo\\\\'bar"})
+        "X='foo\\\\'\\''bar'"
+        >>> sh_prepare({'X': "foo\\x01'bar"})
+        "X='foo\\x01'\\''bar'"
+        >>> sh_prepare({'X': "foo\\x01'bar"}, export = True)
+        "export X='foo\\x01'\\''bar'"
+        >>> sh_prepare({'X': "foo\\x01'bar\\n"})
+        "X='foo\\x01'\\''bar\\n'"
+        >>> sh_prepare({'X': "foo\\x01'bar\\n"})
+        "X='foo\\x01'\\''bar\\n'"
+        >>> sh_prepare({'X': "foo\\x01'bar\\n"}, export = True)
+        "export X='foo\\x01'\\''bar\\n'"
+    """
+
+    out = []
+    export = 'export ' if export else ''
+
+    for k, v in variables.items():
+        out.append('%s%s=%s' % (export, k, sh_string(v)))
+
+    return ';'.join(out)
+
+def sh_command_with(f, *args):
+    r"""sh_command_with(f, arg0, ..., argN) -> command
+
+    Returns a command create by evaluating `f(new_arg0, ..., new_argN)`
+    whenever `f` is a function and `f % (new_arg0, ..., new_argN)` otherwise.
+
+    If the arguments are purely alphanumeric, then they are simply passed to
+    function. If they are simple to escape, they will be escaped and passed to
+    the function.
+
+    If the arguments contain trailing newlines, then it is hard to use them
+    directly because of a limitation in the posix shell. In this case the
+    output from `f` is prepended with a bit of code to create the variables.
+
+    Examples:
+
+        >>> sh_command_with(lambda: "echo hello")
+        'echo hello'
+        >>> sh_command_with(lambda x: "echo " + x, "hello")
+        'echo hello'
+        >>> sh_command_with(lambda x: "/bin/echo " + x, "\\x01")
+        "/bin/echo '\\x01'"
+        >>> sh_command_with(lambda x: "/bin/echo " + x, "\\x01\\n")
+        "/bin/echo '\\x01\\n'"
+        >>> sh_command_with("/bin/echo %s", "\\x01\\n")
+        "/bin/echo '\\x01\\n'"
+    """
+
+    args = list(args)
+    out = []
+
+    for n in range(len(args)):
+        args[n] = sh_string(args[n])
+    if hasattr(f, '__call__'):
+        out.append(f(*args))
+    else:
+        out.append(f % tuple(args))
+    return ';'.join(out)

pwnlib/util/web.py

@@ -21,10 +21,10 @@ def wget(url, save=None, timeout=5, **kwargs):
     Example:
 
       >>> url    = 'https://httpbin.org/robots.txt'
-      >>> result = wget(url)
+      >>> result = wget(url, timeout=60)
       >>> result
       'User-agent: *\\nDisallow: /deny\\n'
-      >>> result2 = wget(url, True)
+      >>> result2 = wget(url, True, timeout=60)
       >>> result == file('robots.txt').read()
       True
     """

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '3.0.2'
+__version__ = '3.2.0beta0'

setup.py

@@ -20,8 +20,16 @@ for dirpath, dirnames, filenames in os.walk(convert_path('pwnlib/shellcraft/temp
 for scheme in INSTALL_SCHEMES.values():
     scheme['data'] = scheme['purelib']
 
-# Find all of the console scripts
-console_scripts = []
+console_scripts = ['pwn=pwnlib.commandline.main:main']
+
+# Find all of the ancillary console scripts
+# We have a magic flag --include-all-scripts
+flag = '--only-use-pwn-command'
+if flag in sys.argv:
+    sys.argv.remove(flag)
+else:
+    flag = False
+
 for filename in glob.glob('pwnlib/commandline/*'):
     filename = os.path.basename(filename)
     filename, ext = os.path.splitext(filename)
@@ -29,12 +37,13 @@ for filename in glob.glob('pwnlib/commandline/*'):
     if ext != '.py' or '__init__' in filename:
         continue
 
-    script = '%s=pwnlib.commandline.%s:main' % (filename, filename)
-    console_scripts.append(script)
+    script = '%s=pwnlib.commandline.common:main' % filename
+    if not flag:
+        console_scripts.append(script)
 
 install_requires     = ['paramiko>=1.15.2',
                         'mako>=1.0.0',
-                        'pyelftools>=0.2.3',
+                        'pyelftools>=0.2.4',
                         'capstone',
                         'ropgadget>=5.3',
                         'pyserial>=2.7',
@@ -43,7 +52,9 @@ install_requires     = ['paramiko>=1.15.2',
                         'tox>=1.8.1',
                         'pygments>=2.0',
                         'pysocks',
-                        'python-dateutil']
+                        'python-dateutil',
+                        'pypandoc',
+                        'packaging']
 
 # This is a hack until somebody ports psutil to OpenBSD
 if platform.system() != 'OpenBSD':
@@ -56,13 +67,25 @@ if not os.path.exists(PythonH):
     print >> sys.stderr, "$ apt-get install python-dev"
     sys.exit(-1)
 
+# Convert README.md to reStructuredText for PyPI
+long_description = ''
+try:
+    import pypandoc
+    try:
+        pypandoc.get_pandoc_path()
+    except OSError:
+        pypandoc.download_pandoc()
+    long_description = pypandoc.convert_file('README.md', 'rst')
+except ImportError:
+    pass
+
+
 setup(
     name                 = 'pwntools',
     packages             = find_packages(),
-    version              = '3.0.2',
+    version              = '3.2.0beta0',
     data_files           = [('',
-                             ['LICENSE-pwntools.txt',
-                             ]),
+                             glob.glob('*.md') + glob.glob('*.txt')),
                             ],
     package_data         = {
         'pwnlib': [
@@ -75,19 +98,32 @@ setup(
     },
     entry_points = {'console_scripts': console_scripts},
     scripts              = glob.glob("bin/*"),
-    description          = "CTF framework and exploit development library.",
+    description          = "Pwntools CTF framework and exploit development library.",
+    long_description     = long_description,
     author               = "Gallopsled et al.",
     author_email         = "#pwntools @ freenode.net",
     url                  = 'https://pwntools.com',
     download_url         = "https://github.com/Gallopsled/pwntools/releases",
     install_requires     = install_requires,
     license              = "Mostly MIT, some GPL/BSD, see LICENSE-pwntools.txt",
+    keywords             = 'pwntools exploit ctf capture the flag binary wargame overflow stack heap defcon',
     classifiers          = [
-        'Topic :: Security',
+        'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
-        'Operating System :: OS Independent',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Science/Research',
+        'Intended Audience :: System Administrators',
         'License :: OSI Approved :: MIT License',
+        'Natural Language :: English',
+        'Operating System :: POSIX :: Linux',
         'Programming Language :: Python :: 2.7',
-        'Intended Audience :: Developers'
+        'Topic :: Security',
+        'Topic :: Software Development :: Assemblers',
+        'Topic :: Software Development :: Debuggers',
+        'Topic :: Software Development :: Disassemblers',
+        'Topic :: Software Development :: Embedded Systems',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Topic :: System :: System Shells',
+        'Topic :: Utilities',
     ]
 )

shippable.yml

@@ -1,17 +0,0 @@
-language: python
-python:
-  - "2.7"
-before_install:
-  - source .travis_install.sh
-  - source .travis_ssh_setup.sh
-install:
-  - pip install -e .
-  - pip install -r docs/requirements.txt
-before_script:
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
-script:
-  - PWNLIB_NOTERM=1 make -C docs doctest
-

tox.ini

@@ -2,5 +2,4 @@
 envlist = py27
 [testenv]
 deps=-r{toxinidir}/requirements.txt
-	 -r{toxinidir}/docs/requirements.txt
-commands=make PWNLIB_NOTERM=1 -C docs clean doctest
+commands=make PWNLIB_NOTERM=1 -C docs doctest

travis/install.sh

@@ -0,0 +1,180 @@
+#!/usr/bin/env bash -e
+set -e
+local_deb_extract()
+{
+    wget $1
+    ar vx *.deb
+    tar xvf data.tar.*
+    rm -f *.tar.* *deb*
+}
+
+get_binutils()
+{
+    BINUTILS_PREFIX='https://launchpad.net/~pwntools/+archive/ubuntu/binutils/+files/binutils-'
+    BINUTILS_SUFFIX='-linux-gnu_2.22-6ubuntu1.1cross0.11pwntools12~precise_amd64.deb'
+    local_deb_extract "${BINUTILS_PREFIX}${1}${BINUTILS_SUFFIX}"
+}
+
+get_qemu()
+{
+    echo "Installing qemu"
+    QEMU_INDEX='http://packages.ubuntu.com/en/yakkety/amd64/qemu-user-static/download'
+    QEMU_URL=$(curl "$QEMU_INDEX" | grep kernel.org | grep -Eo 'http.*\.deb')
+    local_deb_extract "$QEMU_URL"
+}
+
+setup_travis()
+{
+    export PATH=$PWD/usr/bin:$PATH
+    export LD_LIBRARY_PATH=$PWD/usr/lib
+
+    # Install libbfd-multiarch and libopcodes-multiarch if not found in the cache
+    if [ ! -f usr/lib/libbfd-2.22-multiarch.so ];
+    then
+        # Install the multiarch binutils
+        local_deb_extract http://mirrors.mit.edu/ubuntu/pool/universe/b/binutils/binutils-multiarch_2.22-6ubuntu1_amd64.deb
+        pushd usr/lib
+        ln -sf libbfd-2.22-multiarch.so libbfd-2.22.so
+        ln -sf libopcodes-2.22-multiarch.so libopcodes-2.22.so
+        popd
+    fi
+
+    # Install/upgrade qemu
+    if ! (which qemu-arm-static && qemu-arm-static -version | grep 2.6.1); then
+        get_qemu
+    fi
+
+    # Install our custom binutils
+    which arm-linux-gnu-as     || get_binutils arm
+    which mips-linux-gnu-as    || get_binutils mips
+    which powerpc-linux-gnu-as || get_binutils powerpc
+
+    # Test that the installs worked
+    which arm-linux-gnu-as
+    which mips-linux-gnu-as
+    which powerpc-linux-gnu-as
+    which qemu-arm-static
+
+    # Get rid of files we don't want cached
+    rm -rf usr/share
+}
+
+setup_linux()
+{
+    sudo apt-get install -y software-properties-common openssh-server libncurses5-dev libncursesw5-dev openjdk-8-jre-headless
+    sudo apt-add-repository --yes ppa:pwntools/binutils
+    sudo apt-get update
+    sudo apt-get install binutils-arm-linux-gnu binutils-mips-linux-gnu binutils-powerpc-linux-gnu
+}
+
+setup_android_emulator()
+{
+    # If we are running on Travis CI, and there were no changes to Android
+    # or ADB code, then we do not need the emulator
+    if [ -n "$TRAVIS" ]; then
+        if ! (git log --stat "$TRAVIS_COMMIT_RANGE" | grep -E "android|adb"); then
+            # In order to avoid running the doctests that require the Android
+            # emulator, while still leaving the code intact, we remove the
+            # RST file that Sphinx searches.
+            rm -f 'docs/source/adb.rst'
+
+            # However, the file needs to be present or else things break.
+            touch 'docs/source/adb.rst'
+
+            echo "Skipping Android emulator install, Android tests disabled."
+            return
+        fi
+    fi
+
+
+    if ! which java; then
+        echo "OpenJDK-8-JRE is required for Android stuff"
+        exit 1
+    fi
+
+    if (uname | grep -i Darwin &>/dev/null); then
+        brew install android-sdk android-ndk
+    else
+        if [ ! -f android-sdk/android ]; then
+            # Install the SDK, which gives us the 'android' and 'emulator' commands
+            wget https://dl.google.com/android/android-sdk_r24.4.1-linux.tgz
+            tar xf android-sdk_r24.4.1-linux.tgz
+            rm  -f android-sdk_r24.4.1-linux.tgz
+
+            # Travis caching causes this to exist already
+            rm -rf android-sdk
+
+            mv android-sdk-linux android-sdk
+            file android-sdk/tools/android
+        fi
+
+        export PATH="$PWD/android-sdk/tools:$PATH"
+        export PATH="$PWD/android-sdk/platform-tools:$PATH"
+        which android
+
+        # Install the NDK, which is required for adb.compile()
+        NDK_VERSION=android-ndk-r12b
+        if [ ! -f android-ndk/ndk-build ]; then
+            wget   https://dl.google.com/android/repository/$NDK_VERSION-linux-x86_64.zip
+            unzip -q android-ndk-*.zip
+            rm -f    android-ndk-*.zip
+
+            # Travis caching causes this to exist already
+            rm -rf android-ndk
+
+            mv     $NDK_VERSION android-ndk
+        fi
+
+        export NDK=$PWD/android-ndk
+        export PATH=$NDK:$PATH
+    fi
+
+    # Grab prerequisites
+    echo y | android update sdk --no-ui --all --filter platform-tools,extra-android-support
+    echo y | android update sdk --no-ui --all --filter android-21
+
+    # Valid ABIs:
+    # - armeabi-v7a
+    # - arm64-v8a
+    # - x86
+    # - x86_64
+    ABI='armeabi-v7a'
+
+    # Grab the emulator image
+    echo y | android update sdk --no-ui --all --filter sys-img-$ABI-android-21
+
+    # Create our emulator Android Virtual Device (AVD)
+    echo no | android --silent create avd --name android-$ABI   --target android-21 --force --snapshot --abi $ABI
+
+    # In the future, it would be nice to be able to use snapshots.
+    # However, I haven't gotten them to work nicely.
+    emulator -avd android-$ABI -no-window -no-boot-anim -no-skin -no-audio -no-window -no-snapshot &
+    adb wait-for-device
+    adb shell id
+    adb shell getprop
+}
+
+setup_osx()
+{
+    brew update
+    brew install binutils
+    brew install capstone
+}
+
+if [[ "$USER" == "travis" ]]; then
+    setup_travis
+    setup_android_emulator
+elif [[ "$USER" == "shippable" ]]; then
+    sudo apt-get update
+    sudo apt-get install openssh-server gcc-multilib
+    sudo /usr/sbin/sshd -f /etc/ssh/sshd_config &
+    setup_travis
+elif [[ "$(uname)" == "Darwin" ]]; then
+    setup_osx
+elif [[ "$(uname)" == "Linux" ]]; then
+    setup_linux
+    setup_android_emulator
+fi
+
+
+dpkg -l

travis/ssh_setup.sh

@@ -4,6 +4,8 @@
 #
 # All of the "conditional sudo" is to do container-based builds on
 # Travis which are much, much faster.
+set -e
+
 U=travis
 H=/home/$U
 
@@ -36,7 +38,7 @@ pubkey=$(cat ~/.ssh/$U.pub)
 
 # Set the authorized_keys entry to only permit login from localhost,
 # and only with
-USUDO mkdir $H/.ssh
+USUDO mkdir $H/.ssh || true
 USUDO tee -a $H/.ssh/authorized_keys <<EOF
 from="127.0.0.1" $pubkey
 EOF

travis/update_demo.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+
+if [ "$TRAVIS_BRANCH" = "stable" ] && [ ! -z "$TRY_KEY" ]; then
+    # Start the SSH agent
+    eval "$(ssh-agent -s)"
+
+    # Add the keys
+    echo "$TRY_KEY" > ~/.ssh/try_key
+    chmod 600         ~/.ssh/try_key
+    ssh-add           ~/.ssh/try_key
+
+    # Update!
+    ssh [email protected] -- 'docker/update'
+fi