Release 3.3.4

We cannot determine the length of the file if passed e.g. a tube or a pipe or PTY or character device (e.g. stdin or /dev/null).

Supercedes: 9c2f0f4e
(cherry picked from commit 14f717b7b4)

.github/ISSUE_TEMPLATE.md

@@ -0,0 +1,9 @@
+# Pwntools Issue Template
+
+Thanks for contributing to Pwntools!
+
+When reporting an issue, be sure that you are running the latest released version of pwntools (`pip install --upgrade pwntools`).
+
+Please verify that your issue occurs on 64-bit Ubuntu 14.04.  You can use the Dockerfile in `extra/docker/Dockerfile` to save time.
+
+If possible, provide a proof-of-concept which demonstrates the problem.  Include any binaries or scripts necessary to reproduce the issue, and please include the full debug output via setting the environment variable `PWNLIB_DEBUG=1`.

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,27 @@
+# Pwntools Pull Request
+
+Thanks for contributing to Pwntools!  Take a moment to look at [`CONTRIBUTING.md`][contributing] to make sure you're familiar with Pwntools development.
+
+Please provide a high-level explanation of what this pull request is for.
+
+## Testing
+
+Pull Requests that introduce new code should try to add doctests for that code.  See [`TESTING.md`][testing] for more information.
+
+## Target Branch
+
+Depending on what the PR is for, it needs to target a different branch.
+
+You can always [change the branch][change] after you create the PR if it's against the wrong branch.
+
+| Branch   | Type of PR                                                       |
+| -------- | ---------------------------------------------------------------- |
+| `dev`    | New features, and enhancements
+| `dev`    | Documentation fixes and new tests
+| `stable` | Bug fixes that affect the current `stable` branch
+| `beta`   | Bug fixes that affect the current `beta` branch, but not `stable`
+| `dev`    | Bug fixes for code that has never been released
+
+[contributing]: https://github.com/Gallopsled/pwntools/blob/dev/CONTRIBUTING.md
+[testing]: https://github.com/Gallopsled/pwntools/blob/dev/TESTING.md
+[change]: https://github.com/blog/2224-change-the-base-branch-of-a-pull-request

.gitignore

@@ -14,3 +14,4 @@ dist
 *.egg-info
 *.core
 .coverage
+.tox

.travis.yml

@@ -5,6 +5,14 @@ addons:
     packages:
     - gcc-multilib
     - gcc-4.6-arm-linux-gnueabihf
+    - lib32stdc++6
+    - ash
+    - bash
+    - dash
+    - ksh
+    - mksh
+    - zsh
+    - pandoc
 cache:
     - pip
     - directories:
@@ -13,14 +21,12 @@ cache:
         - /home/travis/virtualenv/python2.7.9/bin/
 python:
   - "2.7"
-after_success:
-  - coveralls
 before_install:
-  - source .travis_install.sh
-  - source .travis_ssh_setup.sh
+  - source travis/install.sh
+  - source travis/ssh_setup.sh
 install:
-  - pip install -e .
-  - pip install -r docs/requirements.txt
+  - pip install --upgrade --editable .
+  - pip install --requirement docs/requirements.txt
 before_script:
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
@@ -28,3 +34,6 @@ before_script:
   - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
 script:
   - PWNLIB_NOTERM=1 coverage run -m sphinx -b doctest docs/source docs/build/doctest
+after_success:
+  - coveralls
+  - source travis/update_demo.sh

.travis_install.sh

@@ -1,88 +0,0 @@
-#!/usr/bin/env bash -e
-local_deb_extract()
-{
-    wget $1
-    ar vx *.deb
-    tar xvf data.tar.*
-    rm -f *.tar.* *deb*
-}
-
-get_binutils()
-{
-    BINUTILS_PREFIX='https://launchpad.net/~pwntools/+archive/ubuntu/binutils/+files/binutils-'
-    BINUTILS_SUFFIX='-linux-gnu_2.22-6ubuntu1.1cross0.11pwntools12~precise_amd64.deb'
-    local_deb_extract "${BINUTILS_PREFIX}${1}${BINUTILS_SUFFIX}"
-}
-
-get_qemu()
-{
-    echo "Installing qemu"
-    QEMU_URL='https://mirrors.kernel.org/ubuntu/pool/universe/q/qemu/qemu-user-static_2.6%2bdfsg-3ubuntu1_amd64.deb'
-    local_deb_extract "$QEMU_URL"
-}
-
-setup_travis()
-{
-    export PATH=$PWD/usr/bin:$PATH
-    export LD_LIBRARY_PATH=$PWD/usr/lib
-
-    if [ ! -d usr/bin ];
-    then
-        # Install our custom binutils
-        which arm-linux-as     || get_binutils arm
-        which mips-linux-as    || get_binutils mips
-        which powerpc-linux-as || get_binutils powerpc
-        which aarch64-linux-as || get_binutils aarch64
-
-        # Install the multiarch binutils
-        local_deb_extract http://mirrors.mit.edu/ubuntu/ubuntu/pool/universe/b/binutils/binutils-multiarch_2.22-6ubuntu1_amd64.deb
-    fi
-
-    if ! (which qemu-arm-static && qemu-arm-static -version | grep 2.6.0); then
-        get_qemu
-    fi
-
-    # Get rid of files we don't want cached
-    rm -rf usr/share
-
-    pushd usr/lib
-    ln -sf libbfd-2.22-multiarch.so libbfd-2.22.so
-    ln -sf libopcodes-2.22-multiarch.so libopcodes-2.22.so
-    popd
-
-    which arm-linux-gnu-as
-    which mips-linux-gnu-as
-    which powerpc-linux-gnu-as
-    which aarch64-linux-gnu-as
-    which qemu-arm-static
-}
-
-setup_linux()
-{
-    sudo apt-get install -y software-properties-common openssh-server libncurses5-dev libncursesw5-dev
-    sudo apt-add-repository --yes ppa:pwntools/binutils
-    sudo apt-get update
-    sudo apt-get install binutils-arm-linux-gnu binutils-mips-linux-gnu binutils-powerpc-linux-gnu
-}
-
-setup_osx()
-{
-    brew update
-    brew install binutils
-    brew install capstone
-}
-
-if [[ "$USER" == "travis" ]]; then
-    setup_travis
-elif [[ "$USER" == "shippable" ]]; then
-    sudo apt-get update
-    sudo apt-get install openssh-server gcc-multilib
-    sudo /usr/sbin/sshd -f /etc/ssh/sshd_config &
-    setup_travis
-elif [[ "$(uname)" == "Darwin" ]]; then
-    setup_osx
-elif [[ "$(uname)" == "Linux" ]]; then
-    setup_linux
-fi
-
-dpkg -l

CHANGELOG.md

@@ -3,17 +3,210 @@
 This changelog only includes added major features and changes. Bugfixes and
 minor changes are omitted.
 
-## 3.0.2 (September 6 2016)
+## Release History
+
+The table below shows which release corresponds to each branch, and what date the version was released.
+
+| Version          | Branch   | Release Date           |
+| ---------------- | -------- | ---------------------- |
+| [3.5.0](#350)    | `dev`    | Mar 18, 2017 (planned)
+| [3.4.0](#340)    | `beta`   | Feb 4, 2017 (planned)
+| [3.3.4](#334)    | `stable` | Jan 12, 2016
+| [3.3.3](#333)    |          | Jan 10, 2016
+| [3.3.2](#332)    |          | Jan 10, 2016
+| [3.3.1](#331)    |          | Jan 10, 2016
+| [3.3.0](#330)    |          | Dec 24, 2016
+| [3.2.1](#321)    |          | Dec 24, 2016
+| [3.2.0](#320)    |          | Nov 12, 2016
+| [3.1.1](#311)    |          | Oct 23, 2016
+| [3.1.0](#310)    |          | Oct 2, 2016
+| [3.0.4](#304)    |          | Sept 19, 2016
+| [3.0.3](#303)    |          | Sept 18, 2016
+| [3.0.2](#302)    |          | Sept 6, 2016
+| [3.0.1](#301)    |          | Aug 20, 2016
+| [3.0.0](#300)    |          | Aug 20, 2016
+| [2.2.0](#220)    |          | Jan 5, 2015
+
+## 3.5.0
+
+To be released on Mar 18, 2017.
+
+## 3.4.0
+
+To be released on Feb 4, 2017.
+
+## 3.3.4
+
+- [#850][850] and [#846][846] fix issues with `hexdump` and the `phd` command-line utility, when using pipes (e.g. `echo foo | phd`)
+- [#852][852] Fixes register ordering in `regsort`
+- [#853][853] Fixes the registers restored in `shellcraft.amd64.popad`
+
+[846]: https://github.com/gallopsled/pwntools/pull/846
+[850]: https://github.com/gallopsled/pwntools/pull/850
+[852]: https://github.com/gallopsled/pwntools/pull/852
+[853]: https://github.com/gallopsled/pwntools/pull/853
+
+## 3.3.3
+
+- [#843][843] fixed a bug in `amd64.mov`.
+
+[843]: https://github.com/gallopsled/pwntools/pull/843
+
+## 3.3.2
+
+- [#840][840] fixed a regression introduced by [#837][837].
+
+[840]: https://github.com/gallopsled/pwntools/pull/840
+
+## 3.3.1
+
+- [#833][833] Fixed a performance-impacting bug in the adb module.
+- [#837][837] Fixed a bug([#836][836]) causing `hexdump(cyclic=True)` to throw an exception.
+
+[833]: https://github.com/Gallopsled/pwntools/pull/833
+[837]: https://github.com/Gallopsled/pwntools/pull/837
+[836]: https://github.com/Gallopsled/pwntools/issues/836
+
+## 3.3.0
+
+- [b198ec8][b198ec8] Added `tube.stream()` function, which is like `tube.interact()` without a prompt or keyboard input.
+    + Effectively, this is similar to `cat file` and just prints data as fast as it is received.
+- [aec3fa6][aec3fa6] Disable update checks against GitHub
+    + These checks frequently broke due to GitHub query limits
+- [#757][757] Fixed `adb.wait_for_device()` re-use of the same connection
+- [f9133b1][f9133b1] Add a `STDERR` magic argument to make logging go to `stderr` instead of `stdout`
+    + Usage is e.g. `python foo.py STDERR` or `PWNLIB_STDERR=1 python foo.py`
+    + Also adds `context.log_console` to log to any file or terminal
+- [67e11a9][67e11a9] Add faster error checking to `cyclic()` when provided very large values
+- [5fda658][5fda658] Expose BitPolynom in `globals()`
+- [#765][765] Added `-d` option for hex-escaped output for `shellcraft` command-line tool
+- [#772][772] Fixed bash completion regressions
+- [30c34b7][30c34b7] Fix `ROP.call()` with `Function` objects from `ELF.functions`
+- [fa402ce][fa402ce] Add `adb.uptime` and `adb.boot_time`
+- [82312ba][82312ba] Add `cyclic_metasploit` and `cyclic_metasploit_find`
+
+[757]: https://github.com/Gallopsled/pwntools/pull/757
+[765]: https://github.com/Gallopsled/pwntools/pull/765
+[772]: https://github.com/Gallopsled/pwntools/pull/772
+[b198ec8]: https://github.com/Gallopsled/pwntools/commit/b198ec8
+[aec3fa6]: https://github.com/Gallopsled/pwntools/commit/aec3fa6
+[f9133b1]: https://github.com/Gallopsled/pwntools/commit/f9133b1
+[67e11a9]: https://github.com/Gallopsled/pwntools/commit/67e11a9
+[5fda658]: https://github.com/Gallopsled/pwntools/commit/5fda658
+[30c34b7]: https://github.com/Gallopsled/pwntools/commit/30c34b7
+[fa402ce]: https://github.com/Gallopsled/pwntools/commit/fa402ce
+[82312ba]: https://github.com/Gallopsled/pwntools/commit/82312ba
+
+## 3.2.1
+
+Multiple bug fixes.
+
+- [#783][783] Fix `adb.uninstall` typo
+- [#787][787] Added error handling for `ssh.process` argument `preexec_fn`
+- [#793][793] Fixed progress message in `remote()` when connections failed
+- [#802][802] Fixed partition listing in `adb.partitions`, which accidentally shelled out to the `adb` binary
+- [#804][804] Fix error message for 32-bit distributions
+- [#805][805] Fix exception in `Core.segments` when a segment has no name
+- [#811][811] Fixes and performance improvements for `adb.wait_for_device()`
+- [#813][813] Fixed a release script
+- [#814][814] Fixed exceptions thrown if the `$HOME` directory is not writable
+- [#815][815] Properly handle `None` in `MemLeak`
+
+[783]: https://github.com/Gallopsled/pwntools/pull/783
+[787]: https://github.com/Gallopsled/pwntools/pull/787
+[793]: https://github.com/Gallopsled/pwntools/pull/793
+[802]: https://github.com/Gallopsled/pwntools/pull/802
+[804]: https://github.com/Gallopsled/pwntools/pull/804
+[805]: https://github.com/Gallopsled/pwntools/pull/805
+[811]: https://github.com/Gallopsled/pwntools/pull/811
+[813]: https://github.com/Gallopsled/pwntools/pull/813
+[814]: https://github.com/Gallopsled/pwntools/pull/814
+[815]: https://github.com/Gallopsled/pwntools/pull/815
+
+## 3.2.0
+
+- [#695][695] Fixed a performance regression in `phd`.
+- [452605e][452605e] Fixed [#629][629] related to correct removal of temporary files.
+- [ea94ee4][ea94ee4] Disallows semi-colons in for the `run_in_terminal` function, since it did not work properly in all cases.
+- [6376d07][6376d07] Added the mips shellcode `pushstr_array`.
+- [#700][700] Added missing MIPS shellcode documentation to readthedocs, and enabled unit tests
+- [#701][701] Command line tools refactored to have a common `pwn` entry point.
+    + Added an option to *not* install the traditional `asm`, `disasm`, `checksec`, etc scripts
+    + All existing tools can be accessed from the `pwn` command (e.g. `pwn asm nop`).
+- [#704][704] The `process` object has a new, optional argument `alarm` for setting a `SIGALRM` timeout for processes.
+- [#705][705] Added the Android Emulator to the test suite and Travis CI.
+    + Android Emulator is now required for the full test suite
+    + Android Emulator tests are skipped if no Android-related changes are detected
+- [#711][711] `DynELF` has a new attribute, `heap`, which leaks the current `brk` address (heap base).  This is useful for finding heap allocations with dlmalloc-derived allocators like those used by Glibc.
+- [#717][717] `sh_string` was rewritten to emit more compact and compatible strings
+    + This was achieved by embedding single-quoted non-printable literals
+    + Much more testing was added
+    + Emitted strings are no longer copy-paste compatible, but work fine with e.g. `tubes` module and the default `subprocess` module
+- [#709][709] The `adb` module now directly talks to the `adb` server process via a new module, `adb.protocol`
+    + Removes the need to shell out to `adb`
+    + Avoids version-compatibility issues with `adb` server vs. client
+- [#703][703] Added new methods to `adb`
+    + `install` - Installs an APK
+    + `uninstall` - Uninstalls a package
+    + `packages` - Lists installed packages
+- [4893819][4893819] Modified `shellcraft.sh` on all platforms to provide `argv[0]` and set `argc==1`
+    + This is needed for systems which have Busybox or other minimal shell for `/bin/sh` which does not behave well with `argc==0` or `argv[0]==NULL`.
+- [1e414af][1e414af] Added `connect()` alias for `remote()`
+    + For example, `io=connect('google.com', 80)`
+    + This also works with `tcp(...)` and `udp(...)` aliases
+- [869ec42][869ec42] Added `ssh.read()` and `ssh.write()` aiases
+- [2af55c9][2af55c9] `AdbDevice` objects exposed via e.g. `adb.devices()` now offer scoped access to all `adb` module properties
+    + It is now possible to e.g. `map(lambda d: d.process(['id']).recvall(), adb.devices())`
+
+
+[629]: https://github.com/Gallopsled/pwntools/issues/629
+[695]: https://github.com/Gallopsled/pwntools/pull/695
+[700]: https://github.com/Gallopsled/pwntools/pull/700
+[701]: https://github.com/Gallopsled/pwntools/pull/701
+[704]: https://github.com/Gallopsled/pwntools/pull/704
+[711]: https://github.com/Gallopsled/pwntools/pull/711
+[717]: https://github.com/Gallopsled/pwntools/pull/717
+[709]: https://github.com/Gallopsled/pwntools/pull/709
+[705]: https://github.com/Gallopsled/pwntools/pull/705
+[703]: https://github.com/Gallopsled/pwntools/pull/703
+[452605e]: https://github.com/Gallopsled/pwntools/commit/452605e854f4870ef5ccfdf7fb110dfd75c50feb
+[ea94ee4]: https://github.com/Gallopsled/pwntools/commit/ea94ee4ca5a8060567cc9bd0dc33796a89ad0b95
+[6376d07]: https://github.com/Gallopsled/pwntools/commit/6376d072660fb2250f48bd22629bbd7e3c61c758
+[1e414af]: https://github.com/Gallopsled/pwntools/commit/1e414afbeb3a01242f4918f111febaa63b640eb7
+[869ec42]: https://github.com/Gallopsled/pwntools/commit/869ec42082b4b98958dfe85103da9b101dde7daa
+[4893819]: https://github.com/Gallopsled/pwntools/commit/4893819b4c23182da570e2f4ea4c14d73af2c0df
+[2af55c9]: https://github.com/Gallopsled/pwntools/commit/2af55c9bc382eca23f89bc0abc7a07c075521f94
+
+## 3.1.1
+
+Fixed a bug in `MemLeak.struct` (PR: #768).
+
+## 3.1.0
+
+A number of smaller bugfixes and documentation tweaks.
+
+## 3.0.4
+
+- Fixed a bug that made 3.0.3 uninstallable (Issue: #751, PR: #752)
+
+## 3.0.3
+
+- Fixed some performance and usability problems with the update system (Issues:
+  #723, #724, #736. PRs: #729, #738, #747).
+- Fixed a bug related to internals in pyelftools (PRs: #730, #746).
+- Fixed an issue with travis (Issue: #741, PRs: #743, #744, #745).
+
+## 3.0.2
 
 - Cherry-pick #695, as this was a regression-fix.
 - Added a fix for the update checker, as it would suggest prereleases as updates to stable releases.
 - Various documentation fixes.
 
-## 3.0.1 (August 20 2016)
+## 3.0.1
 
 A small bugfix release. There were a lot of references to the `master`-branch, however after 3.0.0 we use the names `stable`, `beta` and `dev` for our branches.
 
-## 3.0.0 (August 20 2016)
+## 3.0.0
 
 This was a large release (1305 commits since 2.2.0) with a lot of bugfixes and changes.  The Binjitsu project, a fork of Pwntools, was merged back into Pwntools.  As such, its features are now available here.
 
@@ -49,7 +242,7 @@ Android support via a new `adb` module, `context.device`, `context.adb_host`, an
 - Added `context.noptrace` setting for disabling actions which require `ptrace` support.  This is useful for turning all `gdb.debug` and `gdb.attach` options into no-ops, and can be set via the `NOPTRACE` magic command-line option.
 - Added `context.proxy` which hooks all connections and sends them to a SOCKS4/SOCKS5.  This can be set via the `PROXY` magic command-line option.
 - Added `context.randomize` to control randommization of settings like XOR keys and register ordering (default off).
-- Added `context.termianl` for setting how to launch commands in a new terminal.
+- Added `context.terminal` for setting how to launch commands in a new terminal.
 
 #### DynELF and MemLeak Module
 

MANIFEST.in

@@ -1 +1,7 @@
-include *.txt
+graft examples
+graft extra
+include *.md *.txt *.sh *.yml MANIFEST.in
+recursive-include docs   *.rst *.png Makefile *.py *.txt
+recursive-include pwnlib *.py *.asm *.rst *.md *.txt *.sh
+recursive-include pwn    *.py *.asm *.rst *.md *.txt *.sh
+recursive-exclude *.pyc

README.md

@@ -2,7 +2,7 @@
 ![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)
 
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![PyPI](https://img.shields.io/pypi/v/pwntools.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
+[![PyPI](https://img.shields.io/badge/pypi-v3.3.4-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
 [![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)
@@ -22,13 +22,10 @@ r.interactive()
 
 # Try It Now!
 
-You can now do a live demo of Pwntools, [right in your browser](http://demo.pwntools.com).  Alternately, you can SSH to the same host, and log in as user `zerocool` with [this private key][key] (password `i_promise_not_to_be_evil`).
-
-It will drop you into a clean, Docker-ized container.  There is nothing of value on the VPS, so please don't be evil.
-
-[key]: https://gist.githubusercontent.com/zachriggle/efa2e0080ae6de2e8344/raw/4b503e9db54f009d97477d03d4ba5678471f8ff0/id_rsa
+You can now do a live demo of Pwntools, [right in your browser](https://demo.pwntools.com).
 
 # Documentation
+
 Our documentation is available at [docs.pwntools.com](https://docs.pwntools.com/)
 
 To get you started, we've provided some example solutions for past CTF challenges in our [write-ups repository](https://github.com/Gallopsled/pwntools-write-ups).
@@ -45,9 +42,6 @@ apt-get install python2.7 python-pip python-dev git libssl-dev
 pip install --upgrade pwntools
 ```
 
-If you'd like to be an early tester of the the current beta release, add
-`--pre` to the command-line.
-
 However, some of the features (assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://docs.pwntools.com/en/stable/install.html).
 
 

TESTING.md

@@ -5,15 +5,13 @@ Pwntools makes extensive use of unit tests and integration tests to ensure every
 ## Test Suite
 
 
-To run the test suite, it is beste to use Ubuntu 12.04 or 14.04, and run the following commands.  **Be aware** that this will add a user to the machine, and create a public key for SSH login!
+To run the test suite, it is best to use Ubuntu 12.04 or 14.04, and run the following commands.  **Be aware** that this will add a user to the machine, and create a public key for SSH login!
 
 ```sh
-bash .travis_install.sh
-bash .travis_ssh_setup.sh
-pip install -e .
-pip install -r docs/requirements.txt
-cd docs
-PWNLIB_NOTERM=1 make clean doctest
+bash travis/install.sh
+bash travis/ssh_setup.sh
+pip install --upgrade --editable .
+PWNLIB_NOTERM=1 make -C docs doctest
 ```
 
 ## New Tests

docs/requirements.txt

@@ -1,5 +1,5 @@
 sphinxcontrib-napoleon
-sphinxcontrib-autoprogram
+sphinxcontrib-autoprogram==0.1.2
 isort
 paramiko>=1.15.2
 mako>=1.0.0

docs/source/adb.rst

@@ -1,9 +1,13 @@
 .. testsetup:: *
 
    from pwn import *
+   adb = pwnlib.adb
 
 :mod:`pwnlib.adb` --- Android Debug Bridge
 =====================================================
 
-.. automodule:: pwnlib.adb
+.. automodule:: pwnlib.adb.adb
+   :members:
+
+.. automodule:: pwnlib.adb.protocol
    :members:

docs/source/args.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.args` --- Magic Command-Line Arguments
+=====================================================
+
+.. automodule:: pwnlib.args
+   :members:

docs/source/commandline.rst

@@ -14,45 +14,5 @@ pwntools comes with a handful of useful command-line utilities which serve as wr
 
 .. toctree::
 
-.. autoprogram:: pwnlib.commandline.asm:parser
-   :prog: asm
-
-.. autoprogram:: pwnlib.commandline.checksec:parser
-   :prog: checksec
-
-.. autoprogram:: pwnlib.commandline.constgrep:p
-   :prog: constgrep
-
-.. autoprogram:: pwnlib.commandline.cyclic:parser
-   :prog: cyclic
-
-.. autoprogram:: pwnlib.commandline.disasm:parser
-   :prog: disasm
-
-.. autoprogram:: pwnlib.commandline.elfdiff:p
-   :prog: elfdiff
-
-.. autoprogram:: pwnlib.commandline.elfpatch:p
-   :prog: elfpatch
-
-.. autoprogram:: pwnlib.commandline.errno:parser
-   :prog: errno
-
-.. autoprogram:: pwnlib.commandline.hex:parser
-   :prog: hex
-
-.. autoprogram:: pwnlib.commandline.phd:parser
-   :prog: phd
-
-.. autoprogram:: pwnlib.commandline.pwnstrip:p
-   :prog: pwnstrip
-
-.. autoprogram:: pwnlib.commandline.scramble:parser
-   :prog: scramble
-
-.. autoprogram:: pwnlib.commandline.shellcraft:p
-   :prog: shellcraft
-
-.. autoprogram:: pwnlib.commandline.unhex:parser
-   :prog: unhex
-
+.. autoprogram:: pwnlib.commandline.main:parser
+   :prog: pwn

docs/source/conf.py

@@ -55,6 +55,16 @@ pwnlib.context.ContextType.defaults['randomize'] = False
 pwnlib.term.text.when = 'never'
 pwnlib.log.install_default_handler()
 pwnlib.log.rootlogger.setLevel(1)
+
+# Sphinx modifies sys.stdout, and context.log_terminal has
+# a reference to the original instance.  We need to update
+# it for logging to be captured.
+class stdout(object):
+    def __getattr__(self, name):
+        return getattr(sys.stdout, name)
+    def __setattr__(self, name, value):
+        return setattr(sys.stdout, name, value)
+pwnlib.context.ContextType.defaults['log_console'] = stdout()
 '''
 
 autodoc_member_order = 'alphabetical'

docs/source/flag.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+
+
+:mod:`pwnlib.flag` --- CTF Flag Management
+==========================================
+
+.. automodule:: pwnlib.flag
+   :members:

docs/source/index.rst

@@ -43,6 +43,7 @@ Each of the ``pwntools`` modules is documented here.
    :glob:
 
    adb
+   args
    asm
    atexception
    atexit
@@ -52,6 +53,7 @@ Each of the ``pwntools`` modules is documented here.
    encoders
    elf
    exception
+   flag
    fmtstr
    gdb
    log
@@ -67,6 +69,7 @@ Each of the ``pwntools`` modules is documented here.
    tubes
    tubes/*
    ui
+   update
    useragents
    util/*
 

docs/source/install.rst

@@ -27,14 +27,6 @@ pwntools is available as a ``pip`` package.
     $ apt-get install python2.7 python2.7-dev python-pip
     $ pip install --upgrade pwntools
 
-Alternatively you can get early access to the beta release if you want to help
-file bugs or simple get newer features:
-
-.. code-block:: bash
-
-    $ pip install --upgrade --pre pwntools
-
-
 Development
 --------------
 
@@ -43,7 +35,6 @@ If you are hacking on Pwntools locally, you'll want to do something like this:
 .. code-block:: bash
 
     $ git clone https://github.com/Gallopsled/pwntools
-    $ cd pwntools
-    $ pip install -e .
+    $ pip install --upgrade --editable ./pwntools
 
 .. _Ubuntu: https://launchpad.net/~pwntools/+archive/ubuntu/binutils

docs/source/install/binutils.rst

@@ -11,7 +11,10 @@ Building `binutils` from source takes about 60 seconds on a modern 8-core machin
 
 Ubuntu
 ^^^^^^^^^^^^^^^^
-First, add our `Personal Package Archive repository <http://binutils.pwntools.com>`__.
+
+For Ubuntu 12.04 through 15.10, you must first add the pwntools `Personal Package Archive repository <http://binutils.pwntools.com>`__.
+
+Ubuntu Xenial (16.04) has official packages for most architectures, and does not require this step.
 
 .. code-block:: bash
 

docs/source/intro.rst

@@ -176,8 +176,8 @@ loaded with useful time-saving shellcodes.
 Let's say that we want to `setreuid(getuid(), getuid())` followed by `dup`ing
 file descriptor 4 to `stdin`, `stdout`, and `stderr`, and then pop a shell!
 
-    >>> asm(shellcraft.setreuid() + shellcraft.dupsh(4)).encode('hex')
-    '6a3158cd8089c36a465889d9cd806a045b6a0359496a3f58cd8075f86a68682f2f2f73682f62696e6a0b5889e331c999cd80'
+    >>> asm(shellcraft.setreuid() + shellcraft.dupsh(4)).encode('hex') # doctest: +ELLIPSIS
+    '6a3158cd80...'
 
 
 Misc Tools

docs/source/shellcraft/mips.rst

@@ -0,0 +1,19 @@
+.. testsetup:: *
+
+   from pwn import *
+   context.clear(arch='mips')
+
+:mod:`pwnlib.shellcraft.mips` --- Shellcode for MIPS
+===========================================================
+
+:mod:`pwnlib.shellcraft.mips`
+-----------------------------
+
+.. automodule:: pwnlib.shellcraft.mips
+   :members:
+
+:mod:`pwnlib.shellcraft.mips.linux`
+-----------------------------------
+
+.. automodule:: pwnlib.shellcraft.mips.linux
+   :members:

docs/source/update.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+   from pwnlib.update import *
+
+:mod:`pwnlib.update` --- Updating Pwntools
+===================================================
+
+.. automodule:: pwnlib.update
+   :members:

docs/source/util/sh_string.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+   test = pwnlib.util.sh_string.test
+
+:mod:`pwnlib.util.sh_string` --- Shell Expansion is Hard
+===============================================================
+
+.. automodule:: pwnlib.util.sh_string
+   :members:

extra/bash_completion.d/shellcraft

@@ -1,5 +1,6 @@
+#!/usr/bin/env bash
 # cache list of shellcodes
-_shellcraft_shellcodes=$(shellcraft)
+_shellcraft_shellcodes=$(pwn shellcraft -l)
 
 _shellcraft()
 {
@@ -43,4 +44,5 @@ _shellcraft()
     COMPREPLY=( $(compgen -W "${_shellcraft_shellcodes}" -- ${cur}) )
 }
 
+complete -F _shellcraft pwn shellcraft
 complete -F _shellcraft shellcraft

pwn/__init__.py

@@ -13,4 +13,5 @@ if not platform.architecture()[0].startswith('64'):
     See Gallopsled/pwntools#518 for more information."""
     log.warn_once('Pwntools does not support 32-bit Python.  Use a 64-bit release.')
 
-pwnlib.update.check_automatically()
+with context.local(log_console=sys.stderr):
+    pwnlib.update.check_automatically()

pwn/toplevel.py

@@ -6,6 +6,7 @@ import operator
 import os
 import re
 import socks
+import signal
 import string
 import struct
 import subprocess
@@ -14,6 +15,8 @@ import tempfile
 import threading
 import time
 
+from pprint import pprint
+
 import pwnlib
 from pwnlib import *
 from pwnlib.asm import *
@@ -26,6 +29,7 @@ from pwnlib.elf import ELF
 from pwnlib.elf import load
 from pwnlib.encoders import *
 from pwnlib.exception import PwnlibException
+from pwnlib.flag import *
 from pwnlib.fmtstr import FmtStr, fmtstr_payload
 from pwnlib.log import getLogger
 from pwnlib.memleak import MemLeak
@@ -37,7 +41,7 @@ from pwnlib.runner import *
 from pwnlib.timeout import Timeout
 from pwnlib.tubes.listen import listen
 from pwnlib.tubes.process import process
-from pwnlib.tubes.remote import remote, tcp, udp
+from pwnlib.tubes.remote import remote, tcp, udp, connect
 from pwnlib.tubes.serialtube import serialtube
 from pwnlib.tubes.ssh import ssh
 from pwnlib.tubes.tube import tube
@@ -47,6 +51,7 @@ from pwnlib.util import iters
 from pwnlib.util import net
 from pwnlib.util import proc
 from pwnlib.util import safeeval
+from pwnlib.util.crc import BitPolynom
 from pwnlib.util.cyclic import *
 from pwnlib.util.fiddling import *
 from pwnlib.util.getdents import *
@@ -55,6 +60,7 @@ from pwnlib.util.lists import *
 from pwnlib.util.misc import *
 from pwnlib.util.packing import *
 from pwnlib.util.proc import pidof
+from pwnlib.util.sh_string import sh_string, sh_prepare, sh_command_with
 from pwnlib.util.splash import *
 from pwnlib.util.web import *
 

pwnlib/adb/__init__.py

@@ -0,0 +1,2 @@
+from .adb import *
+from .protocol import Client

pwnlib/adb/protocol.py

@@ -0,0 +1,553 @@
+"""
+Implementation of the Android Debug Bridge (ADB) protocol, as far as Binjitsu needs it.
+
+Documentation is available here:
+https://android.googlesource.com/platform/system/core/+/master/adb/protocol.txt
+"""
+import functools
+import stat
+import time
+
+from ..context import context
+from ..log import Logger
+from ..log import getLogger
+from ..tubes.listen import listen
+from ..tubes.process import process
+from ..tubes.remote import remote
+from ..util.lists import group
+from ..util.misc import size
+from ..util.packing import p32
+from ..util.proc import pidof
+from ..util.sh_string import sh_string
+
+log = getLogger(__name__)
+
+def pack(val):
+    return '%04x' % val
+
+def unpack(val):
+    return int(val, 16)
+
+OKAY = "OKAY"
+FAIL = "FAIL"
+
+class Message(object):
+    """An ADB hex-length-prefixed message"""
+    def __init__(self, string):
+        self.string = string
+    def __str__(self):
+        return ('%04x' % len(self.string)) + self.string
+    def __flat__(self):
+        return str(self)
+
+class Connection(remote):
+    """Connection to the ADB server"""
+    def __init__(self, host, port, level=None, *a, **kw):
+        super(Connection, self).__init__(host, port, level=level, *a, **kw)
+
+        self._executable = None
+        self._argv       = None
+        self._pid        = None
+        self._cwd        = None
+        self._env        = None
+
+    def close(self):
+        with context.quiet:
+            super(Connection, self).close()
+
+    def adb_send(self, message):
+        self.send(str(Message(message)))
+        return self.recvn(4)
+
+    def adb_unpack(self):
+        return unpack(self.recvn(4))
+
+    def flat(self, *a, **kw):
+        kw.setdefault('word_size', 32)
+        return super(Connection, self).flat(*a, **kw)
+
+class Process(Connection):
+    """Duck-typed ``tubes.remote`` object to add properties of a ``tubes.process``"""
+
+class Client(Logger):
+    """ADB Client"""
+    def __init__(self, level=None):
+        super(Client, self).__init__()
+
+        if level is not None:
+            self.setLevel(level)
+
+        self.host = context.adb_host
+        self.port = context.adb_port
+        self._c   = None
+
+    @property
+    def c(self):
+        """Client's connection to the ADB server"""
+        if not self._c:
+            try:
+                self._c = Connection(self.host, self.port, level=self.level)
+            except Exception:
+                # If the connection fails, try starting a server on that port
+                # as long as it's the *default* port.
+                if self.host == context.defaults['adb_host'] \
+                and self.port == context.defaults['adb_port']:
+                    log.warn("Could not connect to ADB server, trying to start it")
+                    process(context.adb + ['start-server']).recvall()
+                else:
+                    log.exception('Could not connect to ADB server')
+
+        # Final attempt...
+        if not self._c:
+            self._c = Connection(self.host, self.port, level=self.level)
+        return self._c
+
+    def _autoclose(fn):
+        """Decorator which automatically closes the connection to the ADB server
+        after calling the decorated function."""
+        @functools.wraps(fn)
+        def wrapper(self, *a, **kw):
+            rv = fn(self, *a, **kw)
+            if self._c:
+                self._c.close()
+                self._c = None
+            return rv
+        return wrapper
+
+    def _with_transport(fn):
+        """Decorator which automatically selects a device transport before calling
+        the decorated function, and closes the connection afterward."""
+        @functools.wraps(fn)
+        def wrapper(self, *a, **kw):
+            self.transport()
+            rv = fn(self, *a, **kw)
+            if self._c:
+                self._c.close()
+                self._c = None
+            return rv
+        return wrapper
+
+    def send(self, *a, **kw):
+        """Sends data to the ADB server"""
+        return self.c.adb_send(*a, **kw)
+
+    def unpack(self, *a, **kw):
+        """Receives a hex-ascii packed integer from the ADB server"""
+        return self.c.adb_unpack(*a, **kw)
+
+    def recvl(self):
+        """Receives a length-prefixed data buffer from the ADB server"""
+        length = self.c.adb_unpack()
+        return self.c.recvn(length)
+
+    @_autoclose
+    def kill(self):
+        """Kills the remote ADB server"
+
+        >>> c=adb.protocol.Client()
+        >>> c.kill()
+
+        The server is automatically re-started on the next request,
+        if the default host/port are used.
+
+        >>> c.version() > (4,0)
+        True
+        """
+        try:
+            self.send('host:kill')
+        except EOFError:
+            pass
+
+    @_autoclose
+    def version(self):
+        """
+        Returns:
+            Tuple containing the ``(major, minor)`` version from the ADB server
+
+        Example:
+
+            >>> adb.protocol.Client().version() # doctest: +SKIP
+            (4, 36)
+        """
+        response = self.send('host:version')
+        if response == OKAY:
+            return (self.c.adb_unpack(), self.c.adb_unpack())
+        self.error("Could not fetch version")
+
+    @_autoclose
+    def devices(self, long=False):
+        """
+        Arguments:
+            long(bool): If ``True``, fetch the long-format listing.
+        Returns:
+            String representation of all available devices.
+        """
+        msg = 'host:devices'
+        if long:
+            msg += '-l'
+        response = self.send(msg)
+        if response == 'OKAY':
+            return self.recvl()
+        self.error("Could not enumerate devices")
+
+    @_autoclose
+    def track_devices(self):
+        """
+        Returns:
+            Generator which returns a short-format listing of available
+            devices each time a device state changes.
+        """
+        self.send('host:track-devices')
+        while True:
+            yield self.recvl()
+
+    def transport(self, serial=None):
+        """Sets the Transport on the rmeote device.
+
+        Examples:
+
+            >>> adb.protocol.Client().transport()
+        """
+        msg = 'host:transport:%s' % (serial or context.device)
+        if self.send(msg) == FAIL:
+            self.error("Could not set transport to %r" % serial)
+
+    @_autoclose
+    @_with_transport
+    def execute(self, argv):
+        r"""Executes a program on the device.
+
+        Returns:
+            A ``tube`` which is connected to the process.
+
+        Examples:
+
+            >>> adb.protocol.Client().execute(['echo','hello']).recvall()
+            'hello\n'
+        """
+        self.transport(context.device)
+        if isinstance(argv, str):
+            argv = [argv]
+        argv = list(map(sh_string, argv))
+        cmd = 'exec:%s' % (' '.join(argv))
+        if OKAY == self.send(cmd):
+            rv = self._c
+            self._c = None
+            return rv
+
+    def _basic_wrapper(string):
+        @_autoclose
+        @_with_transport
+        def wrapper(self):
+            self.send(string)
+            return self.c.recvall()
+        return wrapper
+
+    @_autoclose
+    @_with_transport
+    def remount(self):
+        self.send('remount:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def root(self):
+        self.send('root:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def unroot(self):
+        self.send('unroot:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def disable_verity(self):
+        self.send('disable-verity:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def enable_verity(self):
+        self.send('enable-verity:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def reconnect(self):
+        self.send('reconnect:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def reboot(self):
+        self.send('reboot:')
+        return self.c.recvall()
+
+    @_autoclose
+    @_with_transport
+    def reboot_bootloader(self):
+        self.send('reboot:bootloader')
+        return self.c.recvall()
+
+    @_autoclose
+    def wait_for_device(self, serial=''):
+        response = self.send('host-serial:%s:wait-for-any-device' % serial)
+
+        # The first OKAY is that the command was understood
+        if response != 'OKAY':
+            self.error("An error occurred while waiting for device with serial %r" % serial)
+
+        # The second OKAY is that the device is available
+        response = self.c.recvn(4)
+        if response != 'OKAY':
+            self.error("An error occurred while waiting for device with serial %r" % serial)
+
+    def _sync(fn):
+        """Decorator which enters 'sync:' mode to the selected transport,
+        then invokes the decorated funciton."""
+        @functools.wraps(fn)
+        def wrapper(self, *a, **kw):
+            rv = None
+            if FAIL != self.send('sync:'):
+                rv = fn(self, *a, **kw)
+            return rv
+        return wrapper
+
+    def list(self, path):
+        """Execute the ``LIST`` command of the ``SYNC`` API.
+
+        Arguments:
+            path(str): Path of the directory to list.
+
+        Return:
+            A dictionary, where the keys are relative filenames,
+            and the values are a dictionary containing the same
+            values as ``stat()`` supplies.
+
+        Note:
+            In recent releases of Android (e.g. 7.0), the domain that
+            adbd executes from does not have access to everything that
+            the shell user does.
+
+            Because of this, while the shell user can get listings of
+            e.g. the root directory ('/'), adbd cannot.
+
+            The SYNC APIs are executed within the adbd context, not the
+            shell user context.
+
+            This issue is not a problem if the phone is rooted via
+            'adb root', since adbd then runs in the ``su`` domain.
+
+        Examples:
+
+            >>> pprint(adb.Client().list('/data/user'))
+            {'0': {'mode': 41471, 'size': 11, 'time': ...}}
+            >>> adb.Client().list('/does/not/exist')
+            Traceback (most recent call last):
+            ...
+            PwnlibException: Cannot list directory '/does/not/exist': Does not exist
+        """
+        st = self.stat(path)
+
+        if not st:
+            log.error("Cannot list directory %r: Does not exist" % path)
+
+        if not stat.S_ISDIR(st['mode']):
+            log.error("Cannot list directory %r: Path is not a directory" % path)
+
+        return self._list(path)
+
+    @_with_transport
+    @_sync
+    def _list(self, path):
+        self.c.flat('LIST', len(path), path)
+        files = {}
+        while True:
+            response = self.c.recvn(4)
+
+            if response == 'DONE':
+                break
+
+            if response != 'DENT':
+                self.error('Unexpected response: %r' % response)
+
+            mode = self.c.u32()
+            size = self.c.u32()
+            time = self.c.u32()
+            name = self.c.recvn(self.c.u32())
+
+            # Ignore the current directory and parent
+            if name in ('', '.', '..'):
+                continue
+
+            files[name] = {'mode': mode,
+                           'size': size,
+                           'time': time}
+
+        return files
+
+    @_with_transport
+    @_sync
+    def stat(self, path):
+        """Execute the ``STAT`` command of the ``SYNC`` API.
+
+        Arguments:
+            path(str): Path to the file to stat.
+
+        Return:
+            On success, a dictionary mapping the values returned.
+            If the file cannot be ``stat()``ed, None is returned.
+
+        Example:
+
+            >>> expected = {'mode': 16749, 'size': 0, 'time': 0}
+            >>> adb.protocol.Client().stat('/proc')           == expected
+            True
+            >>> adb.protocol.Client().stat('/does/not/exist') == None
+            True
+        """
+        self.c.flat('STAT', len(path), path)
+        if self.c.recvn(4) != 'STAT':
+            self.error("An error occured while attempting to STAT a file.")
+
+        mode = self.c.u32()
+        size = self.c.u32()
+        time = self.c.u32()
+
+        if (mode,size,time) == (0,0,0):
+            return None
+
+        return {'mode': mode, 'size': size, 'time': time}
+
+    def write(self, path, data, mode=0o755, timestamp=None, callback=None):
+        """Execute the ``WRITE`` command of the ``SYNC`` API.
+
+        Arguments:
+            path(str): Path to the file to write
+            data(str): Data to write to the file
+            mode(int): File mode to set (e.g. ``0o755``)
+            timestamp(int): Unix timestamp to set the file date to
+            callback(callable): Callback function invoked as data
+                is written.  Arguments provided are:
+
+                - File path
+                - All data
+                - Expected size of all data
+                - Current chunk
+                - Expected size of chunk
+        """
+        # We must ensure that 'path' is not a directory
+        # Writing to a directory is supported, but creates a temporary file
+        st = self.stat(path)
+
+        if st and stat.S_ISDIR(st['mode']):
+            log.error("Cannot write to %r: Path is a directory" % path)
+
+        return self._write(path, data, mode=0o755, timestamp=None, callback=None)
+
+    @_with_transport
+    @_sync
+    def _write(self, path, data, mode=0o755, timestamp=None, callback=None):
+        path += ',' + str(mode)
+        self.c.flat('SEND', len(path), path)
+
+        sent = 0
+
+        # Data needs to be broken up into chunks!
+        for chunk in group(0x10000, data):
+            self.c.flat('DATA', len(chunk), chunk)
+            if callback:
+                callback(path, data[:sent], len(data), chunk, len(chunk))
+            sent += len(chunk)
+
+        # Send completion notification and timestamp
+        if timestamp is None:
+            timestamp = int(time.time())
+        self.c.flat('DONE', timestamp)
+
+        result = self.c.recvn(4)
+        if result != 'OKAY':
+            log.error("Sync write failed: %r (expected OKAY)" % result)
+
+        return
+
+    @_with_transport
+    @_sync
+    def read(self, path, filesize=0, callback=lambda *a: True):
+        """Execute the ``READ`` command of the ``SYNC`` API.
+
+        Arguments:
+            path(str): Path to the file to read
+            filesize(int): Size of the file, in bytes.  Optional.
+            callback(callable): Callback function invoked as data
+                becomes available.  Arguments provided are:
+
+                - File path
+                - All data
+                - Expected size of all data
+                - Current chunk
+                - Expected size of chunk
+
+        Return:
+            The data received as a string.
+        """
+        self.c.send('RECV' + p32(len(path)) + path)
+
+        # Accumulate all data here
+        all_data = ''
+
+        while True:
+            magic = self.c.recvn(4)
+
+            # adbd says there is no more data to send
+            if magic == 'DONE':
+                break
+
+            if magic == 'FAIL':
+                self.error('Could not read file %r: Got FAIL.' % path)
+
+            # did we expect to be done?
+            if magic != 'DATA':
+                self.error('Error after file read: %r (expected DATA)' % magic)
+
+            # receive all of the data in the chunk
+            chunk_size = self.c.u32()
+            chunk_data  = ''
+            while len(chunk_data) != chunk_size:
+                chunk_data += self.c.recv(chunk_size - len(chunk_data))
+
+                if callback:
+                    callback(path,
+                             all_data,
+                             filesize,
+                             chunk_data,
+                             chunk_size)
+
+            # add the chunk onto what we have
+            all_data += chunk_data
+
+        zero = self.c.u32()
+        if zero != 0:
+            self.error('Error after file read: %r (expected ZERO)' % zero)
+
+        return all_data
+
+    @_with_transport
+    def forward(self, device, host_proto, host_port, device_proto, device_port):
+        self.send('host:forward:%s:%s;%s:%s' % (host_proto, host_port, device_proto, device_port))
+        self.c.recvall()
+
+    def __enter__(self, *a, **kw):
+        return self
+
+    @_autoclose
+    def __exit__(self, *a, **kw): pass
+
+def proxy(port=9999):
+    """Starts an ADB proxy on the specified port, for debugging purposes."""
+    l = listen(port)
+    l.wait_for_connection()
+    r = remote(context.adb_host, context.adb_port, level='debug')
+    l <> r

pwnlib/args.py

@@ -1,5 +1,43 @@
 #!/usr/bin/env python2
 """
+Pwntools exposes several magic command-line arguments and environment
+variables when operating in `from pwn import *` mode.
+
+The arguments extracted from the command-line and removed from ``sys.argv``.
+
+Arguments can be set by appending them to the command-line, or setting
+them in the environment prefixed by ``PWNLIB_``.
+
+The easiest example is to enable more verbose debugging.  Just set ``DEBUG``.
+
+.. code-block:: bash
+
+    $ PWNLIB_DEBUG=1 python exploit.py
+    $ python exploit.py DEBUG
+
+These arguments are automatically extracted, regardless of their name, and
+exposed via ``pwnlib.args.args``, which is exposed as the global variable
+``args``.  Arguments which ``pwntools`` reserves internally are not exposed
+this way.
+
+.. code-block:: bash
+
+    $ python -c 'from pwn import *; print args' A=1 B=Hello HOST=1.2.3.4 DEBUG
+    defaultdict(<type 'str'>, {'A': '1', 'HOST': '1.2.3.4', 'B': 'Hello'})
+
+This is very useful for conditional code, for example determining whether to
+run an exploit locally or to connect to a remote server.  Arguments which are
+not specified evaluate to an empty string.
+
+.. code-block:: python
+
+    if args['REMOTE']:
+        io = remote('exploitme.com', 4141)
+    else:
+        io = process('./pwnable')
+
+The full list of supported "magic arguments" and their effects are listed
+below.
 """
 import collections
 import logging
@@ -42,46 +80,67 @@ def asbool(s):
     else:
         raise ValueError('must be integer or boolean: %r' % s)
 
-def set_log_level(x):
+def LOG_LEVEL(x):
+    """Sets the logging verbosity used via ``context.log_level``,
+    e.g. ``LOG_LEVEL=debug``.
+    """
     with context.local(log_level=x):
         context.defaults['log_level']=context.log_level
 
-def set_log_file(x):
+def LOG_FILE(x):
+    """Sets a log file to be used via ``context.log_file``, e.g.
+    ``LOG_FILE=./log.txt``"""
     context.log_file=x
 
-def set_log_level_error(x):
-    set_log_level('error')
+def SILENT(x):
+    """Sets the logging verbosity to ``error`` which silences most
+    output."""
+    LOG_LEVEL('error')
 
-def set_log_level_debug(x):
-    set_log_level('debug')
+def DEBUG(x):
+    """Sets the logging verbosity to ``debug`` which displays much
+    more information, including logging each byte sent by tubes."""
+    LOG_LEVEL('debug')
 
-def set_noterm(v):
+def NOTERM(v):
+    """Disables pretty terminal settings and animations."""
     if asbool(v):
         global term_mode
         term_mode = False
 
-def set_timeout(v):
+def TIMEOUT(v):
+    """Sets a timeout for tube operations (in seconds) via
+    ``context.timeout``, e.g. ``TIMEOUT=30``"""
     context.defaults['timeout'] = int(v)
 
-def set_randomize(v):
+def RANDOMIZE(v):
+    """Enables randomization of various pieces via ``context.randomize``"""
     context.defaults['randomize'] = asbool(v)
 
-def set_aslr(v):
+def NOASLR(v):
+    """Disables ASLR via ``context.aslr``"""
     context.defaults['aslr'] = not asbool(v)
 
-def set_noptrace(v):
+def NOPTRACE(v):
+    """Disables facilities which require ``ptrace`` such as ``gdb.attach()``
+    statements, via ``context.noptrace``."""
     context.defaults['noptrace'] = asbool(v)
 
+def STDERR(v):
+    """Sends logging to ``stderr`` by default, instead of ``stdout``"""
+    context.log_console = sys.stderr
+
 hooks = {
-    'LOG_LEVEL': set_log_level,
-    'LOG_FILE': set_log_file,
-    'DEBUG': set_log_level_debug,
-    'NOTERM': set_noterm,
-    'SILENT': set_log_level_error,
-    'RANDOMIZE': set_randomize,
-    'TIMEOUT': set_timeout,
-    'NOASLR': set_aslr,
-    'NOPTRACE': set_noptrace,
+    'LOG_LEVEL': LOG_LEVEL,
+    'LOG_FILE': LOG_FILE,
+    'DEBUG': DEBUG,
+    'NOTERM': NOTERM,
+    'SILENT': SILENT,
+    'RANDOMIZE': RANDOMIZE,
+    'TIMEOUT': TIMEOUT,
+    'NOASLR': NOASLR,
+    'NOPTRACE': NOPTRACE,
+    'STDERR': STDERR,
 }
 
 def initialize():

pwnlib/asm.py

@@ -27,8 +27,8 @@ Assembly
     Finally, :func:`asm` is used to assemble shellcode provided by ``pwntools``
     in the :mod:`shellcraft` module.
 
-        >>> asm(shellcraft.sh())
-        'jhh///sh/binj\x0bX\x89\xe31\xc9\x99\xcd\x80'
+        >>> asm(shellcraft.nop())
+        '\x90'
 
 Disassembly
 ------------------------
@@ -66,6 +66,60 @@ __all__ = ['asm', 'cpp', 'disasm', 'make_elf', 'make_elf_from_assembly']
 _basedir = path.split(__file__)[0]
 _incdir  = path.join(_basedir, 'data', 'includes')
 
+def dpkg_search_for_binutils(arch, util):
+    """Use dpkg to search for any available assemblers which will work.
+
+    Returns:
+        A list of candidate package names.
+    """
+
+    # Example output:
+    # $ dpkg -S 'arm*linux*-as'
+    # binutils-arm-linux-gnu: /usr/bin/arm-linux-gnu-as
+    # binutils-arm-linux-gnueabihf: /usr/bin/arm-linux-gnueabihf-as
+    # binutils-arm-linux-gnueabihf: /usr/x86_64-linux-gnu/arm-linux-gnueabihf/include/dis-asm.h
+    # binutils-arm-linux-gnu: /usr/x86_64-linux-gnu/arm-linux-gnu/include/dis-asm.h
+    packages = []
+
+    try:
+        filename = 'bin/%s*linux*-%s' % (arch, util)
+        output = subprocess.check_output(['dpkg','-S',filename])
+        for line in output.strip().splitlines():
+            package, path = line.split(':', 1)
+            packages.append(package)
+    except OSError:
+        pass
+    except subprocess.CalledProcessError:
+        pass
+
+    return packages
+
+def print_binutils_instructions(util, context):
+    """On failure to find a binutils utility, inform the user of a way
+    they can get it easily.
+    """
+    # This links to our instructions on how to manually install binutils
+    # for several architectures.
+    instructions = 'https://docs.pwntools.com/en/stable/install/binutils.html'
+
+    # However, if we can directly provide a useful command, go for it.
+    binutils_arch = {
+        'amd64': 'x86_64',
+        'arm':   'armeabi',
+        'thumb': 'armeabi',
+    }.get(context.arch, context.arch)
+
+    packages = dpkg_search_for_binutils(binutils_arch, util)
+
+    if packages:
+        instructions = '$ sudo apt-get install %s' % packages[0]
+
+    log.error("""
+Could not find %(util)r installed for %(context)s
+Try installing binutils for this architecture:
+%(instructions)s
+""".strip() % locals())
+
 @LocalContext
 def which_binutils(util):
     """
@@ -129,13 +183,8 @@ def which_binutils(util):
                 if res:
                     return res[0]
 
-    locals()['context'] = context
-    log.warning("""
-Could not find %(util)r installed for %(context)s
-Try installing binutils for this architecture:
-https://docs.pwntools.com/en/stable/install/binutils.html
-""".strip() % locals())
-    raise Exception('Could not find %(util)r installed for %(context)s' % locals())
+    # No dice!
+    print_binutils_instructions(util, context)
 
 checked_assembler_version = defaultdict(lambda: False)
 

pwnlib/commandline/__init__.py

@@ -1 +1 @@
-# empty
+#!/usr/bin/env python2

pwnlib/commandline/asm.py

@@ -6,8 +6,9 @@ from pwn import *
 
 from . import common
 
-parser = argparse.ArgumentParser(
-    description = 'Assemble shellcode into bytes'
+parser = common.parser_commands.add_parser(
+    'asm',
+    help = 'Assemble shellcode into bytes'
 )
 
 parser.add_argument(
@@ -91,8 +92,7 @@ parser.add_argument(
     action='store_true'
 )
 
-def main():
-    args   = parser.parse_args()
+def main(args):
     tty    = args.output.isatty()
 
     data   = '\n'.join(args.lines) or args.infile.read()
@@ -123,4 +123,5 @@ def main():
     if tty and fmt is not 'raw':
         args.output.write('\n')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/checksec.py

@@ -6,11 +6,10 @@ from pwn import *
 
 from . import common
 
-parser = argparse.ArgumentParser(
-    description = 'Check binary security settings'
+parser = common.parser_commands.add_parser(
+    'checksec',
+    help = 'Check binary security settings'
 )
-
-
 parser.add_argument(
     'elf',
     nargs='*',
@@ -26,8 +25,7 @@ parser.add_argument(
     help='File to check (for compatibility with checksec.sh)'
 )
 
-def main():
-    args   = parser.parse_args()
+def main(args):
     files  = args.elf or args.elf2 or []
 
     if not files:
@@ -37,4 +35,5 @@ def main():
     for f in files:
         e = ELF(f.name)
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/common.py

@@ -1,10 +1,10 @@
+import argparse
+import os
 import sys
 
 import pwnlib
 from pwnlib.context import context
 
-pwnlib.log.console.stream = sys.stderr
-
 choices = map(str, [16,32,64])
 choices += list(context.oses)
 choices += list(context.architectures)
@@ -20,3 +20,13 @@ def context_arg(arg):
     try: context.endian = arg
     except Exception: pass
     return arg
+
+parser = argparse.ArgumentParser(description='Pwntools Command-line Interface',
+                                 prog='pwn')
+parser_commands = parser.add_subparsers(dest='command')
+
+def main(file=sys.argv[0]):
+    import pwnlib.commandline.main
+    name = os.path.splitext(os.path.basename(file))[0]
+    sys.argv.insert(1, name)
+    pwnlib.commandline.main.main()

pwnlib/commandline/constgrep.py

@@ -5,10 +5,12 @@ import os
 import re
 
 from pwn import *
+
 from . import common
 
-p = argparse.ArgumentParser(
-    description = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
+p = common.parser_commands.add_parser(
+    'constgrep',
+    help = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
     formatter_class = argparse.RawDescriptionHelpFormatter,
 )
 
@@ -55,9 +57,7 @@ p.add_argument(
     help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
 )
 
-def main():
-    args = p.parse_args()
-
+def main(args):
     if args.exact:
         # This is the simple case
         print cpp(args.exact).strip()
@@ -134,4 +134,5 @@ def main():
                 print
                 print '(%s) == %s' % (' | '.join(k for v, k in good), args.constant)
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/cyclic.py

@@ -7,8 +7,9 @@ from pwn import *
 
 from . import common
 
-parser = argparse.ArgumentParser(
-    description = "Cyclic pattern creator/finder"
+parser = common.parser_commands.add_parser(
+    'cyclic',
+    help = "Cyclic pattern creator/finder"
 )
 
 parser.add_argument(
@@ -50,8 +51,7 @@ group.add_argument(
     help = 'Number of characters to print'
 )
 
-def main():
-    args = parser.parse_args()
+def main(args):
     alphabet = args.alphabet
     subsize  = args.length
 
@@ -90,4 +90,5 @@ def main():
         if sys.stdout.isatty():
             sys.stdout.write('\n')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/disasm.py

@@ -7,8 +7,9 @@ from pwn import *
 
 from . import common
 
-parser = argparse.ArgumentParser(
-    description = 'Disassemble bytes into text format'
+parser = common.parser_commands.add_parser(
+    'disasm',
+    help = 'Disassemble bytes into text format'
 )
 
 parser.add_argument(
@@ -52,9 +53,7 @@ parser.add_argument(
 )
 
 
-def main():
-    args = parser.parse_args()
-
+def main(args):
     if len(args.hex) > 0:
         dat = ''.join(args.hex)
         dat = dat.translate(None, string.whitespace)
@@ -88,4 +87,5 @@ def main():
 
     print disasm(dat, vma=safeeval.const(args.address))
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/elfdiff.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python2
 import shutil
-
 from argparse import ArgumentParser
 from subprocess import CalledProcessError
 from subprocess import check_output
@@ -8,6 +7,9 @@ from tempfile import NamedTemporaryFile
 
 from pwn import *
 
+from . import common
+
+
 def dump(objdump, path):
     n = NamedTemporaryFile(delete=False)
     o = check_output([objdump,'-d','-x','-s',path])
@@ -20,13 +22,15 @@ def diff(a,b):
     except CalledProcessError as e:
         return e.output
 
-p = ArgumentParser()
+p = common.parser_commands.add_parser(
+    'elfdiff',
+    help = 'Compare two ELF files'
+)
+
 p.add_argument('a')
 p.add_argument('b')
 
-def main():
-    a = p.parse_args()
-
+def main(a):
     with context.silent:
         x = ELF(a.a)
         y = ELF(a.b)
@@ -49,4 +53,5 @@ def main():
 
     print diff(x, y)
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/elfpatch.py

@@ -4,15 +4,20 @@ import sys
 
 from pwn import *
 
+from . import common
+
+p = common.parser_commands.add_parser(
+    'elfpatch',
+    help = 'Patch an ELF file'
+)
+
 p = argparse.ArgumentParser()
 p.add_argument('elf',help="File to patch")
 p.add_argument('offset',help="Offset to patch in virtual address (hex encoded)")
 p.add_argument('bytes',help='Bytes to patch (hex encoded)')
 
 
-def main():
-    a = p.parse_args()
-
+def main(a):
     if not a.offset.startswith('0x'):
         a.offset = '0x' + a.offset
 
@@ -25,4 +30,5 @@ def main():
     elf.write(offset, bytes)
     sys.stdout.write(elf.get_data())
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/errno.py

@@ -1,17 +1,18 @@
 import argparse
 import os
 
-parser = argparse.ArgumentParser(
-    description = 'Prints out error messages'
+from . import common
+
+parser = common.parser_commands.add_parser(
+    'errno',
+    help = 'Prints out error messages'
 )
 
 parser.add_argument(
     'error', help='Error message or value', type=str
 )
 
-def main():
-  args, unknown = parser.parse_known_args()
-
+def main(args):
   try:
     value = int(args.error, 0)
 
@@ -40,4 +41,5 @@ def main():
   print '#define', name, value
   print os.strerror(value)
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/hex.py

@@ -2,17 +2,21 @@
 import argparse
 import sys
 
-parser = argparse.ArgumentParser(description='''
-Hex-encodes data provided on the command line or via stdin.
+from . import common
+
+parser = common.parser_commands.add_parser(
+    'hex',
+    help = '''
+Hex-encodes data provided on the command line or stdin
 ''')
 parser.add_argument('data', nargs='*',
     help='Data to convert into hex')
 
-def main():
-    args = parser.parse_args()
+def main(args):
     if not args.data:
         print sys.stdin.read().encode('hex')
     else:
         print ' '.join(args.data).encode('hex')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/main.py

@@ -0,0 +1,46 @@
+import sys
+
+from . import asm
+from . import checksec
+from . import common
+from . import constgrep
+from . import cyclic
+from . import disasm
+from . import elfdiff
+from . import elfpatch
+from . import errno
+from . import hex
+from . import phd
+from . import pwnstrip
+from . import scramble
+from . import shellcraft
+from . import unhex
+from . import update
+from .common import parser
+from ..context import context
+
+commands = {
+    'asm': asm.main,
+    'checksec': checksec.main,
+    'constgrep': constgrep.main,
+    'cyclic': cyclic.main,
+    'disasm': disasm.main,
+    'elfdiff': elfdiff.main,
+    'elfpatch': elfpatch.main,
+    'errno': errno.main,
+    'hex': hex.main,
+    'phd': phd.main,
+    'pwnstrip': pwnstrip.main,
+    'scramble': scramble.main,
+    'shellcraft': shellcraft.main,
+    'unhex': unhex.main,
+    'update': update.main,
+}
+
+def main():
+    args = parser.parse_args()
+    with context.local(log_console = sys.stderr):
+        commands[args.command](args)
+
+if __name__ == '__main__':
+    main()

pwnlib/commandline/phd.py

@@ -5,8 +5,11 @@ import sys
 
 from pwn import *
 
-parser = argparse.ArgumentParser(
-    description = 'Pwnlib HexDump'
+from . import common
+
+parser = common.parser_commands.add_parser(
+    'phd',
+    help = 'Pwnlib HexDump'
 )
 
 parser.add_argument(
@@ -64,9 +67,7 @@ def asint(s):
     else:
         return int(s, 10)
 
-def main():
-    args = parser.parse_args()
-
+def main(args):
     infile = args.file
     width  = asint(args.width)
     skip   = asint(args.skip)
@@ -95,4 +96,5 @@ def main():
     except (KeyboardInterrupt, IOError):
         pass
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/pwnstrip.py

@@ -4,9 +4,9 @@ from pwn import *
 
 from . import common
 
-p = argparse.ArgumentParser(
-    description = 'Strip binaries for CTF usage',
-    formatter_class = argparse.RawDescriptionHelpFormatter,
+p = common.parser_commands.add_parser(
+    'pwnstrip',
+    help = 'Strip binaries for CTF usage',
 )
 
 g = p.add_argument_group("actions")
@@ -15,9 +15,7 @@ g.add_argument('-p', '--patch', metavar='FUNCTION', help="Patch function", actio
 p.add_argument('-o', '--output', type=file, default=sys.stdout)
 p.add_argument('file', type=file)
 
-def main():
-    args = p.parse_args()
-
+def main(args):
     if not (args.patch or args.build_id):
         sys.stderr.write("Must specify at least one action\n")
         sys.stderr.write(p.format_usage())
@@ -48,6 +46,5 @@ def main():
 
     args.output.write(result)
 
-
 if __name__ == '__main__':
-    main()
+    pwnlib.common.main(__file__)

pwnlib/commandline/scramble.py

@@ -5,11 +5,11 @@ from pwn import *
 
 from . import common
 
-parser = argparse.ArgumentParser(
-    description = 'Shellcode encoder'
+parser = common.parser_commands.add_parser(
+    'scramble',
+    help = 'Shellcode encoder'
 )
 
-
 parser.add_argument(
     "-f", "--format",
     help="Output format (defaults to hex for ttys, otherwise raw)",
@@ -68,8 +68,7 @@ parser.add_argument(
     action='store_true'
 )
 
-def main():
-    args   = parser.parse_args()
+def main(args):
     tty    = args.output.isatty()
 
     if sys.stdin.isatty():
@@ -100,4 +99,6 @@ def main():
     if tty and fmt is not 'raw':
         args.output.write('\n')
 
-if __name__ == '__main__': main()
+
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/shellcraft.py

@@ -8,18 +8,6 @@ from pwn import *
 
 from . import common
 
-r = text.red
-g = text.green
-b = text.blue
-
-banner = '\n'.join(['  ' + r('____') + '  ' + g('_') + '          ' + r('_') + ' ' + g('_') + '                 ' + b('__') + ' ' + r('_'),
-                    ' ' + r('/ ___|') + g('| |__') + '   ' + b('___') + r('| |') + ' ' + g('|') + ' ' + b('___') + ' ' + r('_ __') + ' ' + g('__ _') + ' ' + b('/ _|') + ' ' + r('|_'),
-                    ' ' + r('\___ \\') + g('| \'_ \\') + ' ' + b('/ _ \\') + ' ' + r('|') + ' ' + g('|') + b('/ __|') + ' ' + r('\'__/') + ' ' + g('_` |') + ' ' + b('|_') + r('| __|'),
-                    '  ' + r('___) |') + ' ' + g('| | |') + '  ' + b('__/') + ' ' + r('|') + ' ' + g('|') + ' ' + b('(__') + r('| |') + ' ' + g('| (_| |') + '  ' + b('_|') + ' ' + r('|_'),
-                    ' ' + r('|____/') + g('|_| |_|') + b('\\___|') + r('_|') + g('_|') + b('\\___|') + r('_|') + '  ' + g('\\__,_|') + b('_|') + '  ' + r('\\__|'),
-                    '\n'
-                    ])
-
 
 #  ____  _          _ _                 __ _
 # / ___|| |__   ___| | | ___ _ __ __ _ / _| |_
@@ -37,9 +25,10 @@ def _string(s):
             out.append('\\x%02x' % co)
     return '"' + ''.join(out) + '"\n'
 
-p = argparse.ArgumentParser(
-    description = 'Microwave shellcode -- Easy, fast and delicious',
-    formatter_class = argparse.RawDescriptionHelpFormatter,
+
+p = common.parser_commands.add_parser(
+    'shellcraft',
+    help = 'Microwave shellcode -- Easy, fast and delicious',
 )
 
 
@@ -68,21 +57,19 @@ p.add_argument(
                'p',
                'i', 'hexii',
                'e', 'elf',
+               'd', 'escaped',
                'default'],
     default = 'default',
-    help = 'Output format (default: hex), choose from {r}aw, {s}tring, {c}-style array, {h}ex string, hex{i}i, {a}ssembly code, {p}reprocssed code',
+    help = 'Output format (default: hex), choose from {e}lf, {r}aw, {s}tring, {c}-style array, {h}ex string, hex{i}i, {a}ssembly code, {p}reprocssed code, escape{d} hex string',
 )
 
 p.add_argument(
     'shellcode',
     nargs = '?',
-    choices = shellcraft.templates,
-    metavar = 'shellcode',
     help = 'The shellcode you want',
+    type = str
 )
 
-p.epilog = 'Available shellcodes are:\n' + '\n'.join(shellcraft.templates)
-
 p.add_argument(
     'args',
     nargs = '*',
@@ -167,6 +154,12 @@ p.add_argument(
     default=None
 )
 
+p.add_argument(
+    '-l', '--list',
+    action='store_true',
+    help='List available shellcodes, optionally provide a filter'
+)
+
 def get_template(name):
     func = shellcraft
     for attr in name.split('.'):
@@ -177,21 +170,25 @@ def is_not_a_syscall_template(name):
     template_src = shellcraft._get_source(name)
     return 'man 2' not in read(template_src)
 
-def main():
-    # Banner must be added here so that it doesn't appear in the autodoc
-    # generation for command line tools
-    p.description = banner + p.description
-    args = p.parse_args()
-
-    if not args.shellcode:
+def main(args):
+    if args.list:
         templates = shellcraft.templates
 
-        if not args.syscalls:
+        if args.shellcode:
+            templates = filter(lambda a: args.shellcode in a, templates)
+        elif not args.syscalls:
             templates = filter(is_not_a_syscall_template, templates)
 
         print '\n'.join(templates)
         exit()
 
+    if not args.shellcode:
+        common.parser.print_usage()
+        exit()
+
+    if args.shellcode not in shellcraft.templates:
+        log.error("Unknown shellcraft template %r. Use --list to see available shellcodes." % args.shellcode)
+
     func = get_template(args.shellcode)
 
     if args.show:
@@ -326,17 +323,19 @@ def main():
         sys.exit(0)
 
     if args.format in ['s', 'str', 'string']:
-        code = _string(code) + '"\n'
+        code = _string(code)
     elif args.format == 'c':
         code = '{' + ', '.join(map(hex, bytearray(code))) + '}' + '\n'
     elif args.format in ['h', 'hex']:
         code = pwnlib.util.fiddling.enhex(code) + '\n'
     elif args.format in ['i', 'hexii']:
         code = hexii(code) + '\n'
-
+    elif args.format in ['d', 'escaped']:
+        code = ''.join('\\x%02x' % ord(c) for c in code) + '\n'
     if not sys.stdin.isatty():
         args.out.write(sys.stdin.read())
 
     args.out.write(code)
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/unhex.py

@@ -3,21 +3,26 @@ import argparse
 import sys
 from string import whitespace
 
-parser = argparse.ArgumentParser(description='''
+from . import common
+
+parser = common.parser_commands.add_parser(
+    'unhex',
+    help = '''
 Decodes hex-encoded data provided on the command line or via stdin.
 ''')
+
 parser.add_argument('hex', nargs='*',
     help='Hex bytes to decode')
 
-def main():
-    args = parser.parse_args()
+def main(args):
     try:
         if not args.hex:
             s = sys.stdin.read().translate(None, whitespace)
             sys.stdout.write(s.decode('hex'))
         else:
-            sys.stdout.write(''.join(sys.argv[1:]).decode('hex'))
+            sys.stdout.write(''.join(args.hex).decode('hex'))
     except TypeError, e:
         sys.stderr.write(str(e) + '\n')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/commandline/update.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python2
+import subprocess
+
+from pwn import *
+
+from . import common
+
+p = common.parser_commands.add_parser(
+    'update',
+    help = 'Check for pwntools updates'
+)
+
+p.add_argument('--install', action='store_true', help='''
+    Install the update automatically.
+''')
+
+p.add_argument('--pre', action='store_true', help='''
+    Check for pre-releases.
+''')
+
+def main(a):
+    result = pwnlib.update.perform_check(prerelease=a.pre)
+    if a.install:
+        subprocess.check_call(result, shell=True)
+
+if __name__ == '__main__':
+    pwnlib.common.main(__file__)

pwnlib/context/__init__.py

@@ -10,6 +10,7 @@ import logging
 import os
 import platform
 import socket
+import stat
 import string
 import subprocess
 import sys
@@ -334,6 +335,7 @@ class ContextType(object):
         'kernel': None,
         'log_level': logging.INFO,
         'log_file': _devnull(),
+        'log_console': sys.stdout,
         'randomize': False,
         'newline': '\n',
         'noptrace': False,
@@ -525,10 +527,21 @@ class ContextType(object):
     def quiet(self, function=None):
         """Disables all non-error logging within the enclosed scope,
         *unless* the debugging level is set to 'debug' or lower."""
-        level = 'error'
-        if context.log_level <= logging.DEBUG:
-            level = None
-        return self.local(function, log_level=level)
+        if not function:
+            level = 'error'
+            if context.log_level <= logging.DEBUG:
+                level = None
+            return self.local(function, log_level=level)
+
+        @functools.wraps(function)
+        def wrapper(*a, **kw):
+            level = 'error'
+            if context.log_level <= logging.DEBUG:
+                level = None
+            with self.local(function, log_level=level):
+                return function(*a, **kw)
+        return wrapper
+
 
     @property
     def verbose(self):
@@ -911,6 +924,24 @@ class ContextType(object):
         value.flush()
         return value
 
+    @_validator
+    def log_console(self, stream):
+        """
+        Sets the default logging console target.
+
+        Examples:
+
+            >>> context.log_level = 'warn'
+            >>> log.warn("Hello")
+            [!] Hello
+            >>> context.log_console=open('/dev/null', 'w')
+            >>> log.warn("Hello")
+            >>> context.clear()
+        """
+        if isinstance(stream, str):
+            stream = open(stream, 'wt')
+        return stream
+
     @property
     def mask(self):
         return (1 << self.bits) - 1
@@ -1086,6 +1117,7 @@ class ContextType(object):
             self.arch = device.arch or self.arch
             self.bits = device.bits or self.bits
             self.endian = device.endian or self.endian
+            self.os = device.os or self.os
         elif isinstance(device, str):
             device = Device(device)
         else:
@@ -1095,8 +1127,13 @@ class ContextType(object):
 
     @property
     def adb(self):
-        """Returns an argument array for connecting to adb."""
-        command = ['adb']
+        """Returns an argument array for connecting to adb.
+
+        Unless ``$ADB_PATH`` is set, uses the default ``adb`` binary in ``$PATH``.
+        """
+        ADB_PATH = os.environ.get('ADB_PATH', 'adb')
+
+        command = [ADB_PATH]
 
         if self.adb_host != self.defaults['adb_host']:
             command += ['-H', self.adb_host]
@@ -1109,6 +1146,45 @@ class ContextType(object):
 
         return command
 
+    @property
+    def cache_dir(self):
+        """Directory used for caching data.
+
+        Note:
+            May be either a path string, or ``None``.
+
+        Example:
+
+            >>> cache_dir = context.cache_dir
+            >>> cache_dir is not None
+            True
+            >>> os.chmod(cache_dir, 0o000)
+            >>> context.cache_dir is None
+            True
+            >>> os.chmod(cache_dir, 0o755)
+            >>> cache_dir == context.cache_dir
+            True
+        """
+        home = os.path.expanduser('~')
+
+        if not os.access(home, os.W_OK):
+            return None
+
+        cache = os.path.join(home, '.pwntools-cache')
+
+        if not os.path.exists(cache):
+            try:
+                os.mkdir(cache)
+            except OSError:
+                return None
+
+        # Some wargames e.g. pwnable.kr have created dummy directories
+        # which cannot be modified by the user account (owned by root).
+        if not os.access(cache, os.W_OK):
+            return None
+
+        return cache
+
 
     #*************************************************************************
     #                               ALIASES

pwnlib/data/useragents/download-useragents.py

@@ -7,7 +7,6 @@ import re
 import urllib
 
 from bs4 import BeautifulSoup
-
 from pwn import *
 
 uas = set()

pwnlib/device.py

@@ -4,12 +4,13 @@ class Device(object):
     bits = None
     endian = None
     serial = None
+    os = None
 
     def __init__(self, serial=None):
         self.serial = serial
 
     def __str__(self):
         return self.serial
- 
+
     def __eq__(self, other):
         return self.serial == other or self.serial == str(other)

pwnlib/dynelf.py

@@ -860,3 +860,14 @@ class DynELF(object):
         self.success('*environ: %#x' % stack)
 
         return stack
+
+    def heap(self):
+        """Finds the beginning of the heap via __curbrk, which is an exported
+        symbol in the linker, which points to the current brk.
+        """
+        curbrk = self.lookup('__curbrk', 'libc')
+        brk    = self.leak.p(curbrk)
+
+        self.success('*curbrk: %#x' % brk)
+
+        return brk

pwnlib/elf/corefile.py

@@ -30,11 +30,11 @@ def iter_notes(self):
     end = self['p_offset'] + self['p_filesz']
     while offset < end:
         note = struct_parse(
-            self._elfstructs.Elf_Nhdr,
+            self.elffile.structs.Elf_Nhdr,
             self.stream,
             stream_pos=offset)
         note['n_offset'] = offset
-        offset += self._elfstructs.Elf_Nhdr.sizeof()
+        offset += self.elffile.structs.Elf_Nhdr.sizeof()
         self.stream.seek(offset)
         # n_namesz is 4-byte aligned.
         disk_namesz = roundup(note['n_namesz'], 2)
@@ -236,7 +236,7 @@ class Core(ELF):
     def libc(self):
         """Return the first mapping in libc"""
         for m in self.mappings:
-            if m.name.startswith('libc') and m.name.endswith('.so'):
+            if m.name and m.name.startswith('libc') and m.name.endswith('.so'):
                 return m
 
     @property

pwnlib/elf/datatypes.py

@@ -539,7 +539,8 @@ class elf_prstatus_i386(ctypes.Structure):
 assert ctypes.sizeof(elf_prstatus_i386) == 0x90
 
 class elf_prstatus_amd64(ctypes.Structure):
-    _fields_ = generate_prstatus_common(64, user_regs_struct_amd64)
+    _fields_ = generate_prstatus_common(64, user_regs_struct_amd64) \
+             + [('padding', ctypes.c_uint32)]
 
 assert ctypes.sizeof(elf_prstatus_amd64) == 0x150
 

pwnlib/elf/elf.py

@@ -1,5 +1,6 @@
 """Exposes functionality for manipulating ELF files
 """
+import codecs
 import mmap
 import os
 import subprocess
@@ -24,12 +25,24 @@ from ..term import text
 from ..tubes.process import process
 from ..util import misc
 from ..util import packing
+from ..util import sh_string
 
 log = getLogger(__name__)
 
 __all__ = ['load', 'ELF']
 
-Function = namedtuple('Function', 'address size')
+class Function(object):
+    def __init__(self, name, address, size):
+        self.name    = name
+        self.address = address
+        self.size    = size
+    def __repr__(self):
+        return '%s(name=%r, address=%#x, size=%#x)' % (
+            self.__class__.__name__,
+            self.name,
+            self.address,
+            self.size
+            )
 
 def load(*args, **kwargs):
     """Compatibility wrapper for pwntools v1"""
@@ -312,7 +325,7 @@ class ELF(ELFFile):
             return
 
         try:
-            cmd = misc.sh_command_with('ulimit -s unlimited; LD_TRACE_LOADED_OBJECTS=1 LD_WARN=1 LD_BIND_NOW=1 %s 2>/dev/null', self.path)
+            cmd = sh_string.sh_command_with('ulimit -s unlimited; LD_TRACE_LOADED_OBJECTS=1 LD_WARN=1 LD_BIND_NOW=1 %s 2>/dev/null', self.path)
 
             data = subprocess.check_output(cmd, shell = True, stderr = subprocess.STDOUT)
             libs = misc.parse_ldd_output(data)
@@ -347,11 +360,15 @@ class ELF(ELFFile):
                     continue
                 if sym.entry.st_info['type'] == 'STT_FUNC' and sym.entry.st_size != 0:
                     name = sym.name
+                    try:
+                        name = codecs.encode(name, 'latin-1')
+                    except Exception:
+                        pass
                     if name not in self.symbols:
                         continue
                     addr = self.symbols[name]
                     size = sym.entry.st_size
-                    self.functions[name] = Function(addr, size)
+                    self.functions[name] = Function(name, addr, size)
 
     def _populate_symbols(self):
         """

pwnlib/flag/__init__.py

@@ -0,0 +1,5 @@
+from .flag import submit_flag
+
+__all__ = [
+    'submit_flag',
+]

pwnlib/flag/flag.py

@@ -0,0 +1,71 @@
+"""Describes a way to submit a key to a key server.
+"""
+import os
+
+from ..args import args
+from ..log import getLogger
+from ..tubes.remote import remote
+
+env_server  = args.get('FLAG_HOST', 'flag-submission-server').strip()
+env_port    = args.get('FLAG_PORT', '31337').strip()
+env_proto   = args.get('FLAG_PROTO', 'tcp').strip()
+env_file    = args.get('FLAG_FILE', '/does/not/exist').strip()
+env_exploit_name = args.get('EXPLOIT_NAME', 'unnamed-exploit').strip()
+env_target_host  = args.get('TARGET_HOST', 'unknown-target').strip()
+env_team_name    = args.get('TEAM_NAME', 'unknown-team').strip()
+
+log = getLogger(__name__)
+
+def submit_flag(flag,
+                exploit=env_exploit_name,
+                target=env_target_host,
+                server=env_server,
+                port=env_port,
+                proto=env_proto,
+                team=env_team_name):
+    """
+    Submits a flag to the game server
+
+    Arguments:
+        flag(str): The flag to submit.
+        exploit(str): Exploit identifier, optional
+        target(str): Target identifier, optional
+        server(str): Flag server host name, optional
+        port(int): Flag server port, optional
+        proto(str), Flag server protocol, optional
+
+    Optional arguments are inferred from the environment,
+    or omitted if none is set.
+
+    Returns:
+        A string indicating the status of the key submission,
+        or an error code.
+
+    Doctest:
+
+        >>> l = listen()
+        >>> _ = submit_flag('flag', server='localhost', port=l.lport)
+        >>> c = l.wait_for_connection()
+        >>> c.recvall().split()
+        ['flag', 'unnamed-exploit', 'unknown-target', 'unknown-team']
+    """
+    flag = flag.strip()
+
+    log.success("Flag: %r" % flag)
+
+    data = "\n".join([flag,
+                      exploit,
+                      target,
+                      team,
+                      ''])
+
+    if os.path.exists(env_file):
+        write(env_file, data)
+        return
+
+    try:
+        with remote(server, int(port)) as r:
+            r.send(data)
+            return r.recvall(timeout=1)
+    except Exception:
+        log.warn("Could not submit flag %r to %s:%s" % (flag, server, port))

pwnlib/gdb.py

@@ -32,9 +32,7 @@ def debug_assembly(asm, execute=None, vma=None):
     tmp_elf = make_elf_from_assembly(asm, vma=vma, extract=False)
     os.chmod(tmp_elf, 0777)
 
-    def unlink():
-        with context.silent: os.unlink(tmp_elf)
-    atexit.register(lambda: unlink)
+    atexit.register(lambda: os.unlink(tmp_elf))
 
     if context.os == 'android':
         android_path = '/data/data/%s' % os.path.basename(tmp_elf)
@@ -60,9 +58,7 @@ def debug_shellcode(data, execute=None, vma=None):
     tmp_elf = make_elf(data, extract=False, vma=vma)
     os.chmod(tmp_elf, 0777)
 
-    def unlink():
-        with context.silent: os.unlink(tmp_elf)
-    atexit.register(lambda: unlink)
+    atexit.register(lambda: os.unlink(tmp_elf))
 
     if context.os == 'android':
         android_path = '/data/data/%s' % os.path.basename(tmp_elf)
@@ -344,6 +340,7 @@ def attach(target, execute = None, exe = None, need_ptrace_scope = True):
         shell = target.parent
 
         tmpfile = shell.mktemp()
+        execute = 'shell rm %s\n%s' % (tmpfile, execute)
         shell.upload_data(execute or '', tmpfile)
 
         cmd = ['ssh', '-C', '-t', '-p', str(shell.port), '-l', shell.user, shell.host]
@@ -351,10 +348,9 @@ def attach(target, execute = None, exe = None, need_ptrace_scope = True):
             cmd = ['sshpass', '-p', shell.password] + cmd
         if shell.keyfile:
             cmd += ['-i', shell.keyfile]
-        cmd += ['gdb %r %s -x "%s" ; rm "%s"' % (target.executable,
-                                                 target.pid,
-                                                 tmpfile,
-                                                 tmpfile)]
+        cmd += ['gdb %r %s -x "%s"' % (target.executable,
+                                       target.pid,
+                                       tmpfile)]
 
         misc.run_in_new_terminal(' '.join(cmd))
         return
@@ -427,10 +423,11 @@ def attach(target, execute = None, exe = None, need_ptrace_scope = True):
         tmp = tempfile.NamedTemporaryFile(prefix = 'pwn', suffix = '.gdb',
                                           delete = False)
         log.debug('Wrote gdb script to %r\n%s' % (tmp.name, execute))
+        execute = 'shell rm %s\n%s' % (tmp.name, execute)
+
         tmp.write(execute)
         tmp.close()
-        atexit.register(lambda: os.unlink(tmp.name))
-        cmd += ' -x "%s" ; rm "%s"' % (tmp.name, tmp.name)
+        cmd += ' -x "%s"' % (tmp.name)
 
     log.info('running in new terminal: %s' % cmd)
     misc.run_in_new_terminal(cmd)

pwnlib/log.py

@@ -113,7 +113,7 @@ from .term import spinners
 from .term import text
 
 # list of prefixes to use for the different message types.  note that the `text`
-# module won't add any escape codes if `sys.stderr.isatty()` is `False`
+# module won't add any escape codes if `pwnlib.context.log_console.isatty()` is `False`
 _msgtype_prefixes = {
     'status'       : [text.magenta, 'x'],
     'success'      : [text.bold_green, '+'],
@@ -166,7 +166,6 @@ class Progress(object):
     using :meth:`Logger.progress`.
     """
     def __init__(self, logger, msg, status, level, args, kwargs):
-        global _progressid
         self._logger = logger
         self._msg = msg
         self._status = status
@@ -264,7 +263,7 @@ class Logger(object):
             # This is a minor hack to permit user-defined classes which inherit
             # from a tube (which do not actually reside in the pwnlib library)
             # to receive logging abilities that behave as they would expect from
-            # the resto f the library
+            # the rest of the library
             module = self.__module__
             if not module.startswith('pwnlib'):
                 module = 'pwnlib.' + module
@@ -490,10 +489,14 @@ class Handler(logging.StreamHandler):
     logger will not be emitted but rather an animated progress line will be
     created.
 
-    This handler outputs to ``sys.stderr``.
-
     An instance of this handler is added to the ``'pwnlib'`` logger.
     """
+    @property
+    def stream(self):
+        return context.log_console
+    @stream.setter
+    def stream(self, value):
+        pass
     def emit(self, record):
         """
         Emit a log record or create/update an animated progress logger
@@ -652,7 +655,7 @@ log_file.setFormatter(logging.Formatter(fmt, iso_8601))
 #     logger.addHandler(myCoolPitchingHandler)
 #
 rootlogger = getLogger('pwnlib')
-console   = Handler(sys.stdout)
+console   = Handler()
 formatter = Formatter()
 console.setFormatter(formatter)
 
@@ -663,8 +666,6 @@ def install_default_handler():
     the ``pwnlib`` root logger.  This function is automatically called from when
     importing :mod:`pwn`.
     '''
-    console.stream = sys.stdout
-
     logger         = logging.getLogger('pwnlib')
 
     if console not in logger.handlers:

pwnlib/memleak.py

@@ -1,3 +1,4 @@
+import ctypes
 import functools
 import string
 
@@ -96,8 +97,18 @@ class MemLeak(object):
             struct(class): A ctypes structure to be instantiated with leaked data
         Return Value:
             An instance of the provided struct class, with the leaked data decoded
+
+        Examples:
+
+            >>> @pwnlib.memleak.MemLeak
+            ... def leaker(addr):
+            ...     return "A"
+            >>> e = leaker.struct(0, pwnlib.elf.Elf32_Phdr)
+            >>> hex(e.p_paddr)
+            '0x41414141'
+
         """
-        size = sizeof(struct)
+        size = ctypes.sizeof(struct)
         data = self.n(address, size)
         obj = struct.from_buffer_copy(data)
         return obj
@@ -118,6 +129,8 @@ class MemLeak(object):
         size   = obj.size
         offset = obj.offset
         data   = self.n(address + offset, size)
+        if not data:
+            return None
         return unpack(data, size*8)
 
     def field_compare(self, address, obj, expected):

pwnlib/regsort.py

@@ -224,6 +224,8 @@ def regsort(in_out, all_regs, tmp = None, xchg = True, randomize = None):
          ('mov', 'b', 'c'),
          ('mov', 'c', 'x'),
          ('mov', 'x', '1')]
+         >>> regsort({'a': 'b', 'b': 'c'}, ['a','b','c'], xchg=0)
+         [('mov', 'a', 'b'), ('mov', 'b', 'c')]
     """
     if randomize is None:
         randomize = context.randomize
@@ -333,7 +335,9 @@ def regsort(in_out, all_regs, tmp = None, xchg = True, randomize = None):
     # If XCHG is expressly disabled, and there is no temporary register,
     # try to see if there is any register which can be used as a temp
     # register instead.
-    if not (xchg or tmp):
+    #
+    # If there are no cycles, there's no need for a temporary register.
+    if in_cycle and not (xchg or tmp):
         for reg in in_out:
             if not depends_on_cycle(reg, in_out, in_cycle):
                 tmp = reg

pwnlib/rop/rop.py

@@ -700,6 +700,9 @@ class ROP(object):
         # If we can find a function with that name, just call it
         if isinstance(resolvable, str):
             addr = self.resolve(resolvable)
+        elif hasattr(resolvable, 'name') and hasattr(resolvable, 'address'):
+            addr = resolvable.address
+            resolvable = str(resolvable.name)
         else:
             addr = resolvable
             resolvable = ''

pwnlib/shellcraft/internal.py

@@ -1,6 +1,8 @@
 import os
 from collections import defaultdict
 
+from ..context import context
+
 __all__ = ['make_function']
 
 loaded = {}
@@ -33,10 +35,14 @@ def init_mako():
 
     render_global = IsInside()
 
+    cache  = context.cache_dir
+    if cache:
+        cache = os.path.join(cache, 'mako')
+
     curdir = os.path.dirname(os.path.abspath(__file__))
     lookup = TemplateLookup(
         directories      = [os.path.join(curdir, 'templates')],
-        module_directory = os.path.expanduser('~/.pwntools-cache/mako')
+        module_directory = cache
     )
 
     # The purpose of this definition is to create a new Tag.

pwnlib/shellcraft/templates/__doc__

@@ -3,14 +3,3 @@ The shellcode module.
 This module contains functions for generating shellcode.
 
 It is organized first by architecture and then by operating system.
-
-Example:
-
-    >>> print shellcraft.i386.nop().strip('\n')
-        nop
-    >>> print shellcraft.i386.linux.sh()
-        /* push '/bin///sh\x00' */
-        push 0x68
-        push 0x732f2f2f
-        push 0x6e69622f
-    ...

pwnlib/shellcraft/templates/amd64/linux/sh.asm

@@ -8,4 +8,4 @@ Execute a different process.
     'Hello\n'
 
 </%docstring>
-${amd64.linux.execve('/bin///sh', 0, 0)}
+${amd64.linux.execve('/bin///sh', ['sh'], 0)}

pwnlib/shellcraft/templates/amd64/mov.asm

@@ -61,17 +61,26 @@ Example:
         mov rax, 0x1010110dfac01fe
         xor [rsp], rax
         pop rax
-   >>> with context.local(os = 'linux'):
-   ...     print shellcraft.amd64.mov('eax', 'SYS_read').rstrip()
-       xor eax, eax /* (SYS_read) */
-   >>> with context.local(os = 'freebsd'):
-   ...     print shellcraft.amd64.mov('eax', 'SYS_read').rstrip()
-       push (SYS_read) /* 3 */
-       pop rax
-   >>> with context.local(os = 'linux'):
-   ...     print shellcraft.amd64.mov('eax', 'PROT_READ | PROT_WRITE | PROT_EXEC').rstrip()
-       push (PROT_READ | PROT_WRITE | PROT_EXEC) /* 7 */
-       pop rax
+    >>> print shellcraft.amd64.mov('rax', 0xffffffff).rstrip()
+        mov eax, 0xffffffff
+    >>> print shellcraft.amd64.mov('rax', 0x7fffffff).rstrip()
+        mov eax, 0x7fffffff
+    >>> print shellcraft.amd64.mov('rax', 0x80010101).rstrip()
+        mov eax, 0x80010101
+    >>> print shellcraft.amd64.mov('rax', 0x80000000).rstrip()
+        mov eax, 0x1010101 /* 2147483648 == 0x80000000 */
+        xor eax, 0x81010101
+    >>> with context.local(os = 'linux'):
+    ...     print shellcraft.amd64.mov('eax', 'SYS_read').rstrip()
+        xor eax, eax /* (SYS_read) */
+    >>> with context.local(os = 'freebsd'):
+    ...     print shellcraft.amd64.mov('eax', 'SYS_read').rstrip()
+        push (SYS_read) /* 3 */
+        pop rax
+    >>> with context.local(os = 'linux'):
+    ...     print shellcraft.amd64.mov('eax', 'PROT_READ | PROT_WRITE | PROT_EXEC').rstrip()
+        push (PROT_READ | PROT_WRITE | PROT_EXEC) /* 7 */
+        pop rax
 
 Args:
   dest (str): The destination register.
@@ -99,7 +108,6 @@ if get_register(src):
     if dest.size == 64 and src.size <= 32:
         dest = get_register(dest.native32)
 
-    src_size = src.size
 else:
     with ctx.local(arch = 'amd64'):
         src = eval(src)
@@ -107,9 +115,9 @@ else:
     if not dest.fits(src):
         log.error("cannot mov %s, %r: dest is smaller than src" % (dest, src))
 
-    src_size = bits_required(src)
+    orig_dest = dest
 
-    if dest.size == 64 and src_size <= 32:
+    if dest.size == 64 and bits_required(src) <= 32:
         dest = get_register(dest.native32)
 
     # Calculate the packed version
@@ -117,7 +125,15 @@ else:
 
     # Calculate the unsigned and signed versions
     srcu = packing.unpack(srcp, dest.size, sign=False)
-    srcs = packing.unpack(srcp, dest.size, sign=True)
+
+    # N.B.: We may have downsized the register for e.g. mov('rax', 0xffffffff)
+    #       In this case, srcp is now a 4-byte packed value, which will expand
+    #       to "-1", which isn't correct.
+    if orig_dest.size != dest.size:
+        srcs = src
+    else:
+        srcs = packing.unpack(srcp, dest.size, sign=True)
+
 %>\
 % if is_register(src):
     % if src == dest:

pwnlib/shellcraft/templates/amd64/popad.asm

@@ -6,8 +6,8 @@ in the same order.
     pop rdi
     pop rsi
     pop rbp
-    pop rsp
-    pop rbp
+    pop rbx /* add rsp, 8 */
+    pop rbx
     pop rdx
     pop rcx
     pop rax

pwnlib/shellcraft/templates/arm/linux/sh.asm

@@ -8,4 +8,4 @@ Execute a different process.
     'Hello\n'
 
 </%docstring>
-    ${arm.linux.execve('/bin///sh', 0, 0)}
+    ${arm.linux.execve('/bin///sh', ['sh'], 0)}

pwnlib/shellcraft/templates/i386/linux/sh.asm

@@ -8,4 +8,4 @@ Execute a different process.
     'Hello\n'
 
 </%docstring>
-${i386.linux.execve('/bin///sh', 0, 0)}
+${i386.linux.execve('/bin///sh', ['sh'], 0)}

pwnlib/shellcraft/templates/mips/android/sh.asm

@@ -1,11 +1,4 @@
 <% from pwnlib.shellcraft import mips %>
 <%docstring>Execute /bin/sh</%docstring>
 
-${mips.push(0)}
-${mips.pushstr('sh')}
-${mips.mov('$a1','$sp')}}
-
-${mips.pushstr('/system/bin//sh')}
-
-${mips.syscall('SYS_execve', '$sp', '$a1', 0)}
-
+${mips.execve('//system/bin/sh', ['sh'], {})}

pwnlib/shellcraft/templates/mips/linux/cat.asm

@@ -1,6 +1,6 @@
 <%
   from pwnlib import constants
-  from pwnlib.shellcraft import mips 
+  from pwnlib.shellcraft import mips
   from pwnlib.shellcraft import common
 %>
 <%page args="filename, fd=1"/>
@@ -11,7 +11,9 @@ Example:
 
     >>> f = tempfile.mktemp()
     >>> write(f, 'FLAG')
-    >>> run_assembly(shellcraft.mips.linux.cat(f)).recvall()
+    >>> asm  = shellcraft.mips.linux.cat(f)
+    >>> asm += shellcraft.mips.linux.exit(0)
+    >>> run_assembly(asm).recvall()
     'FLAG'
 
 </%docstring>

pwnlib/shellcraft/templates/mips/linux/execve.asm

@@ -1,15 +1,47 @@
-
 <%
-    from pwnlib.shellcraft.mips.linux import syscall
+from pwnlib.shellcraft import mips, registers
+from pwnlib.abi import linux_mips_syscall as abi
 %>
-<%page args="path, argv, envp"/>
 <%docstring>
-Invokes the syscall execve.  See 'man 2 execve' for more information.
+Execute a different process.
 
-Arguments:
-    path(char): path
-    argv(char): argv
-    envp(char): envp
+Attempts to perform some automatic detection of types.
+Otherwise, the arguments behave as normal.
+
+- If ``path`` is a string that is not a known register,
+  it is pushed onto the stack.
+- If ``argv`` is an array of strings, it is pushed onto
+  the stack, and NULL-terminated.
+- If ``envp`` is an dictionary of {string:string},
+  it is pushed onto the stack, and NULL-terminated.
+
+Example:
+
+    >>> path = '/bin/sh'
+    >>> argv = ['sh', '-c', 'echo Hello, $NAME; exit $STATUS']
+    >>> envp = {'NAME': 'zerocool', 'STATUS': 3}
+    >>> sc = shellcraft.mips.linux.execve(path, argv, envp)
+    >>> io = run_assembly(sc)
+    >>> io.recvall()
+    'Hello, zerocool\n'
+    >>> io.poll(True)
+    3
 </%docstring>
-
-    ${syscall('SYS_execve', path, argv, envp)}
+<%page args="path = '/system/bin//sh', argv=0, envp=0"/>
+<%
+if isinstance(envp, dict):
+    envp = ['%s=%s' % (k,v) for (k,v) in envp.items()]
+%>
+% if isinstance(argv, (list, tuple)):
+    ${mips.pushstr_array(abi.register_arguments[3], argv)}
+    <% argv = abi.register_arguments[3] %>
+% endif
+% if isinstance(envp, (list, tuple)):
+    ${mips.pushstr_array(abi.register_arguments[2], envp)}
+    <% envp = abi.register_arguments[2] %>
+% endif
+% if isinstance(path, str) and not registers.is_register(path):
+    ${mips.pushstr(path)}
+    <% path = '$sp' %>
+%endif
+    ${mips.syscall('SYS_execve', path, argv, envp)}

pwnlib/shellcraft/templates/mips/linux/mov.asm

@@ -11,8 +11,7 @@ Thin wrapper around :func:`pwnlib.shellcraft.mips.mov`, which sets
 Example:
 
     >>> print pwnlib.shellcraft.mips.linux.mov('$a1', 'SYS_execve').rstrip()
-        .set noat
-        ori $a1, $zero, SYS_execve
+        ori $a1, $zero, (SYS_execve)
 
 </%docstring>
 

pwnlib/shellcraft/templates/mips/linux/pushstr.asm

@@ -12,26 +12,14 @@ Example:
 
     >>> print pwnlib.shellcraft.mips.linux.pushstr('Hello, World').rstrip()
         /* push 'Hello, World\x00' */
-        .set noat
-        slti $at, $zero, -1
-        sw $at, -4($sp)
-        add $sp, $sp, -4
-        .set noat
-        lui $at, 25708
-        ori $at, $at, 29295
-        sw $at, -4($sp)
-        add $sp, $sp, -4
-        .set noat
-        lui $at, 22304
-        ori $at, $at, 11375
-        sw $at, -4($sp)
-        add $sp, $sp, -4
-        .set noat
-        lui $at, 27756
-        ori $at, $at, 25928
-        sw $at, -4($sp)
-        add $sp, $sp, -4
-
+        li $t1, 0x6c6c6548
+        sw $t1, -16($sp)
+        li $t1, 0x57202c6f
+        sw $t1, -12($sp)
+        li $t1, 0x646c726f
+        sw $t1, -8($sp)
+        sw $zero, -4($sp)
+        addiu $sp, $sp, -16
 Args:
   string (str): The string to push.
   append_null (bool): Whether to append a single NULL-byte before pushing.

pwnlib/shellcraft/templates/mips/linux/sh.asm

@@ -1,7 +1,4 @@
 <% from pwnlib.shellcraft import mips %>
 <%docstring>Execute /bin/sh</%docstring>
 
-${mips.pushstr('//bin/sh')}
-
-${mips.syscall('SYS_execve', '$sp', 0, 0)}
-
+${mips.execve('//bin/sh', ['sh'], {})}

pwnlib/shellcraft/templates/mips/linux/syscall.asm

@@ -12,63 +12,55 @@ Any of the arguments can be expressions to be evaluated by :func:`pwnlib.constan
 
 Example:
 
-        >>> print pwnlib.shellcraft.i386.linux.syscall('SYS_execve', 1, 'esp', 2, 0).rstrip()
-            /* call execve(1, 'esp', 2, 0) */
-            push 0x1
-            pop ebx
-            mov ecx, esp
-            push 0x2
-            pop edx
-            xor esi, esi
-            push 0xb
-            pop eax
-            int 0x80
-        >>> print pwnlib.shellcraft.i386.linux.syscall('SYS_execve', 2, 1, 0, 20).rstrip()
-            /* call execve(2, 1, 0, 20) */
-            push 0x2
-            pop ebx
-            push 0x1
-            pop ecx
-            push 0x14
-            pop esi
-            push 0xb
-            pop eax
-            cdq /* Set edx to 0, eax is known to be positive */
-            int 0x80
-        >>> print pwnlib.shellcraft.i386.linux.syscall().rstrip()
+        >>> print pwnlib.shellcraft.mips.linux.syscall('SYS_execve', 1, '$sp', 2, 0).rstrip()
+            /* call execve(1, '$sp', 2, 0) */
+            li $t9, ~1
+            not $a0, $t9
+            add $a1, $sp, $0 /* mov $a1, $sp */
+            li $t9, ~2
+            not $a2, $t9
+            slti $a3, $zero, 0xFFFF /* $a3 = 0 */
+            ori $v0, $zero, (SYS_execve)
+            syscall 0x40404
+        >>> print pwnlib.shellcraft.mips.linux.syscall('SYS_execve', 2, 1, 0, 20).rstrip()
+            /* call execve(2, 1, 0, 0x14) */
+            li $t9, ~2
+            not $a0, $t9
+            li $t9, ~1
+            not $a1, $t9
+            slti $a2, $zero, 0xFFFF /* $a2 = 0 */
+            li $t9, ~0x14
+            not $a3, $t9
+            ori $v0, $zero, (SYS_execve)
+            syscall 0x40404
+        >>> print pwnlib.shellcraft.mips.linux.syscall().rstrip()
             /* call syscall() */
-            int 0x80
-        >>> print pwnlib.shellcraft.i386.linux.syscall('eax', 'ebx', 'ecx').rstrip()
-            /* call syscall('eax', 'ebx', 'ecx') */
-            /* moving ebx into ebx, but this is a no-op */
-            /* moving ecx into ecx, but this is a no-op */
-            /* moving eax into eax, but this is a no-op */
-            int 0x80
-        >>> print pwnlib.shellcraft.i386.linux.syscall('ebp', None, None, 1).rstrip()
-            /* call syscall('ebp', ?, ?, 1) */
-            push 0x1
-            pop edx
-            mov eax, ebp
-            int 0x80
-        >>> print pwnlib.shellcraft.i386.linux.syscall(
+            syscall 0x40404
+        >>> print pwnlib.shellcraft.mips.linux.syscall('$v0', '$a0', '$a1').rstrip()
+            /* call syscall('$v0', '$a0', '$a1') */
+            /* setregs noop */
+            syscall 0x40404
+        >>> print pwnlib.shellcraft.mips.linux.syscall('$a3', None, None, 1).rstrip()
+            /* call syscall('$a3', ?, ?, 1) */
+            li $t9, ~1
+            not $a2, $t9
+            sw $a3, -4($sp) /* mov $v0, $a3 */
+            lw $v0, -4($sp)
+            syscall 0x40404
+        >>> print pwnlib.shellcraft.mips.linux.syscall(
         ...               'SYS_mmap2', 0, 0x1000,
         ...               'PROT_READ | PROT_WRITE | PROT_EXEC',
         ...               'MAP_PRIVATE | MAP_ANONYMOUS',
         ...               -1, 0).rstrip()
-            /* call mmap2(0, 4096, 'PROT_READ | PROT_WRITE | PROT_EXEC', 'MAP_PRIVATE | MAP_ANONYMOUS', -1, 0) */
-            xor ebx, ebx
-            xor ecx, ecx
-            mov ch, 0x10
-            push 0x7
-            pop edx
-            push 0x22
-            pop esi
-            push -1
-            pop edi
-            xor ebp, ebp
-            xor eax, eax
-            mov al, 0xc0
-            int 0x80
+            /* call mmap2(0, 0x1000, 'PROT_READ | PROT_WRITE | PROT_EXEC', 'MAP_PRIVATE | MAP_ANONYMOUS', -1, 0) */
+            slti $a0, $zero, 0xFFFF /* $a0 = 0 */
+            li $t9, ~0x1000
+            not $a1, $t9
+            li $t9, ~(PROT_READ | PROT_WRITE | PROT_EXEC) /* 7 */
+            not $a2, $t9
+            ori $a3, $zero, (MAP_PRIVATE | MAP_ANONYMOUS)
+            ori $v0, $zero, (SYS_mmap2)
+            syscall 0x40404
 </%docstring>
 <%
   append_cdq = False

pwnlib/shellcraft/templates/mips/mov.asm

@@ -120,6 +120,8 @@ if src_reg == 0:
 % elif okay(src):
 ## Nice and easy
     li ${dst}, ${pretty(src)}
+% elif 0 < src <= 0xffff and okay(src, bytes=2):
+    ori ${dst}, $zero, ${src}
 % elif okay((~srcu) & 0xffffffff):
 ## Almost nice and easy
     li $t9, ~${pretty(src)}

pwnlib/shellcraft/templates/mips/pushstr.asm

@@ -21,8 +21,7 @@ Example:
         addiu $sp, $sp, -4
     >>> print shellcraft.mips.pushstr('aa').rstrip()
         /* push 'aa\x00' */
-        li $t9, ~0x6161
-        not $t1, $t9
+        ori $t1, $zero, 24929
         sw $t1, -4($sp)
         addiu $sp, $sp, -4
     >>> print shellcraft.mips.pushstr('aaa').rstrip()

pwnlib/shellcraft/templates/mips/pushstr_array.asm

@@ -0,0 +1,38 @@
+<% from pwnlib.shellcraft import mips %>
+<%docstring>
+Pushes an array/envp-style array of pointers onto the stack.
+
+Arguments:
+    reg(str):
+        Destination register to hold the pointer.
+    array(str,list):
+        Single argument or list of arguments to push.
+        NULL termination is normalized so that each argument
+        ends with exactly one NULL byte.
+</%docstring>
+<%page args="reg, array"/>
+<%
+if isinstance(array, (str)):
+    array = [array]
+
+array_str = ''
+
+# Normalize all of the arguments' endings
+array      = [arg.rstrip('\x00') + '\x00' for arg in array]
+array_str  = ''.join(array)
+
+word_size = 4
+offset = len(array_str) + word_size
+
+%>\
+    /* push argument array ${repr(array)} */
+    ${mips.pushstr(array_str)}
+    ${mips.mov(reg, 0)}
+    ${mips.push(reg)} /* null terminate */
+% for i,arg in enumerate(reversed(array)):
+    ${mips.mov(reg, offset + word_size*i - len(arg))}
+    add ${reg}, $sp
+    ${mips.push(reg)} /* ${repr(arg)} */
+    <% offset -= len(arg) %>\
+% endfor
+    ${mips.mov(reg,'$sp')}

pwnlib/shellcraft/templates/thumb/linux/sh.asm

@@ -8,4 +8,4 @@ Execute a different process.
     'Hello\n'
 
 </%docstring>
-${thumb.linux.execve('/bin///sh', 0, 0)}
+${thumb.linux.execve('/bin///sh', ['sh'], 0)}

pwnlib/term/__init__.py

@@ -7,7 +7,6 @@ from . import readline
 from . import term
 from . import termcap
 from . import text
-from . import completer
 
 # Re-exports (XXX: Are these needed?)
 output = term.output

pwnlib/tubes/process.py

@@ -3,9 +3,11 @@ import errno
 import fcntl
 import logging
 import os
+import platform
 import pty
 import resource
 import select
+import signal
 import subprocess
 import tty
 
@@ -97,6 +99,8 @@ class process(tube):
             Where the process is running, used for logging purposes.
         display(list):
             List of arguments to display, instead of the main executable name.
+        alarm(int):
+            Set a SIGALRM alarm timeout on the process.
 
     Attributes:
         proc(subprocess)
@@ -183,6 +187,13 @@ class process(tube):
 
         >>> process(['sh','-c','ulimit -s'], aslr=0).recvline()
         'unlimited\n'
+
+        >>> io = process(['sh','-c','sleep 10; exit 7'], alarm=2)
+        >>> io.poll(block=True) == -signal.SIGALRM
+        True
+
+        >>> binary = ELF.from_assembly('nop', arch='mips')
+        >>> p = process(binary.path)
     """
 
     PTY = PTY
@@ -190,7 +201,7 @@ class process(tube):
     #: Have we seen the process stop?
     _stop_noticed = False
 
-    def __init__(self, argv,
+    def __init__(self, argv = None,
                  shell = False,
                  executable = None,
                  cwd = None,
@@ -206,9 +217,18 @@ class process(tube):
                  aslr = None,
                  setuid = None,
                  where = 'local',
-                 display = None):
+                 display = None,
+                 alarm = None):
         super(process, self).__init__(timeout, level = level)
 
+        # Permit using context.binary
+        if argv is None:
+            if context.binary:
+                argv = [context.binary.path]
+            else:
+                raise TypeError('Must provide argv or set context.binary')
+
+
         #: `subprocess.Popen` object
         self.proc = None
 
@@ -253,6 +273,9 @@ class process(tube):
         #: Directory the process was created in
         self.cwd          = cwd or os.path.curdir
 
+        #: Alarm timeout of the process
+        self.alarm        = alarm
+
         self.preexec_fn = preexec_fn
         self.display    = display or self.program
 
@@ -265,7 +288,7 @@ class process(tube):
         with self.progress(message) as p:
 
             if not self.aslr:
-                log.warn_once("ASLR is disabled!")
+                self.warn_once("ASLR is disabled!")
 
             # In the event the binary is a foreign architecture,
             # and binfmt is not installed (e.g. when running on
@@ -275,12 +298,6 @@ class process(tube):
             executables = [executable]
             exception = None
 
-            try:
-                if not context.native:
-                    qemu = get_qemu_user()
-                    prefixes.append(([qemu], qemu))
-            except: pass
-
             for prefix, executable in prefixes:
                 try:
                     self.proc = subprocess.Popen(args = prefix + argv,
@@ -297,11 +314,7 @@ class process(tube):
                 except OSError as exception:
                     if exception.errno != errno.ENOEXEC:
                         raise
-            else:
-                try:
-                    raise exception
-                except:
-                    log.exception(str(prefixes))
+                    prefixes.append(self.__on_enoexec(exception))
 
         if self.pty is not None:
             if stdin is slave:
@@ -338,7 +351,7 @@ class process(tube):
 
                 resource.setrlimit(resource.RLIMIT_STACK, (-1, -1))
             except:
-                log.exception("Could not disable ASLR")
+                self.exception("Could not disable ASLR")
 
         # Assume that the user would prefer to have core dumps.
         resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
@@ -357,15 +370,48 @@ class process(tube):
             except:
                 pass
 
+        if self.alarm is not None:
+            signal.alarm(self.alarm)
+
         self.preexec_fn()
 
+    def __on_enoexec(self, exception):
+        """We received an 'exec format' error (ENOEXEC)
+
+        This implies that the user tried to execute e.g.
+        an ARM binary on a non-ARM system, and does not have
+        binfmt helpers installed for QEMU.
+        """
+        # Get the ELF binary for the target executable
+        with context.quiet:
+            # XXX: Cyclic imports :(
+            from ..elf import ELF
+            binary = ELF(self.executable)
+
+        # If we're on macOS, this will never work.  Bail now.
+        # if platform.mac_ver()[0]:
+            # self.error("Cannot run ELF binaries on macOS")
+
+        # Determine what architecture the binary is, and find the
+        # appropriate qemu binary to run it.
+        qemu = get_qemu_user(arch=binary.arch)
+        if qemu:
+            args = [qemu]
+            if self.argv:
+                args += ['-0', self.argv[0]]
+            args += ['--']
+            return [args, qemu]
+
+        # If we get here, we couldn't run the binary directly, and
+        # we don't have a qemu which can run it.
+        self.exception(exception)
+
     @property
     def program(self):
         """Alias for ``executable``, for backward compatibility"""
         return self.executable
 
-    @staticmethod
-    def _validate(cwd, executable, argv, env):
+    def _validate(self, cwd, executable, argv, env):
         """
         Perform extended validation on the executable path, argv, and envp.
 
@@ -384,14 +430,14 @@ class process(tube):
             argv = [argv]
 
         if not all(isinstance(arg, (str, unicode)) for arg in argv):
-            log.error("argv must be strings: %r" % argv)
+            self.error("argv must be strings: %r" % argv)
 
         # Create a duplicate so we can modify it
         argv = list(argv or [])
 
         for i, arg in enumerate(argv):
             if '\x00' in arg[:-1]:
-                log.error('Inappropriate nulls in argv[%i]: %r' % (i, arg))
+                self.error('Inappropriate nulls in argv[%i]: %r' % (i, arg))
 
             argv[i] = arg.rstrip('\x00')
 
@@ -403,7 +449,7 @@ class process(tube):
         #
         if not executable:
             if not argv:
-                log.error("Must specify argv or executable")
+                self.error("Must specify argv or executable")
             executable = argv[0]
 
         # Do not change absolute paths to binaries
@@ -423,11 +469,11 @@ class process(tube):
             executable = os.path.join(cwd, executable)
 
         if not os.path.exists(executable):
-            log.error("%r does not exist"  % executable)
+            self.error("%r does not exist"  % executable)
         if not os.path.isfile(executable):
-            log.error("%r is not a file" % executable)
+            self.error("%r is not a file" % executable)
         if not os.access(executable, os.X_OK):
-            log.error("%r is not marked as executable (+x)" % executable)
+            self.error("%r is not marked as executable (+x)" % executable)
 
         #
         # Validate environment
@@ -441,13 +487,13 @@ class process(tube):
 
         for k,v in env.items():
             if not isinstance(k, (str, unicode)):
-                log.error('Environment keys must be strings: %r' % k)
+                self.error('Environment keys must be strings: %r' % k)
             if not isinstance(k, (str, unicode)):
-                log.error('Environment values must be strings: %r=%r' % (k,v))
+                self.error('Environment values must be strings: %r=%r' % (k,v))
             if '\x00' in k[:-1]:
-                log.error('Inappropriate nulls in env key: %r' % (k))
+                self.error('Inappropriate nulls in env key: %r' % (k))
             if '\x00' in v[:-1]:
-                log.error('Inappropriate nulls in env value: %r=%r' % (k, v))
+                self.error('Inappropriate nulls in env value: %r=%r' % (k, v))
 
             env[k.rstrip('\x00')] = v.rstrip('\x00')
 
@@ -757,7 +803,7 @@ class process(tube):
         """
         # If it's running under qemu-user, don't leak anything.
         if 'qemu-' in os.path.realpath('/proc/%i/exe' % self.pid):
-            log.error("Cannot use leaker on binaries under QEMU.")
+            self.error("Cannot use leaker on binaries under QEMU.")
 
         with open('/proc/%i/mem' % self.pid, 'rb') as mem:
             mem.seek(address)

pwnlib/tubes/remote.py

@@ -108,38 +108,30 @@ class remote(sock):
         sock    = None
         timeout = self.timeout
 
-        h = self.waitfor('Opening connection to %s on port %d' % (self.rhost, self.rport))
+        with self.waitfor('Opening connection to %s on port %d' % (self.rhost, self.rport)) as h:
+            for res in socket.getaddrinfo(self.rhost, self.rport, fam, typ, 0, socket.AI_PASSIVE):
+                self.family, self.type, self.proto, _canonname, sockaddr = res
 
-        for res in socket.getaddrinfo(self.rhost, self.rport, fam, typ, 0, socket.AI_PASSIVE):
-            self.family, self.type, self.proto, _canonname, sockaddr = res
+                if self.type not in [socket.SOCK_STREAM, socket.SOCK_DGRAM]:
+                    continue
 
-            if self.type not in [socket.SOCK_STREAM, socket.SOCK_DGRAM]:
-                continue
+                h.status("Trying %s" % sockaddr[0])
 
-            h.status("Trying %s" % sockaddr[0])
+                sock = socket.socket(self.family, self.type, self.proto)
 
-            sock = socket.socket(self.family, self.type, self.proto)
+                if timeout != None and timeout <= 0:
+                    sock.setblocking(0)
+                else:
+                    sock.setblocking(1)
+                    sock.settimeout(timeout)
 
-            if timeout != None and timeout <= 0:
-                sock.setblocking(0)
-            else:
-                sock.setblocking(1)
-                sock.settimeout(timeout)
-
-            try:
-                sock.connect(sockaddr)
-                break
-            except socket.error:
-                pass
-        else:
-            h.failure()
+                try:
+                    sock.connect(sockaddr)
+                    return sock
+                except socket.error:
+                    pass
             self.error("Could not connect to %s on port %d" % (self.rhost, self.rport))
 
-        h.success()
-        return sock
-
-
-
     @classmethod
     def fromsocket(cls, socket):
         """
@@ -157,13 +149,18 @@ class remote(sock):
         return remote(host, port, fam=s.family, typ=s.type, sock=s)
 
 class tcp(remote):
+    __doc__ = remote.__doc__
     def __init__(self, host, port,
                  fam = "any", typ = "tcp",
                  timeout = Timeout.default, ssl=False, sock=None, level = None):
         return super(tcp, self).__init__(host, port, fam, typ, timeout, ssl, sock, level)
 
 class udp(remote):
+    __doc__ = remote.__doc__
     def __init__(self, host, port,
                  fam = "any", typ = "udp",
                  timeout = Timeout.default, ssl=False, sock=None, level = None):
         return super(udp, self).__init__(host, port, fam, typ, timeout, ssl, sock, level)
+
+class connect(remote):
+    __doc__ = remote.__doc__

pwnlib/tubes/ssh.py

@@ -19,6 +19,7 @@ from ..timeout import Timeout
 from ..util import hashes
 from ..util import misc
 from ..util import safeeval
+from ..util import sh_string
 from .process import process
 from .sock import sock
 
@@ -84,16 +85,16 @@ class ssh_channel(sock):
 
         if isinstance(process, (list, tuple)):
             fmt = ' '.join('%s' for s in process)
-            process = misc.sh_command_with(fmt, *process)
+            process = sh_string.sh_command_with(fmt, *process)
 
         if process and wd:
-            process = misc.sh_command_with('cd %s >/dev/null 2>&1;', wd) + process
+            process = sh_string.sh_command_with('cd %s >/dev/null 2>&1;', wd) + process
 
         if process and env:
             for name, value in env.items():
                 if not re.match('^[a-zA-Z_][a-zA-Z0-9_]*$', name):
                     self.error('run(): Invalid environment key $r' % name)
-                process = '%s;%s' % (misc.sh_prepare(name, value, export=True), process)
+                process = '%s;%s' % (sh_string.sh_prepare(name, value, export=True), process)
 
         if process and tty:
             if raw:
@@ -290,7 +291,7 @@ class ssh_channel(sock):
         """Retrieve the address of an environment variable in the remote process.
         """
         if not hasattr(self, 'argv'):
-            log.error("Can only call getenv() on ssh_channel objects created with ssh.process")
+            self.error("Can only call getenv() on ssh_channel objects created with ssh.process")
 
         argv0 = self.argv[0]
 
@@ -667,8 +668,12 @@ class ssh(Timeout, Logger):
                 See ``stdin``.
             preexec_fn(callable):
                 Function which is executed on the remote side before execve().
+                This **MUST** be a self-contained function -- it must perform
+                all of its own imports, and cannot refer to variables outside
+                its scope.
             preexec_args(object):
                 Argument passed to ``preexec_fn``.
+                This **MUST** only consist of native Python objects.
             raw(bool):
                 If ``True``, disable TTY control code interpretation.
             aslr(bool):
@@ -715,6 +720,23 @@ class ssh(Timeout, Logger):
             ''
             >>> s.process('/usr/bin/env', env={'A':'B'}).recvall()
             'A=B\n'
+
+            >>> s.process('false', preexec_fn=1234)
+            Traceback (most recent call last):
+            ...
+            PwnlibException: preexec_fn must be a function
+
+            >>> s.process('false', preexec_fn=lambda: 1234)
+            Traceback (most recent call last):
+            ...
+            PwnlibException: preexec_fn cannot be a lambda
+
+            >>> def uses_globals():
+            ...     foo = bar
+            >>> print s.process('false', preexec_fn=uses_globals).recvall().strip() # doctest: +ELLIPSIS
+            Traceback (most recent call last):
+            ...
+            NameError: global name 'bar' is not defined
         """
         if not argv and not executable:
             self.error("Must specify argv or executable")
@@ -766,15 +788,16 @@ class ssh(Timeout, Logger):
         # Allow the user to provide a self-contained function to run
         def func(): pass
         func      = preexec_fn or func
-        func_src  = inspect.getsource(func).strip()
-        func_name = func.__name__
         func_args = preexec_args
 
         if not isinstance(func, types.FunctionType):
-            log.error("preexec_fn must be a function")
-        if func_name == (lambda: 0).__name__:
-            log.error("preexec_fn cannot be a lambda")
+            self.error("preexec_fn must be a function")
 
+        func_name = func.__name__
+        if func_name == (lambda: 0).__name__:
+            self.error("preexec_fn cannot be a lambda")
+
+        func_src  = inspect.getsource(func).strip()
         setuid = setuid if setuid is None else bool(setuid)
 
         script = r"""
@@ -876,7 +899,7 @@ os.execve(exe, argv, os.environ)
             if not aslr:
                 self.warn_once("ASLR is disabled!")
 
-            script = misc.sh_command_with('for py in python2.7 python2 python; do test -x "$(which $py 2>&1)" && exec $py -c %s check; done; echo 2', script)
+            script = sh_string.sh_command_with('for py in python2.7 python2 python; do test -x "$(which $py 2>&1)" && exec $py -c %s check; done; echo 2', script)
             with context.local(log_level='error'):
                 python = self.run(script, raw=raw)
             result = safeeval.const(python.recvline())
@@ -1147,7 +1170,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
     def _libs_remote(self, remote):
         """Return a dictionary of the libraries used by a remote file."""
         cmd = '(ulimit -s unlimited; ldd %s > /dev/null && (LD_TRACE_LOADED_OBJECTS=1 %s || ldd %s)) 2>/dev/null'
-        cmd = misc.sh_command_with(lambda arg: cmd % (arg, arg, arg), remote)
+        cmd = sh_string.sh_command_with(lambda arg: cmd % (arg, arg, arg), remote)
         data, status = self.run_to_end(cmd)
         if status != 0:
             self.error('Unable to find libraries for %r' % remote)
@@ -1157,7 +1180,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
 
     def _get_fingerprint(self, remote):
         cmd = '(openssl sha256 || sha256 || sha256sum) 2>/dev/null < %s'
-        cmd = misc.sh_command_with(cmd, remote)
+        cmd = sh_string.sh_command_with(cmd, remote)
         data, status = self.run_to_end(cmd)
 
         if status != 0:
@@ -1200,7 +1223,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             self.sftp.get(remote, local, update)
             return
 
-        cmd = misc.sh_command_with('wc -c < %s', remote)
+        cmd = sh_string.sh_command_with('wc -c < %s', remote)
         total, exitcode = self.run_to_end(cmd)
 
         if exitcode != 0:
@@ -1210,7 +1233,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         total = int(total)
 
         with context.local(log_level = 'ERROR'):
-            cmd = misc.sh_command_with('cat < %s', remote)
+            cmd = sh_string.sh_command_with('cat < %s', remote)
             c = self.run(cmd)
         data = ''
 
@@ -1321,7 +1344,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             remote = str(self.sftp.normalize(remote))
         else:
             with context.local(log_level='error'):
-                remote = self.system(misc.sh_command_with('readlink -f %s', remote))
+                remote = self.system(sh_string.sh_command_with('readlink -f %s', remote))
 
         dirname  = os.path.dirname(remote)
         basename = os.path.basename(remote)
@@ -1333,7 +1356,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
 
         with context.local(log_level='error'):
             remote_tar = self.mktemp()
-            tar = self.system(misc.sh_command_with('tar -C %s -czf %s %s', dirname, remote_tar, basename))
+            tar = self.system(sh_string.sh_command_with('tar -C %s -czf %s %s', dirname, remote_tar, basename))
 
             if 0 != tar.wait():
                 self.error("Could not create remote tar")
@@ -1377,7 +1400,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
                 return
 
         with context.local(log_level = 'ERROR'):
-            cmd = misc.sh_command_with('cat>%s', remote)
+            cmd = sh_string.sh_command_with('cat>%s', remote)
             s = self.run(cmd, tty=False)
             s.send(data)
             s.shutdown('send')
@@ -1386,7 +1409,6 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             if result != 0:
                 self.error("Could not upload file %r (%r)\n%s" % (remote, result, data))
 
-
     def upload_file(self, filename, remote = None):
         """Uploads a file to the remote server. Returns the remote filename.
 
@@ -1454,14 +1476,14 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         if os.path.isdir(file_or_directory):
             return self.upload_dir(file_or_directory, remote)
 
-        log.error('%r does not exist' % file_or_directory)
+        self.error('%r does not exist' % file_or_directory)
 
 
     def download(self, file_or_directory, remote=None):
         if not self.sftp:
             self.error("Cannot determine remote file type without SFTP")
 
-        if 0 == self.system(misc.sh_command_with('test -d %s', file_or_directory)).wait():
+        if 0 == self.system(sh_string.sh_command_with('test -d %s', file_or_directory)).wait():
             self.download_dir(file_or_directory, remote)
         else:
             self.download_file(file_or_directory, remote)
@@ -1517,7 +1539,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         s = self.shell(shell)
 
         if self.cwd != '.':
-            cmd = misc.sh_command_with('cd %s', self.cwd)
+            cmd = sh_string.sh_command_with('cd %s', self.cwd)
             s.sendline(cmd)
 
         s.interactive()
@@ -1559,7 +1581,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
                 self.error("Could not generate a temporary directory (%i)\n%s" % (status, wd))
 
         else:
-            cmd = misc.sh_command_with('ls %s', wd)
+            cmd = sh_string.sh_command_with('ls %s', wd)
             _, status = self.run_to_end(cmd, wd = '.')
 
             if status:
@@ -1568,3 +1590,11 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         self.info("Working directory: %r" % wd)
         self.cwd = wd
         return self.cwd
+
+    def write(self, path, data):
+        """Wrapper around upload_data to match ``pwnlib.util.misc.write``"""
+        return self.upload_data(data, path)
+
+    def read(self, path):
+        """Wrapper around download_data to match ``pwnlib.util.misc.read``"""
+        return self.download_data(path)

pwnlib/tubes/tube.py

@@ -826,6 +826,35 @@ class tube(Timeout, Logger):
         while t.is_alive():
             t.join(timeout = 0.1)
 
+    def stream(self, line_mode=True):
+        """stream()
+
+        Receive data until the tube exits, and print it to stdout.
+
+        Similar to :func:`interactive`, except that no input is sent.
+
+        Similar to ``print tube.recvall()`` except that data is printed
+        as it is received, rather than after all data is received.
+
+        Arguments:
+            line_mode(bool): Whether to receive line-by-line or raw data.
+
+        Returns:
+            All data printed.
+        """
+        buf = Buffer()
+        function = self.recvline if line_mode else self.recv
+        try:
+            while True:
+                buf.add(function())
+                sys.stdout.write(buf.data[-1])
+        except KeyboardInterrupt:
+            pass
+        except EOFError:
+            pass
+
+        return buf.get()
+
     def clean(self, timeout = 0.05):
         """clean(timeout = 0.05)
 

pwnlib/update.py

@@ -1,58 +1,63 @@
+"""
+# Pwntools Update
+
+In order to ensure that Pwntools users always have the latest and
+greatest version, Pwntools automatically checks for updates.
+
+Since this update check takes a moment, it is only performed once
+every week.  It can be permanently disabled via:
+
+.. code-block:: bash
+
+    $ echo never > ~/.pwntools-cache/update
+
+"""
 import datetime
 import json
-import pip
-import pkg_resources
 import os
 import time
+import xmlrpclib
+
+import packaging.version
 
 from .context import context
 from .log import getLogger
-from .version import __version__
 from .util.misc import read
 from .util.misc import write
 from .util.web import wget
+from .version import __version__
 
 log = getLogger(__name__)
 
-current_version = pkg_resources.parse_version(__version__)
+current_version = packaging.version.Version(__version__)
 package_name    = 'pwntools'
 package_repo    = 'Gallopsled/pwntools'
 update_freq     = datetime.timedelta(days=7).total_seconds()
 
-def available_on_github(prerelease=False):
-    """Return True if an update is available on Github."""
-    url = 'https://api.github.com/repos/%s/tags' % package_repo
+def available_on_pypi(prerelease=current_version.is_prerelease):
+    """Return True if an update is available on PyPI.
 
-    with context.quiet:
-        tags = json.loads(wget(url))
-
-
-    versions = map(pkg_resources.parse_version, [t['name'] for t in tags])
+    >>> available_on_pypi() # doctest: +ELLIPSIS
+    <Version('...')>
+    >>> available_on_pypi(prerelease=False).is_prerelease
+    False
+    """
+    client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
+    versions = client.package_releases('pwntools', True)
+    versions = map(packaging.version.Version, versions)
 
     if not prerelease:
         versions = filter(lambda v: not v.is_prerelease, versions)
 
     return max(versions)
 
-def available_on_pypi(prerelease=False):
-    """Return True if an update is available on PyPI."""
-    search_command = pip.commands.search.SearchCommand()
-    options, _ = search_command.parse_args([package_name])
-    pypi_hits = search_command.search(package_name, options)
-    for hit in pip.commands.search.transform_hits(pypi_hits):
-        if hit['name'] != 'pwntools':
-            continue
-
-        versions = map(pkg_resources.parse_version, hit['versions'])
-
-        if not prerelease:
-            versions = filter(lambda v: not v.is_prerelease, versions)
-
-        return max(versions)
-
 def cache_file():
     """Returns the path of the file used to cache update data, and ensures that it exists."""
-    cache_dir  = os.path.expanduser('~/.pwntools-cache')
+    cache_dir = context.cache_dir
+
+    if not cache_dir:
+        return None
+
     cache_file = os.path.join(cache_dir, 'update')
 
     if not os.path.isdir(cache_dir):
@@ -65,47 +70,86 @@ def cache_file():
 
 def last_check():
     """Return the date of the last check"""
-    return os.path.getmtime(cache_file())
+    cache = cache_file()
+    if cache:
+        return os.path.getmtime(cache_file())
+
+    # Fallback
+    return time.time()
 
 def should_check():
     """Return True if we should check for an update"""
-    if read(cache_file()).strip() == 'never':
+    filename = cache_file()
+
+    if not filename:
         return False
+
+    if read(filename).strip() == 'never':
+        return False
+
     return time.time() > (last_check() + update_freq)
 
-def perform_check(prerelease=False):
-    """Perform the update check, and report to the user."""
+def perform_check(prerelease=current_version.is_prerelease):
+    """Perform the update check, and report to the user.
+
+    Arguments:
+        prerelease(bool): Whether or not to include pre-release versions.
+
+    Returns:
+        A list of arguments to the update command.
+
+    >>> from packaging.version import Version
+    >>> pwnlib.update.current_version = Version("999.0.0")
+    >>> print perform_check()
+    None
+    >>> pwnlib.update.current_version = Version("0.0.0")
+    >>> perform_check() # doctest: +ELLIPSIS
+    ['pip', 'install', '-U', ...]
+
+    >>> def bail(*a): raise Exception()
+    >>> pypi   = pwnlib.update.available_on_pypi
+
+    >>> perform_check(prerelease=False)
+    ['pip', 'install', '-U', 'pwntools']
+    >>> perform_check(prerelease=True)  # doctest: +ELLIPSIS
+    ['pip', 'install', '-U', 'pwntools...']
+    """
     pypi = current_version
     try:
         pypi = available_on_pypi(prerelease)
-    except:
+    except Exception:
         log.warning("An issue occurred while checking PyPI")
 
-    github = current_version
-    try:
-        github = available_on_github(prerelease)
-    except:
-        log.warning("An issue occurred while checking Github")
-
-    best = max(pypi, github, current_version)
+    best = max(pypi, current_version)
     where = None
     command = None
 
-    os.utime(cache_file(), None)
+    cache = cache_file()
+
+    if cache:
+        os.utime(cache, None)
 
     if best == current_version:
         log.info("You have the latest version of Pwntools (%s)" % best)
         return
 
+    command = [
+        'pip',
+        'install',
+        '-U'
+    ]
+
     if best == pypi:
         where = 'pypi'
-        command = 'pip install -U %s' % package_name
-    else:
-        where = 'GitHub'
-        command = 'pip install -U git+https://github.com/%s.git@%s' % (package_repo, github)
+        pypi_package = package_name
+        if best.is_prerelease:
+            pypi_package += '==%s' % (best)
+        command += [pypi_package]
+
+    command_str = ' '.join(command)
 
     log.info("A newer version of %s is available on %s (%s --> %s).\n" % (package_name, where, current_version, best) +
-             "Update with: $ %s" % command)
+             "Update with: $ %s" % command_str)
 
     return command
 

pwnlib/util/__init__.py

@@ -15,4 +15,5 @@ from . import net
 from . import packing
 from . import proc
 from . import safeeval
+from . import sh_string
 from . import web

pwnlib/util/crc/__init__.py

@@ -1,10 +1,10 @@
 """Module for calculating CRC-sums.
 
-Contains all crc implementations know on the interwebz. For most implementations it
-contains only the core crc algorithm and not e.g. padding schemes.
+Contains all crc implementations know on the interwebz. For most implementations
+it contains only the core crc algorithm and not e.g. padding schemes.
 
 It is horribly slow, as implements a naive algorithm working direclty on
-bit polynomials.
+bit polynomials. This class is exposed as `BitPolynom`.
 
 The current algorithm is super-linear and takes about 4 seconds to calculate
 the crc32-sum of ``'A'*40000``.
@@ -18,15 +18,70 @@ import types
 from . import known
 from .. import fiddling
 from .. import packing
+from .. import safeeval
 
 
 class BitPolynom(object):
+    """Class for representing GF(2)[X], i.e. the field of polynomials over
+    GF(2).
+
+    In practice the polynomials are represented as numbers such that `x**n`
+    corresponds to `1 << n`. In this representation calculations are easy: Just
+    do everything as normal, but forget about everything the carries.
+
+    Addition becomes xor and multiplication becomes carry-less multiplication.
+
+    Examples:
+
+        >>> p1 = BitPolynom("x**3 + x + 1")
+        >>> p1
+        BitPolynom('x**3 + x + 1')
+        >>> int(p1)
+        11
+        >>> p1 == BitPolynom(11)
+        True
+        >>> p2 = BitPolynom("x**2 + x + 1")
+        >>> p1 + p2
+        BitPolynom('x**3 + x**2')
+        >>> p1 * p2
+        BitPolynom('x**5 + x**4 + 1')
+        >>> p1 / p2
+        BitPolynom('x + 1')
+        >>> p1 % p2
+        BitPolynom('x')
+        >>> d, r = divmod(p1, p2)
+        >>> d * p2 + r == p1
+        True
+        >>> BitPolynom(-1)
+        Traceback (most recent call last):
+            ...
+        ValueError: Polynomials cannot be negative: -1
+        >>> BitPolynom('y')
+        Traceback (most recent call last):
+            ...
+        ValueError: Not a valid polynomial: y
+    """
+
+
     def __init__(self, n):
-        if not isinstance(n, (int, long)):
-            raise TypeError("Polynomial must be called with an integer or a list")
-        if n < 0:
-            raise ValueError("Polynomials cannot be negative: %d" % n)
-        self.n = n
+        if isinstance(n, (str, unicode)):
+            self.n = 0
+            x = BitPolynom(2)
+            try:
+                for p in n.split('+'):
+                    k = safeeval.values(p.strip(), {'x': x, 'X': x})
+                    assert isinstance(k, (BitPolynom, int, long))
+                    assert k >= 0
+                    self.n ^= int(k)
+            except (ValueError, NameError, AssertionError):
+                raise ValueError("Not a valid polynomial: %s" % n)
+        elif isinstance(n, (int, long)):
+            if n >= 0:
+                self.n = n
+            else:
+                raise ValueError("Polynomials cannot be negative: %d" % n)
+        else:
+            raise TypeError("Polynomial must be called with a string or integer")
 
     def __int__(self):
         return self.n
@@ -126,7 +181,30 @@ class BitPolynom(object):
     def __rrshift__(self, other):
         return BitPolynom(int(other) >> int(self))
 
+    def __pow__(self, other):
+        r = BitPolynom(1)
+        for _ in range(other):
+            r *= self
+        return r
+
     def degree(self):
+        """Returns the degree of the polynomial.
+
+        Examples:
+
+            >>> BitPolynom(0).degree()
+            0
+            >>> BitPolynom(1).degree()
+            0
+            >>> BitPolynom(2).degree()
+            1
+            >>> BitPolynom(7).degree()
+            2
+            >>> BitPolynom((1 << 10) - 1).degree()
+            9
+            >>> BitPolynom(1 << 10).degree()
+            10
+        """
         return max(0, int(self).bit_length()-1)
 
     def __repr__(self):
@@ -134,17 +212,20 @@ class BitPolynom(object):
             return '0'
 
         out = []
-        for n in range(self.degree(), 0, -1):
+        for n in range(self.degree(), 1, -1):
             if int(self) & (1 << n):
                 out.append("x**%d" % n)
+        if int(self) & 2:
+            out.append("x")
         if int(self) & 1:
             out.append("1")
-        return ' + '.join(out)
+        return 'BitPolynom(%r)' % ' + '.join(out)
 
 class Module(types.ModuleType):
     def __init__(self):
         super(Module, self).__init__(__name__)
         self._cached_crcs = None
+        self.BitPolynom = BitPolynom
         self.__dict__.update({
             '__file__'    : __file__,
             '__package__' : __package__,
@@ -154,7 +235,7 @@ class Module(types.ModuleType):
         crcs = known.all_crcs
 
         if attr == '__all__':
-            return ['generic_crc', 'cksum', 'find_crc_function'] + sorted(crcs.keys())
+            return ['BitPolynom', 'generic_crc', 'cksum', 'find_crc_function'] + sorted(crcs.keys())
 
         info = crcs.get(attr, None)
         if not info:

pwnlib/util/cyclic.py

@@ -1,8 +1,8 @@
 import string
 
+from . import packing
 from ..context import context
 from ..log import getLogger
-from . import packing
 
 log = getLogger(__name__)
 
@@ -43,8 +43,8 @@ def de_bruijn(alphabet = string.ascii_lowercase, n = None):
 def cyclic(length = None, alphabet = string.ascii_lowercase, n = None):
     """cyclic(length = None, alphabet = string.ascii_lowercase, n = 4) -> list/str
 
-    A simple wrapper over :func:`de_bruijn`. This function returns a
-    at most `length` elements.
+    A simple wrapper over :func:`de_bruijn`. This function returns at most
+    `length` elements.
 
     If the given alphabet is a string, a string is returned from this function. Otherwise
     a list is returned.
@@ -66,6 +66,10 @@ def cyclic(length = None, alphabet = string.ascii_lowercase, n = None):
     if n is None:
         n = 4
 
+    if len(alphabet) ** n < length:
+        log.error("Can't create a pattern length=%i with len(alphabet)==%i and n==%i" \
+                  % (length, len(alphabet), n))
+
     out = []
     for ndx, c in enumerate(de_bruijn(alphabet, n)):
         if length != None and ndx >= length:
@@ -93,8 +97,8 @@ def cyclic_find(subseq, alphabet = string.ascii_lowercase, n = None):
        https://www.sciencedirect.com/science/article/pii/S0012365X00001175
 
     Arguments:
-        subseq: The subsequence to look for. This can either be a string, a list
-                or an integer. If an integer is provided it will be packed as a
+        subseq: The subsequence to look for. This can be a string, a list or an
+                integer. If an integer is provided it will be packed as a
                 little endian integer.
         alphabet: List or string to generate the sequence over.
         n(int): The length of subsequences that should be unique.
@@ -124,8 +128,98 @@ def cyclic_find(subseq, alphabet = string.ascii_lowercase, n = None):
 
     return _gen_find(subseq, de_bruijn(alphabet, n))
 
+def metasploit_pattern(sets = None):
+    """metasploit_pattern(sets = [ string.ascii_uppercase, string.ascii_lowercase, string.digits ]) -> generator
+
+    Generator for a sequence of characters as per Metasploit Framework's
+    `Rex::Text.pattern_create` (aka `pattern_create.rb`).
+
+    The returned generator will yield up to
+    ``len(sets) * reduce(lambda x,y: x*y, map(len, sets))`` elements.
+
+    Arguments:
+        sets: List of strings to generate the sequence over.
+    """
+    sets = sets or [ string.ascii_uppercase, string.ascii_lowercase, string.digits ]
+    offsets = [ 0 ] * len(sets)
+    offsets_indexes_reversed = list(reversed(range(len(offsets))))
+
+    while True:
+        for i, j in zip(sets, offsets):
+            yield i[j]
+        # increment offsets with cascade
+        for i in offsets_indexes_reversed:
+            offsets[i] = (offsets[i] + 1) % len(sets[i])
+            if offsets[i] != 0:
+                break
+        # finish up if we've exhausted the sequence
+        if offsets == [ 0 ] * len(sets):
+            return
+
+def cyclic_metasploit(length = None, sets = None):
+    """cyclic_metasploit(length = None, sets = [ string.ascii_uppercase, string.ascii_lowercase, string.digits ]) -> str
+
+    A simple wrapper over :func:`metasploit_pattern`. This function returns a
+    string of length `length`.
+
+    Arguments:
+        length: The desired length of the string or None if the entire sequence is desired.
+        sets: List of strings to generate the sequence over.
+
+    Example:
+        >>> cyclic_metasploit(32)
+        'Aa0Aa1Aa2Aa3Aa4Aa5Aa6Aa7Aa8Aa9Ab'
+        >>> cyclic_metasploit(sets = ["AB","ab","12"])
+        'Aa1Aa2Ab1Ab2Ba1Ba2Bb1Bb2'
+        >>> cyclic_metasploit()[1337:1341]
+        '5Bs6'
+        >>> len(cyclic_metasploit())
+        20280
+    """
+    sets = sets or [ string.ascii_uppercase, string.ascii_lowercase, string.digits ]
+    out = []
+
+    for ndx, c in enumerate(metasploit_pattern(sets)):
+        if length != None and ndx >= length:
+            break
+        else:
+            out.append(c)
+
+    out = ''.join(out)
+
+    if len(out) < length:
+        log.error("Can't create a pattern of length %i with sets of lengths %s. Maximum pattern length is %i." \
+                  % (length, map(len, sets), len(out)))
+
+    return ''.join(out)
+
+def cyclic_metasploit_find(subseq, sets = None):
+    """cyclic_metasploit_find(subseq, sets = [ string.ascii_uppercase, string.ascii_lowercase, string.digits ]) -> int
+
+    Calculates the position of a substring into a Metasploit Pattern sequence.
+
+    Arguments:
+        subseq: The subsequence to look for. This can be a string or an
+                integer. If an integer is provided it will be packed as a
+                little endian integer.
+        sets: List of strings to generate the sequence over.
+
+    Examples:
+
+        >>> cyclic_metasploit_find(cyclic_metasploit(1000)[514:518])
+        514
+        >>> cyclic_metasploit_find(0x61413161)
+        4
+    """
+    sets = sets or [ string.ascii_uppercase, string.ascii_lowercase, string.digits ]
+
+    if isinstance(subseq, (int, long)):
+        subseq = packing.pack(subseq, 'all', 'little', False)
+
+    return _gen_find(subseq, metasploit_pattern(sets))
+
 def _gen_find(subseq, generator):
-    """Returns the first position of subseq in the generator or -1 if there is no such position."""
+    """Returns the first position of `subseq` in the generator or -1 if there is no such position."""
     subseq = list(subseq)
     pos = 0
     saved = []

pwnlib/util/fiddling.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import base64
 import random
+import os
 import re
 import string
 import StringIO
@@ -10,7 +11,7 @@ from . import packing
 from ..context import context
 from ..log import getLogger
 from ..term import text
-from .cyclic import cyclic
+from .cyclic import de_bruijn
 from .cyclic import cyclic_find
 
 log = getLogger(__name__)
@@ -559,13 +560,15 @@ default_style = {
 }
 
 cyclic_pregen = ''
+de_bruijn_gen = de_bruijn()
 
 def sequential_lines(a,b):
     return (a+b) in cyclic_pregen
 
 def update_cyclic_pregenerated(size):
     global cyclic_pregen
-    cyclic_pregen = cyclic(size)
+    while size > len(cyclic_pregen):
+        cyclic_pregen += de_bruijn_gen.next()
 
 def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
                  highlight=None, cyclic=False):
@@ -627,30 +630,49 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
             return hbyte, abyte
         cache = [style_byte(chr(b)) for b in range(256)]
 
-    if cyclic:
-        update_cyclic_pregenerated(len(s))
-
     numb = 0
     while True:
         offset = begin + numb
-        chunk = fd.read(width)
+
+        # If a tube is passed in as fd, it will raise EOFError when it runs
+        # out of data, unlike a file or StringIO object, which return an empty
+        # string.
+        try:
+            chunk = fd.read(width)
+        except EOFError:
+            chunk = ''
+
+        # We have run out of data, exit the loop
         if chunk == '':
             break
+
+        # Advance the cursor by the number of bytes we actually read
         numb += len(chunk)
+
+        # Update the cyclic pattern in case
+        if cyclic:
+            update_cyclic_pregenerated(numb)
+
         # If this chunk is the same as the last unique chunk,
         # use a '*' instead.
-        if skip and \
-           (last_unique == chunk or \
-            (cyclic and sequential_lines(last_unique, chunk))):
+        if skip and last_unique:
+            same_as_last_line = (last_unique == chunk)
+            lines_are_sequential = (cyclic and sequential_lines(last_unique, chunk))
             last_unique = chunk
-            if not skipping:
-                yield '*'
-                skipping = True
-            continue
 
-        # Chunk is unique, save for next iteration
-        last_unique = chunk
+            if same_as_last_line or lines_are_sequential:
+
+                # If we have not already printed a "*", do so
+                if not skipping:
+                    yield '*'
+                    skipping = True
+
+                # Move on to the next chunk
+                continue
+
+        # Chunk is unique, no longer skipping
         skipping = False
+        last_unique = chunk
 
         # Generate contents for line
         hexbytes = ''

pwnlib/util/misc.py

@@ -7,8 +7,8 @@ import socket
 import stat
 import string
 
-from . import lists
 from . import fiddling
+from . import lists
 from ..context import context
 from ..log import getLogger
 
@@ -189,7 +189,6 @@ def run_in_new_terminal(command, terminal = None, args = None):
 
     Returns:
       None
-
     """
 
     if not terminal:
@@ -214,8 +213,12 @@ def run_in_new_terminal(command, terminal = None, args = None):
     argv = [terminal_path] + args
 
     if isinstance(command, str):
+        if ';' in command:
+            log.error("Cannot use commands with semicolon.  Create a script and invoke that directly.")
         argv += [command]
     elif isinstance(command, (list, tuple)):
+        if any(';' in c for c in command):
+            log.error("Cannot use commands with semicolon.  Create a script and invoke that directly.")
         argv += list(command)
 
     log.debug("Launching a new terminal: %r" % argv)
@@ -271,163 +274,6 @@ def mkdir_p(path):
         else:
             raise
 
-def sh_string(s):
-    """Outputs a string in a format that will be understood by /bin/sh.
-
-    If the string does not contain any bad characters, it will simply be
-    returned, possibly with quotes. If it contains bad characters, it will
-    be escaped in a way which is compatible with most known systems.
-
-    Examples:
-
-        >>> print sh_string('foobar')
-        foobar
-        >>> print sh_string('foo bar')
-        'foo bar'
-        >>> print sh_string("foo'bar")
-        "foo'bar"
-        >>> print sh_string("foo\\\\bar")
-        'foo\\bar'
-        >>> print sh_string("foo\\\\'bar")
-        "foo\\\\'bar"
-        >>> print sh_string("foo\\x01'bar")
-        "$(printf 'foo\\001\\047bar')"
-        >>> print `subprocess.check_output("echo -n " + sh_string("foo\\x01'bar"), shell = True)`
-        "foo\\x01'bar"
-    """
-
-    very_good = set(string.ascii_letters + string.digits)
-    good      = (very_good | set(string.punctuation + ' ')) - set("'")
-    alt_good  = (very_good | set(string.punctuation + ' ')) - set('!')
-
-    if '\x00' in s:
-        log.error("sh_string(): Cannot create a null-byte")
-    if s.endswith('\n'):
-        log.error("sh_string(): Cannot create a newline-terminated string")
-
-    if all(c in very_good for c in s):
-        return s
-    elif all(c in good for c in s):
-        return "'%s'" % s
-    elif all(c in alt_good for c in s):
-        fixed = ''
-        for c in s:
-            if c in '"\\$`':
-                fixed += '\\' + c
-            else:
-                fixed += c
-        return '"%s"' % fixed
-    else:
-        fixed = ''
-        for c in s:
-            if c == '\\':
-                fixed += '\\\\'
-            elif c == '\n':
-                fixed += '\\n'
-            elif c in good:
-                fixed += c
-            else:
-                fixed += '\\%03o' % ord(c)
-        return '"$(printf \'%s\')"' % fixed
-
-def sh_prepare(variables, export = False):
-    """Outputs a posix compliant shell command that will put the data specified
-    by the dictionary into the environment.
-
-    It is assumed that the keys in the dictionary are valid variable names that
-    does not need any escaping.
-
-    Arguments:
-      variables(dict): The variables to set.
-      export(bool): Should the variables be exported or only stored in the shell environment?
-      output(str): A valid posix shell command that will set the given variables.
-
-    It is assumed that `var` is a valid name for a variable in the shell.
-
-    Examples:
-
-        >>> print sh_prepare({'X': 'foobar'})
-        X=foobar
-        >>> r = sh_prepare({'X': 'foobar', 'Y': 'cookies'})
-        >>> r == 'X=foobar;Y=cookies' or r == 'Y=cookies;X=foobar'
-        True
-        >>> print sh_prepare({'X': 'foo bar'})
-        X='foo bar'
-        >>> print sh_prepare({'X': "foo'bar"})
-        X="foo'bar"
-        >>> print sh_prepare({'X': "foo\\\\bar"})
-        X='foo\\bar'
-        >>> print sh_prepare({'X': "foo\\\\'bar"})
-        X="foo\\\\'bar"
-        >>> print sh_prepare({'X': "foo\\x01'bar"})
-        X="$(printf 'foo\\001\\047bar')"
-        >>> print sh_prepare({'X': "foo\\x01'bar"}, export = True)
-        export X="$(printf 'foo\\001\\047bar')"
-        >>> print sh_prepare({'X': "foo\\x01'bar\\n"})
-        X="$(printf 'foo\\001\\047bar\\nx')";X=${X%x}
-        >>> print sh_prepare({'X': "foo\\x01'bar\\n"}, export = True)
-        X="$(printf 'foo\\001\\047bar\\nx')";export X=${X%x}
-        >>> print `subprocess.check_output('%s;echo -n "$X"' % sh_prepare({'X': "foo\\x01'bar"}), shell = True)`
-        "foo\\x01'bar"
-    """
-
-    out = []
-    export = 'export ' if export else ''
-
-    for k, v in variables.items():
-        if v.endswith('\n'):
-            out.append('%s=%s;%s%s=${%s%%x}' % (k, sh_string(v + "x"), export, k, k))
-        else:
-            out.append('%s%s=%s' % (export, k, sh_string(v)))
-    return ';'.join(out)
-
-def sh_command_with(f, *args):
-    """sh_command_with(f, arg0, ..., argN) -> command
-
-    Returns a command create by evaluating `f(new_arg0, ..., new_argN)`
-    whenever `f` is a function and `f % (new_arg0, ..., new_argN)` otherwise.
-
-    If the arguments are purely alphanumeric, then they are simply passed to
-    function. If they are simple to escape, they will be escaped and passed to
-    the function.
-
-    If the arguments contain trailing newlines, then it is hard to use them
-    directly because of a limitation in the posix shell. In this case the
-    output from `f` is prepended with a bit of code to create the variables.
-
-    Examples:
-
-        >>> print sh_command_with(lambda: "echo hello")
-        echo hello
-        >>> print sh_command_with(lambda x: "echo " + x, "hello")
-        echo hello
-        >>> print sh_command_with(lambda x: "echo " + x, "\\x01")
-        echo "$(printf '\\001')"
-        >>> import random
-        >>> random.seed(1)
-        >>> print sh_command_with(lambda x: "echo " + x, "\\x01\\n")
-        dwtgmlqu="$(printf '\\001\\nx')";dwtgmlqu=${dwtgmlqu%x};echo "$dwtgmlqu"
-        >>> random.seed(1)
-        >>> print sh_command_with("echo %s", "\\x01\\n")
-        dwtgmlqu="$(printf '\\001\\nx')";dwtgmlqu=${dwtgmlqu%x};echo "$dwtgmlqu"
-    """
-
-    args = list(args)
-    out = []
-
-    for n in range(len(args)):
-        if args[n].endswith('\n'):
-            v = fiddling.randoms(8)
-            out.append(sh_prepare({v: args[n]}))
-            args[n] = '"$%s"' % v
-        else:
-            args[n] = sh_string(args[n])
-    if hasattr(f, '__call__'):
-        out.append(f(*args))
-    else:
-        out.append(f % tuple(args))
-    return ';'.join(out)
-
 def dealarm_shell(tube):
     """Given a tube which is a shell, dealarm it.
     """

pwnlib/util/net.py

@@ -2,9 +2,9 @@ import ctypes
 import ctypes.util
 import socket
 
-from packing import pack
 from packing import p16
 from packing import p32
+from packing import pack
 
 __all__ = ['getifaddrs', 'interfaces', 'interfaces4', 'interfaces6', 'sockaddr']
 

pwnlib/util/sh_string.py

@@ -0,0 +1,491 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""
+# sh_string
+
+Routines here are for getting any NULL-terminated sequence of bytes evaluated
+intact by any shell.  This includes all variants of quotes, whitespace, and
+non-printable characters.
+
+## Supported Shells
+
+The following shells have been evaluated:
+
+- Ubuntu (dash/sh)
+- MacOS (GNU Bash)
+- Zsh
+- FreeBSD (sh)
+- OpenBSD (sh)
+- NetBSD (sh)
+
+### Debian Almquist shell (Dash)
+
+Ubuntu 14.04 and 16.04 use the Dash shell, and /bin/sh is actually just a
+symlink to /bin/dash.  The feature set supported when invoked as "sh" instead
+of "dash" is different, and we focus exclusively on the "/bin/sh" implementation.
+
+From the [Ubuntu Man Pages][ubuntu], every character except for single-quote
+can be wrapped in single-quotes, and a backslash can be used to escape unquoted
+single-quotes.
+
+   Quoting
+     Quoting is used to remove the special meaning of certain characters or
+     words to the shell, such as operators, whitespace, or keywords.  There
+     are three types of quoting: matched single quotes, matched double quotes,
+     and backslash.
+
+   Backslash
+     A backslash preserves the literal meaning of the following character,
+     with the exception of ⟨newline⟩.  A backslash preceding a ⟨newline⟩ is
+     treated as a line continuation.
+
+   Single Quotes
+     Enclosing characters in single quotes preserves the literal meaning of
+     all the characters (except single quotes, making it impossible to put
+     single-quotes in a single-quoted string).
+
+   Double Quotes
+     Enclosing characters within double quotes preserves the literal meaning
+     of all characters except dollarsign ($), backquote (`), and backslash
+     (\).  The backslash inside double quotes is historically weird, and
+     serves to quote only the following characters:
+           $ ` " \ <newline>.
+     Otherwise it remains literal.
+
+### GNU Bash
+
+The Bash shell is default on many systems, though it is not generally the default
+system-wide shell (i.e., the `system` syscall does not generally invoke it).
+
+That said, its prevalence suggests that it also be addressed.
+
+From the [GNU Bash Manual][bash], every character except for single-quote
+can be wrapped in single-quotes, and a backslash can be used to escape unquoted
+single-quotes.
+
+    3.1.2.1 Escape Character
+
+    A non-quoted backslash ‘\’ is the Bash escape character. It preserves the
+    literal value of the next character that follows, with the exception of
+    newline. If a ``\\newline`` pair appears, and the backslash itself is not
+    quoted, the ``\\newline`` is treated as a line continuation (that is, it
+    is removed from the input stream and effectively ignored).
+
+    3.1.2.2 Single Quotes
+
+    Enclosing characters in single quotes (‘'’) preserves the literal value of
+    each character within the quotes. A single quote may not occur between single
+    uotes, even when preceded by a backslash.
+
+    3.1.2.3 Double Quotes
+
+    Enclosing characters in double quotes (‘"’) preserves the literal value of a
+    ll characters within the quotes, with the exception of ‘$’, ‘`’, ‘\’, and,
+    when history expansion is enabled, ‘!’. The characters ‘$’ and ‘`’ retain their
+    pecial meaning within double quotes (see Shell Expansions). The backslash retains
+    its special meaning only when followed by one of the following characters:
+    ‘$’, ‘`’, ‘"’, ‘\’, or newline. Within double quotes, backslashes that are
+    followed by one of these characters are removed. Backslashes preceding
+    characters without a special meaning are left unmodified. A double quote may
+    be quoted within double quotes by preceding it with a backslash. If enabled,
+    history expansion will be performed unless an ‘!’ appearing in double quotes
+    is escaped using a backslash. The backslash preceding the ‘!’ is not removed.
+
+    The special parameters ‘*’ and ‘@’ have special meaning when in double quotes
+    see Shell Parameter Expansion).
+
+### Z Shell
+
+The Z shell is also a relatively common user shell, even though it's not generally
+the default system-wide shell.
+
+From the [Z Shell Manual][zsh], every character except for single-quote
+can be wrapped in single-quotes, and a backslash can be used to escape unquoted
+single-quotes.
+
+    A character may be quoted (that is, made to stand for itself) by preceding
+    it with a ‘\’. ‘\’ followed by a newline is ignored.
+
+    A string enclosed between ‘$'’ and ‘'’ is processed the same way as the
+    string arguments of the print builtin, and the resulting string is considered
+    o be entirely quoted. A literal ‘'’ character can be included in the string
+    by using the ‘\\'’ escape.
+
+    All characters enclosed between a pair of single quotes ('') that is not
+    preceded by a ‘$’ are quoted. A single quote cannot appear within single
+    quotes unless the option RC_QUOTES is set, in which case a pair of single
+    quotes are turned into a single quote. For example,
+
+    print ''''
+    outputs nothing apart from a newline if RC_QUOTES is not set, but one single
+    quote if it is set.
+
+    Inside double quotes (""), parameter and command substitution occur, and
+    ‘\’ quotes the characters ‘\’, ‘`’, ‘"’, and ‘$’.
+
+### FreeBSD Shell
+
+Compatibility with the FreeBSD shell is included for completeness.
+
+From the [FreeBSD man pages][freebsd], every character except for single-quote
+can be wrapped in single-quotes, and a backslash can be used to escape unquoted
+single-quotes.
+
+     Quoting is used to remove the special meaning of certain characters or
+     words to the shell, such as operators, whitespace, keywords, or alias
+     names.
+
+     There are four types of quoting: matched single quotes, dollar-single
+     quotes, matched double quotes, and backslash.
+
+     Single Quotes
+         Enclosing characters in single quotes preserves the literal mean-
+         ing of all the characters (except single quotes, making it impos-
+         sible to put single-quotes in a single-quoted string).
+
+     Dollar-Single Quotes
+         Enclosing characters between $' and ' preserves the literal mean-
+         ing of all characters except backslashes and single quotes.  A
+         backslash introduces a C-style escape sequence:
+
+         ...
+
+     Double Quotes
+         Enclosing characters within double quotes preserves the literal
+         meaning of all characters except dollar sign (`$'), backquote
+         (``'), and backslash (`\\').  The backslash inside double quotes
+         is historically weird.  It remains literal unless it precedes the
+         following characters, which it serves to quote:
+
+           $     `     "     \     \\n
+
+     Backslash
+         A backslash preserves the literal meaning of the following char-
+         acter, with the exception of the newline character (`\n').  A
+         backslash preceding a newline is treated as a line continuation.
+
+### OpenBSD Shell
+
+From the [OpenBSD Man Pages][openbsd], every character except for single-quote
+can be wrapped in single-quotes, and a backslash can be used to escape unquoted
+single-quotes.
+
+
+    A backslash (\) can be used to quote any character except a newline.
+    If a newline follows a backslash the shell removes them both, effectively
+    making the following line part of the current one.
+
+    A group of characters can be enclosed within single quotes (') to quote
+    every character within the quotes.
+
+    A group of characters can be enclosed within double quotes (") to quote
+    every character within the quotes except a backquote (`) or a dollar
+    sign ($), both of which retain their special meaning. A backslash (\)
+    within double quotes retains its special meaning, but only when followed
+    by a backquote, dollar sign, double quote, or another backslash.
+    An at sign (@) within double quotes has a special meaning
+    (see SPECIAL PARAMETERS, below).
+
+### NetBSD Shell
+
+The NetBSD shell's documentation is identical to the Dash documentation.
+
+### Android Shells
+
+Android has gone through some number of shells.
+
+- Mksh, a Korn shell, was used with Toolbox releases (5.0 and prior)
+- Toybox, also derived from the Almquist Shell (6.0 and newer)
+
+Notably, the Toolbox implementation is not POSIX compliant
+as it lacks a "printf" builtin (e.g. Android 5.0 emulator images).
+
+### Toybox Shell
+
+Android 6.0 (and possibly other versions) use a shell based on ``toybox``.
+
+While it does not include a ``printf`` builtin, ``toybox`` itself includes
+a POSIX-compliant ``printf`` binary.
+
+The Ash shells should be feature-compatible with ``dash``.
+
+### BusyBox Shell
+
+[BusyBox's Wikipedia page][busybox] claims to use an ``ash``-compliant shell,
+and should therefore be compatible with ``dash``.
+
+
+[ubuntu]: http://manpages.ubuntu.com/manpages/trusty/man1/dash.1.html
+[bash]: https://www.gnu.org/software/bash/manual/bash.html#Quoting
+[zsh]: http://zsh.sourceforge.net/Doc/Release/Shell-Grammar.html#Quoting
+[freebsd]: https://www.freebsd.org/cgi/man.cgi?query=sh
+[openbsd]: http://man.openbsd.org/cgi-bin/man.cgi?query=sh#SHELL_GRAMMAR
+[busybox]: https://en.wikipedia.org/wiki/BusyBox#Features
+"""
+import string
+import subprocess
+
+from . import fiddling
+from ..context import context
+from ..log import getLogger
+from ..tubes.process import process
+from .misc import which
+
+log = getLogger(__name__)
+
+def test_all():
+    test('a')
+    test('ab')
+    test('a b')
+    test(r"a\'b")
+    everything_1 = ''.join(chr(c) for c in range(1,256))
+    for s in everything_1:
+        test(s)
+        test(s*4)
+        test(s * 2 + 'X')
+        test('X' + s * 2)
+        test((s*2 + 'X') * 2)
+        test(s + 'X' + s)
+        test(s*2 + 'X' + s*2)
+        test('X' + s*2 + 'X')
+    test(everything_1)
+    test(everything_1 * 2)
+    test(everything_1 * 4)
+    everything_2 = ''.join(chr(c) * 2 for c in range(1,256))
+    test(everything_2)
+
+    test(randoms(1000, everything_1))
+
+
+def test(original):
+    r"""Tests the output provided by a shell interpreting a string
+
+    >>> test('foobar')
+    >>> test('foo bar')
+    >>> test('foo bar\n')
+    >>> test("foo'bar")
+    >>> test("foo\\\\bar")
+    >>> test("foo\\\\'bar")
+    >>> test("foo\\x01'bar")
+    >>> test('\n')
+    >>> test('\xff')
+    >>> test(os.urandom(16 * 1024).replace('\x00', ''))
+    """
+    input = sh_string(original)
+
+    cmdstr = '/bin/echo %s' % input
+
+    SUPPORTED_SHELLS = [
+        ['ash', '-c', cmdstr],
+        ['bash', '-c', cmdstr],
+        ['bash', '-o', 'posix', '-c', cmdstr],
+        ['ksh', '-c', cmdstr],
+        ['busybox', 'ash', '-c', cmdstr],
+        ['busybox', 'sh', '-c', cmdstr],
+        ['zsh', '-c', cmdstr],
+        ['posh', '-c', cmdstr],
+        ['dash', '-c', cmdstr],
+        ['mksh', '-c', cmdstr],
+        ['sh', '-c', cmdstr],
+        # ['adb', 'exec-out', cmdstr]
+    ]
+
+    for shell in SUPPORTED_SHELLS:
+        binary = shell[0]
+
+        if not which(binary):
+            log.warn_once('Shell %r is not available' % binary)
+            continue
+
+        progress = log.progress('%s: %r' % (binary, original))
+
+        with context.quiet:
+            with process(shell) as p:
+                data = p.recvall(timeout=2)
+                p.kill()
+
+        # Remove exactly one trailing newline added by echo
+        # We cannot assume "echo -n" exists.
+        data = data[:-1]
+
+        if data != original:
+            for i,(a,b) in enumerate(zip(data, original)):
+                if a == b:
+                    continue
+                log.error(('Shell %r failed\n' +
+                          'Expect %r\n' +
+                          'Sent   %r\n' +
+                          'Output %r\n' +
+                          'Mismatch @ %i: %r vs %r') \
+                        % (binary, original, input, data, i, a, b))
+
+        progress.success()
+
+
+
+SINGLE_QUOTE = "'"
+ESCAPED_SINGLE_QUOTE = r"\'"
+
+ESCAPED = {
+    # The single quote itself must be escaped, outside of single quotes.
+    "'": "\\'",
+
+    # Slashes must themselves be escaped
+    #
+    # Additionally, some shells coalesce any number N>1 of '\' into
+    # a single backslash literal.
+    # '\\': '"\\\\\\\\"'
+}
+
+def sh_string(s):
+    r"""Outputs a string in a format that will be understood by /bin/sh.
+
+    If the string does not contain any bad characters, it will simply be
+    returned, possibly with quotes. If it contains bad characters, it will
+    be escaped in a way which is compatible with most known systems.
+
+    Warning:
+        This does not play along well with the shell's built-in "echo".
+        It works exactly as expected to set environment variables and
+        arguments, **unless** it's the shell-builtin echo.
+
+    Argument:
+        s(str): String to escape.
+
+    Examples:
+
+        >>> sh_string('foobar')
+        'foobar'
+        >>> sh_string('foo bar')
+        "'foo bar'"
+        >>> sh_string("foo'bar")
+        "'foo'\\''bar'"
+        >>> sh_string("foo\\\\bar")
+        "'foo\\\\bar'"
+        >>> sh_string("foo\\\\'bar")
+        "'foo\\\\'\\''bar'"
+        >>> sh_string("foo\\x01'bar")
+        "'foo\\x01'\\''bar'"
+    """
+    if '\x00' in s:
+        log.error("sh_string(): Cannot create a null-byte")
+
+    chars = set(s)
+    very_good = set(string.ascii_letters + string.digits + "_+.,/")
+
+    # Alphanumeric can always just be used verbatim.
+    if chars <= very_good:
+        return s
+
+    # If there are no single-quotes, the entire thing can be single-quoted
+    if not (chars & set(ESCAPED)):
+        return "'%s'" % s
+
+    # If there are single-quotes, we can single-quote around them, and simply
+    # escape the single-quotes.
+    quoted_string = ''
+    quoted = False
+    for char in s:
+        if char not in ESCAPED:
+            if not quoted:
+                quoted_string += SINGLE_QUOTE
+                quoted = True
+            quoted_string += char
+        else:
+            if quoted:
+                quoted = False
+                quoted_string += SINGLE_QUOTE
+            quoted_string += ESCAPED[char]
+
+    if quoted:
+        quoted_string += SINGLE_QUOTE
+
+    return quoted_string
+
+def sh_prepare(variables, export = False):
+    r"""Outputs a posix compliant shell command that will put the data specified
+    by the dictionary into the environment.
+
+    It is assumed that the keys in the dictionary are valid variable names that
+    does not need any escaping.
+
+    Arguments:
+      variables(dict): The variables to set.
+      export(bool): Should the variables be exported or only stored in the shell environment?
+      output(str): A valid posix shell command that will set the given variables.
+
+    It is assumed that `var` is a valid name for a variable in the shell.
+
+    Examples:
+
+        >>> sh_prepare({'X': 'foobar'})
+        'X=foobar'
+        >>> r = sh_prepare({'X': 'foobar', 'Y': 'cookies'})
+        >>> r == 'X=foobar;Y=cookies' or r == 'Y=cookies;X=foobar'
+        True
+        >>> sh_prepare({'X': 'foo bar'})
+        "X='foo bar'"
+        >>> sh_prepare({'X': "foo'bar"})
+        "X='foo'\\''bar'"
+        >>> sh_prepare({'X': "foo\\\\bar"})
+        "X='foo\\\\bar'"
+        >>> sh_prepare({'X': "foo\\\\'bar"})
+        "X='foo\\\\'\\''bar'"
+        >>> sh_prepare({'X': "foo\\x01'bar"})
+        "X='foo\\x01'\\''bar'"
+        >>> sh_prepare({'X': "foo\\x01'bar"}, export = True)
+        "export X='foo\\x01'\\''bar'"
+        >>> sh_prepare({'X': "foo\\x01'bar\\n"})
+        "X='foo\\x01'\\''bar\\n'"
+        >>> sh_prepare({'X': "foo\\x01'bar\\n"})
+        "X='foo\\x01'\\''bar\\n'"
+        >>> sh_prepare({'X': "foo\\x01'bar\\n"}, export = True)
+        "export X='foo\\x01'\\''bar\\n'"
+    """
+
+    out = []
+    export = 'export ' if export else ''
+
+    for k, v in variables.items():
+        out.append('%s%s=%s' % (export, k, sh_string(v)))
+
+    return ';'.join(out)
+
+def sh_command_with(f, *args):
+    r"""sh_command_with(f, arg0, ..., argN) -> command
+
+    Returns a command create by evaluating `f(new_arg0, ..., new_argN)`
+    whenever `f` is a function and `f % (new_arg0, ..., new_argN)` otherwise.
+
+    If the arguments are purely alphanumeric, then they are simply passed to
+    function. If they are simple to escape, they will be escaped and passed to
+    the function.
+
+    If the arguments contain trailing newlines, then it is hard to use them
+    directly because of a limitation in the posix shell. In this case the
+    output from `f` is prepended with a bit of code to create the variables.
+
+    Examples:
+
+        >>> sh_command_with(lambda: "echo hello")
+        'echo hello'
+        >>> sh_command_with(lambda x: "echo " + x, "hello")
+        'echo hello'
+        >>> sh_command_with(lambda x: "/bin/echo " + x, "\\x01")
+        "/bin/echo '\\x01'"
+        >>> sh_command_with(lambda x: "/bin/echo " + x, "\\x01\\n")
+        "/bin/echo '\\x01\\n'"
+        >>> sh_command_with("/bin/echo %s", "\\x01\\n")
+        "/bin/echo '\\x01\\n'"
+    """
+
+    args = list(args)
+    out = []
+
+    for n in range(len(args)):
+        args[n] = sh_string(args[n])
+    if hasattr(f, '__call__'):
+        out.append(f(*args))
+    else:
+        out.append(f % tuple(args))
+    return ';'.join(out)

pwnlib/util/web.py

@@ -21,10 +21,10 @@ def wget(url, save=None, timeout=5, **kwargs):
     Example:
 
       >>> url    = 'https://httpbin.org/robots.txt'
-      >>> result = wget(url)
+      >>> result = wget(url, timeout=60)
       >>> result
       'User-agent: *\\nDisallow: /deny\\n'
-      >>> result2 = wget(url, True)
+      >>> result2 = wget(url, True, timeout=60)
       >>> result == file('robots.txt').read()
       True
     """

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '3.0.2'
+__version__ = '3.3.4'

setup.py

@@ -20,8 +20,16 @@ for dirpath, dirnames, filenames in os.walk(convert_path('pwnlib/shellcraft/temp
 for scheme in INSTALL_SCHEMES.values():
     scheme['data'] = scheme['purelib']
 
-# Find all of the console scripts
-console_scripts = []
+console_scripts = ['pwn=pwnlib.commandline.main:main']
+
+# Find all of the ancillary console scripts
+# We have a magic flag --include-all-scripts
+flag = '--only-use-pwn-command'
+if flag in sys.argv:
+    sys.argv.remove(flag)
+else:
+    flag = False
+
 for filename in glob.glob('pwnlib/commandline/*'):
     filename = os.path.basename(filename)
     filename, ext = os.path.splitext(filename)
@@ -29,12 +37,13 @@ for filename in glob.glob('pwnlib/commandline/*'):
     if ext != '.py' or '__init__' in filename:
         continue
 
-    script = '%s=pwnlib.commandline.%s:main' % (filename, filename)
-    console_scripts.append(script)
+    script = '%s=pwnlib.commandline.common:main' % filename
+    if not flag:
+        console_scripts.append(script)
 
 install_requires     = ['paramiko>=1.15.2',
                         'mako>=1.0.0',
-                        'pyelftools>=0.2.3',
+                        'pyelftools>=0.2.4',
                         'capstone',
                         'ropgadget>=5.3',
                         'pyserial>=2.7',
@@ -43,7 +52,9 @@ install_requires     = ['paramiko>=1.15.2',
                         'tox>=1.8.1',
                         'pygments>=2.0',
                         'pysocks',
-                        'python-dateutil']
+                        'python-dateutil',
+                        'pypandoc',
+                        'packaging']
 
 # This is a hack until somebody ports psutil to OpenBSD
 if platform.system() != 'OpenBSD':
@@ -56,13 +67,25 @@ if not os.path.exists(PythonH):
     print >> sys.stderr, "$ apt-get install python-dev"
     sys.exit(-1)
 
+# Convert README.md to reStructuredText for PyPI
+long_description = ''
+try:
+    import pypandoc
+    try:
+        pypandoc.get_pandoc_path()
+    except OSError:
+        pypandoc.download_pandoc()
+    long_description = pypandoc.convert_file('README.md', 'rst')
+except ImportError:
+    pass
+
+
 setup(
     name                 = 'pwntools',
     packages             = find_packages(),
-    version              = '3.0.2',
+    version              = '3.3.4',
     data_files           = [('',
-                             ['LICENSE-pwntools.txt',
-                             ]),
+                             glob.glob('*.md') + glob.glob('*.txt')),
                             ],
     package_data         = {
         'pwnlib': [
@@ -75,19 +98,32 @@ setup(
     },
     entry_points = {'console_scripts': console_scripts},
     scripts              = glob.glob("bin/*"),
-    description          = "CTF framework and exploit development library.",
+    description          = "Pwntools CTF framework and exploit development library.",
+    long_description     = long_description,
     author               = "Gallopsled et al.",
     author_email         = "#pwntools @ freenode.net",
     url                  = 'https://pwntools.com',
     download_url         = "https://github.com/Gallopsled/pwntools/releases",
     install_requires     = install_requires,
     license              = "Mostly MIT, some GPL/BSD, see LICENSE-pwntools.txt",
+    keywords             = 'pwntools exploit ctf capture the flag binary wargame overflow stack heap defcon',
     classifiers          = [
-        'Topic :: Security',
+        'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
-        'Operating System :: OS Independent',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Science/Research',
+        'Intended Audience :: System Administrators',
         'License :: OSI Approved :: MIT License',
+        'Natural Language :: English',
+        'Operating System :: POSIX :: Linux',
         'Programming Language :: Python :: 2.7',
-        'Intended Audience :: Developers'
+        'Topic :: Security',
+        'Topic :: Software Development :: Assemblers',
+        'Topic :: Software Development :: Debuggers',
+        'Topic :: Software Development :: Disassemblers',
+        'Topic :: Software Development :: Embedded Systems',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Topic :: System :: System Shells',
+        'Topic :: Utilities',
     ]
 )

shippable.yml

@@ -1,17 +0,0 @@
-language: python
-python:
-  - "2.7"
-before_install:
-  - source .travis_install.sh
-  - source .travis_ssh_setup.sh
-install:
-  - pip install -e .
-  - pip install -r docs/requirements.txt
-before_script:
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
-  - PWNLIB_NOTERM=1 python -c 'from pwn import *; print pwnlib.term.term_mode'
-script:
-  - PWNLIB_NOTERM=1 make -C docs doctest
-

tox.ini

@@ -2,5 +2,4 @@
 envlist = py27
 [testenv]
 deps=-r{toxinidir}/requirements.txt
-	 -r{toxinidir}/docs/requirements.txt
-commands=make PWNLIB_NOTERM=1 -C docs clean doctest
+commands=make PWNLIB_NOTERM=1 -C docs doctest

travis/install.sh

@@ -0,0 +1,186 @@
+#!/usr/bin/env bash -e
+set -e
+local_deb_extract()
+{
+    wget $1
+    ar vx *.deb
+    tar xvf data.tar.*
+    rm -f *.tar.* *deb*
+}
+
+get_binutils()
+{
+    BINUTILS_PREFIX='https://launchpad.net/~pwntools/+archive/ubuntu/binutils/+files/binutils-'
+    BINUTILS_SUFFIX='-linux-gnu_2.22-6ubuntu1.1cross0.11pwntools12~precise_amd64.deb'
+    local_deb_extract "${BINUTILS_PREFIX}${1}${BINUTILS_SUFFIX}"
+}
+
+get_qemu()
+{
+    echo "Installing qemu"
+    QEMU_INDEX='http://packages.ubuntu.com/en/yakkety/amd64/qemu-user-static/download'
+    QEMU_URL=$(curl "$QEMU_INDEX" | grep kernel.org | grep -Eo 'http.*\.deb')
+    local_deb_extract "$QEMU_URL"
+}
+
+setup_travis()
+{
+    export PATH=$PWD/usr/bin:$PATH
+    export LD_LIBRARY_PATH=$PWD/usr/lib
+
+    # Install libbfd-multiarch and libopcodes-multiarch if not found in the cache
+    if [ ! -f usr/lib/libbfd-2.22-multiarch.so ];
+    then
+        # Install the multiarch binutils
+        local_deb_extract http://mirrors.mit.edu/ubuntu/pool/universe/b/binutils/binutils-multiarch_2.22-6ubuntu1_amd64.deb
+        pushd usr/lib
+        ln -sf libbfd-2.22-multiarch.so libbfd-2.22.so
+        ln -sf libopcodes-2.22-multiarch.so libopcodes-2.22.so
+        popd
+    fi
+
+    # Install/upgrade qemu
+    if ! (which qemu-arm-static && qemu-arm-static -version | grep 2.6.1); then
+        get_qemu
+    fi
+
+    # Install our custom binutils
+    which arm-linux-gnu-as     || get_binutils arm
+    which mips-linux-gnu-as    || get_binutils mips
+    which powerpc-linux-gnu-as || get_binutils powerpc
+
+    # Test that the installs worked
+    which arm-linux-gnu-as
+    which mips-linux-gnu-as
+    which powerpc-linux-gnu-as
+    which qemu-arm-static
+
+    # Get rid of files we don't want cached
+    rm -rf usr/share
+}
+
+setup_linux()
+{
+    sudo apt-get install -y software-properties-common openssh-server libncurses5-dev libncursesw5-dev openjdk-8-jre-headless
+    sudo apt-add-repository --yes ppa:pwntools/binutils
+    sudo apt-get update
+    sudo apt-get install binutils-arm-linux-gnu binutils-mips-linux-gnu binutils-powerpc-linux-gnu
+}
+
+setup_android_emulator()
+{
+    # If we are running on Travis CI, and there were no changes to Android
+    # or ADB code, then we do not need the emulator
+    if [ -n "$TRAVIS" ]; then
+        if [ -z "$TRAVIS_COMMIT_RANGE" ]; then
+            echo "TRAVIS_COMMIT_RANGE is empty, forcing Android Emulator installation"
+        elif ! git show "$TRAVIS_COMMIT_RANGE" ; then
+            echo "TRAVIS_COMMIT_RANGE is invalid, forcing Android Emulator installation"
+        elif (git log --stat "$TRAVIS_COMMIT_RANGE" | grep -iE "android|adb"); then
+            echo "Found Android-related commits, forcing Android Emulator installation"
+        else
+            # In order to avoid running the doctests that require the Android
+            # emulator, while still leaving the code intact, we remove the
+            # RST file that Sphinx searches.
+            rm -f 'docs/source/adb.rst'
+
+            # However, the file needs to be present or else things break.
+            touch 'docs/source/adb.rst'
+
+            echo "Skipping Android emulator install, Android tests disabled."
+            return
+        fi
+    fi
+
+
+    if ! which java; then
+        echo "OpenJDK-8-JRE is required for Android stuff"
+        exit 1
+    fi
+
+    if (uname | grep -i Darwin &>/dev/null); then
+        brew install android-sdk android-ndk
+    else
+        if [ ! -f android-sdk/android ]; then
+            # Install the SDK, which gives us the 'android' and 'emulator' commands
+            wget https://dl.google.com/android/android-sdk_r24.4.1-linux.tgz
+            tar xf android-sdk_r24.4.1-linux.tgz
+            rm  -f android-sdk_r24.4.1-linux.tgz
+
+            # Travis caching causes this to exist already
+            rm -rf android-sdk
+
+            mv android-sdk-linux android-sdk
+            file android-sdk/tools/android
+        fi
+
+        export PATH="$PWD/android-sdk/tools:$PATH"
+        export PATH="$PWD/android-sdk/platform-tools:$PATH"
+        which android
+
+        # Install the NDK, which is required for adb.compile()
+        NDK_VERSION=android-ndk-r12b
+        if [ ! -f android-ndk/ndk-build ]; then
+            wget   https://dl.google.com/android/repository/$NDK_VERSION-linux-x86_64.zip
+            unzip -q android-ndk-*.zip
+            rm -f    android-ndk-*.zip
+
+            # Travis caching causes this to exist already
+            rm -rf android-ndk
+
+            mv     $NDK_VERSION android-ndk
+        fi
+
+        export NDK=$PWD/android-ndk
+        export PATH=$NDK:$PATH
+    fi
+
+    # Grab prerequisites
+    echo y | android update sdk --no-ui --all --filter platform-tools,extra-android-support
+    echo y | android update sdk --no-ui --all --filter android-21
+
+    # Valid ABIs:
+    # - armeabi-v7a
+    # - arm64-v8a
+    # - x86
+    # - x86_64
+    ABI='armeabi-v7a'
+
+    # Grab the emulator image
+    echo y | android update sdk --no-ui --all --filter sys-img-$ABI-android-21
+
+    # Create our emulator Android Virtual Device (AVD)
+    echo no | android --silent create avd --name android-$ABI   --target android-21 --force --snapshot --abi $ABI
+
+    # In the future, it would be nice to be able to use snapshots.
+    # However, I haven't gotten them to work nicely.
+    emulator -avd android-$ABI -no-window -no-boot-anim -no-skin -no-audio -no-window -no-snapshot &
+    adb wait-for-device
+    adb shell id
+    adb shell getprop
+}
+
+setup_osx()
+{
+    brew update
+    brew install binutils
+    brew install capstone
+}
+
+if [[ "$USER" == "travis" ]]; then
+    setup_travis
+    setup_android_emulator
+elif [[ "$USER" == "shippable" ]]; then
+    sudo apt-get update
+    sudo apt-get install openssh-server gcc-multilib
+    sudo /usr/sbin/sshd -f /etc/ssh/sshd_config &
+    setup_travis
+elif [[ "$(uname)" == "Darwin" ]]; then
+    setup_osx
+elif [[ "$(uname)" == "Linux" ]]; then
+    setup_linux
+    setup_android_emulator
+fi
+
+
+dpkg -l

travis/ssh_setup.sh

@@ -4,6 +4,8 @@
 #
 # All of the "conditional sudo" is to do container-based builds on
 # Travis which are much, much faster.
+set -e
+
 U=travis
 H=/home/$U
 
@@ -36,7 +38,7 @@ pubkey=$(cat ~/.ssh/$U.pub)
 
 # Set the authorized_keys entry to only permit login from localhost,
 # and only with
-USUDO mkdir $H/.ssh
+USUDO mkdir $H/.ssh || true
 USUDO tee -a $H/.ssh/authorized_keys <<EOF
 from="127.0.0.1" $pubkey
 EOF