Merge pull request #625 from cebrusfs/patch-1

Fix wrong format string parameters in memleak.py

.gitattributes

@@ -0,0 +1 @@
+pwnlib/data/* linguist-vendored

.gitignore

@@ -2,9 +2,11 @@
 *.hi
 *.pyc
 *~
+*.swp
 *.pdf
 */auto/*
 */bin/*
+docs/build
 build
 MANIFEST
 dist

.isort.cfg

@@ -0,0 +1,4 @@
+[settings]
+indent='    '
+not_skip = __init__.py
+force_single_line = 1

.travis.yml

@@ -0,0 +1,23 @@
+sudo: false
+language: python
+cache:
+    - pip
+    - directories:
+        - usr
+        - /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages/
+        - /home/travis/virtualenv/python2.7.9/bin/
+python:
+  - "2.7"
+before_install:
+  - source .travis_ssh_setup.sh
+  - source .travis_install.sh
+install:
+  - pip install -e .
+  - pip install -r docs/requirements.txt
+before_script:
+  - python -c 'from pwn import *; print pwnlib.term.term_mode' NOTERM
+  - python -c 'from pwn import *; print pwnlib.term.term_mode' NOTERM
+  - python -c 'from pwn import *; print pwnlib.term.term_mode' NOTERM
+  - python -c 'from pwn import *; print pwnlib.term.term_mode' NOTERM
+script: PWNLIB_NOTERM=1 make -C docs doctest
+

.travis_install.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+local_deb_extract()
+{
+    wget $1
+    ar vx *.deb
+    tar xvf data.tar.gz
+    rm -f *.tar.gz *deb*
+}
+
+get_binutils()
+{
+    BINUTILS_PREFIX='https://launchpad.net/~pwntools/+archive/ubuntu/binutils/+files/binutils-'
+    BINUTILS_SUFFIX='-linux-gnu_2.22-6ubuntu1.1cross0.11pwntools12~precise_amd64.deb'
+    local_deb_extract "${BINUTILS_PREFIX}${1}${BINUTILS_SUFFIX}"
+}
+
+setup_travis()
+{
+    export PATH=$PWD/usr/bin:$PATH
+    export LD_LIBRARY_PATH=$PWD/usr/lib
+
+    if [ ! -d usr/bin ]; 
+    then
+        which arm-linux-as     || get_binutils arm
+        which mips-linux-as    || get_binutils mips
+        which powerpc-linux-as || get_binutils powerpc
+        local_deb_extract http://mirrors.mit.edu/ubuntu/ubuntu/pool/universe/b/binutils/binutils-multiarch_2.22-6ubuntu1_amd64.deb
+        rm -rf usr/share
+    fi
+
+    pushd usr/lib
+    ln -sf libbfd-2.22-multiarch.so libbfd-2.22.so
+    ln -sf libopcodes-2.22-multiarch.so libopcodes-2.22.so
+    popd
+
+    which arm-linux-gnu-as
+    which mips-linux-gnu-as
+    which powerpc-linux-gnu-as
+}
+
+setup_linux()
+{
+    sudo apt-get install software-properties-common pwgen
+    sudo apt-add-repository --yes ppa:pwntools/binutils
+    sudo apt-get update
+    sudo apt-get install binutils-arm-linux-gnu binutils-mips-linux-gnu binutils-powerpc-linux-gnu
+}
+
+setup_osx()
+{
+    brew update
+    brew install binutils
+    brew install capstone
+}
+
+if [[ "$TRAVIS" ]]; then
+    setup_travis
+elif [[ "$(uname)" == "Darwin" ]]; then
+    setup_osx
+elif [[ "$(uname)" == "Linux" ]]; then
+    setup_linux
+fi

.travis_ssh_setup.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# Install a demo user for SSH purposes
+#
+U=travis
+H=/home/$U
+
+USUDO()
+{
+if [ "$TRAVIS" ];
+then
+    $*
+else
+    sudo -u $U $*
+fi
+}
+
+
+if [ "$TRAVIS" ];
+then
+    rm -f ~/.ssh/*
+else
+    # Create the new user account
+    # Disable password login for the user, and ensure the account is not locked
+    sudo useradd -m $U
+    sudo passwd --delete --unlock $U
+fi
+
+# Generate a new key so that we can log into it
+ssh-keygen -t rsa -f ~/.ssh/$U -N ''
+
+# Load the public key into a memory for below
+pubkey=$(cat ~/.ssh/$U.pub)
+
+# Set the authorized_keys entry to only permit login from localhost,
+# and only with
+USUDO mkdir $H/.ssh
+USUDO tee -a $H/.ssh/authorized_keys <<EOF
+from="127.0.0.1" $pubkey
+EOF
+
+# In the pwntools examples, we ssh to 'example.pwnme'
+# Set up an SSH config entry to make this actually work
+cat >> ~/.ssh/config <<EOF
+
+Host example.pwnme
+    User travis
+    HostName 127.0.0.1
+    IdentityFile ~/.ssh/$U
+EOF
+
+ssh-keyscan -t rsa example.pwnme >>~/.ssh/known_hosts
+ssh -o "StrictHostKeyChecking no" -v [email protected] id

CHANGELOG.md

@@ -0,0 +1,26 @@
+# Changelog
+
+This changelog only includes added major features and changes. Bugfixes and
+minor changes are omitted.
+
+## 2.2.0
+
+This was a large release with a lot of bugfixes and changes. Only the most significant
+are mentioned here.
+
+- Added shellcodes
+- Added phd
+- Re-added our expansion of itertools
+- Added replacements for some semi-broken python standard library modules
+- Re-implemented the rop module
+- Added a serial tube
+- Huge performance gains in the buffering for tubes
+- Re-added user agents
+- Begun using Travis CI with lots of test
+- Removed bundled binutils in favor of documenting how to build them yourselves
+- Added support for port forwarding though our SSH module
+- Added dependency for capstone and ropgadget
+- Added a lots of shellcodes
+- Stuff we forgot
+- Lots of documentation fixes
+- Lots of bugfixes

CONTRIBUTING.md

@@ -0,0 +1,38 @@
+# Contributing
+
+We love pull requests!
+
+Github has a great guide for contributing to open source projects:
+
+- [Contributing to a project](https://guides.github.com/activities/forking/)
+- [Fork the repository](https://guides.github.com/activities/forking/#fork)
+- [Clone your fork](https://guides.github.com/activities/forking/#clone)
+- [Making and pushing changes](https://guides.github.com/activities/forking/#making-changes)
+- [Making a Pull Request](https://guides.github.com/activities/forking/#making-a-pull-request)
+- [Huzzah!](https://guides.github.com/activities/forking/#huzzah)
+
+## Pwntools Specifics
+
+In general, we like to keep things documented.  You should add documentation to any new functionality, and update it for any changed functionality.  Our docstrings use the [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.org/en/latest/example_google.html#example-google).
+
+After you have documentation, you should add a [doctest](https://docs.python.org/2/library/doctest.html).
+
+Finally, it is probably a good idea to run the test suite locally before doing
+the pull-request to make sure everything works, however this is not a
+requirement.
+
+Once you do the pull-request Travis CI will run the test-suite on it. Once it
+passes one of the core developers will look at your pull request, possibly
+comment on it and then hopefully merge it into master.
+
+## Test Suite
+
+To run the test suite, you should be running Ubuntu 12.04 or 14.04, and run the following commands.  **Be aware** that this will add a user to your machine, and create a public key for SSH login!
+
+```sh
+bash .travis_install.sh
+bash .travis_ssh_setup.sh
+cd docs
+pip install -r requirements.txt
+make doctest
+```

LICENSE-pwntools.txt

@@ -13,7 +13,7 @@ directories:
 - pwnlib/data/
 
 
-Copyright (c) 2013 Gallopsled
+Copyright (c) 2015 Gallopsled and Zach Riggle
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,38 +1,59 @@
-# Introduction
+![pwntools logo](docs/source/logo.png?raw=true)
+
+[![Docs latest](https://readthedocs.org/projects/pwntools/badge/)](https://pwntools.readthedocs.org/en/latest)
+[![Docs 2.2.0](https://readthedocs.org/projects/pwntools/badge/?version=2.2.0)](https://pwntools.readthedocs.org/en/2.2.0)
+[![PyPI](https://img.shields.io/pypi/v/pwntools.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
+[![Gittip](https://img.shields.io/gittip/gallopsled.svg?style=flat)](https://www.gittip.com/gallopsled/)
+[![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
+[![Twitter](https://img.shields.io/badge/twitter-Gallopsled-4099FF.svg?style=flat)](https://twitter.com/Gallopsled)
+[![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
 
 This is the CTF framework used by Gallopsled in every CTF.
 
-Most code is inside the pwnlib folder with some functionality inside pwn or
-bin. It is typically used as:
-
 ```python
 from pwn import *
 context(arch = 'i386', os = 'linux')
 
-# EXPLOIT HERE
+r = remote('exploitme.example.com', 31337)
+# EXPLOIT CODE GOES HERE
+r.send(asm(shellcraft.sh()))
+r.interactive()
 ```
 
 However we have made command-line frontends for some of the functionality
-inside the pwnlib. These are:
+in `pwnlib`. These are:
 
-* `asm`/`disasm`: Small wrapper for various assemblers
-* `constgrep`: Tool for finding constants defined in header files
-* `cyclic`: De Bruijn sequence generator and lookup tool
-* `hex`/`unhex`: Command line tools for doing common hexing/unhexing operations
-* `shellcraft`: Frontend to our shellcode
+* `asm`/`disasm`: Small wrapper for various assemblers.
+* `constgrep`: Tool for finding constants defined in header files.
+* `cyclic`: De Bruijn sequence generator and lookup tool.
+* `hex`/`unhex`: Command line tools for doing common hexing/unhexing operations.
+* `shellcraft`: Frontend to our shellcode.
+* `phd`: Replacement for `hexdump` with colors.
 
 # Documentation
-Our documentation is available on http://pwntools.readthedocs.org/
+Our documentation is available at [pwntools.readthedocs.org](https://pwntools.readthedocs.org/en/latest/)
+
+To get you started, we've provided some example solutions for past CTF challenges in our [write-ups repository](https://github.com/Gallopsled/pwntools-write-ups).
 
 # Installation
-Pwntools is available as a pip package. You can install it by running
-`pip install pwntools`.
 
-Alternatively if you prefer to have the latest version in git, you can
-simply clone this repository, run `pip install -r requirements.txt`
-and add entries in your `PATH` and `PYTHONPATH` variables. The script
-`install_local.sh` will help you do so, in case you are using bash.
+pwntools is best supported on Ubuntu 12.04 and 14.04, but most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).
+
+Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with
+
+```sh
+pip install pwntools
+```
+
+However, some of the features (ROP generation and assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://pwntools.readthedocs.org/en/latest/install.html).
+
+
+# Contribution
+
+See [CONTRIBUTING.md](CONTRIBUTING.md)
 
 # Contact
-If you have any questions not worthy of a bug report, feel free to join us
-at `#gallopsled` on Freenode and ask away.
+If you have any questions not worthy of a [bug report](https://github.com/Gallopsled/pwntools/issues), feel free to join us
+at [`#pwntools` on Freenode](irc://irc.freenode.net/pwntools) and ask away.
+Click [here](https://kiwiirc.com/client/irc.freenode.net/pwntools) to connect.
+

bin/asm

@@ -1,59 +0,0 @@
-#!/usr/bin/env python2
-
-import argparse, sys
-from pwnlib.asm           import asm
-from pwnlib.context       import __possible__ as contexts
-from pwnlib.util.fiddling import enhex
-
-parser = argparse.ArgumentParser(
-    description = 'Assemble shellcode into bytes'
-)
-
-parser.add_argument(
-    'lines',
-    metavar='line',
-    nargs='*',
-    help='Lines to assemble. If none are supplied, use stdin'
-)
-
-parser.add_argument(
-    "-f", "--format",
-    help="Output format (defaults to hex for ttys, otherwise raw)",
-    choices=['raw', 'hex', 'string']
-)
-
-parser.add_argument(
-    "-o","--output",
-    metavar='file',
-    help="Output file (defaults to stdout)",
-    type=argparse.FileType('w'),
-    default=sys.stdout
-)
-
-parser.add_argument(
-    '-c', '--context',
-    metavar = '<opt>',
-    choices = contexts['os'] + contexts['arch'],
-    default = ['i386','linux'],
-    action = 'append',
-    help = 'The os/architecture the shellcode will run in (default: linux/i386), choose from: %s' % \
-      ', '.join(contexts['os'] + contexts['arch'])
-)
-
-args   = parser.parse_args()
-tty    = args.output.isatty()
-
-for arch in args.context[::-1]:
-    if arch in contexts['arch']: break
-for os in args.context[::-1]:
-    if os in contexts['os']: break
-
-data   = '\n'.join(args.lines) or sys.stdin.read()
-output = asm(data.replace(';', '\n'), arch = arch, os = os)
-fmt    = args.format or ('hex' if tty else 'raw')
-formatters = {'r':str, 'h':enhex, 's':repr}
-
-args.output.write(formatters[fmt[0]](output))
-
-if tty and fmt is not 'raw':
-    args.output.write('\n')

bin/constgrep

@@ -1,136 +0,0 @@
-#!/usr/bin/env python2
-
-import argparse, sys, os, types, re
-from pwnlib import asm, util, constants
-from pwnlib.util import safeeval
-from pwnlib.context import __possible__ as contexts
-
-p = argparse.ArgumentParser(
-    description = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",
-    formatter_class = argparse.RawDescriptionHelpFormatter,
-)
-
-group = p.add_mutually_exclusive_group()
-group.add_argument(
-    '-e', '--exact',
-    metavar = '<constant name>',
-    # nargs = 1,
-    default = None,
-    help = 'Do an exact match for a constant instead of searching for a regex',
-)
-group.add_argument(
-    'regex',
-    nargs = '?',
-    default = '',
-    help = 'The regex matching constant you want to find',
-)
-
-p.add_argument(
-    'constant',
-    nargs = '?',
-    default = None,
-    help = 'The constant to find',
-)
-
-p.add_argument(
-    '-i', '--case-insensitive',
-    action = 'store_true',
-    help = 'Search case insensitive',
-)
-
-p.add_argument(
-    '-m', '--mask-mode',
-    action = 'store_true',
-    help = 'Instead of searching for a specific constant value, search for values not containing strictly less bits that the given value.',
-)
-
-p.add_argument(
-    '-c', '--context',
-    metavar = '<opt>',
-    choices = contexts['os'] + contexts['arch'],
-    default = ['i386','linux'],
-    action = 'append',
-    help = 'The os/architecture to find constants for (default: linux/i386), choose from: %s' % \
-    ', '.join(dir(constants) + dir(constants.linux))
-)
-
-args = p.parse_args()
-
-# Find the architecture and os from the list of contexts
-for arch in args.context[::-1]:
-    if arch in contexts['arch']: break
-for os in args.context[::-1]:
-    if os in contexts['os']: break
-
-if args.exact:
-    # This is the simple case
-    print asm.cpp(args.exact, os = os, arch = arch).strip()
-else:
-    # New we search in the right module.
-    # But first: We find the right module
-    if os == 'freebsd':
-        mod = constants.freebsd
-    else:
-        mod = getattr(getattr(constants, os), arch)
-
-    # Compile the given regex, for optimized lookup
-    if args.case_insensitive:
-        matcher = re.compile(args.regex, re.IGNORECASE)
-    else:
-        matcher = re.compile(args.regex)
-
-    # Evaluate the given constant
-    if args.constant:
-        constant = safeeval.expr_eval(args.constant)
-    else:
-        constant = None
-
-    # The found matching constants and the length of the longest string
-    out    = []
-    maxlen = 0
-
-    for k in dir(mod):
-        # No python stuff
-        if k.endswith('__') and k.startswith('__'):
-            continue
-
-        # Run the regex
-        if not matcher.search(k):
-            continue
-
-        # Check the constant
-        if constant != None:
-            val = getattr(mod, k)
-            if args.mask_mode:
-                if constant & val != val:
-                    continue
-            else:
-                if constant != val:
-                    continue
-
-        # Append it
-        out.append((getattr(mod, k), k))
-        maxlen = max(len(k), maxlen)
-
-    # Output all matching constants
-    for _, k in sorted(out):
-        print '#define %s %s' % (k.ljust(maxlen), asm.cpp(k, os = os, arch = arch).strip())
-
-    # If we are in match_mode, then try to find a combination of
-    # constants that yield the exact given value
-    # We do not want to find combinations using the value 0.
-    if not (constant == None or constant == 0) and args.mask_mode:
-        mask = constant
-        good = []
-        out = [(v, k) for v, k in out if v != 0]
-
-        while mask and out:
-            cur = out.pop()
-            mask &= ~cur[0]
-            good.append(cur)
-
-            out = [(v, k) for v, k in out if mask & v == v]
-
-        if reduce(lambda x, cur: x | cur[0], good, 0) == constant:
-            print
-            print '(%s) == %s' % (' | '.join(k for v, k in good), args.constant)

bin/cyclic

@@ -1,69 +0,0 @@
-#!/usr/bin/env python2
-import argparse, string
-from pwnlib.util import cyclic, packing
-from pwnlib import log
-
-parser = argparse.ArgumentParser(
-    description = "Cyclic pattern creator/finder"
-)
-
-parser.add_argument(
-    '-a', '--alphabet',
-    metavar = 'alphabet',
-    default = string.ascii_lowercase,
-    help = 'The alphabet to use in the cyclic pattern (defaults to all lower case letters)',
-)
-
-parser.add_argument(
-    '-n', '--length',
-    metavar = 'length',
-    default = 4,
-    type = int,
-    help = 'Size of the unique subsequences (defaults to 4).'
-)
-
-group = parser.add_mutually_exclusive_group(required = True)
-group.add_argument(
-    '-l', '-o', '--offset', '--lookup',
-    metavar = '<lookup value>',
-    help = 'Do a lookup instead printing the alphabet',
-)
-
-group.add_argument(
-    'count',
-    type = int,
-    nargs = '?',
-    help = 'Number of characters to print'
-)
-
-args = parser.parse_args()
-alphabet = args.alphabet
-subsize  = args.length
-
-if args.lookup:
-    pat = args.lookup
-
-    if pat.startswith('0x'):
-        pat = packing.pack(int(pat[2:], 16), subsize*8, 'little', 'unsigned')
-    elif pat.isdigit():
-        pat = packing.pack(int(pat, 10), subsize*8, 'little', 'unsigned')
-
-    if len(pat) != 4:
-        log.fatal('Subpattern must be 4 bytes', 1)
-
-    if not all(c in alphabet for c in pat):
-        log.fatal('Pattern contains characters not present in the alphabet', 1)
-
-    offset = cyclic.cyclic_find(pat, alphabet, subsize)
-
-    if offset == -1:
-        log.fatal('Given pattern does not exist in cyclic pattern', 1)
-    else:
-        print offset
-else:
-    want   = args.count
-    result = cyclic.cyclic(want, alphabet, subsize)
-    got    = len(result)
-    if got < want:
-        log.failure("Alphabet too small (max length = %i)" % got)
-    print result

bin/disasm

@@ -1,38 +0,0 @@
-#!/usr/bin/env python2
-
-import argparse, sys
-from pwnlib import asm, context
-from string import whitespace, hexdigits
-
-parser = argparse.ArgumentParser(
-    description = 'Disassemble bytes into text format'
-)
-
-parser.add_argument(
-    'hex',
-    metavar = 'hex',
-    nargs = '*',
-    help = 'Hex-string to disasemble. If none are supplied, then it uses stdin in non-hex mode.'
-)
-
-parser.add_argument(
-    '-c', '--context',
-    metavar = '<opt>',
-    choices = context.__possible__['arch'],
-    default = 'i386',
-    help = 'The architecture of the shellcode (default: i386), choose from:\n%s' % ', '.join(context.__possible__['arch'])
-)
-
-args = parser.parse_args()
-
-if len(args.hex) > 0:
-    dat = ''.join(args.hex)
-    dat = dat.translate(None, whitespace)
-    if not set(hexdigits) >= set(dat):
-        print "This is not a hex string"
-        exit(-1)
-    dat = dat.decode('hex')
-else:
-    dat = sys.stdin.read()
-
-print asm.disasm(dat, arch = args.context)

bin/hex

@@ -1,7 +0,0 @@
-#!/usr/bin/env python2
-import sys
-
-if len(sys.argv) == 1:
-    print sys.stdin.read().encode('hex')
-else:
-    print ' '.join(sys.argv[1:]).encode('hex')

bin/shellcraft

@@ -1,173 +0,0 @@
-#!/usr/bin/env python2
-import argparse, sys, os, types
-import pwnlib
-from pwnlib import asm, log, util
-from pwnlib.context import __possible__ as contexts
-import pwnlib.term.text as text
-
-r = text.red
-g = text.green
-b = text.blue
-banner = '\n'.join(['  ' + r('____') + '  ' + g('_') + '          ' + r('_') + ' ' + g('_') + '                 ' + b('__') + ' ' + r('_'),
-                    ' ' + r('/ ___|') + g('| |__') + '   ' + b('___') + r('| |') + ' ' + g('|') + ' ' + b('___') + ' ' + r('_ __') + ' ' + g('__ _') + ' ' + b('/ _|') + ' ' + r('|_'),
-                    ' ' + r('\___ \\') + g('| \'_ \\') + ' ' + b('/ _ \\') + ' ' + r('|') + ' ' + g('|') + b('/ __|') + ' ' + r('\'__/') + ' ' + g('_` |') + ' ' + b('|_') + r('| __|'),
-                    '  ' + r('___) |') + ' ' + g('| | |') + '  ' + b('__/') + ' ' + r('|') + ' ' + g('|') + ' ' + b('(__') + r('| |') + ' ' + g('| (_| |') + '  ' + b('_|') + ' ' + r('|_'),
-                    ' ' + r('|____/') + g('|_| |_|') + b('\\___|') + r('_|') + g('_|') + b('\\___|') + r('_|') + '  ' + g('\\__,_|') + b('_|') + '  ' + r('\\__|'),
-                    ''
-                    ])
-#  ____  _          _ _                 __ _
-# / ___|| |__   ___| | | ___ _ __ __ _ / _| |_
-# \___ \| '_ \ / _ \ | |/ __| '__/ _` | |_| __|
-#  ___) | | | |  __/ | | (__| | | (_| |  _| |_
-# |____/|_| |_|\___|_|_|\___|_|  \__,_|_|  \__|
-
-def _string(s):
-    out = []
-    for c in s:
-        co = ord(c)
-        if co >= 0x20 and co <= 0x7e and c not in '/$\'"`':
-            out.append(c)
-        else:
-            out.append('\\x%02x' % co)
-    return '"' + ''.join(out) + '"\n'
-
-def _carray(s):
-    out = []
-    for c in s:
-        out.append('0x' + util.fiddling.enhex(c))
-    return '{' + ', '.join(out) + '};\n'
-
-def _hex(s):
-    return pwnlib.util.fiddling.enhex(s) + '\n'
-
-p = argparse.ArgumentParser(
-    description = banner + '\nMicrowave shellcode -- Easy, fast and delicious',
-    formatter_class = argparse.RawDescriptionHelpFormatter,
-)
-
-p.add_argument(
-    'shellcode',
-    nargs = '?',
-    default = '',
-    metavar = '<shellcode>',
-    help = 'The shellcode you want',
-)
-
-p.add_argument(
-    'args',
-    nargs = '*',
-    metavar = '<arg>',
-    default = (),
-    help = 'Argument to the chosen shellcode',
-)
-
-p.add_argument(
-    '-?', '--show',
-    action = 'store_true',
-    help = 'Show shellcode documentation',
-)
-
-p.add_argument(
-    '-o', '--out',
-    metavar = '<file>',
-    type = argparse.FileType('w'),
-    default = sys.stdout,
-    help = 'Output file (default: stdout)',
-)
-
-p.add_argument(
-    '-f', '--format',
-    metavar = '<format>',
-    choices = ['r', 'raw',
-               's', 'str', 'string',
-               'c',
-               'h', 'hex',
-               'a', 'asm', 'assembly',
-               'p',
-               'i', 'hexii'],
-    default = 'hex',
-    help = 'Output format (default: hex), choose from {r}aw, {s}tring, {c}-style array, {h}ex string, hex{i}i, {a}ssembly code, {p}reprocssed code',
-)
-
-args = p.parse_args()
-
-def get_tree(path, val, result):
-    if path:
-        path += '.'
-
-    if path.startswith('common.'):
-        return
-
-    mods = []
-
-    for k in sorted(dir(val)):
-        if k and k[0] != '_':
-            cur = getattr(val, k)
-            if isinstance(cur, types.ModuleType):
-                mods.append((path + k, cur))
-            else:
-                result.append((path + k, cur))
-
-    for path, val in mods:
-        get_tree(path, val, result)
-    return result
-
-vals = get_tree('', pwnlib.shellcraft, [])
-if args.shellcode:
-    vals = [(k, val) for k, val in vals if k.startswith(args.shellcode + '.') or k == args.shellcode]
-
-if len(vals) == 0:
-    log.fatal("Cannot find subtree by the name of %r" % args.shellcode)
-elif len(vals) > 1:
-    for k, _ in vals:
-        print k
-    exit()
-else:
-    func = vals[0][1]
-
-if args.show:
-    print func.__doc__
-    exit()
-
-defargs = len(func.func_defaults or ())
-reqargs = func.func_code.co_argcount - defargs
-if len(args.args) < reqargs:
-    if defargs > 0:
-        log.fatal('%s takes at least %d arguments' % (args.shellcode, reqargs))
-    else:
-        log.fatal('%s takes exactly %d arguments' % (args.shellcode, reqargs))
-
-# Captain uglyness saves the day!
-for i in range(len(args.args)):
-    if args.args[i] in ['False', 'false']:
-        args.args[i] = False
-
-# And he strikes again!
-os = arch = None
-for k in args.shellcode.split('.')[:-1]:
-    if k in contexts['arch']:
-        arch = k
-    elif k in contexts['os']:
-        os = k
-
-code = func(*args.args)
-
-if args.format in ['a', 'asm', 'assembly']:
-    print code
-    exit()
-if args.format == 'p':
-    print pwnlib.asm.cpp(code, arch = arch, os = os)
-    exit()
-
-code = pwnlib.asm.asm(code, arch = arch, os = os)
-
-if args.format in ['s', 'str', 'string']:
-    code = _string(code)
-elif args.format == 'c':
-    code = _carray(code)
-elif args.format in ['h', 'hex']:
-    code = _hex(code)
-elif args.format in ['i', 'hexii']:
-    code = pwnlib.util.fiddling.hexii(code) + '\n'
-
-sys.stdout.write(code)

bin/unhex

@@ -1,11 +0,0 @@
-#!/usr/bin/env python2
-import sys
-from string import whitespace
-try:
-    if len(sys.argv) == 1:
-        s = sys.stdin.read().translate(None, whitespace)
-        sys.stdout.write(s.decode('hex'))
-    else:
-        sys.stdout.write(''.join(sys.argv[1:]).decode('hex'))
-except TypeError, e:
-    sys.stderr.write(str(e) + '\n')

docs/.gitignore

@@ -1 +1,2 @@
 build
+robots.txt

docs/Makefile

@@ -6,6 +6,8 @@ SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = build
+DASHBUILD     = ./dashbuild.py
+TAR           = tar
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
@@ -14,7 +16,8 @@ ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) sou
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext dash
 
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
@@ -34,6 +37,7 @@ help:
 	@echo "  texinfo    to make Texinfo files"
 	@echo "  info       to make Texinfo files and run them through makeinfo"
 	@echo "  gettext    to make PO message catalogs"
+	@echo "  dash       to make a Dash docset"
 	@echo "  changes    to make an overview of all changed/added/deprecated items"
 	@echo "  linkcheck  to check all external links for integrity"
 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
@@ -136,6 +140,11 @@ gettext:
 	@echo
 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
 
+dash:
+	$(DASHBUILD) source $(BUILDDIR)/dash
+	$(TAR) --exclude='.DS_Store' -cvzf $(BUILDDIR)/dash/pwntools.tgz -C $(BUILDDIR)/dash pwntools.docset
+	@echo "Build finished. The Dash docset is in $(BUILDDIR)/dash."
+
 changes:
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 	@echo

docs/dashbuild.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+
+# -*- coding: utf-8 -*-
+
+#
+# Helper to build Dash docset from sphinx source files.
+#
+# Dash docsets can be read by various applications:
+#
+# Dash         OS X, iOS         https://kapeli.com
+# Zeal         Linux, Windows    https://zealdocs.org
+# Velocity     Windows           http://velocity.silverlakesoftware.com
+# LovelyDocs   Android           http://lovelydocs.io
+# dasht        POSIX             https://github.com/sunaku/dasht
+# Helm Dash    emacs             https://github.com/areina/helm-dash
+#
+
+import argparse
+import doc2dash.__main__
+import os, os.path
+import sphinx
+import sqlite3
+import sys
+
+sys.path.append(os.path.abspath(os.path.join('..', 'pwnlib')))
+import version
+
+def main(args):
+    """Generate a Dash docset from Sphinx source files."""
+
+    srcdir = args.srcdir
+    dstdir = args.dstdir
+    name = args.name
+
+    if not os.path.exists(dstdir):
+        os.makedirs(dstdir)
+
+    # Generate HTML without indices.
+    sphinx.build_main([ "sphinx-build", "-b", "html", "-d", os.path.join(dstdir, "doctrees"), \
+        "-t", "dash", srcdir, os.path.join(dstdir, "html") ])
+
+    # Convert to docset.
+    try:
+        doc2dash.__main__.main.main( \
+            [ os.path.join(dstdir, "html"), "-d", dstdir, "-n", name, \
+                    "-f", "-I", "index.html"], "doc2dash", False)
+    except SystemExit,e:
+        pass
+
+    # Insert a link to the online version.
+    online = args.online
+    if online is not None and online != "":
+        url = online.replace("@VERSION", args.version)
+        with open(os.path.join(dstdir, name+".docset", "Contents", "Info.plist"), "r+") as f_info:
+            pl = f_info.read()
+            pl = pl.replace("</dict>", \
+                "\t<key>DashDocSetFallbackURL</key>\n\t<string>%s</string>\n</dict>" % url)
+            f_info.seek(0)
+            f_info.write(pl)
+            f_info.truncate()
+
+    # Modify the CSS to hide the menu included in the HTML.
+    with open(os.path.join(dstdir, name+".docset", "Contents", "Resources", "Documents", "_static", "css", "theme.css"), "r+") as f_css:
+        css = f_css.read()
+        css = css.replace( \
+            '@media screen and (max-width: 768px){.wy-body-for-nav{background:#fcfcfc}.wy-nav-top{display:block}',\
+            '@media screen {.wy-body-for-nav{background:#fcfcfc}' )
+        css = css.replace( \
+            '@media screen and (max-width: 480px)', \
+            '@media screen ')
+        f_css.seek(0)
+        f_css.write(css)
+        f_css.truncate()
+
+    # Modify the index
+    db_conn = sqlite3.connect(os.path.join(dstdir, name+".docset", "Contents", "Resources", "docSet.dsidx"))
+    try:
+        db_conn.execute('INSERT INTO "searchIndex" ("name","type","path") VALUES ' \
+            '("1 Contents", "Guide", "index.html"), ' \
+            '("2 About pwntools", "Guide", "about.html"), ' \
+            '("3 Installation", "Guide", "install.html"), ' \
+            '("4 Getting Started", "Guide", "intro.html"), ' \
+            '("5 Globals (pwn)", "Guide", "globals.html"), ' \
+            '("6 Command Line Tools", "Guide", "commandline.html")')
+        db_conn.execute('DELETE FROM "searchIndex" WHERE "type" = "Module" AND ("name" = "pwn" OR "name" = "pwnlib")')
+        db_conn.commit()
+    finally:
+        db_conn.close()
+
+    return 0
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--name", help="docset name", default="pwntools")
+parser.add_argument("--online", help="URL for online docs", default="https://pwntools.readthedocs.org/en/@VERSION/")
+parser.add_argument("--version", help="pwntools version", default=version.__version__)
+parser.add_argument("srcdir", help="Source directory containing .rst files")
+parser.add_argument("dstdir", help="Destination and working directory")
+
+main(parser.parse_args())

docs/requirements.txt

@@ -3,3 +3,6 @@ argparse
 mako
 pyelftools
 sphinxcontrib-napoleon
+sphinxcontrib-autoprogram
+sphinx_rtd_theme
+doc2dash

docs/source/about.rst

@@ -0,0 +1,56 @@
+About pwntools
+========================
+
+Whether you're using it to write exploits, or as part
+of another software project will dictate how you use it.
+
+Historically pwntools was used as a sort of exploit-writing DSL. Simply doing
+``from pwn import *`` in a previous version of pwntools would bring all sorts of
+nice side-effects.
+
+When redesigning pwntools for 2.0, we noticed two contrary goals:
+
+* We would like to have a "normal" python module structure, to allow other
+  people to familiarize themselves with pwntools quickly.
+* We would like to have even more side-effects, especially by putting the
+  terminal in raw-mode.
+
+To make this possible, we decided to have two different modules. :mod:`pwnlib`
+would be our nice, clean Python module, while :mod:`pwn` would be used during
+CTFs.
+
+:mod:`pwn` --- Toolbox optimized for CTFs
+-----------------------------------------
+
+.. module:: pwn
+
+As stated, we would also like to have the ability to get a lot of these
+side-effects by default. That is the purpose of this module. It does
+the following:
+
+* Imports everything from the toplevel :mod:`pwnlib` along with
+  functions from a lot of submodules. This means that if you do
+  ``import pwn`` or ``from pwn import *``, you will have access to
+  everything you need to write an exploit.
+* Calls :func:`pwnlib.term.init` to put your terminal in raw mode
+  and implements functionality to make it appear like it isn't.
+* Setting the :data:`pwnlib.context.log_level` to `"info"`.
+* Tries to parse some of the values in :data:`sys.argv` and every
+  value it succeeds in parsing it removes.
+
+:mod:`pwnlib` --- Normal python library
+---------------------------------------
+
+.. module:: pwnlib
+
+This module is our "clean" python-code. As a rule, we do not think that
+importing :mod:`pwnlib` or any of the submodules should have any significant
+side-effects (besides e.g. caching).
+
+For the most part, you will also only get the bits you import. You for instance
+not get access to :mod:`pwnlib.util.packing` simply by doing ``import
+pwnlib.util``.
+
+Though there are a few exceptions (such as :mod:`pwnlib.shellcraft`), that does
+not quite fit the goals of being simple and clean, but they can still be
+imported without implicit side-effects.

docs/source/asm.rst

@@ -1,6 +1,12 @@
 .. testsetup:: *
 
+   import tempfile
+   import os
+   from pwnlib.context import context
    from pwnlib.asm import *
+   from pwnlib import shellcraft
+   from pwnlib.tubes.process import process
+
 
 :mod:`pwnlib.asm` --- Assembler functions
 =========================================

docs/source/atexception.rst

@@ -0,0 +1,5 @@
+:mod:`pwnlib.atexception` --- Callbacks on unhandled exception
+==============================================================
+
+.. automodule:: pwnlib.atexception
+   :members:

docs/source/atexit.rst

@@ -0,0 +1,5 @@
+:mod:`pwnlib.atexit` --- Replacement for atexit
+===============================================
+
+.. automodule:: pwnlib.atexit
+   :members:

docs/source/commandline.rst

@@ -0,0 +1,46 @@
+.. testsetup:: *
+
+   from pwn import *
+   old = context.defaults.copy()
+
+.. testcleanup:: *
+
+    context.defaults.copy = old
+
+Command Line Tools
+========================
+
+pwntools comes with a handful of useful command-line utilities which serve as wrappers for some of the internal functionality.
+
+.. toctree::
+
+.. autoprogram:: pwnlib.commandline.asm:parser
+   :prog: asm
+
+.. autoprogram:: pwnlib.commandline.constgrep:p
+   :prog: constgrep
+
+.. autoprogram:: pwnlib.commandline.cyclic:parser
+   :prog: cyclic
+
+.. autoprogram:: pwnlib.commandline.disasm:parser
+   :prog: disasm
+
+.. autoprogram:: pwnlib.commandline.elfdiff:p
+   :prog: elfdiff
+
+.. autoprogram:: pwnlib.commandline.elfpatch:p
+   :prog: elfpatch
+
+.. autoprogram:: pwnlib.commandline.hex:parser
+   :prog: hex
+
+.. autoprogram:: pwnlib.commandline.phd:parser
+   :prog: phd
+
+.. autoprogram:: pwnlib.commandline.shellcraft:p
+   :prog: shellcraft
+
+.. autoprogram:: pwnlib.commandline.unhex:parser
+   :prog: unhex
+

docs/source/conf.py

@@ -11,7 +11,11 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys, os
+import os
+import subprocess
+import sys
+
+build_dash = tags.has('dash')
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -31,17 +35,23 @@ extensions = [
     'sphinx.ext.doctest',
     'sphinx.ext.linkcode',
     'sphinx.ext.autosummary',
+    'sphinx.ext.coverage',
     'sphinx.ext.todo',
     'sphinx.ext.intersphinx',
+    'sphinxcontrib.autoprogram',
     'sphinxcontrib.napoleon'
 ]
 
 doctest_global_setup = '''
-import pwnlib
-pwnlib.context.reset_local()
+import pwnlib, sys
+pwnlib.context.context.reset_local()
+pwnlib.context.ContextType.defaults['log_level'] = 'ERROR'
+pwnlib.term.text.when = 'never'
+pwnlib.log.install_default_handler()
+pwnlib.log.console.stream = sys.stdout
 '''
 
-autodoc_member_order = 'bysource'
+autodoc_member_order = 'alphabetical'
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -52,7 +62,7 @@ doctest_test_doctest_blocks = True
 source_suffix = '.rst'
 
 # The encoding of source files.
-#source_encoding = 'utf-8-sig'
+source_encoding = 'utf-8-sig'
 
 # The master toctree document.
 master_doc = 'index'
@@ -156,10 +166,10 @@ html_static_path = []
 #html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+html_domain_indices = not build_dash
 
 # If false, no index is generated.
-#html_use_index = True
+html_use_index = not build_dash
 
 # If true, the index is split into individual pages for each letter.
 #html_split_index = False
@@ -205,7 +215,8 @@ latex_documents = [
    u'Gallopsled et al', 'manual'),
 ]
 
-intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)}
+intersphinx_mapping = {'python': ('https://docs.python.org/2.7', None),
+                       'paramiko': ('https://paramiko-docs.readthedocs.org/en/1.15/', None)}
 
 # The name of an image file (relative to this directory) to place at the top of
 # the title page.
@@ -261,6 +272,19 @@ texinfo_documents = [
 # How to display URL addresses: 'footnote', 'no', or 'inline'.
 #texinfo_show_urls = 'footnote'
 
+branch = release
+
+try:
+    git_branch = subprocess.check_output('git describe --tags', shell = True)
+except subprocess.CalledProcessError:
+    git_branch = '-'
+
+try:
+    if '-' in git_branch:
+        branch = subprocess.check_output('git rev-parse HEAD', shell = True).strip()[:10]
+except subprocess.CalledProcessError:
+    pass
+
 def linkcode_resolve(domain, info):
     if domain != 'py':
         return None
@@ -285,21 +309,26 @@ def linkcode_resolve(domain, info):
     else:
         filename = info['module'].replace('.', '/') + '.py'
 
-        # Fixup for context properties
-        if info['module'].startswith('pwnlib.context'):
-            filename = 'pwnlib/context.py'
-
-            if not isinstance(val, (types.FunctionType, types.MethodType)):
-                import pwnlib.context.defaults
-                val = getattr(pwnlib.context.defaults.__class__, info['fullname'], None)
-                if isinstance(val, property):
-                    val = val.fget
-                    val = getattr(val, '_inner', val)
-
         if isinstance(val, (types.ModuleType, types.ClassType, types.MethodType, types.FunctionType, types.TracebackType, types.FrameType, types.CodeType)):
             try:
                 lines, first = inspect.getsourcelines(val)
                 filename += '#L%d-%d' % (first, first + len(lines) - 1)
             except IOError:
                 pass
-    return "https://github.com/Gallopsled/pwntools/blob/%s/%s" % (release, filename)
+
+    return "https://github.com/Gallopsled/pwntools/blob/%s/%s" % (branch, filename)
+
+
+# The readthedocs theme is used by the Dash generator. (Can be used for HTML too.)
+
+if build_dash:
+
+    # on_rtd is whether we are on readthedocs.org
+    on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+    if not on_rtd:  # only import and set the theme if we're building docs locally
+        import sphinx_rtd_theme
+        html_theme = 'sphinx_rtd_theme'
+        html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+    # otherwise, readthedocs.org uses their theme by default, so no need to specify it

docs/source/constants.rst

@@ -1,6 +1,7 @@
 .. testsetup:: *
 
-   from pwnlib import context, constants
+   from pwnlib import constants
+   from pwnlib.context import context
 
 :mod:`pwnlib.constants` --- Easy access to header file constants
 ================================================================

docs/source/context.rst

@@ -1,49 +1,16 @@
 .. testsetup:: *
 
-   from pwnlib import context
+   import pwnlib
+   from   pwnlib.context import context
+   import threading, logging
 
 
 :mod:`pwnlib.context` --- Setting runtime variables
 =====================================================
 
+.. autodata:: pwnlib.context.context
 
-:mod:`pwnlib.context`
----------------------
+.. autoclass:: pwnlib.context.ContextType
+    :members:
 
-.. automodule:: pwnlib.context
-
-   .. exec::
-      import pwnlib
-      from inspect import cleandoc
-      mod = pwnlib.context.defaults.__class__
-      for key in sorted(dir(mod)):
-          val = getattr(mod, key)
-          if isinstance(val, property):
-              print ".. data:: " + key
-              print ""
-              print cleandoc(val.__doc__)
-              print ""
-
-
-   .. autofunction:: pwnlib.context.__call__(**kwargs)
-   .. autofunction:: pwnlib.context.local(**kwargs)
-   .. autofunction:: pwnlib.context.reset_local()
-
-:mod:`pwnlib.context.defaults`
-------------------------------
-
-.. automodule:: pwnlib.context.defaults
-
-   .. exec::
-      import pwnlib
-      mod = pwnlib.context.defaults.__class__
-      for key in sorted(dir(mod)):
-          val = getattr(mod, key)
-          if isinstance(val, property):
-              print ".. data:: " + key
-              print ""
-              print "This is the global version of :data:`pwnlib.context.%s`." % key
-              print
-
-
-   .. autofunction:: pwnlib.context.defaults.__call__(**kwargs)
+.. autoclass:: pwnlib.context.Thread

docs/source/elf.rst

@@ -1,6 +1,7 @@
 .. testsetup:: *
 
    from pwnlib.elf import *
+   from pwnlib.util.misc import which
 
 :mod:`pwnlib.elf` --- Working with ELF binaries
 ===============================================

docs/source/globals.rst

@@ -0,0 +1,102 @@
+.. testsetup:: *
+
+   from pwn import *
+
+``from pwn import *``
+========================
+
+The most common way that you'll see pwntools used is
+
+    >>> from pwn import *
+
+Which imports a bazillion things into the global namespace to make your life easier.
+
+This is a quick list of most of the objects and routines imported, in rough order of importance and frequency of use.
+
+- ``context``
+    - :data:`pwnlib.context.context`
+    - Responsible for most of the pwntools convenience settings
+    - Set `context.log_level = 'debug'` when troubleshooting your exploit
+    - Scope-aware, so you can disable logging for a subsection of code via ``pwnlib.context.ContextType.local``
+- ``remote``, ``listen``, ``ssh``, ``process``
+    - :mod:`pwnlib.tubes`
+    - Super convenient wrappers around all of the common functionality for CTF challenges
+    - Connect to anything, anywhere, and it works the way you want it to
+    - Helpers for common tasks like ``recvline``, ``recvuntil``, ``clean``, etc.
+    - Interact directly with the application via ``.interactive()``
+- ``p32`` and ``u32``
+    - :mod:`pwnlib.util.packing`
+    - Useful functions to make sure you never have to remember if ``'>'`` means signed or unsigned for ``struct.pack``, and no more ugly ``[0]`` index at the end.
+    - Set ``signed`` and ``endian`` in sane manners (also these can be set once on ``context`` and not bothered with again)
+    - Most common sizes are pre-defined (``u8``, ``u64``, etc), and :func:`pwnlib.util.packing.pack` lets you define your own.
+- ``log``
+    - :mod:`pwnlib.log`
+    - Make your output pretty!
+- ``cyclic`` and ``cyclic_func``
+    - :mod:`pwnlib.util.cyclic`
+    - Utilities for generating strings such that you can find the offset of any given substring given only N (usually 4) bytes.  This is super useful for straight buffer overflows.  Instead of looking at 0x41414141, you could know that 0x61616171 means you control EIP at offset 64 in your buffer.
+- ``asm`` and ``disasm``
+    - :mod:`pwnlib.asm`
+    - Quickly turn assembly into some bytes, or vice-versa, without mucking about
+    - Supports any architecture for which you have a binutils installed
+    - Over 20 different architectures have pre-built binaries at `ppa:pwntools/binutils <https://launchpad.net/~pwntools/+archive/ubuntu/binutils>`_.
+- ``shellcraft``
+    - :mod:`pwnlib.shellcraft`
+    - Library of shellcode ready to go
+    - ``asm(shellcraft.sh())`` gives you a shell
+    - Templating library for reusability of shellcode fragments
+- ``ELF``
+    - :mod:`pwnlib.elf`
+    - ELF binary manipulation tools, including symbol lookup, virtual memory to file offset helpers, and the ability to modify and save binaries back to disk
+- ``DynELF``
+    - :mod:`pwnlib.dynelf`
+    - Dynamically resolve functions given only a pointer to any loaded module, and a function which can leak data at any address
+- ``ROP``
+    - :mod:`pwnlib.rop`
+    - Automatically generate ROP chains using a DSL to describe what you want to do, rather than raw addresses
+- ``gdb.debug`` and ``gdb.attach``
+    - :mod:`pwnlib.gdb`
+    - Launch a binary under GDB and pop up a new terminal to interact with it.  Automates setting breakpoints and makes iteration on exploits MUCH faster.
+    - Alternately, attach to a running process given a PID, ``pwnlib.tubes`` object, or even just a socket that's connected to it
+- ``args``
+    - Dictionary contining all-caps command-line arguments for quick access
+    - Run via ``python foo.py REMOTE=1`` and ``args['REMOTE'] == '1'``.
+    - Can also control logging verbosity and terminal fancyness
+        - `NOTERM`
+        - `SILENT`
+        - `DEBUG`
+- ``randoms``, ``rol``, ``ror``, ``xor``, ``bits``
+    - :mod:`pwnlib.util.fiddling`
+    - Useful utilities for generating random data from a given alphabet, or simplifying math operations that usually require masking off with `0xffffffff` or calling `ord` and `chr` an ugly number of times
+- ``net``
+    - :mod:`pwnlib.util.net`
+    - Routines for querying about network interfaces
+- ``proc``
+    - :mod:`pwnlib.util.proc`
+    - Routines for querying about processes
+- ``pause``
+    - It's the new ``getch``
+- ``safeeval``
+    - :mod:`pwnlib.util.safeeval`
+    - Functions for safely evalutaing python code without nasty side-effects.
+
+These are all pretty self explanatory, but are useful to have in the global namespace.
+
+- ``hexdump``
+- ``read`` and ``write``
+- ``enhex`` and ``unhex``
+- ``more``
+- ``group``
+- ``align`` and ``align_down``
+- ``urlencode`` and ``urldecode``
+- ``which``
+- ``wget``
+
+Additionally, all of the following modules are auto-imported for you.  You were going to do it anyway.
+
+- ``os``
+- ``sys``
+- ``time``
+- ``requests``
+- ``re``
+- ``random``

docs/source/index.rst

@@ -1,14 +1,36 @@
-Welcome to pwntools's documentation!
+pwntools
 ====================================
 
-Contents:
+``pwntools`` is a CTF framework and exploit development library.
+Written in Python, it is designed for rapid prototyping and development,
+and intended to make exploit writing as simple as possible.
+
+Getting Started
+---------------
 
 .. toctree::
    :maxdepth: 3
    :glob:
 
+   about
+   install
    intro
+   globals
+   commandline
+
+
+Module Index
+------------
+
+Each of the ``pwntools`` modules is documented here.
+
+.. toctree::
+   :maxdepth: 1
+   :glob:
+
    asm
+   atexception
+   atexit
    constants
    context
    dynelf
@@ -17,15 +39,23 @@ Contents:
    gdb
    log
    memleak
-   tubes
+   replacements
+   rop
    shellcraft
+   shellcraft/*
    term
+   timeout
+   tubes
+   tubes/*
    ui
+   useragents
    util/*
 
-Indices and tables
-==================
+.. only:: not dash
 
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+   Indices and tables
+   ==================
+
+   * :ref:`genindex`
+   * :ref:`modindex`
+   * :ref:`search`

docs/source/install.rst

@@ -0,0 +1,40 @@
+Installation
+============
+
+pwntools is best supported on Ubuntu 12.04 and 14.04, but most
+functionality should work on any Posix-like distribution (Debian, Arch,
+FreeBSD, OSX, etc.).
+
+Prerequisites
+-------------
+
+In order to get the most out of ``pwntools``, you should have the
+following system libraries installed.
+
+.. toctree::
+   :maxdepth: 3
+   :glob:
+
+   install/*
+
+Released Version
+-----------------
+
+Pwntools is available as a ``pip`` package.
+
+.. code-block:: bash
+
+    $ apt-get install python2.7 python2.7-dev python-pip
+    $ pip install pwntools
+
+Latest Version
+--------------
+
+Alternatively if you prefer to use the latest version from the
+repository, pip can install directly from github.
+
+.. code-block:: bash
+
+    $ pip install git+https://github.com/Gallopsled/pwntools#egg=pwntools
+
+.. _Ubuntu: https://launchpad.net/~pwntools/+archive/ubuntu/binutils

docs/source/install/binutils.rst

@@ -0,0 +1,84 @@
+Binutils
+-------------
+
+Assembly of foreign architectures (e.g. assembling Sparc shellcode on
+Mac OS X) requires cross-compiled versions of ``binutils`` to be
+installed. We've made this process as smooth as we can.
+
+In these examples, replace ``$ARCH`` with your target architecture (e.g., arm, mips64, vax, etc.).
+
+Building `binutils` from source takes about 60 seconds on a modern 8-core machine.
+
+Ubuntu
+^^^^^^^^^^^^^^^^
+First, add our `Personal Package Archive repository <http://binutils.pwntools.com>`__.
+
+.. code-block:: bash
+
+    $ apt-get install software-properties-common
+    $ apt-add-repository ppa:pwntools/binutils
+    $ apt-get update
+
+Then, install the binutils for your architecture.
+
+.. code-block:: bash
+
+    $ apt-get install binutils-$ARCH-linux-gnu
+
+Mac OS X
+^^^^^^^^^^^^^^^^
+
+Mac OS X is just as easy, but requires building binutils from source.
+However, we've made ``homebrew`` recipes to make this a single command.
+After installing `brew <http://brew.sh>`__, grab the appropriate
+recipe from our `binutils
+repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
+
+.. code-block:: bash
+
+    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/osx/binutils-$ARCH.rb
+
+Alternate OSes
+^^^^^^^^^^^^^^^^
+
+If you want to build everything by hand, or don't use any of the above
+OSes, ``binutils`` is simple to build by hand.
+
+.. code-block:: bash
+
+    #!/usr/bin/env bash
+
+    V=2.25   # Binutils Version
+    ARCH=arm # Target architecture
+
+    cd /tmp
+    wget -nc https://ftp.gnu.org/gnu/binutils/binutils-$V.tar.gz
+    wget -nc https://ftp.gnu.org/gnu/binutils/binutils-$V.tar.gz.sig
+
+    gpg --keyserver keys.gnupg.net --recv-keys 4AE55E93
+    gpg --verify binutils-$V.tar.gz.sig
+
+    rm -rf binutils-*
+
+    tar xf binutils-$V.tar.gz
+
+    mkdir binutils-build
+    cd binutils-build
+
+    export AR=ar
+    export AS=as
+
+    ../binutils-$V/configure \
+        --prefix=/usr/local \
+        --target=$ARCH-unknown-linux-gnu \
+        --disable-static \
+        --disable-multilib \
+        --disable-werror \
+        --disable-nls
+
+    MAKE=gmake
+    hash gmake || MAKE=make
+
+    $MAKE -j
+    sudo $MAKE install
+

docs/source/install/headers.rst

@@ -0,0 +1,18 @@
+Python Development Headers
+-----------------------------
+
+Some of pwntools' Python dependencies require native extensions (for example, Paramiko requires PyCrypto).
+
+In order to build these native extensions, the development headers for Python must be installed.
+
+Ubuntu
+^^^^^^^^^^^^^^^^
+
+.. code-block:: bash
+
+    $ apt-get install python-dev
+
+Mac OS X
+^^^^^^^^^^^^^^^^
+
+No action needed.

docs/source/intro.rst

@@ -1,53 +1,221 @@
-Introduction to pwntools
+.. testsetup:: *
+
+   from pwn import *
+
+Getting Started
 ========================
 
-Historically pwntools was used as a sort of exploit-writing DSL. Simply doing
-``from pwn import *`` in a previous version of pwntools would bring all sorts of
-nice side-effects.
+To get your feet wet with pwntools, let's first go through a few examples.
 
-When redesigning pwntools for 2.0, we noticed two contrary goals:
+When writing exploits, pwntools generally follows the "kitchen sink" approach.
 
-* We would like to have a "normal" python module structure, to allow other
-  people to faster get familiar with how pwntools works.
-* We would like to have even more side-effects, especially by putting the
-  terminal in raw-mode.
+    >>> from pwn import *
 
-To make this possible, we decided to have two different modules. :mod:`pwnlib`
-would be our nice, clean python module, while :mod:`pwn` would be used during
-CTFs.
+This imports a lot of functionality into the global namespace.  You can now
+assemble, disassemble, pack, unpack, and many other things with a single function.
 
-:mod:`pwnlib` --- Normal python library
----------------------------------------
+A full list of everything that is imported is available on :doc:`globals`.
 
-.. module:: pwnlib
 
-This module is our "clean" python-code. As a rule, we do not think that
-importing :mod:`pwnlib` or any of the submodules should have any significant
-side-effects (besides e.g. caching).
+Making Connections
+------------------
 
-For the most part, you will also only get the bits you import. You for instance
-not get access to :mod:`pwnlib.util.packing` simply by doing ``import
-pwnlib.util``.
+You need to talk to the challenge binary in order to pwn it, right?
+pwntools makes this stupid simple with its :mod:`pwnlib.tubes` module.
 
-Though there are a few exceptions (such as :mod:`pwnlib.shellcraft`), that does
-not quite fit the goals of being simple and clean, but they can still be
-imported without implicit side-effects.
+This exposes a standard interface to talk to processes, sockets, serial ports,
+and all manner of things, along with some nifty helpers for common tasks.
+For example, remote connections via :mod:`pwnlib.tubes.remote`.
 
-:mod:`pwn` --- Toolbox optimized for CTFs
------------------------------------------
+    >>> conn = remote('ftp.debian.org',21)
+    >>> conn.recvline() # doctest: +ELLIPSIS
+    '220 ...'
+    >>> conn.send('USER anonymous\r\n')
+    >>> conn.recvuntil(' ', drop=True)
+    '331'
+    >>> conn.recvline()
+    'Please specify the password.\r\n'
+    >>> conn.close()
 
-.. module:: pwn
+It's also easy to spin up a listener
 
-As stated, we would also like to have the ability to get a lot of these
-side-effects by default. That is the purpose of this module. It does
-the following:
+    >>> l = listen()
+    >>> r = remote('localhost', l.lport)
+    >>> c = l.wait_for_connection()
+    >>> r.send('hello')
+    >>> c.recv()
+    'hello'
+
+Interacting with processes is easy thanks to :mod:`pwnlib.tubes.process`.
+
+::
+
+    >>> sh = process('/bin/sh')
+    >>> sh.sendline('sleep 3; echo hello world;')
+    >>> sh.recvline(timeout=1)
+    ''
+    >>> sh.recvline(timeout=5)
+    'hello world\n'
+    >>> sh.close()
+
+Not only can you interact with processes programmatically, but you can
+actually **interact** with processes.
+
+    >>> sh.interactive() # doctest: +SKIP
+    $ whoami
+    user
+
+There's even an SSH module for when you've got to SSH into a box to perform
+a local/setuid exploit with :mod:`pwnlib.tubes.ssh`.  You can quickly spawn
+processes and grab the output, or spawn a process and interact iwth it like
+a ``process`` tube.
+
+::
+
+    >>> shell = ssh('bandit0', 'bandit.labs.overthewire.org', password='bandit0')
+    >>> shell['whoami']
+    'bandit0'
+    >>> shell.download_file('/etc/motd')
+    >>> sh = shell.run('sh')
+    >>> sh.sendline('sleep 3; echo hello world;') # doctest: +SKIP
+    >>> sh.recvline(timeout=1)
+    ''
+    >>> sh.recvline(timeout=5)
+    'hello world\n'
+    >>> shell.close()
+
+Packing Integers
+------------------
+
+A common task for exploit-writing is converting between integers as Python
+sees them, and their representation as a sequence of bytes.
+Usually folks resort to the built-in ``struct`` module.
+
+pwntools makes this easier with :mod:`pwnlib.util.packing`.  No more remembering
+unpacking codes, and littering your code with helper routines.
+
+    >>> import struct
+    >>> p32(0xdeadbeef) == struct.pack('I', 0xdeadbeef)
+    True
+    >>> leet = '37130000'.decode('hex')
+    >>> u32('abcd') == struct.unpack('I', 'abcd')[0]
+    True
+
+The packing/unpacking operations are defined for many common bit-widths.
+
+    >>> u8('A') == 0x41
+    True
+
+Setting the Target Architecture and OS
+--------------------------------------
+
+The target architecture can generally be specified as an argument to the routine that requires it.
+
+    >>> asm('nop')
+    '\x90'
+    >>> asm('nop', arch='arm')
+    '\x00\xf0 \xe3'
+
+However, it can also be set once in the global ``context``.  The operating system, word size, and endianness can also be set here.
+
+    >>> context.arch      = 'i386'
+    >>> context.os        = 'linux'
+    >>> context.endian    = 'little'
+    >>> context.word_size = 32
+
+Additionally, you can use a shorthand to set all of the values at once.
+
+    >>> asm('nop')
+    '\x90'
+    >>> context(arch='arm', os='linux', endian='big', word_size=32)
+    >>> asm('nop')
+    '\xe3 \xf0\x00'
+
+.. doctest::
+   :hide:
+
+    >>> context.clear()
+
+Setting Logging Verbosity
+-------------------------
+
+You can control the verbosity of the standard pwntools logging via ``context``.
+
+For example, setting
+
+    >>> context.log_level = 'debug'
+
+Will cause all of the data sent and received by a ``tube`` to be printed to the screen.
+
+.. doctest::
+   :hide:
+
+    >>> context.clear()
+
+Assembly and Disassembly
+------------------------
+
+Never again will you need to run some already-assembled pile of shellcode
+from the internet!  The :mod:`pwnlib.asm` module is full of awesome.
+
+    >>> asm('mov eax, 0').encode('hex')
+    'b800000000'
+
+But if you do, it's easy to suss out!
+
+    >>> print disasm('6a0258cd80ebf9'.decode('hex'))
+       0:   6a 02                   push   0x2
+       2:   58                      pop    eax
+       3:   cd 80                   int    0x80
+       5:   eb f9                   jmp    0x0
+
+However, you shouldn't even need to write your own shellcode most of the
+time!  Pwntools comes with the :mod:`pwnlib.shellcraft` module, which is
+loaded with useful time-saving shellcodes.
+
+Let's say that we want to `setreuid(getuid(), getuid())` followed by `dup`ing
+file descriptor 4 to `stdin`, `stdout`, and `stderr`, and then pop a shell!
+
+    >>> asm(shellcraft.setreuid() + shellcraft.dupsh(4)).encode('hex')
+    '6a3158cd8089c389d96a4658cd806a045b6a0359496a3f58cd8075f86a68682f2f2f73682f62696e89e331c96a0b5899cd80'
+
+
+Misc Tools
+----------------------
+
+Never write another hexdump, thanks to :mod:`pwnlib.util.fiddling`.
+
+
+Find offsets in your buffer that cause a crash, thanks to :mod:`pwnlib.cyclic`.
+
+    >>> print cyclic(20)
+    aaaabaaacaaadaaaeaaa
+    >>> # Assume EIP = 0x62616166 ('faab' which is pack(0x62616166))  at crash time
+    >>> print cyclic_find('faab')
+    120
+
+ELF Manipulation
+----------------
+
+Stop hard-coding things!  Look them up at runtime with :mod:`pwnlib.elf`.
+
+    >>> e = ELF('/bin/cat')
+    >>> print hex(e.address) #doctest: +SKIP
+    0x400000
+    >>> print hex(e.symbols['write']) #doctest: +SKIP
+    0x401680
+    >>> print hex(e.got['write']) #doctest: +SKIP
+    0x60b070
+    >>> print hex(e.plt['write']) #doctest: +SKIP
+    0x401680
+
+You can even patch and save the files.
+
+    >>> e = ELF('/bin/cat')
+    >>> e.read(e.address+1, 3)
+    'ELF'
+    >>> e.asm(e.address, 'ret')
+    >>> e.save('/tmp/quiet-cat')
+    >>> disasm(file('/tmp/quiet-cat','rb').read(1))
+    '   0:   c3                      ret'
 
-* Imports everything from the toplevel :mod:`pwnlib` along with
-  functions from a lot of submodules. This means that if you do
-  ``import pwn`` or ``from pwn import *``, you will access to
-  everything you need to write an exploit.
-* Calls :func:`pwnlib.term.init` to put your terminal in raw mode
-  and implementing functionality to make it look like it is not.
-* Setting the :data:`pwnlib.context.log_level` to `"info"`.
-* Tries to parse some of the values in :data:`sys.argv` and every
-  value it succeeds in parsing it removes.

docs/source/log.rst

@@ -1,11 +1,21 @@
-:mod:`pwnlib.log` and   --- Logging stuff
-=================================================================
+:mod:`pwnlib.log` --- Logging stuff
+===================================
 
 .. automodule:: pwnlib.log
-   :members:
+   :members: getLogger, install_default_handler, rootlogger
 
-:mod:`pwnlib.log_levels` --- Possible log levels
-------------------------------------------------
-
-.. automodule:: pwnlib.log_levels
+.. autoclass:: pwnlib.log.Progress
    :members:
+   :member-order: bysource
+
+.. autoclass:: pwnlib.log.Logger
+   :members:
+   :member-order: bysource
+
+.. autoclass:: pwnlib.log.Handler
+   :members:
+   :member-order: bysource
+
+.. autoclass:: pwnlib.log.Formatter
+   :members:
+   :member-order: bysource

docs/source/memleak.rst

@@ -1,3 +1,7 @@
+.. testsetup:: *
+
+   from pwnlib.memleak import *
+
 :mod:`pwnlib.memleak` --- Helper class for leaking memory
 =========================================================
 

docs/source/replacements.rst

@@ -0,0 +1,5 @@
+:mod:`pwnlib.replacements` --- Replacements for various functions
+=================================================================
+
+.. automodule:: pwnlib.replacements
+   :members:

docs/source/rop.rst

@@ -0,0 +1,11 @@
+.. testsetup:: *
+
+   from pwnlib.elf import *
+   from pwnlib.rop import *
+   from pwnlib.util.misc import which
+
+:mod:`pwnlib.rop` --- Return Oriented Programming
+=================================================
+
+.. automodule:: pwnlib.rop
+   :members:

docs/source/shellcraft/amd64.rst

@@ -0,0 +1,18 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.shellcraft.amd64` --- Shellcode for AMD64
+===========================================================
+
+:mod:`pwnlib.shellcraft.amd64`
+---------------------------------------
+
+.. automodule:: pwnlib.shellcraft.amd64
+   :members:
+
+:mod:`pwnlib.shellcraft.amd64.linux`
+---------------------------------------
+
+.. automodule:: pwnlib.shellcraft.amd64.linux
+   :members:

docs/source/shellcraft/arm.rst

@@ -0,0 +1,18 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.shellcraft.arm` --- Shellcode for ARM
+===========================================================
+
+:mod:`pwnlib.shellcraft.arm`
+-----------------------------
+
+.. automodule:: pwnlib.shellcraft.arm
+   :members:
+
+:mod:`pwnlib.shellcraft.arm.linux`
+-----------------------------------
+
+.. automodule:: pwnlib.shellcraft.arm.linux
+   :members:

docs/source/shellcraft/i386.rst

@@ -1,3 +1,7 @@
+.. testsetup:: *
+
+   from pwn import *
+
 :mod:`pwnlib.shellcraft.i386` --- Shellcode for Intel 80386
 ===========================================================
 

docs/source/shellcraft/regsort.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+    from pwnlib.regsort import *
+
+:mod:`pwnlib.regsort` --- Register sorting
+===========================================================
+
+.. automodule:: pwnlib.regsort
+   :members:

docs/source/shellcraft/thumb.rst

@@ -0,0 +1,18 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.shellcraft.thumb` --- Shellcode for Thumb Mode
+===========================================================
+
+:mod:`pwnlib.shellcraft.thumb`
+-------------------------------
+
+.. automodule:: pwnlib.shellcraft.thumb
+   :members:
+
+:mod:`pwnlib.shellcraft.thumb.linux`
+---------------------------------------
+
+.. automodule:: pwnlib.shellcraft.thumb.linux
+   :members:

docs/source/timeout.rst

@@ -0,0 +1,11 @@
+.. testsetup:: *
+
+   from pwnlib.context import context
+   from pwnlib.timeout import Timeout
+   import time
+
+:mod:`pwnlib.timeout` --- Timeout handling
+========================================
+
+.. automodule:: pwnlib.timeout
+   :members:

docs/source/tubes.rst

@@ -1,56 +1,30 @@
+.. testsetup:: *
+
+   from pwn import *
+
 :mod:`pwnlib.tubes` --- Talking to the World!
 =============================================
 
 .. automodule:: pwnlib.tubes
 
-   Sockets
-   -------
 
-   .. automodule:: pwnlib.tubes.remote
+Types of Tubes
+-------------------
 
-      .. autoclass:: pwnlib.tubes.remote.remote(host, port, fam = None, typ = None, proto = 0, timeout = 'default', log_level = INFO)
-         :members:
-         :show-inheritance:
+.. toctree::
+    :maxdepth: 3
+    :glob:
 
-   .. automodule:: pwnlib.tubes.listen
-
-      .. autoclass:: pwnlib.tubes.listen.listen(port, bindaddr = "0.0.0.0", fam = "any", typ = "tcp", timeout = 'default', log_level = INFO)
-         :members:
-         :show-inheritance:
-
-   .. automodule:: pwnlib.tubes.sock
-
-      .. autoclass:: pwnlib.tubes.sock.sock()
-         :show-inheritance:
-
-   Processes
-   ---------
-
-   .. automodule:: pwnlib.tubes.process
-
-      .. autoclass:: pwnlib.tubes.process.process(args, shell = False, executable = None, env = None, timeout = 'default', log_level = INFO)
-         :members: kill, poll, communicate
-         :show-inheritance:
-
-   SSH
-   ---
-
-   .. automodule:: pwnlib.tubes.ssh
-
-      .. autoclass:: pwnlib.tubes.ssh.ssh(user, host, port = 22, password = None, key = None, keyfile = None, proxy_command = None, proxy_sock = None, timeout = 'default', log_level = INFO)
-         :members:
-
-      .. autoclass:: pwnlib.tubes.ssh.ssh_channel()
-         :members: kill, poll, interactive
-         :show-inheritance:
+    tubes/*
 
 
-   Common functionality
-   --------------------
+:mod:`pwnlib.tubes.tube` --- Common Functionality
+-------------------------------------------------
 
-   .. automodule:: pwnlib.tubes.tube
 
-      .. autoclass:: pwnlib.tubes.tube.tube()
-         :members:
-         :exclude-members: recv_raw, send_raw, settimeout_raw,
-                           can_recv_raw
+.. automodule:: pwnlib.tubes.tube
+
+  .. autoclass:: pwnlib.tubes.tube.tube()
+     :members:
+     :exclude-members: recv_raw, send_raw, settimeout_raw,
+                       can_recv_raw

docs/source/tubes/processes.rst

@@ -0,0 +1,12 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.tubes.process` --- Processes
+===========================================================
+
+.. automodule:: pwnlib.tubes.process
+
+  .. autoclass:: pwnlib.tubes.process.process
+     :members: kill, poll, communicate
+     :show-inheritance:

docs/source/tubes/serial.rst

@@ -0,0 +1,11 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.tubes.serialtube` --- Serial Ports
+===========================================================
+
+.. automodule:: pwnlib.tubes.serialtube
+
+   .. autoclass:: pwnlib.tubes.serialtube.serialtube
+      :members:

docs/source/tubes/sockets.rst

@@ -0,0 +1,25 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.tubes.sock` --- Sockets
+===========================================================
+
+
+.. automodule:: pwnlib.tubes.sock
+
+   .. autoclass:: pwnlib.tubes.sock.sock()
+      :show-inheritance:
+
+
+.. automodule:: pwnlib.tubes.remote
+
+   .. autoclass:: pwnlib.tubes.remote.remote
+      :members:
+      :show-inheritance:
+
+.. automodule:: pwnlib.tubes.listen
+
+   .. autoclass:: pwnlib.tubes.listen.listen
+      :members:
+      :show-inheritance:

docs/source/tubes/ssh.rst

@@ -0,0 +1,21 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.tubes.ssh` --- SSH
+===========================================================
+
+.. automodule:: pwnlib.tubes.ssh
+
+   .. autoclass:: pwnlib.tubes.ssh.ssh
+      :members:
+
+   .. autoclass:: pwnlib.tubes.ssh.ssh_channel()
+      :members: kill, poll, interactive
+      :show-inheritance:
+
+   .. autoclass:: pwnlib.tubes.ssh.ssh_connecter()
+      :show-inheritance:
+
+   .. autoclass:: pwnlib.tubes.ssh.ssh_listener()
+      :show-inheritance:

docs/source/useragents.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+   from pwnlib.useragents import *
+
+:mod:`pwnlib.useragents` --- A database of useragent strings
+============================================================
+
+.. automodule:: pwnlib.useragents
+   :members:

docs/source/util/hashes.rst

@@ -0,0 +1,5 @@
+:mod:`pwnlib.util.hashes` --- Hashing functions
+===============================================
+
+.. automodule:: pwnlib.util.hashes
+   :members:

docs/source/util/iters.rst

@@ -0,0 +1,113 @@
+.. testsetup:: *
+
+   import string, operator
+   from pwnlib.util.iters import *
+
+
+:mod:`pwnlib.util.iters` --- Extension of standard module :mod:`itertools`
+==========================================================================
+
+.. automodule:: pwnlib.util.iters
+   :members:
+   :exclude-members:
+     chain                                  ,
+     combinations                           ,
+     combinations_with_replacement          ,
+     compress                               ,
+     count                                  ,
+     cycle                                  ,
+     dropwhile                              ,
+     groupby                                ,
+     ifilter                                ,
+     ifilterfalse                           ,
+     imap                                   ,
+     islice                                 ,
+     izip                                   ,
+     izip_longest                           ,
+     permutations                           ,
+     product                                ,
+     repeat                                 ,
+     starmap                                ,
+     takewhile                              ,
+     tee
+
+.. function:: chain
+
+   Alias for :func:`itertools.chain`.
+
+.. function:: combinations
+
+   Alias for :func:`itertools.combinations`
+
+.. function:: combinations_with_replacement
+
+   Alias for :func:`itertools.combinations_with_replacement`
+
+.. function:: compress
+
+   Alias for :func:`itertools.compress`
+
+.. function:: count
+
+   Alias for :func:`itertools.count`
+
+.. function:: cycle
+
+   Alias for :func:`itertools.cycle`
+
+.. function:: dropwhile
+
+   Alias for :func:`itertools.dropwhile`
+
+.. function:: groupby
+
+   Alias for :func:`itertools.groupby`
+
+.. function:: ifilter
+
+   Alias for :func:`itertools.ifilter`
+
+.. function:: ifilterfalse
+
+   Alias for :func:`itertools.ifilterfalse`
+
+.. function:: imap
+
+   Alias for :func:`itertools.imap`
+
+.. function:: islice
+
+   Alias for :func:`itertools.islice`
+
+.. function:: izip
+
+   Alias for :func:`itertools.izip`
+
+.. function:: izip_longest
+
+   Alias for :func:`itertools.izip_longest`
+
+.. function:: permutations
+
+   Alias for :func:`itertools.permutations`
+
+.. function:: product
+
+   Alias for :func:`itertools.product`
+
+.. function:: repeat
+
+   Alias for :func:`itertools.repeat`
+
+.. function:: starmap
+
+   Alias for :func:`itertools.starmap`
+
+.. function:: takewhile
+
+   Alias for :func:`itertools.takewhile`
+
+.. function:: tee
+
+   Alias for :func:`itertools.tee`
+

docs/source/util/misc.rst

@@ -1,7 +1,7 @@
 .. testsetup:: *
 
    from pwnlib.util.misc import *
-   import os
+   import os, subprocess
    os.chdir("..")
 
 :mod:`pwnlib.util.misc` --- We could not fit it any other place

docs/source/util/proc.rst

@@ -1,6 +1,7 @@
 .. testsetup:: *
 
    from pwnlib.util.proc import *
+   import os, sys
 
 
 :mod:`pwnlib.util.proc` --- Working with ``/proc/``

docs/source/util/web.rst

@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwnlib.context import context
+   from pwnlib.util.web import *
+
+:mod:`pwnlib.util.web` --- Utilities for working with the WWW
+=============================================================
+
+.. automodule:: pwnlib.util.web
+   :members:

examples/README

@@ -1,6 +0,0 @@
-While these examples should all work, they are not very representative of
-the pwntools project.
-
-We have a plan to create a separate repository with examples, primarily
-exploits. Until we do so, we recommend new users to look at
-http://pwntools.readthedocs.org, as this is a better overview of our features.

examples/README.md

@@ -0,0 +1,109 @@
+# Examples
+While these examples should all work, they are not very representative of
+the pwntools project.
+
+We have a plan to create a separate repository with examples, primarily
+exploits. Until we do so, we recommend new users to look at
+https://pwntools.readthedocs.org, as this is a better overview of our features.
+
+In no particular order the docstrings for each example:
+
+* `args.py`
+```
+When not in lib-mode (import `pwn` rather than `pwnlib`) we parse the
+commandline for variables definitions.  A variable definition has the form::
+
+  <var>=<val>
+
+where ``<var>`` contains only uppercase letters, digits and underscores and
+doesn't start with a digit.
+
+Try running this example with::
+
+  $ python args.py RHOST=localhost RPORT=1337
+```
+* `asm.py`
+```
+Example showing the interface to `pwnlib.asm.asm` and `pwnlib.shellcraft`.
+```
+* `clean_and_log.py`
+```
+Use case for `pwnlib.tubes.tube.clean_and_log`.
+
+Sometimes you will have a solution to a challenge but you don't know what it
+will look like when you get the flag.  Sometimes that will leave you with a
+top-level exception, no flag, and angry team members.
+
+Solution:
+ 1. Always run wireshark or tcpdump.  Always.
+ 2. Register <your socket>.clean or <your socket>.clean_and_log to run at exit.
+```
+* `indented.py`
+```
+When running in term-mode (import `pwn` rather than `pwnlib`, stdout is a TTY
+and not running in a REPL), we can do proper indentation where lines too long to
+fit on a screen are split into multiple individually indented lines.
+
+Too see the difference try running with::
+
+  $ python indented.py
+
+and
+
+  $ python -i indented.py
+
+Also notice that `pause()` can react on any key when in `term_mode`.
+```
+* `listen_uroboros.py`
+```
+An example showing interconnection of sockets.  This script will wait for three
+connections on port 1337, then connect them like a three-way Uroboros.
+```
+* `options.py`
+```
+Example showing `pwnlib.ui.options()`
+```
+* `port_forward.py`
+```
+A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.  Notice
+that `<>` is just a shorthand.
+```
+* `readline_completers.py`
+```
+Example showing pwnlib's readline implementation and a few completers.  This
+part of pwnlib will probably see some major changes soon, but we wanted to show
+off some proof-of-concepts.
+```
+* `remote_gdb_debugging.py`
+```
+Simple example showing how to use the remote
+gdb debugging features available in pwntools.
+```
+* `spinners.py`
+```
+Just a lot of spinners!
+```
+* `splash.py`
+```
+"Easteregg"
+```
+* `ssh.py`
+```
+Example showing how to use the ssh class.
+```
+* `text.py`
+```
+Example showing how to use `pwnlib.term.text`.
+
+Try running with::
+
+  $ TERM=xterm python text.py
+
+and::
+
+  $ TERM=xterm-256color python text.py
+```
+* `remote.py`
+```
+Example showing how to use the remote class.
+```

examples/args.py

@@ -0,0 +1,18 @@
+"""
+When not in lib-mode (import `pwn` rather than `pwnlib`) we parse the
+commandline for variables definitions.  A variable definition has the form::
+
+  <var>=<val>
+
+where ``<var>`` contains only uppercase letters, digits and underscores and
+doesn't start with a digit.
+
+Try running this example with::
+
+  $ python args.py RHOST=localhost RPORT=1337
+"""
+
+from pwn import *
+
+print args['RHOST'] or 'RHOST is not set'
+print args['RPORT'] or 'RPORT is not set'

examples/argstest.py

@@ -1,4 +0,0 @@
-import pwn
-
-print pwn.args['RHOST'],
-print pwn.args['RPORT']

examples/asm.py

@@ -1,5 +1,9 @@
+"""
+Example showing the interface to `pwnlib.asm.asm` and `pwnlib.shellcraft`.
+"""
+
 from pwn import *
-context(arch='i386', os='linux', net='tcp4')
+context(arch='i386', os='linux')
 
 shellcode = shellcraft.i386_to_amd64()
 shellcode_asm = asm(shellcode)

examples/attach.py

@@ -0,0 +1,12 @@
+"""
+Example showing `pwnlib.gdb.attach()`
+"""
+
+from pwn import *
+
+bash = process('/bin/bash')
+gdb.attach(bash, execute = '''
+p "hello from pwnlib"
+c
+''')
+bash.interactive()

examples/clean_and_log.py

@@ -0,0 +1,29 @@
+"""
+Use case for `pwnlib.tubes.tube.clean_and_log`.
+
+Sometimes you will have a solution to a challenge but you don't know what it
+will look like when you get the flag.  Sometimes that will leave you with a
+top-level exception, no flag, and angry team members.
+
+Solution:
+ 1. Always run wireshark or tcpdump.  Always.
+ 2. Register <your socket>.clean or <your socket>.clean_and_log to run at exit.
+"""
+
+from pwn import *
+
+os.system('''((
+echo prefix sometext ;
+echo prefix someothertext ;
+echo here comes the flag ;
+echo LostInTheInterTubes
+) | nc -l 1337) &
+''')
+
+r = remote('localhost', 1337)
+atexit.register(r.clean_and_log)
+
+while True:
+    line = r.recvline()
+    print re.findall('^prefix (\S+)$', line)[0]
+

examples/completertest.py

@@ -1,15 +0,0 @@
-import pwn
-import pwnlib.term.completer
-
-c1 = pwn.term.completer.LongestPrefixCompleter([
-    'foobar',
-    'foobaz',
-    'fooquux',
-    ])
-
-c2 = pwn.term.completer.PathCompleter(mask = '*.py')
-
-with c1:
-    while True:
-        s = pwn.term.readline.readline(prompt = '> ')
-        print 'You wrote', s

examples/contexttest.py

@@ -1,6 +0,0 @@
-import pwn
-
-print pwn.context
-
-pwn.context.arch = 'i386'
-pwn.context.arch = 'yolo'

examples/gen-README.py

@@ -0,0 +1,41 @@
+"""
+Script to generate README.md
+"""
+
+from pwn import *
+
+out = '''# Examples
+While these examples should all work, they are not very representative of
+the pwntools project.
+
+We have a plan to create a separate repository with examples, primarily
+exploits. Until we do so, we recommend new users to look at
+https://pwntools.readthedocs.org, as this is a better overview of our features.
+
+In no particular order the docstrings for each example:
+
+'''
+
+def append_example(_arg, top, names):
+    global out
+    for name in names:
+        if not (name.endswith('.py') and name != __file__):
+            continue
+        path = os.path.join(top, name)[2:] # strip './'
+        log.info('-> %s' % path)
+        data = read(path).strip()
+        if data[0:3] not in ('"""', "'''"):
+            log.warning('  Has no docstring!')
+            continue
+        try:
+            i = data.index(data[0:3], 3)
+        except ValueError:
+            log.warning('  Docstring is weird')
+            continue
+        doc = util.safeeval.const(data[0:i + 3])
+        out += '* `%s`\n' % path
+        out += '```%s```\n' % doc
+
+os.path.walk('.', append_example, None)
+
+write('README.md', out)

examples/indented.py

@@ -0,0 +1,25 @@
+"""
+When running in term-mode (import `pwn` rather than `pwnlib`, stdout is a TTY
+and not running in a REPL), we can do proper indentation where lines too long to
+fit on a screen are split into multiple individually indented lines.
+
+Too see the difference try running with::
+
+  $ python indented.py
+
+and
+
+  $ python -i indented.py
+
+Also notice that `pause()` can react on any key when in `term_mode`.
+"""
+
+from pwn import *
+
+context.log_level = 'info'
+
+log.indented('A' * 100)
+log.indented('B' * 100)
+log.indented('C' * 100)
+
+pause()

examples/indenttest.py

@@ -1,8 +0,0 @@
-import pwn
-
-pwn.context.log_level = 'info'
-
-# print pwn2.lib
-pwn.log.info(('A' * 100 + '\n') * 4)
-
-raw_input('>')

examples/inputtest.py

@@ -1,48 +0,0 @@
-# raise Exception('1234')
-
-# import sys
-# _write = sys.stdout.write
-# class Proxy:
-#     def __init__(self, fd):
-#         self._fd = fd
-#     def readline(self):
-#         _write('Input plox: ')
-#         return self._fd.readline()
-#     def read(self, n):
-#         _write('xxx')
-#         return self._fd.read(n)
-#     def __getattr__(self, k):
-#         # _write('[%s]\n' % k)
-#         return self._fd.__getattribute__(k)
-
-# # sys.stdout = Proxy(sys.stdout)
-# sys.stdin = Proxy(sys.stdin)
-
-# print raw_input()
-
-
-import pwn
-import pwnlib.term.readline
-
-h = pwn.term.output('yolo\n')
-i = 0
-def foo(_):
-    global i
-    i += 1
-    pwn.term.output('foooooo%d\n' % i, frozen = True, before = h)
-
-def bar():
-    print
-
-def baz(_):
-    import sys
-    sys.exit()
-
-pwnlib.term.readline.startup_hook = bar
-pwnlib.term.readline.keymap.register({
-    'C-x C-a': foo,
-    'C-x C-s': baz,
-    })
-
-while True:
-    print raw_input('> ')

examples/iotest.py

@@ -1,70 +0,0 @@
-import termios, tty, sys, string, time
-
-def get_term_size():
-    fd = os.open(os.ctermid(), os.O_RDONLY)
-    height, width = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234'))
-    os.close(fd)
-    return width, height
-
-def get_cursor_pos():
-    sys.stdout.write("\033[6n")
-    s = ''
-    while True:
-        c = sys.stdin.read(1)
-        if c == 'R':
-            break
-        s += c
-    col, row = map(int, s[2:].split(';'))
-    return col, row
-
-fd = sys.stdin.fileno()
-old_settings = termios.tcgetattr(fd)
-tty.setraw(sys.stdin.fileno())
-
-while True:
-    # print 'At (%d, %d)' % get_cursor_pos()
-    c = sys.stdin.read(1)
-    if c == 'q':
-        break
-    if c == '\r':
-        print
-    if c == 'w':
-        s = '\x01\x1b[s\x1b[50Ayolo\x1b[u\x02'
-        sys.stdout.write(s)
-    if c == 'e':
-        sys.stdout.write('\x1b[s')
-        for i in range(100):
-            sys.stdout.write('\x1b[0;15f%s' % '/-\\|'[i % 4])
-            sys.stdout.flush()
-            time.sleep(0.1)
-        sys.stdout.write('\x1b[u')
-    if c == 'E':
-        sys.stdout.write('\x1b[s')
-        for i in range(100):
-            sys.stdout.write('\x1b[0;15H%s' % '.oO*'[i % 4])
-            sys.stdout.flush()
-            time.sleep(0.1)
-        sys.stdout.write('\x1b[u')
-    if c == 'r':
-        sys.stdout.write('\x1b[S')
-    if c == 't':
-        sys.stdout.write('\x1b[T')
-    if c == 'y':
-        sys.stdout.write('\x1b[syolo\r\n\x1b[ufoo')
-    if c == 'u':
-        x = ord('a')
-        for i in range(10):
-            sys.stdout.write('\x1b[A' + chr(x))
-            sys.stdout.flush()
-            time.sleep(0.3)
-            x += 1
-            sys.stdout.write('\x1b[B' + chr(x))
-            sys.stdout.flush()
-            time.sleep(0.3)
-            x += 1
-    if c in string.printable:
-        sys.stdout.write(c)
-    else:
-        sys.stdout.write('\\x%02x' % ord(c))
-
-termios.tcsetattr(fd, termios.TCSADRAIN, old_settings)

examples/keymaptest.py

@@ -1,27 +0,0 @@
-import pwn
-
-def match(t):
-    print 'match', t
-
-def nomatch(t):
-    print 'nomatch', t
-
-def key(t):
-    print 'key', t
-
-def tab(t):
-    print 'tab', t
-
-def tabtab(t):
-    print 'tabtab', t
-
-import pwnlib.term.keymap
-km = pwnlib.term.keymap.Keymap({
-    '<match>'     : match,
-    '<nomatch>'   : nomatch,
-    '<any>'       : key,
-    '<tab>'       : tab,
-    '<tab> <tab>' : tabtab,
-    })
-
-km.handle_input()

examples/listen_uroboros.py

@@ -0,0 +1,14 @@
+"""
+An example showing interconnection of sockets.  This script will wait for three
+connections on port 1337, then connect them like a three-way Uroboros.
+"""
+
+from pwn import *
+
+cs = [listen(1337).wait_for_connection() for _ in range(3)]
+
+cs[0] << cs[1] << cs[2] << cs[0]
+
+cs[0].wait_for_close()
+cs[1].wait_for_close()
+cs[2].wait_for_close()

examples/logtest.py

@@ -1,55 +0,0 @@
-# import pwn2.lib as lib
-import pwn
-import time
-
-pwn.context.log_level = 0
-
-n = 0
-
-h = pwn.log.waitfor('spinners running', status = '0')
-
-hs = []
-while True:
-    s = raw_input('> ')
-    if s == 'q':
-        break
-    hs.append(pwn.log.waitfor(s, status = 'running'))
-    n += 1
-    h.status(str(n))
-
-h.success()
-
-for h in hs:
-    h.failure()
-
-# pwn.log.waitfor('waiting for stuff')
-# time.sleep(0.5)
-# pwn.log.info('something happened')
-# time.sleep(0.5)
-# pwn.log.success('something went well')
-# time.sleep(0.5)
-# pwn.log.waitfor('waiting for some minor stuff')
-# time.sleep(0.5)
-# pwn.log.failure('went wrong')
-# raw_input('please hit <enter> ')
-# time.sleep(0.5)
-# pwn.log.done_failure()
-# time.sleep(0.5)
-# pwn.log.info('all in all it went ok')
-# time.sleep(0.5)
-# pwn.log.done_success()
-
-
-# def foo():
-#     plib.log.stub()
-
-# foo()
-
-# try:
-#     try:
-#         raise KeyError
-#     except:
-#         plib.log.die('no reason')
-# except Exception as e:
-#     print `e`
-#     # raise

examples/memleaktest.py

@@ -1,12 +0,0 @@
-from pwn import *
-
-context(endianness='little', sign='unsigned')
-
-def legit_leak(addr):
-    return p32(addr)
-
-mem = MemLeak(legit_leak)
-print mem.b(0x08040001) # prints 1
-mem.setd(0x08040000, 0xaabbccdd)
-print hex(mem.b(0x08040001))   # prints 0xcc
-print mem.raw(0x08040000, 20)  # prints stuff

examples/options.py

@@ -0,0 +1,8 @@
+"""
+Example showing `pwnlib.ui.options()`
+"""
+
+from pwn import *
+
+opts = [string.letters[x] for x in range(10)]
+print 'You choose "%s"' % opts[options('Pick one:', opts)]

examples/port_forward.py

@@ -0,0 +1,11 @@
+"""
+A very simple port forwarder using `pwnlib.tubes.tube.connect_both()`.  Notice
+that `<>` is just a shorthand.
+"""
+
+from pwn import *
+
+while True:
+    listen(1337).wait_for_connection() <> remote('google.com', 80)
+
+# now point your browser (or curl(1)) to http://localhost:1337

examples/ptytest.py

@@ -1,37 +0,0 @@
-import pty, os, select, sys, time, pwn
-
-pid, fd = pty.fork()
-
-if pid == 0:
-    argv = ('/usr/bin/vim',)
-    argv = ('/bin/bash',)
-    os.execlp(argv[0], *argv)
-else:
-    # time.sleep(1)
-    # sys.stdout.write(os.read(fd, 4096))
-    while True:
-        rfds, wfds, xfds = select.select([fd, 0], [], [])
-        if fd in rfds:
-            s = os.read(fd, 4096)
-            if not s:
-                raise EOFError
-            sys.stdout.write(s)
-            sys.stdout.flush()
-        if 0 in rfds:
-            s = os.read(0, 4096)
-            if not s:
-                raise EOFError
-            os.write(fd, s)
-
-
-# def master_read(fd):
-#     import os
-#     s = os.read(fd, 4096)
-#     return s
-
-# def stdin_read(fd):
-#     import os
-#     s = os.read(fd, 4096)
-#     return s
-
-# pty.spawn('/bin/sh', master_read = master_read, stdin_read = stdin_read)

examples/readline_completers.py

@@ -0,0 +1,31 @@
+"""
+Example showing pwnlib's readline implementation and a few completers.  This
+part of pwnlib will probably see some major changes soon, but we wanted to show
+off some proof-of-concepts.
+"""
+
+from pwn import *
+from pwnlib.term.completer import LongestPrefixCompleter, \
+                                  PathCompleter
+
+c1 = LongestPrefixCompleter([
+    'foobar',
+    'foobaz',
+    'fooqux',
+    'exit',
+    'enough!',
+    ])
+
+c2 = PathCompleter(mask = '*.py')
+
+with c1:
+    print 'type "exit" to exit'
+    while True:
+        s = term.readline.readline(prompt = '> ').strip()
+        if s in ('exit', 'enough!'):
+            break
+        print 'You wrote', s
+with c2:
+    print 'choose a file'
+    s = term.readline.readline(prompt = text.bold_green('$ ')).strip()
+    print 'You picked', s

examples/readlineexc.py

@@ -1,15 +0,0 @@
-
-# import sys
-# class Wrapper:
-#     def __init__(self, fd):
-#         self._fd = fd
-#     def readline(self, size = None):
-#         return ''
-#     def __getattr__(self, k):
-#         return self._fd.__getattribute__(k)
-# sys.stdin = Wrapper(sys.stdin)
-# print `raw_input()`
-
-import readline
-import pwn
-raw_input('> ')

examples/remote.py

@@ -0,0 +1,11 @@
+"""
+Example showing how to use the remote class.
+"""
+
+from pwn import *
+
+sock = remote('127.0.0.1', 9001)
+
+print sock.recvline()
+sock.send('foo')
+sock.interactive()

examples/remote_gdb_debugging.py

@@ -0,0 +1,16 @@
+"""
+Simple example showing how to use the remote
+gdb debugging features available in pwntools.
+"""
+
+from pwn import *
+import getpass
+
+s = ssh(getpass.getuser(), '127.0.0.1', port = 22, keyfile = "~/.ssh/id_rsa")
+c = gdb.ssh_gdb(s, '/bin/sh', execute = '''
+p/x $pc
+c''')
+
+c.sendline('ls -la')
+c.sendline('exit')
+print c.recvall()

examples/spinners.py

@@ -0,0 +1,25 @@
+"""
+Just a lot of spinners!
+"""
+
+from pwn import *
+
+context.log_level = 0
+
+n = 1
+h = log.waitfor('spinners running', status = str(n))
+
+hs = []
+print 'type "q" to quit'
+while True:
+    s = raw_input('> ')
+    if s == 'q':
+        break
+    hs.append(log.waitfor(s, status = 'running'))
+    n += 1
+    h.status(str(n))
+
+h.success()
+
+for h in hs:
+    h.failure()

examples/splash.py

@@ -1,9 +1,12 @@
-import pwn
+"""
+"Easteregg"
+"""
 
-pwn.splash()
-pwn.pause()
+from pwn import *
 
-h = pwn.log.waitfor("Inputting:")
+splash()
+
+h = log.waitfor("You wrote", status = "--")
 
 while True:
     l = raw_input('> ')

examples/ssh.py

@@ -0,0 +1,40 @@
+"""
+Example showing how to use the ssh class.
+"""
+
+from pwn import *
+
+shell = ssh(host='bandit.labs.overthewire.org',user='bandit0',password='bandit0')
+
+# Show basic command syntax
+log.info("username: %s" % shell.whoami())
+log.info("pwd: %s" % shell.pwd())
+
+# Show full tube syntax
+tube = shell.run('cat')
+tube.send("Hello, cat")
+tube.shutdown("out")
+print tube.recvall()
+
+# Show automatic working directories
+shell.set_working_directory()
+log.info("pwd: %s" % shell.pwd())
+
+shell.upload_data("""
+#include <stdio.h>
+int main() {
+    return printf("Hello, world");
+}
+""", 'example.c')
+
+shell.gcc(['example.c','-o','example'])
+
+print shell['./example']
+
+# Show the different styles of calling
+print shell.echo("single string")
+print shell.echo(["list","of","strings"])
+print shell["echo single statement"]
+
+# Show off the interactive shell
+shell.interactive()

examples/termtest.py

@@ -1,93 +0,0 @@
-import pwn
-import pwnlib.term
-from pwnlib.term import *
-import time, os, sys, threading
-
-from pwnlib.log import *
-
-try:
-    # output(os.urandom(10))
-    # print 'foo'
-    # print 'bar'
-    # h = output('')
-    # for i in range(250):
-    #     time.sleep(0.005)
-    #     print 'foo%d' % i
-    #     # print 'xx'
-    #     # time.sleep(1)
-    #     # update(h, 'hello \x1b[3%dmthere\x1b[m\n' % (i % 6 + 1))
-    # # freeze(h)
-    # # print pwn2.nonlib.term.parse('hello \x1b[32mthere\x1b[m\n')
-    # # output('hello \x1b[32mthere\x1b[m\n')
-    # # update(h, 'asdf')
-    # print 'hello'
-
-    # output('fooxxx')
-    # h1 = output('')
-    # h2 = output('bar')
-    # output(' baz\n')
-
-    # # update(h2, 'BAR')
-    # update(h1, '\x1b[K\x1b[2A\x1b[1;31mYOLO\x1b[m\x1b[2B')
-
-    # print 'fooxxx\x1b[2Dbar', 'baz'
-
-    clock = output(float = True, frozen = False)
-    def loop():
-        while True:
-            clock.update('\n' + time.asctime(time.localtime()) + '\n')
-            time.sleep(1)
-    t = threading.Thread(target = loop)
-    t.daemon = True
-    t.start()
-    i = 0
-    # numlines = output()
-    numlines = info(frozen = False)
-    while True:
-        s = ''
-        numlines.update('You have written %d lines' % i)
-        i += 1
-        s = raw_input('> ')
-        output('you wrote: ')
-        output('%s\n' % s, indent = 20)
-        if s == 'q':
-            break
-
-    # output('\xc3\xb8' * 85)
-    # output('\x1b[44m \x1b[m')
-    # # output('\x1b[J')
-    # time.sleep(1)
-
-    # h = output('last\n', float = True)
-    # output('foo')
-    # # update(h, 'LAST\n')
-    # h = output('bar')
-    # output('baz\n')
-    # # delete(h)
-    # update(h, 'BAR')
-
-
-    # output('\x1b[s\x1b[20;40H')
-    # h = output('yolo')
-    # update(h, 'YOLO\x1b[u')
-
-    # output('\n')
-    # pwn2.nonlib.term.goto((1, 0))
-    # pwn2.nonlib.term.put('%d' % pwn2.nonlib.term.scroll)
-    # pwn2.nonlib.term.flush()
-    # time.sleep(1)
-
-    # pwn2.nonlib.term.put('\n'*10)
-
-    pass
-
-except:
-    # pwn2.nonlib.term.fini()
-    # raise e
-    # pwn2.nonlib.term.put(str(e))
-    pass
-    raise
-
-pwn2.nonlib.term.put('\n' * 20)
-for c in pwn2.nonlib.term.cells:
-    pwn2.nonlib.term.put('%s %s %s %s\n' % (c.start, c.end, c.float, `c.content`))

examples/text.py

@@ -0,0 +1,19 @@
+'''
+Example showing how to use `pwnlib.term.text`.
+
+Try running with::
+
+  $ TERM=xterm python text.py
+
+and::
+
+  $ TERM=xterm-256color python text.py
+'''
+from pwn import *
+
+s = 'hello from pwntools'
+print text.black_on_green(s)
+print text.black_on_bright_green(s)
+print text.green_on_black(s)
+print text.bright_green_on_black(s)
+print text.bold_green_on_black(s)

examples/yesno.py

@@ -0,0 +1,10 @@
+"""
+Example showing `pwnlib.ui.yesno()`
+"""
+
+from pwn import *
+
+if ui.yesno('Do you like Pwntools?'):
+    print ':D'
+else:
+    print ':('

extra/bash_completion.d/shellcraft

@@ -0,0 +1,46 @@
+# cache list of shellcodes
+_shellcraft_shellcodes=$(shellcraft)
+
+_shellcraft()
+{
+    COMPREPLY=()
+    local cur prev opts word code
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    opts="-h --help -? --show -o --out -f --format"
+
+    if [ "${prev}" == "-f" -o    \
+         "${prev}" == "--format" \
+        ] ; then
+        opts="raw str c hex asm p hexii"
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}))
+        return 0
+    fi
+
+    if [ "${prev}" == "-o" -o \
+         "${prev}" == "--out" \
+        ] ; then
+        COMPREPLY=( $(compgen -f ${cur}))
+        return 0
+    fi
+
+    if [[ "${cur}" == -* ]] ; then
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+        return 0
+    fi
+
+    # check if a shellcode was already chosen
+    for word in "${COMP_WORDS[@]}" ; do
+        for code in ${_shellcraft_shellcodes} ; do
+            if [ "${word}" == "${code}" ] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+                return 0
+            fi
+        done
+    done
+
+    # complete shellcode name
+    COMPREPLY=( $(compgen -W "${_shellcraft_shellcodes}" -- ${cur}) )
+}
+
+complete -F _shellcraft shellcraft

install_local.sh

@@ -1,12 +0,0 @@
-#!/bin/sh
-sudo pip install -r requirements.txt
-
-echo "================================================"
-echo "Now please put this directory in your PYTHONPATH"
-echo "and the bin directory in your PATH."
-echo
-echo "If you are using bash, you do can this by putting"
-echo "These in your .bashrc:"
-echo
-echo "export PATH=\"`realpath bin`:\$PATH\""
-echo "export PYTHONPATH=\"`realpath .`:\$PYTHONPATH\""

pwn/__init__.py

@@ -1,15 +1,10 @@
 # Promote useful stuff to toplevel
-import sys
 from .toplevel import *
 
-# Put the terminal in raw mode
-term.init()
-
-# default log level in non-lib mode
-context.defaults.log_level = 'info'
 
 # look for special args in argv
 def closure():
+    term_mode = True
     import sys
     if not hasattr(sys, 'argv'):
         return
@@ -26,18 +21,99 @@ def closure():
         if not all(c in body for c in s[1:]):
             return False
         return True
-    for arg in sys.argv[:]:
+    def asbool(s):
+        if   s.lower() == 'true':
+            return True
+        elif s.lower() == 'false':
+            return False
+        elif s.isdigit():
+            return bool(int(s))
+        else:
+            raise ValueError('must be integer or boolean')
+    # parse environtment variables
+    for k, v in os.environ.items():
+        if not k.startswith('PWNLIB_'):
+            continue
+        k = k[7:]
+        if   k == 'DEBUG':
+            if asbool(v):
+                context.log_level = 'DEBUG'
+        elif k == 'SILENT':
+            if asbool(v):
+                context.log_level = 'ERROR'
+        elif k == 'NOTERM':
+            if asbool(v):
+                term_mode = False
+        elif isident(k):
+            args[k] = v
+    # parse command line
+    # save a copy of argv for the log file header (see below)
+    argv = sys.argv[:]
+    for arg in argv:
         if   arg == 'DEBUG':
             sys.argv.remove(arg)
-            context.log_level = 'debug'
-        elif arg == 'NOINFO':
+            context.log_level = 'DEBUG'
+        elif arg == 'SILENT':
             sys.argv.remove(arg)
-            context.log_level = 'silent'
+            context.log_level = 'ERROR'
+        elif arg == 'NOTERM':
+            term_mode = False
         elif arg.find('=') > 0:
             k, v = arg.split('=', 1)
             if not isident(k):
                 continue
             sys.argv.remove(arg)
             args[k] = v
+    if 'LOG_LEVEL' in args:
+        context.log_level = args['LOG_LEVEL']
+    if 'LOG_FILE' in args:
+        # install a file logger
+        import logging, time
+        modes = ('w', 'wb', 'a', 'ab')
+        filename = args['LOG_FILE']
+        mode = 'a'
+        # check if mode was specified as "[filename],[mode]"
+        if ',' in filename:
+            filename_, mode_ = filename.rsplit(',', 1)
+            if mode in modes:
+                filename = filename_
+                mode = mode_
+        # ISO 8601
+        dfmt = '%Y-%m-%dT%H:%M:%S'
+        # write a "header" to the file, which makes it easier to find the start
+        # of a session
+        with open(filename, mode) as fd:
+            lines = [
+                '=' * 78,
+                ' Started at %s ' % time.strftime(dfmt),
+                ' sys.argv = [',
+                ]
+            for arg in argv:
+                lines.append('   %r,' % arg)
+            lines.append(' ]')
+            lines.append('=' * 78)
+            for line in lines:
+                fd.write('=%s=\n' % line.ljust(78))
+        # if the mode was 'w' or 'wb' we need to change it to 'a'/'ab' now so
+        # the logging module wont overwrite the header
+        mode = mode.replace('w', 'a')
+        # create a formatter and a handler and install them for the pwnlib root
+        # logger (i.e. 'pwnlib')
+        handler = logging.FileHandler(filename, mode)
+        fmt = '%(asctime)s:%(levelname)s:%(name)s:%(message)s'
+        formatter = logging.Formatter(fmt, dfmt)
+        handler.setFormatter(formatter)
+        logging.root.addHandler(handler)
+    # put the terminal in rawmode unless NOTERM was specified
+    if term_mode:
+        term.init()
+    # install a log handler and turn logging all the way up
+    import pwnlib.log as log
+    import logging
+    log.rootlogger.setLevel(logging.DEBUG)
+    log.install_default_handler()
+
 closure()
 del closure
+
+log = getLogger('pwnlib.exploit')

pwn/toplevel.py

@@ -1,26 +1,67 @@
+# Promote these modules, so that "from pwn import *" will let you access them
+import collections
+import math
+import operator
+import os
+import re
+import string
+import struct
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+
+import pwnlib
+
 # Get all the modules from pwnlib
-from pwnlib                 import *
+from pwnlib import *
 
 # Promote functions from these modules to toplevel
-from pwnlib.asm             import *
-from pwnlib.dynelf          import DynELF
-from pwnlib.elf             import ELF, load
-from pwnlib.log_levels      import *
-from pwnlib.memleak         import MemLeak
-from pwnlib.tubes.listen    import *
-from pwnlib.tubes.process   import *
-from pwnlib.tubes.remote    import *
-from pwnlib.tubes.ssh       import *
-from pwnlib.ui              import *
-from pwnlib.util            import crc, net, proc
-from pwnlib.util.cyclic     import *
-from pwnlib.util.fiddling   import *
-from pwnlib.util.hashes     import *
-from pwnlib.util.lists      import *
-from pwnlib.util.misc       import *
-from pwnlib.util.packing    import *
-from pwnlib.util.proc       import pidof
-from pwnlib.util.splash     import *
+from pwnlib.asm import asm
+from pwnlib.asm import cpp
+from pwnlib.asm import disasm
+from pwnlib.context import Thread
+from pwnlib.context import context
+from pwnlib.dynelf import DynELF
+from pwnlib.elf import ELF
+from pwnlib.elf import load
+from pwnlib.exception import PwnlibException
+from pwnlib.fmtstr import FmtStr
+from pwnlib.log import getLogger
+from pwnlib.memleak import MemLeak
+from pwnlib.regsort import *
+from pwnlib.replacements import *
+from pwnlib.rop import ROP
+from pwnlib.timeout import Timeout
+from pwnlib.tubes.listen import listen
+from pwnlib.tubes.process import process
+from pwnlib.tubes.remote import remote
+from pwnlib.tubes.serialtube import serialtube
+from pwnlib.tubes.ssh import ssh
+from pwnlib.tubes.tube import tube
+from pwnlib.ui import *
+from pwnlib.util import crc
+from pwnlib.util import iters
+from pwnlib.util import net
+from pwnlib.util import proc
+from pwnlib.util import safeeval
+from pwnlib.util.cyclic import *
+from pwnlib.util.fiddling import *
+from pwnlib.util.hashes import *
+from pwnlib.util.lists import *
+from pwnlib.util.misc import *
+from pwnlib.util.packing import *
+from pwnlib.util.proc import pidof
+from pwnlib.util.splash import *
+from pwnlib.util.web import *
 
-# Promote these modules, so that "from pwn import *" will let you access them
-import pwn, pwnlib, re, time, sys, os, string, threading, subprocess, struct
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO

pwnlib/__init__.py

@@ -1,12 +1,35 @@
-__all__ = [
-    'asm', 'constants', 'context', 'dynelf',
-    'elf', 'exception', 'gdb', 'log_levels',
-    'log', 'memleak', 'shellcraft', 'term',
-    'tubes', 'ui', 'util'
-]
-from . import asm, constants, context, dynelf
-from . import elf, exception, gdb, log_levels
-from . import log, memleak, shellcraft, term
-from . import tubes, ui, util
+import importlib
 
 from .version import __version__
+
+version = __version__
+
+__all__ = [
+    'asm',
+    'atexception',
+    'atexit',
+    'commandline',
+    'constants',
+    'context',
+    'dynelf',
+    'elf',
+    'exception',
+    'fmtstr',
+    'gdb',
+    'libcdb',
+    'log',
+    'memleak',
+    'pep237',
+    'regsort',
+    'replacements',
+    'rop',
+    'shellcraft',
+    'term',
+    'tubes',
+    'ui',
+    'useragents',
+    'util'
+]
+
+for module in __all__:
+    importlib.import_module('.%s' % module, 'pwnlib')

pwnlib/asm.py

@@ -1,118 +1,284 @@
-from . import context, log
-import tempfile, subprocess, shutil, tempfile, errno
+# -*- coding: utf-8 -*-
+r"""
+Utilities for assembling and disassembling code.
+
+Architecture Selection
+------------------------
+
+    Architecture, endianness, and word size are selected by using :mod:`pwnlib.context`.
+
+    Any parameters which can be specified to ``context`` can also be specified as
+    keyword arguments to either :func:`asm` or :func:`disasm`.
+
+Assembly
+------------------------
+
+    To assemble code, simply invoke :func:`asm` on the code to assemble.
+
+        >>> asm('mov eax, 0')
+        '\xb8\x00\x00\x00\x00'
+
+    Additionally, you can use constants as defined in the :mod:`pwnlib.constants`
+    module.
+
+        >>> asm('mov eax, SYS_execve')
+        '\xb8\x0b\x00\x00\x00'
+
+    Finally, :func:`asm` is used to assemble shellcode provided by ``pwntools``
+    in the :mod:`shellcraft` module.
+
+        >>> asm(shellcraft.sh())
+        'jhh///sh/bin\x89\xe31\xc9j\x0bX\x99\xcd\x80'
+
+Disassembly
+------------------------
+
+    To disassemble code, simply invoke :func:`disasm` on the bytes to disassemble.
+
+    >>> disasm('\xb8\x0b\x00\x00\x00')
+    '   0:   b8 0b 00 00 00          mov    eax,0xb'
+
+"""
+import errno
+import os
+import platform
+import re
+import shutil
+import string
+import subprocess
+import sys
+import tempfile
+from collections import defaultdict
+from glob import glob
+from os import environ
 from os import path
 
-__all__ = ['asm', 'cpp', 'disasm']
+from .context import context
+from .log import getLogger
+
+log = getLogger(__name__)
+
+__all__ = ['asm', 'cpp', 'disasm', 'make_elf']
 
 _basedir = path.split(__file__)[0]
-_bindir  = path.join(_basedir, 'data', 'binutils')
 _incdir  = path.join(_basedir, 'data', 'includes')
 
-def _assembler(arch):
+def which_binutils(util, **kwargs):
+    """
+    Finds a binutils in the PATH somewhere.
+    Expects that the utility is prefixed with the architecture name.
+
+    Examples:
+
+        >>> import platform
+        >>> which_binutils('as', arch=platform.machine())
+        '.../bin/as'
+        >>> which_binutils('as', arch='arm') #doctest: +ELLIPSIS
+        '.../bin/arm-...-as'
+        >>> which_binutils('as', arch='powerpc') #doctest: +ELLIPSIS
+        '.../bin/powerpc...-as'
+        >>> which_binutils('as', arch='msp430') #doctest: +SKIP
+        ...
+        Traceback (most recent call last):
+        ...
+        Exception: Could not find 'as' installed for ContextType(arch = 'msp430')
+    """
+    with context.local(**kwargs):
+        arch = context.arch
+        bits = context.bits
+
+        # Fix up pwntools vs Debian triplet naming, and account
+        # for 'thumb' being its own pwntools architecture.
+        arches = [arch] + {
+            'thumb':  ['arm',    'aarch64'],
+            'i386':   ['x86_64', 'amd64'],
+            'amd64':  ['x86_64', 'i386'],
+        }.get(arch, [])
+
+        # If one of the candidate architectures matches the native
+        # architecture, use that as a last resort.
+        machine = platform.machine()
+        try:
+            with context.local(arch = machine):
+                if context.arch in arches:
+                    arches.append(None)
+        except AttributeError:
+            log.warn_once("Your local binutils won't be used because architecture %r is not supported." % machine)
+
+        for arch in arches:
+            # hack for homebrew-installed binutils on mac
+            for gutil in ['g'+util, util]:
+                # e.g. objdump
+                if arch is None: pattern = gutil
+
+                # e.g. aarch64-linux-gnu-objdump
+                else:       pattern = '%s*linux*-%s' % (arch,gutil)
+
+                for dir in environ['PATH'].split(':'):
+                    res = sorted(glob(path.join(dir, pattern)))
+                    if res:
+                        return res[0]
+
+        locals()['context'] = context
+        log.warning("""
+Could not find %(util)r installed for %(context)s
+Try installing binutils for this architecture:
+    https://pwntools.readthedocs.org/en/latest/install/binutils.html
+""".strip() % locals())
+        raise Exception('Could not find %(util)r installed for %(context)s' % locals())
+
+checked_assembler_version = defaultdict(lambda: False)
+
+def _assembler():
+    gas = which_binutils('as')
+
+    E = {
+        'big':    '-EB',
+        'little': '-EL'
+    }[context.endianness]
+
+    B = '-%s' % context.bits
+
     assemblers = {
-        'i386'   : ['nasm', '-Ox'],
-        'amd64'  : ['nasm', '-Ox'],
-        'thumb'  : [path.join(_bindir, 'arm-as')],
-        'mips'   : [path.join(_bindir, 'mips-as' ), '--EB'],
-        'mipsel' : [path.join(_bindir, 'mips-as' ), '--EL'],
+        'i386'   : [gas, B],
+        'amd64'  : [gas, B],
+
+        # Most architectures accept -EL or -EB
+        'thumb'  : [gas, '-mthumb', E],
+        'arm'    : [gas, E],
+        'aarch64': [gas, E],
+        'mips'   : [gas, E, B],
+        'mips64' : [gas, E, B],
+        'sparc':   [gas, E, B],
+        'sparc64': [gas, E, B],
+
+        # Powerpc wants -mbig or -mlittle, and -mppc32 or -mppc64
+        'powerpc':   [gas, '-m%s' % context.endianness, '-mppc%s' % context.bits],
+        'powerpc64': [gas, '-m%s' % context.endianness, '-mppc%s' % context.bits],
+
+        # ia64 only accepts -mbe or -mle
+        'ia64':    [gas, '-m%ce' % context.endianness[0]]
     }
 
-    if arch in assemblers:
-        return assemblers[arch]
-    else:
-        return [path.join(_bindir, arch + '-as')]
+    assembler = assemblers.get(context.arch, [gas])
+
+    if not checked_assembler_version[gas]:
+        checked_assembler_version[gas] = True
+        result = subprocess.check_output([gas, '--version','/dev/null'],
+                                         stderr=subprocess.STDOUT)
+        version = re.search(r' (\d\.\d+)', result).group(1)
+        if version < '2.19':
+            log.warn_once('Your binutils version is too old and may not work!\n'  + \
+                'Try updating with: https://pwntools.readthedocs.org/en/latest/install/binutils.html\n' + \
+                'Reported Version: %r' % result.strip())
 
 
-def _objcopy(arch):
-    if arch in ['i386', 'amd64']:
-        return 'objcopy'
-    else:
-        return path.join(_bindir, 'objcopy')
+    return assembler
 
+def _linker():
+    ld  = [which_binutils('ld')]
+    bfd = ['--oformat=' + _bfdname()]
 
-def _objdump(arch):
-    if arch in ['i386', 'amd64']:
-        return ['objdump', '-Mintel']
-    else:
-        return [path.join(_bindir, 'objdump')]
+    E = {
+        'big':    '-EB',
+        'little': '-EL'
+    }[context.endianness]
 
+    arguments = {
+        'i386': ['-m', 'elf_i386'],
+    }.get(context.arch, [])
+
+    return ld + bfd + [E] + arguments
+
+def _objcopy():
+    return [which_binutils('objcopy')]
+
+def _objdump():
+    path = [which_binutils('objdump')]
+
+    if context.arch in ('i386', 'amd64'):
+        path += ['-Mintel']
+
+    return path
+
+def _include_header():
+    os   = context.os
+    arch = context.arch
 
-def _include_header(arch, os):
     if os == 'freebsd':
-        return '#include <freebsd.h>\n'
-    elif os == 'linux' and arch:
-        return '#include <linux/%s.h>\n' % arch
-    else:
-        return ''
+        include = 'freebsd.h'
+    elif os == 'linux':
+        include = 'linux/%s.h' % arch
+
+    if not include or not path.exists(path.join(_incdir, include)):
+        log.warn_once("Could not find system include headers for %s-%s" % (arch,os))
+        return '\n'
+
+    return '#include <%s>\n' % include
 
 
-def _arch_header(arch):
+def _arch_header():
+    prefix  = ['.section .shellcode,"ax"']
     headers = {
-        'i386'  : ['bits 32'],
-        'amd64' : ['bits 64'],
-        'arm'   : [
-            '.section .shellcode,"ax"',
-            '.syntax unified',
-            '.arch armv7-a',
-            '.arm'
-        ],
-        'thumb' : [
-            '.section .shellcode,"ax"',
-            '.syntax unified',
-            '.arch armv7-a',
-            '.thumb'
-        ],
-        'mips'  : [
-            '.section .shellcode,"ax"',
-            '.set mips2',
-            '.set noreorder',
-        ],
-        'mipsel'  : [
-            '.section .shellcode,"ax"',
-            '.set mips2',
-            '.set noreorder',
-        ],
+        'i386'  :  ['.intel_syntax noprefix'],
+        'amd64' :  ['.intel_syntax noprefix'],
+        'arm'   : ['.syntax unified',
+                   '.arch armv7-a',
+                   '.arm'],
+        'thumb' : ['.syntax unified',
+                   '.arch armv7-a',
+                   '.thumb'],
+        'mips'  : ['.set mips2',
+                   '.set noreorder'],
     }
 
-    if arch in headers:
-        return '\n'.join(headers[arch]) + '\n'
-    else:
-        return ''
+    return '\n'.join(prefix + headers.get(context.arch, [])) + '\n'
+
+def _bfdname():
+    arch = context.arch
+    E    = context.endianness
 
-def _bfdname(arch):
     bfdnames = {
         'i386'    : 'elf32-i386',
+        'aarch64' : 'elf64-%saarch64' % E,
         'amd64'   : 'elf64-x86-64',
-        'arm'     : 'elf32-littlearm',
-        'thumb'   : 'elf32-littlearm',
-        'mips'    : 'elf32-bigmips',
-        'mipsel'  : 'elf32-littlemips',
+        'arm'     : 'elf32-%sarm' % E,
+        'thumb'   : 'elf32-%sarm' % E,
+        'mips'    : 'elf32-trad%smips' % E,
+        'mips64'  : 'elf64-trad%smips' % E,
         'alpha'   : 'elf64-alpha',
         'cris'    : 'elf32-cris',
-        'ia64'    : 'elf64-ia64-little',
+        'ia64'    : 'elf64-ia64-%s' % E,
         'm68k'    : 'elf32-m68k',
         'powerpc' : 'elf32-powerpc',
+        'powerpc64' : 'elf64-powerpc',
         'vax'     : 'elf32-vax',
+        'sparc'   : 'elf32-sparc',
+        'sparc64' : 'elf64-sparc',
     }
 
     if arch in bfdnames:
         return bfdnames[arch]
     else:
-        log.error("Cannot find bfd name for architecture %r" % arch)
+        raise Exception("Cannot find bfd name for architecture %r" % arch)
 
 
-def _bfdarch(arch):
-    if arch == 'amd64':
-        return 'i386:x86-64'
-    elif arch == 'thumb':
-        return 'arm'
-    elif arch == 'ia64':
-        return 'ia64-elf64'
-    else:
-        return arch
+def _bfdarch():
+    arch = context.arch
+    convert = {
+    'i386': 'i386',
+    'amd64': 'i386:x86-64',
+    'thumb': 'arm',
+    'ia64': 'ia64-elf64'
+    }
 
+    if arch in convert:
+        return convert[arch]
+
+    return arch
 
 def _run(cmd, stdin = None):
+    log.debug(subprocess.list2cmdline(cmd))
     try:
         proc = subprocess.Popen(
             cmd,
@@ -124,60 +290,140 @@ def _run(cmd, stdin = None):
         exitcode = proc.wait()
     except OSError as e:
         if e.errno == errno.ENOENT:
-            log.error('Could not run %r the program' % cmd[0])
+            log.exception('Could not run %r the program' % cmd[0])
         else:
             raise
 
     if (exitcode, stderr) != (0, ''):
-        msg = 'There was an error running %s:\n' % repr(cmd[0])
+        msg = 'There was an error running %s:\n' % repr(cmd)
         if exitcode != 0:
             msg += 'It had the exitcode %d.\n' % exitcode
         if stderr != '':
             msg += 'It had this on stdout:\n%s\n' % stderr
         log.error(msg)
 
-
     return stdout
 
-def cpp(shellcode, arch = None, os = None):
-    """cpp(shellcode, arch = None, os = None) -> str
+def cpp(shellcode, **kwargs):
+    r"""cpp(shellcode, ...) -> str
 
     Runs CPP over the given shellcode.
 
     The output will always contain exactly one newline at the end.
 
+    Arguments:
+        shellcode(str): Shellcode to preprocess
+
+    Kwargs:
+        Any arguments/properties that can be set on ``context``
+
     Examples:
-      >>> cpp("testing: SYS_setresuid")
-      'testing: SYS_setresuid\\n'
-      >>> cpp("mov al, SYS_setresuid", arch = "i386", os = "linux")
-      'mov al, 164\\n'
-      >>> cpp("weee SYS_setresuid", arch = "arm", os = "linux")
-      'weee (0x900000+164)\\n'
-      >>> cpp("SYS_setresuid", arch = "thumb", os = "linux")
-      '(0+164)\\n'
-      >>> cpp("SYS_setresuid", os = "freebsd")
-      '311\\n'
+
+        .. doctest::
+
+            >>> cpp("mov al, SYS_setresuid", arch = "i386", os = "linux")
+            'mov al, 164\n'
+            >>> cpp("weee SYS_setresuid", arch = "arm", os = "linux")
+            'weee (0x900000+164)\n'
+            >>> cpp("SYS_setresuid", arch = "thumb", os = "linux")
+            '(0+164)\n'
+            >>> cpp("SYS_setresuid", os = "freebsd")
+            '311\n'
     """
 
-    arch = arch or context.arch
-    os   = os   or context.os
-    code = _include_header(arch, os) + shellcode
-    cmd  = [
-        'cpp',
-        '-C',
-        '-nostdinc',
-        '-undef',
-        '-P',
-        '-I' + _incdir,
-        '/dev/stdin'
-    ]
-    return _run(cmd, code).strip('\n').rstrip() + '\n'
+    with context.local(**kwargs):
+        arch = context.arch
+        os   = context.os
+        code = _include_header() + shellcode
+        cmd  = [
+            'cpp',
+            '-C',
+            '-nostdinc',
+            '-undef',
+            '-P',
+            '-I' + _incdir,
+            '/dev/stdin'
+        ]
+        return _run(cmd, code).strip('\n').rstrip() + '\n'
 
+elf_template = '''
+.global _start
+.global __start
+.text
+_start:
+__start:
+'''
 
-def asm(shellcode, arch = None, os = None):
-    """asm(code, arch = None, os = None) -> str
+def make_elf(data, vma = None, strip=True, **kwargs):
+    r"""
+    Builds an ELF file with the specified binary data as its
+    executable code.
 
-    Runs CPP over a given shellcode and then assembles it into bytes.
+    Arguments:
+        data(str): Assembled code
+        vma(int):  Load address for the ELF file
+
+    Examples:
+
+        This example creates an i386 ELF that just does
+        execve('/bin/sh',...).
+
+        >>> context.clear()
+        >>> context.arch = 'i386'
+        >>> context.bits = 32
+        >>> filename = tempfile.mktemp()
+        >>> bin_sh = '6a68682f2f2f73682f62696e89e331c96a0b5899cd80'.decode('hex')
+        >>> data = make_elf(bin_sh)
+        >>> with open(filename,'wb+') as f:
+        ...     f.write(data)
+        ...     f.flush()
+        >>> os.chmod(filename,0777)
+        >>> p = process(filename)
+        >>> p.sendline('echo Hello; exit')
+        >>> p.recvline()
+        'Hello\n'
+    """
+    with context.local(**kwargs):
+        assembler = _assembler()
+        linker    = _linker()
+        code      = elf_template
+        code      += '.string "%s"' % ''.join('\\x%02x' % c for c in bytearray(data))
+        code      += '\n'
+
+        log.debug(code)
+
+        tmpdir    = tempfile.mkdtemp(prefix = 'pwn-asm-')
+        step1     = path.join(tmpdir, 'step1-asm')
+        step2     = path.join(tmpdir, 'step2-obj')
+        step3     = path.join(tmpdir, 'step3-elf')
+
+        try:
+            with open(step1, 'wb+') as f:
+                f.write(code)
+
+            _run(assembler + ['-o', step2, step1])
+
+            load_addr = []
+            if vma is not None:
+                load_addr = ['-Ttext-segment=%#x' % vma]
+
+            _run(linker    + load_addr + ['-N', '-o', step3, step2])
+
+            if strip:
+                _run([which_binutils('objcopy'), '-Sg', step3])
+                _run([which_binutils('strip'), '--strip-unneeded', step3])
+
+            with open(step3, 'r') as f:
+                return f.read()
+        except Exception:
+            log.exception("An error occurred while building an ELF:\n%s" % code)
+        else:
+            shutil.rmtree(tmpdir)
+
+def asm(shellcode, vma = 0, **kwargs):
+    r"""asm(code, vma = 0, ...) -> str
+
+    Runs :func:`cpp` over a given shellcode and then assembles it into bytes.
 
     To see which architectures or operating systems are supported,
     look in :mod:`pwnlib.contex`.
@@ -185,65 +431,81 @@ def asm(shellcode, arch = None, os = None):
     To support all these architecture, we bundle the GNU assembler
     and objcopy with pwntools.
 
-    Args:
+    Arguments:
       shellcode(str): Assembler code to assemble.
-      arch: A supported architecture or None. In case of None,
-            :data:`pwnlib.context.arch` will be used.
-      os: A supported operating system or None. In case of None,
-          :data:`pwnlib.context.os` will be used.
+      vma(int):       Virtual memory address of the beginning of assembly
+
+    Kwargs:
+        Any arguments/properties that can be set on ``context``
 
     Examples:
-      >>> asm("mov eax, SYS_select", arch = 'i386', os = 'freebsd')
-      '\\xb8]\\x00\\x00\\x00'
-      >>> asm("mov rax, SYS_select", arch = 'amd64', os = 'linux')
-      '\\xb8\\x17\\x00\\x00\\x00'
-      >>> asm("ldr r0, =SYS_select", arch = 'arm', os = 'linux')
-      '\\x04\\x00\\x1f\\xe5R\\x00\\x90\\x00'
+
+        .. doctest::
+
+            >>> asm("mov eax, SYS_select", arch = 'i386', os = 'freebsd')
+            '\xb8]\x00\x00\x00'
+            >>> asm("mov eax, SYS_select", arch = 'amd64', os = 'linux')
+            '\xb8\x17\x00\x00\x00'
+            >>> asm("mov rax, SYS_select", arch = 'amd64', os = 'linux')
+            'H\xc7\xc0\x17\x00\x00\x00'
+            >>> asm("ldr r0, =SYS_select", arch = 'arm', os = 'linux', bits=32)
+            '\x04\x00\x1f\xe5R\x00\x90\x00'
     """
+    result = ''
 
-    arch = arch or context.arch
-    os   = os   or context.os
-    if not arch:
-        raise ValueError(
-            "asm() needs to get 'arch' through an argument or the context"
-        )
+    with context.local(**kwargs):
+        assembler = _assembler()
+        linker    = _linker()
+        objcopy   = _objcopy() + ['-j', '.shellcode', '-Obinary']
+        code      = ''
+        code      += _arch_header()
+        code      += cpp(shellcode)
 
-    tmpdir  = tempfile.mkdtemp(prefix = 'pwn-asm-')
-    def tmp(name):
-        return path.join(tmpdir, name)
+        log.debug('Assembling\n%s' % code)
 
-    assembler = _assembler(arch)
-    objcopy   = [_objcopy(arch), '-j.shellcode', '-Obinary']
-    code      = _arch_header(arch) + cpp(shellcode, arch, os)
+        tmpdir    = tempfile.mkdtemp(prefix = 'pwn-asm-')
+        step1     = path.join(tmpdir, 'step1')
+        step2     = path.join(tmpdir, 'step2')
+        step3     = path.join(tmpdir, 'step3')
+        step4     = path.join(tmpdir, 'step4')
 
-    try:
-        with open(tmp('step1'), 'w') as fd:
-            fd.write(code)
+        try:
+            with open(step1, 'w') as fd:
+                fd.write(code)
 
-        _run(assembler + ['-o', tmp('step2'), tmp('step1')])
+            _run(assembler + ['-o', step2, step1])
 
-        if arch in ['i386', 'amd64']:
-            with open(tmp('step2')) as fd:
-                return fd.read()
+            if not vma:
+                shutil.copy(step2, step3)
 
-        # Sanity check for seeing if the output has relocations
-        relocs = subprocess.check_output(
-            ['readelf', '-r', tmp('step2')]
-        ).strip()
-        if len(relocs.split('\n')) > 1:
-            raise Exception(
-                'There were relocations in the shellcode:\n\n%s' % relocs
-            )
+            if vma:
+                 _run(linker + ['--section-start=.shellcode=%#x' % vma,
+                                '--entry=%#x' % vma,
+                                '-o', step3, step2])
 
-        _run(objcopy + [tmp('step2'), tmp('step3')])
+            elif file(step2,'rb').read(4) == '\x7fELF':
+                # Sanity check for seeing if the output has relocations
+                relocs = subprocess.check_output(
+                    [which_binutils('readelf'), '-r', step2]
+                ).strip()
+                if len(relocs.split('\n')) > 1:
+                    log.error('Shellcode contains relocations:\n%s' % relocs)
+            else:
+                shutil.copy(step2, step3)
 
-        with open(tmp('step3')) as fd:
-            return fd.read()
-    finally:
-        shutil.rmtree(tmpdir)
+            _run(objcopy + [step3, step4])
 
-def disasm(data, arch = None):
-    """disasm(data, arch = None) -> str
+            with open(step4) as fd:
+                result = fd.read()
+
+        except Exception:
+            log.exception("An error occurred while assembling:\n%s" % code)
+        else:
+            shutil.rmtree(tmpdir)
+            return result
+
+def disasm(data, vma = 0, byte = True, offset = True, **kwargs):
+    """disasm(data, ...) -> str
 
     Disassembles a bytestring into human readable assembler.
 
@@ -253,62 +515,93 @@ def disasm(data, arch = None):
     To support all these architecture, we bundle the GNU objcopy
     and objdump with pwntools.
 
-    Args:
+    Arguments:
       data(str): Bytestring to disassemble.
-      arch: A supported architecture or None. In case of None,
-            :data:`pwnlib.context.arch` will be used.
+      vma(int): Passed through to the --adjust-vma argument of objdump
+      byte(bool): Include the hex-printed bytes in the disassembly
+      offset(bool): Include the virtual memory address in the disassembly
+
+    Kwargs:
+      Any arguments/properties that can be set on ``context``
 
     Examples:
-      >>> print disasm('b85d000000'.decode('hex'), arch = 'i386')
-         0:   b8 5d 00 00 00          mov    eax,0x5d
-      >>> print disasm('b817000000'.decode('hex'), arch = 'amd64')
-         0:   b8 17 00 00 00          mov    eax,0x17
-      >>> print disasm('04001fe552009000'.decode('hex'), arch = 'arm')
-         0:   e51f0004        ldr     r0, [pc, #-4]   ; 0x4
-         4:   00900052        addseq  r0, r0, r2, asr r0
-      >>> print disasm('4ff00500'.decode('hex'), arch = 'thumb')
-         0:   f04f 0005       mov.w   r0, #5
+
+        .. doctest::
+
+          >>> print disasm('b85d000000'.decode('hex'), arch = 'i386')
+             0:   b8 5d 00 00 00          mov    eax,0x5d
+          >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0)
+             0:   mov    eax,0x5d
+          >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0, offset = 0)
+          mov    eax,0x5d
+          >>> print disasm('b817000000'.decode('hex'), arch = 'amd64')
+             0:   b8 17 00 00 00          mov    eax,0x17
+          >>> print disasm('48c7c017000000'.decode('hex'), arch = 'amd64')
+             0:   48 c7 c0 17 00 00 00    mov    rax,0x17
+          >>> print disasm('04001fe552009000'.decode('hex'), arch = 'arm')
+             0:   e51f0004        ldr     r0, [pc, #-4]   ; 0x4
+             4:   00900052        addseq  r0, r0, r2, asr r0
+          >>> print disasm('4ff00500'.decode('hex'), arch = 'thumb', bits=32)
+             0:   f04f 0005       mov.w   r0, #5
+          >>>
     """
+    result = ''
 
-    arch = arch or context.arch
-    if not arch:
-        raise ValueError(
-            "asm() needs to get 'arch' through an argument or the context"
-        )
+    with context.local(**kwargs):
+        arch   = context.arch
+        os     = context.os
 
-    tmpdir = tempfile.mkdtemp(prefix = 'pwn-disasm-')
-    def tmp(name):
-        return path.join(tmpdir, name)
+        tmpdir = tempfile.mkdtemp(prefix = 'pwn-disasm-')
+        step1  = path.join(tmpdir, 'step1')
+        step2  = path.join(tmpdir, 'step2')
 
-    bfdarch = _bfdarch(arch)
-    bfdname = _bfdname(arch)
-    objdump = _objdump(arch) + ['-d']
-    objcopy = [
-        _objcopy(arch),
-        '-I', 'binary',
-        '-O', bfdname,
-        '-B', bfdarch,
-        '--set-section-flags', '.data=code',
-        '--rename-section', '.data=.text',
-    ]
+        bfdarch = _bfdarch()
+        bfdname = _bfdname()
+        objdump = _objdump() + ['-d', '--adjust-vma', str(vma), '-b', bfdname]
+        objcopy = _objcopy() + [
+            '-I', 'binary',
+            '-O', bfdname,
+            '-B', bfdarch,
+            '--set-section-flags', '.data=code',
+            '--rename-section', '.data=.text',
+        ]
 
-    if arch == 'thumb':
-        objcopy += ['--prefix-symbol=$t.']
-    else:
-        objcopy += ['-w', '-N', '*']
-
-    try:
-        with open(tmp('step1'), 'w') as fd:
-            fd.write(data)
-        _run(objcopy + [tmp('step1'), tmp('step2')])
-
-        output0 = subprocess.check_output(objdump + [tmp('step2')])
-        output1 = output0.split('<.text>:\n')
-        if len(output1) != 2:
-            raise IOError(
-                'Something went wrong with objdump:\n\n%s' % output0
-            )
+        if arch == 'thumb':
+            objcopy += ['--prefix-symbol=$t.']
         else:
-            return output1[1].strip('\n').rstrip().expandtabs()
-    finally:
-        shutil.rmtree(tmpdir)
+            objcopy += ['-w', '-N', '*']
+
+        try:
+
+            with open(step1, 'w') as fd:
+                fd.write(data)
+
+            res = _run(objcopy + [step1, step2])
+
+            output0 = subprocess.check_output(objdump + [step2])
+            output1 = output0.split('<.text>:\n')
+
+            if len(output1) != 2:
+                log.error('Could not find .text in objdump output:\n%s' % output0)
+
+            result = output1[1].strip('\n').rstrip().expandtabs()
+        except Exception:
+            log.exception("An error occurred while disassembling:\n%s" % data)
+        else:
+            shutil.rmtree(tmpdir)
+
+    lines = []
+    pattern = '^( *[0-9a-f]+: *)((?:[0-9a-f]+ )+ *)(.*)'
+    for line in result.splitlines():
+        o, b, i = re.search(pattern, line).groups()
+
+        line = ''
+
+        if offset:
+            line += o
+        if byte:
+            line += b
+        line += i
+        lines.append(line)
+
+    return '\n'.join(lines)

pwnlib/atexception.py

@@ -0,0 +1,104 @@
+"""
+Analogous to atexit, this module allows the programmer to register functions to
+be run if an unhandled exception occurs.
+"""
+
+import sys
+import threading
+import traceback
+
+from .context import context
+
+__all__ = ['register', 'unregister']
+
+_lock = threading.Lock()
+_ident = 0
+_handlers = {}
+
+def register(func, *args, **kwargs):
+    """register(func, *args, **kwargs)
+
+    Registers a function to be called when an unhandled exception occurs.  The
+    function will be called with positional arguments `args` and keyword
+    arguments `kwargs`, i.e. ``func(*args, **kwargs)``.  The current `context`
+    is recorded and will be the one used when the handler is run.
+
+    E.g. to suppress logging output from an exception-handler one could write::
+
+      with context.local(log_level = 'error'):
+        atexception.register(handler)
+
+    An identifier is returned which can be used to unregister the
+    exception-handler.
+
+    This function can be used as a decorator::
+
+      @atexception.register
+      def handler():
+        ...
+
+    Notice however that this will bind ``handler`` to the identifier and not the
+    actual exception-handler.  The exception-handler can then be unregistered
+    with::
+
+      atexception.unregister(handler)
+
+    This function is thread safe.
+
+    """
+    global _ident
+    with _lock:
+        ident = _ident
+        _ident += 1
+    _handlers[ident] = (func, args, kwargs, vars(context))
+    return ident
+
+def unregister(func):
+    """unregister(func)
+
+    Remove `func` from the collection of registered functions.  If `func` isn't
+    registered this is a no-op.
+    """
+    if func in _handlers:
+        del _handlers[func]
+
+def _run_handlers():
+    """_run_handlers()
+
+    Run registered handlers.  They run in the reverse order of which they were
+    registered.
+
+    If a handler raises an exception, it will be printed but nothing else
+    happens, i.e. other handlers will be run.
+    """
+    context.clear()
+    for _ident, (func, args, kwargs, ctx) in \
+        sorted(_handlers.items(), reverse = True):
+        try:
+            with context.local(**ctx):
+                func(*args, **kwargs)
+        except SystemExit:
+            pass
+        except:
+            # extract the current exception and rewind the traceback to where it
+            # originated
+            typ, val, tb = sys.exc_info()
+            traceback.print_exception(typ, val, tb.tb_next)
+
+# we rely on the existing excepthook to print exceptions
+if hasattr(sys, 'excepthook'):
+    _oldhook = sys.excepthook
+else:
+    _oldhook = None
+
+def _newhook(typ, val, tb):
+    """_newhook(typ, val, tb)
+
+    Our excepthook replacement.  First the original hook is called to print the
+    exception, then each handler is called.
+    """
+    if _oldhook:
+        _oldhook(typ, val, tb)
+    _run_handlers()
+
+sys.excepthook = _newhook

pwnlib/atexit.py

@@ -0,0 +1,96 @@
+"""
+Replacement for the Python standard library's atexit.py.
+
+Whereas the standard :mod:`atexit` module only defines :func:`atexit.register`,
+this replacement module also defines :func:`unregister`.
+
+This module also fixes a the issue that exceptions raised by an exit handler is
+printed twice when the standard :mod:`atexit` is used.
+"""
+
+import sys
+import threading
+import traceback
+
+from .context import context
+
+__all__ = ['register', 'unregister']
+
+_lock = threading.Lock()
+_ident = 0
+_handlers = {}
+
+def register(func, *args, **kwargs):
+    """register(func, *args, **kwargs)
+
+    Registers a function to be called on program termination.  The function will
+    be called with positional arguments `args` and keyword arguments `kwargs`,
+    i.e. ``func(*args, **kwargs)``.  The current `context` is recorded and will
+    be the one used when the handler is run.
+
+    E.g. to suppress logging output from an exit-handler one could write::
+
+      with context.local(log_level = 'error'):
+        atexit.register(handler)
+
+    An identifier is returned which can be used to unregister the exit-handler.
+
+    This function can be used as a decorator::
+
+      @atexit.register
+      def handler():
+        ...
+
+    Notice however that this will bind ``handler`` to the identifier and not the
+    actual exit-handler.  The exit-handler can then be unregistered with::
+
+      atexit.unregister(handler)
+
+    This function is thread safe.
+
+    """
+    global _ident
+    with _lock:
+        ident = _ident
+        _ident += 1
+    _handlers[ident] = (func, args, kwargs, vars(context))
+    return ident
+
+def unregister(ident):
+    """unregister(ident)
+
+    Remove the exit-handler identified by `ident` from the list of registered
+    handlers.  If `ident` isn't registered this is a no-op.
+    """
+    if ident in _handlers:
+        del _handlers[ident]
+
+def _run_handlers():
+    """_run_handlers()
+
+    Run registered exit-handlers.  They run in the reverse order of which they
+    were registered.
+
+    If a handler raises an exception, it will be printed but nothing else
+    happens, i.e. other handlers will be run and `sys.excepthook` will not be
+    called for that reason.
+    """
+    context.clear()
+    for _ident, (func, args, kwargs, ctx) in \
+        sorted(_handlers.items(), reverse = True):
+        try:
+            with context.local(**ctx):
+                func(*args, **kwargs)
+        except SystemExit:
+            pass
+        except:
+            # extract the current exception and rewind the traceback to where it
+            # originated
+            typ, val, tb = sys.exc_info()
+            traceback.print_exception(typ, val, tb.tb_next)
+
+# if there's already an exitfunc registered be sure to run that too
+if hasattr(sys, "exitfunc"):
+    register(sys.exitfunc)
+
+sys.exitfunc = _run_handlers

pwnlib/commandline/asm.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python2
+import argparse
+import sys
+
+from pwn import *
+
+from . import common
+
+parser = argparse.ArgumentParser(
+    description = 'Assemble shellcode into bytes'
+)
+
+parser.add_argument(
+    'lines',
+    metavar='line',
+    nargs='*',
+    help='Lines to assemble. If none are supplied, use stdin'
+)
+
+parser.add_argument(
+    "-f", "--format",
+    help="Output format (defaults to hex for ttys, otherwise raw)",
+    choices=['raw', 'hex', 'string']
+)
+
+parser.add_argument(
+    "-o","--output",
+    metavar='file',
+    help="Output file (defaults to stdout)",
+    type=argparse.FileType('w'),
+    default=sys.stdout
+)
+
+parser.add_argument(
+    '-c', '--context',
+    metavar = 'context',
+    action = 'append',
+    type   = common.context_arg,
+    choices = common.choices,
+    help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
+)
+
+
+parser.add_argument(
+    '-d',
+    '--debug',
+    help='Debug the shellcode with GDB',
+    action='store_true'
+)
+
+def main():
+    args   = parser.parse_args()
+    tty    = args.output.isatty()
+
+    data   = '\n'.join(args.lines) or sys.stdin.read()
+    output = asm(data.replace(';', '\n'))
+    fmt    = args.format or ('hex' if tty else 'raw')
+    formatters = {'r':str, 'h':enhex, 's':repr}
+
+    if args.debug:
+        proc = gdb.debug_shellcode(output, arch=context.arch)
+        proc.interactive()
+        sys.exit(0)
+
+    args.output.write(formatters[fmt[0]](output))
+
+    if tty and fmt is not 'raw':
+        args.output.write('\n')
+
+if __name__ == '__main__': main()

pwnlib/commandline/checksec.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python2
+import argparse
+import sys
+
+from pwn import *
+
+from . import common
+
+parser = argparse.ArgumentParser(
+    description = 'Check binary security settings'
+)
+
+parser.add_argument(
+    'elf',
+    nargs='+',
+    type=file,
+    help='Files to check'
+)
+
+def main():
+    args   = parser.parse_args()
+    for f in args.elf:
+        e = ELF(f.name)
+
+if __name__ == '__main__': main()