Merge pull request #625 from cebrusfs/patch-1

Fix wrong format string parameters in memleak.py

.gitignore

@@ -2,6 +2,7 @@
 *.hi
 *.pyc
 *~
+*.swp
 *.pdf
 */auto/*
 */bin/*

.isort.cfg

@@ -0,0 +1,4 @@
+[settings]
+indent='    '
+not_skip = __init__.py
+force_single_line = 1

.travis.yml

@@ -1,15 +1,23 @@
+sudo: false
 language: python
+cache:
+    - pip
+    - directories:
+        - usr
+        - /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages/
+        - /home/travis/virtualenv/python2.7.9/bin/
 python:
   - "2.7"
-branches:
-  only:
-    - master
-install:
-  - source .travis_install.sh
+before_install:
   - source .travis_ssh_setup.sh
+  - source .travis_install.sh
+install:
+  - pip install -e .
+  - pip install -r docs/requirements.txt
+before_script:
   - python -c 'from pwn import *; print pwnlib.term.term_mode' NOTERM
   - python -c 'from pwn import *; print pwnlib.term.term_mode' NOTERM
   - python -c 'from pwn import *; print pwnlib.term.term_mode' NOTERM
   - python -c 'from pwn import *; print pwnlib.term.term_mode' NOTERM
+script: PWNLIB_NOTERM=1 make -C docs doctest
 
-script: make -C docs doctest

.travis_install.sh

@@ -1,15 +1,49 @@
 #!/usr/bin/env bash
+local_deb_extract()
+{
+    wget $1
+    ar vx *.deb
+    tar xvf data.tar.gz
+    rm -f *.tar.gz *deb*
+}
+
+get_binutils()
+{
+    BINUTILS_PREFIX='https://launchpad.net/~pwntools/+archive/ubuntu/binutils/+files/binutils-'
+    BINUTILS_SUFFIX='-linux-gnu_2.22-6ubuntu1.1cross0.11pwntools12~precise_amd64.deb'
+    local_deb_extract "${BINUTILS_PREFIX}${1}${BINUTILS_SUFFIX}"
+}
+
+setup_travis()
+{
+    export PATH=$PWD/usr/bin:$PATH
+    export LD_LIBRARY_PATH=$PWD/usr/lib
+
+    if [ ! -d usr/bin ]; 
+    then
+        which arm-linux-as     || get_binutils arm
+        which mips-linux-as    || get_binutils mips
+        which powerpc-linux-as || get_binutils powerpc
+        local_deb_extract http://mirrors.mit.edu/ubuntu/ubuntu/pool/universe/b/binutils/binutils-multiarch_2.22-6ubuntu1_amd64.deb
+        rm -rf usr/share
+    fi
+
+    pushd usr/lib
+    ln -sf libbfd-2.22-multiarch.so libbfd-2.22.so
+    ln -sf libopcodes-2.22-multiarch.so libopcodes-2.22.so
+    popd
+
+    which arm-linux-gnu-as
+    which mips-linux-gnu-as
+    which powerpc-linux-gnu-as
+}
 
 setup_linux()
 {
-    sudo apt-get update
     sudo apt-get install software-properties-common pwgen
     sudo apt-add-repository --yes ppa:pwntools/binutils
     sudo apt-get update
     sudo apt-get install binutils-arm-linux-gnu binutils-mips-linux-gnu binutils-powerpc-linux-gnu
-    wget -nc http://www.capstone-engine.org/download/2.1.2/capstone-2.1.2_amd64.deb
-    (echo "a30bcb58fd82d32b135eec978eaa22230c44e722 *capstone-2.1.2_amd64.deb" | sha1sum -c) || exit
-    sudo dpkg -i *.deb
 }
 
 setup_osx()
@@ -19,11 +53,10 @@ setup_osx()
     brew install capstone
 }
 
-case $(uname) in
-Darwin) setup_osx   ;;
-Linux)  setup_linux ;;
-esac
-
-pip install -r requirements.txt
-pip install -r docs/requirements.txt
-pip install .
+if [[ "$TRAVIS" ]]; then
+    setup_travis
+elif [[ "$(uname)" == "Darwin" ]]; then
+    setup_osx
+elif [[ "$(uname)" == "Linux" ]]; then
+    setup_linux
+fi

.travis_ssh_setup.sh

@@ -2,17 +2,52 @@
 #
 # Install a demo user for SSH purposes
 #
-U=demouser
+U=travis
 H=/home/$U
 
-sudo useradd -m $U
-ssh-keygen -t rsa -f /tmp/$U -N ''
+USUDO()
+{
+if [ "$TRAVIS" ];
+then
+    $*
+else
+    sudo -u $U $*
+fi
+}
 
-eval $(ssh-agent -s)
-ssh-add    /tmp/$U
 
-sudo -u $U mkdir $H/.ssh
-sudo -u $U cp    /tmp/$U.pub $H/.ssh/authorized_keys
-sudo chmod -R 700 $H
+if [ "$TRAVIS" ];
+then
+    rm -f ~/.ssh/*
+else
+    # Create the new user account
+    # Disable password login for the user, and ensure the account is not locked
+    sudo useradd -m $U
+    sudo passwd --delete --unlock $U
+fi
 
-(echo -n "$U:"; pwgen 100 1) | sudo chpasswd
+# Generate a new key so that we can log into it
+ssh-keygen -t rsa -f ~/.ssh/$U -N ''
+
+# Load the public key into a memory for below
+pubkey=$(cat ~/.ssh/$U.pub)
+
+# Set the authorized_keys entry to only permit login from localhost,
+# and only with
+USUDO mkdir $H/.ssh
+USUDO tee -a $H/.ssh/authorized_keys <<EOF
+from="127.0.0.1" $pubkey
+EOF
+
+# In the pwntools examples, we ssh to 'example.pwnme'
+# Set up an SSH config entry to make this actually work
+cat >> ~/.ssh/config <<EOF
+
+Host example.pwnme
+    User travis
+    HostName 127.0.0.1
+    IdentityFile ~/.ssh/$U
+EOF
+
+ssh-keyscan -t rsa example.pwnme >>~/.ssh/known_hosts
+ssh -o "StrictHostKeyChecking no" -v [email protected] id

CHANGELOG.md

@@ -3,7 +3,7 @@
 This changelog only includes added major features and changes. Bugfixes and
 minor changes are omitted.
 
-## 2.2.0 - 2014-08-28 (pre)
+## 2.2.0
 
 This was a large release with a lot of bugfixes and changes. Only the most significant
 are mentioned here.
@@ -16,3 +16,11 @@ are mentioned here.
 - Added a serial tube
 - Huge performance gains in the buffering for tubes
 - Re-added user agents
+- Begun using Travis CI with lots of test
+- Removed bundled binutils in favor of documenting how to build them yourselves
+- Added support for port forwarding though our SSH module
+- Added dependency for capstone and ropgadget
+- Added a lots of shellcodes
+- Stuff we forgot
+- Lots of documentation fixes
+- Lots of bugfixes

LICENSE-pwntools.txt

@@ -13,7 +13,7 @@ directories:
 - pwnlib/data/
 
 
-Copyright (c) 2013 Gallopsled
+Copyright (c) 2015 Gallopsled and Zach Riggle
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,9 +1,12 @@
-# pwntools - CTF toolkit
+![pwntools logo](docs/source/logo.png?raw=true)
+
 [![Docs latest](https://readthedocs.org/projects/pwntools/badge/)](https://pwntools.readthedocs.org/en/latest)
-[![Docs 2.1.3](https://readthedocs.org/projects/pwntools/badge/?version=2.1.3)](https://pwntools.readthedocs.org/en/2.1.3)
-[![PyPI](https://img.shields.io/pypi/v/pwntools.svg)](https://pypi.python.org/pypi/pwntools/)
-[![Gittip](https://img.shields.io/gittip/gallopsled.svg)](https://www.gittip.com/gallopsled/)
+[![Docs 2.2.0](https://readthedocs.org/projects/pwntools/badge/?version=2.2.0)](https://pwntools.readthedocs.org/en/2.2.0)
+[![PyPI](https://img.shields.io/pypi/v/pwntools.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
+[![Gittip](https://img.shields.io/gittip/gallopsled.svg?style=flat)](https://www.gittip.com/gallopsled/)
 [![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
+[![Twitter](https://img.shields.io/badge/twitter-Gallopsled-4099FF.svg?style=flat)](https://twitter.com/Gallopsled)
+[![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
 
 This is the CTF framework used by Gallopsled in every CTF.
 
@@ -51,6 +54,6 @@ See [CONTRIBUTING.md](CONTRIBUTING.md)
 
 # Contact
 If you have any questions not worthy of a [bug report](https://github.com/Gallopsled/pwntools/issues), feel free to join us
-at [`#gallopsled` on Freenode](irc://irc.freenode.net/gallopsled) and ask away.
-Click [here](https://kiwiirc.com/client/irc.freenode.net/gallopsled) to connect.
+at [`#pwntools` on Freenode](irc://irc.freenode.net/pwntools) and ask away.
+Click [here](https://kiwiirc.com/client/irc.freenode.net/pwntools) to connect.
 

docs/Makefile

@@ -6,6 +6,8 @@ SPHINXOPTS    =
 SPHINXBUILD   = sphinx-build
 PAPER         =
 BUILDDIR      = build
+DASHBUILD     = ./dashbuild.py
+TAR           = tar
 
 # Internal variables.
 PAPEROPT_a4     = -D latex_paper_size=a4
@@ -14,7 +16,8 @@ ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) sou
 # the i18n builder cannot share the environment and doctrees with the others
 I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
 
-.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext dash
 
 help:
 	@echo "Please use \`make <target>' where <target> is one of"
@@ -34,6 +37,7 @@ help:
 	@echo "  texinfo    to make Texinfo files"
 	@echo "  info       to make Texinfo files and run them through makeinfo"
 	@echo "  gettext    to make PO message catalogs"
+	@echo "  dash       to make a Dash docset"
 	@echo "  changes    to make an overview of all changed/added/deprecated items"
 	@echo "  linkcheck  to check all external links for integrity"
 	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
@@ -136,6 +140,11 @@ gettext:
 	@echo
 	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
 
+dash:
+	$(DASHBUILD) source $(BUILDDIR)/dash
+	$(TAR) --exclude='.DS_Store' -cvzf $(BUILDDIR)/dash/pwntools.tgz -C $(BUILDDIR)/dash pwntools.docset
+	@echo "Build finished. The Dash docset is in $(BUILDDIR)/dash."
+
 changes:
 	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
 	@echo

docs/dashbuild.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python
+
+# -*- coding: utf-8 -*-
+
+#
+# Helper to build Dash docset from sphinx source files.
+#
+# Dash docsets can be read by various applications:
+#
+# Dash         OS X, iOS         https://kapeli.com
+# Zeal         Linux, Windows    https://zealdocs.org
+# Velocity     Windows           http://velocity.silverlakesoftware.com
+# LovelyDocs   Android           http://lovelydocs.io
+# dasht        POSIX             https://github.com/sunaku/dasht
+# Helm Dash    emacs             https://github.com/areina/helm-dash
+#
+
+import argparse
+import doc2dash.__main__
+import os, os.path
+import sphinx
+import sqlite3
+import sys
+
+sys.path.append(os.path.abspath(os.path.join('..', 'pwnlib')))
+import version
+
+def main(args):
+    """Generate a Dash docset from Sphinx source files."""
+
+    srcdir = args.srcdir
+    dstdir = args.dstdir
+    name = args.name
+
+    if not os.path.exists(dstdir):
+        os.makedirs(dstdir)
+
+    # Generate HTML without indices.
+    sphinx.build_main([ "sphinx-build", "-b", "html", "-d", os.path.join(dstdir, "doctrees"), \
+        "-t", "dash", srcdir, os.path.join(dstdir, "html") ])
+
+    # Convert to docset.
+    try:
+        doc2dash.__main__.main.main( \
+            [ os.path.join(dstdir, "html"), "-d", dstdir, "-n", name, \
+                    "-f", "-I", "index.html"], "doc2dash", False)
+    except SystemExit,e:
+        pass
+
+    # Insert a link to the online version.
+    online = args.online
+    if online is not None and online != "":
+        url = online.replace("@VERSION", args.version)
+        with open(os.path.join(dstdir, name+".docset", "Contents", "Info.plist"), "r+") as f_info:
+            pl = f_info.read()
+            pl = pl.replace("</dict>", \
+                "\t<key>DashDocSetFallbackURL</key>\n\t<string>%s</string>\n</dict>" % url)
+            f_info.seek(0)
+            f_info.write(pl)
+            f_info.truncate()
+
+    # Modify the CSS to hide the menu included in the HTML.
+    with open(os.path.join(dstdir, name+".docset", "Contents", "Resources", "Documents", "_static", "css", "theme.css"), "r+") as f_css:
+        css = f_css.read()
+        css = css.replace( \
+            '@media screen and (max-width: 768px){.wy-body-for-nav{background:#fcfcfc}.wy-nav-top{display:block}',\
+            '@media screen {.wy-body-for-nav{background:#fcfcfc}' )
+        css = css.replace( \
+            '@media screen and (max-width: 480px)', \
+            '@media screen ')
+        f_css.seek(0)
+        f_css.write(css)
+        f_css.truncate()
+
+    # Modify the index
+    db_conn = sqlite3.connect(os.path.join(dstdir, name+".docset", "Contents", "Resources", "docSet.dsidx"))
+    try:
+        db_conn.execute('INSERT INTO "searchIndex" ("name","type","path") VALUES ' \
+            '("1 Contents", "Guide", "index.html"), ' \
+            '("2 About pwntools", "Guide", "about.html"), ' \
+            '("3 Installation", "Guide", "install.html"), ' \
+            '("4 Getting Started", "Guide", "intro.html"), ' \
+            '("5 Globals (pwn)", "Guide", "globals.html"), ' \
+            '("6 Command Line Tools", "Guide", "commandline.html")')
+        db_conn.execute('DELETE FROM "searchIndex" WHERE "type" = "Module" AND ("name" = "pwn" OR "name" = "pwnlib")')
+        db_conn.commit()
+    finally:
+        db_conn.close()
+
+    return 0
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--name", help="docset name", default="pwntools")
+parser.add_argument("--online", help="URL for online docs", default="https://pwntools.readthedocs.org/en/@VERSION/")
+parser.add_argument("--version", help="pwntools version", default=version.__version__)
+parser.add_argument("srcdir", help="Source directory containing .rst files")
+parser.add_argument("dstdir", help="Destination and working directory")
+
+main(parser.parse_args())

docs/requirements.txt

@@ -3,4 +3,6 @@ argparse
 mako
 pyelftools
 sphinxcontrib-napoleon
-sphinxcontrib-autoprogram
+sphinxcontrib-autoprogram
+sphinx_rtd_theme
+doc2dash

docs/source/about.rst

@@ -11,7 +11,7 @@ nice side-effects.
 When redesigning pwntools for 2.0, we noticed two contrary goals:
 
 * We would like to have a "normal" python module structure, to allow other
-  people to faster get familiar with how pwntools works.
+  people to familiarize themselves with pwntools quickly.
 * We would like to have even more side-effects, especially by putting the
   terminal in raw-mode.
 
@@ -30,10 +30,10 @@ the following:
 
 * Imports everything from the toplevel :mod:`pwnlib` along with
   functions from a lot of submodules. This means that if you do
-  ``import pwn`` or ``from pwn import *``, you will access to
+  ``import pwn`` or ``from pwn import *``, you will have access to
   everything you need to write an exploit.
 * Calls :func:`pwnlib.term.init` to put your terminal in raw mode
-  and implementing functionality to make it look like it is not.
+  and implements functionality to make it appear like it isn't.
 * Setting the :data:`pwnlib.context.log_level` to `"info"`.
 * Tries to parse some of the values in :data:`sys.argv` and every
   value it succeeds in parsing it removes.

docs/source/asm.rst

@@ -1,7 +1,12 @@
 .. testsetup:: *
 
+   import tempfile
+   import os
+   from pwnlib.context import context
    from pwnlib.asm import *
    from pwnlib import shellcraft
+   from pwnlib.tubes.process import process
+
 
 :mod:`pwnlib.asm` --- Assembler functions
 =========================================

docs/source/conf.py

@@ -11,7 +11,11 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys, os, subprocess
+import os
+import subprocess
+import sys
+
+build_dash = tags.has('dash')
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -39,11 +43,12 @@ extensions = [
 ]
 
 doctest_global_setup = '''
-import pwnlib
+import pwnlib, sys
 pwnlib.context.context.reset_local()
 pwnlib.context.ContextType.defaults['log_level'] = 'ERROR'
 pwnlib.term.text.when = 'never'
-
+pwnlib.log.install_default_handler()
+pwnlib.log.console.stream = sys.stdout
 '''
 
 autodoc_member_order = 'alphabetical'
@@ -161,10 +166,10 @@ html_static_path = []
 #html_additional_pages = {}
 
 # If false, no module index is generated.
-#html_domain_indices = True
+html_domain_indices = not build_dash
 
 # If false, no index is generated.
-#html_use_index = True
+html_use_index = not build_dash
 
 # If true, the index is split into individual pages for each letter.
 #html_split_index = False
@@ -312,3 +317,18 @@ def linkcode_resolve(domain, info):
                 pass
 
     return "https://github.com/Gallopsled/pwntools/blob/%s/%s" % (branch, filename)
+
+
+# The readthedocs theme is used by the Dash generator. (Can be used for HTML too.)
+
+if build_dash:
+
+    # on_rtd is whether we are on readthedocs.org
+    on_rtd = os.environ.get('READTHEDOCS', None) == 'True'
+
+    if not on_rtd:  # only import and set the theme if we're building docs locally
+        import sphinx_rtd_theme
+        html_theme = 'sphinx_rtd_theme'
+        html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+
+    # otherwise, readthedocs.org uses their theme by default, so no need to specify it

docs/source/index.rst

@@ -42,16 +42,20 @@ Each of the ``pwntools`` modules is documented here.
    replacements
    rop
    shellcraft
+   shellcraft/*
    term
    timeout
    tubes
+   tubes/*
    ui
    useragents
    util/*
 
-Indices and tables
-==================
+.. only:: not dash
 
-* :ref:`genindex`
-* :ref:`modindex`
-* :ref:`search`
+   Indices and tables
+   ==================
+
+   * :ref:`genindex`
+   * :ref:`modindex`
+   * :ref:`search`

docs/source/install.rst

@@ -31,16 +31,10 @@ Latest Version
 --------------
 
 Alternatively if you prefer to use the latest version from the
-repository:
+repository, pip can install directly from github.
 
 .. code-block:: bash
 
-    $ git clone https://github.com/Gallopsled/pwntools
-    $ cd pwntools
-    $ pip install -e .
+    $ pip install git+https://github.com/Gallopsled/pwntools#egg=pwntools
 
 .. _Ubuntu: https://launchpad.net/~pwntools/+archive/ubuntu/binutils
-.. _libcapstone 2.1: http://www.capstone-engine.org
-.. _i386: http://www.capstone-engine.org/download/2.1.2/capstone-2.1.2_i386.deb
-.. _amd64: http://www.capstone-engine.org/download/2.1.2/capstone-2.1.2_amd64.deb
-

docs/source/install/binutils.rst

@@ -7,6 +7,8 @@ installed. We've made this process as smooth as we can.
 
 In these examples, replace ``$ARCH`` with your target architecture (e.g., arm, mips64, vax, etc.).
 
+Building `binutils` from source takes about 60 seconds on a modern 8-core machine.
+
 Ubuntu
 ^^^^^^^^^^^^^^^^
 First, add our `Personal Package Archive repository <http://binutils.pwntools.com>`__.
@@ -15,7 +17,7 @@ First, add our `Personal Package Archive repository <http://binutils.pwntools.co
 
     $ apt-get install software-properties-common
     $ apt-add-repository ppa:pwntools/binutils
-    $ apt-get updatez
+    $ apt-get update
 
 Then, install the binutils for your architecture.
 

docs/source/install/capstone.rst

@@ -1,27 +0,0 @@
-Capstone
--------------
-
-Capstone is a disassembly library required for gathering ROP gadgets
-and ROP chain generation.
-
-It's a separate requirement from ``binutils`` because it's used by
-Jon Salwan's ``ROPgadget`` tool which we use under the covers.
-
-In particular, version ``2.1.2`` should be used.  Capstone can be downloaded here_, or installed with the steps below.
-
-Ubuntu
-^^^^^^^^^^^^^^^^
-
-.. code-block:: bash
-
-    $ wget -nc http://www.capstone-engine.org/download/2.1.2/capstone-2.1.2_amd64.deb
-    $ apt-get install capstone-2.1.2_amd64.deb
-
-Mac OS X
-^^^^^^^^^^^^^^^^
-
-.. code-block:: bash
-
-    $ brew install capstone
-
-.. _here:  http://www.capstone-engine.org/download.html

docs/source/install/headers.rst

@@ -1,5 +1,5 @@
 Python Development Headers
--------------
+-----------------------------
 
 Some of pwntools' Python dependencies require native extensions (for example, Paramiko requires PyCrypto).
 

docs/source/intro.rst

@@ -190,7 +190,7 @@ Find offsets in your buffer that cause a crash, thanks to :mod:`pwnlib.cyclic`.
 
     >>> print cyclic(20)
     aaaabaaacaaadaaaeaaa
-    >>> # Assume EIP = 0x62616166 ('faab') at crash time
+    >>> # Assume EIP = 0x62616166 ('faab' which is pack(0x62616166))  at crash time
     >>> print cyclic_find('faab')
     120
 

docs/source/log.rst

@@ -1,7 +1,21 @@
-:mod:`pwnlib.log` and   --- Logging stuff
-=================================================================
+:mod:`pwnlib.log` --- Logging stuff
+===================================
 
 .. automodule:: pwnlib.log
+   :members: getLogger, install_default_handler, rootlogger
+
+.. autoclass:: pwnlib.log.Progress
+   :members:
+   :member-order: bysource
 
 .. autoclass:: pwnlib.log.Logger
-   :members: debug, info, warn, error, progress, success, failure, indented, info_once, warn_once
+   :members:
+   :member-order: bysource
+
+.. autoclass:: pwnlib.log.Handler
+   :members:
+   :member-order: bysource
+
+.. autoclass:: pwnlib.log.Formatter
+   :members:
+   :member-order: bysource

docs/source/shellcraft/regsort.rst

@@ -0,0 +1,9 @@
+.. testsetup:: *
+
+    from pwnlib.regsort import *
+
+:mod:`pwnlib.regsort` --- Register sorting
+===========================================================
+
+.. automodule:: pwnlib.regsort
+   :members:

docs/source/tubes.rst

@@ -1,69 +1,30 @@
 .. testsetup:: *
 
-   from pwnlib.tubes.tube import tube
-   from pwnlib.tubes.ssh import *
-   from pwnlib.tubes.remote import remote
-   from pwnlib.util.misc import which
-   from pwnlib.context import context
+   from pwn import *
 
 :mod:`pwnlib.tubes` --- Talking to the World!
 =============================================
 
 .. automodule:: pwnlib.tubes
 
-   Sockets
-   -------
 
-   .. automodule:: pwnlib.tubes.remote
+Types of Tubes
+-------------------
 
-      .. autoclass:: pwnlib.tubes.remote.remote
-         :members:
-         :show-inheritance:
+.. toctree::
+    :maxdepth: 3
+    :glob:
 
-   .. automodule:: pwnlib.tubes.listen
+    tubes/*
 
-      .. autoclass:: pwnlib.tubes.listen.listen
-         :members:
-         :show-inheritance:
 
-   .. automodule:: pwnlib.tubes.sock
+:mod:`pwnlib.tubes.tube` --- Common Functionality
+-------------------------------------------------
 
-      .. autoclass:: pwnlib.tubes.sock.sock()
-         :show-inheritance:
 
-   Processes
-   ---------
+.. automodule:: pwnlib.tubes.tube
 
-   .. automodule:: pwnlib.tubes.process
-
-      .. autoclass:: pwnlib.tubes.process.process
-         :members: kill, poll, communicate
-         :show-inheritance:
-
-   SSH
-   ---
-
-   .. automodule:: pwnlib.tubes.ssh
-
-      .. autoclass:: pwnlib.tubes.ssh.ssh
-         :members:
-
-      .. autoclass:: pwnlib.tubes.ssh.ssh_channel()
-         :members: kill, poll, interactive
-         :show-inheritance:
-
-      .. autoclass:: pwnlib.tubes.ssh.ssh_connecter()
-         :show-inheritance:
-
-      .. autoclass:: pwnlib.tubes.ssh.ssh_listener()
-         :show-inheritance:
-
-   Common functionality
-   --------------------
-
-   .. automodule:: pwnlib.tubes.tube
-
-      .. autoclass:: pwnlib.tubes.tube.tube()
-         :members:
-         :exclude-members: recv_raw, send_raw, settimeout_raw,
-                           can_recv_raw
+  .. autoclass:: pwnlib.tubes.tube.tube()
+     :members:
+     :exclude-members: recv_raw, send_raw, settimeout_raw,
+                       can_recv_raw

docs/source/tubes/processes.rst

@@ -0,0 +1,12 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.tubes.process` --- Processes
+===========================================================
+
+.. automodule:: pwnlib.tubes.process
+
+  .. autoclass:: pwnlib.tubes.process.process
+     :members: kill, poll, communicate
+     :show-inheritance:

docs/source/tubes/serial.rst

@@ -0,0 +1,11 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.tubes.serialtube` --- Serial Ports
+===========================================================
+
+.. automodule:: pwnlib.tubes.serialtube
+
+   .. autoclass:: pwnlib.tubes.serialtube.serialtube
+      :members:

docs/source/tubes/sockets.rst

@@ -0,0 +1,25 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.tubes.sock` --- Sockets
+===========================================================
+
+
+.. automodule:: pwnlib.tubes.sock
+
+   .. autoclass:: pwnlib.tubes.sock.sock()
+      :show-inheritance:
+
+
+.. automodule:: pwnlib.tubes.remote
+
+   .. autoclass:: pwnlib.tubes.remote.remote
+      :members:
+      :show-inheritance:
+
+.. automodule:: pwnlib.tubes.listen
+
+   .. autoclass:: pwnlib.tubes.listen.listen
+      :members:
+      :show-inheritance:

docs/source/tubes/ssh.rst

@@ -0,0 +1,21 @@
+.. testsetup:: *
+
+   from pwn import *
+
+:mod:`pwnlib.tubes.ssh` --- SSH
+===========================================================
+
+.. automodule:: pwnlib.tubes.ssh
+
+   .. autoclass:: pwnlib.tubes.ssh.ssh
+      :members:
+
+   .. autoclass:: pwnlib.tubes.ssh.ssh_channel()
+      :members: kill, poll, interactive
+      :show-inheritance:
+
+   .. autoclass:: pwnlib.tubes.ssh.ssh_connecter()
+      :show-inheritance:
+
+   .. autoclass:: pwnlib.tubes.ssh.ssh_listener()
+      :show-inheritance:

examples/asm.py

@@ -3,7 +3,7 @@ Example showing the interface to `pwnlib.asm.asm` and `pwnlib.shellcraft`.
 """
 
 from pwn import *
-context(arch='i386', os='linux', net='tcp4')
+context(arch='i386', os='linux')
 
 shellcode = shellcraft.i386_to_amd64()
 shellcode_asm = asm(shellcode)

examples/attach.py

@@ -0,0 +1,12 @@
+"""
+Example showing `pwnlib.gdb.attach()`
+"""
+
+from pwn import *
+
+bash = process('/bin/bash')
+gdb.attach(bash, execute = '''
+p "hello from pwnlib"
+c
+''')
+bash.interactive()

examples/yesno.py

@@ -0,0 +1,10 @@
+"""
+Example showing `pwnlib.ui.yesno()`
+"""
+
+from pwn import *
+
+if ui.yesno('Do you like Pwntools?'):
+    print ':D'
+else:
+    print ':('

extra/bash_completion.d/shellcraft

@@ -0,0 +1,46 @@
+# cache list of shellcodes
+_shellcraft_shellcodes=$(shellcraft)
+
+_shellcraft()
+{
+    COMPREPLY=()
+    local cur prev opts word code
+    cur="${COMP_WORDS[COMP_CWORD]}"
+    prev="${COMP_WORDS[COMP_CWORD-1]}"
+    opts="-h --help -? --show -o --out -f --format"
+
+    if [ "${prev}" == "-f" -o    \
+         "${prev}" == "--format" \
+        ] ; then
+        opts="raw str c hex asm p hexii"
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}))
+        return 0
+    fi
+
+    if [ "${prev}" == "-o" -o \
+         "${prev}" == "--out" \
+        ] ; then
+        COMPREPLY=( $(compgen -f ${cur}))
+        return 0
+    fi
+
+    if [[ "${cur}" == -* ]] ; then
+        COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+        return 0
+    fi
+
+    # check if a shellcode was already chosen
+    for word in "${COMP_WORDS[@]}" ; do
+        for code in ${_shellcraft_shellcodes} ; do
+            if [ "${word}" == "${code}" ] ; then
+                COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+                return 0
+            fi
+        done
+    done
+
+    # complete shellcode name
+    COMPREPLY=( $(compgen -W "${_shellcraft_shellcodes}" -- ${cur}) )
+}
+
+complete -F _shellcraft shellcraft

install_local.sh

@@ -1,12 +0,0 @@
-#!/bin/sh
-sudo pip2 install -r requirements.txt
-
-echo "================================================"
-echo "Now please put this directory in your PYTHONPATH"
-echo "and the bin directory in your PATH."
-echo
-echo "If you are using bash, you do can this by putting"
-echo "These in your .bashrc:"
-echo
-echo "export PATH=\"`realpath bin`:\$PATH\""
-echo "export PYTHONPATH=\"`realpath .`:\$PYTHONPATH\""

pwn/__init__.py

@@ -1,6 +1,7 @@
 # Promote useful stuff to toplevel
 from .toplevel import *
 
+
 # look for special args in argv
 def closure():
     term_mode = True
@@ -36,22 +37,22 @@ def closure():
         k = k[7:]
         if   k == 'DEBUG':
             if asbool(v):
-                context.log_level = 'debug'
+                context.log_level = 'DEBUG'
         elif k == 'SILENT':
             if asbool(v):
                 context.log_level = 'ERROR'
         elif k == 'NOTERM':
             if asbool(v):
                 term_mode = False
-        elif k == 'LOG_LEVEL':
-            context.log_level = v
         elif isident(k):
             args[k] = v
     # parse command line
-    for arg in sys.argv[:]:
+    # save a copy of argv for the log file header (see below)
+    argv = sys.argv[:]
+    for arg in argv:
         if   arg == 'DEBUG':
             sys.argv.remove(arg)
-            context.log_level = 'debug'
+            context.log_level = 'DEBUG'
         elif arg == 'SILENT':
             sys.argv.remove(arg)
             context.log_level = 'ERROR'
@@ -63,12 +64,56 @@ def closure():
                 continue
             sys.argv.remove(arg)
             args[k] = v
+    if 'LOG_LEVEL' in args:
+        context.log_level = args['LOG_LEVEL']
+    if 'LOG_FILE' in args:
+        # install a file logger
+        import logging, time
+        modes = ('w', 'wb', 'a', 'ab')
+        filename = args['LOG_FILE']
+        mode = 'a'
+        # check if mode was specified as "[filename],[mode]"
+        if ',' in filename:
+            filename_, mode_ = filename.rsplit(',', 1)
+            if mode in modes:
+                filename = filename_
+                mode = mode_
+        # ISO 8601
+        dfmt = '%Y-%m-%dT%H:%M:%S'
+        # write a "header" to the file, which makes it easier to find the start
+        # of a session
+        with open(filename, mode) as fd:
+            lines = [
+                '=' * 78,
+                ' Started at %s ' % time.strftime(dfmt),
+                ' sys.argv = [',
+                ]
+            for arg in argv:
+                lines.append('   %r,' % arg)
+            lines.append(' ]')
+            lines.append('=' * 78)
+            for line in lines:
+                fd.write('=%s=\n' % line.ljust(78))
+        # if the mode was 'w' or 'wb' we need to change it to 'a'/'ab' now so
+        # the logging module wont overwrite the header
+        mode = mode.replace('w', 'a')
+        # create a formatter and a handler and install them for the pwnlib root
+        # logger (i.e. 'pwnlib')
+        handler = logging.FileHandler(filename, mode)
+        fmt = '%(asctime)s:%(levelname)s:%(name)s:%(message)s'
+        formatter = logging.Formatter(fmt, dfmt)
+        handler.setFormatter(formatter)
+        logging.root.addHandler(handler)
     # put the terminal in rawmode unless NOTERM was specified
     if term_mode:
         term.init()
-    else:
-        term.text.when = 'never'
+    # install a log handler and turn logging all the way up
+    import pwnlib.log as log
+    import logging
+    log.rootlogger.setLevel(logging.DEBUG)
+    log.install_default_handler()
+
 closure()
 del closure
 
-log = logging.getLogger('pwnlib.exploit')
+log = getLogger('pwnlib.exploit')

pwn/toplevel.py

@@ -1,36 +1,67 @@
+# Promote these modules, so that "from pwn import *" will let you access them
+import collections
+import math
+import operator
+import os
+import re
+import string
+import struct
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+
+import pwnlib
+
 # Get all the modules from pwnlib
-from pwnlib                  import *
+from pwnlib import *
 
 # Promote functions from these modules to toplevel
-from pwnlib.asm              import asm, disasm, cpp
-from pwnlib.context          import context, Thread
-from pwnlib.dynelf           import DynELF
-from pwnlib.elf              import ELF, load
-from pwnlib.exception        import PwnlibException
-from pwnlib.memleak          import MemLeak
-from pwnlib.replacements     import *
-from pwnlib.rop              import ROP
-from pwnlib.timeout          import Timeout
-from pwnlib.tubes.listen     import listen
-from pwnlib.tubes.process    import process
-from pwnlib.tubes.remote     import remote
+from pwnlib.asm import asm
+from pwnlib.asm import cpp
+from pwnlib.asm import disasm
+from pwnlib.context import Thread
+from pwnlib.context import context
+from pwnlib.dynelf import DynELF
+from pwnlib.elf import ELF
+from pwnlib.elf import load
+from pwnlib.exception import PwnlibException
+from pwnlib.fmtstr import FmtStr
+from pwnlib.log import getLogger
+from pwnlib.memleak import MemLeak
+from pwnlib.regsort import *
+from pwnlib.replacements import *
+from pwnlib.rop import ROP
+from pwnlib.timeout import Timeout
+from pwnlib.tubes.listen import listen
+from pwnlib.tubes.process import process
+from pwnlib.tubes.remote import remote
 from pwnlib.tubes.serialtube import serialtube
-from pwnlib.tubes.ssh        import ssh
-from pwnlib.tubes.tube       import tube
-from pwnlib.ui               import *
-from pwnlib.util             import crc, net, proc, iters, safeeval
-from pwnlib.util.cyclic      import *
-from pwnlib.util.fiddling    import *
-from pwnlib.util.hashes      import *
-from pwnlib.util.lists       import *
-from pwnlib.util.misc        import *
-from pwnlib.util.packing     import *
-from pwnlib.util.proc        import pidof
-from pwnlib.util.splash      import *
-from pwnlib.util.web         import *
+from pwnlib.tubes.ssh import ssh
+from pwnlib.tubes.tube import tube
+from pwnlib.ui import *
+from pwnlib.util import crc
+from pwnlib.util import iters
+from pwnlib.util import net
+from pwnlib.util import proc
+from pwnlib.util import safeeval
+from pwnlib.util.cyclic import *
+from pwnlib.util.fiddling import *
+from pwnlib.util.hashes import *
+from pwnlib.util.lists import *
+from pwnlib.util.misc import *
+from pwnlib.util.packing import *
+from pwnlib.util.proc import pidof
+from pwnlib.util.splash import *
+from pwnlib.util.web import *
 
-# Promote these modules, so that "from pwn import *" will let you access them
-import \
-    collections   , operator      , os            , pwnlib        , \
-    re            , string        , struct        , subprocess    , \
-    sys           , threading     , time          , tempfile
+try:
+    import cPickle as pickle
+except ImportError:
+    import pickle
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO

pwnlib/__init__.py

@@ -1,19 +1,35 @@
+import importlib
+
+from .version import __version__
+
+version = __version__
+
 __all__ = [
-    'atexception' , 'atexit'      , 'asm'         , 'constants'   ,
-    'context'     , 'dynelf'      , 'elf'         , 'exception'   ,
-    'gdb'                         , 'log'         , 'memleak'     ,
-    'replacements', 'rop'         , 'shellcraft'  , 'term'        ,
-    'tubes'       , 'ui'          , 'useragents'  , 'util'
+    'asm',
+    'atexception',
+    'atexit',
+    'commandline',
+    'constants',
+    'context',
+    'dynelf',
+    'elf',
+    'exception',
+    'fmtstr',
+    'gdb',
+    'libcdb',
+    'log',
+    'memleak',
+    'pep237',
+    'regsort',
+    'replacements',
+    'rop',
+    'shellcraft',
+    'term',
+    'tubes',
+    'ui',
+    'useragents',
+    'util'
 ]
 
-
-from . import \
-    atexception   , atexit        , asm           , constants     , \
-                    dynelf        , elf           , exception     , \
-    gdb                           , log           , memleak       , \
-    replacements  , rop           , shellcraft    , term          , \
-    tubes         , ui            , useragents    , util,           \
-    pep237
-
-# from .context import context
-from .version import __version__
+for module in __all__:
+    importlib.import_module('.%s' % module, 'pwnlib')

pwnlib/asm.py

@@ -39,15 +39,26 @@ Disassembly
     '   0:   b8 0b 00 00 00          mov    eax,0xb'
 
 """
-import tempfile, subprocess, shutil, tempfile, errno, logging, platform
-from os import path, environ
+import errno
+import os
+import platform
+import re
+import shutil
+import string
+import subprocess
+import sys
+import tempfile
+from collections import defaultdict
 from glob import glob
-from . import log
+from os import environ
+from os import path
+
 from .context import context
+from .log import getLogger
 
-log = logging.getLogger(__name__)
+log = getLogger(__name__)
 
-__all__ = ['asm', 'cpp', 'disasm', 'which_binutils']
+__all__ = ['asm', 'cpp', 'disasm', 'make_elf']
 
 _basedir = path.split(__file__)[0]
 _incdir  = path.join(_basedir, 'data', 'includes')
@@ -104,7 +115,7 @@ def which_binutils(util, **kwargs):
                 else:       pattern = '%s*linux*-%s' % (arch,gutil)
 
                 for dir in environ['PATH'].split(':'):
-                    res = glob(path.join(dir, pattern))
+                    res = sorted(glob(path.join(dir, pattern)))
                     if res:
                         return res[0]
 
@@ -116,6 +127,8 @@ Try installing binutils for this architecture:
 """.strip() % locals())
         raise Exception('Could not find %(util)r installed for %(context)s' % locals())
 
+checked_assembler_version = defaultdict(lambda: False)
+
 def _assembler():
     gas = which_binutils('as')
 
@@ -124,28 +137,58 @@ def _assembler():
         'little': '-EL'
     }[context.endianness]
 
+    B = '-%s' % context.bits
+
     assemblers = {
-        'i386'   : [gas, '--32'],
-        'amd64'  : [gas, '--64'],
+        'i386'   : [gas, B],
+        'amd64'  : [gas, B],
 
         # Most architectures accept -EL or -EB
         'thumb'  : [gas, '-mthumb', E],
         'arm'    : [gas, E],
         'aarch64': [gas, E],
-        'mips'   : [gas, E],
-        'mips64' : [gas, E],
-        'sparc':   [gas, E],
-        'sparc64': [gas, E],
+        'mips'   : [gas, E, B],
+        'mips64' : [gas, E, B],
+        'sparc':   [gas, E, B],
+        'sparc64': [gas, E, B],
 
-        # Powerpc wants -mbig or -mlittle
-        'powerpc':   [gas, '-m%s' % context.endianness],
-        'powerpc64': [gas, '-m%s' % context.endianness],
+        # Powerpc wants -mbig or -mlittle, and -mppc32 or -mppc64
+        'powerpc':   [gas, '-m%s' % context.endianness, '-mppc%s' % context.bits],
+        'powerpc64': [gas, '-m%s' % context.endianness, '-mppc%s' % context.bits],
 
         # ia64 only accepts -mbe or -mle
         'ia64':    [gas, '-m%ce' % context.endianness[0]]
     }
 
-    return assemblers.get(context.arch, [gas])
+    assembler = assemblers.get(context.arch, [gas])
+
+    if not checked_assembler_version[gas]:
+        checked_assembler_version[gas] = True
+        result = subprocess.check_output([gas, '--version','/dev/null'],
+                                         stderr=subprocess.STDOUT)
+        version = re.search(r' (\d\.\d+)', result).group(1)
+        if version < '2.19':
+            log.warn_once('Your binutils version is too old and may not work!\n'  + \
+                'Try updating with: https://pwntools.readthedocs.org/en/latest/install/binutils.html\n' + \
+                'Reported Version: %r' % result.strip())
+
+
+    return assembler
+
+def _linker():
+    ld  = [which_binutils('ld')]
+    bfd = ['--oformat=' + _bfdname()]
+
+    E = {
+        'big':    '-EB',
+        'little': '-EL'
+    }[context.endianness]
+
+    arguments = {
+        'i386': ['-m', 'elf_i386'],
+    }.get(context.arch, [])
+
+    return ld + bfd + [E] + arguments
 
 def _objcopy():
     return [which_binutils('objcopy')]
@@ -193,19 +236,25 @@ def _arch_header():
 
 def _bfdname():
     arch = context.arch
+    E    = context.endianness
 
     bfdnames = {
         'i386'    : 'elf32-i386',
+        'aarch64' : 'elf64-%saarch64' % E,
         'amd64'   : 'elf64-x86-64',
-        'arm'     : 'elf32-littlearm',
-        'thumb'   : 'elf32-littlearm',
-        'mips'    : 'elf32-%smips' % context.endianness,
+        'arm'     : 'elf32-%sarm' % E,
+        'thumb'   : 'elf32-%sarm' % E,
+        'mips'    : 'elf32-trad%smips' % E,
+        'mips64'  : 'elf64-trad%smips' % E,
         'alpha'   : 'elf64-alpha',
         'cris'    : 'elf32-cris',
-        'ia64'    : 'elf64-ia64-%s' % context.endianness,
+        'ia64'    : 'elf64-ia64-%s' % E,
         'm68k'    : 'elf32-m68k',
         'powerpc' : 'elf32-powerpc',
+        'powerpc64' : 'elf64-powerpc',
         'vax'     : 'elf32-vax',
+        'sparc'   : 'elf32-sparc',
+        'sparc64' : 'elf64-sparc',
     }
 
     if arch in bfdnames:
@@ -297,6 +346,79 @@ def cpp(shellcode, **kwargs):
         ]
         return _run(cmd, code).strip('\n').rstrip() + '\n'
 
+elf_template = '''
+.global _start
+.global __start
+.text
+_start:
+__start:
+'''
+
+def make_elf(data, vma = None, strip=True, **kwargs):
+    r"""
+    Builds an ELF file with the specified binary data as its
+    executable code.
+
+    Arguments:
+        data(str): Assembled code
+        vma(int):  Load address for the ELF file
+
+    Examples:
+
+        This example creates an i386 ELF that just does
+        execve('/bin/sh',...).
+
+        >>> context.clear()
+        >>> context.arch = 'i386'
+        >>> context.bits = 32
+        >>> filename = tempfile.mktemp()
+        >>> bin_sh = '6a68682f2f2f73682f62696e89e331c96a0b5899cd80'.decode('hex')
+        >>> data = make_elf(bin_sh)
+        >>> with open(filename,'wb+') as f:
+        ...     f.write(data)
+        ...     f.flush()
+        >>> os.chmod(filename,0777)
+        >>> p = process(filename)
+        >>> p.sendline('echo Hello; exit')
+        >>> p.recvline()
+        'Hello\n'
+    """
+    with context.local(**kwargs):
+        assembler = _assembler()
+        linker    = _linker()
+        code      = elf_template
+        code      += '.string "%s"' % ''.join('\\x%02x' % c for c in bytearray(data))
+        code      += '\n'
+
+        log.debug(code)
+
+        tmpdir    = tempfile.mkdtemp(prefix = 'pwn-asm-')
+        step1     = path.join(tmpdir, 'step1-asm')
+        step2     = path.join(tmpdir, 'step2-obj')
+        step3     = path.join(tmpdir, 'step3-elf')
+
+        try:
+            with open(step1, 'wb+') as f:
+                f.write(code)
+
+            _run(assembler + ['-o', step2, step1])
+
+            load_addr = []
+            if vma is not None:
+                load_addr = ['-Ttext-segment=%#x' % vma]
+
+            _run(linker    + load_addr + ['-N', '-o', step3, step2])
+
+            if strip:
+                _run([which_binutils('objcopy'), '-Sg', step3])
+                _run([which_binutils('strip'), '--strip-unneeded', step3])
+
+            with open(step3, 'r') as f:
+                return f.read()
+        except Exception:
+            log.exception("An error occurred while building an ELF:\n%s" % code)
+        else:
+            shutil.rmtree(tmpdir)
 
 def asm(shellcode, vma = 0, **kwargs):
     r"""asm(code, vma = 0, ...) -> str
@@ -309,7 +431,7 @@ def asm(shellcode, vma = 0, **kwargs):
     To support all these architecture, we bundle the GNU assembler
     and objcopy with pwntools.
 
-    Args:
+    Arguments:
       shellcode(str): Assembler code to assemble.
       vma(int):       Virtual memory address of the beginning of assembly
 
@@ -333,8 +455,9 @@ def asm(shellcode, vma = 0, **kwargs):
 
     with context.local(**kwargs):
         assembler = _assembler()
+        linker    = _linker()
         objcopy   = _objcopy() + ['-j', '.shellcode', '-Obinary']
-        code      = '.org %#x\n' % vma
+        code      = ''
         code      += _arch_header()
         code      += cpp(shellcode)
 
@@ -344,6 +467,7 @@ def asm(shellcode, vma = 0, **kwargs):
         step1     = path.join(tmpdir, 'step1')
         step2     = path.join(tmpdir, 'step2')
         step3     = path.join(tmpdir, 'step3')
+        step4     = path.join(tmpdir, 'step4')
 
         try:
             with open(step1, 'w') as fd:
@@ -351,26 +475,36 @@ def asm(shellcode, vma = 0, **kwargs):
 
             _run(assembler + ['-o', step2, step1])
 
-            if file(step2,'rb').read(4) == '\x7fELF':
+            if not vma:
+                shutil.copy(step2, step3)
+
+            if vma:
+                 _run(linker + ['--section-start=.shellcode=%#x' % vma,
+                                '--entry=%#x' % vma,
+                                '-o', step3, step2])
+
+            elif file(step2,'rb').read(4) == '\x7fELF':
                 # Sanity check for seeing if the output has relocations
                 relocs = subprocess.check_output(
                     [which_binutils('readelf'), '-r', step2]
                 ).strip()
                 if len(relocs.split('\n')) > 1:
                     log.error('Shellcode contains relocations:\n%s' % relocs)
+            else:
+                shutil.copy(step2, step3)
 
-            _run(objcopy + [step2, step3])
+            _run(objcopy + [step3, step4])
 
-            with open(step3) as fd:
+            with open(step4) as fd:
                 result = fd.read()
 
-        except:
+        except Exception:
             log.exception("An error occurred while assembling:\n%s" % code)
         else:
             shutil.rmtree(tmpdir)
             return result
 
-def disasm(data, vma = 0, **kwargs):
+def disasm(data, vma = 0, byte = True, offset = True, **kwargs):
     """disasm(data, ...) -> str
 
     Disassembles a bytestring into human readable assembler.
@@ -381,9 +515,11 @@ def disasm(data, vma = 0, **kwargs):
     To support all these architecture, we bundle the GNU objcopy
     and objdump with pwntools.
 
-    Args:
+    Arguments:
       data(str): Bytestring to disassemble.
       vma(int): Passed through to the --adjust-vma argument of objdump
+      byte(bool): Include the hex-printed bytes in the disassembly
+      offset(bool): Include the virtual memory address in the disassembly
 
     Kwargs:
       Any arguments/properties that can be set on ``context``
@@ -394,6 +530,10 @@ def disasm(data, vma = 0, **kwargs):
 
           >>> print disasm('b85d000000'.decode('hex'), arch = 'i386')
              0:   b8 5d 00 00 00          mov    eax,0x5d
+          >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0)
+             0:   mov    eax,0x5d
+          >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0, offset = 0)
+          mov    eax,0x5d
           >>> print disasm('b817000000'.decode('hex'), arch = 'amd64')
              0:   b8 17 00 00 00          mov    eax,0x17
           >>> print disasm('48c7c017000000'.decode('hex'), arch = 'amd64')
@@ -403,6 +543,7 @@ def disasm(data, vma = 0, **kwargs):
              4:   00900052        addseq  r0, r0, r2, asr r0
           >>> print disasm('4ff00500'.decode('hex'), arch = 'thumb', bits=32)
              0:   f04f 0005       mov.w   r0, #5
+          >>>
     """
     result = ''
 
@@ -444,9 +585,23 @@ def disasm(data, vma = 0, **kwargs):
                 log.error('Could not find .text in objdump output:\n%s' % output0)
 
             result = output1[1].strip('\n').rstrip().expandtabs()
-        except:
+        except Exception:
             log.exception("An error occurred while disassembling:\n%s" % data)
         else:
             shutil.rmtree(tmpdir)
-            return result
 
+    lines = []
+    pattern = '^( *[0-9a-f]+: *)((?:[0-9a-f]+ )+ *)(.*)'
+    for line in result.splitlines():
+        o, b, i = re.search(pattern, line).groups()
+
+        line = ''
+
+        if offset:
+            line += o
+        if byte:
+            line += b
+        line += i
+        lines.append(line)
+
+    return '\n'.join(lines)

pwnlib/atexception.py

@@ -3,7 +3,10 @@ Analogous to atexit, this module allows the programmer to register functions to
 be run if an unhandled exception occurs.
 """
 
-import sys, traceback, threading
+import sys
+import threading
+import traceback
+
 from .context import context
 
 __all__ = ['register', 'unregister']

pwnlib/atexit.py

@@ -8,7 +8,10 @@ This module also fixes a the issue that exceptions raised by an exit handler is
 printed twice when the standard :mod:`atexit` is used.
 """
 
-import sys, traceback, threading
+import sys
+import threading
+import traceback
+
 from .context import context
 
 __all__ = ['register', 'unregister']

pwnlib/commandline/asm.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python2
+import argparse
+import sys
 
-import argparse, sys
-from pwnlib.asm           import asm
-from pwnlib.context       import context
-from pwnlib.util.fiddling import enhex
+from pwn import *
+
+from . import common
 
 parser = argparse.ArgumentParser(
     description = 'Assemble shellcode into bytes'
@@ -32,31 +33,38 @@ parser.add_argument(
 
 parser.add_argument(
     '-c', '--context',
-    metavar = '<opt>',
-    choices = context.oses + list(context.architectures),
-    default = ['i386','linux'],
+    metavar = 'context',
     action = 'append',
-    help = 'The os/architecture the shellcode will run in (default: linux/i386), choose from: %s' % \
-      ', '.join(sorted(context.oses + list(context.architectures)))
+    type   = common.context_arg,
+    choices = common.choices,
+    help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
+)
+
+
+parser.add_argument(
+    '-d',
+    '--debug',
+    help='Debug the shellcode with GDB',
+    action='store_true'
 )
 
 def main():
     args   = parser.parse_args()
     tty    = args.output.isatty()
 
-    for arch in args.context[::-1]:
-        if arch in context.architectures: break
-    for os in args.context[::-1]:
-        if os in context.oses: break
-
     data   = '\n'.join(args.lines) or sys.stdin.read()
-    output = asm(data.replace(';', '\n'), arch = arch, os = os)
+    output = asm(data.replace(';', '\n'))
     fmt    = args.format or ('hex' if tty else 'raw')
     formatters = {'r':str, 'h':enhex, 's':repr}
 
+    if args.debug:
+        proc = gdb.debug_shellcode(output, arch=context.arch)
+        proc.interactive()
+        sys.exit(0)
+
     args.output.write(formatters[fmt[0]](output))
 
     if tty and fmt is not 'raw':
         args.output.write('\n')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__': main()

pwnlib/commandline/checksec.py

@@ -0,0 +1,25 @@
+#!/usr/bin/env python2
+import argparse
+import sys
+
+from pwn import *
+
+from . import common
+
+parser = argparse.ArgumentParser(
+    description = 'Check binary security settings'
+)
+
+parser.add_argument(
+    'elf',
+    nargs='+',
+    type=file,
+    help='Files to check'
+)
+
+def main():
+    args   = parser.parse_args()
+    for f in args.elf:
+        e = ELF(f.name)
+
+if __name__ == '__main__': main()

pwnlib/commandline/common.py

@@ -0,0 +1,19 @@
+from pwn import *
+
+pwnlib.log.console.stream = sys.stderr
+
+choices = map(str, [16,32,64])
+choices += list(context.oses)
+choices += list(context.architectures)
+choices += list(context.endiannesses)
+
+def context_arg(arg):
+    try: context.arch = arg
+    except Exception: pass
+    try: context.os = arg
+    except Exception: pass
+    try: context.bits = int(arg)
+    except Exception: arg
+    try: context.endian = arg
+    except Exception: pass
+    return arg

pwnlib/commandline/constgrep.py

@@ -1,10 +1,16 @@
 #!/usr/bin/env python2
 
-import argparse, os, re
-from pwnlib import asm, constants
-from pwnlib.util import safeeval
-# from pwnlib.context import __possible__ as contexts
+import argparse
+import os
+import re
+
+import pwnlib.log
+from pwnlib import asm
+from pwnlib import constants
 from pwnlib.context import context
+from pwnlib.util import safeeval
+
+pwnlib.log.install_default_handler()
 
 p = argparse.ArgumentParser(
     description = "Looking up constants from header files.\n\nExample: constgrep -c freebsd -m  ^PROT_ '3 + 4'",

pwnlib/commandline/cyclic.py

@@ -1,7 +1,16 @@
 #!/usr/bin/env python2
-import argparse, string, sys
-from pwnlib.util import cyclic, packing
-from pwnlib import log
+import argparse
+import string
+import sys
+
+from pwnlib.log import getLogger
+from pwnlib.log import install_default_handler
+from pwnlib.util import cyclic
+from pwnlib.util import packing
+
+install_default_handler()
+
+log = getLogger('pwnlib.commandline.cyclic')
 
 parser = argparse.ArgumentParser(
     description = "Cyclic pattern creator/finder"
@@ -46,20 +55,23 @@ def main():
         pat = args.lookup
 
         if pat.startswith('0x'):
-            pat = packing.pack(int(pat[2:], 16), subsize*8, 'little', 'unsigned')
+            pat = packing.pack(int(pat[2:], 16), subsize*8, 'little', False)
         elif pat.isdigit():
-            pat = packing.pack(int(pat, 10), subsize*8, 'little', 'unsigned')
+            pat = packing.pack(int(pat, 10), subsize*8, 'little', False)
 
-        if len(pat) != 4:
-            log.fatal('Subpattern must be 4 bytes', 1)
+        if len(pat) != subsize:
+            log.critical('Subpattern must be %d bytes' % subsize)
+            sys.exit(1)
 
         if not all(c in alphabet for c in pat):
-            log.fatal('Pattern contains characters not present in the alphabet', 1)
+            log.critical('Pattern contains characters not present in the alphabet')
+            sys.exit(1)
 
         offset = cyclic.cyclic_find(pat, alphabet, subsize)
 
         if offset == -1:
-            log.fatal('Given pattern does not exist in cyclic pattern', 1)
+            log.critical('Given pattern does not exist in cyclic pattern')
+            sys.exit(1)
         else:
             print offset
     else:
@@ -74,4 +86,4 @@ def main():
         if sys.stdout.isatty():
             sys.stdout.write('\n')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__': main()

pwnlib/commandline/disasm.py

@@ -1,9 +1,14 @@
 #!/usr/bin/env python2
+import argparse
+import sys
+from string import hexdigits
+from string import whitespace
 
-import argparse, sys
+import pwnlib.log
 from pwnlib import asm
 from pwnlib.context import context
-from string import whitespace, hexdigits
+
+pwnlib.log.install_default_handler()
 
 parser = argparse.ArgumentParser(
     description = 'Disassemble bytes into text format'
@@ -39,4 +44,4 @@ def main():
 
     print asm.disasm(dat, arch = args.context)
 
-if __name__ == '__main__': main()
+if __name__ == '__main__': main()

pwnlib/commandline/elfdiff.py

@@ -1,8 +1,10 @@
 #!/usr/bin/env python2
 from argparse import ArgumentParser
-from subprocess import check_output, CalledProcessError
+from subprocess import CalledProcessError
+from subprocess import check_output
 from tempfile import NamedTemporaryFile
 
+
 def dump(x):
     n = NamedTemporaryFile(delete=False)
     o = check_output(['objdump','-d','-x','-s',x])
@@ -24,4 +26,4 @@ def main():
     a = p.parse_args()
     print diff(dump(a.a), dump(a.b))
 
-if __name__ == '__main__': main()
+if __name__ == '__main__': main()

pwnlib/commandline/elfpatch.py

@@ -1,8 +1,13 @@
 #!/usr/bin/env python2
+import argparse
+import sys
 
-import argparse, sys
+import pwnlib.log
 from pwnlib.elf import ELF
 from pwnlib.util.fiddling import unhex
+
+pwnlib.log.install_default_handler()
+
 p = argparse.ArgumentParser()
 p.add_argument('elf',help="File to patch")
 p.add_argument('offset',help="Offset to patch in virtual address (hex encoded)")

pwnlib/commandline/hex.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python2
-import sys, argparse
+import argparse
+import sys
 
 parser = argparse.ArgumentParser(description='''
 Hex-encodes data provided on the command line or via stdin.
@@ -14,4 +15,4 @@ def main():
     else:
         print ' '.join(args.data).encode('hex')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__': main()

pwnlib/commandline/phd.py

@@ -1,9 +1,15 @@
 #!/usr/bin/env python2
+import argparse
+import os
+import re
+import sys
 
-import argparse, sys, os
+import pwnlib.log
 import pwnlib.term.text as text
 from pwnlib.util.fiddling import hexdump_iter
 
+pwnlib.log.install_default_handler()
+
 parser = argparse.ArgumentParser(
     description = 'Pwnlib HexDump'
 )
@@ -25,7 +31,9 @@ parser.add_argument(
 
 parser.add_argument(
     "-l", "--highlight",
-    help="Byte to highlight.",
+    help="Byte sequence to highlight.  Use '?' to match arbitrary bytes and "\
+         "'\\?' to match an actual question mark.  Use '\\xXX' for non-"\
+         "printable bytes.",
     nargs="*",
 )
 
@@ -82,18 +90,41 @@ def main():
         else:
             infile.seek(skip, os.SEEK_CUR)
 
-    data = infile.read(count)
-
-    hl = []
     if args.highlight:
-        for hs in args.highlight:
-            for h in hs.split(','):
-                hl.append(asint(h))
+        def canon(hl):
+            out = []
+            i = 0
+            while i < len(hl):
+                c = hl[i]
+                if c == '\\' and len(hl) > i + 1:
+                    c2 = hl[i + 1]
+                    if   c2 == 'x':
+                        try:
+                            b = chr(int(hl[i + 2: i + 4], 16))
+                        except:
+                            print 'Bad escape sequence:', hl[i:]
+                            sys.exit(1)
+                        out.append(b)
+                        i += 3
+                    elif c2 in '\\?':
+                        out.append(c2)
+                        i += 1
+                    else:
+                        out.append(c)
+                elif c == '?':
+                    out.append(None)
+                else:
+                    out.append(c)
+                i += 1
+            return out
+        highlight = map(canon, args.highlight)
+    else:
+        highlight = []
 
     try:
-        for line in hexdump_iter(data, width, highlight = hl, begin = offset + skip):
+        for line in hexdump_iter(infile, width, highlight = highlight, begin = offset + skip):
             print line
     except (KeyboardInterrupt, IOError):
         pass
 
-if __name__ == '__main__': main()
+if __name__ == '__main__': main()

pwnlib/commandline/shellcraft.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python2
-import argparse, sys, os, types
-import pwnlib
-from pwnlib import log, util
-import pwnlib.term.text as text
-from pwnlib.context import context
+import argparse
+import os
+import sys
+import types
+
+from pwn import *
+
+from . import common
 
 r = text.red
 g = text.green
@@ -34,15 +37,6 @@ def _string(s):
             out.append('\\x%02x' % co)
     return '"' + ''.join(out) + '"\n'
 
-def _carray(s):
-    out = []
-    for c in s:
-        out.append('0x' + util.fiddling.enhex(c))
-    return '{' + ', '.join(out) + '};\n'
-
-def _hex(s):
-    return pwnlib.util.fiddling.enhex(s) + '\n'
-
 p = argparse.ArgumentParser(
     description = 'Microwave shellcode -- Easy, fast and delicious',
     formatter_class = argparse.RawDescriptionHelpFormatter,
@@ -57,7 +51,7 @@ p.add_argument(
 
 p.add_argument(
     '-o', '--out',
-    metavar = '<file>',
+    metavar = 'file',
     type = argparse.FileType('w'),
     default = sys.stdout,
     help = 'Output file (default: stdout)',
@@ -65,7 +59,7 @@ p.add_argument(
 
 p.add_argument(
     '-f', '--format',
-    metavar = '<format>',
+    metavar = 'format',
     choices = ['r', 'raw',
                's', 'str', 'string',
                'c',
@@ -78,99 +72,106 @@ p.add_argument(
     help = 'Output format (default: hex), choose from {r}aw, {s}tring, {c}-style array, {h}ex string, hex{i}i, {a}ssembly code, {p}reprocssed code',
 )
 
-
-class NoDefaultContextValues(object):
-    def __enter__(self):
-        self.old = context.defaults.copy()
-        context.defaults['os'] = None
-        context.defaults['arch'] = None
-    def __exit__(self, *a):
-        context.defaults.update(self.old)
-
-
-
-def get_tree(path, val, result):
-    with NoDefaultContextValues():
-        if path:
-            path += '.'
-
-        if path.startswith('common.'):
-            return
-
-        mods = []
-
-        for k in sorted(dir(val)):
-            if k and k[0] != '_':
-                cur = getattr(val, k)
-                if isinstance(cur, types.ModuleType):
-                    mods.append((path + k, cur))
-                else:
-                    result.append((path + k, cur))
-
-        for path, val in mods:
-            get_tree(path, val, result)
-        return result
-
-# Enumearte all of the shellcode names
-all_shellcodes = get_tree('', pwnlib.shellcraft, [])
-names = '\n'.join('    ' + sc[0] for sc in all_shellcodes)
-
 p.add_argument(
     'shellcode',
     nargs = '?',
-    default = '',
-    metavar = '<shellcode>',
+    choices = shellcraft.templates,
+    metavar = 'shellcode',
     help = 'The shellcode you want',
 )
 
-p.epilog = 'Available shellcodes are:\n' + names
-
+p.epilog = 'Available shellcodes are:\n' + '\n'.join(shellcraft.templates)
 
 p.add_argument(
     'args',
     nargs = '*',
-    metavar = '<arg>',
+    metavar = 'arg',
     default = (),
     help = 'Argument to the chosen shellcode',
 )
 
+p.add_argument(
+    '-d',
+    '--debug',
+    help='Debug the shellcode with GDB',
+    action='store_true'
+)
+
 def main():
     # Banner must be added here so that it doesn't appear in the autodoc
     # generation for command line tools
     p.description = banner + p.description
     args = p.parse_args()
 
+    if not args.shellcode:
+        print '\n'.join(shellcraft.templates)
+        exit()
+
     if args.format == 'default':
         if sys.stdout.isatty():
             args.format = 'hex'
         else:
             args.format = 'raw'
 
-
-    vals = get_tree('', pwnlib.shellcraft, [])
-    if args.shellcode:
-        vals = [(k, val) for k, val in vals if k.startswith(args.shellcode + '.') or k == args.shellcode]
-
-    if len(vals) == 0:
-        log.fatal("Cannot find subtree by the name of %r" % args.shellcode)
-    elif len(vals) > 1:
-        for k, _ in vals:
-            print k
-        exit()
-    else:
-        func = vals[0][1]
+    func = shellcraft
+    for attr in args.shellcode.split('.'):
+        func = getattr(func, attr)
 
     if args.show:
-        print func.__doc__
+        # remove doctests
+        doc = []
+        in_doctest = False
+        block_indent = None
+        caption = None
+        lines = func.__doc__.splitlines()
+        i = 0
+        while i < len(lines):
+            line = lines[i]
+            if line.lstrip().startswith('>>>'):
+                # this line starts a doctest
+                in_doctest = True
+                block_indent = None
+                if caption:
+                    # delete back up to the caption
+                    doc = doc[:caption - i]
+                    caption = None
+            elif line == '':
+                # skip blank lines
+                pass
+            elif in_doctest:
+                # indentation marks the end of a doctest
+                indent = len(line) - len(line.lstrip())
+                if block_indent is None:
+                    if not line.lstrip().startswith('...'):
+                        block_indent = indent
+                elif indent < block_indent:
+                    in_doctest = False
+                    block_indent = None
+                    # re-evalutate this line
+                    continue
+            elif line.endswith(':'):
+                # save index of caption
+                caption = i
+            else:
+                # this is not blank space and we're not in a doctest, so the
+                # previous caption (if any) was not for a doctest
+                caption = None
+
+            if not in_doctest:
+                doc.append(line)
+            i += 1
+        print '\n'.join(doc).rstrip()
         exit()
 
     defargs = len(func.func_defaults or ())
     reqargs = func.func_code.co_argcount - defargs
     if len(args.args) < reqargs:
         if defargs > 0:
-            log.fatal('%s takes at least %d arguments' % (args.shellcode, reqargs))
+            log.critical('%s takes at least %d arguments' % (args.shellcode, reqargs))
+            sys.exit(1)
         else:
-            log.fatal('%s takes exactly %d arguments' % (args.shellcode, reqargs))
+            log.critical('%s takes exactly %d arguments' % (args.shellcode, reqargs))
+            sys.exit(1)
 
     # Captain uglyness saves the day!
     for i, val in enumerate(args.args):
@@ -180,32 +181,32 @@ def main():
             pass
 
     # And he strikes again!
-    os = arch = None
-    for k in args.shellcode.split('.')[:-1]:
-        if k in context.architectures:
-            arch = k
-        elif k in context.oses:
-            os = k
-
+    map(common.context_arg, args.shellcode.split('.'))
     code = func(*args.args)
 
     if args.format in ['a', 'asm', 'assembly']:
         print code
         exit()
     if args.format == 'p':
-        print pwnlib.asm.cpp(code, arch = arch, os = os)
+        print cpp(code)
         exit()
 
-    code = pwnlib.asm.asm(code, arch = arch, os = os)
+    code = asm(code)
+
+    if args.debug:
+        arch = args.shellcode.split('.')[0]
+        proc = gdb.debug_shellcode(code, arch=arch)
+        proc.interactive()
+        sys.exit(0)
 
     if args.format in ['s', 'str', 'string']:
         code = _string(code)
     elif args.format == 'c':
-        code = _carray(code)
+        code = '{' + ', '.join(map(hex, bytearray(code))) + '}' + '\n'
     elif args.format in ['h', 'hex']:
-        code = _hex(code)
+        code = pwnlib.util.fiddling.enhex(code) + '\n'
     elif args.format in ['i', 'hexii']:
-        code = pwnlib.util.fiddling.hexii(code) + '\n'
+        code = hexii(code) + '\n'
 
     if not sys.stdin.isatty():
         sys.stdout.write(sys.stdin.read())

pwnlib/commandline/unhex.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python2
-import sys, argparse
+import argparse
+import sys
 from string import whitespace
 
 parser = argparse.ArgumentParser(description='''
@@ -19,4 +20,4 @@ def main():
     except TypeError, e:
         sys.stderr.write(str(e) + '\n')
 
-if __name__ == '__main__': main()
+if __name__ == '__main__': main()

pwnlib/constants/__init__.py

@@ -3,13 +3,21 @@
 The purpose of this module is to provide quick access to constants from
 different architectures and operating systems.
 
+The constants are wrapped by a convenience class that allows accessing
+the name of the constant, while performing all normal mathematical
+operations on it.
+
 Example:
 
-    >>> print constants.freebsd.SYS_stat
+    >>> str(constants.freebsd.SYS_stat)
+    'SYS_stat'
+    >>> int(constants.freebsd.SYS_stat)
     188
-    >>> print constants.linux.i386.SYS_stat
+    >>> hex(constants.freebsd.SYS_stat)
+    '0xbc'
+    >>> 0 | constants.linux.i386.SYS_stat
     106
-    >>> print constants.linux.amd64.SYS_stat
+    >>> 0 + constants.linux.amd64.SYS_stat
     4
 
 The submodule ``freebsd`` contains all constants for FreeBSD, while the
@@ -22,54 +30,119 @@ what happens in :mod:`pwnlib.shellcraft`.
 Example:
 
     >>> with context.local(os = 'freebsd'):
-    ...     print constants.SYS_stat
+    ...     print int(constants.SYS_stat)
     188
     >>> with context.local(os = 'linux', arch = 'i386'):
-    ...     print constants.SYS_stat
+    ...     print int(constants.SYS_stat)
     106
     >>> with context.local(os = 'linux', arch = 'amd64'):
-    ...     print constants.SYS_stat
+    ...     print int(constants.SYS_stat)
     4
 
 """
+import importlib
+import sys
 from types import ModuleType
-import importlib, sys
+
 from ..context import context
+from ..util import safeeval
 
-class module(ModuleType):
-    def __init__(self, submodules):
-        super(module, self).__init__(__name__)
 
-        # Insert nice properties
-        self.__dict__.update({
-            '__doc__':     __doc__,
-            '__file__':    __file__,
-            '__package__': __package__,
-            '__path__':    __path__,
-            '__all__':     submodules,
-        })
+class ConstantsModule(ModuleType):
+    """
+    ModuleType specialization in order to automatically
+    route queries down to the correct module based on the
+    current context arch / os.
+
+        >>> with context.local(arch = 'i386', os = 'linux'):
+        ...    print constants.SYS_execve + constants.PROT_WRITE
+        13
+        >>> with context.local(arch = 'amd64', os = 'linux'):
+        ...    print constants.SYS_execve + constants.PROT_WRITE
+        61
+        >>> with context.local(arch = 'amd64', os = 'linux'):
+        ...    print constants.SYS_execve + constants.PROT_WRITE
+        61
+        >>> False
+        True
+
+    """
+    possible_submodules = set(context.oses) | set(context.architectures)
+
+    def __init__(self, name, module):
+        super(ConstantsModule, self).__init__(name)
+        self.__dict__.update(module.__dict__)
+        self._env_store = {}
+
+    def guess(self):
+        if context.os in self.__name__ and context.arch in self.__name__:
+            return self
+
+        mod = self
+        mod = getattr(mod, context.os, mod)
+        mod = getattr(mod, context.arch, mod)
+        return mod
+
+    def __dir__(self):
+        return self.__all__
 
     def __getattr__(self, key):
-        if key in self.__all__:
-            mod = importlib.import_module('.' + key, __package__)
-            setattr(self, key, mod)
-            return mod
+        # Special case for __all__, we want to return the contextually
+        # relevant module.
+        if key == '__all__':
+            return self.guess().__dict__.keys()
 
-        if context.os in self.__all__:
-            return getattr(getattr(self, context.os), key)
+        # Special case for all other special properties which aren't defined
+        if key.endswith('__'):
+            raise AttributeError
+
+        # This code is only hit if the attribute doesn't already exist.
+        # Attempt to import a module by the specified name.
+        if key in self.possible_submodules:
+            try:
+                mod = importlib.import_module('.' + key, self.__name__)
+                mod = ConstantsModule(mod.__name__, mod)
+                setattr(self, key, mod)
+                sys.modules[mod.__name__] = mod
+                return mod
+            except ImportError:
+                pass
+        else:
+            mod = self.guess()
+            if hasattr(mod, key):
+                return getattr(mod, key)
 
         raise AttributeError("'module' object has no attribute '%s'" % key)
 
-    def __dir__(self):
-        result = list(self.__all__)
-        if context.os in self.__all__:
-            result.extend(dir(getattr(self, context.os)))
+    def eval(self, string):
+        """eval(string) -> value
 
+        Evaluates a string in the context of values of this module.
+
+        Example:
+
+            >>> with context.local(arch = 'i386', os = 'linux'):
+            ...    print constants.eval('SYS_execve + PROT_WRITE')
+            13
+            >>> with context.local(arch = 'amd64', os = 'linux'):
+            ...    print constants.eval('SYS_execve + PROT_WRITE')
+            61
+            >>> with context.local(arch = 'amd64', os = 'linux'):
+            ...    print constants.eval('SYS_execve + PROT_WRITE')
+            61
+        """
+        if not isinstance(string, str):
+            return string
+
+        key = context.os, context.arch
+        if key not in self._env_store:
+            self._env_store[key] = {key: getattr(self, key) for key in dir(self) if not key.endswith('__')}
+
+        return safeeval.values(string, self._env_store[key])
 
-        return result
 
 # To prevent garbage collection
 tether = sys.modules[__name__]
 
 # Create the module structure
-sys.modules[__name__] = module(['linux', 'freebsd'])
+sys.modules[__name__] = ConstantsModule(__name__, tether)

pwnlib/constants/constant.py

@@ -0,0 +1,9 @@
+class Constant(int):
+    def __new__(cls, s, i):
+        obj = super(Constant, cls).__new__(cls, i)
+        obj.s = s
+        return obj
+    def __str__(self):
+        return self.s
+    def __repr__(self):
+        return 'Constant(%r, %#x)' % (self.s,int(self))

pwnlib/constants/linux/__init__.py

@@ -1,40 +0,0 @@
-from types import ModuleType
-import importlib, sys
-from ...context import context
-
-class module(ModuleType):
-    def __init__(self, submodules):
-        super(module, self).__init__(__name__)
-
-        # Insert nice properties
-        self.__dict__.update({
-            '__file__':    __file__,
-            '__package__': __package__,
-            '__path__':    __path__,
-            '__all__':     submodules,
-        })
-
-    def __getattr__(self, key):
-        if key in self.__all__:
-            mod = importlib.import_module('.' + key, __package__)
-            setattr(self, key, mod)
-            return mod
-
-        if context.arch in self.__all__:
-            return getattr(getattr(self, context.arch), key)
-
-        raise AttributeError("'module' object has no attribute '%s'" % key)
-
-    def __dir__(self):
-        result = list(self.__all__)
-        if context.arch in self.__all__:
-            result.extend(dir(getattr(self, context.arch)))
-
-
-        return result
-
-# To prevent garbage collection
-tether = sys.modules[__name__]
-
-# Create the module structure
-sys.modules[__name__] = module(['amd64', 'arm', 'i386', 'mips', 'thumb'])

pwnlib/constants/linux/powerpc.py

@@ -0,0 +1 @@
+from pwnlib.constants.constant import Constant

pwnlib/constants/linux/powerpc64.py

@@ -0,0 +1 @@
+from pwnlib.constants.constant import Constant

pwnlib/constants/linux/s390x.py

@@ -0,0 +1 @@
+from pwnlib.constants.constant import Constant

pwnlib/constants/linux/sparc64.py

@@ -0,0 +1 @@
+from pwnlib.constants.constant import Constant

pwnlib/context/__init__.py

@@ -4,9 +4,14 @@
 Implements context management so that nested/scoped contexts and threaded
 contexts work properly and as expected.
 """
-import threading, collections, string, logging
+import collections
+import logging
+import string
+import threading
+
 from ..timeout import Timeout
 
+
 class _defaultdict(dict):
     """
     Dictionary which loads missing keys from another dictionary.
@@ -301,6 +306,7 @@ class ContextType(object):
     #: Default values for :class:`pwnlib.context.ContextType`
     defaults = {
         'arch': 'i386',
+        'binary': None,
         'bits': 32,
         'endian': 'little',
         'log_level': logging.INFO,
@@ -308,6 +314,7 @@ class ContextType(object):
         'os': 'linux',
         'signed': False,
         'timeout': Timeout.maximum,
+        'terminal': None,
     }
 
     #: Valid values for :meth:`pwnlib.context.ContextType.os`
@@ -413,7 +420,7 @@ class ContextType(object):
 
             context.update({'os': 'linux', 'arch': 'arm'})
 
-        Args:
+        Arguments:
           kwargs: Variables to be assigned in the environment.
 
         Examples:
@@ -440,7 +447,7 @@ class ContextType(object):
 
         For more information, see the example below or PEP 343.
 
-        Args:
+        Arguments:
           kwargs: Variables to be assigned in the new environment.
 
         Returns:
@@ -559,14 +566,14 @@ class ContextType(object):
             True
         """
 
-        # Lowercase, remove everything non-alphanumeric
+        # Lowercase
         arch = arch.lower()
-        arch = arch.replace(string.punctuation, '')
 
         # Attempt to perform convenience and legacy compatibility
         # transformations.
-        transform = {'x86':'i386', 'ppc': 'powerpc', 'x86_64': 'amd64'}
-        for k, v in transform.items():
+        # We have to make sure that x86_64 appears before x86 for this to work correctly.
+        transform = [('ppc', 'powerpc'), ('x86_64', 'amd64'), ('x86', 'i386')]
+        for k, v in transform:
             if arch.startswith(k):
                 arch = arch.replace(k,v,1)
 
@@ -598,15 +605,44 @@ class ContextType(object):
             >>> context.bits = -1 #doctest: +ELLIPSIS
             Traceback (most recent call last):
             ...
-            AttributeError: bits must be >= 0 (-1)
+            AttributeError: bits must be > 0 (-1)
         """
         bits = int(bits)
 
         if bits <= 0:
-            raise AttributeError("bits must be >= 0 (%r)" % bits)
+            raise AttributeError("bits must be > 0 (%r)" % bits)
 
         return bits
 
+    @_validator
+    def binary(self, binary):
+        """
+        Infer target architecture, bit-with, and endianness from a binary file.
+        Data type is a :class:`pwnlib.elf.ELF` object.
+
+        Examples:
+
+            >>> context.clear()
+            >>> context.arch, context.bits
+            ('i386', 32)
+            >>> context.binary = '/bin/bash'
+            >>> context.arch, context.bits
+            ('amd64', 64)
+            >>> context.binary
+            ELF('/bin/bash')
+
+        """
+        # Cyclic imports... sorry Idolf.
+        from ..elf     import ELF
+
+        e = ELF(binary)
+
+        self.arch   = e.arch
+        self.bits   = e.bits
+        self.endian = e.endian
+
+        return e
+
     @property
     def bytes(self):
         """
@@ -623,7 +659,7 @@ class ContextType(object):
             >>> context.bytes = 0 #doctest: +ELLIPSIS
             Traceback (most recent call last):
             ...
-            AttributeError: bits must be >= 0 (0)
+            AttributeError: bits must be > 0 (0)
         """
         return self.bits/8
     @bytes.setter
@@ -673,6 +709,10 @@ class ContextType(object):
         """
         Sets the verbosity of ``pwntools`` logging mechanism.
 
+        More specifically it controls the filtering of messages that happens
+        inside the handler for logging to the screen. So if you want e.g. log
+        all messages to a file, then this attribute makes no difference to you.
+
         Valid values are specified by the standard Python ``logging`` module.
 
         Default value is set to ``INFO``.
@@ -776,6 +816,17 @@ class ContextType(object):
         """
         return Timeout(value).timeout
 
+    @_validator
+    def terminal(self, value):
+        """
+        Default terminal used by :meth:`pwnlib.util.misc.run_in_new_terminal`.
+        Can be a string or an iterable of strings.  In the latter case the first
+        entry is the terminal and the rest are default arguments.
+        """
+        if isinstance(value, (str, unicode)):
+            return [value]
+        return value
+
     #*************************************************************************
     #                               ALIASES
     #*************************************************************************

pwnlib/data/includes/generator/generate_constants.sh

@@ -9,7 +9,7 @@ headers="../"
 
 rm -rf "$headers/linux" "$headers/freebsd.h"
 mkdir -p "$headers/linux"
-find "$python" -type f \( -name '*.py' -o -name '*.pyc' \) -not -name "__init__.py" -delete
+find "$python" -type f \( -name '*.py' -o -name '*.pyc' \) -not -name "__init__.py"  -not -name 'constant.py' -delete
 find "$python" -type d -empty -delete
 
 for arch in $ARCHS; do

pwnlib/data/includes/generator/linux/aarch64.h

@@ -0,0 +1,4 @@
+#define __arm__
+#define __aarch64__
+#include <aarch64/syscalls.h>
+#include <common.h>

pwnlib/data/includes/generator/linux/alpha.h

@@ -0,0 +1,3 @@
+#define __alpha__
+#include <alpha/syscalls.h>
+#include <common.h>

pwnlib/data/includes/generator/linux/diet/aarch64/syscalls.h

@@ -0,0 +1,323 @@
+/*
+ * This file contains the system call numbers.
+ */
+
+#define __NR_io_setup 0
+#define __NR_io_destroy 1
+#define __NR_io_submit 2
+#define __NR_io_cancel 3
+#define __NR_io_getevents 4
+#define __NR_setxattr 5
+#define __NR_lsetxattr 6
+#define __NR_fsetxattr 7
+#define __NR_getxattr 8
+#define __NR_lgetxattr 9
+#define __NR_fgetxattr 10
+#define __NR_listxattr 11
+#define __NR_llistxattr 12
+#define __NR_flistxattr 13
+#define __NR_removexattr 14
+#define __NR_lremovexattr 15
+#define __NR_fremovexattr 16
+#define __NR_getcwd 17
+#define __NR_lookup_dcookie 18
+#define __NR_eventfd2 19
+#define __NR_epoll_create1 20
+#define __NR_epoll_ctl 21
+#define __NR_epoll_pwait 22
+#define __NR_dup 23
+#define __NR_dup3 24
+#define __NR_fcntl 25
+#define __NR_inotify_init1 26
+#define __NR_inotify_add_watch 27
+#define __NR_inotify_rm_watch 28
+#define __NR_ioctl 29
+#define __NR_ioprio_set 30
+#define __NR_ioprio_get 31
+#define __NR_flock 32
+#define __NR_mknodat 33
+#define __NR_mkdirat 34
+#define __NR_unlinkat 35
+#define __NR_symlinkat 36
+#define __NR_linkat 37
+#define __NR_renameat 38
+#define __NR_umount2 39
+#define __NR_mount 40
+#define __NR_pivot_root 41
+#define __NR_nfsservctl 42
+#define __NR_statfs 43
+#define __NR_fstatfs 44
+#define __NR_truncate 45
+#define __NR_ftruncate 46
+#define __NR_fallocate 47
+#define __NR_faccessat 48
+#define __NR_chdir 49
+#define __NR_fchdir 50
+#define __NR_chroot 51
+#define __NR_fchmod 52
+#define __NR_fchmodat 53
+#define __NR_fchownat 54
+#define __NR_fchown 55
+#define __NR_openat 56
+#define __NR_close 57
+#define __NR_vhangup 58
+#define __NR_pipe2 59
+#define __NR_quotactl 60
+#define __NR_getdents64 61
+#define __NR_lseek 62
+#define __NR_read 63
+#define __NR_write 64
+#define __NR_readv 65
+#define __NR_writev 66
+#define __NR_pread64 67
+#define __NR_pwrite64 68
+#define __NR_preadv 69
+#define __NR_pwritev 70
+#define __NR_sendfile 71
+#define __NR_pselect6 72
+#define __NR_ppoll 73
+#define __NR_signalfd4 74
+#define __NR_vmsplice 75
+#define __NR_splice 76
+#define __NR_tee 77
+#define __NR_readlinkat 78
+#define __NR_fstatat64 79
+#define __NR_fstat 80
+#define __NR_sync 81
+#define __NR_fsync 82
+#define __NR_fdatasync 83
+#define __NR_sync_file_range2 84
+/* #define __NR_sync_file_range 84 */
+#define __NR_timerfd_create 85
+#define __NR_timerfd_settime 86
+#define __NR_timerfd_gettime 87
+#define __NR_utimensat 88
+#define __NR_acct 89
+#define __NR_capget 90
+#define __NR_capset 91
+#define __NR_personality 92
+#define __NR_exit 93
+#define __NR_exit_group 94
+#define __NR_waitid 95
+#define __NR_set_tid_address 96
+#define __NR_unshare 97
+#define __NR_futex 98
+#define __NR_set_robust_list 99
+#define __NR_get_robust_list 100
+#define __NR_nanosleep 101
+#define __NR_getitimer 102
+#define __NR_setitimer 103
+#define __NR_kexec_load 104
+#define __NR_init_module 105
+#define __NR_delete_module 106
+#define __NR_timer_create 107
+#define __NR_timer_gettime 108
+#define __NR_timer_getoverrun 109
+#define __NR_timer_settime 110
+#define __NR_timer_delete 111
+#define __NR_clock_settime 112
+#define __NR_clock_gettime 113
+#define __NR_clock_getres 114
+#define __NR_clock_nanosleep 115
+#define __NR_syslog 116
+#define __NR_ptrace 117
+#define __NR_sched_setparam 118
+#define __NR_sched_setscheduler 119
+#define __NR_sched_getscheduler 120
+#define __NR_sched_getparam 121
+#define __NR_sched_setaffinity 122
+#define __NR_sched_getaffinity 123
+#define __NR_sched_yield 124
+#define __NR_sched_get_priority_max 125
+#define __NR_sched_get_priority_min 126
+#define __NR_sched_rr_get_interval 127
+#define __NR_restart_syscall 128
+#define __NR_kill 129
+#define __NR_tkill 130
+#define __NR_tgkill 131
+#define __NR_sigaltstack 132
+#define __NR_rt_sigsuspend 133
+#define __NR_rt_sigaction 134
+#define __NR_rt_sigprocmask 135
+#define __NR_rt_sigpending 136
+#define __NR_rt_sigtimedwait 137
+#define __NR_rt_sigqueueinfo 138
+#define __NR_rt_sigreturn 139
+#define __NR_setpriority 140
+#define __NR_getpriority 141
+#define __NR_reboot 142
+#define __NR_setregid 143
+#define __NR_setgid 144
+#define __NR_setreuid 145
+#define __NR_setuid 146
+#define __NR_setresuid 147
+#define __NR_getresuid 148
+#define __NR_setresgid 149
+#define __NR_getresgid 150
+#define __NR_setfsuid 151
+#define __NR_setfsgid 152
+#define __NR_times 153
+#define __NR_setpgid 154
+#define __NR_getpgid 155
+#define __NR_getsid 156
+#define __NR_setsid 157
+#define __NR_getgroups 158
+#define __NR_setgroups 159
+#define __NR_uname 160
+#define __NR_sethostname 161
+#define __NR_setdomainname 162
+#define __NR_getrlimit 163
+#define __NR_setrlimit 164
+#define __NR_getrusage 165
+#define __NR_umask 166
+#define __NR_prctl 167
+#define __NR_getcpu 168
+#define __NR_gettimeofday 169
+#define __NR_settimeofday 170
+#define __NR_adjtimex 171
+#define __NR_getpid 172
+#define __NR_getppid 173
+#define __NR_getuid 174
+#define __NR_geteuid 175
+#define __NR_getgid 176
+#define __NR_getegid 177
+#define __NR_gettid 178
+#define __NR_sysinfo 179
+#define __NR_mq_open 180
+#define __NR_mq_unlink 181
+#define __NR_mq_timedsend 182
+#define __NR_mq_timedreceive 183
+#define __NR_mq_notify 184
+#define __NR_mq_getsetattr 185
+#define __NR_msgget 186
+#define __NR_msgctl 187
+#define __NR_msgrcv 188
+#define __NR_msgsnd 189
+#define __NR_semget 190
+#define __NR_semctl 191
+#define __NR_semtimedop 192
+#define __NR_semop 193
+#define __NR_shmget 194
+#define __NR_shmctl 195
+#define __NR_shmat 196
+#define __NR_shmdt 197
+#define __NR_socket 198
+#define __NR_socketpair 199
+#define __NR_bind 200
+#define __NR_listen 201
+#define __NR_accept 202
+#define __NR_connect 203
+#define __NR_getsockname 204
+#define __NR_getpeername 205
+#define __NR_sendto 206
+#define __NR_recvfrom 207
+#define __NR_setsockopt 208
+#define __NR_getsockopt 209
+#define __NR_shutdown 210
+#define __NR_sendmsg 211
+#define __NR_recvmsg 212
+#define __NR_readahead 213
+#define __NR_brk 214
+#define __NR_munmap 215
+#define __NR_mremap 216
+#define __NR_add_key 217
+#define __NR_request_key 218
+#define __NR_keyctl 219
+#define __NR_clone 220
+#define __NR_execve 221
+#define __NR_mmap 222
+#define __NR_fadvise64 223
+#define __NR_swapon 224
+#define __NR_swapoff 225
+#define __NR_mprotect 226
+#define __NR_msync 227
+#define __NR_mlock 228
+#define __NR_munlock 229
+#define __NR_mlockall 230
+#define __NR_munlockall 231
+#define __NR_mincore 232
+#define __NR_madvise 233
+#define __NR_remap_file_pages 234
+#define __NR_mbind 235
+#define __NR_get_mempolicy 236
+#define __NR_set_mempolicy 237
+#define __NR_migrate_pages 238
+#define __NR_move_pages 239
+#define __NR_rt_tgsigqueueinfo 240
+#define __NR_perf_event_open 241
+#define __NR_accept4 242
+#define __NR_recvmmsg 243
+#define __NR_arch_specific_syscall 244
+#define __NR_wait4 260
+#define __NR_prlimit64 261
+#define __NR_fanotify_init 262
+#define __NR_fanotify_mark 263
+#define __NR_name_to_handle_at         264
+#define __NR_open_by_handle_at         265
+#define __NR_clock_adjtime 266
+#define __NR_syncfs 267
+#define __NR_setns 268
+#define __NR_sendmmsg 269
+#define __NR_process_vm_readv 270
+#define __NR_process_vm_writev 271
+#define __NR_kcmp 272
+#define __NR_finit_module 273
+#define __NR_open 1024
+#define __NR_link 1025
+#define __NR_unlink 1026
+#define __NR_mknod 1027
+#define __NR_chmod 1028
+#define __NR_chown 1029
+#define __NR_mkdir 1030
+#define __NR_rmdir 1031
+#define __NR_lchown 1032
+#define __NR_access 1033
+#define __NR_rename 1034
+#define __NR_readlink 1035
+#define __NR_symlink 1036
+#define __NR_utimes 1037
+#define __NR_stat 1038
+#define __NR_lstat 1039
+#define __NR_pipe 1040
+#define __NR_dup2 1041
+#define __NR_epoll_create 1042
+#define __NR_inotify_init 1043
+#define __NR_eventfd 1044
+#define __NR_signalfd 1045
+#define __NR_sendfile64 1046
+#define __NR_ftruncate64 1047
+#define __NR_truncate64 1048
+#define __NR_stat64 1049
+#define __NR_lstat64 1050
+#define __NR_fstat64 1051
+#define __NR_fcntl64 1052
+/* #define __NR_fadvise64 1053 */
+#define __NR_newfstatat 1054
+#define __NR_fstatfs64 1055
+#define __NR_statfs64 1056
+#define __NR_lseek64 1057
+#define __NR_mmap64 1058
+#define __NR_alarm 1059
+#define __NR_getpgrp 1060
+#define __NR_pause 1061
+#define __NR_time 1062
+#define __NR_utime 1063
+#define __NR_creat 1064
+#define __NR_getdents 1065
+#define __NR_futimesat 1066
+#define __NR_select 1067
+#define __NR_poll 1068
+#define __NR_epoll_wait 1069
+#define __NR_ustat 1070
+#define __NR_vfork 1071
+#define __NR_oldwait4 1072
+#define __NR_recv 1073
+#define __NR_send 1074
+#define __NR_bdflush 1075
+#define __NR_umount 1076
+#define __NR_uselib 1077
+#define __NR__sysctl 1078
+#define __NR_fork 1079
+#define __NR_syscalls (__NR_fork+1)
+
+#define __NR_sigreturn 1999

pwnlib/data/includes/generator/linux/ia64.h

@@ -0,0 +1,3 @@
+#define __ia64__
+#include <ia64/syscalls.h>
+#include <common.h>

pwnlib/data/includes/generator/linux/powerpc.h

@@ -0,0 +1,3 @@
+#define __ppc__
+#include <ppc/syscalls.h>
+#include <common.h>

pwnlib/data/includes/generator/linux/powerpc64.h

@@ -0,0 +1,3 @@
+#define __ppc64__
+#include <ppc64/syscalls.h>
+#include <common.h>

pwnlib/data/includes/generator/linux/s390.h

@@ -0,0 +1,3 @@
+#define __s390__
+#include <s390/syscalls.h>
+#include <common.h>

pwnlib/data/includes/generator/linux/s390x.h

@@ -0,0 +1,3 @@
+#define __s390x__
+#include <s390x/syscalls.h>
+#include <common.h>

pwnlib/data/includes/generator/linux/sparc.h

@@ -0,0 +1,3 @@
+#define __sparc__
+#include <sparc/syscalls.h>
+#include <common.h>

pwnlib/data/includes/generator/linux/sparc64.h

@@ -0,0 +1,3 @@
+#define __sparc64__
+#include <sparc64/syscalls.h>
+#include <common.h>

pwnlib/data/includes/generator/load_constants.py

@@ -1,10 +1,14 @@
 #!/usr/bin/env python2
-import sys, re
+import re
+import sys
+
 from pwnlib.util import safeeval
 
 python = open(sys.argv[1], "w")
 header = open(sys.argv[2], "w")
 
+print >>python, 'from pwnlib.constants.constant import Constant'
+
 data = sys.stdin.read().strip().split('\n')
 
 res = ""
@@ -25,7 +29,7 @@ for l in data:
     elif val.endswith('L'):
         val = val[:-1]
 
-    print >> python, "%s = %s" % (key, val)
+    print >> python, "{key} = Constant({key!r},{val})".format(**locals())
     if re.match(r'^0[0-9]', val) or re.match(r'[^0-9a-fA-Fx]0[0-9]', val):
         print >> header, "#define %s %s" % (key, hex(safeeval.expr(val)))
     else:

pwnlib/data/useragents/LICENSE.txt

@@ -6,8 +6,8 @@ The user agent strings are downloaded from:
  - http://www.user-agents.org/
  - http://techpatterns.com/downloads/firefox/useragentswitcher.xml
 
-The latter list is part of the Firefox `User Agent Switcher' add-on which is
-released under GPL3.0.
+The latter list is part of the Firefox `User Agent Switcher' add-on and released
+under BSD 2 Clause.
 
 
 As mentioned elsewhere we are not lawyers and do not know the legal status of

pwnlib/data/useragents/download-useragents.py

@@ -3,8 +3,12 @@
 Script for downloading lists of user agent strings
 """
 
+import os
+import re
+import urllib
+
 from bs4 import BeautifulSoup
-import re, urllib, os
+
 from pwn import *
 
 uas = set()

pwnlib/dynelf.py

@@ -47,16 +47,22 @@ Example
 
 DynELF
 """
-from . import elf
-from .elf     import ELF, constants
-from .memleak import MemLeak
-from .context import context
+import ctypes
 
 from elftools.elf.enums import ENUM_D_TAG
 
-import logging, ctypes
+from . import elf
+from . import libcdb
+from .context import context
+from .elf import ELF
+from .elf import constants
+from .log import getLogger
+from .memleak import MemLeak
+from .util.fiddling import enhex
+from .util.packing import unpack
+from .util.web import wget
 
-log    = logging.getLogger(__name__)
+log    = getLogger(__name__)
 sizeof = ctypes.sizeof
 
 def sysv_hash(symbol):
@@ -133,7 +139,7 @@ class DynELF(object):
     .. _SYSV:      https://refspecs.linuxbase.org/elf/gabi4+/ch5.dynamic.html#hash
     .. _GNU:       https://blogs.oracle.com/ali/entry/gnu_hash_elf_sections
     .. _DT_DEBUG:  https://reverseengineering.stackexchange.com/questions/6525/elf-link-map-when-linked-as-relro
-    .. _link map:  https://google.com
+    .. _link map:  https://sourceware.org/git/?p=glibc.git;a=blob;f=elf/link.h;h=eaca8028e45a859ac280301a6e955a14eed1b887;hb=HEAD#l84
     '''
 
     def __init__(self, leak, pointer=None, elf=None):
@@ -418,6 +424,46 @@ class DynELF(object):
         else:
             self._waitfor.status(msg)
 
+    def libc(self):
+        """libc(self) -> ELF
+
+        Leak the Build ID of the remote libc.so, download the file,
+        and load an ``ELF`` object with the correct base address.
+
+        Returns:
+            An ELF object, or None.
+        """
+        lib = 'libc.so'
+
+        with self.waitfor('Downloading libc'):
+            dynlib = self._dynamic_load_dynelf(lib)
+
+            self.status("Trying lookup based on Build ID")
+            build_id = dynlib._lookup_build_id(lib)
+
+            libbase = self.libbase
+
+            if lib is not None:
+                libbase = self.lookup(symb = None, lib = lib)
+
+            for offset in libcdb.get_build_id_offsets():
+                address = libbase + offset
+                if self.leak.d(address + 0xC) == unpack("GNU\x00", 32):
+                    return enhex(''.join(self.leak.raw(address + 0x10, 20)))
+
+            if not build_id:
+                return None
+
+            self.status("Trying lookup based on Build ID: %s" % build_id)
+            path = libcdb.search_by_build_id(build_id)
+
+            if not path:
+                return None
+
+            libc = ELF(path)
+            libc.address = dynlib.libbase
+            return libc
+
     def lookup (self, symb = None, lib = None):
         """lookup(symb = None, lib = None) -> int
 
@@ -589,12 +635,12 @@ class DynELF(object):
         leak       = self.leak
         Sym        = {32: elf.Elf32_Sym, 64: elf.Elf64_Sym}[self.elfclass]
 
-        nbuckets   = leak.field(hshtab, elf.Elf_HashTable.nbuckets)
+        nbucket   = leak.field(hshtab, elf.Elf_HashTable.nbucket)
         bucketaddr = hshtab + sizeof(elf.Elf_HashTable)
-        chain      = bucketaddr + (nbuckets * 4)
+        chain      = bucketaddr + (nbucket * 4)
 
         self.status('hashmap')
-        hsh = sysv_hash(symb) % nbuckets
+        hsh = sysv_hash(symb) % nbucket
 
         # Get the index out of the bucket for the hash we computed
         idx = leak.d(bucketaddr, hsh)

pwnlib/elf/__init__.py

@@ -1,17 +1,25 @@
 """Exposes functionality for manipulating ELF files
 """
-from ..term import text
-from .datatypes import *
-from ..asm import asm, disasm
-from ..util import misc
+import mmap
+import os
+import subprocess
 
-import mmap, subprocess, os, logging
-from elftools.elf.elffile import ELFFile
-from elftools.elf.sections import SymbolTableSection
+from elftools.elf.constants import P_FLAGS
+from elftools.elf.constants import SHN_INDICES
 from elftools.elf.descriptions import describe_e_type
-from elftools.elf.constants import P_FLAGS, SHN_INDICES
+from elftools.elf.elffile import ELFFile
+from elftools.elf.gnuversions import GNUVerDefSection
+from elftools.elf.relocation import RelocationSection
+from elftools.elf.sections import SymbolTableSection
 
-log = logging.getLogger(__name__)
+from ..asm import asm
+from ..asm import disasm
+from ..log import getLogger
+from ..term import text
+from ..util import misc
+from .datatypes import *
+
+log = getLogger(__name__)
 
 __all__ = ['load', 'ELF'] + sorted(filter(lambda x: not x.startswith('_'), datatypes.__dict__.keys()))
 
@@ -19,7 +27,22 @@ def load(*args, **kwargs):
     """Compatibility wrapper for pwntools v1"""
     return ELF(*args, **kwargs)
 
+# Monkey-patch some things inside elftools to make life easier
+
 class ELF(ELFFile):
+
+    #: Bit-ness of the file
+    bits = None
+
+    #: Architecture of the file
+    arch = None
+
+    #: Endianness of the file
+    endian = None
+
+    #: Path to the file
+    path = None
+
     """Encapsulates information about an ELF file.
 
     :ivar path: Path to the binary on disk
@@ -53,14 +76,17 @@ class ELF(ELFFile):
 
         super(ELF,self).__init__(self.mmap)
 
-        self.path     = os.path.abspath(path)
-
-
-        # Fix difference between elftools and pwntools
+        self.path = os.path.abspath(path)
         self.arch = self.get_machine_arch().lower()
-        if self.arch == 'x64':
-            self.arch = 'amd64'
 
+        # Check endianness
+        self.endian = {
+            'ELFDATANONE': 'little',
+            'ELFDATA2LSB': 'little',
+            'ELFDATA2MSB': 'big'
+        }[self['e_ident']['EI_DATA']]
+
+        self.bits = self.elfclass
 
         self._populate_got_plt()
         self._populate_symbols()
@@ -72,12 +98,31 @@ class ELF(ELFFile):
             self._address = min(filter(bool, (s.header.p_vaddr for s in self.segments)))
         self.load_addr = self._address
 
-        if self.execstack:
-            log.info('Stack is executable!')
+        self._describe()
+
+    def _describe(self):
+        log.info_once('\n'.join((repr(self.path),
+                                'Arch:          %s-%s-%s' % (self.arch, self.bits, self.endian),
+                                self.checksec())))
 
     def __repr__(self):
         return "ELF(%r)" % self.path
 
+    def get_machine_arch(self):
+        return {
+            'EM_X86_64': 'amd64',
+            'EM_386' :'i386',
+            'EM_486': 'i386',
+            'EM_ARM': 'arm',
+            'EM_AARCH64': 'aarch64',
+            'EM_MIPS': 'mips',
+            'EM_PPC': 'powerpc',
+            'EM_PPC64': 'powerpc64',
+            'EM_SPARC32PLUS': 'sparc',
+            'EM_SPARCV9': 'sparc64',
+            'EM_IA_64': 'ia64'
+        }.get(self['e_machine'], self['e_machine'])
+
     @property
     def entry(self):
         """Entry point to the ELF"""
@@ -145,7 +190,7 @@ class ELF(ELFFile):
     def section(self, name):
         """Gets data for the named section
 
-        Args:
+        Arguments:
             name(str): Name of the section
 
         Returns:
@@ -153,9 +198,21 @@ class ELF(ELFFile):
         """
         return self.get_section_by_name(name).data()
 
+    @property
+    def rwx_segments(self):
+        """Returns: list of all segments which are writeable and executable."""
+        if not self.nx:
+            return self.writable_segments
+
+        wx = P_FLAGS.PF_X | P_FLAGS.PF_W
+        return [s for s in self.segments if s.header.p_flags & wx == wx]
+
     @property
     def executable_segments(self):
         """Returns: list of all segments which are executable."""
+        if not self.nx:
+            return list(self.segments)
+
         return [s for s in self.segments if s.header.p_flags & P_FLAGS.PF_X]
 
     @property
@@ -247,7 +304,17 @@ class ELF(ELFFile):
             return
 
         # Find the relocation section for PLT
-        rel_plt = next(s for s in self.sections if s.header.sh_info == self.sections.index(plt))
+        try:
+            rel_plt = next(s for s in self.sections if
+                            s.header.sh_info == self.sections.index(plt) and
+                            isinstance(s, RelocationSection))
+        except StopIteration:
+            # Evidently whatever android-ndk uses to build binaries zeroes out sh_info for rel.plt
+            rel_plt = self.get_section_by_name('.rel.plt') or self.get_section_by_name('.rela.plt')
+
+        if not rel_plt:
+            log.warning("Couldn't find relocations against PLT to get symbols")
+            return
 
         if rel_plt.header.sh_link != SHN_INDICES.SHN_UNDEF:
             # Find the symbols for the relocation section
@@ -268,10 +335,11 @@ class ELF(ELFFile):
 
         # Map architecture: offset, multiplier
         header_size, entry_size = {
-            'x86':   (0x10, 0x10),
+            'i386':   (0x10, 0x10),
             'amd64': (0x10, 0x10),
-            'arm':   (0x14, 0xC)
-        }[self.arch]
+            'arm':   (0x14, 0xC),
+            'aarch64': (0x20, 0x20),
+        }.get(self.arch, (0,0))
 
 
         # Based on the ordering of the GOT symbols, populate the PLT
@@ -283,7 +351,7 @@ class ELF(ELFFile):
 
         Search the ELF's virtual address space for the specified string.
 
-        Args:
+        Arguments:
             needle(str): String to search for.
             writable(bool): Search only writable sections.
 
@@ -324,7 +392,7 @@ class ELF(ELFFile):
     def offset_to_vaddr(self, offset):
         """Translates the specified offset to a virtual address.
 
-        Args:
+        Arguments:
             offset(int): Offset to translate
 
         Returns:
@@ -353,7 +421,7 @@ class ELF(ELFFile):
     def vaddr_to_offset(self, address):
         """Translates the specified virtual address to a file address
 
-        Args:
+        Arguments:
             address(int): Virtual address to translate
 
         Returns:
@@ -384,7 +452,7 @@ class ELF(ELFFile):
     def read(self, address, count):
         """Read data from the specified virtual address
 
-        Args:
+        Arguments:
             address(int): Virtual address to read
             count(int): Number of bytes to read
 
@@ -410,7 +478,7 @@ class ELF(ELFFile):
     def write(self, address, data):
         """Writes data to the specified virtual address
 
-        Args:
+        Arguments:
             address(int): Virtual address to write
             data(str): Bytes to write
 
@@ -468,6 +536,10 @@ class ELF(ELFFile):
         self.stream.seek(old)
         return data
 
+    @property
+    def data(self):
+        return self.get_data()
+
     def disasm(self, address, n_bytes):
         """Returns a string of disassembled instructions at
         the specified virtual memory address"""
@@ -530,7 +602,12 @@ class ELF(ELFFile):
 
     @property
     def nx(self):
-        return not any('GNU_STACK' in seg.header.p_type for seg in self.executable_segments)
+        if not any('GNU_STACK' in seg.header.p_type for seg in self.segments):
+            return False
+
+        # Can't call self.executable_segments because of dependency loop.
+        exec_seg = [s for s in self.segments if s.header.p_flags & P_FLAGS.PF_X]
+        return not any('GNU_STACK' in seg.header.p_type for seg in exec_seg)
 
     @property
     def execstack(self):
@@ -589,19 +666,32 @@ class ELF(ELFFile):
             "PIE:".ljust(15) + {
                 True: green("PIE enabled"),
                 False: red("No PIE")
-            }[self.pie],
-            "RPATH:".ljust(15) + {
-                False:  green("No RPATH"),
-                True:   red(repr(self.rpath))
-            }.get(bool(self.rpath)),
-            "RUNPATH:".ljust(15) + {
-                False:  green("No RUNPATH"),
-                True:   red(repr(self.runpath))
-            }.get(bool(self.runpath))
+            }[self.pie]
         ]
 
+        # Are there any RWX areas in the binary?
+        #
+        # This will occur if NX is disabled and *any* area is
+        # RW, or can expressly occur.
+        rwx = self.rwx_segments
+
+        if self.nx and rwx:
+            res += [ "RWX:".ljust(15) + red("Has RWX segments") ]
+
+        if self.rpath:
+            res += [ "RPATH:".ljust(15) + red(repr(self.rpath)) ]
+
+        if self.runpath:
+            res += [ "RUNPATH:".ljust(15) + red(repr(self.runpath)) ]
+
         if self.packed:
             res.append('Packer:'.ljust(15) + red("Packed with UPX"))
 
         return '\n'.join(res)
 
+    @property
+    def buildid(self):
+        section = self.get_section_by_name('.note.gnu.build-id')
+        if section:
+            return section.data()[16:]
+        return None

pwnlib/exception.py

@@ -1,5 +1,7 @@
 __all__ = ['PwnlibException']
-import traceback, sys
+import sys
+import traceback
+
 
 class PwnlibException(Exception):
     '''Exception thrown by :func:`pwnlib.log.error`.

pwnlib/fmtstr.py

@@ -0,0 +1,93 @@
+import logging
+import re
+
+from pwnlib.log import getLogger
+from pwnlib.memleak import MemLeak
+from pwnlib.util.cyclic import *
+from pwnlib.util.fiddling import randoms
+from pwnlib.util.packing import *
+
+log = getLogger(__name__)
+
+class FmtStr(object):
+    def __init__(self, execute_fmt, offset = None, padlen = 0, numbwritten = 0):
+        self.execute_fmt = execute_fmt
+        self.offset = offset
+        self.padlen = padlen
+        self.numbwritten = numbwritten
+
+
+        if self.offset == None:
+            self.offset, self.padlen = self.find_offset()
+            log.info("Found format string offset: %d", self.offset)
+
+        self.writes = []
+        self.leaker = MemLeak(self._leaker)
+
+    def leak_stack(self, offset, prefix=""):
+        leak = self.execute_fmt(prefix+"START%%%d$pEND" % offset)
+        try:
+            leak = re.findall(r"START(.*)END", leak, re.MULTILINE | re.DOTALL)[0]
+            leak = int(leak, 16)
+        except ValueError:
+            leak = 0
+        return leak
+
+    def find_offset(self):
+        marker = cyclic(20)
+        for off in range(1,1000):
+            leak = self.leak_stack(off, marker)
+            leak = pack(leak)
+
+            pad = cyclic_find(leak)
+            if pad >= 0 and pad < 20:
+                return off, pad
+        else:
+            log.error("Could not find offset to format string on stack")
+            return None, None
+
+    def _leaker(self, addr):
+        # Hack: elfheaders often start at offset 0 in a page,
+        # but we often can't leak addresses containing null bytes,
+        # and the page below elfheaders is often not mapped.
+        # Thus the solution to this problem is to check if the next 3 bytes are
+        # "ELF" and if so we lie and leak "\x7f"
+        # unless it is leaked otherwise.
+        if addr & 0xfff == 0 and self.leaker._leak(addr+1, 3, False) == "ELF":
+            return "\x7f"
+
+        fmtstr = randoms(self.padlen) + pack(addr) + "START%%%d$sEND" % self.offset
+
+        leak = self.execute_fmt(fmtstr)
+        leak = re.findall(r"START(.*)END", leak, re.MULTILINE | re.DOTALL)[0]
+
+        leak += "\x00"
+
+        return leak
+
+    def execute_writes(self):
+        addrs = []
+        bytes = []
+
+        #convert every write into single-byte writes
+        for addr, data in self.writes:
+            data = flat(data)
+            for off, b in enumerate(data):
+                addrs.append(addr+off)
+                bytes.append(u8(b))
+
+        fmtstr = randoms(self.padlen) + flat(addrs)
+        n = self.numbwritten + len(fmtstr)
+
+        for i, b in enumerate(bytes):
+            b -= (n % 256)
+            if b <= 0:
+                b += 256
+            fmtstr += "%%%dc%%%d$hhn" % (b, self.offset + i)
+            n += b
+
+        self.execute_fmt(fmtstr)
+        self.writes = []
+
+    def write(self, addr, data):
+        self.writes.append((addr, data))

pwnlib/gdb.py

@@ -1,16 +1,60 @@
-import os, tempfile, re, shlex, logging
-from .util import misc, proc
-from . import tubes, elf
+import os
+import random
+import re
+import shlex
+import tempfile
 
-log = logging.getLogger(__name__)
+from . import atexit
+from . import elf
+from . import tubes
+from .asm import make_elf
+from .context import context
+from .log import getLogger
+from .util import misc
+from .util import proc
 
-def debug(args, exe=None, execute=None, ssh=None):
+log = getLogger(__name__)
+
+def debug_shellcode(data, execute=None, **kwargs):
+    """
+    Creates an ELF file, and launches it with GDB.
+
+    Arguments:
+        data(str): Assembled shellcode bytes
+        kwargs(dict): Arguments passed to context (e.g. arch='arm')
+
+    Returns:
+        A ``process`` tube connected to the shellcode on stdin/stdout/stderr.
+    """
+    with context.local(**kwargs):
+        tmp_elf  = tempfile.mktemp(prefix='pwn', suffix='.elf')
+        elf_data = make_elf(data)
+        with open(tmp_elf,'wb+') as f:
+            f.write(elf_data)
+            f.flush()
+        os.chmod(tmp_elf, 0777)
+        atexit.register(lambda: os.unlink(tmp_elf))
+        return debug(tmp_elf, execute=None, arch=context.arch)
+
+def get_qemu_arch(arch):
+    if arch == 'mips' and context.endian == 'little':
+        return 'mipsel'
+    if arch == 'arm' and context.endian == 'big':
+        return 'armeb'
+    if arch == 'amd64':
+        return 'x86_64'
+
+    arch = arch.replace('powerpc', 'ppc')
+
+    return arch
+
+def debug(args, exe=None, execute=None, ssh=None, arch=None):
     """debug(args) -> tube
 
     Launch a GDB server with the specified command line,
     and launches GDB to attach to it.
 
-    Args:
+    Arguments:
         args: Same args as passed to pwnlib.tubes.process
         ssh: Remote ssh session to use to launch the process.
           Automatically sets up port forwarding so that gdb runs locally.
@@ -18,27 +62,41 @@ def debug(args, exe=None, execute=None, ssh=None):
     Returns:
         A tube connected to the target process
     """
-    args      = ['gdbserver', 'localhost:0'] + args
+    if isinstance(args, (str, unicode)):
+        args = [args]
+
+    orig_args = args
+
+    if not arch:
+        args = ['gdbserver', 'localhost:0'] + args
+    else:
+        qemu_port = random.randint(1024, 65535)
+        qemu_arch = get_qemu_arch(arch)
+        args = ['qemu-%s-static' % qemu_arch, '-g', str(qemu_port)] + args
 
     if not ssh:
-        execute = tubes.process.process
+        runner  = tubes.process.process
         which   = misc.which
     if ssh:
-        execute = ssh.run
+        runner  = ssh.run
         which   = ssh.which
 
     # Make sure gdbserver is installed
-    if not which('gdbserver'):
-        log.error("gdbserver is not installed")
+    if not which(args[0]):
+        log.error("%s is not installed" % args[0])
 
-    gdbserver = execute(args)
+    with context.local(log_level='debug'):
+        gdbserver = runner(args)
 
-    # Process /bin/bash created; pid = 14366
-    # Listening on port 34816
-    process_created = gdbserver.recvline()
-    listening_on    = gdbserver.recvline()
+    if not arch:
+        # Process /bin/bash created; pid = 14366
+        # Listening on port 34816
+        process_created = gdbserver.recvline()
+        listening_on    = gdbserver.recvline()
 
-    port     = int(listening_on.split()[-1])
+        port = int(listening_on.split()[-1])
+    else:
+        port = qemu_port
 
     listener = remote = None
 
@@ -47,15 +105,23 @@ def debug(args, exe=None, execute=None, ssh=None):
         listener = tubes.listen.listen(0)
         port     = listener.lport
     elif not exe:
-        exe = misc.which(args[0])
+        exe = misc.which(orig_args[0])
 
-    attach(('127.0.0.1', port), exe=exe)
+    attach(('127.0.0.1', port), exe=orig_args[0], execute=execute, arch=context.arch)
 
     if ssh:
         remote <> listener.wait_for_connection()
 
     return gdbserver
 
+def get_gdb_arch(arch):
+    return {
+        'amd64': 'i386:x86-64',
+        'powerpc': 'powerpc:403',
+        'powerpc64': 'powerpc:e5500'
+    }.get(arch, arch)
+
+
 def attach(target, execute = None, exe = None, arch = None):
     """attach(target, execute = None, exe = None, arch = None) -> None
 
@@ -72,7 +138,7 @@ def attach(target, execute = None, exe = None, arch = None):
 
     If `gdb-multiarch` is installed we use that or 'gdb' otherwise.
 
-    Args:
+    Arguments:
       target: The target to attach to.
       execute (str or file): GDB script to run after attaching.
       exe (str): The path of the target binary.
@@ -89,6 +155,7 @@ def attach(target, execute = None, exe = None, arch = None):
             msg =  'Disable ptrace_scope to attach to running processes.\n'
             msg += 'More info: https://askubuntu.com/q/41629'
             log.warning(msg)
+            return
     except IOError:
         pass
 
@@ -100,10 +167,18 @@ def attach(target, execute = None, exe = None, arch = None):
             execute = fd.read()
             fd.close()
 
+    # enable gdb.attach(p, 'continue')
+    if execute and not execute.endswith('\n'):
+        execute += '\n'
+
     # gdb script to run before `execute`
     pre = ''
     if arch:
-        pre += 'set architechture %s\n' % arch
+        if not misc.which('gdb-multiarch'):
+            log.warn_once('Cross-architecture debugging usually requires gdb-multiarch\n' \
+                '$ apt-get install gdb-multiarch')
+        pre += 'set endian %s\n' % context.endian
+        pre += 'set architecture %s\n' % get_gdb_arch(arch)
 
     # let's see if we can find a pid to attach to
     pid = None
@@ -116,8 +191,27 @@ def attach(target, execute = None, exe = None, arch = None):
         if not pids:
             log.error('no such process: %s' % target)
         pid = pids[0]
-        log.info('attaching you youngest process "%s" (PID = %d)' %
+        log.info('attaching to youngest process "%s" (PID = %d)' %
                  (target, pid))
+    elif isinstance(target, tubes.ssh.ssh_channel):
+        if not target.pid:
+            log.error("PID unknown for channel")
+
+        shell = target.parent
+
+        tmpfile = shell.mktemp()
+        shell.upload_data(execute or '', tmpfile)
+
+        cmd = ['ssh', '-t', '-p', str(shell.port), '-l', shell.user, shell.host]
+        if shell.password:
+            cmd = ['sshpass', '-p', shell.password] + cmd
+        if shell.keyfile:
+            cmd += ['-i', shell.keyfile]
+        cmd += ['gdb %r %s -x "%s" ; rm "%s"' % (target.exe, target.pid, tmpfile, tmpfile)]
+
+        misc.run_in_new_terminal(' '.join(cmd))
+        return
+
     elif isinstance(target, tubes.sock.sock):
         pids = proc.pidof(target)
         if not pids:
@@ -178,7 +272,6 @@ def attach(target, execute = None, exe = None, arch = None):
                 return os.path.join(proc.cwd(spid), exe)
 
         exe = exe or findexe()
-
     else:
         log.error("don't know how to attach to target: %r" % target)
 
@@ -213,6 +306,7 @@ def attach(target, execute = None, exe = None, arch = None):
                                           delete = False)
         tmp.write(execute)
         tmp.close()
+        atexit.register(lambda: os.unlink(tmp.name))
         cmd += ' -x "%s" ; rm "%s"' % (tmp.name, tmp.name)
 
     log.info('running in new terminal: %s' % cmd)
@@ -283,7 +377,7 @@ def find_module_addresses(binary, ssh=None, ulimit=False):
     3. Loading each library into an ELF
     4. Fixing up the base address vs. the ``.text`` segment address
 
-    Args:
+    Arguments:
         binary(str): Path to the binary on the remote server
         ssh(pwnlib.tubes.tube): SSH connection through which to load the libraries.
             If left as ``None``, will use a ``pwnlib.tubes.process.process``.

pwnlib/internal/dochelper.py

@@ -1,13 +1,15 @@
 import sys
 from os.path import basename
 
+from docutils import nodes
+from docutils import statemachine
+from sphinx.util.compat import Directive
+
 try:
     from StringIO import StringIO
 except ImportError:
     from io import StringIO
 
-from sphinx.util.compat import Directive
-from docutils import nodes, statemachine
 
 class ExecDirective(Directive):
     """Execute the specified python code and insert the output into the document"""

pwnlib/libcdb.py

@@ -0,0 +1,115 @@
+"""
+Fetch a LIBC binary based on some heuristics.
+"""
+import codecs
+import json
+import os
+import tempfile
+import urlparse
+
+from .context import context
+from .elf import ELF
+from .log import getLogger
+from .util.fiddling import b64d
+from .util.fiddling import hexdump
+from .util.misc import read
+from .util.misc import write
+from .util.safeeval import const
+from .util.web import wget
+
+log = getLogger(__name__)
+
+cache_dir = os.path.join(tempfile.gettempdir(), 'pwn')
+
+def search_by_build_id(hex_encoded_id):
+    """
+    Given a hex-encoded Build ID, return the path to an ELF with that Build ID
+    only the local system.
+
+    If it can't be found, return None.
+
+    Arguments:
+        hex_encoded_id(str):
+            Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
+
+    Returns:
+        Path to the downloaded library on disk, or ``None``.
+    """
+    cache = cache_dir + '-libc.so.' + hex_encoded_id
+
+    if os.path.exists(cache) and read(cache).startswith('\x7FELF'):
+        log.info_once("Using cached data from %r" % cache)
+        return cache
+
+    log.info("Downloading data from GitHub")
+
+    url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/build_id/"
+    url      = urlparse.urljoin(url_base, hex_encoded_id)
+
+    data   = ""
+    while not data.startswith('\x7fELF'):
+        data = wget(url)
+
+        if not data:
+            return None
+
+        if data.startswith('..'):
+            url = os.path.dirname(url) + '/'
+            url = urlparse.urljoin(url, data)
+
+    write(cache, data)
+    return cache
+
+def get_build_id_offsets():
+    """
+    Returns a list of file offsets where the Build ID should reside within
+    an ELF file of the currentlys-elected architecture.
+    """
+    # Given the corpus of almost all libc to have been released with
+    # RedHat, Fedora, Ubuntu, Debian, etc. over the past several years,
+    # we can say with 99% certainty that the GNU Build ID section will
+    # be at one of the specified addresses.
+    #
+    # The point here is to get an easy win by reading less DWORDs than would
+    # have otherwise been required to walk the section table and the string
+    # stable.
+    #
+    # function check_arch() {
+    # readelf -n $(file -L * | grep -i "$1" | cut -d ':' -f 1) \
+    #       | grep -B3 BUILD_ID \
+    #       | grep offset \
+    #       | sort \
+    #       | uniq -c
+    # }
+
+    return {
+    # $ check_arch 80386
+    #     181 Displaying notes found at file offset 0x00000174 with length 0x00000024:
+        'i386': [0x174],
+    # $ check_arch "ARM, EABI5"
+    #      69 Displaying notes found at file offset 0x00000174 with length 0x00000024:
+        'arm':  [0x174],
+        'thumb':  [0x174],
+    # $ check_arch "ARM aarch64"
+    #       1 Displaying notes found at file offset 0x00000238 with length 0x00000024:
+        'aarch64': [0x238],
+    # $ check_arch "x86-64"
+    #       6 Displaying notes found at file offset 0x00000174 with length 0x00000024:
+    #      82 Displaying notes found at file offset 0x00000270 with length 0x00000024:
+        'amd64': [0x270, 0x174],
+    # $ check_arch "PowerPC or cisco"
+    #      88 Displaying notes found at file offset 0x00000174 with length 0x00000024:
+        'powerpc': [0x174],
+    # $ check_arch "64-bit PowerPC"
+    #      30 Displaying notes found at file offset 0x00000238 with length 0x00000024:
+        'powerpc64': [0x238],
+    # $ check_arch "SPARC32"
+    #      32 Displaying notes found at file offset 0x00000174 with length 0x00000024:
+        'sparc': [0x174],
+    # $ check_arch "SPARC V9"
+    #      33 Displaying notes found at file offset 0x00000270 with length 0x00000024:
+        'sparc64': [0x270]
+    }.get(context.arch, [])
+
+
+__all__ = ['get_build_id_offsets', 'search_by_build_id']