Release 3.13.0

Merge branch 'stable' into beta
Reduce cache filename length on rop.py (fix #1180 ) (#1204 )
2026-11-09 01:59:33 -10:00 · 2019-11-05 18:39:55 +01:00 · 2019-11-05 10:40:48 +01:00 · 2019-11-04 17:11:16 +01:00 · 2019-10-30 15:49:06 +01:00 · 2019-10-30 15:46:22 +01:00
94 changed files with 2335 additions and 697 deletions
@@ -4,6 +4,11 @@ Thanks for contributing to Pwntools!

 When reporting an issue, be sure that you are running the latest released version of pwntools (`pip install --upgrade pwntools`).

-Please verify that your issue occurs on 64-bit Ubuntu 14.04.  You can use the Dockerfile in `extra/docker/Dockerfile` to save time.
+Please verify that your issue occurs on 64-bit Ubuntu 14.04.  You can use the Dockerfile on `docker.io` for quick testing.
+
+```
+$ docker pull pwntools/pwntools:stable
+$ docker run -it pwntools/pwntools:stable
+```

 If possible, provide a proof-of-concept which demonstrates the problem.  Include any binaries or scripts necessary to reproduce the issue, and please include the full debug output via setting the environment variable `PWNLIB_DEBUG=1`.
@@ -1,10 +1,9 @@
-sudo: false
+dist: xenial
 language: python
 addons:
  apt:
    packages:
    - gcc-multilib
-    - gcc-4.6-arm-linux-gnueabihf
    - lib32stdc++6
    - ash
    - bash
@@ -14,14 +13,22 @@ addons:
    - zsh
    - pandoc
    - gdb
+    - binutils
+    - qemu-user-static
+    - binutils-multiarch
+    - binutils-aarch64-linux-gnu
+    - binutils-arm-linux-gnueabihf
+    - binutils-mips-linux-gnu
+    - binutils-powerpc-linux-gnu
 cache:
    - pip
    - directories:
        - usr
-        - /home/travis/virtualenv/python2.7.9/lib/python2.7/site-packages/
-        - /home/travis/virtualenv/python2.7.9/bin/
+        - /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/
+        - /home/travis/virtualenv/python2.7.15/bin/
+sudo: required
 python:
-  - "2.7"
+  - "2.7.15"
 before_install:
  - echo $-
  - source travis/install.sh
@@ -30,6 +37,7 @@ before_install:
 install:
  - pip install --upgrade pip
  - pip install --upgrade appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
+  - python setup.py egg_info      # pypa/pip#6275
  - pip install --upgrade --editable .
  - pip install --upgrade --requirement docs/requirements.txt
 before_script:
@@ -9,9 +9,18 @@ The table below shows which release corresponds to each branch, and what date th

 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [3.9.0](#390)    | `dev`    | Sep 8, 2017 (planned)
-| [3.8.0](#380)    | `beta`   | Jul 28, 2017 (planned)
-| [3.7.0](#370)    | `stable` | Jun 19, 2017
+| [4.1.0](#410)    | `dev`    | Jan 20, 2020 (planned)
+| [4.0.0](#400)    | `beta`   | Dec 20, 2019 (planned)
+| [3.13.0](#3130)  | `stable` | Nov 5, 2019
+| [3.12.0](#3120)  |          | Feb 22, 2018
+| [3.11.0](#3110)  |          | Jan 3, 2018
+| [3.10.0](#3100)  |          | Oct 25, 2017
+| [3.9.2](#392)    |          | Oct 5, 2017
+| [3.9.1](#391)    |          | Sep 28, 2017
+| [3.9.0](#390)    |          | Sep 11, 2017
+| [3.8.0](#380)    |          | Jul 29, 2017
+| [3.7.1](#371)    |          | Jul 14, 2017
+| [3.7.0](#370)    |          | Jun 19, 2017
 | [3.6.1](#361)    |          | May 12, 2017
 | [3.6.0](#360)    |          | May 8, 2017
 | [3.5.1](#351)    |          | Apr 15, 2017
@@ -34,19 +43,153 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015

+## 4.1.0
+
+To be released on Jan 20, 2020.
+
+## 4.0.0
+
+To be released on Dec 20, 2019.
+
+- **Python 3 support! <3**
+- [#1317][1317] Tubes with `context.encoding`
+- [#1216][1216] Improve format string generator
+- [#1285][1285] Add freebsd generic syscall templates
+- [76413f][76413f] Add pwnlib.adb.bootimg for 'ANDROID!' format boot.img images
+- [#1202][1202] Docker: Kill 14 layers in pwntools base images
+- [#1182][1182] shellcraft.dupio() for mips
+
+[1317]: https://github.com/Gallopsled/pwntools/pulls/1317
+[1285]: https://github.com/Gallopsled/pwntools/pulls/1285
+[1216]: https://github.com/Gallopsled/pwntools/pulls/1216
+[1202]: https://github.com/Gallopsled/pwntools/pulls/1202
+[1182]: https://github.com/Gallopsled/pwntools/pulls/1182
+[76413f]: https://github.com/Gallopsled/pwntools/commit/76413f
+
+## 3.13.0
+
+- [#1204][1204] Reduce ROP cache filename length
+- [#1175][1175] Fix nested SSH connectors
+- [#1355][1355] Fix 'break' syscall
+- [#1277][1277] Fix timeout parameter passing in sendlineafter and other similar functions
+- [#1292][1292] Provide correct arch name to gdb for sparc64
+
+[1175]: https://github.com/Gallopsled/pwntools/pulls/1175
+[1204]: https://github.com/Gallopsled/pwntools/pulls/1355
+[1277]: https://github.com/Gallopsled/pwntools/pulls/1277
+[1292]: https://github.com/Gallopsled/pwntools/pulls/1292
+[1355]: https://github.com/Gallopsled/pwntools/pulls/1355
+
+## 3.12.1
+
+- [#1104][1104] Add `DynELF.dump()` for dumping remote ELF files
+- [#1101][1101] Set `context.os` via `context.binary`, useful for Android exploitation
+- [5fdc08][5fdc08] Work around broken `pidof` on Android
+- [63dfed][63dfed] Print warning when Corefile deletion fails instead of throwing an exception
+- [#1094][1094] Make hexdump output alignment more consistent
+- [#1096][1096] `flat()` and `fit()` are now the same function
+
+[1104]: https://github.com/Gallopsled/pwntools/pull/1104
+[1101]: https://github.com/Gallopsled/pwntools/pull/1101
+[1094]: https://github.com/Gallopsled/pwntools/pull/1094
+[1096]: https://github.com/Gallopsled/pwntools/pull/1096
+[5fdc08]: https://github.com/Gallopsled/pwntools/commit/5fdc08
+[63dfed]: https://github.com/Gallopsled/pwntools/commit/63dfed
+
+## 3.12.0
+
+- [#1083][1083] Better error messages for `gdb` when `LD_PRELOAD` is incorrect
+- [#1085][1085] Add support for extracting Android `BOOTLDR!` images
+- [#1075][1075] Add support for detecting GNU Screen for `run_in_new_terminal`
+- [#1074][1074] Add support for running `pwntools-gdb` wrapper script instead of `gdb`
+- [#1068][1068] Work around very old OpenSSL versions which don't have sha256 support *AND* don't exit with an error code when trying to use it
+- [#1067][1067] Add `pwnlib.tubes.server` module, which adds a reusable `server` listener
+- [#1063][1063] Add support for labels in `fit()`, allowing dynamic contents to be injected.  (This feature is really cool, check out the pull request!)
+
+[1083]: https://github.com/Gallopsled/pwntools/pull/1083
+[1085]: https://github.com/Gallopsled/pwntools/pull/1085
+[1075]: https://github.com/Gallopsled/pwntools/pull/1075
+[1074]: https://github.com/Gallopsled/pwntools/pull/1074
+[1068]: https://github.com/Gallopsled/pwntools/pull/1068
+[1067]: https://github.com/Gallopsled/pwntools/pull/1067
+[1063]: https://github.com/Gallopsled/pwntools/pull/1063
+
+## 3.11.0
+
+- [#1044][1044] Enhancements to ROP
+    + Much better support for 64-bit Intel (amd64) ROP
+    + ROP gadget selection is optimized to favor multi-pops instead of multiple single-pop gadgets
+    + Added support for blacklisting byte values in ROP gadget addresses
+- [#1049][1049] Enhancements to `cyclic`
+    + `context` now has two additional attributes, `cyclic_alphabet` and `cyclic_length`, which correspond to the arguments `alphabet` and `n` to `cyclic()` and `cyclic_find()` and related routines.
+    + The motivation for this change is to allow setting the `alphabet` globally, so that any padding / patterns generated internally to pwntools can be controlled.  The specific motivation is blacklisting values in ROP padding.
+- [#1052][1052] Enhancements for detecting `QEMU_LD_PREFIX` used by QEMU user-mode emulation for sysroots
+- [#1035][1035] Minor documentation changes
+- [#1032][1032] Enhancements to `pwn template`
+- [#1031][1031] More accurate `Coredump.fault_addr` on amd64
+- [#1084][1084] Fix broken tests due to `ftp.debian.org` going down
+
+[1044]: https://github.com/Gallopsled/pwntools/pull/1044
+[1049]: https://github.com/Gallopsled/pwntools/pull/1049
+[1052]: https://github.com/Gallopsled/pwntools/pull/1052
+[1035]: https://github.com/Gallopsled/pwntools/pull/1035
+[1032]: https://github.com/Gallopsled/pwntools/pull/1032
+[1031]: https://github.com/Gallopsled/pwntools/pull/1031
+[1084]: https://github.com/Gallopsled/pwntools/pull/1084
+
+## 3.10.0
+
+- [#1007][1007] Add support for setting a `gdbinit` file in the context
+- [#1055][1055] Fixes for `Corefile` stack parsing, speed up `ELF.string()`
+- [#1057][1057] Fix a variable name typo in `DynELF` logging which results in an exception being thrown
+- [#1058][1058] Fix an edge case in `ssh_process.exe`
+
+[1007]: https://github.com/Gallopsled/pwntools/pull/1007
+[1055]: https://github.com/Gallopsled/pwntools/pull/1055
+[1057]: https://github.com/Gallopsled/pwntools/pull/1057
+[1058]: https://github.com/Gallopsled/pwntools/pull/1058
+
+## 3.9.2
+
+- [#1043][1043] Do not attempt to populate the libraries used by statically-linked binaries
+
+[1043]: https://github.com/Gallopsled/pwntools/pull/1043
+
+## 3.9.1
+
+- [#1038][1038] Fix an issue with `process()` where glibc would buffer data internally, causing a hang on `select()`
+- [#1036][1036] Fix Travis CI logging verbosity
+- [#1029][1029] Fix some `unicode` issues when using the `readline` command history in `tube.interactive()`
+
+[1038]: https://github.com/Gallopsled/pwntools/pull/1038
+[1036]: https://github.com/Gallopsled/pwntools/pull/1036
+[1029]: https://github.com/Gallopsled/pwntools/pull/1029
+
 ## 3.9.0

-To be released on Sep 8, 2017.
+- [#1003][1003] Make `concat_all` faster while also simplifying it's logic
+- [#1014][1014] Fix for overwritten env when parsing core file
+- [#1023][1023] Fixes to Travis CI
+
+[1003]: https://github.com/Gallopsled/pwntools/pull/1003
+[1014]: https://github.com/Gallopsled/pwntools/pull/1014
+[1023]: https://github.com/Gallopsled/pwntools/pull/1023

 ## 3.8.0

-To be released on Jul 28, 2017.
-
 - [#981][981] Fixed RELRO detection logic
 - [#986][986] Enhancements to DynELF for controlling usage of LibcDB
+- A few documentation fixes
+- A few fixes for the Docker image

-[981]: https://github.com/Gallopsled/pwntools/issues/981
-[986]: https://github.com/Gallopsled/pwntools/issues/986
+[981]: https://github.com/Gallopsled/pwntools/pull/981
+[986]: https://github.com/Gallopsled/pwntools/pull/986
+
+## 3.7.1
+
+- [#998][998] Fix a bug where integer values could not be set in `.pwn.conf`.
+
+[998]: https://github.com/Gallopsled/pwntools/pull/998

 ## 3.7.0

@@ -54,8 +197,8 @@ To be released on Jul 28, 2017.
 - [#952][952] A few small fixes were made to `pwn template`, and the CRC database was updated.
 - [5c72d62c][5c72d62c] Updated the CRC database

-[933]: https://github.com/Gallopsled/pwntools/issues/933
-[952]: https://github.com/Gallopsled/pwntools/issues/952
+[933]: https://github.com/Gallopsled/pwntools/pull/933
+[952]: https://github.com/Gallopsled/pwntools/pull/952
 [5c72d62c]: https://github.com/Gallopsled/pwntools/commit/5c72d62c

 ## 3.6.1
@@ -2,13 +2,13 @@
 ![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)

 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![PyPI](https://img.shields.io/badge/pypi-v3.7.0-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
+[![PyPI](https://img.shields.io/badge/pypi-v3.13.0-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
 [![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)

-pwntools is a CTF framework and exploit development library. Written in Python, it is designed for rapid prototyping and development, and intended to make exploit writing as simple as possible.
+Pwntools is a CTF framework and exploit development library. Written in Python, it is designed for rapid prototyping and development, and intended to make exploit writing as simple as possible.

 ```python
 from pwn import *
@@ -32,7 +32,7 @@ To get you started, we've provided some example solutions for past CTF challenge

 # Installation

-pwntools is best supported on 64-bit Ubuntu 12.04 and 14.04, but most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  Python 2.7 is required.
+Pwntools is best supported on 64-bit Ubuntu LTE releases (12.04, 14.04, and 16.04).  Most functionality should work on any Posix-like distribution (Debian, Arch, FreeBSD, OSX, etc.).  Python 2.7 is required.

 Most of the functionality of pwntools is self-contained and Python-only.  You should be able to get running quickly with

@@ -1,5 +1,5 @@
 sphinxcontrib-napoleon
-sphinxcontrib-autoprogram==0.1.2
+sphinxcontrib-autoprogram==0.1.4
 isort
 paramiko>=1.15.2
 mako>=1.0.0
@@ -14,3 +14,4 @@ pygments>=2.0
 sphinx_rtd_theme
 doc2dash
 coveralls
+sphinx<1.8.0
@@ -15,7 +15,7 @@
   :members:

 Internal Functions
-=========================================
+-----------------------------------------

 These are only included so that their tests are run.

@@ -24,8 +24,6 @@ sys.path.insert(0, os.path.abspath('../..'))

 import pwnlib

-# -- General configuration -----------------------------------------------------
-
 # If your documentation needs a minimal Sphinx version, state it here.
 #needs_sphinx = '1.0'

@@ -44,6 +42,23 @@ extensions = [
    'sphinxcontrib.napoleon'
 ]

+# Disable "info" logging directly to stdout by Sphinx
+import logging
+
+class SphinxPwnlibFilter(logging.Filter):
+    def filter(self, record):
+        if record.name.startswith('pwn'):
+            return False
+        if record.name.startswith('paramiko'):
+            return False
+        return True
+
+log_filter = SphinxPwnlibFilter()
+
+for i, handler in enumerate(logging.root.handlers):
+    print("Filtering Sphinx handler", handler)
+    handler.addFilter(log_filter)
+
 # Napoleon settings
 napoleon_use_ivar = True
 napoleon_use_rtype = False
@@ -363,27 +378,25 @@ from sphinx.util.inspect import safe_getmembers, safe_getattr
 def dont_skip_any_doctests(app, what, name, obj, skip, options):
    return False

-# Test non-exported members
-class ModuleDocumenter(sphinx.ext.autodoc.ModuleDocumenter):
-    def get_object_members(self, want_all):
-        if want_all:
-            # if not hasattr(self.object, '__all__'):
-            #     for implicit module members, check __module__ to avoid
-            #     documenting imported objects
-                return True, safe_getmembers(self.object)
-            # else:
-            #     memberlist = self.object.__all__
-            #     # Sometimes __all__ is broken...
-            #     if not isinstance(memberlist, (list, tuple)) or not \
-            #        all(isinstance(entry, string_types) for entry in memberlist):
-            #         self.directive.warn(
-            #             '__all__ should be a list of strings, not %r '
-            #             '(in module %s) -- ignoring __all__' %
-            #             (memberlist, self.fullname))
-            #         # fall back to all members
-            #         return True, safe_getmembers(self.object)
-        else:
-            memberlist = self.options.members or []
+def get_object_members_all(self, want_all):
+    if want_all:
+        # if not hasattr(self.object, '__all__'):
+        #     for implicit module members, check __module__ to avoid
+        #     documenting imported objects
+        return True, safe_getmembers(self.object)
+    # else:
+    #     memberlist = self.object.__all__
+    #     # Sometimes __all__ is broken...
+    #     if not isinstance(memberlist, (list, tuple)) or not \
+        #        all(isinstance(entry, string_types) for entry in memberlist):
+    #         self.directive.warn(
+    #             '__all__ should be a list of strings, not %r '
+    #             '(in module %s) -- ignoring __all__' %
+    #             (memberlist, self.fullname))
+    #         # fall back to all members
+    #         return True, safe_getmembers(self.object)
+    else:
+        memberlist = self.options.members or []
        ret = []
        for mname in memberlist:
            try:
@@ -399,4 +412,4 @@ if 'doctest' in sys.argv:
    def setup(app):
        app.connect('autodoc-skip-member', dont_skip_any_doctests)

-    sphinx.ext.autodoc.ModuleDocumenter = ModuleDocumenter
+    sphinx.ext.autodoc.ModuleDocumenter.get_object_members = get_object_members_all
@@ -8,7 +8,7 @@
 :mod:`pwnlib.context` --- Setting runtime variables
 =====================================================

-Many settings in ``pwntools`` are controlled via the global variable :data:`.context, such as the selected target operating system, architecture, and bit-width.
+Many settings in ``pwntools`` are controlled via the global variable :data:`.context`, such as the selected target operating system, architecture, and bit-width.

 In general, exploits will start with something like:

@@ -1,5 +1,6 @@
 .. testsetup:: *

+   from glob import glob
   from pwn import *

 :mod:`pwnlib.elf.corefile` --- Core Files
@@ -13,7 +13,7 @@ Which imports a bazillion things into the global namespace to make your life eas

 This is a quick list of most of the objects and routines imported, in rough order of importance and frequency of use.

- :obj:`.context`
+- :mod:`pwnlib.context`
    - :data:`pwnlib.context.context`
    - Responsible for most of the pwntools convenience settings
    - Set `context.log_level = 'debug'` when troubleshooting your exploit
@@ -57,9 +57,11 @@ Each of the ``pwntools`` modules is documented here.
   flag
   fmtstr
   gdb
+   libcdb
   log
   memleak
   protocols
+   qemu
   replacements
   rop
   rop/*
@@ -27,7 +27,7 @@ This exposes a standard interface to talk to processes, sockets, serial ports,
 and all manner of things, along with some nifty helpers for common tasks.
 For example, remote connections via :mod:`pwnlib.tubes.remote`.

-    >>> conn = remote('ftp.debian.org',21)
+    >>> conn = remote('ftp.ubuntu.com',21)
    >>> conn.recvline() # doctest: +ELLIPSIS
    '220 ...'
    >>> conn.send('USER anonymous\r\n')
@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+   from pwnlib.libcdb import *
+
+:mod:`pwnlib.libcdb` --- Libc Database
+===========================================
+
+.. automodule:: pwnlib.libcdb
+   :members:
@@ -0,0 +1,10 @@
+.. testsetup:: *
+
+   from pwn import *
+
+
+:mod:`pwnlib.qemu` --- QEMU Utilities
+==========================================
+
+.. automodule:: pwnlib.qemu
+   :members:
@@ -1,6 +1,7 @@
 .. testsetup:: *

   from pwn import *
+   from pwnlib.tubes.server import server

 :mod:`pwnlib.tubes.sock` --- Sockets
 ===========================================================
@@ -23,3 +24,9 @@
   .. autoclass:: pwnlib.tubes.listen.listen
      :members:
      :show-inheritance:
+
+.. automodule:: pwnlib.tubes.server
+
+   .. autoclass:: pwnlib.tubes.server.server
+      :members:
+      :show-inheritance:
@@ -1,5 +1,6 @@
 .. testsetup:: *

+   from pwn import *
   from pwnlib.util.cyclic import *


@@ -1,6 +1,7 @@
 .. testsetup:: *

   from pwnlib.util.proc import *
+   from pwnlib.tubes.process import process
   import os, sys


@@ -1,6 +1,6 @@
 subdirs=xenial stable $(wildcard */)
 makefiles=$(wildcard */Makefile)
-tags=stable beta dev
+tags=base stable beta dev

 all: $(makefiles) $(subdirs)

@@ -0,0 +1 @@
+xenial
@@ -1,6 +1,6 @@
 all: Dockerfile

-Dockerfile: ../stable/Dockerfile FORCE
+Dockerfile: ../xenial/Dockerfile FORCE
 	sed "s|ubuntu:xenial|kalilinux/kali-linux-docker|" < "$<" > "$@"

 FORCE:
@@ -1,26 +1,6 @@
-############################################################
-# Dockerfile to build Pwntools container
-# Based on Ubuntu
-############################################################
+FROM pwntools/pwntools:base

-FROM ubuntu:xenial
-MAINTAINER Maintainer Gallopsled et al.
-
-RUN apt-get update
-RUN apt-get install -y build-essential
-RUN apt-get install -y git
-RUN apt-get install -y libssl-dev
-RUN apt-get install -y libffi-dev
-RUN apt-get install -y python2.7
-RUN apt-get install -y python-pip
-RUN apt-get install -y python-dev
-RUN pip install --upgrade pip
-RUN python -m pip install --upgrade pwntools
+USER root
+RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
-
-RUN apt-get install -y sudo
-RUN useradd -m pwntools
-RUN passwd --delete --unlock pwntools
-RUN echo "pwntools ALL=(ALL:ALL) NOPASSWD: ALL" > /etc/sudoers.d/pwntools
 USER pwntools
-WORKDIR /home/pwntools
@@ -1,8 +0,0 @@
-all: Dockerfile
-
-Dockerfile: ../xenial/Dockerfile FORCE
-	cp "$<" "$@"
-
-FORCE:
-
-.PHONY: all
@@ -43,6 +43,7 @@ from pwnlib.tubes.listen import listen
 from pwnlib.tubes.process import process, PTY, PIPE, STDOUT
 from pwnlib.tubes.remote import remote, tcp, udp, connect
 from pwnlib.tubes.serialtube import serialtube
+from pwnlib.tubes.server import server
 from pwnlib.tubes.ssh import ssh
 from pwnlib.tubes.tube import tube
 from pwnlib.ui import *
@@ -124,8 +124,3 @@ linux_aarch64_sigreturn = SigreturnABI('sp', ['x8'], 16, 0)

 windows_i386  = ABI('esp', [], 4, 0)
 windows_amd64 = ABI('rsp', ['rcx','rdx','r8','r9'], 32, 32)
-
-# Fake ABIs used by SROP
-linux_i386_srop = ABI('esp', ['eax'], 4, 0)
-linux_amd64_srop = ABI('rsp', ['rax'], 4, 0)
-linux_arm_srop = ABI('sp', ['r7'], 4, 0)
@@ -328,7 +328,7 @@ class AdbDevice(Device):
            kwargs['port'] = 'emulator'

        for field in fields[2:]:
-            k,v = field.split(':')
+            k,v = field.split(':', 1)
            kwargs[k] = v

        return AdbDevice(serial, type, **kwargs)
@@ -820,6 +820,7 @@ echo $PATH | while read -d: directory; do
    [ -x "$directory/{name}" ] || continue;
    echo -n "$directory/{name}\\x00";
 done
+[ -x "{name}" ] && echo -n "$PWD/{name}\\x00"
 '''.format(name=name)

    which_cmd = which_cmd.strip()
@@ -880,9 +881,16 @@ def logcat(stream=False):
 def pidof(name):
    """Returns a list of PIDs for the named process."""
    with context.quiet:
-        io = process(['pidof', name])
-        data = io.recvall().split()
-    return list(map(int, data))
+        # Older devices have a broken 'pidof', apparently.
+        # Try pgrep first.
+        io = process(['pgrep', name])
+        data = io.recvall()
+
+        if 'not found' in data:
+            io = process(['pidof', name])
+            data = io.recvall()
+
+    return list(map(int, data.split()))

@with_device
 def proc_exe(pid):
@@ -0,0 +1,103 @@
+from __future__ import unicode_literals
+
+import ctypes
+import io
+import os
+import sys
+
+class img_info(ctypes.Structure):
+    _fields_ = [
+        ('name', ctypes.c_char * 64),
+        ('size', ctypes.c_uint32)
+    ]
+
+class bootloader_images_header(ctypes.Structure):
+    _fields_ = [
+        ('magic', ctypes.c_char * 8),
+        ('num_images', ctypes.c_uint32),
+        ('start_offset', ctypes.c_uint32),
+        ('bootldr_size', ctypes.c_uint32),
+    ]
+
+    def __init__(self, *a, **kw):
+        super(bootloader_images_header, self).__init__(*a, **kw)
+        if self.magic != self.MAGIC:
+            raise ValueError("Incorrect magic (%r, expected %r)" % (self.magic, self.MAGIC))
+    MAGIC = 'BOOTLDR!'
+
+class BootloaderImage(object):
+    def __init__(self, data):
+        """Android Bootloader image
+
+        Arguments:
+            data(str): Binary data from the image file.
+        """
+        self.data = data
+        self.header = bootloader_images_header.from_buffer_copy(data)
+
+        imgarray = ctypes.ARRAY(img_info, self.header.num_images)
+        self.img_info = imgarray.from_buffer_copy(data, ctypes.sizeof(self.header))
+
+    def extract(self, index_or_name):
+        """extract(index_or_name) -> bytes
+
+        Extract the contents of an image.
+
+        Arguments:
+            index_or_name(str,int): Either an image index or name.
+
+        Returns:
+            Contents of the image.
+        """
+        if isinstance(index_or_name, int):
+            index = index_or_name
+        else:
+            for i in range(len(self.img_info)):
+                if self.img_info[i].name == index_or_name:
+                    index = i
+                    break
+            else:
+                raise ValueError("Invalid img name: %r" % index_or_name)
+
+        if index >= len(self.img_info):
+            raise ValueError("index out of range (%s, max %s)" % (index, len(self.img_info)))
+
+        offset = self.header.start_offset
+
+        for i in range(index):
+            offset += self.img_info[i].size
+
+        return self.data[offset:offset + self.img_info[index].size]
+
+    def extract_all(self, path):
+        """extract_all(path)
+
+        Extracts all images to the provided path.  The filenames are taken
+        from the image name, with '.img' appended.
+        """
+        if not os.path.isdir(path):
+            raise ValueError("%r does not exist or is not a directory" % path)
+
+        for img in self.img_info:
+            imgpath = os.path.join(path, img.name + '.img')
+            with open(imgpath, 'wb+') as f:
+                data = self.extract(img.name)
+                f.write(data)
+
+    def __str__(self):
+        rv = []
+        rv.append("Bootloader")
+        rv.append("  Magic:  %r" % self.header.magic)
+        rv.append("  Offset: %#x" % self.header.start_offset)
+        rv.append("  Size:   %#x" % self.header.bootldr_size)
+        rv.append("  Images: %s" % self.header.num_images)
+        for img in self.img_info:
+            rv.append("    Name: %s" % img.name)
+            rv.append("    Size: %#x" % img.size)
+            rv.append("    Data: %r..." % self.extract(img.name)[:32])
+        return '\n'.join(rv)
+
+if __name__ == '__main__':
+    # Easy sanity checking
+    b = BootloaderImage(open(sys.argv[1]).read())
+    print(b)
@@ -37,8 +37,18 @@ not specified evaluate to an empty string.
    else:
        io = process('./pwnable')

+Arguments can also be accessed directly with the dot operator, e.g.:
+
+.. code-block:: python
+
+    if args.REMOTE:
+        ...
+
+Any undefined arguments evaluate to an empty string, ``''``.
+
 The full list of supported "magic arguments" and their effects are listed
 below.
+
 """
 from __future__ import absolute_import

@@ -184,7 +194,7 @@ def initialize():
        value = 'True'

        if '=' in arg:
-            arg, value = arg.split('=')
+            arg, value = arg.split('=', 1)

        if arg in hooks:
            sys.argv.remove(orig)
@@ -36,7 +36,7 @@ Disassembly
    To disassemble code, simply invoke :func:`disasm` on the bytes to disassemble.

    >>> disasm('\xb8\x0b\x00\x00\x00')
-    '   0:   b8 0b 00 00 00          mov    eax,0xb'
+    '   0:   b8 0b 00 00 00          mov    eax, 0xb'

 """
 from __future__ import absolute_import
@@ -614,7 +614,7 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
    Runs :func:`cpp` over a given shellcode and then assembles it into bytes.

    To see which architectures or operating systems are supported,
-    look in :mod:`pwnlib.contex`.
+    look in :mod:`pwnlib.context`.

    Assembling shellcode requires that the GNU assembler is installed
    for the target architecture.
@@ -737,15 +737,15 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
    Examples:

        >>> print disasm('b85d000000'.decode('hex'), arch = 'i386')
-           0:   b8 5d 00 00 00          mov    eax,0x5d
+           0:   b8 5d 00 00 00          mov    eax, 0x5d
        >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0)
-           0:   mov    eax,0x5d
+           0:   mov    eax, 0x5d
        >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0, offset = 0)
-        mov    eax,0x5d
+        mov    eax, 0x5d
        >>> print disasm('b817000000'.decode('hex'), arch = 'amd64')
-           0:   b8 17 00 00 00          mov    eax,0x17
+           0:   b8 17 00 00 00          mov    eax, 0x17
        >>> print disasm('48c7c017000000'.decode('hex'), arch = 'amd64')
-           0:   48 c7 c0 17 00 00 00    mov    rax,0x17
+           0:   48 c7 c0 17 00 00 00    mov    rax, 0x17
        >>> print disasm('04001fe552009000'.decode('hex'), arch = 'arm')
           0:   e51f0004        ldr     r0, [pc, #-4]   ; 0x4
           4:   00900052        addseq  r0, r0, r2, asr r0
@@ -816,4 +816,4 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
            line += i
        lines.append(line)

-    return '\n'.join(lines)
+    return re.sub(',([^ ])', r', \1', '\n'.join(lines))
@@ -30,13 +30,25 @@ parser.add_argument(
    help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
 )
 parser.add_argument(
-    '--exec', type=file, dest='executable',
+    '--exec',
+
+    # NOTE: Type cannot be "file" because we may be referring to a remote
+    #       file, or a file on an Android device.
+    type=str,
+
+    dest='executable',
    help='File to debug'
 )
 parser.add_argument(
    '--process', metavar='PROCESS_NAME',
    help='Name of the process to attach to (e.g. "bash")'
 )
+parser.add_argument(
+    '--sysroot', metavar='SYSROOT',
+    type=str,
+    default='',
+    help="GDB sysroot path"
+)

 def main(args):
    gdbscript = ''
@@ -47,23 +59,46 @@ def main(args):
        context.device = adb.wait_for_device()

    if args.executable:
-        context.binary = ELF(args.executable.name)
-        target = context.binary.path
+        if os.path.exists(args.executable):
+            context.binary = ELF(args.executable)
+            target = context.binary.path
+
+        # This path does nothing, but avoids the "print_usage()"
+        # path below.
+        elif context.os == 'android':
+            target = args.executable
    elif args.pid:
        target = int(args.pid)
    elif args.process:
        if context.os == 'android':
-            target = adb.pidof(process)
+            target = adb.pidof(args.process)
        else:
-            target = pidof(process)
+            target = pidof(args.process)
+
+        # pidof() returns a list
+        if not target:
+            log.error("Could not find a PID for %r", args.process)
+
+        target = target[0]
+
+        # pidof will sometimes return all PIDs, including init
+        if target == 1:
+            log.error("Got PID 1 from pidof.  Check the process name, or use --pid 1 to debug init")
    else:
        parser.print_usage()
        return 1

    if args.pid or args.process:
-        gdb.attach(target, gdbscript=gdbscript)
+        pid = gdb.attach(target, gdbscript=gdbscript, sysroot=args.sysroot)
+
+        # Since we spawned the gdbserver process, and process registers an
+        # atexit handler to close itself, gdbserver will be terminated when
+        # we exit.  This will manifest as a "remote connected ended" or
+        # similar error message.  Hold it open for the user.
+        log.info("GDB connection forwarding will terminate when you press enter")
+        pause()
    else:
-        gdb.debug(target, gdbscript=gdbscript).interactive()
+        gdb.debug(target, gdbscript=gdbscript, sysroot=args.sysroot).interactive()

 if __name__ == '__main__':
    pwnlib.commandline.common.main(__file__)
@@ -164,6 +164,12 @@ p.add_argument(
    help='List available shellcodes, optionally provide a filter'
 )

+p.add_argument(
+    '-s', '--shared',
+    action='store_true',
+    help='Generated ELF is a shared library'
+)
+
 def get_template(name):
    func = shellcraft
    for attr in name.split('.'):
@@ -296,11 +302,11 @@ def main(args):


        if not args.avoid:
-            code = read(make_elf_from_assembly(assembly, vma=vma))
+            code = read(make_elf_from_assembly(assembly, vma=vma, shared=args.shared))
        else:
            code = asm(assembly)
            code = encode(code, args.avoid)
-            code = make_elf(code, vma=vma)
+            code = make_elf(code, vma=vma, shared=args.shared)
            # code = read(make_elf(encode(asm(code), args.avoid)))
    else:
        code = encode(asm(assembly), args.avoid)
@@ -19,6 +19,7 @@ parser.add_argument('--port', help='Remote port / SSH port', type=int)
 parser.add_argument('--user', help='SSH Username')
 parser.add_argument('--pass', help='SSH Password', dest='password')
 parser.add_argument('--path', help='Remote path of file on SSH server')
+parser.add_argument('--quiet', help='Less verbose template comments', action='store_true')

 def main(args):
    cache = None
@@ -49,7 +50,8 @@ def main(args):
                             args.port,
                             args.user,
                             args.password,
-                             args.path)
+                             args.path,
+                             args.quiet)

    # Fix Mako formatting bs
    output = re.sub('\n\n\n', '\n\n', output)
@@ -150,7 +150,7 @@ class _Tls_DictStack(threading.local, _DictStack):

 def _validator(validator):
    """
-    Validator that tis tightly coupled to the implementation
+    Validator that is tightly coupled to the implementation
    of the classes here.

    This expects that the object has a ._tls property which
@@ -337,9 +337,12 @@ class ContextType(object):
        'binary': None,
        'bits': 32,
        'buffer_size': 4096,
+        'cyclic_alphabet': string.ascii_lowercase,
+        'cyclic_size': 4,
        'delete_corefiles': False,
        'device': os.getenv('ANDROID_SERIAL', None) or None,
        'endian': 'little',
+        'gdbinit': "",
        'kernel': None,
        'log_level': logging.INFO,
        'log_file': _devnull(),
@@ -774,6 +777,7 @@ class ContextType(object):
        self.arch   = binary.arch
        self.bits   = binary.bits
        self.endian = binary.endian
+        self.os     = binary.os

        return binary

@@ -1231,6 +1235,49 @@ class ContextType(object):
        """
        return bool(v)

+
+    @_validator
+    def gdbinit(self, value):
+        """Path to the gdbinit that is used when running GDB locally.
+
+        This is useful if you want pwntools-launched GDB to include some additional modules,
+        like PEDA but you do not want to have GDB include them by default.
+
+        The setting will only apply when GDB is launched locally since remote hosts may not have
+        the necessary requirements for the gdbinit.
+
+        If set to an empty string, GDB will use the default `~/.gdbinit`.
+
+        Default value is ``""``.
+        """
+        return str(value)
+
+    @_validator
+    def cyclic_alphabet(self, alphabet):
+        """Cyclic alphabet.
+
+        Default value is `string.ascii_lowercase`.
+        """
+
+        # Do not allow multiple occurrences
+        if len(set(alphabet)) != len(alphabet):
+            raise AttributeError("cyclic alphabet cannot contain duplicates")
+
+        return str(alphabet)
+
+    @_validator
+    def cyclic_size(self, size):
+        """Cyclic pattern size.
+
+        Default value is `4`.
+        """
+        size = int(size)
+
+        if size > self.bytes:
+            raise AttributeError("cyclic pattern size cannot be larger than word size")
+
+        return size
+
    #*************************************************************************
    #                               ALIASES
    #*************************************************************************
@@ -1359,9 +1406,14 @@ def update_context_defaults(section):
        if key not in ContextType.defaults:
            log.warn("Unknown configuration option %r in section %r" % (key, 'context'))
            continue
-        if isinstance(ContextType.defaults[key], (str, unicode, tuple)):
-            value = safeeval.expr(value)

-        ContextType.defaults[key] = value
+        default = ContextType.defaults[key]
+
+        if isinstance(default, (str, unicode, tuple, int, long, list, dict)):
+            value = safeeval.expr(value)
+        else:
+            log.warn("Unsupported configuration option %r in section %r" % (key, 'context'))
+
+        ContextType.defaults[key] = type(default)(value)

 register_config('context', update_context_defaults)
@@ -1,3 +1,6 @@
+from __future__ import absolute_import
+from pwnlib.data.elf import relro
+
 import os
 path = os.path.dirname(__file__)

@@ -0,0 +1,5 @@
+import os
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)
@@ -1,13 +1,19 @@
-<%page args="binary, host=None, port=None, user=None, password=None, remote_path=None"/>\
+<%page args="binary, host=None, port=None, user=None, password=None, remote_path=None, quiet=False"/>\
 <%
+import os
+import sys
+
 from pwnlib.context import context as ctx
 from pwnlib.elf.elf import ELF
+from pwnlib.util.sh_string import sh_string
 from elftools.common.exceptions import ELFError

-import os
+argv = list(sys.argv)
+argv[0] = os.path.basename(argv[0])
+
 try:
    if binary:
-        ctx.binary = ELF(binary, checksec=False)
+       ctx.binary = ELF(binary, checksec=False)
 except ELFError:
    pass

@@ -28,9 +34,15 @@ binary_repr = repr(binary)
 %>\
 #!/usr/bin/env python2
 # -*- coding: utf-8 -*-
+%if not quiet:
+# This exploit template was generated via:
+# $ ${' '.join(map(sh_string, argv))}
+%endif
 from pwn import *

+%if not quiet:
 # Set up pwntools for the correct architecture
+%endif
 %if ctx.binary:
 exe = context.binary = ELF(${binary_repr})
 <% binary_repr = 'exe.path' %>
@@ -40,6 +52,7 @@ exe = ${binary_repr}
 <% binary_repr = 'exe' %>
 %endif

+%if not quiet:
 # Many built-in settings can be controlled on the command-line and show up
 # in "args".  For example, to dump all data sent/received, and disable ASLR
 # for all created processes...
@@ -47,6 +60,7 @@ exe = ${binary_repr}
 %if host or port or user:
 # ./exploit.py GDB HOST=example.com PORT=4141
 %endif
+%endif
 %if host:
 host = args.HOST or ${repr(host)}
 %endif
@@ -61,10 +75,60 @@ password = args.PASSWORD or ${repr(password)}
 remote_path = ${repr(remote_path)}
 %endif

+%if ssh:
+# Connect to the remote SSH server
+shell = None
+if not args.LOCAL:
+    shell = ssh(user, host, port, password)
+    shell.set_working_directory(symlink=True)
+%endif
+
+%if host:
+def local(argv=[], *a, **kw):
+    '''Execute the target binary locally'''
+    if args.GDB:
+        return gdb.debug([${binary_repr}] + argv, gdbscript=gdbscript, *a, **kw)
+    else:
+        return process([${binary_repr}] + argv, *a, **kw)
+
+def remote(argv=[], *a, **kw):
+  %if ssh:
+    '''Execute the target binary on the remote host'''
+    if args.GDB:
+        return gdb.debug([remote_path] + argv, gdbscript=gdbscript, ssh=shell, *a, **kw)
+    else:
+        return shell.process([remote_path] + argv, *a, **kw)
+  %else:
+    '''Connect to the process on the remote host'''
+    io = connect(host, port)
+    if args.GDB:
+        gdb.attach(io, gdbscript=gdbscript)
+    return io
+  %endif
+%endif
+
+%if host:
+def start(argv=[], *a, **kw):
+    '''Start the exploit against the target.'''
+    if args.LOCAL:
+        return local(argv, *a, **kw)
+    else:
+        return remote(argv, *a, **kw)
+%else:
+def start(argv=[], *a, **kw):
+    '''Start the exploit against the target.'''
+    if args.GDB:
+        return gdb.debug([${binary_repr}] + argv, gdbscript=gdbscript, *a, **kw)
+    else:
+        return process([${binary_repr}] + argv, *a, **kw)
+%endif
+
 %if exe or remote_path:
+%if not quiet:
 # Specify your GDB script here for debugging
 # GDB will be launched if the exploit is run via e.g.
 # ./exploit.py GDB
+%endif
 gdbscript = '''
 %if ctx.binary:
  %if 'main' in ctx.binary.symbols:
@@ -77,56 +141,27 @@ continue
 '''.format(**locals())
 %endif

-%if ssh:
-# Connect to the remote SSH server
-shell = None
-if not args.LOCAL:
-    shell = ssh(user, host, port, password)
-    shell.set_working_directory(symlink=True)
-%endif
-
-%if host:
-# Execute the target binary locally
-def local(argv=[], *a, **kw):
-    if args.GDB:
-        return gdb.debug([exe.path] + argv, gdbscript=gdbscript, *a, **kw)
-    else:
-        return process([exe.path] + argv, *a, **kw)
-
-%if ssh:
-# Execute the target binary on the remote host
-%else:
-# Connect to the process on the remote host
-%endif
-def remote(argv=[], *a, **kw):
-  %if ssh:
-    if args.GDB:
-        return gdb.debug([remote_path] + argv, gdbscript=gdbscript, ssh=shell, *a, **kw)
-    else:
-        return shell.process([remote_path] + argv, *a, **kw)
-  %else:
-    io = connect(host, port)
-    if args.GDB:
-        gdb.attach(io, gdbscript=gdbscript)
-    return io
-  %endif
-%endif
-
-%if host:
-start = local if args.LOCAL else remote
-%else:
-def start(argv=[], *a, **kw):
-    if args.GDB:
-        return gdb.debug([${binary_repr}] + argv, gdbscript=gdbscript, *a, **kw)
-    else:
-        return process([${binary_repr}] + argv, *a, **kw)
-%endif

+%if not quiet:
 #===========================================================
 #                    EXPLOIT GOES HERE
 #===========================================================
+%else:
+# -- Exploit goes here --
+%endif
+%if ctx.binary and not quiet:
+# ${'%-10s%s-%s-%s' % ('Arch:',
+                       ctx.binary.arch,
+                       ctx.binary.bits,
+                       ctx.binary.endian)}
+%for line in ctx.binary.checksec(color=False).splitlines():
+# ${line}
+%endfor
+%endif
+
 io = start()

+%if not quiet:
 # shellcode = asm(shellcraft.sh())
 # payload = fit({
 #     32: 0xdeadbeef,
@@ -135,5 +170,6 @@ io = start()
 # io.send(payload)
 # flag = io.recv(...)
 # log.success(flag)
+%endif

 io.interactive()
@@ -1,13 +1,12 @@
-This directory contains a script `download-useragents.py' which downloads a list
-of user agent strings from the Internet.
+This directory contains a script `download-useragents.py' which downloads a
+list of user agent strings from the Internet.

 The user agent strings are downloaded from:
- - http://www.useragentstring.com/
- - http://www.user-agents.org/
+ - http://www.user-agents.org
 - http://techpatterns.com/downloads/firefox/useragentswitcher.xml

-The latter list is part of the Firefox `User Agent Switcher' add-on and released
-under BSD 2 Clause.
+The latter list is part of the Firefox `User Agent Switcher' add-on and
+released under BSD 2 Clause.


 As mentioned elsewhere we are not lawyers and do not know the legal status of
@@ -1,13 +1,9 @@
 #!/usr/bin/env python2
-"""
-Script for downloading lists of user agent strings
-"""
+"""Script for downloading lists of user agent strings."""
 import os
-import re
 import urllib

 from bs4 import BeautifulSoup
-
 from pwn import *

 uas = set()
@@ -23,39 +19,9 @@ def getxml(url):
    f.close()
    return xml

-with log.waitfor('Fetching from from http://www.useragentstring.com') as l:
-    html = getxml('http://www.useragentstring.com/pages/All/')
-    soup = BeautifulSoup(html)
-    l.success()
-
-with log.waitfor('Parsing list') as l:
-    liste = soup.select('#liste a')
-    for a in liste:
-        href = a.get('href')
-        if not href or href[0] != '/':
-            continue
-        m = re.match('More (.+) user agents strings -->>', a.string)
-        if m:
-            target = m.group(1)
-            with log.waitfor('Fetching additional user agents for %s' % target) as l:
-                html = getxml('http://www.useragentstring.com' + href)
-                soup = BeautifulSoup(html)
-                subliste = soup.select('#liste a')
-                for a in subliste:
-                    ua = a.string.strip()
-                    try:
-                        ua = ua.decode('utf-8')
-                    except UnicodeEncodeError:
-                        continue
-                    uas.add(ua)
-                l.success()
-        else:
-            uas.add(a.string.strip())
-    l.success()
-
 with log.waitfor('Fetching from from http://techpatterns.com') as l:
    xml = getxml('http://techpatterns.com/downloads/firefox/useragentswitcher.xml')
-    soup = BeautifulSoup(xml)
+    soup = BeautifulSoup(xml, 'html.parser')
    l.success()

 def loop(xml):
@@ -67,16 +33,16 @@ def loop(xml):
            uas.add(item['useragent'].strip())

 with log.waitfor('Parsing list') as l:
-    loop(soup.body.useragentswitcher)
+    loop(soup.useragentswitcher)
    l.success()

 with log.waitfor('Fetching from from http://www.user-agents.org') as l:
    xml = getxml('http://www.user-agents.org/allagents.xml')
-    soup = BeautifulSoup(xml)
+    soup = BeautifulSoup(xml, 'html.parser')
    l.success()

 with log.waitfor('Parsing list') as l:
-    for item in soup.body.__getattr__('user-agents'):
+    for item in soup.__getattr__('user-agents'):
        if item.name == 'user-agent':
            ua = item.select('string')[0].string.strip()
            try:
@@ -8211,6 +8211,7 @@ Mozilla/5.0 (BlackBerry; U; BlackBerry 9850; en-US) AppleWebKit/534.11+ (KHTML,
 Mozilla/5.0 (BlackBerry; U; BlackBerry 9850; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0.254 Mobile Safari/534.11+
 Mozilla/5.0 (BlackBerry; U; BlackBerry 9860; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0.254 Mobile Safari/534.11+
 Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+
+Mozilla/5.0 (BlackBerry; U; BlackBerry 9930; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.267 Mobile Safari/534.11+
 Mozilla/5.0 (Clustered-Search-Bot/1.0; [email protected]; http://www.clush.com/)
 Mozilla/5.0 (Darwin; FreeBSD 5.6; en-GB; rv:1.8.1.17pre) Gecko/20080716 K-Meleon/1.5.0
 Mozilla/5.0 (Darwin; FreeBSD 5.6; en-GB; rv:1.9.1b3pre)Gecko/20081211 K-Meleon/1.5.2
@@ -8244,13 +8245,26 @@ Mozilla/5.0 (Linux; Android 4.4.4; XT1032 Build/KXB21.14-L1.61) AppleWebKit/537.
 Mozilla/5.0 (Linux; Android 4.4; Nexus 5 Build/BuildID) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/30.0.0.0 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 5.0.1; SCH-R970 Build/LRX22C) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 5.0.2; SAMSUNG SM-T530NU Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.2 Chrome/38.0.2125.102 Safari/537.36
+Mozilla/5.0 (Linux; Android 5.1.1; Coolpad 3622A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.78 Safari/537.36 OPR/30.0.1856.93524
 Mozilla/5.0 (Linux; Android 5.1; C6740N Build/LMY47O) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 6.0.1; SM-G900H Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 6.0; ALE-L21 Build/HuaweiALE-L21) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.89 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 6.0; HTC One M9 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 6.0; LG-D850 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.97 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 6.0; Nexus 5X Build/MDB08L) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.0; LG-H918 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.0; LGL84VL Build/NRD90U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.0; LGUS997 Build/NRD90U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.0; Moto G (5) Plus Build/NPNS25.137-35-5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 7.0; Nexus 9 Build/NRD90R) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Safari/537.36
+Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-G925R6 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.1.1; BBB100-1 Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.1.1; Coolpad 3632A Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.1.1; XT1710-02 Build/NDS26.74-36) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.1.2; Nexus 6P Build/N2G48C) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.1.2; Pixel Build/NHG47N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 8.0.0; Pixel XL Build/OPR6.170623.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36
 Mozilla/5.0 (Linux; U) Opera 6.02  [en]
 Mozilla/5.0 (Linux; U; Android 0.5; en-us) AppleWebKit/522  (KHTML, like Gecko) Safari/419.3
 Mozilla/5.0 (Linux; U; Android 1.0; en-us; dream) AppleWebKit/525.10  (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2
@@ -8369,6 +8383,7 @@ Mozilla/5.0 (Macintosh; I; Intel Mac OS X 11_7_9; de-LI; rv:1.9b4) Gecko/2012010
 Mozilla/5.0 (Macintosh; I; PPC Mac OS X Mach-O; en-US; rv:1.9a1) Gecko/20061204 Firefox/3.0a1
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:40.0) Gecko/20100101 Firefox/40.0
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:55.0) Gecko/20100101 Firefox/55.0
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:5.0) Gecko/20100101 Firefox/5.0
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:15.0) Gecko/20120909 Firefox/15.0.1 SeaMonkey/2.12.1
@@ -8405,17 +8420,20 @@ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.1.25 (KHTML, like Gecko)
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 OPR/28.0.1750.51
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.82 Safari/537.36 OPR/29.0.1795.41
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.105 Safari/537.36 Vivaldi/1.0.162.9
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2859.0 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.49 Safari/537.36
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Maxthon/4.5.2
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 Safari/600.8.9
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11) AppleWebKit/601.1.56 (KHTML, like Gecko) Version/9.0 Safari/601.1.56
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.8 (KHTML, like Gecko) Version/10.1 Safari/603.1.30
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.68 Safari/534.24
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/534.31 (KHTML, like Gecko) Chrome/13.0.748.0 Safari/534.31
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.801.0 Safari/535.1
@@ -10005,6 +10023,8 @@ Mozilla/5.0 (OS/2; Warp 4.5; rv:31.0) Gecko/20100101 Firefox/31.0
 Mozilla/5.0 (OS/2; Warp 4.5; rv:31.0) Gecko/20100101 Firefox/31.0 SeaMonkey/2.28
 Mozilla/5.0 (OS/2; Warp 4.5; rv:38.0) Gecko/20100101 Firefox/38.0
 Mozilla/5.0 (OS/2; Warp 4.5; rv:38.0) Gecko/20100101 Firefox/38.0 SeaMonkey/2.35
+Mozilla/5.0 (OS/2; Warp 4.5; rv:45.0) Gecko/20100101 Firefox/45.0
+Mozilla/5.0 (OS/2; Warp 4.5; rv:45.0) Gecko/20100101 Firefox/45.0 SeaMonkey/2.42.9esr
 Mozilla/5.0 (PLAYSTATION 3; 1.0)
 Mozilla/5.0 (PLAYSTATION 3; 1.00)
 Mozilla/5.0 (PLAYSTATION 3; 1.10)
@@ -10018,6 +10038,7 @@ Mozilla/5.0 (PlayBook; U; RIM Tablet OS 2.1.0; en-US) AppleWebKit/536.2+ (KHTML
 Mozilla/5.0 (S60; SymbOS; Opera Mobi/1181; U; en-GB; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.1
 Mozilla/5.0 (S60; SymbOS; Opera Mobi/1209; U; it; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.1
 Mozilla/5.0 (S60; SymbOS; Opera Mobi/SYB-1103211396; U; es-LA; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00
+Mozilla/5.0 (SMART-TV; X11; Linux armv7l) AppleWebkit/537.42 (KHTML, like Gecko) Chromium/25.0.1349.2 Chrome/25.0.1349.2 Safari/537.42
 Mozilla/5.0 (Sage)
 Mozilla/5.0 (Slurp/cat; [email protected]; http://www.inktomi.com/slurp.html)
 Mozilla/5.0 (Slurp/si; [email protected]; http://www.inktomi.com/slurp.html)
@@ -10072,18 +10093,27 @@ Mozilla/5.0 (Windows ME; U) Opera 6.05  [de]
 Mozilla/5.0 (Windows ME; U; en) Opera 8.51
 Mozilla/5.0 (Windows ME; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.63
 Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240
+Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063
+Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.91 Safari/537.36 Vivaldi/1.92.917.39
 Mozilla/5.0 (Windows NT 10.0; ARM; Lumia 950 Dual SIM) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393
 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36
 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 UBrowser/5.6.13705.206 Safari/537.36
 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36 OPR/36.0.2130.46
 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36
 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.90 Safari/537.36 Vivaldi/1.4.589.11
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 AOL/11.0 AOLBUILD/11.0.1305 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36 OPR/47.0.2631.55
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/5.0.4.3000 Chrome/47.0.2526.73 Safari/537.36
 Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; MATBJS; rv:11.0) like Gecko
+Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; MALNJS; rv:11.0) like Gecko
 Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0
 Mozilla/5.0 (Windows NT 10.0; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0
 Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36 Edge/12.0
 Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2869.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3191.0 Safari/537.36
 Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:52.0) Gecko/20100101 Firefox/52.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0
+Mozilla/5.0 (Windows NT 10.0; rv:45.9) Gecko/20100101 Goanna/3.2 Firefox/45.9 PaleMoon/27.4.0
 Mozilla/5.0 (Windows NT 4.0; U) Opera 6.05  [en]
 Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36
 Mozilla/5.0 (Windows NT 5.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1150.1  Iron/20.0.1150.1 Safari/536.11
@@ -10145,6 +10175,8 @@ Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.
 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36
 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36
 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36
+Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 YaBrowser/17.3.0.1785 Yowser/2.5 Safari/537.36
+Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) MxBrowser/4.5.10.7000 Chrome/30.0.1551.0 Safari/537.36
 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1250.0 Iron/22.0.2150.0 Safari/537.4
 Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0
 Mozilla/5.0 (Windows NT 5.1; U) Opera 7.03  [de]
@@ -15192,6 +15224,7 @@ Mozilla/5.0 (X11; ; Linux i686; en-US; rv:1.8.1.3) Gecko
 Mozilla/5.0 (X11; ; Linux i686; rv:1.9.2.20) Gecko/20110805
 Mozilla/5.0 (X11; ; Linux x86_64; rv:1.8.1.6) Gecko/20070802 Firefox
 Mozilla/5.0 (X11; Arch Linux i686; rv:2.0) Gecko/20110321 Firefox/4.0
+Mozilla/5.0 (X11; CentOS; Linux x86_64; rv:36.0) Gecko/20100101 Firefox/36.0
 Mozilla/5.0 (X11; CrOS i686 0.13.507) AppleWebKit/534.35 (KHTML, like Gecko) Chrome/13.0.763.0 Safari/534.35
 Mozilla/5.0 (X11; CrOS i686 0.13.587) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.14 Safari/535.1
 Mozilla/5.0 (X11; CrOS i686 1193.158.0) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7
@@ -15205,6 +15238,7 @@ Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) C
 Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36
 Mozilla/5.0 (X11; CrOS x86_64 5841.83.0) AppleWebKit/537.36 (KHTML like Gecko) Chrome/36.0.1985.138 Safari/537.36
 Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36
+Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3178.0 Safari/537.36
 Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0
 Mozilla/5.0 (X11; FreeBSD 7.1-RELEASE i386; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.63
 Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.65 Safari/535.11
@@ -15215,6 +15249,7 @@ Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/3
 Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.79 Safari/537.4
 Mozilla/5.0 (X11; FreeBSD amd64; rv:27.0) Gecko/20100101 Firefox/27.0 SeaMonkey/2.24
 Mozilla/5.0 (X11; FreeBSD amd64; rv:5.0) Gecko/20100101 Firefox/5.0
+Mozilla/5.0 (X11; FreeBSD amd64; rv:54.0) Gecko/20100101 Firefox/54.0
 Mozilla/5.0 (X11; FreeBSD amd64; rv:6.0) Gecko/20110818 Firefox/6.0 SeaMonkey/2.3
 Mozilla/5.0 (X11; FreeBSD i386) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2
 Mozilla/5.0 (X11; FreeBSD i386; rv:28.0) Gecko/20100101 Firefox/28.0 SeaMonkey/2.25
@@ -15239,6 +15274,7 @@ Mozilla/5.0 (X11; Linux armv7l; rv:2.0b4pre) Gecko/20100818 Firefox/4.0b4pre Fen
 Mozilla/5.0 (X11; Linux i386; U) Opera 7.60  [en-GB]
 Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0
 Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.130 Safari/537.36
+Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3187.0 Safari/537.366
 Mozilla/5.0 (X11; Linux i686 on x86_64; rv:10.0.6) Gecko/20100101 Firefox/10.0.6 Iceweasel/10.0.6
 Mozilla/5.0 (X11; Linux i686 on x86_64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1
 Mozilla/5.0 (X11; Linux i686 on x86_64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 Fennec/2.0.1
@@ -15284,6 +15320,7 @@ Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0
 Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2166.2 Safari/537.36
 Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36 OPR/40.0.2308.62
 Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/51.0.2704.79 Chrome/51.0.2704.79 Safari/537.36
+Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/60.0.3112.78 Chrome/60.0.3112.78 Safari/537.36
 Mozilla/5.0 (X11; Linux i686) AppleWebKit/538.1 (KHTML, like Gecko) QupZilla/1.8.6 Safari/538.1
 Mozilla/5.0 (X11; Linux i686; Debian Testing; rv:7.0.1) Gecko/20100101 Firefox/7.0.1 Iceweasel/7.0.1
 Mozilla/5.0 (X11; Linux i686; U) Opera 7.54  [en]
@@ -15387,6 +15424,7 @@ Mozilla/5.0 (X11; Linux i686; rv:40.0) Gecko/20100101 Firefox/40.0
 Mozilla/5.0 (X11; Linux i686; rv:43.0) Gecko/20100101 Firefox/43.0
 Mozilla/5.0 (X11; Linux i686; rv:46.0) Gecko/20100101 Firefox/46.0
 Mozilla/5.0 (X11; Linux i686; rv:49.0) Gecko/20100101 Firefox/49.0
+Mozilla/5.0 (X11; Linux i686; rv:49.0) Gecko/20100101 Firefox/49.0 SeaMonkey/2.46
 Mozilla/5.0 (X11; Linux i686; rv:5.0) Gecko/20100101 Firefox/5.0
 Mozilla/5.0 (X11; Linux i686; rv:5.0) Gecko/20100101 Firefox/5.0 Iceweasel/5.0
 Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0
@@ -15453,6 +15491,7 @@ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/33.0.1750.152 Chrome/33.0.1750.152 Safari/537.36
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.56 Safari/537.4
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) QupZilla/1.9.0 Safari/538.1
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/604.1 (KHTML, like Gecko) Version/11.0 Safari/604.1 Ubuntu/17.04 (3.24.1-0ubuntu1) Epiphany/3.24.1
 Mozilla/5.0 (X11; Linux x86_64) Gecko Firefox/5.0
 Mozilla/5.0 (X11; Linux x86_64; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.62
 Mozilla/5.0 (X11; Linux x86_64; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0
@@ -15516,6 +15555,7 @@ Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0 Iceweasel/3
 Mozilla/5.0 (X11; Linux x86_64; rv:5.0) Gecko/20100101 Firefox/5.0 FirePHP/0.5
 Mozilla/5.0 (X11; Linux x86_64; rv:5.0) Gecko/20100101 Firefox/5.0 Firefox/5.0
 Mozilla/5.0 (X11; Linux x86_64; rv:5.0) Gecko/20100101 Firefox/5.0 Iceweasel/5.0
+Mozilla/5.0 (X11; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0
 Mozilla/5.0 (X11; Linux x86_64; rv:6.0.1) Gecko/20110831 conkeror/0.9.3
 Mozilla/5.0 (X11; Linux x86_64; rv:7.0.1) Gecko/20100101 Firefox/7.0.1 Iceweasel/7.0.1 Debian
 Mozilla/5.0 (X11; Linux x86_64; rv:7.0a1) Gecko/20110602 Firefox/7.0a1 SeaMonkey/2.2a1pre Lightning/1.1a1pre
@@ -18563,6 +18603,7 @@ Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0
 Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0
 Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:35.0) Gecko/20100101 Firefox/35.0
 Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0
 Mozilla/5.0 (X11; debian; Linux x86_64; rv:15.0) Gecko/20100101 Iceweasel/15.0
 Mozilla/5.0 (X11;U; Linux i686; en-GB; rv:1.9.1) Gecko/20090624 Ubuntu/9.04 (jaunty) Firefox/3.5
 Mozilla/5.0 (X11;U;Linux i686;en-US;rv:1.8.1) Gecko/2006101022 Firefox/2.0
@@ -19300,15 +19341,19 @@ Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)
 Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
 Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
 Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexNews/4.0; +http://yandex.com/bots)
 Mozilla/5.0 (compatible; YesupBot/1.0; +http://www.yesup.net/bot.html)
 Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; )
 Mozilla/5.0 (compatible; Yoono; http://www.yoono.com/)
 Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )
 Mozilla/5.0 (compatible; Zenbot/1.3; +http://zen.co.za/webmasters/)
 Mozilla/5.0 (compatible; aiHitBot/1.0; +http://www.aihit.com/)
+Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )
+Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)
 Mozilla/5.0 (compatible; archive.org_bot/1.10.0 +http://www.loc.gov/minerva/crawl.html)
 Mozilla/5.0 (compatible; archive.org_bot/1.13.1x http://crawler.archive.org)
 Mozilla/5.0 (compatible; archive.org_bot/1.5.0-200506132127 http://crawler.archive.org) Hurricane Katrina
+Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)
 Mozilla/5.0 (compatible; bingbot/2.0  http://www.bing.com/bingbot.htm)
 Mozilla/5.0 (compatible; bingbot/2.0 +http://www.bing.com/bingbot.htm)
 Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
@@ -19352,6 +19397,7 @@ Mozilla/5.0 (en-US;rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2
 Mozilla/5.0 (en-us) AppleWebKit/525.13 (KHTML, like Gecko; Google Web Preview) Version/3.1 Safari/525.13
 Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.2; U; de-DE) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.40.1 Safari/534.6 TouchPad/1.0
 Mozilla/5.0 (iPad; CPU OS 10_0 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/49.0.2623.109 Mobile/14A5335b Safari/601.1.46
+Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A5362a Safari/604.1
 Mozilla/5.0 (iPad; CPU OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Mobile/9B176
 Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3
 Mozilla/5.0 (iPad; CPU OS 5_1_1 like Mac OS X; ru-ru) AppleWebKit/534.46.0 (KHTML, like Gecko) CriOS/21.0.1180.80 Mobile/9B206 Safari/7534.48.3
@@ -19390,6 +19436,8 @@ Mozilla/5.0 (iPhone Simulator; U; CPU iPhone OS 3_2 like Mac OS X; en-us) AppleW
 Mozilla/5.0 (iPhone; CPU  iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16
 Mozilla/5.0 (iPhone; CPU iPhone OS 10_0 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) GSA/18.0.130791545 Mobile/14A5345a Safari/600.1.4
 Mozilla/5.0 (iPhone; CPU iPhone OS 10_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A346 Safari/602.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A5362a Safari/604.1
 Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A403 Safari/8536.25
 Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53
 Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4
@@ -20490,6 +20538,7 @@ Opera/9.80 (Android; Opera Mini/7.29530/27.1407; U; en) Presto/2.8.119 Version/1
 Opera/9.80 (Android; Opera Mini/7.5.33361/31.1350; U; en) Presto/2.8.119 Version/11.10
 Opera/9.80 (Android; Opera Mini/7.5.33361/31.1543; U; en) Presto/2.8.119 Version/11.1010
 Opera/9.80 (Android; Opera Mini/7.6.35766/35.5706; U; en) Presto/2.8.119 Version/11.10
+Opera/9.80 (Android; Opera Mini/9.0.1829/66.318; U; en) Presto/2.12.423 Version/12.16
 Opera/9.80 (Android;Opera Mini/6.0.24212/24.746 U;en) Presto/2.5.25 Version/10.5454
 Opera/9.80 (BlackBerry; Opera Mini/5.1.22303/22.387; U; en) Presto/2.5.25 Version/10.54
 Opera/9.80 (BlackBerry; Opera Mini/6.24209/27.1366; U; en) Presto/2.8.119 Version/11.10
@@ -21055,6 +21104,7 @@ Scooter_trk3-3.0.3
 Scope (Mars+)
 ScoutAbout
 ScoutAnt/0.1; +http://www.ant.com/what_is_ant.com/
+Screaming Frog SEO Spider/8.1
 Scrubby/2.1 (http://www.scrubtheweb.com/)
 Scrubby/2.2 (http://www.scrubtheweb.com/)
 Scrubby/2.x (http://www.scrubtheweb.com/)
@@ -21311,6 +21361,7 @@ Trampelpfad-Spider-v0.1
 TulipChain/5.x (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://java.sun.com/) Linux/2.4.17
 TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8
 Tumblr/1.0 RSS syndication (+http://www.tumblr.com/) ([email protected])
+TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)
 TurnitinBot/1.5 (http://www.turnitin.com/robot/crawlerinfo.html)
 TurnitinBot/1.5 http://www.turnitin.com/robot/crawlerinfo.html
 TurnitinBot/2.0 http://www.turnitin.com/robot/crawlerinfo.html
@@ -21323,6 +21374,7 @@ TwengaBot
 Twiceler www.cuill.com/robots.html
 Twiceler-0.9 http://www.cuill.com/twiceler/robot.html
 Twisted PageGetter
+Twitterbot/1.0
 Twitturly / v0.x
 Twotrees Reactive Filter V2.0
 Tycoon Agent/Nutch-1.0-dev
@@ -21584,6 +21636,7 @@ Xenu Link Sleuth 1.2h
 Xenu Link Sleuth 1.2i
 Xenu Link Sleuth 1.xx
 Xenu Link Sleuth/1.3.7
+Xenu Link Sleuth/1.3.8
 Xenu's Link Sleuth 1.x[a-z]
 Xerka WebBot v1.0.0 [UPVOpenDir]
 Xylix
@@ -145,7 +145,7 @@ class DynELF(object):
    .. _DT_PLTGOT: http://refspecs.linuxfoundation.org/ELF/zSeries/lzsabi0_zSeries/x2251.html
    '''

-    def __init__(self, leak, pointer=None, elf=None):
+    def __init__(self, leak, pointer=None, elf=None, libcdb=True):
        '''
        Instantiates an object which can resolve symbols in a running binary
        given a :class:`pwnlib.memleak.MemLeak` leaker and a pointer inside
@@ -155,7 +155,9 @@ class DynELF(object):
            leak(MemLeak): Instance of pwnlib.memleak.MemLeak for leaking memory
            pointer(int):  A pointer into a loaded ELF file
            elf(str,ELF):  Path to the ELF file on disk, or a loaded :class:`pwnlib.elf.ELF`.
+            libcdb(bool):  Attempt to use libcdb to speed up libc lookups
        '''
+        self.libcdb    = libcdb
        self._elfclass = None
        self._elftype  = None
        self._link_map = None
@@ -415,7 +417,7 @@ class DynELF(object):
    def _find_linkmap(self, pltgot=None, debug=None):
        """
        The linkmap is a chained structure created by the loader at runtime
-        which contains information on the names and load addresses osf all
+        which contains information on the names and load addresses of all
        libraries.

        For non-RELRO binaries, a pointer to this is stored in the .got.plt
@@ -429,7 +431,7 @@ class DynELF(object):
        Got     = {32: elf.Elf_i386_GOT, 64: elf.Elf_x86_64_GOT}[self.elfclass]
        r_debug = {32: elf.Elf32_r_debug, 64: elf.Elf64_r_debug}[self.elfclass]

-        result = None
+        linkmap = None

        if not pltgot:
            w.status("Finding linkmap: DT_PLTGOT")
@@ -437,24 +439,24 @@ class DynELF(object):

        if pltgot:
            w.status("GOT.linkmap")
-            result = self.leak.field(pltgot, Got.linkmap)
-            w.status("GOT.linkmap %#x" % result)
+            linkmap = self.leak.field(pltgot, Got.linkmap)
+            w.status("GOT.linkmap %#x" % linkmap)

-        if not result:
+        if not linkmap:
            debug = debug or self._find_dt(constants.DT_DEBUG)
            if debug:
                w.status("r_debug.linkmap")
-                result = self.leak.field(debug, r_debug.r_map)
-                w.status("r_debug.linkmap %#x" % result)
+                linkmap = self.leak.field(debug, r_debug.r_map)
+                w.status("r_debug.linkmap %#x" % linkmap)

-        if not (pltgot or debug):
+        if not linkmap:
            w.failure("Could not find DT_PLTGOT or DT_DEBUG")
            return None

-        result = self._make_absolute_ptr(result)
+        linkmap = self._make_absolute_ptr(linkmap)

-        w.success('%#x' % result)
-        return result
+        w.success('%#x' % linkmap)
+        return linkmap

    def waitfor(self, msg):
        if not self._waitfor:
@@ -521,6 +523,7 @@ class DynELF(object):

        Arguments:
            symb(str): Named routine to look up
+              If omitted, the base address of the library will be returned.
            lib(str): Substring to match for the library name.
              If omitted, the current library is searched.
              If set to ``'libc'``, ``'libc.so'`` is assumed.
@@ -560,11 +563,10 @@ class DynELF(object):
        #
        # If we are resolving a symbol in the library, find it.
        #
-        if symb:
+        if symb and self.libcdb:
            # Try a quick lookup by build ID
            self.status("Trying lookup based on Build ID")
            build_id = dynlib._lookup_build_id(lib=lib)
-            result   = None
            if build_id:
                log.info("Trying lookup based on Build ID: %s" % build_id)
                path = libcdb.search_by_build_id(build_id)
@@ -573,11 +575,10 @@ class DynELF(object):
                        e = ELF(path)
                        e.address = dynlib.libbase
                        result = e.symbols[symb]
-
-            if not result:
-                self.status("Trying remote lookup")
-                result = dynlib._lookup(symb)
-        else:
+        if symb and not result:
+            self.status("Trying remote lookup")
+            result = dynlib._lookup(symb)
+        if not symb:
            result = dynlib.libbase

        #
@@ -837,6 +838,9 @@ class DynELF(object):
    def _lookup_build_id(self, lib = None):

        libbase = self.libbase
+        if not self.link_map:
+            self.status("No linkmap found")
+            return None

        if lib is not None:
            libbase = self.lookup(symb = None, lib = lib)
@@ -906,3 +910,65 @@ class DynELF(object):
        self.success('*curbrk: %#x' % brk)

        return brk
+
+    def _find_mapped_pages(self, readonly = False, page_size = 0x1000):
+        """
+        A generator of all mapped pages, as found using the Program Headers.
+
+        Yields tuples of the form: (virtual address, memory size)
+        """
+        leak  = self.leak
+        base  = self.libbase
+
+        Ehdr  = {32: elf.Elf32_Ehdr, 64: elf.Elf64_Ehdr}[self.elfclass]
+        Phdr  = {32: elf.Elf32_Phdr, 64: elf.Elf64_Phdr}[self.elfclass]
+
+        phead = base + leak.field(base, Ehdr.e_phoff)
+        phnum = leak.field(base, Ehdr.e_phnum)
+
+        for i in range(phnum):
+            if leak.field_compare(phead, Phdr.p_type, constants.PT_LOAD) :
+                # the interesting pages are those that are aligned to PAGE_SIZE
+                if leak.field_compare(phead, Phdr.p_align, page_size) and \
+                    (readonly or leak.field(phead, Phdr.p_flags) & 0x02 != 0):
+                    vaddr = leak.field(phead, Phdr.p_vaddr)
+                    memsz = leak.field(phead, Phdr.p_memsz)
+                    # fix relative offsets
+                    if vaddr < base :
+                        vaddr += base
+                    yield vaddr, memsz
+            phead += sizeof(Phdr)
+
+    def dump(self, libs = False, readonly = False):
+        """dump(libs = False, readonly = False)
+
+        Dumps the ELF's memory pages to allow further analysis.
+
+        Arguments:
+            libs(bool, optional): True if should dump the libraries too (False by default)
+            readonly(bool, optional): True if should dump read-only pages (False by default)
+
+        Returns:
+            a dictionary of the form: { address : bytes }
+        """
+        leak      = self.leak
+        page_size = 0x1000
+        pages     = {}
+
+        for vaddr, memsz in self._find_mapped_pages(readonly, page_size) :
+            offset    = vaddr % page_size
+            if offset != 0 :
+                memsz += offset
+                vaddr -= offset
+            memsz += (page_size - (memsz % page_size)) % page_size
+            pages[vaddr] = leak.n(vaddr, memsz)
+
+        if libs :
+            for lib_name in self.bases() :
+                if len(lib_name) == 0 :
+                    continue
+                dyn_lib = self._dynamic_load_dynelf(lib_name)
+                if dyn_lib is not None :
+                    pages.update(dyn_lib.dump(readonly = readonly))
+
+        return pages
@@ -87,6 +87,7 @@ from pwnlib.tubes.process import process
 from pwnlib.tubes.ssh import ssh_channel
 from pwnlib.tubes.tube import tube
 from pwnlib.util.fiddling import b64d
+from pwnlib.util.fiddling import enhex
 from pwnlib.util.fiddling import unhex
 from pwnlib.util.misc import read
 from pwnlib.util.misc import write
@@ -223,6 +224,8 @@ class Mapping(object):
        return self._core.read(item, 1)

    def __contains__(self, item):
+        if isinstance(item, Mapping):
+            return (self.start <= item.start) and (item.stop <= self.stop)
        return self.start <= item < self.stop

    def find(self, sub, start=None, end=None):
@@ -389,7 +392,7 @@ class Corefile(ELF):

        >>> core.vdso.data[:4]
        '\x7fELF'
-        >>> core.libc # docteset: +ELLIPSIS
+        >>> core.libc # doctest: +ELLIPSIS
        Mapping('/lib/x86_64-linux-gnu/libc-...', ...)

        The corefile also contains a :attr:`.Corefile.stack` property, which gives
@@ -421,7 +424,7 @@ class Corefile(ELF):

        >>> s = ssh('travis', 'example.pwnme')
        >>> _ = s.set_working_directory()
-        >>> elf = ELF.from_assembly('int3')
+        >>> elf = ELF.from_assembly(shellcraft.trap())
        >>> path = s.upload(elf.path)
        >>> _ =s.chmod('+x', path)
        >>> io = s.process(path)
@@ -429,6 +432,56 @@ class Corefile(ELF):
        -1
        >>> io.corefile.signal == signal.SIGTRAP # doctest: +SKIP
        True
+
+        Make sure fault_addr synthesis works for amd64 on ret.
+
+        >>> context.clear(arch='amd64')
+        >>> elf = ELF.from_assembly('push 1234; ret')
+        >>> io = elf.process()
+        >>> io.wait()
+        >>> io.corefile.fault_addr
+        1234
+
+    Tests:
+
+        These are extra tests not meant to serve as examples.
+
+        Corefile.getenv() works correctly, even if the environment variable's
+        value contains embedded '='. Corefile is able to find the stack, even
+        if the stack pointer doesn't point at the stack.
+
+        >>> elf = ELF.from_assembly(shellcraft.crash())
+        >>> io = elf.process(env={'FOO': 'BAR=BAZ'})
+        >>> io.wait()
+        >>> core = io.corefile
+        >>> core.getenv('FOO')
+        'BAR=BAZ'
+        >>> core.sp == 0
+        True
+        >>> core.sp in core.stack
+        False
+
+        Corefile gracefully handles the stack being filled with garbage, including
+        argc / argv / envp being overwritten.
+
+        >>> context.clear(arch='i386')
+        >>> assembly = '''
+        ... LOOP:
+        ...   mov dword ptr [esp], 0x41414141
+        ...   pop eax
+        ...   jmp LOOP
+        ... '''
+        >>> elf = ELF.from_assembly(assembly)
+        >>> io = elf.process()
+        >>> io.wait()
+        >>> core = io.corefile
+        [!] End of the stack is corrupted, skipping stack parsing (got: 4141414141414141)
+        >>> core.argc, core.argv, core.env
+        (0, [], {})
+        >>> core.stack.data.endswith('AAAA')
+        True
+        >>> core.fault_addr == core.sp
+        True
    """

    _fill_gaps = False
@@ -454,6 +507,9 @@ class Corefile(ELF):
        #: variable.
        #:
        #: Note: Use with the :meth:`.ELF.string` method to extract them.
+        #:
+        #: Note: If FOO=BAR is in the environment, self.env['FOO'] is the
+        #:       address of the string "BAR\x00".
        self.env = {}

        #: :class:`list`: List of addresses of arguments on the stack.
@@ -493,36 +549,40 @@ class Corefile(ELF):
            for segment in self.segments:
                if not isinstance(segment, elftools.elf.segments.NoteSegment):
                    continue
+
+
+                # Note that older versions of pyelftools (<=0.24) are missing enum values
+                # for NT_PRSTATUS, NT_PRPSINFO, NT_AUXV, etc.
+                # For this reason, we have to check if note.n_type is any of several values.
                for note in iter_notes(segment):
-                    # Try to find NT_PRSTATUS.  Note that pyelftools currently
-                    # mis-identifies the enum name as 'NT_GNU_ABI_TAG'.
+                    # Try to find NT_PRSTATUS.
                    if prstatus_type and \
                       note.n_descsz == ctypes.sizeof(prstatus_type) and \
-                       note.n_type == 'NT_GNU_ABI_TAG':
+                       note.n_type in ('NT_GNU_ABI_TAG', 'NT_PRSTATUS'):
                        self.NT_PRSTATUS = note
                        self.prstatus = prstatus_type.from_buffer_copy(note.n_desc)

                    # Try to find NT_PRPSINFO
-                    # Note that pyelftools currently mis-identifies the enum name
-                    # as 'NT_GNU_BUILD_ID'
-                    if note.n_descsz == ctypes.sizeof(prpsinfo_type) and \
-                       note.n_type == 'NT_GNU_BUILD_ID':
+                    if prpsinfo_type and \
+                       note.n_descsz == ctypes.sizeof(prpsinfo_type) and \
+                       note.n_type in ('NT_GNU_ABI_TAG', 'NT_PRPSINFO'):
                        self.NT_PRPSINFO = note
                        self.prpsinfo = prpsinfo_type.from_buffer_copy(note.n_desc)

                    # Try to find NT_SIGINFO so we can see the fault
-                    if note.n_type == 0x53494749:
+                    if note.n_type in (0x53494749, 'NT_SIGINFO'):
                        self.NT_SIGINFO = note
                        self.siginfo = siginfo_type.from_buffer_copy(note.n_desc)

                    # Try to find the list of mapped files
-                    if note.n_type == constants.NT_FILE:
+                    if note.n_type in (constants.NT_FILE, 'NT_FILE'):
                        with context.local(bytes=self.bytes):
                            self._parse_nt_file(note)

                    # Try to find the auxiliary vector, which will tell us
                    # where the top of the stack is.
-                    if note.n_type == constants.NT_AUXV:
+                    if note.n_type in (constants.NT_AUXV, 'NT_AUXV'):
+                        self.NT_AUXV = note
                        with context.local(bytes=self.bytes):
                            self._parse_auxv(note)

@@ -531,21 +591,15 @@ class Corefile(ELF):

            if self.stack and self.mappings:
                for mapping in self.mappings:
-                    if mapping.stop == self.stack:
+                    if self.stack in mapping or self.stack == mapping.stop:
                        mapping.name = '[stack]'
                        self.stack   = mapping
                        break
                else:
-                    for mapping in self.mappings:
-                        if self.stack in mapping:
-                            mapping.name = '[stack]'
-                            self.stack   = mapping
-                            break
-                    else:
-                        log.warn('Could not find the stack!')
-                        self.stack = None
+                    log.warn('Could not find the stack!')
+                    self.stack = None

-            with context.local(bytes=self.bytes, log_level='error'):
+            with context.local(bytes=self.bytes, log_level='warn'):
                try:
                    self._parse_stack()
                except ValueError:
@@ -664,7 +718,22 @@ class Corefile(ELF):

    @property
    def signal(self):
-        """:class:`int`: Signal which caused the core to be dumped."""
+        """:class:`int`: Signal which caused the core to be dumped.
+
+        Example:
+
+            >>> elf = ELF.from_assembly(shellcraft.trap())
+            >>> io = elf.process()
+            >>> io.wait()
+            >>> io.corefile.signal == signal.SIGTRAP
+            True
+
+            >>> elf = ELF.from_assembly(shellcraft.crash())
+            >>> io = elf.process()
+            >>> io.wait()
+            >>> io.corefile.signal == signal.SIGSEGV
+            True
+        """
        if self.siginfo:
            return int(self.siginfo.si_signo)
        if self.prstatus:
@@ -675,11 +744,49 @@ class Corefile(ELF):
        """:class:`int`: Address which generated the fault, for the signals
            SIGILL, SIGFPE, SIGSEGV, SIGBUS.  This is only available in native
            core dumps created by the kernel.  If the information is unavailable,
-            this returns the address of the instruction pointer."""
-        if self.siginfo:
-            return int(self.siginfo.sigfault_addr)
+            this returns the address of the instruction pointer.

-        return getattr(self, 'pc', 0)
+
+        Example:
+
+            >>> elf = ELF.from_assembly('mov eax, 0xdeadbeef; jmp eax', arch='i386')
+            >>> io = elf.process()
+            >>> io.wait()
+            >>> io.corefile.fault_addr == io.corefile.eax == 0xdeadbeef
+            True
+        """
+        if not self.siginfo:
+            return getattr(self, 'pc', 0)
+
+        fault_addr = int(self.siginfo.sigfault_addr)
+
+        # The fault_addr is zero if the crash occurs due to a
+        # "protection fault", e.g. a dereference of 0x4141414141414141
+        # because this is technically a kernel address.
+        #
+        # A protection fault does not set "fault_addr" in the siginfo.
+        # (http://elixir.free-electrons.com/linux/v4.14-rc8/source/kernel/signal.c#L1052)
+        #
+        # Since a common use for corefiles is to spray the stack with a
+        # cyclic pattern to find the offset to get control of $PC,
+        # check for a "ret" instruction ("\xc3").
+        #
+        # If we find a RET at $PC, extract the "return address" from the
+        # top of the stack.
+        if fault_addr == 0 and self.siginfo.si_code == 0x80:
+            try:
+                code = self.read(self.pc, 1)
+                RET = '\xc3'
+                if code == RET:
+                    fault_addr = self.unpack(self.sp)
+            except Exception:
+                # Could not read $rsp or $rip
+                pass
+
+        return fault_addr
+
+        # No embedded siginfo structure, so just return the
+        # current instruction pointer.

    @property
    def _pc_register(self):
@@ -722,8 +829,8 @@ class Corefile(ELF):
        fields = [
            repr(self.path),
            '%-10s %s' % ('Arch:', gnu_triplet),
-            '%-10s %#x' % ('%s:' % self._pc_register.upper(), getattr(self, 'pc', 0)),
-            '%-10s %#x' % ('%s:' % self._sp_register.upper(), getattr(self, 'sp', 0)),
+            '%-10s %#x' % ('%s:' % self._pc_register.upper(), self.pc or 0),
+            '%-10s %#x' % ('%s:' % self._sp_register.upper(), self.sp or 0),
        ]

        if self.exe and self.exe.name:
@@ -790,18 +897,25 @@ class Corefile(ELF):
        if not stack:
            return

+        # If the stack does not end with zeroes, something is very wrong.
+        if not stack.data.endswith('\x00' * 8):
+            log.warn_once("End of the stack is corrupted, skipping stack parsing (got: %s)",
+                          enhex(self.data[-8:]))
+            return
+
        # AT_EXECFN is the start of the filename, e.g. '/bin/sh'
        # Immediately preceding is a NULL-terminated environment variable string.
        # We want to find the beginning of it
-        if self.at_execfn:
-            address = self.at_execfn-1
-        else:
-            log.debug('No AT_EXECFN')
+        if not self.at_execfn:
            address = stack.stop
            address -= 2*self.bytes
            address -= 1
            address = stack.rfind('\x00', None, address)
            address += 1
+            self.at_execfn = address
+
+        address = self.at_execfn-1
+

        # Sanity check!
        try:
@@ -826,6 +940,10 @@ class Corefile(ELF):
        # find a pointer to this address, followed by a NULL.
        last_env_addr = address + 1
        p_last_env_addr = stack.find(pack(last_env_addr), None, last_env_addr)
+        if p_last_env_addr < 0:
+            # Something weird is happening.  Just don't touch it.
+            log.warn_once("Found bad environment at %#x", last_env_addr)
+            return

        # Sanity check that we did correctly find the envp NULL terminator.
        envp_nullterm = p_last_env_addr+context.bytes
@@ -844,11 +962,31 @@ class Corefile(ELF):
        # Now we can fill in the environment
        env_pointer_data = stack[start_of_envp:p_last_env_addr+self.bytes]
        for pointer in unpack_many(env_pointer_data):
-            end = stack.find('=', last_env_addr)

-            if pointer in stack and end in stack:
-                name = stack[pointer:end]
-                self.env[name] = pointer
+            # If the stack is corrupted, the pointer will be outside of
+            # the stack.
+            if pointer not in stack:
+                continue
+
+            try:
+                name_value = self.string(pointer)
+            except Exception:
+                continue
+
+            name, value = name_value.split('=', 1)
+
+            # "end" points at the byte after the null terminator
+            end = pointer + len(name_value) + 1
+
+            # Do not mark things as environment variables if they point
+            # outside of the stack itself, or we had to cross into a different
+            # mapping (after the stack) to read it.
+            # This may occur when the entire stack is filled with non-NUL bytes,
+            # and we NULL-terminate on a read failure in .string().
+            if end not in stack:
+                continue
+
+            self.env[name] = pointer + len(name) + len('=')

        # May as well grab the arguments off the stack as well.
        # argc comes immediately before argv[0] on the stack, but
@@ -900,15 +1038,32 @@ class Corefile(ELF):

        Returns:
            :class:`str`: The contents of the environment variable.
+
+        Example:
+
+            >>> elf = ELF.from_assembly(shellcraft.trap())
+            >>> io = elf.process(env={'GREETING': 'Hello!'})
+            >>> io.wait()
+            >>> io.corefile.getenv('GREETING')
+            'Hello!'
        """
        if name not in self.env:
            log.error("Environment variable %r not set" % name)

-        return self.string(self.env[name]).split('=',1)[-1]
+        return self.string(self.env[name])

    @property
    def registers(self):
-        """:class:`dict`: All available registers in the coredump."""
+        """:class:`dict`: All available registers in the coredump.
+
+        Example:
+
+            >>> elf = ELF.from_assembly('mov eax, 0xdeadbeef;' + shellcraft.trap(), arch='i386')
+            >>> io = elf.process()
+            >>> io.wait()
+            >>> io.corefile.registers['eax'] == 0xdeadbeef
+            True
+        """
        if not self.prstatus:
            return {}

@@ -956,7 +1111,7 @@ class Coredump(Corefile):
 class CorefileFinder(object):
    def __init__(self, proc):
        if proc.poll() is None:
-            log.error("Process %i has not exited" % (process.pid))
+            log.error("Process %i has not exited" % (proc.pid))

        self.process = proc
        self.pid = proc.pid
@@ -1016,6 +1171,10 @@ class CorefileFinder(object):
            new_path = 'core.%i' % core_pid
            if core_pid > 0 and new_path != self.core_path:
                write(new_path, self.read(self.core_path))
+                try:
+                    self.unlink(self.core_path)
+                except (IOError, OSError):
+                    log.warn("Could not delete %r" % self.core_path)
                self.core_path = new_path

        # Check the PID
@@ -252,6 +252,40 @@ class constants:
    AT_L2_CACHESHAPE        = 36
    AT_L3_CACHESHAPE        = 37

+    # Legal flags used in the d_val field of the DT_FLAGS dynamic entry.
+    DF_ORIGIN               = 0x01
+    DF_SYMBOLIC             = 0x02
+    DF_TEXTREL              = 0x04
+    DF_BIND_NOW             = 0x08
+    DF_STATIC_TLS           = 0x10
+
+    # Legal flags used in the d_val field of the DT_FLAGS_1 dynamic entry.
+    DF_1_NOW                = 0x00000001
+    DF_1_GLOBAL             = 0x00000002
+    DF_1_GROUP              = 0x00000004
+    DF_1_NODELETE           = 0x00000008
+    DF_1_LOADFLTR           = 0x00000010
+    DF_1_INITFIRST          = 0x00000020
+    DF_1_NOOPEN             = 0x00000040
+    DF_1_ORIGIN             = 0x00000080
+    DF_1_DIRECT             = 0x00000100
+    DF_1_TRANS              = 0x00000200
+    DF_1_INTERPOSE          = 0x00000400
+    DF_1_NODEFLIB           = 0x00000800
+    DF_1_NODUMP             = 0x00001000
+    DF_1_CONFALT            = 0x00002000
+    DF_1_ENDFILTEE          = 0x00004000
+    DF_1_DISPRELDNE         = 0x00008000
+    DF_1_DISPRELPND         = 0x00010000
+    DF_1_NODIRECT           = 0x00020000
+    DF_1_IGNMULDEF          = 0x00040000
+    DF_1_NOKSYMS            = 0x00080000
+    DF_1_NOHDR              = 0x00100000
+    DF_1_EDITED             = 0x00200000
+    DF_1_NORELOC            = 0x00400000
+    DF_1_SYMINTPOSE         = 0x00800000
+    DF_1_GLOBAUDIT          = 0x01000000
+    DF_1_SINGLETON          = 0x02000000


 class Elf32_Ehdr(ctypes.Structure):
@@ -51,22 +51,29 @@ from elftools.elf.constants import P_FLAGS
 from elftools.elf.constants import SHN_INDICES
 from elftools.elf.descriptions import describe_e_type
 from elftools.elf.elffile import ELFFile
-from elftools.elf.enums import ENUM_P_TYPE
 from elftools.elf.gnuversions import GNUVerDefSection
 from elftools.elf.relocation import RelocationSection
 from elftools.elf.sections import SymbolTableSection
+from elftools.elf.segments import InterpSegment
+
+# See https://github.com/Gallopsled/pwntools/issues/1189
+try:
+    from elftools.elf.enums import ENUM_P_TYPE
+except ImportError:
+    from elftools.elf.enums import ENUM_P_TYPE_BASE as ENUM_P_TYPE

 import intervaltree

 from pwnlib import adb
+from pwnlib import qemu
 from pwnlib.asm import *
 from pwnlib.context import LocalContext
 from pwnlib.context import context
 from pwnlib.elf.config import kernel_configuration
 from pwnlib.elf.config import parse_kconfig
+from pwnlib.elf.datatypes import constants
 from pwnlib.elf.plt import emulate_plt_instructions
 from pwnlib.log import getLogger
-from pwnlib.qemu import get_qemu_arch
 from pwnlib.term import text
 from pwnlib.tubes.process import process
 from pwnlib.util import misc
@@ -181,6 +188,7 @@ class ELF(ELFFile):
    functions = {}
    endian = 'little'
    address = 0x400000
+    linker = None

    # Whether to fill gaps in memory with zeroed pages
    _fill_gaps = True
@@ -259,6 +267,15 @@ class ELF(ELFFile):
                self.arch = 'mips64'
                self.bits = 64

+        # Is this a native binary? Should we be checking QEMU?
+        try:
+            with context.local(arch=self.arch):
+                #: Whether this ELF should be able to run natively
+                self.native = context.native
+        except AttributeError:
+            # The architecture may not be supported in pwntools
+            self.native = False
+
        self._address = 0
        if self.elftype != 'DYN':
            for seg in self.iter_segments_by_type('PT_LOAD'):
@@ -297,8 +314,20 @@ class ELF(ELFFile):

        for seg in self.iter_segments_by_type('PT_INTERP'):
            self.executable = True
+
+            #: ``True`` if the ELF is statically linked
            self.statically_linked = False

+            #: Path to the linker for the ELF
+            self.linker = self.read(seg.header.p_vaddr, seg.header.p_memsz)
+            self.linker = self.linker.rstrip('\x00')
+
+        #: Operating system of the ELF
+        self.os = 'linux'
+
+        if self.linker and self.linker.startswith('/system/bin/linker'):
+            self.os = 'android'
+
        #: ``True`` if the ELF is a shared library
        self.library = not self.executable and self.elftype == 'DYN'

@@ -407,10 +436,10 @@ class ELF(ELFFile):
        import pwnlib.gdb
        return pwnlib.gdb.debug([self.path] + argv, *a, **kw)

-    def _describe(self):
+    def _describe(self, *a, **kw):
        log.info_once('\n'.join((repr(self.path),
                                '%-10s%s-%s-%s' % ('Arch:', self.arch, self.bits, self.endian),
-                                self.checksec())))
+                                self.checksec(*a, **kw))))

    def __repr__(self):
        return "ELF(%r)" % self.path
@@ -604,10 +633,20 @@ class ELF(ELFFile):
        >>> any(map(lambda x: 'libc' in x, bash.libs.keys()))
        True
        """
-        if not self.get_section_by_name('.dynamic'):
+
+        # We need a .dynamic section for dynamically linked libraries
+        if not self.get_section_by_name('.dynamic') or self.statically_linked:
            self.libs= {}
            return

+        # We must also specify a 'PT_INTERP', otherwise it's a 'statically-linked'
+        # binary which is also position-independent (and as such has a .dynamic).
+        for segment in self.iter_segments_by_type('PT_INTERP'):
+            break
+        else:
+            self.libs = {}
+            return
+
        try:
            cmd = 'ulimit -s unlimited; LD_TRACE_LOADED_OBJECTS=1 LD_WARN=1 LD_BIND_NOW=1 %s 2>/dev/null' % sh_string(self.path)

@@ -618,10 +657,11 @@ class ELF(ELFFile):
                if os.path.exists(lib):
                    continue

-                qemu_lib = '/etc/qemu-binfmt/%s/%s' % (get_qemu_arch(arch=self.arch), lib)
-
-                if os.path.exists(qemu_lib):
-                    libs[os.path.realpath(qemu_lib)] = libs.pop(lib)
+                if not self.native:
+                    ld_prefix = qemu.ld_prefix()
+                    qemu_lib = os.path.exists(os.path.join(ld_prefix, lib))
+                    if qemu_lib:
+                        libs[os.path.realpath(qemu_lib)] = libs.pop(lib)

            self.libs = libs

@@ -1327,14 +1367,28 @@ class ELF(ELFFile):
        .. _PT_GNU_RELRO: https://refspecs.linuxbase.org/LSB_3.1.1/LSB-Core-generic/LSB-Core-generic.html#PROGHEADER
        .. _DF_BIND_NOW: http://refspecs.linuxbase.org/elf/gabi4+/ch5.dynamic.html#df_bind_now

+        >>> path = pwnlib.data.elf.relro.path
+        >>> for test in glob(os.path.join(path, 'test-*')):
+        ...     e = ELF(test)
+        ...     expected = os.path.basename(test).split('-')[2]
+        ...     actual = str(e.relro).lower()
+        ...     assert actual == expected
        """
+        if not any('GNU_RELRO' in str(s.header.p_type) for s in self.segments):
+            return None
+
        if self.dynamic_by_tag('DT_BIND_NOW'):
            return "Full"

-        if any('GNU_RELRO' in str(s.header.p_type) for s in self.segments):
-            return "Partial"
+        flags = self.dynamic_value_by_tag('DT_FLAGS')
+        if flags and flags & constants.DF_BIND_NOW:
+            return "Full"

-        return None
+        flags_1 = self.dynamic_value_by_tag('DT_FLAGS_1')
+        if flags_1 and flags_1 & constants.DF_1_NOW:
+            return "Full"
+
+        return "Partial"

    @property
    def nx(self):
@@ -1473,9 +1527,9 @@ class ELF(ELFFile):
        if not dt_runpath:
            return None

-        return self.dynamic_string(dt_rpath.entry.d_ptr)
+        return self.dynamic_string(dt_runpath.entry.d_ptr)

-    def checksec(self, banner=True):
+    def checksec(self, banner=True, color=True):
        """checksec(banner=True)

        Prints out information in the binary, similar to ``checksec.sh``.
@@ -1483,9 +1537,9 @@ class ELF(ELFFile):
        Arguments:
            banner(bool): Whether to print the path to the ELF binary.
        """
-        red    = text.red
-        green  = text.green
-        yellow = text.yellow
+        red    = text.red if color else str
+        green  = text.green if color else str
+        yellow = text.yellow if color else str

        res = []

@@ -1660,16 +1714,33 @@ class ELF(ELFFile):
        return packing.unpack(self.read(address, context.bytes), *a, **kw)

    def string(self, address):
-        """Reads a null-terminated string from the specified ``address``"""
+        """string(address) -> str
+
+        Reads a null-terminated string from the specified ``address``
+
+        Returns:
+            A ``str`` with the string contents (NUL terminator is omitted),
+            or an empty string if no NUL terminator could be found.
+        """
        data = ''
        while True:
-            c = self.read(address, 1)
+            read_size = 0x1000
+            partial_page = address & 0xfff
+
+            if partial_page:
+                read_size -= partial_page
+
+            c = self.read(address, read_size)
+
            if not c:
                return ''
-            if c == '\x00':
-                return data
+
            data += c
-            address += 1
+
+            if '\x00' in c:
+                return data[:data.index('\x00')]
+
+            address += len(c)

    def flat(self, address, *a, **kw):
        """Writes a full array of values to the specified address.
@@ -1714,4 +1785,3 @@ class ELF(ELFFile):
                return

        log.error("Could not find PT_GNU_STACK, stack should already be executable")
-
@@ -14,11 +14,9 @@ class ArmEncoder(Encoder):
    arch = 'arm'

    blacklist  = {chr(c) for c in range(256) if chr(c) in (string.ascii_letters + string.digits)}
+    icache_flush = 1

    def __call__(self, input, avoid, pcreg=None):
-
-        icache_flush = 1
-
        # If randomization is disabled, ensure that the seed
        # is always the same for the builder.
        state = random.getstate()
@@ -29,9 +27,9 @@ class ArmEncoder(Encoder):
            b = builder.builder()

            enc_data = b.enc_data_builder(input)
-            dec_loop = b.DecoderLoopBuilder(icache_flush)
+            dec_loop = b.DecoderLoopBuilder(self.icache_flush)
            enc_dec_loop = b.encDecoderLoopBuilder(dec_loop)
-            dec = b.DecoderBuilder(dec_loop, icache_flush)
+            dec = b.DecoderBuilder(dec_loop, self.icache_flush)

            output,dec = b.buildInit(dec);

@@ -100,6 +100,7 @@ import time
 from pwnlib import adb
 from pwnlib import atexit
 from pwnlib import elf
+from pwnlib import qemu
 from pwnlib import tubes
 from pwnlib.asm import _bfdname
 from pwnlib.asm import make_elf
@@ -107,7 +108,6 @@ from pwnlib.asm import make_elf_from_assembly
 from pwnlib.context import LocalContext
 from pwnlib.context import context
 from pwnlib.log import getLogger
-from pwnlib.qemu import get_qemu_user
 from pwnlib.util import misc
 from pwnlib.util import proc

@@ -127,10 +127,19 @@ def debug_assembly(asm, gdbscript=None, vma=None):
        asm(str): Assembly code to debug
        gdbscript(str): Script to run in GDB
        vma(int): Base address to load the shellcode at
-        **kwargs: Override any :obj:`.context` values.
+        **kwargs: Override any :obj:`pwnlib.context.context` values.

    Returns:
        :class:`.process`
+
+    Example:
+
+        .. code-block:: python
+
+            assembly = shellcraft.echo("Hello world!\n")
+            io = gdb.debug_assembly(assembly)
+            io.recvline()
+            # 'Hello world!'
    """
    tmp_elf = make_elf_from_assembly(asm, vma=vma, extract=False)
    os.chmod(tmp_elf, 0777)
@@ -153,10 +162,20 @@ def debug_shellcode(data, gdbscript=None, vma=None):
        data(str): Assembled shellcode bytes
        gdbscript(str): Script to run in GDB
        vma(int): Base address to load the shellcode at
-        **kwargs: Override any :obj:`.context` values.
+        **kwargs: Override any :obj:`pwnlib.context.context` values.

    Returns:
        :class:`.process`
+
+    Example:
+
+        .. code-block:: python
+
+            assembly = shellcraft.echo("Hello world!\n")
+            shellcode = asm(assembly)
+            io = gdb.debug_shellcode(shellcode)
+            io.recvline()
+            # 'Hello world!'
    """
    if isinstance(data, unicode):
        log.error("Shellcode is cannot be unicode.  Did you mean debug_assembly?")
@@ -210,7 +229,7 @@ def _gdbserver_args(pid=None, path=None, args=None, which=None):

    orig_args = args

-    gdbserver_args = [gdbserver]
+    gdbserver_args = [gdbserver, '--multi']
    if context.aslr:
        gdbserver_args += ['--no-disable-randomization']
    else:
@@ -230,6 +249,12 @@ def _gdbserver_port(gdbserver, ssh):
    # Process /bin/bash created; pid = 14366
    # Listening on port 34816
    process_created = gdbserver.recvline()
+
+    if process_created.startswith('ERROR:'):
+        raise ValueError(
+            'Failed to spawn process under gdbserver. gdbserver error message: %s' % process_created
+        )
+
    gdbserver.pid   = int(process_created.split()[-1], 0)

    listening_on = ''
@@ -268,7 +293,7 @@ def _get_runner(ssh=None):
    else:                          return tubes.process.process

@LocalContext
-def debug(args, gdbscript=None, exe=None, ssh=None, env=None, **kwargs):
+def debug(args, gdbscript=None, exe=None, ssh=None, env=None, sysroot=None, **kwargs):
    """debug(args) -> tube

    Launch a GDB server with the specified command line,
@@ -280,6 +305,8 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, **kwargs):
        exe(str): Path to the executable on disk
        env(dict): Environment to start the binary in
        ssh(:class:`.ssh`): Remote ssh session to use to launch the process.
+        sysroot(str): Foreign-architecture sysroot, used for QEMU-emulated binaries
+            and Android targets.

    Returns:
        :class:`.process` or :class:`.ssh_channel`: A tube connected to the target process
@@ -290,14 +317,6 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, **kwargs):
        from the very beginning.  This requires that both ``gdb`` and ``gdbserver``
        are installed on your machine.

-        .. code-block:: python
-
-            # Create a new process, and stop it at 'main'
-            io = gdb.debug('bash', '''
-            break main
-            continue
-            ''')
-
        When GDB opens via :func:`debug`, it will initially be stopped on the very first
        instruction of the dynamic linker (``ld.so``) for dynamically-linked binaries.

@@ -322,6 +341,22 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, **kwargs):
            - Generally, you just add a few ``continue`` commands until things are set up
              the way you want it to be.

+    Examples:
+
+        .. code-block:: python
+
+            # Create a new process, and stop it at 'main'
+            io = gdb.debug('bash', '''
+            break main
+            continue
+            ''')
+
+            # Send a command to Bash
+            io.sendline("echo hello")
+
+            # Interact with the process
+            io.interactive()
+
        .. code-block:: python

            # Create a new process, and stop it at 'main'
@@ -336,6 +371,12 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, **kwargs):
            continue
            ''')

+            # Send a command to Bash
+            io.sendline("echo hello")
+
+            # Interact with the process
+            io.interactive()
+
        You can use :func:`debug` to spawn new processes on remote machines as well,
        by using the ``ssh=`` keyword to pass in your :class:`.ssh` instance.

@@ -345,12 +386,18 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, **kwargs):
            shell = ssh('passcode', 'pwnable.kr', 2222, password='guest')

            # Start a process on the server
-            bash = gdb.debug(['bash'],
+            io = gdb.debug(['bash'],
                            ssh=shell,
                            gdbscript='''
            break main
            continue
            ''')
+
+            # Send a command to Bash
+            io.sendline("echo hello")
+
+            # Interact with the process
+            io.interactive()
    """
    if isinstance(args, (int, tubes.process.process, tubes.ssh.ssh_channel)):
        log.error("Use gdb.attach() to debug a running process")
@@ -365,6 +412,7 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, **kwargs):

    runner = _get_runner(ssh)
    which  = _get_which(ssh)
+    gdbscript = gdbscript or ''

    if context.noptrace:
        log.warn_once("Skipping debugger since context.noptrace==True")
@@ -374,7 +422,15 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, **kwargs):
        args = _gdbserver_args(args=args, which=which)
    else:
        qemu_port = random.randint(1024, 65535)
-        args = [get_qemu_user(), '-g', str(qemu_port)] + args
+        qemu_user = qemu.user_path()
+        sysroot = sysroot or qemu.ld_prefix(env=env)
+        if not qemu_user:
+            log.error("Cannot debug %s binaries without appropriate QEMU binaries" % context.arch)
+        args = [qemu_user, '-g', str(qemu_port)] + args
+
+    # Use a sane default sysroot for Android
+    if not sysroot and context.os == 'android':
+        sysroot = 'remote:/'

    # Make sure gdbserver/qemu is installed
    if not which(args[0]):
@@ -383,6 +439,8 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, **kwargs):
    exe = exe or which(orig_args[0])
    if not exe:
        log.error("%s does not exist" % orig_args[0])
+    else:
+        gdbscript = 'file %s\n%s' % (exe, gdbscript)

    # Start gdbserver/qemu
    # (Note: We override ASLR here for the gdbserver process itself.)
@@ -401,7 +459,7 @@ def debug(args, gdbscript=None, exe=None, ssh=None, env=None, **kwargs):
    if not ssh and context.os == 'android':
        host = context.adb_host

-    attach((host, port), exe=exe, gdbscript=gdbscript, need_ptrace_scope = False, ssh=ssh)
+    attach((host, port), exe=exe, gdbscript=gdbscript, need_ptrace_scope = False, ssh=ssh, sysroot=sysroot)

    # gdbserver outputs a message when a client connects
    garbage = gdbserver.recvline(timeout=1)
@@ -417,7 +475,8 @@ def get_gdb_arch():
        'powerpc': 'powerpc:common',
        'powerpc64': 'powerpc:common64',
        'mips64': 'mips:isa64',
-        'thumb': 'arm'
+        'thumb': 'arm',
+        'sparc64': 'sparc:v9'
    }.get(context.arch, context.arch)

 def binary():
@@ -425,8 +484,13 @@ def binary():

    Returns:
        str: Path to the appropriate ``gdb`` binary to use.
+
+    Example:
+
+        >>> gdb.binary() # doctest: +SKIP
+        '/usr/bin/gdb'
    """
-    gdb = misc.which('gdb')
+    gdb = misc.which('pwntools-gdb') or misc.which('gdb')

    if not context.native:
        multiarch = misc.which('gdb-multiarch')
@@ -443,7 +507,7 @@ def binary():
    return gdb

@LocalContext
-def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_args = None, ssh = None):
+def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_args = None, ssh = None, sysroot = None):
    """attach(target, gdbscript = None, exe = None, arch = None, ssh = None) -> None

    Start GDB in a new terminal and attach to `target`.
@@ -455,6 +519,8 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
        arch(str): Architechture of the target binary.  If `exe` known GDB will
          detect the architechture automatically (if it is supported).
        gdb_args(list): List of additional arguments to pass to GDB.
+        sysroot(str): Foreign-architecture sysroot, used for QEMU-emulated binaries
+            and Android targets.

    Returns:
        PID of the GDB process (or the window which it is running in).
@@ -479,6 +545,8 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
            This will use the GDB installed on the remote machine.
            If a password is required to connect, the ``sshpass`` program must be installed.

+    Examples:
+
        .. code-block:: python

            # Attach directly to pid 1234
@@ -552,11 +620,17 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
    if gdbscript and not gdbscript.endswith('\n'):
        gdbscript += '\n'

+    # Use a sane default sysroot for Android
+    if not sysroot and context.os == 'android':
+        sysroot = 'remote:/'
+
    # gdb script to run before `gdbscript`
    pre = ''
    if not context.native:
        pre += 'set endian %s\n' % context.endian
        pre += 'set architecture %s\n' % get_gdb_arch()
+        if sysroot:
+            pre += 'set sysroot %s\n' % sysroot

        if context.os == 'android':
            pre += 'set gnutarget ' + _bfdname() + '\n'
@@ -611,9 +685,18 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
        pid = pids[0]
    elif isinstance(target, tubes.process.process):
        pid = proc.pidof(target)[0]
+        exe = exe or target.executable
    elif isinstance(target, tuple) and len(target) == 2:
        host, port = target
-        pre += 'target remote %s:%d\n' % (host, port)
+
+        if context.os != 'android':
+            pre += 'target remote %s:%d\n' % (host, port)
+        else:
+            # Android debugging is done over gdbserver, which can't follow
+            # new inferiors (tldr; follow-fork-mode child) unless it is run
+            # in extended-remote mode.
+            pre += 'target extended-remote %s:%d\n' % (host, port)
+            pre += 'set detach-on-fork off\n'

        def findexe():
            for spid in proc.pidof(target):
@@ -640,12 +723,20 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
    if not pid and not exe:
        log.error('could not find target process')

+    if exe:
+        # The 'file' statement should go first
+        pre = 'file %s\n%s' % (exe, pre)
+
    cmd = binary()

    if gdb_args:
        cmd += ' '
        cmd += ' '.join(gdb_args)

+    if context.gdbinit:
+        cmd += ' -nh '                     # ignore ~/.gdbinit
+        cmd += ' -x %s ' % context.gdbinit # load custom gdbinit
+
    cmd += ' -q '

    if exe and context.native:
@@ -666,7 +757,11 @@ def attach(target, gdbscript = None, exe = None, need_ptrace_scope = True, gdb_a
        gdbserver = runner(gdb_cmd)
        port    = _gdbserver_port(gdbserver, None)
        host    = context.adb_host
-        pre    += 'target remote %s:%i' % (context.adb_host, port)
+        pre    += 'target extended-remote %s:%i\n' % (context.adb_host, port)
+
+        # gdbserver on Android sets 'detach-on-fork on' which breaks things
+        # when you're trying to debug anything that forks.
+        pre += 'set detach-on-fork off\n'

    gdbscript = pre + (gdbscript or '')

@@ -694,7 +789,7 @@ def ssh_gdb(ssh, argv, gdbscript = None, arch = None, **kwargs):
        argv = [argv]

    exe = argv[0]
-    argv = ["gdbserver", "127.0.0.1:0"] + argv
+    argv = ["gdbserver", "--multi", "127.0.0.1:0"] + argv

    # Download the executable
    local_exe = os.path.basename(exe)
@@ -3,7 +3,7 @@ from os.path import basename

 from docutils import nodes
 from docutils import statemachine
-from sphinx.util.compat import Directive
+from docutils.parsers.rst import Directive

 try:
    from StringIO import StringIO
@@ -21,14 +21,63 @@ from pwnlib.util.web import wget

 log = getLogger(__name__)

-cache_dir = os.path.join(tempfile.gettempdir(), 'pwn')
+HASHES = ['build_id', 'sha1', 'sha256', 'md5']
+
+def search_by_hash(hex_encoded_id, hash_type='build_id'):
+    assert hash_type in HASHES, hash_type
+
+    # Ensure that the libcdb cache directory exists
+    cache_dir = os.path.join(context.cache_dir, 'libcdb', hash_type)
+
+    if not os.path.isdir(cache_dir):
+        os.makedirs(cache_dir)
+
+    # If we already downloaded the file, and it looks even passingly like
+    # a valid ELF file, return it.
+    cache = os.path.join(cache_dir, hex_encoded_id)
+
+    if os.path.exists(cache):
+        log.debug("Found existing cached libc at %r", cache)
+
+        data = read(cache)
+        if data.startswith('\x7FELF'):
+            log.info_once("Using cached data from %r", cache)
+            return cache
+        else:
+            log.info_once("Skipping unavialable libc %s", hex_encoded_id)
+            return None
+
+    # Build the URL using the requested hash type
+    url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/%s/" % hash_type
+    url      = urlparse.urljoin(url_base, hex_encoded_id)
+
+    data   = ""
+    while not data.startswith('\x7fELF'):
+        log.debug("Downloading data from LibcDB: %s", url)
+        data = wget(url)
+
+        if not data:
+            log.warn_once("Could not fetch libc for build_id %s", hex_encoded_id)
+            break
+
+        # GitLab serves up symlinks with
+        if data.startswith('..'):
+            url = os.path.dirname(url) + '/'
+            url = urlparse.urljoin(url, data)
+
+    # Save whatever we got to the cache
+    write(cache, data or '')
+
+    # Return ``None`` if we did not get a valid ELF file
+    if not data or not data.startswith('\x7FELF'):
+        return None
+
+    return cache
+

 def search_by_build_id(hex_encoded_id):
    """
-    Given a hex-encoded Build ID, return the path to an ELF with that Build ID
-    only the local system.
-
-    If it can't be found, return None.
+    Given a hex-encoded Build ID, attempt to download a matching libc from libcdb.

    Arguments:
        hex_encoded_id(str):
@@ -36,31 +85,79 @@ def search_by_build_id(hex_encoded_id):

    Returns:
        Path to the downloaded library on disk, or :const:`None`.
+
+    Examples:
+        >>> filename = search_by_build_id('fe136e485814fee2268cf19e5c124ed0f73f4400')
+        >>> hex(ELF(filename).symbols.read)
+        '0xda260'
+        >>> None == search_by_build_id('XX')
+        True
    """
-    cache = cache_dir + '-libc.so.' + hex_encoded_id
+    return search_by_hash(hex_encoded_id, 'build_id')

-    if os.path.exists(cache) and read(cache).startswith('\x7FELF'):
-        log.info_once("Using cached data from %r" % cache)
-        return cache
+def search_by_md5(hex_encoded_id):
+    """
+    Given a hex-encoded md5sum, attempt to download a matching libc from libcdb.

-    log.info("Downloading data from GitHub")
+    Arguments:
+        hex_encoded_id(str):
+            Hex-encoded Build ID (e.g. 'ABCDEF...') of the library

-    url_base = "https://gitlab.com/libcdb/libcdb/raw/master/hashes/build_id/"
-    url      = urlparse.urljoin(url_base, hex_encoded_id)
+    Returns:
+        Path to the downloaded library on disk, or :const:`None`.

-    data   = ""
-    while not data.startswith('\x7fELF'):
-        data = wget(url)
+    Examples:
+        >>> filename = search_by_md5('7a71dafb87606f360043dcd638e411bd')
+        >>> hex(ELF(filename).symbols.read)
+        '0xda260'
+        >>> None == search_by_build_id('XX')
+        True
+    """
+    return search_by_hash(hex_encoded_id, 'md5')
+
+def search_by_sha1(hex_encoded_id):
+    """
+    Given a hex-encoded sha1, attempt to download a matching libc from libcdb.
+
+    Arguments:
+        hex_encoded_id(str):
+            Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
+
+    Returns:
+        Path to the downloaded library on disk, or :const:`None`.
+
+    Examples:
+        >>> filename = search_by_sha1('34471e355a5e71400b9d65e78d2cd6ce7fc49de5')
+        >>> hex(ELF(filename).symbols.read)
+        '0xda260'
+        >>> None == search_by_sha1('XX')
+        True
+    """
+    return search_by_hash(hex_encoded_id, 'sha1')
+
+
+def search_by_sha256(hex_encoded_id):
+    """
+    Given a hex-encoded sha256, attempt to download a matching libc from libcdb.
+
+    Arguments:
+        hex_encoded_id(str):
+            Hex-encoded Build ID (e.g. 'ABCDEF...') of the library
+
+    Returns:
+        Path to the downloaded library on disk, or :const:`None`.
+
+    Examples:
+        >>> filename = search_by_sha256('5e877a8272da934812d2d1f9ee94f73c77c790cbc5d8251f5322389fc9667f21')
+        >>> hex(ELF(filename).symbols.read)
+        '0xda260'
+        >>> None == search_by_sha256('XX')
+        True
+    """
+    return search_by_hash(hex_encoded_id, 'sha256')

-        if not data:
-            return None

-        if data.startswith('..'):
-            url = os.path.dirname(url) + '/'
-            url = urlparse.urljoin(url, data)

-    write(cache, data)
-    return cache

 def get_build_id_offsets():
    """
@@ -114,4 +211,4 @@ def get_build_id_offsets():
    }.get(context.arch, [])


-__all__ = ['get_build_id_offsets', 'search_by_build_id']
+__all__ = ['get_build_id_offsets', 'search_by_build_id', 'search_by_sha1', 'search_by_sha256', 'search_by_md5']
@@ -363,7 +363,7 @@ class Logger(object):
        """
        m = message % args
        if m not in self._one_time_warnings:
-            if self.isEnabledFor(logging.INFO):
+            if self.isEnabledFor(logging.WARNING):
                self._one_time_warnings.add(m)
            self._log(logging.WARNING, message, args, kwargs, 'warning_once')

@@ -1,5 +1,81 @@
+"""Run foreign-architecture binaries
+
+Overview
+--------
+
+So you want to exploit ARM binaries on your Intel PC?
+
+Pwntools has a good level of integration with QEMU user-mode emulation,
+in order to run, debug, and pwn foreign architecture binaries.
+
+In general, everything magic happens "behind the scenes", and pwntools
+attempts to make your life easier.
+
+When using :class:`.process.process`, pwntools will attempt to blindly
+execute the binary, in case your system is configured to use ``binfmt-misc``.
+
+If this fails, pwntools will attempt to manually launch the binary under
+qemu user-mode emulation.  Preference is given to statically-linked variants,
+i.e. ``qemu-arm-static`` will be selected before ``qemu-arm``.
+
+Debugging
+~~~~~~~~~
+
+When debugging binaries with :func:`.gdb.debug`, pwntools automatically adds
+the appropriate command-line flags to QEMU to start its GDB stub, and
+automatically informs GDB of the correct architecture and sysroot.
+
+Sysroot
+~~~~~~~
+You can override the default sysroot by setting the ``QEMU_LD_PREFIX``
+environment variable.  This affects where ``qemu`` will look for files when
+``open()`` is called, e.g. when the linker is attempting to resolve ``libc.so``.
+
+Required Setup
+--------------
+
+For Ubuntu 16.04 and newer, the setup is relatively straightforward for most
+architectures.
+
+First, install the QEMU emulator itself.  If your binary is statically-linked,
+thsi is sufficient.
+
+    $ sudo apt-get install qemu-user
+
+If your binary is dynamically linked, you need to install libraries like libc.
+Generally, this package is named ``libc6-$ARCH-cross``, e.g. ``libc-mips-cross``.
+ARM comes in both soft-float and hard-float variants, e.g. ``armhf``.
+
+    $ sudo apt-get install libc6-arm64-cross
+
+If your binary relies on additional libraries, you can generally find them
+easily with ``apt-cache search``.  For example, if it's a C++ binary it
+may require ``libstdc++``.
+
+    $ apt-cache search 'libstdc++' | grep arm64
+
+Any other libraries that you require you'll have to find some other way.
+
+Telling QEMU Where Libraries Are
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The libraries are now installed on your system at e.g. ``/usr/aarch64-linux-gnu``.
+
+QEMU does not know where they are, and expects them to be at e.g. ``/etc/qemu-binfmt/aarch64``.
+If you try to run your library now, you'll probably see an error about ``libc.so.6`` missing.
+
+Create the ``/etc/qemu-binfmt`` directory if it does not exist, and create a symlink to
+the appropriate path.
+
+    $ sudo mkdir /etc/qemu-binfmt
+    $ sudo ln -s /usr/aarch64-linux-gnu /etc/qemu-binfmt/aarch64
+
+Now QEMU should be able to run the libraries.
+"""
 from __future__ import absolute_import

+import os
+
 from pwnlib.context import LocalContext
 from pwnlib.context import context
 from pwnlib.log import getLogger
@@ -8,14 +84,14 @@ from pwnlib.util import misc
 log = getLogger(__name__)

@LocalContext
-def get_qemu_arch():
+def archname():
    """
    Returns the name which QEMU uses for the currently selected
    architecture.

-    >>> get_qemu_arch()
+    >>> pwnlib.qemu.archname()
    'i386'
-    >>> get_qemu_arch(arch='powerpc')
+    >>> pwnlib.qemu.archname(arch='powerpc')
    'ppc'
    """
    return {
@@ -31,17 +107,17 @@ def get_qemu_arch():
    }.get((context.arch, context.endian), context.arch)

@LocalContext
-def get_qemu_user():
+def user_path():
    """
    Returns the path to the QEMU-user binary for the currently
    selected architecture.

-    >>> get_qemu_user()
+    >>> pwnlib.qemu.user_path()
    'qemu-i386-static'
-    >>> get_qemu_user(arch='thumb')
+    >>> pwnlib.qemu.user_path(arch='thumb')
    'qemu-arm-static'
    """
-    arch   = get_qemu_arch()
+    arch   = archname()
    normal = 'qemu-' + arch
    static = normal + '-static'

@@ -52,3 +128,32 @@ def get_qemu_user():
        return normal

    log.warn_once("Neither %r nor %r are available" % (normal, static))
+
+@LocalContext
+def ld_prefix(path=None, env=None):
+    """Returns the linker prefix for the selected qemu-user binary
+
+    >>> pwnlib.qemu.ld_prefix(arch='arm')
+    '/etc/qemu-binfmt/arm'
+    """
+    if path is None:
+        path = user_path()
+
+    # Did we explicitly specify the path in an environment variable?
+    if env and 'QEMU_LD_PREFIX' in env:
+        return env['QEMU_LD_PREFIX']
+
+    if 'QEMU_LD_PREFIX' in os.environ:
+        return os.environ['QEMU_LD_PREFIX']
+
+    # Cyclic imports!
+    from pwnlib.tubes.process import process
+
+    with context.quiet:
+        with process([path, '--help'], env=env) as io:
+            line = io.recvline_regex('QEMU_LD_PREFIX *=')
+
+    name, libpath = line.split('=', 1)
+
+    return libpath.strip()
+
@@ -233,4 +233,12 @@ class Call(object):
                args.extend(map(repr, arg.values))
            else:
                args.append(arg)
-        return '%s(%s)' % (self.name or fmt % self.target, ', '.join(map(str, args)))
+
+        name = self.name or (fmt % self.target)
+        arg_str = []
+        for arg in args:
+            if isinstance(arg, (int,long)) and arg > 0x100:
+                arg_str.append(hex(arg))
+            else:
+                arg_str.append(str(arg))
+        return '%s(%s)' % (name, ', '.join(arg_str))
@@ -40,7 +40,7 @@ standard Linux ABIs.
    0x0004:       0x64636261
    0x0008:              0x2
    0x000c:       0xdeadbeef read(4, 5, 6)
-    0x0010:           'eaaa' <pad>
+    0x0010:           'eaaa' <return address>
    0x0014:              0x4 arg0
    0x0018:              0x5 arg1
    0x001c:              0x6 arg2
@@ -55,19 +55,18 @@ The stack is automatically adjusted for the next frame
    0x0004:       0x64636261
    0x0008:              0x2
    0x000c:       0xdeadbeef read(4, 5, 6)
-    0x0010:       0x10000000 <adjust: add esp, 0x10; ret>
+    0x0010:       0x10000000 <adjust @0x24> add esp, 0x10; ret
    0x0014:              0x4 arg0
    0x0018:              0x5 arg1
    0x001c:              0x6 arg2
    0x0020:           'iaaa' <pad>
    0x0024:       0xdecafbad write(7, 8, 9)
-    0x0028:       0x10000000 <adjust: add esp, 0x10; ret>
+    0x0028:       0x10000000 <adjust @0x3c> add esp, 0x10; ret
    0x002c:              0x7 arg0
    0x0030:              0x8 arg1
    0x0034:              0x9 arg2
    0x0038:           'oaaa' <pad>
    0x003c:       0xfeedface exit()
-    0x0040:           'qaaa' <pad>

 ROP Example
 -------------------
@@ -106,20 +105,19 @@ Finally, let's build our ROP stack
    >>> rop.write(c.STDOUT_FILENO, binary.symbols['flag'], 8)
    >>> rop.exit()
    >>> print rop.dump()
-    0x0000:       0x10000012 write(STDOUT_FILENO, 268435494, 8)
-    0x0004:       0x1000000e <adjust: add esp, 0x10; ret>
+    0x0000:       0x10000012 write(STDOUT_FILENO, 0x10000026, 8)
+    0x0004:       0x1000000e <adjust @0x18> add esp, 0x10; ret
    0x0008:              0x1 arg0
    0x000c:       0x10000026 flag
    0x0010:              0x8 arg2
    0x0014:           'faaa' <pad>
    0x0018:       0x1000002f exit()
-    0x001c:           'haaa' <pad>

 The raw data from the ROP stack is available via `str`.

    >>> raw_rop = str(rop)
    >>> print enhex(raw_rop)
-    120000100e000010010000002600001008000000666161612f00001068616161
+    120000100e000010010000002600001008000000666161612f000010

 Let's try it out!

@@ -128,6 +126,63 @@ Let's try it out!
    >>> print p.recvall(timeout=5)
    The flag

+ROP Example (amd64)
+-------------------
+
+For amd64 binaries, the registers are loaded off the stack.  Pwntools can do
+basic reasoning about simple "pop; pop; add; ret"-style gadgets, and satisfy
+requirements so that everything "just works".
+
+    >>> context.clear(arch='amd64')
+    >>> assembly = 'pop rdx; pop rdi; pop rsi; add rsp, 0x20; ret; target: ret'
+    >>> binary = ELF.from_assembly(assembly)
+    >>> rop = ROP(binary)
+    >>> rop.target(1,2,3)
+    >>> print rop.dump()
+    0x0000:       0x10000000 pop rdx; pop rdi; pop rsi; add rsp, 0x20; ret
+    0x0008:              0x3 [arg2] rdx = 3
+    0x0010:              0x1 [arg0] rdi = 1
+    0x0018:              0x2 [arg1] rsi = 2
+    0x0020:       'iaaajaaa' <pad 0x20>
+    0x0028:       'kaaalaaa' <pad 0x18>
+    0x0030:       'maaanaaa' <pad 0x10>
+    0x0038:       'oaaapaaa' <pad 0x8>
+    0x0040:       0x10000008 target
+    >>> rop.target(1)
+    >>> print rop.dump()
+    0x0000:       0x10000000 pop rdx; pop rdi; pop rsi; add rsp, 0x20; ret
+    0x0008:              0x3 [arg2] rdx = 3
+    0x0010:              0x1 [arg0] rdi = 1
+    0x0018:              0x2 [arg1] rsi = 2
+    0x0020:       'iaaajaaa' <pad 0x20>
+    0x0028:       'kaaalaaa' <pad 0x18>
+    0x0030:       'maaanaaa' <pad 0x10>
+    0x0038:       'oaaapaaa' <pad 0x8>
+    0x0040:       0x10000008 target
+    0x0048:       0x10000001 pop rdi; pop rsi; add rsp, 0x20; ret
+    0x0050:              0x1 [arg0] rdi = 1
+    0x0058:       'waaaxaaa' <pad rsi>
+    0x0060:       'yaaazaab' <pad 0x20>
+    0x0068:       'baabcaab' <pad 0x18>
+    0x0070:       'daabeaab' <pad 0x10>
+    0x0078:       'faabgaab' <pad 0x8>
+    0x0080:       0x10000008 target
+
+Pwntools will also filter out some bad instructions while setting the registers
+( e.g. syscall, int 0x80... )
+
+    >>> assembly = 'syscall; pop rdx; pop rsi; ret ; pop rdi ; int 0x80; pop rsi; pop rdx; ret ; pop rdi ; ret'
+    >>> binary = ELF.from_assembly(assembly)
+    >>> rop = ROP(binary)
+    >>> rop.call(0xdeadbeef, [1, 2, 3])
+    >>> print rop.dump()
+    0x0000:       0x1000000b pop rdi; ret
+    0x0008:              0x1 [arg0] rdi = 1
+    0x0010:       0x10000008 pop rsi; pop rdx; ret
+    0x0018:              0x2 [arg1] rsi = 2
+    0x0020:              0x3 [arg2] rdx = 3
+    0x0028:       0xdeadbeef
+
 ROP + Sigreturn
 -----------------------

@@ -160,7 +215,7 @@ That's all there is to it.

    >>> print rop.dump()
    0x0000:       0x1000000e pop eax; ret
-    0x0004:             0x77
+    0x0004:             0x77 [arg0] eax = SYS_sigreturn
    0x0008:       0x1000000b int 0x80
    0x000c:              0x0 gs
    0x0010:              0x0 fs
@@ -197,8 +252,10 @@ from __future__ import absolute_import
 import collections
 import copy
 import hashlib
+import itertools
 import os
 import re
+import string
 import sys
 import tempfile

@@ -215,9 +272,9 @@ from pwnlib.rop.call import CurrentStackPointer
 from pwnlib.rop.call import NextGadgetAddress
 from pwnlib.rop.call import StackAdjustment
 from pwnlib.rop.gadgets import Gadget
-from pwnlib.util import cyclic
 from pwnlib.util import lists
 from pwnlib.util import packing
+from pwnlib.util.cyclic import cyclic
 from pwnlib.util.packing import *

 log = getLogger(__name__)
@@ -228,6 +285,8 @@ class Padding(object):
    """
    Placeholder for exactly one pointer-width of padding.
    """
+    def __init__(self, name='<pad>'):
+        self.name = name

 class DescriptiveStack(list):
    """
@@ -306,16 +365,16 @@ class ROP(object):
    >>> r.execve(4, 5, 6)
    >>> print r.dump()
    0x0000:       0x10001234 funcname(1, 2)
-    0x0004:       0x10000003 <adjust: add esp, 0x10; ret>
+    0x0004:       0x10000003 <adjust @0x18> add esp, 0x10; ret
    0x0008:              0x1 arg0
    0x000c:              0x2 arg1
    0x0010:           'eaaa' <pad>
    0x0014:           'faaa' <pad>
    0x0018:       0x10001234 funcname(3)
-    0x001c:       0x10000007 <adjust: pop eax; ret>
+    0x001c:       0x10000007 <adjust @0x24> pop eax; ret
    0x0020:              0x3 arg0
    0x0024:       0x10000007 pop eax; ret
-    0x0028:             0x77
+    0x0028:             0x77 [arg0] eax = SYS_sigreturn
    0x002c:       0x10000000 int 0x80
    0x0030:              0x0 gs
    0x0034:              0x0 fs
@@ -344,16 +403,16 @@ class ROP(object):
    >>> r.execve(4, 5, 6)
    >>> print r.dump()
    0x8048000:       0x10001234 funcname(1, 2)
-    0x8048004:       0x10000003 <adjust: add esp, 0x10; ret>
+    0x8048004:       0x10000003 <adjust @0x8048018> add esp, 0x10; ret
    0x8048008:              0x1 arg0
    0x804800c:              0x2 arg1
    0x8048010:           'eaaa' <pad>
    0x8048014:           'faaa' <pad>
    0x8048018:       0x10001234 funcname(3)
-    0x804801c:       0x10000007 <adjust: pop eax; ret>
+    0x804801c:       0x10000007 <adjust @0x8048024> pop eax; ret
    0x8048020:              0x3 arg0
    0x8048024:       0x10000007 pop eax; ret
-    0x8048028:             0x77
+    0x8048028:             0x77 [arg0] eax = SYS_sigreturn
    0x804802c:       0x10000000 int 0x80
    0x8048030:              0x0 gs
    0x8048034:              0x0 fs
@@ -375,25 +434,24 @@ class ROP(object):
    0x8048074:              0x0 esp_at_signal
    0x8048078:             0x2b ss
    0x804807c:              0x0 fpstate
+
+
+    >>> elf = ELF.from_assembly('ret')
+    >>> r = ROP(elf)
+    >>> r.ret.address == 0x10000000
+    True
+    >>> r = ROP(elf, badchars='\x00')
+    >>> r.gadgets == {}
+    True
+    >>> r.ret is None
+    True
    """
-    #: List of individual ROP gadgets, ROP calls, SROP frames, etc.
-    #: This is intended to be the highest-level abstraction that we can muster.
-    _chain = []
-
-    #: List of ELF files which are available for mining gadgets
-    elfs = []
-
-    #: Stack address where the first byte of the ROP chain lies, if known.
-    base = 0
-
-    #: Whether or not the ROP chain directly sets the stack pointer to a value
-    #: which is not contiguous
-    migrated = False
-
-    def __init__(self, elfs, base = None, **kwargs):
+    def __init__(self, elfs, base = None, badchars = '', **kwargs):
        """
        Arguments:
            elfs(list): List of :class:`.ELF` objects for mining
+            base(int): Stack address where the first byte of the ROP chain lies, if known.
+            badchars(str): Characters which should not appear in ROP gadget addresses.
        """
        import ropgadget

@@ -402,10 +460,24 @@ class ROP(object):
            elfs = [elfs]
        elif isinstance(elfs, (str, unicode)):
            elfs = [ELF(elfs)]
-        self.elfs = elfs
+
+        #: List of individual ROP gadgets, ROP calls, SROP frames, etc.
+        #: This is intended to be the highest-level abstraction that we can muster.
        self._chain = []
+
+        #: List of ELF files which are available for mining gadgets
+        self.elfs = elfs
+
+        #: Stack address where the first byte of the ROP chain lies, if known.
        self.base = base
+
+        #: Whether or not the ROP chain directly sets the stack pointer to a value
+        #: which is not contiguous
        self.migrated = False
+
+        #: Characters which should not appear in ROP gadget addresses.
+        self._badchars = set(badchars)
+
        self.__load()

    @staticmethod
@@ -415,24 +487,102 @@ class ROP(object):

    def setRegisters(self, registers):
        """
-        Returns an OrderedDict of addresses/values which will set the specified
-        register context.
+        Returns an list of addresses/values which will set the specified register context.

        Arguments:
            registers(dict): Dictionary of ``{register name: value}``

        Returns:
-            An OrderedDict of ``{register: sequence of gadgets, values, etc.}``.
+            A list of tuples, ordering the stack.
+
+            Each tuple is in the form of ``(value, name)`` where ``value`` is either a
+            gadget address or literal value to go on the stack, and ``name`` is either
+            a string name or other item which can be "unresolved".
+
+        Note:
+            This is basically an implementation of the Set Cover Problem, which is
+            NP-hard.  This means that we will take polynomial time N**2, where N is
+            the number of gadgets.  We can reduce runtime by discarding useless and
+            inferior gadgets ahead of time.
        """
-        reg_order = collections.OrderedDict()
+        if not registers:
+            return []

-        for reg, value in registers.items():
-            gadget = self.find_gadget(['pop ' + reg, 'ret'])
-            if not gadget:
-                log.error("can't set %r" % reg)
-            reg_order[reg] = [gadget, value]
+        regset = set(registers)

-        return reg_order
+        bad_instructions = set(('syscall', 'sysenter', 'int 0x80'))
+        
+        # Collect all gadgets which use these registers
+        # Also collect the "best" gadget for each combination of registers
+        gadgets = []
+        best_gadgets = {}
+
+        for gadget in self.gadgets.values():
+            # Do not use gadgets which doesn't end with 'ret'
+            if gadget.insns[-1] != 'ret':
+                continue
+            # Do not use gadgets which contain 'syscall' or 'int'
+            if set(gadget.insns) & bad_instructions:
+                continue
+
+            touched = tuple(regset & set(gadget.regs))
+
+            if not touched:
+                continue
+
+            old = best_gadgets.get(touched, gadget)
+
+            # if we have a new gadget for the touched registers, choose it
+            # if the new gadget requires less stack space, choose it
+            # if both gadgets require same stack space, choose the one with less instructions
+            if (old is gadget) \
+              or (old.move > gadget.move) \
+              or (old.move == gadget.move and len(old.insns) > len(gadget.insns)):
+                best_gadgets[touched] = gadget
+
+        winner = None
+        budget = 999999999
+
+        for num_gadgets in range(len(registers)):
+            for combo in itertools.combinations(best_gadgets.values(), 1+num_gadgets):
+                # Is this better than what we can already do?
+                cost = sum((g.move for g in combo))
+                if cost > budget:
+                    continue
+
+                # Does it hit all of the registers we want?
+                coverage = set(sum((g.regs for g in combo), [])) & regset
+
+                if coverage != regset:
+                    continue
+
+                # It is better than what we had, and hits all of the registers.
+                winner = combo
+                budget = cost
+
+        if not winner:
+            log.error("Could not satisfy setRegisters(%r)", registers)
+
+        # We have our set of "winner" gadgets, let's build a stack!
+        stack = []
+
+        for gadget in winner:
+            moved = 8 # Account for the gadget itself
+            goodregs = set(gadget.regs) & regset
+            name = ",".join(goodregs)
+            stack.append((gadget.address, gadget))
+            for r in gadget.regs:
+                moved += 8
+                if r in registers:
+                    stack.append((registers[r], r))
+                else:
+                    stack.append((Padding('<pad %s>' % r), r))
+
+            for slot in range(moved, gadget.move, context.bytes):
+                left = gadget.move - slot
+                stack.append((Padding('<pad %#x>' % left), 'stack padding'))
+
+        return stack

    def resolve(self, resolvable):
        """Resolves a symbol to an address
@@ -486,8 +636,12 @@ class ROP(object):
        ''

        """
+
+        # Ensure we don't generate a cyclic pattern which contains badchars
+        alphabet = ''.join(c for c in string.ascii_lowercase if c not in self._badchars)
+
        if count:
-            return cyclic.cyclic(offset + count)[-count:]
+            return cyclic(offset + count, alphabet=alphabet)[-count:]
        return ''

    def describe(self, object):
@@ -575,13 +729,17 @@ class ROP(object):
                stack.describe(self.describe(slot))

                registers    = dict(zip(slot.abi.register_arguments, slot.args))
-                setRegisters = self.setRegisters(registers)

-                for register, gadgets in setRegisters.items():
-                    value       = registers[register]
-                    description = self.describe(value) or 'arg%i' % slot.args.index(value)
-                    stack.describe('set %s = %s' % (register, description))
-                    stack.extend(gadgets)
+                for value, name in self.setRegisters(registers):
+                    if name in registers:
+                        index = slot.abi.register_arguments.index(name)
+                        description = self.describe(value) or value
+                        stack.describe('[arg%s] %s = %s' % (index, name, description))
+                    elif isinstance(name, Gadget):
+                        stack.describe('; '.join(name.insns))
+                    elif isinstance(name, str):
+                        stack.describe(name)
+                    stack.append(value)

                if address != stack.next:
                    stack.describe(slot.name)
@@ -599,24 +757,33 @@ class ROP(object):
                # properly, but likely also need to adjust the stack past the
                # arguments.
                if slot.abi.returns:
-                    if remaining:
-                        fix_size  = (1 + len(stackArguments))
-                        fix_bytes = fix_size * context.bytes
-                        adjust   = self.search(move = fix_bytes)

-                        if not adjust:
-                            log.error("Could not find gadget to adjust stack by %#x bytes" % fix_bytes)
+                    # Save off the address of the next gadget
+                    if remaining or stackArguments:
+                        nextGadgetAddr = stack.next

-                        nextGadgetAddr = stack.next + adjust.move
+                    # If there were arguments on the stack, we need to stick something
+                    # in the slot where the return address goes.
+                    if len(stackArguments) > 0:
+                        if remaining:
+                            fix_size  = (1 + len(stackArguments))
+                            fix_bytes = fix_size * context.bytes
+                            adjust   = self.search(move = fix_bytes)

-                        stack.describe('<adjust: %s>' % self.describe(adjust))
-                        stack.append(adjust.address)
+                            if not adjust:
+                                log.error("Could not find gadget to adjust stack by %#x bytes" % fix_bytes)

-                        for pad in range(fix_bytes, adjust.move, context.bytes):
-                            stackArguments.append(Padding())
-                    else:
-                        stack.describe('<pad>')
-                        stack.append(Padding())
+                            nextGadgetAddr += adjust.move
+
+                            stack.describe('<adjust @%#x> %s' % (nextGadgetAddr, self.describe(adjust)))
+                            stack.append(adjust.address)
+
+                            for pad in range(fix_bytes, adjust.move, context.bytes):
+                                stackArguments.append(Padding())
+
+                        # We could not find a proper "adjust" gadget, but also didn't need one.
+                        else:
+                            stack.append(Padding("<return address>"))


                for i, argument in enumerate(stackArguments):
@@ -657,7 +824,7 @@ class ROP(object):

            elif isinstance(slot, Padding):
                stack[i] = self.generatePadding(i * context.bytes, context.bytes)
-                stack.describe('<pad>', slot_address)
+                stack.describe(slot.name, slot_address)

            elif isinstance(slot, Gadget):
                stack[i] = slot.address
@@ -835,13 +1002,20 @@ class ROP(object):
        """Returns: Raw bytes of the ROP chain"""
        return self.chain()

-    def __get_cachefile_name(self, elf):
-        basename = os.path.basename(elf.file.name)
-        sha256 = hashlib.sha256(elf.get_data()).hexdigest()
+    def __get_cachefile_name(self, files):
+        """Given an ELF or list of ELF objects, return a cache file for the set of files"""
        cachedir = os.path.join(tempfile.gettempdir(), 'pwntools-rop-cache')
        if not os.path.exists(cachedir):
            os.mkdir(cachedir)
-        return os.path.join(cachedir, sha256)
+
+        if isinstance(files, ELF):
+            files = [files]
+
+        sha256 = hashlib.sha256()
+        for elf_data in sorted(elf.get_data() for elf in self.elfs):
+            sha256.update(elf_data)
+
+        return os.path.join(cachedir, sha256.hexdigest())

    def __cache_load(self, elf):
        filename = self.__get_cachefile_name(elf)
@@ -871,7 +1045,7 @@ class ROP(object):
        #

        pop   = re.compile(r'^pop (.{3})')
-        add   = re.compile(r'^add .sp, (\S+)$')
+        add   = re.compile(r'^add [er]sp, (\S+)$')
        ret   = re.compile(r'^ret$')
        leave = re.compile(r'^leave$')
        int80 = re.compile(r'int +0x80')
@@ -950,6 +1124,11 @@ class ROP(object):
        }[context.bytes]

        for addr, insns in gadgets.items():
+
+            # Filter out gadgets by address against badchars
+            if set(pack(addr)) & self._badchars:
+                continue
+
            sp_move = 0
            regs = []
            for insn in insns:
@@ -999,7 +1178,10 @@ class ROP(object):
        regs = set(regs or ())

        for addr, gadget in self.gadgets.items():
-            if gadget.move < move:          continue
+            addr_bytes = set(pack(gadget.address))
+            if addr_bytes & self._badchars:     continue
+            if gadget.insns[-1] != 'ret':        continue
+            if gadget.move < move:               continue
            if not (regs <= set(gadget.regs)):   continue
            yield gadget

@@ -165,7 +165,7 @@ tether = sys.modules[__name__]
 shellcraft = module(__name__, '')

 class LazyImporter:
-    def find_module(self, fullname, path):
+    def find_module(self, fullname, path=None):
        if not fullname.startswith('pwnlib.shellcraft.'):
            return None

@@ -40,6 +40,8 @@ mips = {
    '$31': 31, '$ra': 31,
 }

+mips_list = list(mips)
+
 arm = map('r{}'.format, range(13))
 arm += ["sp", "lr", "pc", "cpsr"]

@@ -211,7 +213,7 @@ def current():
        'arm': arm,
        'thumb': arm,
        'aarch64': aarch64,
-        'mips': mips,
+        'mips': mips_list,
        'powerpc': powerpc
    }[context.arch]

@@ -1,7 +1,7 @@
 <%
  from pwnlib import shellcraft
 %>
-<%page args="filename, mode='O_RDONLY'"/>
+<%page args="filename, flags='O_RDONLY', mode=0"/>
 <%docstring>
 Opens a file
 </%docstring>
@@ -9,4 +9,4 @@ Opens a file
  AT_FDCWD=-100
 %>
    ${shellcraft.pushstr(filename)}
-    ${shellcraft.syscall('SYS_openat', AT_FDCWD, 'sp', mode, 0)}
+    ${shellcraft.syscall('SYS_openat', AT_FDCWD, 'sp', flags, mode)}
@@ -8,5 +8,5 @@ Opens a file and writes its contents to the specified file descriptor.
 </%docstring>

    ${pushstr(filename)}
-    ${syscall('SYS_open', 'rsp', 0, 'O_RDONLY')}
+    ${syscall('SYS_open', 'rsp', 'O_RDONLY', 0)}
    ${syscall('SYS_sendfile', fd, 'rax', 0, 0x7fffffff)}
@@ -10,7 +10,7 @@ Args: [sock (imm/reg) = rbp]
  after     = common.label("after")
 %>

-
+    /* dup() file descriptor ${sock} into stdin/stdout/stderr */
 ${dup}:
    ${amd64.mov('rbp', sock)}

@@ -15,5 +15,5 @@ Example:

 </%docstring>
    ${arm.pushstr(filename)}
-    ${arm.linux.open('sp', 0, int(constants.O_RDONLY))}
+    ${arm.linux.open('sp', int(constants.O_RDONLY), 0)}
    ${arm.linux.sendfile(fd, 'r0', 0, 0x7fffffff)}
@@ -1,8 +1,2 @@
 <%docstring>A trap instruction.</%docstring>
-    nop
-    nop
-    nop
-    mov  r0, 12
-    mov  r1, sp
-    BKPT 0xab
-    B .
+    BKPT
@@ -4,7 +4,7 @@ import pwnlib.abi
 import pwnlib.constants
 import pwnlib.shellcraft
 %>
-<%docstring>break(vararg_0, vararg_1, vararg_2, vararg_3, vararg_4) -> str
+<%docstring>break_(vararg_0, vararg_1, vararg_2, vararg_3, vararg_4) -> str

 Invokes the syscall break.

@@ -83,7 +83,7 @@ Returns:
    else:
        raise Exception("Could not locate any syscalls: %r" % syscalls)
 %>
-    /* break(${', '.join(syscall_repr)}) */
+    /* break_(${', '.join(syscall_repr)}) */
 %for name, arg in string_arguments.items():
    ${pwnlib.shellcraft.pushstr(arg, append_null=('\x00' not in arg))}
    ${pwnlib.shellcraft.mov(regs[argument_names.index(name)], abi.stack)}
@@ -95,4 +95,4 @@ Returns:
    ${pwnlib.shellcraft.push(arg)}
 %endfor
    ${pwnlib.shellcraft.setregs(register_arguments)}
-    ${pwnlib.shellcraft.syscall(syscall)}
+    ${pwnlib.shellcraft.syscall(syscall)}
@@ -19,5 +19,5 @@ label = common.label("sendfile_loop")
 %>

    ${sc.pushstr(filename)}
-    ${sc.open('esp', 0, 'O_RDONLY')}
+    ${sc.open('esp', 'O_RDONLY', 0)}
    ${sc.sendfile(fd, 'eax', 0, 0x7fffffff)}
@@ -24,7 +24,7 @@ Examples:
    'Hello'

    >>> l = listen(fam='ipv6', timeout=5)
-    >>> assembly   = shellcraft.i386.linux.connect('ip6-localhost', l.lport, 'ipv6')
+    >>> assembly = shellcraft.i386.linux.connect('::1', l.lport, 'ipv6')
    >>> p = run_assembly(assembly)
    >>> assert l.wait_for_connection()

@@ -11,6 +11,7 @@ Args: [sock (imm/reg) = ebp]
  looplabel = common.label("loop")
 %>

+    /* dup() file descriptor ${sock} into stdin/stdout/stderr */
 ${dup}:
    ${mov('ebx', sock)}
    ${mov('ecx', 3)}
@@ -22,5 +22,5 @@ label = common.label("sendfile_loop")
 %>

    ${mips.pushstr(filename)}
-    ${mips.open('$sp', 0, int(constants.O_RDONLY))}
+    ${mips.open('$sp', int(constants.O_RDONLY), 0)}
    ${mips.sendfile(fd, '$v0', 0, 0x7fffffff)}
@@ -1,4 +1,13 @@
 <% from pwnlib.shellcraft import mips %>
-<%docstring>Execute /bin/sh</%docstring>
+<%docstring>Execute /bin/sh
+
+Example:
+
+    >>> p = run_assembly(shellcraft.mips.linux.sh())
+    >>> p.sendline('echo Hello')
+    >>> p.recv()
+    'Hello\n'
+
+</%docstring>

 ${mips.execve('//bin/sh', ['sh'], {})}
@@ -20,6 +20,6 @@ label = common.label("sendfile_loop")
 %>

    ${thumb.pushstr(filename)}
-    ${thumb.linux.open('sp', 0, constants.O_RDONLY)}
+    ${thumb.linux.open('sp', constants.O_RDONLY, 0)}
    ${thumb.mov('r5', 'r0')}
    ${thumb.linux.sendfile(fd, 'r5', 0, 0x7fffffff)}
@@ -9,6 +9,7 @@ Args: [sock (imm/reg) = r6]
  dup       = common.label("dup")
  looplabel = common.label("loop")
 %>
+    /* dup() file descriptor ${sock} into stdin/stdout/stderr */
 ${dup}:
        ${mov('r1', 2)}
        ${mov('r7', 'SYS_dup2')}
@@ -1,4 +1,4 @@
-<% from pwnlib.shellcraft.thumb.linux import mov %>
+<% from pwnlib.shellcraft.thumb import mov %>
 <% from socket import htons %>
 <%page args="port = None"/>
 <%docstring>
@@ -6,6 +6,10 @@

    Finds a connected socket. If port is specified it is checked
    against the peer port. Resulting socket is left in r6.
+
+    Example:
+        >>> enhex(asm(shellcraft.findpeer(1337)))
+        '6ff00006ee4606f101064ff001074fea072707f11f07f54630461fb401a96a4601df0130efdd01994fea11414ff039024fea022202f105029142e4d1'
 </%docstring>
 findpeer:
    /* File descriptor in r6 */
@@ -52,7 +56,7 @@ compare_port:
    ldr r1, [sp, #4]
    lsr r1, #16

-    /* Put the port (${port}) to search for into r1 */
+    /* Put the port (${port}) to search for into r2 */
    ${mov('r2', htons(int(port)))}

    /* Is it the one we have been searching for? */
@@ -1,4 +1,4 @@
-<% from pwnlib.shellcraft.thumb.linux import mov %>
+<% from pwnlib.shellcraft.thumb import mov %>
 <% from socket import htons %>
 <%page args="port, network='ipv4'"/>
 <%docstring>
@@ -6,6 +6,10 @@

    Listens on a TCP port, accept a client and leave his socket in r6.
    Port is the TCP port to listen on, network is either 'ipv4' or 'ipv6'.
+
+    Example:
+        >>> enhex(asm(shellcraft.listen(1337, 'ipv4')))
+        '4ff001074fea072707f119074ff002004ff0010182ea020201df0646004901e00200053906b469464ff0100207f1010701df30464ff0010107f1020701df304681ea010182ea020207f1010701df0646'
 </%docstring>
    /* First create listening socket */
    ${mov('r7', 'SYS_socket')}
@@ -1,3 +1,5 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
 from __future__ import absolute_import

 from pwnlib.term import keyconsts as kc
@@ -7,7 +9,7 @@ from pwnlib.term import text

 cursor = text.reverse

-buffer_left, buffer_right = u'', u''
+buffer_left, buffer_right = '', ''
 saved_buffer = None
 history = []
 history_idx = None
@@ -29,6 +31,14 @@ suggest_hook = None

 tabs = 0

+def force_to_bytes(data):
+    if isinstance(data, bytes):
+        return data
+    try:
+        return data.encode('utf-8')
+    except Exception:
+        return data.encode('latin-1')
+
 def set_completer(completer):
    global complete_hook, suggest_hook
    if completer is None:
@@ -80,7 +90,7 @@ def handle_keypress(trace):

 def clear():
    global buffer_left, buffer_right, history_idx, search_idx
-    buffer_left, buffer_right = u'', u''
+    buffer_left, buffer_right = '', ''
    history_idx = None
    search_idx = None
    redisplay()
@@ -141,7 +151,7 @@ def cancel_search(*_):
 def commit_search():
    global search_idx
    if search_idx is not None and search_results:
-        set_buffer(history[search_results[search_idx][0]], u'')
+        set_buffer(history[search_results[search_idx][0]], '')
        search_idx = None
        redisplay()

@@ -169,7 +179,7 @@ def update_search_results():
 def search_history(*_):
    global buffer_left, buffer_right, history_idx, search_idx
    if search_idx is None:
-        buffer_left, buffer_right = buffer_left + buffer_right, u''
+        buffer_left, buffer_right = buffer_left + buffer_right, ''
        history_idx = None
        search_idx = 0
        update_search_results()
@@ -187,7 +197,7 @@ def history_prev(*_):
        history_idx = -1
    if history_idx < len(history) - 1:
        history_idx += 1
-        set_buffer(history[history_idx], u'')
+        set_buffer(history[history_idx], '')

 def history_next(*_):
    global history_idx, saved_buffer
@@ -200,7 +210,7 @@ def history_next(*_):
        saved_buffer = None
    else:
        history_idx -= 1
-        set_buffer(history[history_idx], u'')
+        set_buffer(history[history_idx], '')

 def backward_char(*_):
    global buffer_left, buffer_right
@@ -319,11 +329,11 @@ def forward_word(*_):

 def go_beginning(*_):
    commit_search()
-    set_buffer(u'', buffer_left + buffer_right)
+    set_buffer('', buffer_left + buffer_right)

 def go_end(*_):
    commit_search()
-    set_buffer(buffer_left + buffer_right, u'')
+    set_buffer(buffer_left + buffer_right, '')

 keymap = km.Keymap({
    '<nomatch>'   : self_insert,
@@ -382,10 +392,10 @@ def readline(_size = None, prompt = '', float = True, priority = 10):
                if eof:
                    return ''
                else:
-                    buffer = (buffer_left + buffer_right).encode('utf-8')
+                    buffer = (buffer_left + buffer_right)
                    if buffer:
                        history.insert(0, buffer)
-                    return buffer + '\n'
+                    return force_to_bytes(buffer) + '\n'
            except KeyboardInterrupt:
                control_c()
    finally:
@@ -15,8 +15,9 @@ from pwnlib.tubes import listen
 from pwnlib.tubes import process
 from pwnlib.tubes import remote
 from pwnlib.tubes import serialtube
+from pwnlib.tubes import server
 from pwnlib.tubes import sock
 from pwnlib.tubes import ssh
 from pwnlib.tubes import tube

-__all__ = ['tube', 'sock', 'remote', 'listen', 'process', 'serialtube', 'ssh']
+__all__ = ['tube', 'sock', 'remote', 'listen', 'process', 'serialtube', 'server', 'ssh']
@@ -16,9 +16,9 @@ import subprocess
 import time
 import tty

+from pwnlib import qemu
 from pwnlib.context import context
 from pwnlib.log import getLogger
-from pwnlib.qemu import get_qemu_user
 from pwnlib.timeout import Timeout
 from pwnlib.tubes.tube import tube
 from pwnlib.util.hashes import sha256file
@@ -108,10 +108,6 @@ class process(tube):
        alarm(int):
            Set a SIGALRM alarm timeout on the process.

-    Attributes:
-
-        proc(subprocess)
-
    Examples:

        >>> p = process('python2')
@@ -238,7 +234,7 @@ class process(tube):
                raise TypeError('Must provide argv or set context.binary')


-        #: `subprocess.Popen` object
+        #: :class:`subprocess.Popen` object that backs this process
        self.proc = None

        if not shell:
@@ -336,11 +332,11 @@ class process(tube):

        if self.pty is not None:
            if stdin is slave:
-                self.proc.stdin = os.fdopen(os.dup(master), 'r+')
+                self.proc.stdin = os.fdopen(os.dup(master), 'r+', 0)
            if stdout is slave:
-                self.proc.stdout = os.fdopen(os.dup(master), 'r+')
+                self.proc.stdout = os.fdopen(os.dup(master), 'r+', 0)
            if stderr is slave:
-                self.proc.stderr = os.fdopen(os.dup(master), 'r+')
+                self.proc.stderr = os.fdopen(os.dup(master), 'r+', 0)

            os.close(master)
            os.close(slave)
@@ -436,21 +432,21 @@ class process(tube):

        # Determine what architecture the binary is, and find the
        # appropriate qemu binary to run it.
-        qemu = get_qemu_user(arch=binary.arch)
+        qemu_path = qemu.user_path(arch=binary.arch)

-        if not qemu:
+        if not qemu_path:
            raise exception

-        qemu = which(qemu)
-        if qemu:
-            self._qemu = qemu
+        qemu_path = which(qemu_path)
+        if qemu_path:
+            self._qemu = qemu_path

-            args = [qemu]
+            args = [qemu_path]
            if self.argv:
                args += ['-0', self.argv[0]]
            args += ['--']

-            return [args, qemu]
+            return [args, qemu_path]

        # If we get here, we couldn't run the binary directly, and
        # we don't have a qemu which can run it.
@@ -569,7 +565,7 @@ class process(tube):
        #

        # Create a duplicate so we can modify it safely
-        env = dict(os.environ if env is None else env)
+        env = (os.environ if env is None else env).copy()

        for k,v in env.items():
            if not isinstance(k, (str, unicode)):
@@ -879,7 +875,7 @@ class process(tube):
        from pwnlib.elf import ELF

        for lib, address in self.libs().items():
-            if 'libc.so' in lib:
+            if 'libc.so' in lib or 'libc-' in lib:
                e = ELF(lib)
                e.address = address
                return e
@@ -935,7 +931,7 @@ class process(tube):

        Example:

-            >>> e = ELF('/bin/sh')
+            >>> e = ELF('/bin/bash')
            >>> p = process(e.path)

            In order to make sure there's not a race condition against
@@ -957,3 +953,25 @@ class process(tube):
        with open('/proc/%i/mem' % self.pid, 'rb') as mem:
            mem.seek(address)
            return mem.read(count) or None
+
+    @property
+    def stdin(self):
+        """Shorthand for ``self.proc.stdin``
+
+        See: :obj:`.process.proc`
+        """
+        return self.proc.stdin
+    @property
+    def stdout(self):
+        """Shorthand for ``self.proc.stdout``
+
+        See: :obj:`.process.proc`
+        """
+        return self.proc.stdout
+    @property
+    def stderr(self):
+        """Shorthand for ``self.proc.stderr``
+
+        See: :obj:`.process.proc`
+        """
+        return self.proc.stderr
@@ -23,7 +23,7 @@ class remote(sock):
        typ: The string "tcp" or "udp" or an integer to pass to :func:`socket.getaddrinfo`.
        timeout: A positive number, None or the string "default".
        ssl(bool): Wrap the socket with SSL
-        sock(socket): Socket to inherit, rather than connecting
+        sock(socket.socket): Socket to inherit, rather than connecting

    Examples:

@@ -0,0 +1,162 @@
+from __future__ import absolute_import
+
+import errno
+import socket
+import threading
+
+from pwnlib.context import context
+from pwnlib.log import getLogger
+from pwnlib.tubes.sock import sock
+from pwnlib.tubes.remote import remote
+
+log = getLogger(__name__)
+
+class server(sock):
+    r"""Creates an TCP or UDP-server to listen for connections. It supports
+    both IPv4 and IPv6.
+
+    Arguments:
+        port(int): The port to connect to.
+            Defaults to a port auto-selected by the operating system.
+        bindaddr(str): The address to bind to.
+            Defaults to ``0.0.0.0`` / `::`.
+        fam: The string "any", "ipv4" or "ipv6" or an integer to pass to :func:`socket.getaddrinfo`.
+        typ: The string "tcp" or "udp" or an integer to pass to :func:`socket.getaddrinfo`.
+        callback: A function to be started on incoming connections. It should take a :class:`pwnlib.tubes.remote` as its only argument.
+
+    Examples:
+
+        >>> s = server(8888)
+        >>> client_conn = remote('localhost', s.lport)
+        >>> server_conn = s.next_connection()
+        >>> client_conn.sendline('Hello')
+        >>> server_conn.recvline()
+        'Hello\n'
+        >>> def cb(r):
+        ...     client_input = r.readline()
+        ...     r.send(client_input[::-1])
+        ...
+        >>> t = server(8889, callback=cb)
+        >>> client_conn = remote('localhost', t.lport)
+        >>> client_conn.sendline('callback')
+        >>> client_conn.recv()
+        '\nkcabllac'
+    """
+
+    #: Local port
+    lport = 0
+
+    #: Local host
+    lhost = None
+
+    #: Socket type (e.g. socket.SOCK_STREAM)
+    type = None
+
+    #: Socket family
+    family = None
+
+    #: Socket protocol
+    protocol = None
+
+    #: Canonical name of the listening interface
+    canonname = None
+
+    #: Sockaddr structure that is being listened on
+    sockaddr = None
+
+    _accepter = None
+
+    def __init__(self, port=0, bindaddr = "0.0.0.0", fam = "any", typ = "tcp",
+                 callback = None, blocking = False, *args, **kwargs):
+        super(server, self).__init__(*args, **kwargs)
+
+        port = int(port)
+        fam  = {socket.AF_INET: 'ipv4',
+                socket.AF_INET6: 'ipv6'}.get(fam, fam)
+
+        fam = self._get_family(fam)
+        typ = self._get_type(typ)
+
+        if fam == socket.AF_INET6 and bindaddr == '0.0.0.0':
+            bindaddr = '::'
+
+        h = self.waitfor('Trying to bind to %s on port %d' % (bindaddr, port))
+
+        for res in socket.getaddrinfo(bindaddr, port, fam, typ, 0, socket.AI_PASSIVE):
+            self.family, self.type, self.proto, self.canonname, self.sockaddr = res
+
+            if self.type not in [socket.SOCK_STREAM, socket.SOCK_DGRAM]:
+                continue
+
+            h.status("Trying %s" % self.sockaddr[0])
+            listen_sock = socket.socket(self.family, self.type, self.proto)
+            listen_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+            listen_sock.bind(self.sockaddr)
+            self.lhost, self.lport = listen_sock.getsockname()[:2]
+            if self.type == socket.SOCK_STREAM:
+                listen_sock.listen(1)
+            break
+        else:
+            h.failure()
+            self.error("Could not bind to %s on port %d" % (bindaddr, port))
+
+        h.success()
+
+        self.sock = listen_sock
+        self.connections_waiting = threading.Event()
+        self.connections = []
+        def accepter():
+            while True:
+                h = self.waitfor('Waiting for connections on %s:%s' % (self.lhost, self.lport))
+                while True:
+                    try:
+                        if self.type == socket.SOCK_STREAM:
+                            sock, rhost = listen_sock.accept()
+                        else:
+                            data, rhost = listen_sock.recvfrom(4096)
+                            listen_sock.connect(rhost)
+                            sock = listen_sock
+                            sock.unrecv(data)
+                        sock.settimeout(self.timeout)
+                        break
+                    except socket.error as e:
+                        if e.errno == errno.EINTR:
+                            continue
+                        h.failure()
+                        self.exception("Socket failure while waiting for connection")
+                        sock = None
+                        return
+
+                self.rhost, self.rport = rhost[:2]
+                r = remote(self.rhost, self.rport, sock = sock)
+                h.success('Got connection from %s on port %d' % (self.rhost, self.rport))
+                if callback:
+                    if not blocking:
+                        t = context.Thread(target = callback, args = (r,))
+                        t.daemon = True
+                        t.start()
+                    else:
+                        callback(r)
+                else:
+                    self.connections.append(r)
+                    if not self.connections_waiting.is_set():
+                        self.connections_waiting.set()
+
+        self._accepter = context.Thread(target = accepter)
+        self._accepter.daemon = True
+        self._accepter.start()
+
+    def next_connection(self):
+        if not self.connections_waiting.is_set():
+            self.connections_waiting.wait()
+        conn = self.connections.pop(0)
+        if not self.connections:
+            self.connections_waiting.clear()
+        return conn
+
+    def close(self):
+        # since `close` is scheduled to run on exit we must check that we got
+        # a connection or the program will hang in the `join` call above
+        if self._accepter and self._accepter.is_alive():
+            return
+        super(server, self).close()
@@ -146,13 +146,17 @@ class ssh_channel(sock):
        # However, we need to wait for the return value to propagate,
        # which may not happen by the time .close() is called by tube.recvall()
        tmp_sock = self.sock
+        tmp_close = self.close
+        self.close = lambda: None

        timeout = self.maximum if self.timeout is self.forever else self.timeout
        data = super(ssh_channel, self).recvall(timeout)

        # Restore self.sock to be able to call wait()
+        self.close = tmp_close
        self.sock = tmp_sock
        self.wait()
+        self.close()

        # Again set self.sock to None
        self.sock = None
@@ -341,7 +345,8 @@ class ssh_process(ssh_channel):
        libs = self.parent.libs(self.executable)

        for lib in libs:
-            if self.executable in lib:
+            # Cannot just check "executable in lib", see issue #1047
+            if lib.endswith(self.executable):
                return pwnlib.elf.elf.ELF(lib)


@@ -417,9 +422,18 @@ class ssh_connecter(sock):
                self.exception(e.message)
                raise

-            sockname = self.sock.get_transport().sock.getsockname()
-            self.lhost = sockname[0]
-            self.lport = sockname[1]
+            try:
+                # Iterate all layers of proxying to get to base-level Socket object
+                curr = self.sock.get_transport().sock
+                while getattr(curr, "get_transport", None):
+                    curr = curr.get_transport().sock
+
+                sockname = curr.getsockname()
+                self.lhost = sockname[0]
+                self.lport = sockname[1]
+            except Exception as e:
+                self.exception("Could not find base-level Socket object.")
+                raise e

            h.success()

@@ -529,7 +543,21 @@ class ssh(Timeout, Logger):
            ssh_agent: If :const:`True`, enable usage of keys via ssh-agent

        NOTE: The proxy_command and proxy_sock arguments is only available if a
-        fairly new version of paramiko is used."""
+        fairly new version of paramiko is used.
+
+        Example proxying:
+
+            >>> s1 = ssh(host='example.pwnme',
+            ...          user='travis',
+            ...          password='demopass')
+            >>> r1 = s1.remote('localhost', 22)
+            >>> s2 = ssh(host='example.pwnme',
+            ...          user='travis',
+            ...          password='demopass',
+            ...          proxy_sock=r1.sock)
+            >>> r2 = s2.remote('localhost', 22) # and so on...
+            >>> for x in r2, s2, r1, s1: x.close()
+        """
        super(ssh, self).__init__(*a, **kw)

        Logger.__init__(self)
@@ -873,6 +901,8 @@ os.chdir(%(cwd)r)
 if env is not None:
    os.environ.clear()
    os.environ.update(env)
+else:
+    env = os.environ

 def is_exe(path):
    return os.path.isfile(path) and os.access(path, os.X_OK)
@@ -960,7 +990,7 @@ except Exception:
 %(func_src)s
 apply(%(func_name)s, %(func_args)r)

-os.execve(exe, argv, os.environ)
+os.execve(exe, argv, env)
 """ % locals()

        script = script.strip()
@@ -1301,7 +1331,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
        return misc.parse_ldd_output(data)

    def _get_fingerprint(self, remote):
-        cmd = '(openssl sha256 || sha256 || sha256sum) 2>/dev/null < '
+        cmd = '(sha256 || sha256sum || openssl sha256) 2>/dev/null < '
        cmd = cmd + sh_string(remote)
        data, status = self.run_to_end(cmd)

@@ -1616,7 +1646,6 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))

        self.error('%r does not exist' % file_or_directory)

-
    def download(self, file_or_directory, local=None):
        """download(file_or_directory, local=None)

@@ -1630,7 +1659,10 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
        if not self.sftp:
            self.error("Cannot determine remote file type without SFTP")

-        if 0 == self.system('test -d ' + sh_string(file_or_directory)).wait():
+        with self.system('test -d ' + sh_string(file_or_directory)) as io:
+            is_dir = io.wait()
+
+        if 0 == is_dir:
            self.download_dir(file_or_directory, local)
        else:
            self.download_file(file_or_directory, local)
@@ -1991,7 +2023,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
        if value is not None:
            with open(path, 'w+') as f:
                f.write(value)
-        else:
+        elif os.path.exists(path):
            with open(path, 'r+') as f:
                return f.read()

@@ -622,7 +622,7 @@ class tube(Timeout, Logger):
        return self.recvline_pred(pred, keepends = keepends, timeout = timeout)

    def recvrepeat(self, timeout = default):
-        """recvrepeat()
+        """recvrepeat(timeout = default) -> str

        Receives data until a timeout or EOF is reached.

@@ -732,37 +732,37 @@ class tube(Timeout, Logger):
    def sendafter(self, delim, data, timeout = default):
        """sendafter(delim, data, timeout = default) -> str

-        A combination of ``recvuntil(delim, timeout)`` and ``send(data)``.
+        A combination of ``recvuntil(delim, timeout=timeout)`` and ``send(data)``.
        """

-        res = self.recvuntil(delim, timeout)
+        res = self.recvuntil(delim, timeout=timeout)
        self.send(data)
        return res

    def sendlineafter(self, delim, data, timeout = default):
        """sendlineafter(delim, data, timeout = default) -> str

-        A combination of ``recvuntil(delim, timeout)`` and ``sendline(data)``."""
+        A combination of ``recvuntil(delim, timeout=timeout)`` and ``sendline(data)``."""

-        res = self.recvuntil(delim, timeout)
+        res = self.recvuntil(delim, timeout=timeout)
        self.sendline(data)
        return res

    def sendthen(self, delim, data, timeout = default):
        """sendthen(delim, data, timeout = default) -> str

-        A combination of ``send(data)`` and ``recvuntil(delim, timeout)``."""
+        A combination of ``send(data)`` and ``recvuntil(delim, timeout=timeout)``."""

        self.send(data)
-        return self.recvuntil(delim, timeout)
+        return self.recvuntil(delim, timeout=timeout)

    def sendlinethen(self, delim, data, timeout = default):
        """sendlinethen(delim, data, timeout = default) -> str

-        A combination of ``sendline(data)`` and ``recvuntil(delim, timeout)``."""
+        A combination of ``sendline(data)`` and ``recvuntil(delim, timeout=timeout)``."""

        self.send(data + self.newline)
-        return self.recvuntil(delim, timeout)
+        return self.recvuntil(delim, timeout=timeout)

    def interactive(self, prompt = term.text.bold_red('$') + ' '):
        """interactive(prompt = pwnlib.term.text.bold_red('$') + ' ')
@@ -2,15 +2,15 @@ from __future__ import absolute_import

 import string

-from pwnlib.context import context
+from pwnlib.context import context, LocalContext
 from pwnlib.log import getLogger
 from pwnlib.util import packing

 log = getLogger(__name__)

 # Taken from https://en.wikipedia.org/wiki/De_Bruijn_sequence but changed to a generator
-def de_bruijn(alphabet = string.ascii_lowercase, n = None):
-    """de_bruijn(alphabet = string.ascii_lowercase, n = 4) -> generator
+def de_bruijn(alphabet = None, n = None):
+    """de_bruijn(alphabet = None, n = None) -> generator

    Generator for a sequence of unique substrings of length `n`. This is implemented using a
    De Bruijn Sequence over the given `alphabet`.
@@ -21,8 +21,10 @@ def de_bruijn(alphabet = string.ascii_lowercase, n = None):
        alphabet: List or string to generate the sequence over.
        n(int): The length of subsequences that should be unique.
    """
+    if alphabet is None:
+        alphabet = context.cyclic_alphabet
    if n is None:
-        n = 4
+        n = context.cyclic_size
    k = len(alphabet)
    a = [0] * k * n
    def db(t, p):
@@ -42,8 +44,8 @@ def de_bruijn(alphabet = string.ascii_lowercase, n = None):

    return db(1,1)

-def cyclic(length = None, alphabet = string.ascii_lowercase, n = None):
-    """cyclic(length = None, alphabet = string.ascii_lowercase, n = 4) -> list/str
+def cyclic(length = None, alphabet = None, n = None):
+    """cyclic(length = None, alphabet = None, n = None) -> list/str

    A simple wrapper over :func:`de_bruijn`. This function returns at most
    `length` elements.
@@ -56,17 +58,68 @@ def cyclic(length = None, alphabet = string.ascii_lowercase, n = None):
        alphabet: List or string to generate the sequence over.
        n(int): The length of subsequences that should be unique.

+    Notes:
+        The maximum length is `len(alphabet)**n`.
+
+        The default values for `alphabet` and `n` restrict the total space to ~446KB.
+
+        If you need to generate a longer cyclic pattern, provide a longer `alphabet`,
+        or if possible a larger `n`.
+
    Example:
-        >>> cyclic(alphabet = "ABC", n = 3)
-        'AAABAACABBABCACBACCBBBCBCCC'
+
+        Cyclic patterns are usually generated by providing a specific `length`.
+
        >>> cyclic(20)
        'aaaabaaacaaadaaaeaaa'
+
+        >>> cyclic(32)
+        'aaaabaaacaaadaaaeaaafaaagaaahaaa'
+
+        The `alphabet` and `n` arguments will control the actual output of the pattern
+
+        >>> cyclic(20, alphabet=string.ascii_uppercase)
+        'AAAABAAACAAADAAAEAAA'
+
+        >>> cyclic(20, n=8)
+        'aaaaaaaabaaaaaaacaaa'
+
+        >>> cyclic(20, n=2)
+        'aabacadaeafagahaiaja'
+
+        The size of `n` and `alphabet` limit the maximum length that can be generated.
+        Without providing `length`, the entire possible cyclic space is generated.
+
+        >>> cyclic(alphabet = "ABC", n = 3)
+        'AAABAACABBABCACBACCBBBCBCCC'
+
+        >>> cyclic(length=512, alphabet = "ABC", n = 3)
+        Traceback (most recent call last):
+        ...
+        PwnlibException: Can't create a pattern length=512 with len(alphabet)==3 and n==3
+
+        The `alphabet` can be set in `context`, which is useful for circumstances
+        when certain characters are not allowed.  See :obj:`.context.cyclic_alphabet`.
+
+        >>> context.cyclic_alphabet = "ABC"
+        >>> cyclic(10)
+        'AAAABAAACA'
+
+        The original values can always be restored with:
+
+        >>> context.clear()
+
+        The following just a test to make sure the length is correct.
+
        >>> alphabet, n = range(30), 3
        >>> len(alphabet)**n, len(cyclic(alphabet = alphabet, n = n))
        (27000, 27000)
    """
    if n is None:
-        n = 4
+        n = context.cyclic_size
+
+    if alphabet is None:
+        alphabet = context.cyclic_alphabet

    if len(alphabet) ** n < length:
        log.error("Can't create a pattern length=%i with len(alphabet)==%i and n==%i" \
@@ -84,8 +137,9 @@ def cyclic(length = None, alphabet = string.ascii_lowercase, n = None):
    else:
        return out

-def cyclic_find(subseq, alphabet = string.ascii_lowercase, n = None):
-    """cyclic_find(subseq, alphabet = string.ascii_lowercase, n = None) -> int
+@LocalContext
+def cyclic_find(subseq, alphabet = None, n = None):
+    """cyclic_find(subseq, alphabet = None, n = None) -> int

    Calculates the position of a substring into a De Bruijn sequence.

@@ -103,25 +157,71 @@ def cyclic_find(subseq, alphabet = string.ascii_lowercase, n = None):
                integer. If an integer is provided it will be packed as a
                little endian integer.
        alphabet: List or string to generate the sequence over.
+                  By default, uses :obj:`.context.cyclic_alphabet`.
        n(int): The length of subsequences that should be unique.
-
+                By default, uses :obj:`.context.cyclic_size`.

    Examples:

+        Let's generate an example cyclic pattern.
+
+        >>> cyclic(16)
+        'aaaabaaacaaadaaa'
+
+        Note that 'baaa' starts at offset 4.  The `cyclic_find` routine shows us this:
+
+        >>> cyclic_find('baaa')
+        4
+
+        The *default* length of a subsequence generated by `cyclic` is `4`.
+        If a longer value is submitted, it is automatically truncated to four bytes.
+
+        >>> cyclic_find('baaacaaa')
+        4
+
+        If you provided e.g. `n=8` to `cyclic` to generate larger subsequences,
+        you must explicitly provide that argument.
+
+        >>> cyclic_find('baaacaaa', n=8)
+        3515208
+
+        We can generate a large cyclic pattern, and grab a subset of it to
+        check a deeper offset.
+
        >>> cyclic_find(cyclic(1000)[514:518])
        514
+
+        Instead of passing in the byte representation of the pattern, you can
+        also pass in the integer value.  Note that this is sensitive to the
+        selected endianness via `context.endian`.
+
        >>> cyclic_find(0x61616162)
        4
-    """
-    if isinstance(subseq, (int, long)):
-        width = 'all' if n is None else n * 8
-        subseq = packing.pack(subseq, width, 'little', False)
+        >>> cyclic_find(0x61616162, endian='big')
+        1

-    if n is None and len(subseq) != 4:
-        log.warn_once("cyclic_find() expects 4-byte subsequences by default, you gave %r\n" % subseq \
-            + "Unless you specified cyclic(..., n=%i), you probably just want the first 4 bytes.\n" % len(subseq) \
-            + "Truncating the data at 4 bytes.  Specify cyclic_find(..., n=%i) to override this." % len(subseq))
-        subseq = subseq[:4]
+        You can use anything for the cyclic pattern, including non-printable
+        characters.
+
+        >>> cyclic_find(0x00000000, alphabet=unhex('DEADBEEF00'))
+        621
+    """
+
+    if n is None:
+        n = context.cyclic_size
+
+    if isinstance(subseq, (int, long)):
+        subseq = packing.pack(subseq, bytes=n)
+
+    if len(subseq) != n:
+        log.warn_once("cyclic_find() expects %i-byte subsequences by default, you gave %r\n"\
+            + "Unless you specified cyclic(..., n=%i), you probably just want the first 4 bytes.\n"\
+            + "Truncating the data at 4 bytes.  Specify cyclic_find(..., n=%i) to override this.",
+            n, subseq, len(subseq), len(subseq))
+        subseq = subseq[:n]
+
+    if alphabet is None:
+        alphabet = context.cyclic_alphabet

    if any(c not in alphabet for c in subseq):
        return -1
@@ -574,9 +574,9 @@ def update_cyclic_pregenerated(size):
        cyclic_pregen += de_bruijn_gen.next()

 def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
-                 highlight=None, cyclic=False):
+                 highlight=None, cyclic=False, groupsize=4):
    r"""hexdump_iter(s, width = 16, skip = True, hexii = False, begin = 0,
-                    style = None, highlight = None, cyclic = False) -> str generator
+                    style = None, highlight = None, cyclic = False, groupsize=4) -> str generator

    Return a hexdump-dump of a string as a generator of lines.  Unless you have
    massive amounts of data you probably want to use :meth:`hexdump`.
@@ -584,6 +584,7 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
    Arguments:
        fd(file): File object to dump.  Use :meth:`StringIO.StringIO` or :meth:`hexdump` to dump a string.
        width(int): The number of characters per line
+        groupsize(int): The number of characters per group
        skip(bool): Set to True, if repeated lines should be replaced by a "*"
        hexii(bool): Set to True, if a hexii-dump should be returned instead of a hexdump.
        begin(int):  Offset of the first byte to print in the left column
@@ -601,7 +602,7 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
        >>> tmp.flush()
        >>> tmp.seek(4)
        >>> print '\n'.join(hexdump_iter(tmp))
-        00000000  48 45 4c 4c  4f 2c 20 57  4f 52 4c 44               │HELL│O, W│ORLD││
+        00000000  48 45 4c 4c  4f 2c 20 57  4f 52 4c 44               │HELL│O, W│ORLD│
        0000000c

        >>> t = tube()
@@ -613,6 +614,9 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
    style     = style or {}
    highlight = highlight or []

+    if groupsize < 1:
+        groupsize = width
+
    for b in highlight:
        if isinstance(b, str):
            b = ord(b)
@@ -625,15 +629,10 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
    lines       = []
    last_unique = ''
    byte_width  = len('00 ')
-    column_sep  = '  '
-    line_fmt    = '%%(offset)08x  %%(hexbytes)-%is │%%(printable)s│' % (len(column_sep)+(width*byte_width))
    spacer      = ' '
    marker      = (style.get('marker') or (lambda s:s))('│')

-    if hexii:
-        column_sep = ''
-        line_fmt   = '%%(offset)08x  %%(hexbytes)-%is│' % (len(column_sep)+(width*byte_width))
-    else:
+    if not hexii:
        def style_byte(b):
            hbyte = '%02x' % ord(b)
            abyte = b if isprint(b) else '·'
@@ -696,31 +695,38 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
        # Generate contents for line
        hexbytes = ''
        printable = ''
+        color_chars = 0
+        abyte = abyte_previous = ''
        for i, b in enumerate(chunk):
            if not hexii:
+                abyte_previous = abyte
                hbyte, abyte = cache[ord(b)]
+                color_chars += len(hbyte) - 2
            else:
                hbyte, abyte = _hexiichar(b), ''

-            if i % 4 == 3 and i < width - 1:
+            if (i + 1) % groupsize == 0 and i < width - 1:
                hbyte += spacer
-                abyte += marker
+                abyte_previous += abyte
+                abyte = marker

            hexbytes += hbyte + ' '
+            printable += abyte_previous
+
+        if abyte != marker:
            printable += abyte

-        if i + 1 < width:
-            delta = width - i - 1
+        dividers_per_line = (width // groupsize)
+        if width % groupsize == 0:
+            dividers_per_line -= 1

-            # How many hex-bytes would we have printed?
-            count = byte_width * delta
-
-            # How many dividers do we need to fill out the line?
-            dividers_per_line = (width // 4) - (1 if width % 4 == 0 else 0)
-            dividers_printed = (i // 4) + (1 if i % 4 == 3 else 0)
-            count += dividers_per_line - dividers_printed
-
-            hexbytes += ' ' * count
+        if hexii:
+            line_fmt = '%%(offset)08x  %%(hexbytes)-%is│' % (width*byte_width)
+        else:
+            line_fmt = '%%(offset)08x  %%(hexbytes)-%is │%%(printable)s│' % (
+                 (width * byte_width)
+                + color_chars
+                + dividers_per_line )

        line = line_fmt % {'offset': offset, 'hexbytes': hexbytes, 'printable': printable}
        yield line
@@ -729,15 +735,16 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
    yield line

 def hexdump(s, width=16, skip=True, hexii=False, begin=0,
-            style=None, highlight=None, cyclic=False):
+            style=None, highlight=None, cyclic=False, groupsize=4):
    r"""hexdump(s, width = 16, skip = True, hexii = False, begin = 0,
-               style = None, highlight = None, cyclic = False) -> str generator
+               style = None, highlight = None, cyclic = False, groupsize=4) -> str generator

    Return a hexdump-dump of a string.

    Arguments:
        s(str): The data to hexdump.
        width(int): The number of characters per line
+        groupsize(int): The number of characters per group
        skip(bool): Set to True, if repeated lines should be replaced by a "*"
        hexii(bool): Set to True, if a hexii-dump should be returned instead of a hexdump.
        begin(int):  Offset of the first byte to print in the left column
@@ -760,7 +767,7 @@ def hexdump(s, width=16, skip=True, hexii=False, begin=0,
        00000020

        >>> print hexdump('A'*32, width=8)
-        00000000  41 41 41 41  41 41 41 41   │AAAA│AAAA│
+        00000000  41 41 41 41  41 41 41 41  │AAAA│AAAA│
        *
        00000020

@@ -880,7 +887,7 @@ def hexdump(s, width=16, skip=True, hexii=False, begin=0,
        00000010
        >>> print hexdump('A'*16, width=12)
        00000000  41 41 41 41  41 41 41 41  41 41 41 41  │AAAA│AAAA│AAAA│
-        0000000c  41 41 41 41                            │AAAA││
+        0000000c  41 41 41 41                            │AAAA│
        00000010
        >>> print hexdump('A'*16, width=13)
        00000000  41 41 41 41  41 41 41 41  41 41 41 41  41  │AAAA│AAAA│AAAA│A│
@@ -894,6 +901,15 @@ def hexdump(s, width=16, skip=True, hexii=False, begin=0,
        00000000  41 41 41 41  41 41 41 41  41 41 41 41  41 41 41  │AAAA│AAAA│AAAA│AAA│
        0000000f  41                                               │A│
        00000010
+
+        >>> print hexdump('A'*24, width=16, groupsize=8)
+        00000000  41 41 41 41 41 41 41 41  41 41 41 41 41 41 41 41  │AAAAAAAA│AAAAAAAA│
+        00000010  41 41 41 41 41 41 41 41                           │AAAAAAAA│
+        00000018
+        >>> print hexdump('A'*24, width=16, groupsize=-1)
+        00000000  41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41  │AAAAAAAAAAAAAAAA│
+        00000010  41 41 41 41 41 41 41 41                          │AAAAAAAA│
+        00000018
    """
    s = packing.flat(s)
    return '\n'.join(hexdump_iter(StringIO.StringIO(s),
@@ -903,7 +919,8 @@ def hexdump(s, width=16, skip=True, hexii=False, begin=0,
                                  begin,
                                  style,
                                  highlight,
-                                  cyclic))
+                                  cyclic,
+                                  groupsize))

 def negate(value, width = None):
    """
@@ -534,8 +534,8 @@ def iter_except(func, exception):
    the end.

    Arguments:
-      func:  The function to call.
-      exception(exception):  The exception that signals the end.  Other
+      func(callable): The function to call.
+      exception(Exception):  The exception that signals the end.  Other
        exceptions will not be caught.

    Returns:
@@ -114,17 +114,15 @@ def concat_all(*args):
       [0, 1, 2, 3, 4, 5, 6]
    """

-    if len(args) != 1:
-        return concat_all(list(args))
+    def go(arg, output):
+        if isinstance(arg, (tuple, list)):
+            for e in arg:
+                go(e, output)
+        else:
+            output.append(arg)
+        return output

-    if not isinstance(args[0], (tuple, list)):
-        return [args[0]]
-
-    res = []
-    for k in args[0]:
-        res.extend(concat_all(k))
-
-    return res
+    return go(args, [])

 def ordlist(s):
    """ordlist(s) -> list
@@ -190,6 +190,8 @@ def run_in_new_terminal(command, terminal = None, args = None):
          variable), ``x-terminal-emulator`` is used.
        - If tmux is detected (by the presence of the ``$TMUX`` environment
          variable), a new pane will be opened.
+        - If GNU Screen is detected (by the presence of the ``$STY`` environment
+          variable), a new screen will be opened.

    Arguments:
        command (str): The command to run.
@@ -219,6 +221,9 @@ def run_in_new_terminal(command, terminal = None, args = None):
        elif 'TMUX' in os.environ and which('tmux'):
            terminal = 'tmux'
            args     = ['splitw']
+        elif 'STY' in os.environ and which('screen'):
+            terminal = 'screen'
+            args     = ['-t','pwntools-gdb','bash','-c']

    if not terminal:
        log.error('Could not find a terminal binary to use. Set context.terminal to your terminal.')
@@ -58,7 +58,7 @@ def pack(number, word_size = None, endianness = None, sign = None, **kwargs):

    Arguments:
        number (int): Number to convert
-        word_size (int): Word size of the converted integer or the string 'all'.
+        word_size (int): Word size of the converted integer or the string 'all' (in bits).
        endianness (str): Endianness of the converted integer ("little"/"big")
        sign (str): Signedness of the converted integer (False/True)
        kwargs: Anything that can be passed to context.local
@@ -171,7 +171,7 @@ def unpack(data, word_size = None):

    Arguments:
        number (int): String to convert
-        word_size (int): Word size of the converted integer or the string "all".
+        word_size (int): Word size of the converted integer or the string "all" (in bits).
        endianness (str): Endianness of the converted integer ("little"/"big")
        sign (str): Signedness of the converted integer (False/True)
        kwargs: Anything that can be passed to context.local
@@ -237,7 +237,7 @@ def unpack_many(data, word_size = None):

    Args
        number (int): String to convert
-        word_size (int): Word size of the converted integers or the string "all".
+        word_size (int): Word size of the converted integers or the string "all" (in bits).
        endianness (str): Endianness of the converted integer ("little"/"big")
        sign (str): Signedness of the converted integer (False/True)
        kwargs: Anything that can be passed to context.local
@@ -366,7 +366,7 @@ def make_packer(word_size = None, sign = None, **kwargs):
    faster to call this function, since it will then use a specialized version.

    Arguments:
-        word_size (int): The word size to be baked into the returned packer or the string all.
+        word_size (int): The word size to be baked into the returned packer or the string all (in bits).
        endianness (str): The endianness to be baked into the returned packer. ("little"/"big")
        sign (str): The signness to be baked into the returned packer. ("unsigned"/"signed")
        kwargs: Additional context flags, for setting by alias (e.g. ``endian=`` rather than index)
@@ -429,7 +429,7 @@ def make_unpacker(word_size = None, endianness = None, sign = None, **kwargs):
    faster to call this function, since it will then use a specialized version.

    Arguments:
-        word_size (int): The word size to be baked into the returned packer.
+        word_size (int): The word size to be baked into the returned packer (in bits).
        endianness (str): The endianness to be baked into the returned packer. ("little"/"big")
        sign (str): The signness to be baked into the returned packer. ("unsigned"/"signed")
        kwargs: Additional context flags, for setting by alias (e.g. ``endian=`` rather than index)
@@ -479,43 +479,121 @@ def make_unpacker(word_size = None, endianness = None, sign = None, **kwargs):
    else:
        return lambda number: unpack(number, word_size, endianness, sign)

+def _fit(pieces, preprocessor, packer, filler):
+    # Pulls bytes from `filler` and adds them to `pad` until it ends in `key`.
+    # Returns the index of `key` in `pad`.
+    pad = []
+    def fill(key):
+        key = list(key)
+        while len(pad) < len(key) or pad[-len(key):] != key:
+            pad.append(filler.next())
+        return len(pad) - len(key)

+    # Key conversion:
+    # - convert str/unicode keys to offsets
+    # - convert large int (no null-bytes in a machine word) keys to offsets
+    pieces_ = dict()
+    large_key = 2**(context.word_size-8)
+    for k, v in pieces.items():
+        if isinstance(k, (int, long)):
+            if k >= large_key:
+                k = fill(pack(k))
+        elif isinstance(k, unicode):
+            k = fill(k.encode('utf8'))
+        elif isinstance(k, bytearray):
+            k = fill(str(k))
+        elif isinstance(k, str):
+            k = fill(k)
+        else:
+            raise TypeError("flat(): offset must be of type int or str, but got '%s'" % type(k))
+        if k in pieces_:
+            raise ValueError("flag(): multiple values at offset %d" % k)
+        pieces_[k] = v
+    pieces = pieces_

-def _flat(args, preprocessor, packer):
+    # We must "roll back" `filler` so each recursive call to `_flat` gets it in
+    # the right position
+    filler = iters.chain(pad, filler)
+
+    # Build output
+    out = ''
+    for k, v in sorted(pieces.items()):
+        if k < len(out):
+            raise ValueError("flat(): data at offset %d overlaps with previous data which ends at offset %d" % (k, len(out)))
+
+        # Fill up to offset
+        while len(out) < k:
+            out += filler.next()
+
+        # Recursively flatten data
+        out += _flat([v], preprocessor, packer, filler)
+
+    return filler, out
+
+def _flat(args, preprocessor, packer, filler):
    out = []
    for arg in args:

-        if not isinstance(arg, (list, tuple)):
+        if not isinstance(arg, (list, tuple, dict)):
            arg_ = preprocessor(arg)
            if arg_ != None:
                arg = arg_

        if hasattr(arg, '__flat__'):
-            out.append(arg.__flat__())
+            val = arg.__flat__()
        elif isinstance(arg, (list, tuple)):
-            out.append(_flat(arg, preprocessor, packer))
+            val = _flat(arg, preprocessor, packer, filler)
+        elif isinstance(arg, dict):
+            filler, val = _fit(arg, preprocessor, packer, filler)
        elif isinstance(arg, str):
-            out.append(arg)
+            val = arg
        elif isinstance(arg, unicode):
-            out.append(arg.encode('utf8'))
+            val = arg.encode('utf8')
        elif isinstance(arg, (int, long)):
-            out.append(packer(arg))
+            val = packer(arg)
        elif isinstance(arg, bytearray):
-            out.append(str(arg))
+            val = str(arg)
        else:
            raise ValueError("flat(): Flat does not support values of type %s" % type(arg))
+
+        out.append(val)
+
+        # Advance `filler` for "non-recursive" values
+        if not isinstance(arg, (list, tuple, dict)):
+            for _ in xrange(len(val)):
+                filler.next()
+
    return ''.join(out)

@LocalContext
 def flat(*args, **kwargs):
-    """flat(*args, preprocessor = None, word_size = None, endianness = None, sign = None)
+    """flat(*args, preprocessor = None, length = None, filler = de_bruijn(),
+     word_size = None, endianness = None, sign = None) -> str

    Flattens the arguments into a string.

-    This function takes an arbitrary number of arbitrarily nested lists and
-    tuples. It will then find every string and number inside those and flatten
-    them out. Strings are inserted directly while numbers are packed using the
-    :func:`pack` function.
+    This function takes an arbitrary number of arbitrarily nested lists, tuples
+    and dictionaries.  It will then find every string and number inside those
+    and flatten them out.  Strings are inserted directly while numbers are
+    packed using the :func:`pack` function.  Unicode strings are UTF-8 encoded.
+
+    Dictionary keys give offsets at which to place the corresponding values
+    (which are recursively flattened).  Offsets are relative to where the
+    flattened dictionary occurs in the output (i.e. `{0: 'foo'}` is equivalent
+    to `'foo'`).  Offsets can be integers, unicode strings or regular strings.
+    Integer offsets >= `2**(word_size-8)` are converted to a string using
+    `:func:pack`.  Unicode strings are UTF-8 encoded.  After these conversions
+    offsets are either integers or strings.  In the latter case, the offset will
+    be the lowest index at which the string occurs in `filler`.  See examples
+    below.
+
+    Space between pieces of data is filled out using the iterable `filler`.  The
+    `n`'th byte in the output will be byte at index ``n % len(iterable)`` byte
+    in `filler` if it has finite length or the byte at index `n` otherwise.
+
+    If `length` is given, the output will be padded with bytes from `filler` to
+    be this size.  If the output is longer than `length`, a :py:exc:`ValueError`
+    exception is raised.

    The three kwargs `word_size`, `endianness` and `sign` will default to using
    values in :mod:`pwnlib.context` if not specified as an argument.
@@ -524,7 +602,9 @@ def flat(*args, **kwargs):
      args: Values to flatten
      preprocessor (function): Gets called on every element to optionally
         transform the element before flattening. If :const:`None` is
-         returned, then the original value is uded.
+         returned, then the original value is used.
+      length: The length of the output.
+      filler: Iterable to use for padding.
      word_size (int): Word size of the converted integer.
      endianness (str): Endianness of the converted integer ("little"/"big").
      sign (str): Signedness of the converted integer (False/True)
@@ -534,21 +614,48 @@ def flat(*args, **kwargs):
      '\\x01\\x00testABABABABABAB'
      >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
      '234'
-"""
+      >>> flat({12: 0x41414141,
+      ...       24: 'Hello',
+      ...      })
+      'aaaabaaacaaaAAAAeaaafaaaHello'
+      >>> flat({'caaa': ''})
+      'aaaabaaa'
+      >>> flat({12: 'XXXX'}, filler = 'AB', length = 20)
+      'ABABABABABABXXXXABAB'
+      >>> flat({ 8: [0x41414141, 0x42424242],
+      ...       20: 'CCCC'})
+      'aaaabaaaAAAABBBBeaaaCCCC'
+      >>> flat({ 0x61616162: 'X'})
+      'aaaaX'
+      >>> flat({4: {0: 'X', 4: 'Y'}})
+      'aaaaXaaaY'
+
+    """
+    # HACK: To avoid circular imports we need to delay the import of `cyclic`
+    from pwnlib.util import cyclic

    preprocessor = kwargs.pop('preprocessor', lambda x: None)
-    word_size    = kwargs.pop('word_size', None)
+    filler       = kwargs.pop('filler', cyclic.de_bruijn())
+    length       = kwargs.pop('length', None)

    if kwargs != {}:
        raise TypeError("flat() does not support argument %r" % kwargs.popitem()[0])

-    return _flat(args, preprocessor, make_packer(word_size))
+    filler = iters.cycle(filler)
+    out = _flat(args, preprocessor, make_packer(), filler)

+    if length:
+        if len(out) > length:
+            raise ValueError("flat(): Arguments does not fit within `length` (= %d) bytes" % length)
+        out += ''.join(filler.next() for _ in xrange(length - len(out)))

-@LocalContext
-def fit(pieces=None, **kwargs):
-    """fit(pieces, filler = de_bruijn(), length = None, preprocessor = None) -> str
+    return out

+def fit(*args, **kwargs):
+    """Legacy alias for `:func:flat`"""
+    return flat(*args, **kwargs)
+
+"""
    Generates a string from a dictionary mapping offsets to data to place at
    that offset.

@@ -577,102 +684,13 @@ def fit(pieces=None, **kwargs):
      preprocessor (function): Gets called on every element to optionally
         transform the element before flattening. If :const:`None` is
         returned, then the original value is used.
-      word_size (int): Word size of the converted integer.
+      word_size (int): Word size of the converted integer (in bits).
      endianness (str): Endianness of the converted integer ("little"/"big").
      sign (str): Signedness of the converted integer (False/True)

    Examples:
-      >>> fit({12: 0x41414141,
-      ...      24: 'Hello',
-      ...     })
-      'aaaabaaacaaaAAAAeaaafaaaHello'
-      >>> fit({'caaa': ''})
-      'aaaabaaa'
-      >>> fit({12: 'XXXX'}, filler = 'AB', length = 20)
-      'ABABABABABABXXXXABAB'
-      >>> fit({ 8: [0x41414141, 0x42424242],
-      ...      20: 'CCCC'})
-      'aaaabaaaAAAABBBBeaaaCCCC'
-      >>> fit({ 0x61616162: 'X'})
-      'aaaaX'

    """
-    # HACK: To avoid circular imports we need to delay the import of `cyclic`
-    from pwnlib.util import cyclic
-
-    filler       = kwargs.pop('filler', cyclic.de_bruijn())
-    length       = kwargs.pop('length', None)
-    preprocessor = kwargs.pop('preprocessor', lambda x: None)
-
-    if kwargs != {}:
-        raise TypeError("fit() does not support argument %r" % kwargs.popitem()[0])
-
-    packer = make_packer()
-    filler = iters.cycle(filler)
-    out = ''
-
-    if not length and not pieces:
-        return ''
-
-    if not pieces:
-        return ''.join(filler.next() for f in range(length))
-
-    def fill(out, value):
-        while value not in out:
-            out += filler.next()
-        return out, out.index(value)
-
-    # convert str keys to offsets
-    # convert large int keys to offsets
-    pieces_ = dict()
-    for k, v in pieces.items():
-        if isinstance(k, (int, long)):
-            # cyclic() generally starts with 'aaaa'
-            if k >= 0x61616161:
-                out, k = fill(out, pack(k))
-        elif isinstance(k, str):
-            out, k = fill(out, k)
-        else:
-            raise TypeError("fit(): offset must be of type int or str, but got '%s'" % type(k))
-        pieces_[k] = v
-    pieces = pieces_
-
-    # convert values to their flattened forms
-    for k,v in pieces.items():
-        pieces[k] = _flat([v], preprocessor, packer)
-
-    # if we were provided a length, make sure everything fits
-    last = max(pieces)
-    if length and last and (last + len(pieces[last]) > length):
-        raise ValueError("fit(): Pieces do not fit within `length` (= %d) bytes" % length)
-
-    # insert data into output
-    out = list(out)
-    l = 0
-    for k, v in sorted(pieces.items()):
-        if k < l:
-            raise ValueError("fit(): data at offset %d overlaps with previous data which ends at offset %d" % (k, l))
-        while len(out) < k:
-            out.append(filler.next())
-        v = _flat([v], preprocessor, packer)
-        l = k + len(v)
-
-        # consume the filler for each byte of actual data
-        for i in range(len(out), l):
-            filler.next()
-
-        out[k:l] = v
-
-    # truncate/pad output
-    if length:
-        if l > length:
-            raise ValueError("fit(): Pieces does not fit within `length` (= %d) bytes" % length)
-        while len(out) < length:
-            out.append(filler.next())
-    else:
-        out = out[:l]
-
-    return ''.join(out)

 def signed(integer):
    return unpack(pack(integer), signed=True)
@@ -99,9 +99,9 @@ def name(pid):
        Name of process as listed in ``/proc/<pid>/status``.

    Example:
-        >>> pid = pidof('init')[0]
-        >>> name(pid) == 'init'
-        True
+        >>> p = process('cat')
+        >>> name(p.pid)
+        'cat'
    """
    return psutil.Process(pid).name()

@@ -245,6 +245,8 @@ def status(pid):
    try:
        with open('/proc/%d/status' % pid) as fd:
            for line in fd:
+                if ':' not in line:
+                    continue
                i = line.index(':')
                key = line[:i]
                val = line[i + 2:-1] # initial :\t and trailing \n
@@ -1,6 +1,6 @@
 _const_codes = [
    'POP_TOP','ROT_TWO','ROT_THREE','ROT_FOUR','DUP_TOP',
-    'BUILD_LIST','BUILD_MAP','BUILD_TUPLE',
+    'BUILD_LIST','BUILD_MAP','BUILD_TUPLE','BUILD_SET',
    'LOAD_CONST','RETURN_VALUE','STORE_SUBSCR', 'STORE_MAP'
    ]

@@ -393,7 +393,7 @@ def sh_string(s):
        return "''"

    chars = set(s)
-    very_good = set(string.ascii_letters + string.digits + "_+.,/")
+    very_good = set(string.ascii_letters + string.digits + "_+.,/-")

    # Alphanumeric can always just be used verbatim.
    if chars <= very_good:
@@ -1 +1 @@
-__version__ = '3.7.0'
+__version__ = '3.13.0'
@@ -2,6 +2,7 @@
 import glob
 import os
 import platform
+import subprocess
 import sys
 import traceback
 from distutils.command.install import INSTALL_SCHEMES
@@ -45,7 +46,7 @@ for filename in glob.glob('pwnlib/commandline/*'):
 install_requires     = ['paramiko>=1.15.2',
                        'mako>=1.0.0',
                        'pyelftools>=0.2.4',
-                        'capstone',
+                        'capstone>=3.0.5rc2', # See Gallopsled/pwntools#971, Gallopsled/pwntools#1160
                        'ropgadget>=5.3',
                        'pyserial>=2.7',
                        'requests>=2.0',
@@ -54,10 +55,10 @@ install_requires     = ['paramiko>=1.15.2',
                        'pygments>=2.0',
                        'pysocks',
                        'python-dateutil',
-                        'pypandoc',
                        'packaging',
                        'psutil>=3.3.0',
-                        'intervaltree',
+                        'intervaltree<3.0', # See Gallopsled/pwntools#1238
+                        'sortedcontainers<2.0', # See Gallopsled/pwntools#1154
                        'unicorn']

 # Check that the user has installed the Python development headers
@@ -70,23 +71,16 @@ if not os.path.exists(PythonH):
 # Convert README.md to reStructuredText for PyPI
 long_description = ''
 try:
-    import pypandoc
-    try:
-        pypandoc.get_pandoc_path()
-    except OSError:
-        pypandoc.download_pandoc()
-    long_description = pypandoc.convert_file('README.md', 'rst')
-except ImportError:
-    pass
+    long_description = subprocess.check_output(['pandoc', 'README.md', '--to=rst'])
 except Exception as e:
    print >>sys.stderr, "Failed to convert README.md through pandoc, proceeding anyway"
    traceback.print_exc()

-
 setup(
    name                 = 'pwntools',
+    python_requires      = '~=2.7',
    packages             = find_packages(),
-    version              = '3.7.0',
+    version              = '3.13.0',
    data_files           = [('',
                             glob.glob('*.md') + glob.glob('*.txt')),
                            ],
@@ -105,7 +99,7 @@ setup(
    description          = "Pwntools CTF framework and exploit development library.",
    long_description     = long_description,
    author               = "Gallopsled et al.",
-    author_email         = "#pwntools @ freenode.net",
+    author_email         = "pwntools[email protected]",
    url                  = 'https://pwntools.com',
    download_url         = "https://github.com/Gallopsled/pwntools/releases",
    install_requires     = install_requires,
@@ -8,7 +8,10 @@ MAINTAINER Maintainer Gallopsled et al.

 USER root

+RUN apt-get update
+
 # Use UTF-8
+RUN apt-get install -y locales
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash -e
-set -e
+set -ex

 local_deb_extract()
 {
@@ -11,11 +11,11 @@ local_deb_extract()

 install_deb()
 {
-    version=${2:-zesty}
+    version=${2:-artful}
    package=$1
    echo "Installing $package"
    INDEX="http://packages.ubuntu.com/en/$version/amd64/$package/download"
-    URL=$(curl "$INDEX" | grep -Eo "https?://.*$package.*\.deb" | head -1)
+    URL=$(curl -L "$INDEX" | grep -Eo "https?://.*$package.*\.deb" | head -1)
    local_deb_extract "$URL"
 }

@@ -90,7 +90,7 @@ setup_android_emulator()
        elif [[ -n "$TRAVIS_TAG" ]]; then
            echo "TRAVIS_TAG ($TRAVIS_TAG) indicates a new relase"
            echo "Forcing Android Emulator installation"
-        elif (git log --stat "$TRAVIS_COMMIT_RANGE" | grep -iE "android|adb"); then
+        elif (git log --stat "$TRAVIS_COMMIT_RANGE" | grep -iE "android|adb" | grep -v "commit "); then
            echo "Found Android-related commits, forcing Android Emulator installation"
        else
            # In order to avoid running the doctests that require the Android
@@ -119,7 +119,7 @@ setup_android_emulator()
    else
        if [ ! -f android-sdk/android ]; then
            # Install the SDK, which gives us the 'android' and 'emulator' commands
-            wget https://dl.google.com/android/android-sdk_r24.4.1-linux.tgz
+            wget -nv https://dl.google.com/android/android-sdk_r24.4.1-linux.tgz
            tar xf android-sdk_r24.4.1-linux.tgz
            rm  -f android-sdk_r24.4.1-linux.tgz

@@ -137,7 +137,7 @@ setup_android_emulator()
        # Install the NDK, which is required for adb.compile()
        NDK_VERSION=android-ndk-r12b
        if [ ! -f android-ndk/ndk-build ]; then
-            wget   https://dl.google.com/android/repository/$NDK_VERSION-linux-x86_64.zip
+            wget -nv https://dl.google.com/android/repository/$NDK_VERSION-linux-x86_64.zip
            unzip -q android-ndk-*.zip
            rm -f    android-ndk-*.zip

@@ -184,7 +184,7 @@ setup_osx()
 }

 if [[ "$USER" == "travis" ]]; then
-    setup_travis
+#   setup_travis
    setup_android_emulator
 elif [[ "$USER" == "shippable" ]]; then
    sudo apt-get update
@@ -198,4 +198,4 @@ elif [[ "$(uname)" == "Linux" ]]; then
    setup_android_emulator
 fi

-set +e
+set +ex
@@ -4,7 +4,7 @@
 #
 # All of the "conditional sudo" is to do container-based builds on
 # Travis which are much, much faster.
-set -e
+set -ex

 U=travis
 H=/home/$U
@@ -55,4 +55,4 @@ EOF

 ssh -o "StrictHostKeyChecking no" -vvvv [email protected] id

-set +e
+set +ex