Release 4.4.0

* Update libcdb buildid offsets

Recent libcs from Debian and Ubuntu have different offsets for the build id.

* Update CHANGELOG

.coveragerc

@@ -9,3 +9,4 @@ source =
     pwnlib
     ~/.pwntools-cache-2.7/
     ~/.pwntools-cache-3.8/
+disable_warnings = module-not-imported

.github/workflows/ci.yml

@@ -14,8 +14,10 @@ jobs:
       with:
         fetch-depth: 0
 
-    - name: Git History 2
-      run: git log --oneline --graph -10
+    - name: Git History
+      run: |
+        git fetch origin
+        git log --oneline --graph -10
 
     - name: Cache for pip
       uses: actions/cache@v1
@@ -46,13 +48,12 @@ jobs:
       id: java-needed
       env:
         GITHUB_REF: ${{ github.event.ref }}
-        GITHUB_COMMIT_RANGE: ${{ github.event.before }}...${{ github.event.after }}
       run: |
         if echo "$GITHUB_REF" | grep -Eq 'staging|tags'; then
           echo "Found release or important branch ($GITHUB_REF), forcing tests."
-        elif ! git show "$GITHUB_COMMIT_RANGE" >/dev/null; then
+        elif ! git show "origin/$GITHUB_BASE_REF"..HEAD >/dev/null; then
           echo 'Incorrect commit range, forcing android tests.'
-        elif git log --stat "$GITHUB_COMMIT_RANGE" | grep -iE 'android|\<adb\>'; then
+        elif git log --stat "origin/$GITHUB_BASE_REF"..HEAD | grep -iE 'android|\<adb\>'; then
           echo 'Found Android-related commits, forcing tests.'
         else
           # clear files that cause Android doctests
@@ -73,7 +74,9 @@ jobs:
           binutils-aarch64-linux-gnu \
           binutils-arm-linux-gnueabihf \
           binutils-mips-linux-gnu \
+          binutils-msp430 \
           binutils-powerpc-linux-gnu \
+          binutils-s390x-linux-gnu \
           binutils-sparc64-linux-gnu \
           gcc-multilib \
           libc6-dbg \

.github/workflows/pylint.yml

@@ -0,0 +1,36 @@
+name: PyLint
+on: [pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: PyLint
+      run: |
+        set -x
+        pip install pylint
+        pylint --exit-zero --errors-only  pwnlib > current.txt
+        git fetch origin
+        git checkout origin/"$GITHUB_BASE_REF"
+        pylint --exit-zero --errors-only  pwnlib > base.txt
+        if diff base.txt current.txt | grep '>'; then
+          false
+        fi

.travis.yml

@@ -19,7 +19,9 @@ addons:
     - binutils-aarch64-linux-gnu
     - binutils-arm-linux-gnueabihf
     - binutils-mips-linux-gnu
+    - binutils-msp430
     - binutils-powerpc-linux-gnu
+    - binutils-s390x-linux-gnu
     - binutils-sparc64-linux-gnu
     - bash-static
 cache:

CHANGELOG.md

@@ -9,9 +9,11 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [4.5.0](#450)    | `dev`    | Dec 20, 2020 (planned)
-| [4.4.0](#440)    | `beta`   | Nov 20, 2020 (planned)
-| [4.3.0](#430)    | `stable` | Oct 20, 2020
+| [4.6.0](#460)    | `dev`    | May 29, 2020 (planned)
+| [4.5.0](#450)    | `beta`   | Apr 29, 2020 (planned)
+| [4.4.0](#440)    | `stable` | Mar 29, 2020
+| [4.3.1](#431)    |          | Nov 29, 2020
+| [4.3.0](#430)    |          | Oct 20, 2020
 | [4.2.0](#420)    |          | Jul 3, 2020
 | [4.1.7](#417)    |          | Jun 30, 2020
 | [4.1.5](#415)    |          | Jun 27, 2020
@@ -56,15 +58,44 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 4.5.0 (`dev`)
+## 4.6.0 (`dev`)
+
+
+
+## 4.5.0 (`beta`)
 
 - [#1261][1261] Misc `run_in_new_terminal` improvements (notably gdb terminated by default)
 - [#1695][1695] Allow using GDB Python API
+- [#1735][1735] Python 3.9 support in safeeval
+- [#1738][1738] Which function support custom search path
+  - process also looks now at `env['PATH']` to find the path for the executable
+- [#1742][1742] New `baremetal` os to debug binaries executed with qemu-system-$(arch)
+- [#1757][1757] update cache directories
+- [#1758][1758] Remove eval from cli
+- [#1780][1780] Re-add Python2 to the official Dockerfile
+- [#1941][1941] Disable all Android tests, `pwnlib.adb` is no longer supported in CI
+- [#1811][1811] Remove unnecessary `pwn.toplevel.__all__`
+- [#1827][1827] Support `$XDG_CONFIG_HOME` dir for `pwn.conf`
+- [#1841][1841] Add colored_traceback
+- [#1839][1839] run_in_new_terminal now creates a runner script if given a list or tuple
+- [#1833][1833] Add pwnlib.filesystem module
 
 [1261]: https://github.com/Gallopsled/pwntools/pull/1261
 [1695]: https://github.com/Gallopsled/pwntools/pull/1695
+[1735]: https://github.com/Gallopsled/pwntools/pull/1735
+[1738]: https://github.com/Gallopsled/pwntools/pull/1738
+[1742]: https://github.com/Gallopsled/pwntools/pull/1742
+[1757]: https://github.com/Gallopsled/pwntools/pull/1757
+[1758]: https://github.com/Gallopsled/pwntools/pull/1758
+[1780]: https://github.com/Gallopsled/pwntools/pull/1780
+[1941]: https://github.com/Gallopsled/pwntools/pull/1941
+[1811]: https://github.com/Gallopsled/pwntools/pull/1811
+[1827]: https://github.com/Gallopsled/pwntools/pull/1827
+[1841]: https://github.com/Gallopsled/pwntools/pull/1841
+[1839]: https://github.com/Gallopsled/pwntools/pull/1839
+[1833]:  https://github.com/Gallopsled/pwntools/pull/1833
 
-## 4.4.0 (`beta`)
+## 4.4.0 (`stable`)
 
 - [#1541][1541] Use `context.newline` for tubes by default
 - [#1602][1602] Fix bytes handling in ssh tubes
@@ -108,7 +139,7 @@ The table below shows which release corresponds to each branch, and what date th
 [1703]: https://github.com/Gallopsled/pwntools/pull/1703
 [1704]: https://github.com/Gallopsled/pwntools/pull/1704
 
-## 4.3.1 (`stable`)
+## 4.3.1
 
 - [#1732][1732] Fix shellcraft SSTI vulnerability (first major pwntools vuln!)
 
@@ -212,7 +243,7 @@ The table below shows which release corresponds to each branch, and what date th
 [1001]: https://github.com/Gallopsled/pwntools/pull/1001
 [1389]: https://github.com/Gallopsled/pwntools/pull/1389
 [1241]: https://github.com/Gallopsled/pwntools/pull/1241
-[1218]: https://github.com/Gallopsled/pwntools/pull/1218  
+[1218]: https://github.com/Gallopsled/pwntools/pull/1218
 
 ## 4.0.1
 

README.md

@@ -25,6 +25,8 @@ r.interactive()
 
 Our documentation is available at [docs.pwntools.com](https://docs.pwntools.com/)
 
+A series of tutorials is also [available online](https://github.com/Gallopsled/pwntools-tutorial#readme)
+
 To get you started, we've provided some example solutions for past CTF challenges in our [write-ups repository](https://github.com/Gallopsled/pwntools-write-ups).
 
 # Installation

docs/source/conf.py

@@ -91,6 +91,7 @@ pwnlib.context.ContextType.defaults['log_console'] = stdout()
 
 github_actions = os.environ.get('USER') == 'runner'
 travis_ci = os.environ.get('USER') == 'travis'
+branch_dev = os.environ.get('GITHUB_BASE_REF') == 'dev'
 '''
 
 autoclass_content = 'both'

docs/source/encoders.rst

@@ -1,13 +1,18 @@
 .. testsetup:: *
 
    from pwn import *
-
+   
 :mod:`pwnlib.encoders` --- Encoding Shellcode
 ===============================================
 
 .. automodule:: pwnlib.encoders.encoder
    :members:
 
+.. automodule:: pwnlib.encoders.i386.ascii_shellcode
+   :members:
+   :special-members:
+   :exclude-members: __init__
+
 .. automodule:: pwnlib.encoders.i386.xor
    :members:
 

docs/source/intro.rst

@@ -17,6 +17,13 @@ assemble, disassemble, pack, unpack, and many other things with a single functio
 A full list of everything that is imported is available on :doc:`globals`.
 
 
+Tutorials
+---------
+
+A series of tutorials for Pwntools exists online, at
+https://github.com/Gallopsled/pwntools-tutorial#readme
+
+
 Making Connections
 ------------------
 
@@ -188,10 +195,10 @@ Never write another hexdump, thanks to :mod:`pwnlib.util.fiddling`.
 
 Find offsets in your buffer that cause a crash, thanks to :mod:`pwnlib.cyclic`.
 
-    >>> print(cyclic(20).decode())
-    aaaabaaacaaadaaaeaaa
+    >>> cyclic(20)
+    b'aaaabaaacaaadaaaeaaa'
     >>> # Assume EIP = 0x62616166 (b'faab' which is pack(0x62616166))  at crash time
-    >>> print(cyclic_find(b'faab'))
+    >>> cyclic_find(b'faab')
     120
 
 ELF Manipulation

extra/docker/beta/Dockerfile

@@ -1,6 +1,6 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@beta
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@beta
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/bionic/Dockerfile

@@ -12,11 +12,19 @@ RUN apt-get update \
         git \
         libssl-dev \
         libffi-dev \
-        python2.7 \
-        python-pip \
-        python-dev \
-    && pip install --upgrade pip \
-    && python -m pip install --upgrade pwntools \
+        python3 \
+        python3-pip \
+        python3-dev \
+        qemu-user-static \
+        binutils-arm-linux-gnueabihf \
+        binutils-aarch64-linux-gnu \
+        binutils-mips-linux-gnu \
+        binutils-mipsel-linux-gnu \
+        binutils-powerpc-linux-gnu \
+        binutils-powerpc64-linux-gnu \
+        binutils-sparc64-linux-gnu \
+    && pip3 install --upgrade pip \
+    && python3 -m pip install --upgrade pwntools \
     && PWNLIB_NOTERM=1 pwn update \
     && apt-get install -y sudo \
     && useradd -m pwntools \

extra/docker/dev/Dockerfile

@@ -1,6 +1,6 @@
 FROM pwntools/pwntools:stable
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@dev
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@dev
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

extra/docker/stable/Dockerfile

@@ -1,6 +1,6 @@
 FROM pwntools/pwntools:base
 
 USER root
-RUN pip install --upgrade git+https://github.com/Gallopsled/pwntools@stable
+RUN pip3 install --upgrade git+https://github.com/Gallopsled/pwntools@stable
 RUN PWNLIB_NOTERM=1 pwn update
 USER pwntools

pwn/toplevel.py

@@ -5,6 +5,7 @@ import math
 import operator
 import os
 import re
+import requests
 import socks
 import signal
 import string

pwnlib/args.py

@@ -1,5 +1,3 @@
-
-#!/usr/bin/env python2
 """
 Pwntools exposes several magic command-line arguments and environment
 variables when operating in `from pwn import *` mode.

pwnlib/asm.py

@@ -334,9 +334,11 @@ def _bfdname():
         'cris'    : 'elf32-cris',
         'ia64'    : 'elf64-ia64-%s' % E,
         'm68k'    : 'elf32-m68k',
+        'msp430'  : 'elf32-msp430',
         'powerpc' : 'elf32-powerpc',
         'powerpc64' : 'elf64-powerpc',
         'vax'     : 'elf32-vax',
+        's390'    : 'elf%d-s390' % context.bits,
         'sparc'   : 'elf32-sparc',
         'sparc64' : 'elf64-sparc',
     }
@@ -646,6 +648,10 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
         b'H\xc7\xc0\x17\x00\x00\x00'
         >>> asm("mov r0, #SYS_select", arch = 'arm', os = 'linux', bits=32)
         b'R\x00\xa0\xe3'
+        >>> asm("mov #42, r0", arch = 'msp430')
+        b'0@*\x00'
+        >>> asm("la %r0, 42", arch = 's390', bits=64)
+        b'A\x00\x00*'
     """
     result = ''
 

pwnlib/commandline/cyclic.py

@@ -70,7 +70,7 @@ def main(args):
             pat = int(pat, 0)
         except ValueError:
             pass
-        pat = flat(pat)
+        pat = flat(pat, bytes=args.length)
 
         if len(pat) != subsize:
             log.critical('Subpattern must be %d bytes' % subsize)

pwnlib/commandline/shellcraft.py

@@ -198,11 +198,11 @@ def main(args):
         common.parser.print_usage()
         exit()
 
-    if args.shellcode not in shellcraft.templates:
+    try:
+        func = get_template(args.shellcode)
+    except AttributeError:
         log.error("Unknown shellcraft template %r. Use --list to see available shellcodes." % args.shellcode)
 
-    func = get_template(args.shellcode)
-
     if args.show:
         # remove doctests
         doc = []

pwnlib/commandline/template.py

@@ -41,7 +41,14 @@ def main(args):
             log.error("Must specify --path or a exe")
 
         s = ssh(args.user, args.host, args.port or 22, args.password or None)
-        s.download(args.path or args.exe)
+
+        try:
+            remote = args.path or args.exe
+            s.download(remote)
+        except Exception:
+            log.warning("Could not download file %r, opening a shell", remote)
+            s.interactive()
+            return
 
         if not args.exe:
             args.exe = os.path.basename(args.path)

pwnlib/context/__init__.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 """
 Implements context management so that nested/scoped contexts and threaded
@@ -359,7 +358,7 @@ class ContextType(object):
         'log_console': sys.stdout,
         'randomize': False,
         'rename_corefiles': True,
-        'newline': '\n',
+        'newline': b'\n',
         'noptrace': False,
         'os': 'linux',
         'proxy': None,
@@ -1279,6 +1278,15 @@ class ContextType(object):
         """
         return bool(v)
 
+    @_validator
+    def newline(self, v):
+        """Line ending used for Tubes by default.
+
+        This configures the newline emitted by e.g. ``sendline`` or that is used
+        as a delimiter for e.g. ``recvline``.
+        """
+        return six.ensure_binary(v)
+
 
     @_validator
     def gdbinit(self, value):

pwnlib/encoders/i386/__init__.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
 
+from pwnlib.encoders.i386 import ascii_shellcode
 from pwnlib.encoders.i386 import delta
 from pwnlib.encoders.i386 import xor

pwnlib/encoders/i386/ascii_shellcode.py

@@ -0,0 +1,300 @@
+""" Encoder to convert shellcode to shellcode that contains only ascii
+characters """
+# https://github.com/Gallopsled/pwntools/pull/1667
+
+from __future__ import absolute_import
+
+from itertools import product
+
+import six
+
+from pwnlib.context import LocalContext
+from pwnlib.context import context
+from pwnlib.encoders.encoder import Encoder
+from pwnlib.encoders.encoder import all_chars
+from pwnlib.util.iters import group
+from pwnlib.util.packing import *
+
+
+class AsciiShellcodeEncoder(Encoder):
+    """ Pack shellcode into only ascii characters that unpacks itself and
+    executes (on the stack)
+
+    The original paper this encoder is based on:
+    http://julianor.tripod.com/bc/bypass-msb.txt
+
+    A more visual explanation as well as an implementation in C:
+    https://github.com/VincentDary/PolyAsciiShellGen/blob/master/README.md#mechanism
+    """
+
+    def __init__(self, slop=20, max_subs=4):
+        """ Init
+
+        Args:
+            slop (int, optional): The amount esp will be increased by in the
+                allocation phase (In addition to the length of the packed
+                shellcode) as well as defines the size of the NOP sled (you can
+                increase/ decrease the size of the NOP sled by adding/removing
+                b'P'-s to/ from the end of the packed shellcode).
+                Defaults to 20.
+            max_subs (int, optional): The maximum amount of subtractions
+                allowed to be taken. This may be increased if you have a
+                relatively  restrictive ``avoid`` set. The more subtractions
+                there are, the bigger the packed shellcode will be.
+                Defaults to 4.
+        """
+        if six.PY2:
+            super(AsciiShellcodeEncoder, self).__init__()
+        elif six.PY3:
+            super().__init__()
+        self.slop = slop
+        self.max_subs = max_subs
+
+    @LocalContext
+    def __call__(self, raw_bytes, avoid=None, pcreg=None):
+        r""" Pack shellcode into only ascii characters that unpacks itself and
+        executes (on the stack)
+
+        Args:
+            raw_bytes (bytes): The shellcode to be packed
+            avoid (set, optional): Characters to avoid. Defaults to allow
+                printable ascii (0x21-0x7e).
+            pcreg (NoneType, optional): Ignored
+
+        Raises:
+            RuntimeError: A required character is in ``avoid`` (required
+                characters are characters which assemble into assembly
+                instructions and are used to unpack the shellcode onto the
+                stack, more details in the paper linked above ``\ - % T X P``).
+            RuntimeError: Not supported architecture
+            ArithmeticError: The allowed character set does not contain
+                two characters that when they are bitwise-anded with eachother
+                their result is 0
+            ArithmeticError: Could not find a correct subtraction sequence
+                to get to the the desired target value with the given ``avoid``
+                parameter
+
+        Returns:
+            bytes: The packed shellcode
+
+        Examples:
+
+            >>> context.update(arch='i386', os='linux')
+            >>> sc = b"\x83\xc4\x181\xc01\xdb\xb0\x06\xcd\x80Sh/ttyh/dev\x89\xe31\xc9f\xb9\x12'\xb0\x05\xcd\x80j\x17X1\xdb\xcd\x80j.XS\xcd\x801\xc0Ph//shh/bin\x89\xe3PS\x89\xe1\x99\xb0\x0b\xcd\x80"
+            >>> encoders.i386.ascii_shellcode.encode(sc)
+            b'TX-!!!!-"_``-~~~~P\\%!!!!%@@@@-!6!!-V~!!-~~<-P-!mha-a~~~P-!!L`-a^~~-~~~~P-!!if-9`~~P-!!!!-aOaf-~~~~P-!&!<-!~`~--~~~P-!!!!-!!H^-+A~~P-U!![-~A1~P-,<V!-~~~!-~~~GP-!2!8-j~O~P-!]!!-!~!r-y~w~P-c!!!-~<(+P-N!_W-~1~~P-!!]!-Mn~!-~~~<P-!<!!-r~!P-~~x~P-fe!$-~~S~-~~~~P-!!\'$-%z~~P-A!!!-~!#!-~*~=P-!7!!-T~!!-~~E^PPPPPPPPPPPPPPPPPPPPP'
+            >>> avoid = {'\x00', '\x83', '\x04', '\x87', '\x08', '\x8b', '\x0c', '\x8f', '\x10', '\x93', '\x14', '\x97', '\x18', '\x9b', '\x1c', '\x9f', ' ', '\xa3', '\xa7', '\xab', '\xaf', '\xb3', '\xb7', '\xbb', '\xbf', '\xc3', '\xc7', '\xcb', '\xcf', '\xd3', '\xd7', '\xdb', '\xdf', '\xe3', '\xe7', '\xeb', '\xef', '\xf3', '\xf7', '\xfb', '\xff', '\x80', '\x03', '\x84', '\x07', '\x88', '\x0b', '\x8c', '\x0f', '\x90', '\x13', '\x94', '\x17', '\x98', '\x1b', '\x9c', '\x1f', '\xa0', '\xa4', '\xa8', '\xac', '\xb0', '\xb4', '\xb8', '\xbc', '\xc0', '\xc4', '\xc8', '\xcc', '\xd0', '\xd4', '\xd8', '\xdc', '\xe0', '\xe4', '\xe8', '\xec', '\xf0', '\xf4', '\xf8', '\xfc', '\x7f', '\x81', '\x02', '\x85', '\x06', '\x89', '\n', '\x8d', '\x0e', '\x91', '\x12', '\x95', '\x16', '\x99', '\x1a', '\x9d', '\x1e', '\xa1', '\xa5', '\xa9', '\xad', '\xb1', '\xb5', '\xb9', '\xbd', '\xc1', '\xc5', '\xc9', '\xcd', '\xd1', '\xd5', '\xd9', '\xdd', '\xe1', '\xe5', '\xe9', '\xed', '\xf1', '\xf5', '\xf9', '\xfd', '\x01', '\x82', '\x05', '\x86', '\t', '\x8a', '\r', '\x8e', '\x11', '\x92', '\x15', '\x96', '\x19', '\x9a', '\x1d', '\x9e', '\xa2', '\xa6', '\xaa', '\xae', '\xb2', '\xb6', '\xba', '\xbe', '\xc2', '\xc6', '\xca', '\xce', '\xd2', '\xd6', '\xda', '\xde', '\xe2', '\xe6', '\xea', '\xee', '\xf2', '\xf6', '\xfa', '\xfe'}
+            >>> sc = shellcraft.echo("Hello world") + shellcraft.exit()
+            >>> ascii = encoders.i386.ascii_shellcode.encode(asm(sc), avoid)
+            >>> ascii += asm('jmp esp') # just for testing, the unpacker should also run on the stack
+            >>> ELF.from_bytes(ascii).process().recvall()
+            b'Hello world'
+        """
+        if not avoid:
+            vocab = bytearray(
+                b"!\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~")
+        else:
+            required_chars = set('\\-%TXP')
+            allowed = set(all_chars)
+            if avoid.intersection(required_chars):
+                raise RuntimeError(
+                    '''These characters ({}) are required because they assemble
+                    into instructions used to unpack the shellcode'''.format(
+                        str(required_chars, 'ascii')))
+            allowed.difference_update(avoid)
+            vocab = bytearray(map(ord, allowed))
+
+        if context.arch != 'i386' or context.bits != 32:
+            raise RuntimeError('Only 32-bit i386 is currently supported')
+
+        int_size = context.bytes
+
+        # Prepend with NOPs for the NOP sled
+        shellcode = bytearray(b'\x90'*int_size + raw_bytes)
+        subtractions = self._get_subtractions(shellcode, vocab)
+        allocator = self._get_allocator(len(subtractions) + self.slop, vocab)
+        nop_sled = b'P' * self.slop  # push eax
+        return bytes(allocator + subtractions + nop_sled)
+
+    @LocalContext
+    def _get_allocator(self, size, vocab):
+        r""" Allocate enough space on the stack for the shellcode
+
+        int_size is taken from the context
+
+        Args:
+            size (int): The allocation size
+            vocab (bytearray): Allowed characters
+
+        Returns:
+            bytearray: The allocator shellcode
+
+        Examples:
+
+            >>> context.update(arch='i386', os='linux')
+            >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~')
+            >>> encoders.i386.ascii_shellcode.encode._get_allocator(300, vocab)
+            bytearray(b'TX-!!!!-!_``-t~~~P\\%!!!!%@@@@')
+        """
+        size += 0x1e  # add typical allocator size
+        int_size = context.bytes
+        # Use eax for subtractions because sub esp, X doesn't assemble to ascii
+        result = bytearray(b'TX')  # push esp; pop eax
+        # Set target to the `size` arg
+        target = bytearray(pack(size))
+        # All we are doing here is adding (subtracting) `size`
+        # to esp (to allocate space on the stack), so we don't care
+        # about esp's actual value. That's why the `last` parameter
+        # for `calc_subtractions` can just be zero
+        for subtraction in self._calc_subtractions(
+                bytearray(int_size), target, vocab):
+            # sub eax, subtraction
+            result += b'-' + subtraction
+        result += b'P\\'  # push eax, pop esp
+        # Zero out eax for the unpacking part
+        pos, neg = self._find_negatives(vocab)
+        # and eax, pos; and eax, neg ; (0b00010101 & 0b00101010 = 0b0)
+        result += flat((b'%', pos, b'%', neg))
+        return result
+
+    @LocalContext
+    def _find_negatives(self, vocab):
+        r""" Find two bitwise negatives in the vocab so that when they are
+        and-ed the result is 0.
+
+        int_size is taken from the context
+
+        Args:
+            vocab (bytearray): Allowed characters
+
+        Returns:
+            Tuple[int, int]: value A, value B
+
+        Raises:
+            ArithmeticError: The allowed character set does not contain
+                two characters that when they are bitwise-and-ed with eachother
+                the result is 0
+
+        Examples:
+
+            >>> context.update(arch='i386', os='linux')
+            >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~')
+            >>> a, b = encoders.i386.ascii_shellcode.encode._find_negatives(vocab)
+            >>> a & b
+            0
+        """
+        int_size = context.bytes
+        for products in product(vocab, vocab):
+            if products[0] & products[1] == 0:
+                return tuple(
+                    # pylint: disable=undefined-variable
+                    unpack(p8(x)*int_size)  # noqa: F405
+                    for x in bytearray(products)
+                )
+        else:
+            raise ArithmeticError(
+                'Could not find two bitwise negatives in the provided vocab')
+
+    @LocalContext
+    def _get_subtractions(self, shellcode, vocab):
+        r""" Covert the sellcode to sub eax and posh eax instructions
+
+        int_size is taken from the context
+
+        Args:
+            shellcode (bytearray): The shellcode to pack
+            vocab (bytearray): Allowed characters
+
+        Returns:
+            bytearray: packed shellcode
+
+        Examples:
+
+            >>> context.update(arch='i386', os='linux')
+            >>> sc = bytearray(b'ABCDEFGHIGKLMNOPQRSTUVXYZ')
+            >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~')
+            >>> encoders.i386.ascii_shellcode.encode._get_subtractions(sc, vocab)
+            bytearray(b'-(!!!-~NNNP-!=;:-f~~~-~~~~P-!!!!-edee-~~~~P-!!!!-eddd-~~~~P-!!!!-egdd-~~~~P-!!!!-eadd-~~~~P-!!!!-eddd-~~~~P')
+        """
+        int_size = context.bytes
+        result = bytearray()
+        last = bytearray(int_size)
+        # Group the shellcode into bytes of stack cell size, pad with NOPs
+        # if the shellcode does not divide into stack cell size and reverse.
+        # The shellcode will be reversed again back to it's original order once
+        # it's pushed onto the stack
+        sc = tuple(group(int_size, shellcode, 0x90))[::-1]
+        # Pack the shellcode to a sub/push sequence
+        for x in sc:
+            for subtraction in self._calc_subtractions(last, x, vocab):
+                result += b'-' + subtraction  # sub eax, ...
+            last = x
+            result += b'P'  # push eax
+        return result
+
+    @LocalContext
+    def _calc_subtractions(self, last, target, vocab):
+        r""" Given `target` and `last`, return a list of integers that when
+         subtracted from `last` will equal `target` while only constructing
+         integers from bytes in `vocab`
+
+        int_size is taken from the context
+
+        Args:
+            last (bytearray): Original value
+            target (bytearray): Desired value
+            vocab (bytearray): Allowed characters
+
+        Raises:
+            ArithmeticError: If a sequence of subtractions could not be found
+
+        Returns:
+            List[bytearray]: List of numbers that would need to be subtracted
+            from `last` to get to `target`
+
+        Examples:
+
+            >>> context.update(arch='i386', os='linux')
+            >>> vocab = bytearray(b'!"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~')
+            >>> print(encoders.i386.ascii_shellcode.encode._calc_subtractions(bytearray(b'\x10'*4), bytearray(b'\x11'*4), vocab))
+            [bytearray(b'!!!!'), bytearray(b'`___'), bytearray(b'~~~~')]
+            >>> print(encoders.i386.ascii_shellcode.encode._calc_subtractions(bytearray(b'\x11\x12\x13\x14'), bytearray(b'\x15\x16\x17\x18'), vocab))
+            [bytearray(b'~}}}'), bytearray(b'~~~~')]
+        """
+        int_size = context.bytes
+        subtractions = [bytearray(int_size)]
+        for sub in range(self.max_subs):
+            carry = success_count = 0
+            for byte in range(int_size):
+                # Try all combinations of all the characters in vocab of
+                # `subtraction` characters in each combination. So if
+                # `max_subs` is 4 and we're on the second subtraction attempt,
+                # products will equal
+                # [\, ", #, %, ...], [\, ", #, %, ...], (0,), (0,)
+                for products in product(
+                    *[x <= sub and vocab or (0,) for x in range(self.max_subs)]
+                ):
+                    # Sum up all the products, carry from last byte and
+                    # the target
+                    attempt = target[byte] + carry + sum(products)
+                    # If the attempt equals last, we've found the combination
+                    if last[byte] == attempt & 0xff:
+                        carry = (attempt & 0xff00) >> 8
+                        # Update the result with the current `products`
+                        for p, i in zip(products, range(sub + 1)):
+                            subtractions[i][byte] = p
+                        success_count += 1
+                        break
+            if success_count == int_size:
+                return subtractions
+            else:
+                subtractions.append(bytearray(int_size))
+        else:
+            raise ArithmeticError(
+                str.format(
+                    '''Could not find the correct subtraction sequence
+                to get the the desired target ({}) from ({})''',
+                    target[byte], last[byte]))
+
+
+encode = AsciiShellcodeEncoder()

pwnlib/encoders/i386/xor.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # Source:
 # http://www.iodigitalsec.com/python-cascading-xor-polymorphic-shellcode-generator/
 #

pwnlib/encoders/mips/xor.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # Source:
 # https://github.com/zcutlip/bowcaster/blob/master/src/bowcaster/encoders/mips.py
 #

pwnlib/fmtstr.py

@@ -851,7 +851,7 @@ class FmtStr(object):
         payload = b"START%%%d$pEND" % offset
         leak = self.execute_fmt(prefix + payload)
         try:
-            leak = re.findall(br"START(.*)END", leak, re.MULTILINE | re.DOTALL)[0]
+            leak = re.findall(br"START(.*?)END", leak, re.MULTILINE | re.DOTALL)[0]
             leak = int(leak, 16)
         except ValueError:
             leak = 0

pwnlib/gdb.py

@@ -241,9 +241,9 @@ def _gdbserver_args(pid=None, path=None, args=None, which=None, env=None):
         env_args = []
         for key in tuple(env):
             if key.startswith('LD_'): # LD_PRELOAD / LD_LIBRARY_PATH etc.
-                env_args.append('{}={}'.format(key, env.pop(key)))
+                env_args.append('{}="{}"'.format(key, env.pop(key)))
             else:
-                env_args.append('{}={}'.format(key, env[key]))
+                env_args.append('{}="{}"'.format(key, env[key]))
         gdbserver_args += ['--wrapper', 'env', '-i'] + env_args + ['--']
 
     gdbserver_args += ['localhost:0']
@@ -263,7 +263,10 @@ def _gdbserver_port(gdbserver, ssh):
             'Failed to spawn process under gdbserver. gdbserver error message: %s' % process_created
         )
 
-    gdbserver.pid   = int(process_created.split()[-1], 0)
+    try:
+        gdbserver.pid   = int(process_created.split()[-1], 0)
+    except ValueError:
+        log.error('gdbserver did not output its pid (maybe chmod +x?): %s', six.ensure_str(process_created))
 
     listening_on = b''
     while b'Listening' not in listening_on:

pwnlib/libcdb.py

@@ -46,7 +46,7 @@ def search_by_hash(hex_encoded_id, hash_type='build_id'):
             log.info_once("Using cached data from %r", cache)
             return cache
         else:
-            log.info_once("Skipping unavialable libc %s", hex_encoded_id)
+            log.info_once("Skipping unavailable libc %s", hex_encoded_id)
             return None
 
     # Build the URL using the requested hash type
@@ -186,7 +186,7 @@ def get_build_id_offsets():
     return {
     # $ check_arch 80386
     #     181 Displaying notes found at file offset 0x00000174 with length 0x00000024:
-        'i386': [0x174],
+        'i386': [0x174, 0x1b4, 0x1d4],
     # $ check_arch "ARM, EABI5"
     #      69 Displaying notes found at file offset 0x00000174 with length 0x00000024:
         'arm':  [0x174],
@@ -197,7 +197,7 @@ def get_build_id_offsets():
     # $ check_arch "x86-64"
     #       6 Displaying notes found at file offset 0x00000174 with length 0x00000024:
     #      82 Displaying notes found at file offset 0x00000270 with length 0x00000024:
-        'amd64': [0x270, 0x174, 0x2e0],
+        'amd64': [0x270, 0x174, 0x2e0, 0x370],
     # $ check_arch "PowerPC or cisco"
     #      88 Displaying notes found at file offset 0x00000174 with length 0x00000024:
         'powerpc': [0x174],

pwnlib/rop/rop.py

@@ -267,7 +267,7 @@ That's all there is to it.
     >>> print(rop.dump())
     0x0000:       0x1000000e pop eax; ret
     0x0004:             0x77 [arg0] eax = SYS_sigreturn
-    0x0008:       0x1000000b int 0x80
+    0x0008:       0x1000000b int 0x80; ret
     0x000c:              0x0 gs
     0x0010:              0x0 fs
     0x0014:              0x0 es
@@ -282,7 +282,7 @@ That's all there is to it.
     0x0038:              0xb eax = SYS_execve
     0x003c:              0x0 trapno
     0x0040:              0x0 err
-    0x0044:       0x1000000b int 0x80
+    0x0044:       0x1000000b int 0x80; ret
     0x0048:             0x23 cs
     0x004c:              0x0 eflags
     0x0050:              0x0 esp_at_signal
@@ -452,7 +452,7 @@ class ROP(object):
     0x0020:              0x3 arg0
     0x0024:       0x10000007 pop eax; ret
     0x0028:             0x77 [arg0] eax = SYS_sigreturn
-    0x002c:       0x10000000 int 0x80
+    0x002c:       0x10000000 int 0x80; ret
     0x0030:              0x0 gs
     0x0034:              0x0 fs
     0x0038:              0x0 es
@@ -467,7 +467,7 @@ class ROP(object):
     0x005c:              0xb eax = SYS_execve
     0x0060:              0x0 trapno
     0x0064:              0x0 err
-    0x0068:       0x10000000 int 0x80
+    0x0068:       0x10000000 int 0x80; ret
     0x006c:             0x23 cs
     0x0070:              0x0 eflags
     0x0074:              0x0 esp_at_signal
@@ -490,7 +490,7 @@ class ROP(object):
     0x8048020:              0x3 arg0
     0x8048024:       0x10000007 pop eax; ret
     0x8048028:             0x77 [arg0] eax = SYS_sigreturn
-    0x804802c:       0x10000000 int 0x80
+    0x804802c:       0x10000000 int 0x80; ret
     0x8048030:              0x0 gs
     0x8048034:              0x0 fs
     0x8048038:              0x0 es
@@ -505,7 +505,7 @@ class ROP(object):
     0x804805c:              0xb eax = SYS_execve
     0x8048060:              0x0 trapno
     0x8048064:              0x0 err
-    0x8048068:       0x10000000 int 0x80
+    0x8048068:       0x10000000 int 0x80; ret
     0x804806c:             0x23 cs
     0x8048070:              0x0 eflags
     0x8048074:              0x0 esp_at_signal
@@ -523,6 +523,14 @@ class ROP(object):
     >>> r.ret is None
     True
     """
+    BAD_ATTRS = [
+        'trait_names',          # ipython tab-complete
+        'download',             # frequent typo
+        'upload',               # frequent typo
+    ]
+    X86_SUFFIXES = ['ax', 'bx', 'cx', 'dx', 'bp', 'sp', 'di', 'si',
+                    'r8', 'r9', '10', '11', '12', '13', '14', '15']
+
     def __init__(self, elfs, base = None, badchars = b'', **kwargs):
         """
         Arguments:
@@ -661,6 +669,43 @@ class ROP(object):
 
         return stack
 
+    def __call__(self, *args, **kwargs):
+        """Set the given register(s)' by constructing a rop chain.
+
+        This is a thin wrapper around :meth:`setRegisters` which
+        actually executes the rop chain.
+
+        You can call this :class:`ROP` instance and provide keyword arguments,
+        or a dictionary.
+
+        Arguments:
+            regs(dict): Mapping of registers to values.
+                        Can instead provide ``kwargs``.
+
+        >>> context.clear(arch='amd64')
+        >>> assembly = 'pop rax; pop rdi; pop rsi; ret; pop rax; ret;'
+        >>> e = ELF.from_assembly(assembly)
+        >>> r = ROP(e)
+        >>> r(rax=0xdead, rdi=0xbeef, rsi=0xcafe)
+        >>> print(r.dump())
+        0x0000:       0x10000000
+        0x0008:           0xdead
+        0x0010:           0xbeef
+        0x0018:           0xcafe
+        >>> r = ROP(e)
+        >>> r({'rax': 0xdead, 'rdi': 0xbeef, 'rsi': 0xcafe})
+        >>> print(r.dump())
+        0x0000:       0x10000000
+        0x0008:           0xdead
+        0x0010:           0xbeef
+        0x0018:           0xcafe
+        """
+        if len(args) == 1 and isinstance(args[0], dict):
+            for value, _ in self.setRegisters(args[0]):
+                self.raw(value)
+        else:
+            self(kwargs)
+
     def resolve(self, resolvable):
         """Resolves a symbol to an address
 
@@ -703,6 +748,7 @@ class ROP(object):
         """
         Generates padding to be inserted into the ROP stack.
 
+        >>> context.clear(arch='i386')
         >>> rop = ROP([])
         >>> val = rop.generatePadding(5,15)
         >>> cyclic_find(val[:4])
@@ -925,17 +971,28 @@ class ROP(object):
     def find_stack_adjustment(self, slots):
         self.search(move=slots * context.bytes)
 
-    def chain(self):
+    def chain(self, base=None):
         """Build the ROP chain
+        
+        Arguments:
+            base(int):
+                The base address to build the rop-chain from. Defaults to
+                :attr:`base`.
 
         Returns:
             str containing raw ROP bytes
         """
-        return packing.flat(self.build())
+        return packing.flat(self.build(base=base))
 
-    def dump(self):
-        """Dump the ROP chain in an easy-to-read manner"""
-        return self.build().dump()
+    def dump(self, base=None):
+        """Dump the ROP chain in an easy-to-read manner
+        
+        Arguments:
+            base(int):
+                The base address to build the rop-chain from. Defaults to
+                :attr:`base`.
+        """
+        return self.build(base=base).dump()
 
     def regs(self, registers=None, **kw):
         if registers is None:
@@ -1050,6 +1107,7 @@ class ROP(object):
         Arguments:
             data(int/str): The raw value to put onto the rop chain.
 
+        >>> context.clear(arch='i386')
         >>> rop = ROP([])
         >>> rop.raw('AAAAAAAA')
         >>> rop.raw('BBBBBBBB')
@@ -1069,7 +1127,7 @@ class ROP(object):
     def migrate(self, next_base):
         """Explicitly set $sp, by using a ``leave; ret`` gadget"""
         if isinstance(next_base, ROP):
-            next_base = self.base
+            next_base = next_base.base
         pop_sp = self.rsp or self.esp
         pop_bp = self.rbp or self.ebp
         leave  = self.leave
@@ -1090,7 +1148,7 @@ class ROP(object):
 
     def __get_cachefile_name(self, files):
         """Given an ELF or list of ELF objects, return a cache file for the set of files"""
-        cachedir = os.path.join(tempfile.gettempdir(), 'pwntools-rop-cache-%d.%d' % sys.version_info[:2])
+        cachedir = os.path.join(context.cache_dir, 'rop-cache')
         if not os.path.exists(cachedir):
             os.mkdir(cachedir)
 
@@ -1106,7 +1164,7 @@ class ROP(object):
     @staticmethod
     def clear_cache():
         """Clears the ROP gadget cache"""
-        cachedir = os.path.join(tempfile.gettempdir(), 'pwntools-rop-cache')
+        cachedir = os.path.join(context.cache_dir, 'rop-cache')
         shutil.rmtree(cachedir)
 
     def __cache_load(self, elf):
@@ -1187,7 +1245,7 @@ class ROP(object):
             try:
                 sys.stdout = Wrapper(sys.stdout)
                 import ropgadget
-                sys.argv = ['ropgadget', '--binary', elf.path, '--only', 'sysenter|syscall|int|add|pop|leave|ret', '--nojop']
+                sys.argv = ['ropgadget', '--binary', elf.path, '--only', 'sysenter|syscall|int|add|pop|leave|ret', '--nojop', '--multibr']
                 args = ropgadget.args.Args().getArgs()
                 core = ropgadget.core.Core(args)
                 core.do_binary(elf.path)
@@ -1341,6 +1399,7 @@ class ROP(object):
         Also provides a shorthand for ``.call()``:
             ``rop.function(args)`` is equivalent to ``rop.call(function, args)``
 
+        >>> context.clear(arch='i386')
         >>> elf=ELF(which('bash'))
         >>> rop=ROP([elf])
         >>> rop.rdi     == rop.search(regs=['rdi'], order = 'regs')
@@ -1351,16 +1410,15 @@ class ROP(object):
         True
         >>> rop.ret is not None
         True
+        >>> with context.local(arch='amd64', bits='64'):
+        ...     r = ROP(ELF.from_assembly('syscall; ret'))
+        >>> r.syscall is not None
+        True
         """
         gadget = collections.namedtuple('gadget', ['address', 'details'])
-        bad_attrs = [
-            'trait_names',          # ipython tab-complete
-            'download',             # frequent typo
-            'upload',               # frequent typo
-        ]
 
         if attr in self.__dict__ \
-        or attr in bad_attrs \
+        or attr in self.BAD_ATTRS \
         or attr.startswith('_'):
             raise AttributeError('ROP instance has no attribute %r' % attr)
 
@@ -1397,17 +1455,14 @@ class ROP(object):
              'syscall': 'syscall',
              'sysenter': 'sysenter'}
             for each in self.gadgets:
-                if self.gadgets[each]['insns'] == [mapping[attr]]:
+                if self.gadgets[each]['insns'][0] == mapping[attr]:
                     return gadget(each, self.gadgets[each])
             return None
 
         #
         # Check for a '_'-delimited list of registers
         #
-        x86_suffixes = ['ax', 'bx', 'cx', 'dx', 'bp', 'sp', 'di', 'si',
-                        'r8', 'r9', '10', '11', '12', '13', '14', '15']
-
-        if all(map(lambda x: x[-2:] in x86_suffixes, attr.split('_'))):
+        if all(map(lambda x: x[-2:] in self.X86_SUFFIXES, attr.split('_'))):
             return self.search(regs=attr.split('_'), order='regs')
 
         #
@@ -1417,3 +1472,42 @@ class ROP(object):
             return self.call(attr, args)
 
         return call
+
+    def __setattr__(self, attr, value):
+        """Helper for setting registers.
+
+        This convenience feature allows one to set the values of registers
+        with simple python assignment syntax.
+
+        Warning:
+            Only one register is set at a time (one per rop chain).
+            This may lead to some previously set to registers be overwritten!
+
+        Note:
+            If you would like to set multiple registers in as few rop chains
+            as possible, see :meth:`__call__`.
+
+        >>> context.clear(arch='amd64')
+        >>> assembly = 'pop rax; pop rdi; pop rsi; ret; pop rax; ret;'
+        >>> e = ELF.from_assembly(assembly)
+        >>> r = ROP(e)
+        >>> r.rax = 0xdead
+        >>> r.rdi = 0xbeef
+        >>> r.rsi = 0xcafe
+        >>> print(r.dump())
+        0x0000:       0x10000004 pop rax; ret
+        0x0008:           0xdead
+        0x0010:       0x10000001 pop rdi; pop rsi; ret
+        0x0018:           0xbeef
+        0x0020:      b'iaaajaaa' <pad rsi>
+        0x0028:       0x10000002 pop rsi; ret
+        0x0030:           0xcafe
+        """
+        if attr in self.BAD_ATTRS:
+            raise AttributeError('ROP instance has no attribute %r' % attr)
+
+        if attr[-2:] in self.X86_SUFFIXES:  # handle setting registers
+            self({attr: value})
+
+        # Otherwise, perform usual setting
+        self.__dict__[attr] = value

pwnlib/rop/srop.py

@@ -240,8 +240,8 @@ stack_pointers = {
 # # XXX Need to add support for Capstone in order to extract ARM and MIPS
 # XXX as the SVC code may vary.
 syscall_instructions = {
-    'amd64': ['int 0x80', 'syscall', 'sysenter'],
-    'i386': ['int 0x80', 'syscall', 'sysenter'],
+    'amd64': ['syscall'],
+    'i386': ['int 0x80'],
     'arm': ['svc 0'],
     'aarch64': ['svc 0'],
     'thumb': ['svc 0'],

pwnlib/term/__init__.py

@@ -39,7 +39,7 @@ def can_init():
 
     # Check fancy REPLs
     mods = sys.modules.keys()
-    for repl in ['IPython', 'bpython', 'dreampielib']:
+    for repl in ['IPython', 'bpython', 'dreampielib', 'jupyter_client._version']:
         if repl in mods:
             return False
 

pwnlib/term/readline.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 from __future__ import division

pwnlib/term/unix_termcap.py

@@ -45,6 +45,14 @@ def get(cap, *args, **kwargs):
 def init():
     global cache
 
+    # Detect running under Jupyter
+    try:
+        if get_ipython().__class__.__name__ == 'ZMQInteractiveShell':
+            os.environ['PWNLIB_NOTERM'] = '1'
+            os.environ['JUPYTER_DETECTED'] ='yes'
+    except NameError:
+        pass
+
     if 'PWNLIB_NOTERM' not in os.environ:
         # Fix for BPython
         try:
@@ -52,6 +60,8 @@ def init():
         except curses.error as e:
             import traceback
             print('Warning:', ''.join(traceback.format_exception_only(e.__class__, e)), file=sys.stderr)
+            print('Terminal features will not be available.  Consider setting TERM variable to your current terminal name (or xterm).', file=sys.stderr)
+            os.environ['PWNLIB_NOTERM'] = '1'
 
     cache = {}
     # Manually add reset sequence into the cache.

pwnlib/timeout.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 """
 Timeout encapsulation, complete with countdowns and scope managers.

pwnlib/tubes/buffer.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 from __future__ import absolute_import
 from __future__ import division
 

pwnlib/tubes/remote.py

@@ -25,6 +25,8 @@ class remote(sock):
         typ: The string "tcp" or "udp" or an integer to pass to :func:`socket.getaddrinfo`.
         timeout: A positive number, None or the string "default".
         ssl(bool): Wrap the socket with SSL
+        ssl_context(ssl.SSLContext): Specify SSLContext used to wrap the socket.
+        sni: Set 'server_hostname' in ssl_args based on the host parameter.
         sock(socket.socket): Socket to inherit, rather than connecting
         ssl_args(dict): Pass ssl.wrap_socket named arguments in a dictionary.
 
@@ -56,7 +58,8 @@ class remote(sock):
 
     def __init__(self, host, port,
                  fam = "any", typ = "tcp",
-                 ssl=False, sock=None, ssl_args=None, *args, **kwargs):
+                 ssl=False, sock=None, ssl_context=None, ssl_args=None, sni=True,
+                 *args, **kwargs):
         super(remote, self).__init__(*args, **kwargs)
 
         self.rport  = int(port)
@@ -82,7 +85,13 @@ class remote(sock):
             self.lhost, self.lport = self.sock.getsockname()[:2]
 
             if ssl:
-                self.sock = _ssl.wrap_socket(self.sock,**(ssl_args or {}))
+                ssl_args = ssl_args or {}
+                ssl_context = ssl_context or _ssl.SSLContext(_ssl.PROTOCOL_TLSv1_2)
+                if isinstance(sni, str):
+                    ssl_args["server_hostname"] = sni
+                elif sni:
+                    ssl_args["server_hostname"] = host
+                self.sock = ssl_context.wrap_socket(self.sock,**ssl_args)
 
     def _connect(self, fam, typ):
         sock    = None

pwnlib/tubes/sock.py

@@ -48,6 +48,8 @@ class sock(tube):
                     raise EOFError
                 elif e.errno == errno.EINTR:
                     continue
+                elif 'timed out' in e.message:
+                    return None
                 else:
                     raise
 

pwnlib/tubes/ssh.py

@@ -69,26 +69,25 @@ class ssh_channel(sock):
         self.process = process
         self.cwd  = wd or '.'
         if isinstance(wd, six.text_type):
-            wd = wd.encode('utf-8')
+            wd = context._encode(wd)
 
         env = env or {}
         msg = 'Opening new channel: %r' % (process or 'shell')
 
         if isinstance(process, (list, tuple)):
-            process = b' '.join((lambda x:x.encode('utf-8') if isinstance(x, six.text_type) else x)(sh_string(s)) for s in process)
+            process = b' '.join(context._encode(sh_string(s)) for s in process)
         if isinstance(process, six.text_type):
-            process = process.encode('utf-8')
+            process = context._encode(process)
 
         if process and wd:
             process = b'cd ' + sh_string(wd) + b' >/dev/null 2>&1;' + process
 
         if process and env:
             for name, value in env.items():
-                if not re.match('^[a-zA-Z_][a-zA-Z0-9_]*$', name):
+                nameb = context._encode(name)
+                if not re.match(b'^[a-zA-Z_][a-zA-Z0-9_]*$', nameb):
                     self.error('run(): Invalid environment key %r' % name)
-                export = 'export %s=%s;' % (name, sh_string(value))
-                if isinstance(export, six.text_type):
-                    export = export.encode('utf-8')
+                export = b'export %s=%s;' % (nameb, sh_string(context._encode(value)))
                 process = export + process
 
         if process and tty:
@@ -264,11 +263,11 @@ class ssh_channel(sock):
                 if not data:
                     event.set()
                 else:
-                    data = [six.byte2int(data)]
+                    data = bytearray(data)
 
             if data:
                 try:
-                    self.send(b''.join(six.int2byte(c) for c in data))
+                    self.send(bytes(bytearray(data)))
                 except EOFError:
                     event.set()
                     self.info('Got EOF while sending in interactive')
@@ -1154,6 +1153,8 @@ os.execve(exe, argv, env)
             >>> py.sendline(b'exit')
             >>> print(repr(py.recvline()))
             b'4\n'
+            >>> s.system('env | grep -a AAAA', env={'AAAA': b'\x90'}).recvall()
+            b'AAAA=\x90\n'
         """
 
         if wd is None:
@@ -1429,7 +1430,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         total, exitcode = self.run_to_end(cmd)
 
         if exitcode != 0:
-            h.failure("%r does not exist or is not accessible" % remote)
+            h.error("%r does not exist or is not accessible" % remote)
             return
 
         total = int(total)
@@ -1479,7 +1480,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             self._download_raw(remote, local, p)
 
             if not self._verify_local_fingerprint(fingerprint):
-                p.failure('Could not download file %r' % remote)
+                p.error('Could not download file %r' % remote)
 
         return local
 

pwnlib/tubes/tube.py

@@ -32,10 +32,6 @@ class tube(Timeout, Logger):
     default = Timeout.default
     forever = Timeout.forever
 
-    #: Delimiter to use for :meth:`sendline`, :meth:`recvline`,
-    #: and related functions.
-    newline = b'\n'
-
     def __init__(self, timeout = default, level = None, *a, **kw):
         super(tube, self).__init__(timeout)
 
@@ -44,8 +40,35 @@ class tube(Timeout, Logger):
             self.setLevel(level)
 
         self.buffer = Buffer(*a, **kw)
+        self._newline = None
         atexit.register(self.close)
 
+    @property
+    def newline(self):
+        r'''Character sent with methods like sendline() or used for recvline().
+
+            >>> t = tube()
+            >>> t.newline = 'X'
+            >>> t.unrecv('A\nB\nCX')
+            >>> t.recvline()
+            b'A\nB\nCX'
+
+            >>> t = tube()
+            >>> context.newline = '\r\n'
+            >>> t.newline
+            b'\r\n'
+
+            # Clean up
+            >>> context.clear()
+        '''
+        if self._newline is not None:
+            return self._newline
+        return context.newline
+
+    @newline.setter
+    def newline(self, newline):
+        self._newline = six.ensure_binary(newline)
+
     # Functions based on functions from subclasses
     def recv(self, numb = None, timeout = default):
         r"""recv(numb = 4096, timeout = default) -> bytes
@@ -589,7 +612,7 @@ class tube(Timeout, Logger):
     def recvline_endswith(self, delims, keepends=False, timeout=default):
         r"""recvline_endswith(delims, keepends=False, timeout=default) -> bytes
 
-        Keep receiving lines until one is found that starts with one of
+        Keep receiving lines until one is found that ends with one of
         `delims`.  Returns the last line received.
 
         If the request is not satisfied before ``timeout`` seconds pass,

pwnlib/ui.py

@@ -43,6 +43,7 @@ atexception.register(lambda:os.kill(os.getppid(), signal.SIGUSR1))
         raise EOFError("process terminated with code: %r (%r)" % (p.poll(True), p.stderr.read()))
     fcntl.ioctl(p.stdout.fileno(), termios.TIOCSWINSZ, struct.pack("hh", 80, 80))
     p.stdout.write(b"\x1b[1;1R")
+    time.sleep(0.5)
     return p
 
 def yesno(prompt, default=None):
@@ -57,6 +58,10 @@ def yesno(prompt, default=None):
       `True` if the answer was "yes", `False` if "no"
 
     Examples:
+
+    .. doctest::
+       :skipif: branch_dev
+
         >>> yesno("A number:", 20)
         Traceback (most recent call last):
         ...
@@ -75,6 +80,10 @@ def yesno(prompt, default=None):
          [?] is it good 3 [yes/No] False
 
     Tests:
+
+    .. doctest::
+       :skipif: branch_dev
+
         >>> p = testpwnproc("print(yesno('is it ok??'))")
         >>> b"is it ok" in p.recvuntil("??")
         True
@@ -136,12 +145,20 @@ def options(prompt, opts, default = None):
       The users choice in the form of an integer.
 
     Examples:
+
+    .. doctest::
+       :skipif: branch_dev
+
         >>> options("Select a color", ("red", "green", "blue"), "green")
         Traceback (most recent call last):
         ...
         ValueError: options(): default must be a number or None
 
     Tests:
+
+    .. doctest::
+       :skipif: branch_dev
+
         >>> p = testpwnproc("print(options('select a color', ('red', 'green', 'blue')))")
         >>> p.sendline(b"\33[C\33[A\33[A\33[B\33[1;5A\33[1;5B 0310")
         >>> _ = p.recvall()
@@ -252,6 +269,10 @@ def pause(n=None):
     r"""Waits for either user input or a specific number of seconds.
 
     Examples:
+
+    .. doctest::
+       :skipif: branch_dev
+
         >>> with context.local(log_level="INFO"):
         ...     pause(1)
         [x] Waiting
@@ -263,6 +284,10 @@ def pause(n=None):
         ValueError: pause(): n must be a number or None
 
     Tests:
+
+    .. doctest::
+       :skipif: branch_dev
+
         >>> saved_stdin = sys.stdin
         >>> try:
         ...     sys.stdin = io.TextIOWrapper(io.BytesIO(b"\n"))
@@ -308,11 +333,16 @@ def more(text):
       :const:`None`
 
     Tests:
+
+    .. doctest::
+       :skipif: branch_dev
+       
         >>> more("text")
         text
         >>> p = testpwnproc("more('text\\n' * (term.height + 2))")
         >>> p.send(b"x")
-        >>> b"text" in p.recvall()
+        >>> data = p.recvall()
+        >>> b"text" in data or data
         True
     """
     if term.term_mode:

pwnlib/util/iters.py

@@ -843,6 +843,13 @@ def bruteforce(func, alphabet, length, method = 'upto', start = None, databag =
     h.failure('No matches found')
 
 
+def _mbruteforcewrap(func, alphabet, length, method, start, databag):
+    oldloglevel = context.log_level
+    context.log_level = 'critical'
+    res = bruteforce(func, alphabet, length, method=method, start=start, databag=databag)
+    context.log_level = oldloglevel
+    databag["result"] = res
+
 
 def mbruteforce(func, alphabet, length, method = 'upto', start = None, threads = None):
     """mbruteforce(func, alphabet, length, method = 'upto', start = None, threads = None)
@@ -864,13 +871,6 @@ def mbruteforce(func, alphabet, length, method = 'upto', start = None, threads =
       '9999'
     """
 
-    def bruteforcewrap(func, alphabet, length, method, start, databag):
-        oldloglevel = context.log_level
-        context.log_level = 'critical'
-        res = bruteforce(func, alphabet, length, method=method, start=start, databag=databag)
-        context.log_level = oldloglevel
-        databag["result"] = res
-
     if start is None:
         start = (1, 1)
 
@@ -896,7 +896,7 @@ def mbruteforce(func, alphabet, length, method = 'upto', start = None, threads =
 
         chunkid = (i2-1) + (i * N2) + 1
 
-        processes[i] = multiprocessing.Process(target=bruteforcewrap,
+        processes[i] = multiprocessing.Process(target=_mbruteforcewrap,
                 args=(func, alphabet, length, method, (chunkid, totalchunks),
                         shareddata[i]))
         processes[i].start()

pwnlib/util/packing.py

@@ -615,10 +615,10 @@ def flat(*args, **kwargs):
 
     Dictionary keys give offsets at which to place the corresponding values
     (which are recursively flattened).  Offsets are relative to where the
-    flattened dictionary occurs in the output (i.e. `{0: 'foo'}` is equivalent
-    to `'foo'`).  Offsets can be integers, unicode strings or regular strings.
+    flattened dictionary occurs in the output (i.e. ``{0: 'foo'}`` is equivalent
+    to ``'foo'``).  Offsets can be integers, unicode strings or regular strings.
     Integer offsets >= ``2**(word_size-8)`` are converted to a string using
-    `:func:pack`.  Unicode strings are UTF-8 encoded.  After these conversions
+    :func:`pack`.  Unicode strings are UTF-8 encoded.  After these conversions
     offsets are either integers or strings.  In the latter case, the offset will
     be the lowest index at which the string occurs in `filler`.  See examples
     below.
@@ -837,8 +837,8 @@ def dd(dst, src, count = 0, skip = 0, seek = 0, truncate = False):
     The seek offset of file objects will be preserved.
 
     Arguments:
-        dst: Supported types are `:class:file`, `:class:list`, `:class:tuple`,
-             `:class:str`, `:class:bytearray` and `:class:unicode`.
+        dst: Supported types are :class:`file`, :class:`list`, :class:`tuple`,
+             :class:`str`, :class:`bytearray` and :class:`unicode`.
         src: An iterable of byte values (characters or integers), a unicode
              string or a file object.
         count (int): How many bytes to copy.  If `count` is 0 or larger than
@@ -846,7 +846,7 @@ def dd(dst, src, count = 0, skip = 0, seek = 0, truncate = False):
                      copied.
         skip (int): Offset in `dst` to copy to.
         seek (int): Offset in `src` to copy from.
-        truncate (bool): If `:const:True`, `dst` is truncated at the last copied
+        truncate (bool): If :const:`True`, `dst` is truncated at the last copied
                          byte.
 
     Returns:

pwnlib/util/sh_string.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python2
 # -*- coding: utf-8 -*-
 r"""
 Routines here are for getting any NULL-terminated sequence of bytes evaluated

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '4.3.1'
+__version__ = '4.4.0'

setup.py

@@ -83,7 +83,7 @@ setup(
     name                 = 'pwntools',
     python_requires      = '>=2.7',
     packages             = find_packages(),
-    version              = '4.3.1',
+    version              = '4.4.0',
     data_files           = [('',
                              glob.glob('*.md') + glob.glob('*.txt')),
                             ],
@@ -118,6 +118,7 @@ setup(
         'Natural Language :: English',
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
         'Topic :: Security',
         'Topic :: Software Development :: Assemblers',
         'Topic :: Software Development :: Debuggers',