Fix features in pack() and add more doctests (#1610)

* [packing] Better alternative to #1580

pack() was failing for dictionaries that are not ordered

Closes #1582
Closes #1579

* [docs] Add better documentation for flat()

* [changelog] Add 4.1.4

* [release] Bump version to 4.1.4

Co-authored-by: Arusekk <[email protected]>

.github/PULL_REQUEST_TEMPLATE.md

@@ -25,3 +25,8 @@ You can always [change the branch][change] after you create the PR if it's again
 [contributing]: https://github.com/Gallopsled/pwntools/blob/dev/CONTRIBUTING.md
 [testing]: https://github.com/Gallopsled/pwntools/blob/dev/TESTING.md
 [change]: https://github.com/blog/2224-change-the-base-branch-of-a-pull-request
+
+## Changelog
+
+After creating your Pull Request, please add and push a commit that updates the changelog for the appropriate branch.  
+You can look at the existing changelog for examples of how to do this.

.github/workflows/changelog.yml

@@ -0,0 +1,20 @@
+name: Changelog
+on: [pull_request]
+env:
+  GITHUB_BASE: origin/${{ github.event.pull_request.base.ref }}
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 5
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Check changelog for updates
+      run: |
+        git log --stat ${GITHUB_BASE}..HEAD | grep CHANGELOG.md

.github/workflows/ci.yml

@@ -8,6 +8,7 @@ jobs:
         python-version: [2.7, 3.8]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
     steps:
     - uses: actions/checkout@v2
       with:
@@ -28,11 +29,18 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Lint
+    - name: Verify tag against version
+      if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
+      env:
+        GITHUB_REF: ${{ github.event.ref }}
       run: |
-        pip install flake8
-        flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=pwnlib/constants,android-?dk,.git,__pycache__
+        set -x
+        GITHUB_TAG=${GITHUB_REF#refs/tags/}
+        echo "$GITHUB_TAG" | grep -E '^[0-9.]*(beta[0-9])?$'
+        vsetup=$(grep -o "version\\s*=\\s*[\"'].*[\"'],$" setup.py | grep -o "[0-9][^\"']*")
+        vpwnlib=$(grep -o "__version__\\s*=\\s*[\"'].*[\"']$" pwnlib/version.py | grep -o "[0-9][^\"']*")
+        [ "$vsetup" = "$vpwnlib" ]
+        [ "$GITHUB_TAG" = "$vsetup" ]
 
     - name: Detect whether java is needed
       id: java-needed
@@ -67,10 +75,8 @@ jobs:
           binutils-mips-linux-gnu \
           binutils-powerpc-linux-gnu \
           gcc-multilib \
+          libc6-dbg \
           $ANDROID_JRE
-        sudo apt-get install -y -o Acquire::Retries=3 \
-          gcc-aarch64-linux-gnu \
-          gcc-arm-linux-gnueabihf
 
     - name: Install android avd
       if:  steps.java-needed.outputs.need
@@ -98,7 +104,6 @@ jobs:
     - name: Disable broken tests
       run: |
         rm -f docs/source/elf/corefile.rst
-        rm -f docs/source/ui.rst
 
     - name: Manually install non-broken Unicorn
       run:  pip install unicorn==1.0.2rc3
@@ -123,26 +128,45 @@ jobs:
     - name: Coverage running commandline tools
       run: |
         export TERM=linux
-        pwn() { coverage run -m pwnlib.commandline.main "$@"; }
+        pwn() { ( set +x; cmd=$1; shift; PYTHONUNBUFFERED=1 exec coverage run -m pwnlib.commandline."$cmd" "$@" ) }
         set -x
 
         pwn cyclic 32
         pwn cyclic -l 0x62616161
+        pwn cyclic -a ab
+        echo
 
+        pwn shellcraft --list |tail
+        pwn shellcraft -l --syscalls |tail
+        pwn shellcraft -l execve
+        pwn shellcraft --show i386.linux.loader_append
         pwn shellcraft -f asm --color amd64.linux.sh
+        pwn shellcraft -f elf amd64.linux.syscalls.exit 0 </dev/null |pwn hex
+        pwn shellcraft -f i   --color amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -f c   amd64.linux.syscalls.exit 0 </dev/null
+        pwn shellcraft -f str aarch64.linux.sh </dev/null
+        pwn shellcraft -abr -f elf -o /dev/null amd64.linux.cat /etc/passwd </dev/null
+        pwn shellcraft -nzr thumb.linux.syscalls.execve /bin/cat '["/bin/cat", "/etc/os-release"]' </dev/null
+        pwn shellcraft -fp aarch64.trap
 
         pwn disasm --color ff3424c3ebfe
+        pwn asm -f hex nop
 
         pwn hex abcd
         pwn unhex 4141 4141
 
-        head -c256 /dev/urandom | pwn phd
+        cat /dev/urandom | pwn phd --color -c 256 -s 2
+        pwn phd -l 0x3d --color=always /etc/os-release
+
+        pwn checksec /bin/bash
 
         pwn errno 2
+        pwn errno -1
+        pwn errno EADDRINUSE
 
         pwn constgrep -c freebsd -m ^PROT_ '3 + 4'
         pwn constgrep ^MAP_ 0
-        pwn constgrep ^O_
+        pwn constgrep -e O_RDWR
 
     - name: Build source and wheel distributions
       run: |

.github/workflows/lint.yml

@@ -0,0 +1,33 @@
+name: Lint
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        python-version: [3.8]
+        os: [ubuntu-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Cache for pip
+      uses: actions/cache@v1
+      id: cache-pip
+      with:
+        path: ~/.cache/pip
+        key: ${{ matrix.os }}-cache-pip
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Lint
+      run: |
+        pip install flake8
+        flake8 . --count --select=E9,F63,F7 --show-source --statistics --exclude=android-?dk  # TODO: Add F82
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude=pwnlib/constants,android-?dk,.git,__pycache__

.travis.yml

@@ -20,10 +20,7 @@ addons:
     - binutils-arm-linux-gnueabihf
     - binutils-mips-linux-gnu
     - binutils-powerpc-linux-gnu
-    - gcc-aarch64-linux-gnu
-    - gcc-arm-linux-gnueabihf
     - bash-static
-
 cache:
     - pip
     - directories:

CHANGELOG.md

@@ -66,7 +66,26 @@ To be released on Jun 5, 2020.
 [fecf9f]: http://github.com/Gallopsled/pwntools/commit/fecf9f
 [1454]: https://github.com/Gallopsled/pwntools/pull/1454
 
-## 4.1.1 (`stable`)
+## 4.1.4 (`stable`)
+
+- [#1698][1609] Fix issues in `packing.flat` with mis-ordred fields
+
+[1609]: https://github.com/Gallopsled/pwntools/pull/1609
+
+## 4.1.3
+
+- [#1590][1590] Fix `gdb.attach()` for `remote`, `listen`, `ssh` tubes
+  - Also fix `run_in_new_terminal` for Py2 unicode strings
+- [#1595][1595] Fix ssh.process(timeout=)
+
+[1590]: https://github.com/Gallopsled/pwntools/pull/1590
+[1595]: https://github.com/Gallopsled/pwntools/pull/1595
+
+## 4.1.2
+
+- Pwntools requires `six` v.1.12.0 or higher
+
+## 4.1.1
 
 - Fix PLT resolution by locking unicorn <1.0.2rc4 (#1538)
 - Fix wrong ELF/context unpack handling (c4c11a37)

docs/source/install/binutils.rst

@@ -39,7 +39,7 @@ repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
 .. code-block:: bash
 
-    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/osx/binutils-$ARCH.rb
+    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
 
 Alternate OSes
 ^^^^^^^^^^^^^^^^

pwnlib/commandline/asm.py

@@ -5,6 +5,9 @@ from __future__ import division
 import argparse
 import sys
 
+import pwnlib
+pwnlib.args.free_form = False
+
 from pwn import *
 from pwnlib.commandline import common
 

pwnlib/commandline/cyclic.py

@@ -43,7 +43,7 @@ parser.add_argument(
     help = 'The os/architecture/endianness/bits the shellcode will run in (default: linux/i386), choose from: %s' % common.choices,
 )
 
-group = parser.add_mutually_exclusive_group(required = True)
+group = parser.add_mutually_exclusive_group(required=False)
 group.add_argument(
     '-l', '-o', '--offset', '--lookup',
     dest = 'lookup',
@@ -53,9 +53,10 @@ group.add_argument(
 
 group.add_argument(
     'count',
-    type = int,
-    nargs = '?',
-    help = 'Number of characters to print'
+    type=int,
+    nargs='?',
+    default=None,
+    help='Number of characters to print'
 )
 
 def main(args):
@@ -90,7 +91,7 @@ def main(args):
         want   = args.count
         result = cyclic(want, alphabet, subsize)
         got    = len(result)
-        if got < want:
+        if want is not None and got < want:
             log.failure("Alphabet too small (max length = %i)" % got)
 
         out = getattr(sys.stdout, 'buffer', sys.stdout)

pwnlib/commandline/errno.py

@@ -45,4 +45,4 @@ def main(args):
   print(os.strerror(value))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/hex.py

@@ -26,4 +26,4 @@ def main(args):
         print(enhex(data))
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/commandline/phd.py

@@ -85,12 +85,12 @@ def main(args):
     text.when = color
 
     if skip:
-        if infile == sys.stdin:
-            infile.read(skip)
-        else:
+        try:
             infile.seek(skip, os.SEEK_CUR)
+        except IOError:
+            infile.read(skip)
 
-    if count:
+    if count != -1:
         infile = io.BytesIO(infile.read(count))
 
     hl = []

pwnlib/commandline/unhex.py

@@ -30,4 +30,4 @@ def main(args):
         sys.stderr.write(str(e) + '\n')
 
 if __name__ == '__main__':
-    pwnlib.commandline.common.main(__file__)
+    common.main(__file__)

pwnlib/data/elf/__init__.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
+from pwnlib.data.elf import fmtstr
 from pwnlib.data.elf import relro
 
 import os

pwnlib/data/elf/fmtstr/Makefile

@@ -0,0 +1,5 @@
+
+all: i386
+
+i386: vuln.c
+	gcc "$^" -fno-stack-protector -no-pie -m32 -o $@

pwnlib/data/elf/fmtstr/__init__.py

@@ -0,0 +1,5 @@
+import os
+path = os.path.dirname(__file__)
+
+def get(x):
+    return os.path.join(path, x)

pwnlib/data/elf/fmtstr/vuln.c

@@ -0,0 +1,25 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#define MEMORY_ADDRESS ((void*)0x11111000)
+#define MEMORY_SIZE 1024
+#define TARGET ((int *) 0x11111110)
+int main(int argc, char const *argv[])
+{
+       char buff[1024];
+       void *ptr = NULL;
+       int *my_var = TARGET;
+       ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+       if(ptr != MEMORY_ADDRESS)
+       {
+               perror("mmap");
+               return EXIT_FAILURE;
+       }
+       *my_var = 0x41414141;
+       write(1, &my_var, sizeof(int *));
+       scanf("%s", buff);
+       dprintf(2, buff);
+       write(1, my_var, sizeof(int));
+       return 0;
+}

pwnlib/fmtstr.py

@@ -1,39 +1,39 @@
 r"""
 Provide some tools to exploit format string bug
 
-Examples:
+Let's use this program as an example:
 
-    >>> program = tempfile.mktemp()
-    >>> source  = program + ".c"
-    >>> write(source, '''
-    ... #include <stdio.h>
-    ... #include <stdlib.h>
-    ... #include <unistd.h>
-    ... #include <sys/mman.h>
-    ... #define MEMORY_ADDRESS ((void*)0x11111000)
-    ... #define MEMORY_SIZE 1024
-    ... #define TARGET ((int *) 0x11111110)
-    ... int main(int argc, char const *argv[])
-    ... {
-    ...        char buff[1024];
-    ...        void *ptr = NULL;
-    ...        int *my_var = TARGET;
-    ...        ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
-    ...        if(ptr != MEMORY_ADDRESS)
-    ...        {
-    ...                perror("mmap");
-    ...                return EXIT_FAILURE;
-    ...        }
-    ...        *my_var = 0x41414141;
-    ...        write(1, &my_var, sizeof(int *));
-    ...        scanf("%s", buff);
-    ...        dprintf(2, buff);
-    ...        write(1, my_var, sizeof(int));
-    ...        return 0;
-    ... }''')
-    >>> gcc = process(["gcc", source, "-Wno-format-security", "-m32", "-o", program])
-    >>> gcc.poll(True) and gcc.recvall()
-    0
+::
+
+    #include <stdio.h>
+    #include <stdlib.h>
+    #include <unistd.h>
+    #include <sys/mman.h>
+    #define MEMORY_ADDRESS ((void*)0x11111000)
+    #define MEMORY_SIZE 1024
+    #define TARGET ((int *) 0x11111110)
+    int main(int argc, char const *argv[])
+    {
+           char buff[1024];
+           void *ptr = NULL;
+           int *my_var = TARGET;
+           ptr = mmap(MEMORY_ADDRESS, MEMORY_SIZE, PROT_READ|PROT_WRITE, MAP_FIXED|MAP_ANONYMOUS|MAP_PRIVATE, 0, 0);
+           if(ptr != MEMORY_ADDRESS)
+           {
+                   perror("mmap");
+                   return EXIT_FAILURE;
+           }
+           *my_var = 0x41414141;
+           write(1, &my_var, sizeof(int *));
+           scanf("%s", buff);
+           dprintf(2, buff);
+           write(1, my_var, sizeof(int));
+           return 0;
+    }
+
+We can automate the exploitation of the process like so:
+
+    >>> program = pwnlib.data.elf.fmtstr.get('i386')
     >>> def exec_fmt(payload):
     ...     p = process(program)
     ...     p.sendline(payload)

pwnlib/gdb.py

@@ -110,6 +110,7 @@ from pwnlib.asm import make_elf_from_assembly
 from pwnlib.context import LocalContext
 from pwnlib.context import context
 from pwnlib.log import getLogger
+from pwnlib.timeout import Timeout
 from pwnlib.util import misc
 from pwnlib.util import proc
 
@@ -545,68 +546,63 @@ def attach(target, gdbscript = '', exe = None, need_ptrace_scope = True, gdb_arg
 
     Examples:
 
-    >>> # Attach directly to pid 1234
-    >>> gdb.attach(1234) # doctest: +SKIP
 
+        Attach to a process by PID
 
-    >>> # Attach to the youngest "bash" process
-    >>> gdb.attach('bash') # doctest: +SKIP
+        >>> pid = gdb.attach(1234) # doctest: +SKIP
 
+        Attach to the youngest process by name
 
-    >>> # Start a process
-    >>> bash = process('bash')
+        >>> pid = gdb.attach('bash') # doctest: +SKIP
 
-    >>> # Attach the debugger
-    >>> pid = gdb.attach(bash, '''
-    ... set follow-fork-mode child
-    ... break execve
-    ... continue
-    ... ''')
+        Attach a debugger to a :class:`.process` tube and automate interaction
 
-    >>> # Interact with the process
-    >>> bash.sendline("/bin/echo hello")
-    >>> bash.recvline()
-    b'hello\n'
-    >>> bash.close()
+        >>> io = process('bash')
+        >>> pid = gdb.attach(io, gdbscript='''
+        ... call puts("Hello from process debugger!")
+        ... detach
+        ... quit
+        ... ''')
+        >>> io.recvline()
+        b'Hello from process debugger!\n'
+        >>> io.sendline('echo Hello from bash && exit')
+        >>> io.recvall()
+        b'Hello from bash\n'
 
-    >>> # Start a forking server
-    >>> server = process(['socat', 'tcp-listen:12345,fork,reuseaddr', 'exec:/bin/bash,nofork'])
-    >>> sleep(1)
+        Attach to the remote process from a :class:`.remote` or :class:`.listen` tube,
+        as long as it is running on the same machine.
 
-    >>> # Connect to the server
-    >>> io = remote('127.0.0.1', 12345)
+        >>> server = process(['socat', 'tcp-listen:12345,reuseaddr,fork', 'exec:/bin/bash,nofork'])
+        >>> sleep(1) # Wait for socat to start
+        >>> io = remote('127.0.0.1', 12345)
+        >>> sleep(1) # Wait for process to fork
+        >>> pid = gdb.attach(io, gdbscript='''
+        ... call puts("Hello from remote debugger!")
+        ... detach
+        ... quit
+        ... ''')
+        >>> io.recvline()
+        b'Hello from remote debugger!\n'
+        >>> io.sendline('echo Hello from bash && exit')
+        >>> io.recvall()
+        b'Hello from bash\n'
 
-    >>> # Connect the debugger to the server-spawned process
-    >>> pid = gdb.attach(io, '''
-    ... break exit
-    ... continue
-    ... ''', exe = '/bin/bash')
-
-    >>> # Talk to the spawned 'sh'
-    >>> io.sendline("echo hello")
-    >>> io.recvline()
-    b'hello\n'
-    >>> io.sendline("exit")
-
-    >>> io.close()
-
-    >>> # Connect to the SSH server
-    >>> shell = ssh('travis', 'example.pwnme', password='demopass')
-
-    >>> # Start a process on the server
-    >>> cat = shell.process(['cat'])
-
-    >>> # Attach a debugger to it
-    >>> gdb.attach(cat, '''
-    ... break exit
-    ... continue
-    ... ''')
-
-    >>> cat.sendline("hello")
-    >>> cat.recvline()
-    b'hello\n'
-    >>> # Cause `cat` to exit
-    >>> cat.close()
+        Attach to processes running on a remote machine via an SSH :class:`.ssh` process
+        
+        >>> shell = ssh('travis', 'example.pwnme', password='demopass')
+        >>> io = shell.process(['cat'])
+        >>> pid = gdb.attach(io, gdbscript='''
+        ... call sleep(5)
+        ... call puts("Hello from ssh debugger!")
+        ... detach
+        ... quit
+        ... ''')
+        >>> io.recvline(timeout=5)  # doctest: +SKIP
+        b'Hello from ssh debugger!\n'
+        >>> io.sendline('This will be echoed back')
+        >>> io.recvline()
+        b'This will be echoed back\n'
+        >>> io.close()
     """
     if context.noptrace:
         log.warn_once("Skipping debug attach since context.noptrace==True")
@@ -687,17 +683,24 @@ def attach(target, gdbscript = '', exe = None, need_ptrace_scope = True, gdb_arg
     elif isinstance(target, tubes.sock.sock):
         pids = proc.pidof(target)
         if not pids:
-            log.error('could not find remote process (%s:%d) on this machine' %
+            log.error('Could not find remote process (%s:%d) on this machine' %
                       target.sock.getpeername())
-        waiting = True
-        if exe:
-            while waiting:
-                for pid in pids:
-                    if proc.exe(pid) == exe:
-                        waiting = False
-                        break
-                else:
-                    time.sleep(0.01)
+        pid = pids[0]
+
+        # Specifically check for socat, since it has an intermediary process
+        # if you do not specify "nofork" to the EXEC: argument
+        # python(2640)───socat(2642)───socat(2643)───bash(2644)
+        if proc.exe(pid).endswith('/socat') and time.sleep(0.1) and proc.children(pid):
+            pid = proc.children(pid)[0]
+
+        # We may attach to the remote process after the fork but before it performs an exec.  
+        # If an exe is provided, wait until the process is actually running the expected exe
+        # before we attach the debugger.
+        t = Timeout()
+        with t.countdown(2):
+            while exe and os.realpath(proc.exe(pid)) != os.realpath(exe) and t.timeout:
+                time.sleep(0.1)
+
     elif isinstance(target, tubes.process.process):
         pid = proc.pidof(target)[0]
         exe = exe or target.executable
@@ -945,13 +948,23 @@ def find_module_addresses(binary, ssh=None, ulimit=False):
     return rv
 
 def corefile(process):
-    r"""Drops a core file for the process.
+    r"""Drops a core file for a running local process.
+
+    Note:
+        You should use :meth:`.process.corefile` instead of using this method directly.
 
     Arguments:
         process: Process to dump
 
     Returns:
         :class:`.Core`: The generated core file
+
+    Example:
+
+        >>> io = process('bash')
+        >>> core = gdb.corefile(io)
+        >>> core.exe.name
+        '/bin/bash'
     """
 
     if context.noptrace:

pwnlib/gdb_faketerminal.py

@@ -3,7 +3,10 @@ from pwnlib.tubes.process import process
 from time import sleep
 from sys import argv
 sleep(1)
-sh = process(argv[1], shell=True)
+if len(argv) == 2:
+	sh = process(argv[1], shell=True)
+else:
+	sh = process(argv[1:])
 sh.sendline('set prompt (gdb)')
 res = sh.sendlineafter('(gdb)', 'c')
 while b'The program is not being run.' not in res:

pwnlib/shellcraft/templates/aarch64/linux/loader_append.asm

@@ -17,18 +17,11 @@ Example:
 The following doctest is commented out because it doesn't work on Travis
 for reasons I cannot diagnose.  However, it should work just fine :-)
 
-    >>> gcc = process(['aarch64-linux-gnu-gcc','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    >>> gcc.write(b'''
-    ... #include <stdio.h>
-    ... int main() {
-    ...     printf("Hello, %s!\\n", "world");
-    ... }
-    ... ''')
-    >>> gcc.shutdown('send')
-    >>> gcc.poll(True) and gcc.recvall()
-    0
-    >>> sc = shellcraft.loader_append('a.out')
-    >>> run_assembly(sc).recvline()
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
     b'Hello, world!\n'
 
 </%docstring>
@@ -47,7 +40,7 @@ ${there}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>

pwnlib/shellcraft/templates/amd64/linux/loader_append.asm

@@ -14,23 +14,15 @@ Arguments:
 
 Example:
 
-    >>> gcc = process(['gcc','-m64','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    >>> gcc.write(b'''
-    ... #include <stdio.h>
-    ... int main() {
-    ...     printf("Hello, %s!\\n", "amd64");
-    ... }
-    ... ''')
-    >>> gcc.shutdown('send')
-    >>> gcc.poll(True) and gcc.recvall()
-    0
-    >>> sc = shellcraft.loader_append('a.out')
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> payloadELF.arch
+    'amd64'
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
+    b'Hello, world!\n'
 
-The following doctest is commented out because it doesn't work on Travis
-for reasons I cannot diagnose.  However, it should work just fine :-)
-
-    >>> run_assembly(sc).recvline() == b'Hello, amd64!\n'
-    True
 
 </%docstring>
 <%page args="data = None"/>
@@ -48,7 +40,7 @@ ${elf_data}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>

pwnlib/shellcraft/templates/i386/linux/loader_append.asm

@@ -14,23 +14,14 @@ Arguments:
 
 Example:
 
-    >>> gcc = process(['gcc','-m32','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    >>> gcc.write(b'''
-    ... #include <stdio.h>
-    ... int main() {
-    ...     printf("Hello, %s!\\n", "i386");
-    ... }
-    ... ''')
-    >>> gcc.shutdown('send')
-    >>> gcc.poll(True) and gcc.recvall()
-    0
-    >>> sc = shellcraft.loader_append('a.out')
-
-The following doctest is commented out because it doesn't work on Travis
-for reasons I cannot diagnose.  However, it should work just fine :-)
-
-    >>> run_assembly(sc).recvline()
-    b'Hello, i386!\n'
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> payloadELF.arch
+    'i386'
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
+    b'Hello, world!\n'
 
 </%docstring>
 <%page args="data = None"/>
@@ -48,7 +39,7 @@ ${elf_data}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>

pwnlib/shellcraft/templates/thumb/linux/loader_append.asm

@@ -17,20 +17,16 @@ Example:
 The following doctest is commented out because it doesn't work on Travis
 for reasons I cannot diagnose.  However, it should work just fine :-)
 
-    >>> gcc = process(['arm-linux-gnueabihf-gcc','-xc','-static','-Wl,-Ttext-segment=0x20000000','-'])
-    >>> gcc.write(b'''
-    ... #include <stdio.h>
-    ... int main() {
-    ...     printf("Hello, %s!\\n", "world");
-    ... }
-    ... ''')
-    >>> gcc.shutdown('send')
-    >>> gcc.poll(True) and gcc.recvall()
-    0
-    >>> sc = shellcraft.loader_append('a.out')
-    >>> run_assembly(sc).recvline()
+    >>> payload = shellcraft.echo(b'Hello, world!\n') + shellcraft.exit(0)
+    >>> payloadELF = ELF.from_assembly(payload)
+    >>> payloadELF.arch
+    'arm'
+    >>> loader = shellcraft.loader_append(payloadELF.data)
+    >>> loaderELF = ELF.from_assembly(loader, vma=0, shared=True)
+    >>> loaderELF.process().recvall()
     b'Hello, world!\n'
 
+
 </%docstring>
 <%page args="data = None"/>
 <%
@@ -45,7 +41,7 @@ ${there}:
 <%
 import os
 
-if os.path.isfile(data):
+if b'\x00' not in data and os.path.isfile(data):
     with open(data, 'rb') as f:
         data = f.read()
 %>

pwnlib/tubes/ssh.py

@@ -832,6 +832,10 @@ class ssh(Timeout, Logger):
 
             >>> s.process('echo hello', shell=True).recvall()
             b'hello\n'
+
+            >>> io = s.process(['cat'], timeout=5)
+            >>> io.recvline()
+            b''
         """
         if not argv and not executable:
             self.error("Must specify argv or executable")
@@ -1044,11 +1048,14 @@ os.execve(exe, argv, env)
 
         msg = 'Starting remote process %s on %s' % (execve_repr, self.host)
 
+        if timeout == Timeout.default:
+            timeout = self.timeout
+
         with self.progress(msg) as h:
 
             script = 'for py in python2.7 python2 python; do test -x "$(which $py 2>&1)" && exec $py -c %s check; done; echo 2' % sh_string(script)
-            with context.local(log_level='error'):
-                python = ssh_process(self, script, tty=True, raw=True, level=self.level, timeout=self.timeout)
+            with context.quiet:
+                python = ssh_process(self, script, tty=True, raw=True, level=self.level, timeout=timeout)
 
             try:
                 result = safeeval.const(python.recvline())

pwnlib/util/misc.py

@@ -5,6 +5,7 @@ import errno
 import os
 import platform
 import re
+import six
 import socket
 import stat
 import string
@@ -238,7 +239,7 @@ def run_in_new_terminal(command, terminal = None, args = None):
 
     argv = [which(terminal)] + args
 
-    if isinstance(command, str):
+    if isinstance(command, six.string_types):
         if ';' in command:
             log.error("Cannot use commands with semicolon.  Create a script and invoke that directly.")
         argv += [command]

pwnlib/util/packing.py

@@ -33,6 +33,7 @@ Examples:
 from __future__ import absolute_import
 from __future__ import division
 
+import collections
 import six
 import struct
 import sys
@@ -491,9 +492,11 @@ def _fit(pieces, preprocessor, packer, filler):
     pad = bytearray()
     def fill(key):
         key = bytearray(key)
-        while len(pad) < len(key) or pad[-len(key):] != key:
+        offset = pad.find(key)
+        while offset == -1:
             pad.append(next(filler))
-        return len(pad) - len(key)
+            offset = pad.find(key, -len(key))
+        return offset
 
     # Key conversion:
     # - convert str/unicode keys to offsets
@@ -614,26 +617,101 @@ def flat(*args, **kwargs):
       sign (str): Signedness of the converted integer (False/True)
 
     Examples:
-      >>> flat(1, "test", [[["AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
-      b'\x01\x00testABABABABABAB'
-      >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
-      b'234'
-      >>> flat({12: 0x41414141,
-      ...       24: 'Hello',
-      ...      })
-      b'aaaabaaacaaaAAAAeaaafaaaHello'
-      >>> flat({'caaa': ''})
-      b'aaaabaaa'
-      >>> flat({12: 'XXXX'}, filler = (ord('A'), ord('B')), length = 20)
-      b'ABABABABABABXXXXABAB'
-      >>> flat({ 8: [0x41414141, 0x42424242],
-      ...       20: 'CCCC'})
-      b'aaaabaaaAAAABBBBeaaaCCCC'
-      >>> flat({ 0x61616162: 'X'})
-      b'aaaaX'
-      >>> flat({4: {0: 'X', 4: 'Y'}})
-      b'aaaaXaaaY'
 
+        (Test setup, please ignore)
+    
+        >>> context.clear()
+
+        Basic usage of :meth:`flat` works similar to the pack() routines.
+
+        >>> flat(4)
+        b'\x04\x00\x00\x00'
+
+        :meth:`flat` works with strings, bytes, lists, and dictionaries.
+
+        >>> flat(b'X')
+        b'X'
+        >>> flat([1,2,3])
+        b'\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00'
+        >>> flat({4:'X'})
+        b'aaaaX'
+
+        :meth:`.flat` flattens all of the values provided, and allows nested lists
+        and dictionaries.
+
+        >>> flat([{4:'X'}] * 2)
+        b'aaaaXaaacX'
+        >>> flat([[[[[[[[[1]]]], 2]]]]])
+        b'\x01\x00\x00\x00\x02\x00\x00\x00'
+
+        You can also provide additional arguments like endianness, word-size, and
+        whether the values are treated as signed or not.
+
+        >>> flat(1, "test", [[["AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
+        b'\x01\x00testABABABABABAB'
+
+        A preprocessor function can be provided in order to modify the values in-flight.
+        This example converts increments each value by 1, then converts to a string.
+
+        >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
+        b'234'
+
+        Using dictionaries is a fast way to get specific values at specific offsets,
+        without having to do ``data += "foo"`` repeatedly.
+
+        >>> flat({12: 0x41414141,
+        ...       24: 'Hello',
+        ...      })
+        b'aaaabaaacaaaAAAAeaaafaaaHello'
+
+        Dictionary usage permits directly using values derived from :func:`.cyclic`.
+        See :func:`.cyclic`, :function:`pwnlib.context.context.cyclic_alphabet`, and :data:`.context.cyclic_size`
+        for more options.  
+
+        The cyclic pattern can be provided as either the text or hexadecimal offset.
+
+        >>> flat({ 0x61616162: 'X'})
+        b'aaaaX'
+        >>> flat({'baaa': 'X'})
+        b'aaaaX'
+
+        Fields do not have to be in linear order, and can be freely mixed.
+        This also works with cyclic offsets.
+
+        >>> flat({2: 'A', 0:'B'})
+        b'BaA'
+        >>> flat({0x61616161:'x', 0x61616162:'y'})
+        b'xaaay'
+        >>> flat({0x61616162:'y', 0x61616161:'x'})
+        b'xaaay'
+
+        Fields do not have to be in order, and can be freely mixed.
+
+        >>> flat({'caaa': 'XXXX', 16: '\x41', 20: 0xdeadbeef})
+        b'aaaabaaaXXXXdaaaAaaa\xef\xbe\xad\xde'
+        >>> flat({ 8: [0x41414141, 0x42424242], 20: 'CCCC'})
+        b'aaaabaaaAAAABBBBeaaaCCCC'
+        >>> fit({
+        ...     0x61616161: 'a',
+        ...     1: 'b',
+        ...     0x61616161+2: 'c',
+        ...     3: 'd',
+        ... })
+        b'abadbaaac'
+
+        By default, gaps in the data are filled in with the :meth:`.cyclic` pattern.
+        You can customize this by providing an iterable or method for the ``filler``
+        argument.
+
+        >>> flat({12: 'XXXX'}, filler = (ord('A'), ord('B')), length = 20)
+        b'ABABABABABABXXXXABAB'
+
+        Nested dictionaries also work as expected.
+
+        >>> flat({4: {0: 'X', 4: 'Y'}})
+        b'aaaaXaaaY'
+        >>> fit({4: {4: 'XXXX'}})
+        b'aaaabaaaXXXX'
     """
     # HACK: To avoid circular imports we need to delay the import of `cyclic`
     from pwnlib.util import cyclic
@@ -656,7 +734,7 @@ def flat(*args, **kwargs):
     return out
 
 def fit(*args, **kwargs):
-    """Legacy alias for `:func:flat`"""
+    """Legacy alias for :func:`flat`"""
     return flat(*args, **kwargs)
 
 """

pwnlib/util/proc.py

@@ -9,6 +9,7 @@ import psutil
 
 from pwnlib import tubes
 from pwnlib.log import getLogger
+from pwnlib.timeout import Timeout
 
 log = getLogger(__name__)
 
@@ -337,17 +338,23 @@ def wait_for_debugger(pid, debugger_pid=None):
     Returns:
         None
     """
-    with log.waitfor('Waiting for debugger') as l:
-        if debugger_pid:
-            debugger = psutil.Process(debugger_pid)
-            while tracer(pid) is None:
-                try:
-                    debugger.wait(0.01)
-                except psutil.TimeoutExpired:
-                    pass
-                else:
-                    l.error("debugger exited! (maybe check /proc/sys/kernel/yama/ptrace_scope)")
+    t = Timeout()
+    with t.countdown(timeout=15):
+        with log.waitfor('Waiting for debugger') as l:
+            if debugger_pid:
+                debugger = psutil.Process(debugger_pid)
+                while t.timeout and tracer(pid) is None:
+                    try:
+                        debugger.wait(0.01)
+                    except psutil.TimeoutExpired:
+                        pass
+                    else:
+                        l.failure("debugger exited! (maybe check /proc/sys/kernel/yama/ptrace_scope)")
+            else:
+                while t.timeout and tracer(pid) is None:
+                    time.sleep(0.01)
+
+        if tracer(pid):
+            l.success()
         else:
-            while tracer(pid) is None:
-                time.sleep(0.01)
-        l.success()
+            l.failure('Debugger did not attach to pid %d within 15 seconds', pid)

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '4.1.2'
+__version__ = '4.1.4'

setup.py

@@ -83,7 +83,7 @@ setup(
     name                 = 'pwntools',
     python_requires      = '>=2.7',
     packages             = find_packages(),
-    version              = '4.1.2',
+    version              = '4.1.4',
     data_files           = [('',
                              glob.glob('*.md') + glob.glob('*.txt')),
                             ],

travis/install.sh

@@ -64,6 +64,11 @@ setup_travis()
     rm -rf usr/share
 }
 
+setup_ipv6()
+{
+    echo 0 | sudo tee /proc/sys/net/ipv6/conf/all/disable_ipv6
+}
+
 setup_gdbserver()
 {
     # https://docs.improbable.io/reference/14.3/shared/debug-cloud-workers#common-issues
@@ -186,6 +191,7 @@ setup_osx()
 
 if [[ "$USER" == "travis" ]]; then
 #   setup_travis
+    setup_ipv6
     setup_gdbserver
     setup_android_emulator
 elif [[ "$USER" == "shippable" ]]; then