[release] Version bump to 4.1.7

* [stable] Fix incorrectly-merged code

* [aarch64] Use byte-strings instead of strings

* [aarch64] Have different tests for str vs bytes

* [aarch64] Convert all items to binary before processing

* [aarch64] Fix doctest results

* [aarch64] Stopgap to fix Py3 CI tests

* [aarch64] Remove failing test, will address in a future release

CHANGELOG.md

@@ -66,7 +66,25 @@ To be released on Jun 5, 2020.
 [fecf9f]: http://github.com/Gallopsled/pwntools/commit/fecf9f
 [1454]: https://github.com/Gallopsled/pwntools/pull/1454
 
-## 4.1.3 (`stable`)
+## 4.1.7 (`stable`)
+
+- [#1615][1615] Fix aarch64 pushstr and pushstr_array
+
+[1615]: https://github.com/Gallopsled/pwntools/pull/1454
+
+## 4.1.5
+
+- [#1517][1517] flat(..., filler=) is fixed for `str` values and Python2 `bytes`
+
+[1517]: https://github.com/Gallopsled/pwntools/pull/1517
+
+## 4.1.4
+
+- [#1698][1609] Fix issues in `packing.flat` with mis-ordred fields
+
+[1609]: https://github.com/Gallopsled/pwntools/pull/1609
+
+## 4.1.3
 
 - [#1590][1590] Fix `gdb.attach()` for `remote`, `listen`, `ssh` tubes
   - Also fix `run_in_new_terminal` for Py2 unicode strings

docs/source/install/binutils.rst

@@ -39,7 +39,7 @@ repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
 .. code-block:: bash
 
-    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/osx/binutils-$ARCH.rb
+    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
 
 Alternate OSes
 ^^^^^^^^^^^^^^^^

pwnlib/shellcraft/templates/aarch64/pushstr.asm

@@ -15,64 +15,19 @@ Args:
 
 Examples:
 
-    >>> print(shellcraft.pushstr("Hello!").rstrip())
-        /* push b'Hello!\x00' */
-        /* Set x14 = 36762444129608 = 0x216f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #8559, lsl #0x20
-        str x14, [sp, #-16]!
-    >>> print(shellcraft.pushstr("Hello, world!").rstrip())
-        /* push b'Hello, world!\x00' */
-        /* Set x14 = 8583909746840200520 = 0x77202c6f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #11375, lsl #0x20
-        movk x14, #30496, lsl #0x30
-        /* Set x15 = 143418749551 = 0x21646c726f */
-        mov  x15, #29295
-        movk x15, #25708, lsl #16
-        movk x15, #33, lsl #0x20
-        stp x14, x15, [sp, #-16]!
-    >>> print(shellcraft.pushstr("Hello, world, bienvenue").rstrip())
-        /* push b'Hello, world, bienvenue\x00' */
-        /* Set x14 = 8583909746840200520 = 0x77202c6f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #11375, lsl #0x20
-        movk x14, #30496, lsl #0x30
-        /* Set x15 = 7593667296735556207 = 0x6962202c646c726f */
-        mov  x15, #29295
-        movk x15, #25708, lsl #16
-        movk x15, #8236, lsl #0x20
-        movk x15, #26978, lsl #0x30
-        stp x14, x15, [sp, #-16]!
-        /* Set x14 = 28558089656888933 = 0x65756e65766e65 */
-        mov  x14, #28261
-        movk x14, #25974, lsl #16
-        movk x14, #30062, lsl #0x20
-        movk x14, #101, lsl #0x30
-        str x14, [sp, #-16]!
-    >>> print(shellcraft.pushstr("Hello, world, bienvenue!").rstrip())
-        /* push b'Hello, world, bienvenue!\x00' */
-        /* Set x14 = 8583909746840200520 = 0x77202c6f6c6c6548 */
-        mov  x14, #25928
-        movk x14, #27756, lsl #16
-        movk x14, #11375, lsl #0x20
-        movk x14, #30496, lsl #0x30
-        /* Set x15 = 7593667296735556207 = 0x6962202c646c726f */
-        mov  x15, #29295
-        movk x15, #25708, lsl #16
-        movk x15, #8236, lsl #0x20
-        movk x15, #26978, lsl #0x30
-        stp x14, x15, [sp, #-16]!
-        /* Set x14 = 2406458692908510821 = 0x2165756e65766e65 */
-        mov  x14, #28261
-        movk x14, #25974, lsl #16
-        movk x14, #30062, lsl #0x20
-        movk x14, #8549, lsl #0x30
-        mov  x15, xzr
-        stp x14, x15, [sp, #-16]!
+    >>> string = "Hello, world!"
+    >>> assembly = shellcraft.pushstr(string)
+    >>> assembly += shellcraft.write(1, 'sp', len(string))
+    >>> assembly += shellcraft.exit()
+    >>> ELF.from_assembly(assembly).process().recvall()
+    b'Hello, world!'
+
+    >>> string = "Hello, world! This is a long string! Wow!"
+    >>> assembly = shellcraft.pushstr(string)
+    >>> assembly += shellcraft.write(1, 'sp', len(string))
+    >>> assembly += shellcraft.exit()
+    >>> ELF.from_assembly(assembly).process().recvall()
+    b'Hello, world! This is a long string! Wow!'
 </%docstring>
 <%
 if isinstance(string, six.text_type):
@@ -91,6 +46,8 @@ while len(string) % 8:
 words = packing.unpack_many(string)
 pairs = lists.group(2, words)
 
+pairs = pairs[::-1]
+
 # The stack must be 16-byte aligned
 total = len(pairs) * 16
 

pwnlib/shellcraft/templates/aarch64/pushstr_array.asm

@@ -2,7 +2,7 @@
     from pwnlib import shellcraft
     from pwnlib.context import context as ctx
     from pwnlib.util.iters import group
-    from six import text_type, binary_type
+    from six import text_type, binary_type, ensure_binary
 %>
 <%docstring>
 Pushes an array/envp-style array of pointers onto the stack.
@@ -14,66 +14,44 @@ Arguments:
         Single argument or list of arguments to push.
         NULL termination is normalized so that each argument
         ends with exactly one NULL byte.
+
+Example:
+    >>> assembly = shellcraft.execve("/bin/sh", ["sh", "-c", "echo Hello string $WORLD"], {"WORLD": "World!"})
+    >>> ELF.from_assembly(assembly).process().recvall()
+    b'Hello string World!\n'
 </%docstring>
 <%page args="reg, array, register1='x14', register2='x15'"/>
 <%
 if isinstance(array, (binary_type, text_type)):
     array = [array]
 
-# Normalize all of the arguments' endings
+# Convert all items to strings
+array = [ensure_binary(x) for x in array]
+
+# Normalize line endings for each item
 array = [arg.rstrip(b'\x00') + b'\x00' for arg in array]
 
+# Join everything in the string-to-be-pushed
+string = b''.join(array)
+
 # Maximum amount that we can adjust SP by at once is 4095,
 # which seems like a safe maximum.
 if len(array) * 8 > 4095:
     raise Exception("Array size is too large (%i), max=4095" % len(array))
-
-# Join them into one big string that can be pushed
-array_str = b''.join(array)
-
-# Create a listing of offsets from what will be the "top" of the stack.
-num_pointers = len(array)
-
-# Account for the NULL terminator
-num_pointers += 1
-
-while num_pointers % 2 != 0:
-    num_pointers += 1
-
-# Offset from the 'top' of the stack, to the data pointed at
-sp_to_data = num_pointers * ctx.bytes
-
-# List of amounts to subtract from $SP
-offsets = {}
-for i,arg in enumerate(reversed(array)):
-    offsets[i] = sp_to_data + len(array_str) - len(arg)
-
-# If the array length is ODD we can sneak in our null terminator at the end
-if len(array) % 2 == 1:
-    offsets[len(offsets)] = 'sp'
-
-sorted_offsets = []
-for key in sorted(offsets):
-    sorted_offsets.append(offsets[key])
-
-# Load the offsets into pairs
-pairwise_offsets = group(2, sorted_offsets)
 %>\
     /* push argument array ${repr(array)} */
-    ${shellcraft.pushstr(array_str, register1=register1, register2=register2)}
+    ${shellcraft.pushstr(string, register1=register1, register2=register2)}
 
-    /* adjust the stack pointer to account for the array of pointers */
-    sub sp, sp, ${num_pointers * ctx.bytes}
+    /* push null terminator */
+    ${shellcraft.mov(register1, 0)}
+    str ${register1}, [sp, #-8]!
 
-    /* push pointers onto the stack in pairs */
-%for i, (a, b) in enumerate(pairwise_offsets):
-    ${shellcraft.mov(register1, a)}
-    ${shellcraft.mov(register2, b)}
-    sub  ${register1}, sp, ${register1}
-    sub  ${register2}, sp, ${register2}
-    stp  ${register1}, ${register2}, [sp], ${i * 16}
+    /* push pointers onto the stack */
+%for i, value in enumerate(reversed(array)):
+   ${shellcraft.mov(register1, (i+1)*8 + string.index(value))}
+   add ${register1}, sp, ${register1}
+   str ${register1}, [sp, #-8]! /* ${array[-i]} */
 %endfor
-%if len(array[-1] != 'sp')
 
     /* set ${reg} to the current top of the stack */
     ${shellcraft.mov(reg,'sp')}

pwnlib/util/packing.py

@@ -492,9 +492,11 @@ def _fit(pieces, preprocessor, packer, filler):
     pad = bytearray()
     def fill(key):
         key = bytearray(key)
-        while len(pad) < len(key) or pad[-len(key):] != key:
+        offset = pad.find(key)
+        while offset == -1:
             pad.append(next(filler))
-        return len(pad) - len(key)
+            offset = pad.find(key, -len(key))
+        return offset
 
     # Key conversion:
     # - convert str/unicode keys to offsets
@@ -549,7 +551,6 @@ def _flat(args, preprocessor, packer, filler):
         elif isinstance(arg, (list, tuple)):
             val = _flat(arg, preprocessor, packer, filler)
         elif isinstance(arg, dict):
-            arg = collections.OrderedDict(((k,v) for k,v in sorted(arg.items())))
             filler, val = _fit(arg, preprocessor, packer, filler)
         elif isinstance(arg, bytes):
             val = arg
@@ -617,37 +618,102 @@ def flat(*args, **kwargs):
 
     Examples:
 
-      >>> context.clear()
-      >>> flat(1, "test", [[["AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
-      b'\x01\x00testABABABABABAB'
-      >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
-      b'234'
-      >>> flat({12: 0x41414141,
-      ...       24: 'Hello',
-      ...      })
-      b'aaaabaaacaaaAAAAeaaafaaaHello'
-      >>> flat({'caaa': ''})
-      b'aaaabaaa'
-      >>> flat({12: 'XXXX'}, filler = (ord('A'), ord('B')), length = 20)
-      b'ABABABABABABXXXXABAB'
-      >>> flat({ 8: [0x41414141, 0x42424242],
-      ...       20: 'CCCC'})
-      b'aaaabaaaAAAABBBBeaaaCCCC'
-      >>> flat({ 0x61616162: 'X'})
-      b'aaaaX'
-      >>> flat({4: {0: 'X', 4: 'Y'}})
-      b'aaaaXaaaY'
-      >>> flat({0x61616161:'x', 0x61616162:'y'})
-      b'xaaay'
-      >>> flat({0x61616162:'y', 0x61616161:'x'})
-      b'xaaay'
-      >>> fit({
-      ...     0x61616161: 'a',
-      ...     1: 'b',
-      ...     0x61616161+2: 'c',
-      ...     3: 'd',
-      ... })
-      b'abadbaaac'
+        (Test setup, please ignore)
+    
+        >>> context.clear()
+
+        Basic usage of :meth:`flat` works similar to the pack() routines.
+
+        >>> flat(4)
+        b'\x04\x00\x00\x00'
+
+        :meth:`flat` works with strings, bytes, lists, and dictionaries.
+
+        >>> flat(b'X')
+        b'X'
+        >>> flat([1,2,3])
+        b'\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00'
+        >>> flat({4:'X'})
+        b'aaaaX'
+
+        :meth:`.flat` flattens all of the values provided, and allows nested lists
+        and dictionaries.
+
+        >>> flat([{4:'X'}] * 2)
+        b'aaaaXaaacX'
+        >>> flat([[[[[[[[[1]]]], 2]]]]])
+        b'\x01\x00\x00\x00\x02\x00\x00\x00'
+
+        You can also provide additional arguments like endianness, word-size, and
+        whether the values are treated as signed or not.
+
+        >>> flat(1, "test", [[["AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
+        b'\x01\x00testABABABABABAB'
+
+        A preprocessor function can be provided in order to modify the values in-flight.
+        This example converts increments each value by 1, then converts to a string.
+
+        >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
+        b'234'
+
+        Using dictionaries is a fast way to get specific values at specific offsets,
+        without having to do ``data += "foo"`` repeatedly.
+
+        >>> flat({12: 0x41414141,
+        ...       24: 'Hello',
+        ...      })
+        b'aaaabaaacaaaAAAAeaaafaaaHello'
+
+        Dictionary usage permits directly using values derived from :func:`.cyclic`.
+        See :func:`.cyclic`, :function:`pwnlib.context.context.cyclic_alphabet`, and :data:`.context.cyclic_size`
+        for more options.  
+
+        The cyclic pattern can be provided as either the text or hexadecimal offset.
+
+        >>> flat({ 0x61616162: 'X'})
+        b'aaaaX'
+        >>> flat({'baaa': 'X'})
+        b'aaaaX'
+
+        Fields do not have to be in linear order, and can be freely mixed.
+        This also works with cyclic offsets.
+
+        >>> flat({2: 'A', 0:'B'})
+        b'BaA'
+        >>> flat({0x61616161:'x', 0x61616162:'y'})
+        b'xaaay'
+        >>> flat({0x61616162:'y', 0x61616161:'x'})
+        b'xaaay'
+
+        Fields do not have to be in order, and can be freely mixed.
+
+        >>> flat({'caaa': 'XXXX', 16: '\x41', 20: 0xdeadbeef})
+        b'aaaabaaaXXXXdaaaAaaa\xef\xbe\xad\xde'
+        >>> flat({ 8: [0x41414141, 0x42424242], 20: 'CCCC'})
+        b'aaaabaaaAAAABBBBeaaaCCCC'
+        >>> fit({
+        ...     0x61616161: 'a',
+        ...     1: 'b',
+        ...     0x61616161+2: 'c',
+        ...     3: 'd',
+        ... })
+        b'abadbaaac'
+
+        By default, gaps in the data are filled in with the :meth:`.cyclic` pattern.
+        You can customize this by providing an iterable or method for the ``filler``
+        argument.
+
+        >>> flat({12: 'XXXX'}, filler = b'_', length = 20)
+        b'____________XXXX____'
+        >>> flat({12: 'XXXX'}, filler = b'AB', length = 20)
+        b'ABABABABABABXXXXABAB'
+
+        Nested dictionaries also work as expected.
+
+        >>> flat({4: {0: 'X', 4: 'Y'}})
+        b'aaaaXaaaY'
+        >>> fit({4: {4: 'XXXX'}})
+        b'aaaabaaaXXXX'
     """
     # HACK: To avoid circular imports we need to delay the import of `cyclic`
     from pwnlib.util import cyclic
@@ -656,6 +722,9 @@ def flat(*args, **kwargs):
     filler       = kwargs.pop('filler', cyclic.de_bruijn())
     length       = kwargs.pop('length', None)
 
+    if isinstance(filler, str):
+        filler = bytearray(six.ensure_binary(filler))
+
     if kwargs != {}:
         raise TypeError("flat() does not support argument %r" % kwargs.popitem()[0])
 
@@ -670,7 +739,7 @@ def flat(*args, **kwargs):
     return out
 
 def fit(*args, **kwargs):
-    """Legacy alias for `:func:flat`"""
+    """Legacy alias for :func:`flat`"""
     return flat(*args, **kwargs)
 
 """

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '4.1.3'
+__version__ = '4.1.7'

setup.py

@@ -83,7 +83,7 @@ setup(
     name                 = 'pwntools',
     python_requires      = '>=2.7',
     packages             = find_packages(),
-    version              = '4.1.3',
+    version              = '4.1.7',
     data_files           = [('',
                              glob.glob('*.md') + glob.glob('*.txt')),
                             ],