Fix features in pack() and add more doctests (#1610)

* [packing] Better alternative to #1580

pack() was failing for dictionaries that are not ordered

Closes #1582
Closes #1579

* [docs] Add better documentation for flat()

* [changelog] Add 4.1.4

* [release] Bump version to 4.1.4

Co-authored-by: Arusekk <[email protected]>

CHANGELOG.md

@@ -66,7 +66,13 @@ To be released on Jun 5, 2020.
 [fecf9f]: http://github.com/Gallopsled/pwntools/commit/fecf9f
 [1454]: https://github.com/Gallopsled/pwntools/pull/1454
 
-## 4.1.3 (`stable`)
+## 4.1.4 (`stable`)
+
+- [#1698][1609] Fix issues in `packing.flat` with mis-ordred fields
+
+[1609]: https://github.com/Gallopsled/pwntools/pull/1609
+
+## 4.1.3
 
 - [#1590][1590] Fix `gdb.attach()` for `remote`, `listen`, `ssh` tubes
   - Also fix `run_in_new_terminal` for Py2 unicode strings

docs/source/install/binutils.rst

@@ -39,7 +39,7 @@ repo <https://github.com/Gallopsled/pwntools-binutils/>`__.
 
 .. code-block:: bash
 
-    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/osx/binutils-$ARCH.rb
+    $ brew install https://raw.githubusercontent.com/Gallopsled/pwntools-binutils/master/macos/binutils-$ARCH.rb
 
 Alternate OSes
 ^^^^^^^^^^^^^^^^

pwnlib/util/packing.py

@@ -492,9 +492,11 @@ def _fit(pieces, preprocessor, packer, filler):
     pad = bytearray()
     def fill(key):
         key = bytearray(key)
-        while len(pad) < len(key) or pad[-len(key):] != key:
+        offset = pad.find(key)
+        while offset == -1:
             pad.append(next(filler))
-        return len(pad) - len(key)
+            offset = pad.find(key, -len(key))
+        return offset
 
     # Key conversion:
     # - convert str/unicode keys to offsets
@@ -549,7 +551,6 @@ def _flat(args, preprocessor, packer, filler):
         elif isinstance(arg, (list, tuple)):
             val = _flat(arg, preprocessor, packer, filler)
         elif isinstance(arg, dict):
-            arg = collections.OrderedDict(((k,v) for k,v in sorted(arg.items())))
             filler, val = _fit(arg, preprocessor, packer, filler)
         elif isinstance(arg, bytes):
             val = arg
@@ -617,37 +618,100 @@ def flat(*args, **kwargs):
 
     Examples:
 
-      >>> context.clear()
-      >>> flat(1, "test", [[["AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
-      b'\x01\x00testABABABABABAB'
-      >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
-      b'234'
-      >>> flat({12: 0x41414141,
-      ...       24: 'Hello',
-      ...      })
-      b'aaaabaaacaaaAAAAeaaafaaaHello'
-      >>> flat({'caaa': ''})
-      b'aaaabaaa'
-      >>> flat({12: 'XXXX'}, filler = (ord('A'), ord('B')), length = 20)
-      b'ABABABABABABXXXXABAB'
-      >>> flat({ 8: [0x41414141, 0x42424242],
-      ...       20: 'CCCC'})
-      b'aaaabaaaAAAABBBBeaaaCCCC'
-      >>> flat({ 0x61616162: 'X'})
-      b'aaaaX'
-      >>> flat({4: {0: 'X', 4: 'Y'}})
-      b'aaaaXaaaY'
-      >>> flat({0x61616161:'x', 0x61616162:'y'})
-      b'xaaay'
-      >>> flat({0x61616162:'y', 0x61616161:'x'})
-      b'xaaay'
-      >>> fit({
-      ...     0x61616161: 'a',
-      ...     1: 'b',
-      ...     0x61616161+2: 'c',
-      ...     3: 'd',
-      ... })
-      b'abadbaaac'
+        (Test setup, please ignore)
+    
+        >>> context.clear()
+
+        Basic usage of :meth:`flat` works similar to the pack() routines.
+
+        >>> flat(4)
+        b'\x04\x00\x00\x00'
+
+        :meth:`flat` works with strings, bytes, lists, and dictionaries.
+
+        >>> flat(b'X')
+        b'X'
+        >>> flat([1,2,3])
+        b'\x01\x00\x00\x00\x02\x00\x00\x00\x03\x00\x00\x00'
+        >>> flat({4:'X'})
+        b'aaaaX'
+
+        :meth:`.flat` flattens all of the values provided, and allows nested lists
+        and dictionaries.
+
+        >>> flat([{4:'X'}] * 2)
+        b'aaaaXaaacX'
+        >>> flat([[[[[[[[[1]]]], 2]]]]])
+        b'\x01\x00\x00\x00\x02\x00\x00\x00'
+
+        You can also provide additional arguments like endianness, word-size, and
+        whether the values are treated as signed or not.
+
+        >>> flat(1, "test", [[["AB"]*2]*3], endianness = 'little', word_size = 16, sign = False)
+        b'\x01\x00testABABABABABAB'
+
+        A preprocessor function can be provided in order to modify the values in-flight.
+        This example converts increments each value by 1, then converts to a string.
+
+        >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
+        b'234'
+
+        Using dictionaries is a fast way to get specific values at specific offsets,
+        without having to do ``data += "foo"`` repeatedly.
+
+        >>> flat({12: 0x41414141,
+        ...       24: 'Hello',
+        ...      })
+        b'aaaabaaacaaaAAAAeaaafaaaHello'
+
+        Dictionary usage permits directly using values derived from :func:`.cyclic`.
+        See :func:`.cyclic`, :function:`pwnlib.context.context.cyclic_alphabet`, and :data:`.context.cyclic_size`
+        for more options.  
+
+        The cyclic pattern can be provided as either the text or hexadecimal offset.
+
+        >>> flat({ 0x61616162: 'X'})
+        b'aaaaX'
+        >>> flat({'baaa': 'X'})
+        b'aaaaX'
+
+        Fields do not have to be in linear order, and can be freely mixed.
+        This also works with cyclic offsets.
+
+        >>> flat({2: 'A', 0:'B'})
+        b'BaA'
+        >>> flat({0x61616161:'x', 0x61616162:'y'})
+        b'xaaay'
+        >>> flat({0x61616162:'y', 0x61616161:'x'})
+        b'xaaay'
+
+        Fields do not have to be in order, and can be freely mixed.
+
+        >>> flat({'caaa': 'XXXX', 16: '\x41', 20: 0xdeadbeef})
+        b'aaaabaaaXXXXdaaaAaaa\xef\xbe\xad\xde'
+        >>> flat({ 8: [0x41414141, 0x42424242], 20: 'CCCC'})
+        b'aaaabaaaAAAABBBBeaaaCCCC'
+        >>> fit({
+        ...     0x61616161: 'a',
+        ...     1: 'b',
+        ...     0x61616161+2: 'c',
+        ...     3: 'd',
+        ... })
+        b'abadbaaac'
+
+        By default, gaps in the data are filled in with the :meth:`.cyclic` pattern.
+        You can customize this by providing an iterable or method for the ``filler``
+        argument.
+
+        >>> flat({12: 'XXXX'}, filler = (ord('A'), ord('B')), length = 20)
+        b'ABABABABABABXXXXABAB'
+
+        Nested dictionaries also work as expected.
+
+        >>> flat({4: {0: 'X', 4: 'Y'}})
+        b'aaaaXaaaY'
+        >>> fit({4: {4: 'XXXX'}})
+        b'aaaabaaaXXXX'
     """
     # HACK: To avoid circular imports we need to delay the import of `cyclic`
     from pwnlib.util import cyclic
@@ -670,7 +734,7 @@ def flat(*args, **kwargs):
     return out
 
 def fit(*args, **kwargs):
-    """Legacy alias for `:func:flat`"""
+    """Legacy alias for :func:`flat`"""
     return flat(*args, **kwargs)
 
 """

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '4.1.3'
+__version__ = '4.1.4'

setup.py

@@ -83,7 +83,7 @@ setup(
     name                 = 'pwntools',
     python_requires      = '>=2.7',
     packages             = find_packages(),
-    version              = '4.1.3',
+    version              = '4.1.4',
     data_files           = [('',
                              glob.glob('*.md') + glob.glob('*.txt')),
                             ],