Release 3.3.4

We cannot determine the length of the file if passed e.g. a tube or a pipe or PTY or character device (e.g. stdin or /dev/null).

Supercedes: 9c2f0f4e
(cherry picked from commit 14f717b7b4)

CHANGELOG.md

@@ -11,7 +11,9 @@ The table below shows which release corresponds to each branch, and what date th
 | ---------------- | -------- | ---------------------- |
 | [3.5.0](#350)    | `dev`    | Mar 18, 2017 (planned)
 | [3.4.0](#340)    | `beta`   | Feb 4, 2017 (planned)
-| [3.3.2](#332)    | `stable` | Jan 10, 2016
+| [3.3.4](#334)    | `stable` | Jan 12, 2016
+| [3.3.3](#333)    |          | Jan 10, 2016
+| [3.3.2](#332)    |          | Jan 10, 2016
 | [3.3.1](#331)    |          | Jan 10, 2016
 | [3.3.0](#330)    |          | Dec 24, 2016
 | [3.2.1](#321)    |          | Dec 24, 2016
@@ -33,11 +35,28 @@ To be released on Mar 18, 2017.
 
 To be released on Feb 4, 2017.
 
+## 3.3.4
+
+- [#850][850] and [#846][846] fix issues with `hexdump` and the `phd` command-line utility, when using pipes (e.g. `echo foo | phd`)
+- [#852][852] Fixes register ordering in `regsort`
+- [#853][853] Fixes the registers restored in `shellcraft.amd64.popad`
+
+[846]: https://github.com/gallopsled/pwntools/pull/846
+[850]: https://github.com/gallopsled/pwntools/pull/850
+[852]: https://github.com/gallopsled/pwntools/pull/852
+[853]: https://github.com/gallopsled/pwntools/pull/853
+
+## 3.3.3
+
+- [#843][843] fixed a bug in `amd64.mov`.
+
+[843]: https://github.com/gallopsled/pwntools/pull/843
+
 ## 3.3.2
 
-- [#840][840] Fixed a regression introduced by [#837][837].
+- [#840][840] fixed a regression introduced by [#837][837].
 
-[840]: https://github.com/Gallopsled/pwntools/pull/840
+[840]: https://github.com/gallopsled/pwntools/pull/840
 
 ## 3.3.1
 

README.md

@@ -2,7 +2,7 @@
 ![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)
 
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![PyPI](https://img.shields.io/badge/pypi-v3.3.2-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
+[![PyPI](https://img.shields.io/badge/pypi-v3.3.4-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
 [![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)

pwnlib/regsort.py

@@ -224,6 +224,8 @@ def regsort(in_out, all_regs, tmp = None, xchg = True, randomize = None):
          ('mov', 'b', 'c'),
          ('mov', 'c', 'x'),
          ('mov', 'x', '1')]
+         >>> regsort({'a': 'b', 'b': 'c'}, ['a','b','c'], xchg=0)
+         [('mov', 'a', 'b'), ('mov', 'b', 'c')]
     """
     if randomize is None:
         randomize = context.randomize
@@ -333,7 +335,9 @@ def regsort(in_out, all_regs, tmp = None, xchg = True, randomize = None):
     # If XCHG is expressly disabled, and there is no temporary register,
     # try to see if there is any register which can be used as a temp
     # register instead.
-    if not (xchg or tmp):
+    #
+    # If there are no cycles, there's no need for a temporary register.
+    if in_cycle and not (xchg or tmp):
         for reg in in_out:
             if not depends_on_cycle(reg, in_out, in_cycle):
                 tmp = reg

pwnlib/shellcraft/templates/amd64/popad.asm

@@ -6,8 +6,8 @@ in the same order.
     pop rdi
     pop rsi
     pop rbp
-    pop rsp
-    pop rbp
+    pop rbx /* add rsp, 8 */
+    pop rbx
     pop rdx
     pop rcx
     pop rax

pwnlib/util/fiddling.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import base64
 import random
+import os
 import re
 import string
 import StringIO
@@ -10,7 +11,7 @@ from . import packing
 from ..context import context
 from ..log import getLogger
 from ..term import text
-from .cyclic import cyclic
+from .cyclic import de_bruijn
 from .cyclic import cyclic_find
 
 log = getLogger(__name__)
@@ -559,13 +560,15 @@ default_style = {
 }
 
 cyclic_pregen = ''
+de_bruijn_gen = de_bruijn()
 
 def sequential_lines(a,b):
     return (a+b) in cyclic_pregen
 
 def update_cyclic_pregenerated(size):
     global cyclic_pregen
-    cyclic_pregen = cyclic(size)
+    while size > len(cyclic_pregen):
+        cyclic_pregen += de_bruijn_gen.next()
 
 def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
                  highlight=None, cyclic=False):
@@ -608,22 +611,6 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
     spacer      = ' '
     marker      = (style.get('marker') or (lambda s:s))('│')
 
-    # Total length of the input stream
-    total = 0
-
-    if hasattr(fd, 'len'):
-        total = fd.len
-    else:
-        # Save the current file offset
-        cur = fd.seek(0, os.SEEK_CUR)
-
-        # Determine the total size of the file
-        fd.seek(0, os.SEEK_END)
-        total = fd.tell()
-
-        # Restore the file offset
-        fd.seek(cur, os.SEEK_SET)
-
     if hexii:
         column_sep = ''
         line_fmt   = '%%(offset)08x  %%(hexbytes)-%is│' % (len(column_sep)+(width*byte_width))
@@ -643,17 +630,29 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
             return hbyte, abyte
         cache = [style_byte(chr(b)) for b in range(256)]
 
-    if cyclic:
-        update_cyclic_pregenerated(total)
-
     numb = 0
     while True:
         offset = begin + numb
-        chunk = fd.read(width)
+
+        # If a tube is passed in as fd, it will raise EOFError when it runs
+        # out of data, unlike a file or StringIO object, which return an empty
+        # string.
+        try:
+            chunk = fd.read(width)
+        except EOFError:
+            chunk = ''
+
+        # We have run out of data, exit the loop
         if chunk == '':
             break
+
+        # Advance the cursor by the number of bytes we actually read
         numb += len(chunk)
 
+        # Update the cyclic pattern in case
+        if cyclic:
+            update_cyclic_pregenerated(numb)
+
         # If this chunk is the same as the last unique chunk,
         # use a '*' instead.
         if skip and last_unique:

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '3.3.2'
+__version__ = '3.3.4'

setup.py

@@ -83,7 +83,7 @@ except ImportError:
 setup(
     name                 = 'pwntools',
     packages             = find_packages(),
-    version              = '3.3.2',
+    version              = '3.3.4',
     data_files           = [('',
                              glob.glob('*.md') + glob.glob('*.txt')),
                             ],