Release 3.0.3

Also backports the PyPi documentation changes to stable.

Fixes #736

.travis_install.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash -e
+set -e
 local_deb_extract()
 {
     wget $1
@@ -17,7 +18,8 @@ get_binutils()
 get_qemu()
 {
     echo "Installing qemu"
-    QEMU_URL='https://mirrors.kernel.org/ubuntu/pool/universe/q/qemu/qemu-user-static_2.6%2bdfsg-3ubuntu1_amd64.deb'
+    QEMU_INDEX='http://packages.ubuntu.com/en/yakkety/amd64/qemu-user-static/download'
+    QEMU_URL=$(curl "$QEMU_INDEX" | grep kernel.org | grep -Eo 'http.*\.deb')
     local_deb_extract "$QEMU_URL"
 }
 
@@ -26,35 +28,35 @@ setup_travis()
     export PATH=$PWD/usr/bin:$PATH
     export LD_LIBRARY_PATH=$PWD/usr/lib
 
-    if [ ! -d usr/bin ];
+    # Install libbfd-multiarch and libopcodes-multiarch if not found in the cache
+    if [ ! -f usr/lib/libbfd-2.22-multiarch.so ];
     then
-        # Install our custom binutils
-        which arm-linux-as     || get_binutils arm
-        which mips-linux-as    || get_binutils mips
-        which powerpc-linux-as || get_binutils powerpc
-        which aarch64-linux-as || get_binutils aarch64
-
         # Install the multiarch binutils
         local_deb_extract http://mirrors.mit.edu/ubuntu/ubuntu/pool/universe/b/binutils/binutils-multiarch_2.22-6ubuntu1_amd64.deb
+        pushd usr/lib
+        ln -sf libbfd-2.22-multiarch.so libbfd-2.22.so
+        ln -sf libopcodes-2.22-multiarch.so libopcodes-2.22.so
+        popd
     fi
 
-    if ! (which qemu-arm-static && qemu-arm-static -version | grep 2.6.0); then
+    # Install/upgrade qemu
+    if ! (which qemu-arm-static && qemu-arm-static -version | grep 2.6.1); then
         get_qemu
     fi
 
-    # Get rid of files we don't want cached
-    rm -rf usr/share
-
-    pushd usr/lib
-    ln -sf libbfd-2.22-multiarch.so libbfd-2.22.so
-    ln -sf libopcodes-2.22-multiarch.so libopcodes-2.22.so
-    popd
+    # Install our custom binutils
+    which arm-linux-gnu-as     || get_binutils arm
+    which mips-linux-gnu-as    || get_binutils mips
+    which powerpc-linux-gnu-as || get_binutils powerpc
 
+    # Test that the installs worked
     which arm-linux-gnu-as
     which mips-linux-gnu-as
     which powerpc-linux-gnu-as
-    which aarch64-linux-gnu-as
     which qemu-arm-static
+
+    # Get rid of files we don't want cached
+    rm -rf usr/share
 }
 
 setup_linux()

.travis_ssh_setup.sh

@@ -4,6 +4,8 @@
 #
 # All of the "conditional sudo" is to do container-based builds on
 # Travis which are much, much faster.
+set -e
+
 U=travis
 H=/home/$U
 
@@ -36,7 +38,7 @@ pubkey=$(cat ~/.ssh/$U.pub)
 
 # Set the authorized_keys entry to only permit login from localhost,
 # and only with
-USUDO mkdir $H/.ssh
+USUDO mkdir $H/.ssh || true
 USUDO tee -a $H/.ssh/authorized_keys <<EOF
 from="127.0.0.1" $pubkey
 EOF

CHANGELOG.md

@@ -3,6 +3,23 @@
 This changelog only includes added major features and changes. Bugfixes and
 minor changes are omitted.
 
+## 3.0.3 (September 18 2016)
+
+- Fixed some performance and usability problems with the update system (Issues:
+  #723, #724, #736. PRs: #729, #738, #747).
+- Fixed a bug related to internals in pyelftools (PRs: #730, #746).
+- Fixed an issue with travis (Issue: #741, PRs: #743, #744, #745).
+
+## 3.0.2 (September 6 2016)
+
+- Cherry-pick #695, as this was a regression-fix.
+- Added a fix for the update checker, as it would suggest prereleases as updates to stable releases.
+- Various documentation fixes.
+
+## 3.0.1 (August 20 2016)
+
+A small bugfix release. There were a lot of references to the `master`-branch, however after 3.0.0 we use the names `stable`, `beta` and `dev` for our branches.
+
 ## 3.0.0 (August 20 2016)
 
 This was a large release (1305 commits since 2.2.0) with a lot of bugfixes and changes.  The Binjitsu project, a fork of Pwntools, was merged back into Pwntools.  As such, its features are now available here.

CONTRIBUTING.md

@@ -19,9 +19,14 @@ Finally, it is probably a good idea to run the test suite locally before doing
 the pull-request to make sure everything works, however this is not a
 requirement.
 
+Once you are ready to do a pull-request, you should figure out if your changes
+constitutes a new feature or a bugfix in stable or beta. If it is a bugfix in
+stable or beta, you should do the pull-request against the branch in question,
+and otherwise your pull-request should be against the dev branch.
+
 Once you do the pull-request Travis CI will run the test-suite on it. Once it
 passes one of the core developers will look at your pull request, possibly
-comment on it and then hopefully merge it into master.
+comment on it and then hopefully merge it into the branch in question.
 
 ## Automated Testing
 

LICENSE-pwntools.txt

@@ -13,7 +13,7 @@ directories:
 - pwnlib/data/
 
 
-Copyright (c) 2015 Gallopsled and Zach Riggle
+Copyright (c) 2015 Gallopsled et al.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,12 +1,10 @@
 # pwntools - CTF toolkit
-![pwntools logo](https://github.com/Gallopsled/pwntools/blob/master/docs/source/logo.png?raw=true)
+![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)
 
-[![Docs latest](https://readthedocs.org/projects/pwntools/badge/?version=latest)](https://pwntools.readthedocs.org/)
-[![Docs 2.2.0](https://readthedocs.org/projects/pwntools/badge/?version=2.2.0)](https://pwntools.readthedocs.org/en/2.2.0)
+[![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
 [![PyPI](https://img.shields.io/pypi/v/pwntools.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
-[![Gittip](https://img.shields.io/gittip/gallopsled.svg?style=flat)](https://www.gittip.com/gallopsled/)
-[![Travis](https://travis-ci.org/Gallopsled/pwntools.svg?branch=master)](https://travis-ci.org/Gallopsled/pwntools)
-[![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/master.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=master)
+[![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
+[![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)
 [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat)](http://choosealicense.com/licenses/mit/)
 
@@ -31,7 +29,7 @@ It will drop you into a clean, Docker-ized container.  There is nothing of value
 [key]: https://gist.githubusercontent.com/zachriggle/efa2e0080ae6de2e8344/raw/4b503e9db54f009d97477d03d4ba5678471f8ff0/id_rsa
 
 # Documentation
-Our documentation is available at [pwntools.readthedocs.org](https://pwntools.readthedocs.org/)
+Our documentation is available at [docs.pwntools.com](https://docs.pwntools.com/)
 
 To get you started, we've provided some example solutions for past CTF challenges in our [write-ups repository](https://github.com/Gallopsled/pwntools-write-ups).
 
@@ -44,10 +42,13 @@ Most of the functionality of pwntools is self-contained and Python-only.  You sh
 ```sh
 apt-get update
 apt-get install python2.7 python-pip python-dev git libssl-dev
-pip install --upgrade git+https://github.com/Gallopsled/pwntools.git
+pip install --upgrade pwntools
 ```
 
-However, some of the features (assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://pwntools.readthedocs.org/en/latest/install.html).
+If you'd like to be an early tester of the the current beta release, add
+`--pre` to the command-line.
+
+However, some of the features (assembling/disassembling foreign architectures) require non-Python dependencies.  For more information, see the [complete installation instructions here](https://docs.pwntools.com/en/stable/install.html).
 
 
 # Contribution

docs/source/conf.py

@@ -75,7 +75,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'pwntools'
-copyright = u'2015, Zach Riggle'
+copyright = u'2016, Gallopsled et al.'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -217,7 +217,7 @@ latex_elements = {
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
   ('index', 'pwntools.tex', u'pwntools Documentation',
-   u'2015, Zach Riggle', 'manual'),
+   u'2016, Gallopsled et al.', 'manual'),
 ]
 
 intersphinx_mapping = {'python': ('https://docs.python.org/2.7', None),
@@ -250,7 +250,7 @@ intersphinx_mapping = {'python': ('https://docs.python.org/2.7', None),
 # (source start file, name, description, authors, manual section).
 man_pages = [
     ('index', 'pwntools', u'pwntools Documentation',
-     [u'2015, Zach Riggle'], 1)
+     [u'2016, Gallopsled et al.'], 1)
 ]
 
 # If true, show URL addresses after external links.

docs/source/index.rst

@@ -5,6 +5,20 @@ pwntools
 Written in Python, it is designed for rapid prototyping and development,
 and intended to make exploit writing as simple as possible.
 
+The primary location for this documentation is at docs.pwntools.com_, which uses
+readthedocs_. It comes in three primary flavors:
+
+- Stable_
+- Beta_
+- Dev_
+
+.. _readthedocs: https://readthedocs.org
+.. _docs.pwntools.com: https://docs.pwntools.com
+.. _Stable: https://docs.pwntools.com/en/stable
+.. _Beta: https://docs.pwntools.com/en/beta
+.. _Dev: https://docs.pwntools.com/en/dev
+
+
 Getting Started
 ---------------
 

docs/source/install.rst

@@ -25,7 +25,15 @@ pwntools is available as a ``pip`` package.
 .. code-block:: bash
 
     $ apt-get install python2.7 python2.7-dev python-pip
-    $ pip install --upgrade git+https://github.com/Gallopsled/pwntools.git
+    $ pip install --upgrade pwntools
+
+Alternatively you can get early access to the beta release if you want to help
+file bugs or simple get newer features:
+
+.. code-block:: bash
+
+    $ pip install --upgrade --pre pwntools
+
 
 Development
 --------------

examples/README.md

@@ -4,7 +4,7 @@ the pwntools project.
 
 We have a plan to create a separate repository with examples, primarily
 exploits. Until we do so, we recommend new users to look at
-https://pwntools.readthedocs.org, as this is a better overview of our features.
+https://docs.pwntools.com, as this is a better overview of our features.
 
 In no particular order the docstrings for each example:
 

examples/gen-README.py

@@ -11,7 +11,7 @@ the pwntools project.
 
 We have a plan to create a separate repository with examples, primarily
 exploits. Until we do so, we recommend new users to look at
-https://pwntools.readthedocs.org, as this is a better overview of our features.
+https://docs.pwntools.com, as this is a better overview of our features.
 
 In no particular order the docstrings for each example:
 

extra/docker/Dockerfile

@@ -4,7 +4,7 @@
 ############################################################
 
 FROM ubuntu:trusty
-MAINTAINER Maintainer Zach Riggle
+MAINTAINER Maintainer Gallopsled et al.
 
 RUN apt-get update && apt-get install -y software-properties-common
 RUN apt-add-repository --yes ppa:pwntools/binutils

pwnlib/asm.py

@@ -133,7 +133,7 @@ def which_binutils(util):
     log.warning("""
 Could not find %(util)r installed for %(context)s
 Try installing binutils for this architecture:
-https://pwntools.readthedocs.org/en/latest/install/binutils.html
+https://docs.pwntools.com/en/stable/install/binutils.html
 """.strip() % locals())
     raise Exception('Could not find %(util)r installed for %(context)s' % locals())
 
@@ -179,7 +179,7 @@ def _assembler():
         version = re.search(r' (\d\.\d+)', result).group(1)
         if version < '2.19':
             log.warn_once('Your binutils version is too old and may not work!\n'  + \
-                'Try updating with: https://pwntools.readthedocs.org/en/latest/install/binutils.html\n' + \
+                'Try updating with: https://docs.pwntools.com/en/stable/install/binutils.html\n' + \
                 'Reported Version: %r' % result.strip())
 
 

pwnlib/commandline/phd.py

@@ -83,8 +83,6 @@ def main():
         else:
             infile.seek(skip, os.SEEK_CUR)
 
-    data = infile.read(count)
-
     hl = []
     if args.highlight:
         for hs in args.highlight:
@@ -92,7 +90,7 @@ def main():
                 hl.append(asint(h))
 
     try:
-        for line in hexdump_iter(data, width, highlight = hl, begin = offset + skip):
+        for line in hexdump_iter(infile, width, highlight = hl, begin = offset + skip):
             print line
     except (KeyboardInterrupt, IOError):
         pass

pwnlib/context/__init__.py

@@ -1202,7 +1202,7 @@ if 'ANDROID_ADB_SERVER_PORT' in os.environ:
 
 def LocalContext(function):
     """
-    Wraps the specied function on a context.local() block, using kwargs.
+    Wraps the specified function on a context.local() block, using kwargs.
 
     Example:
 

pwnlib/elf/corefile.py

@@ -6,7 +6,6 @@ from elftools.common.py3compat import bytes2str
 from elftools.common.utils import roundup
 from elftools.common.utils import struct_parse
 from elftools.construct import CString
-
 from ..context import context
 from ..log import getLogger
 from ..tubes.tube import tube
@@ -30,11 +29,11 @@ def iter_notes(self):
     end = self['p_offset'] + self['p_filesz']
     while offset < end:
         note = struct_parse(
-            self._elfstructs.Elf_Nhdr,
+            self.elffile.structs.Elf_Nhdr,
             self.stream,
             stream_pos=offset)
         note['n_offset'] = offset
-        offset += self._elfstructs.Elf_Nhdr.sizeof()
+        offset += self.elffile.structs.Elf_Nhdr.sizeof()
         self.stream.seek(offset)
         # n_namesz is 4-byte aligned.
         disk_namesz = roundup(note['n_namesz'], 2)

pwnlib/testexample.py

@@ -3,7 +3,7 @@ Module-level documentation would go here, along with a general description
 of the functionality.  You can also add module-level doctests.
 
 You can see what the documentation for this module will look like here:
-https://pwntools.readthedocs.org/en/latest/testexample.html
+https://docs.pwntools.com/en/stable/testexample.html
 
 The tests for this module are run when the documentation is automatically-generated
 by Sphinx.  This particular module is invoked by an "automodule" directive, which

pwnlib/tubes/tube.py

@@ -144,7 +144,7 @@ class tube(Timeout, Logger):
     def _recv(self, numb = 4096, timeout = default):
         """_recv(numb = 4096, timeout = default) -> str
 
-        Recieves one chunk of from the internal buffer or from the OS if the
+        Receives one chunk of from the internal buffer or from the OS if the
         buffer is empty.
         """
         data = ''
@@ -198,7 +198,7 @@ class tube(Timeout, Logger):
     def recvn(self, numb, timeout = default):
         """recvn(numb, timeout = default) -> str
 
-        Recieves exactly `n` bytes.
+        Receives exactly `n` bytes.
 
         If the request is not satisfied before ``timeout`` seconds pass,
         all data is buffered and an empty string (``''``) is returned.
@@ -246,7 +246,7 @@ class tube(Timeout, Logger):
     def recvuntil(self, delims, drop=False, timeout = default):
         """recvuntil(delims, timeout = default) -> str
 
-        Recieve data until one of `delims` is encountered.
+        Receive data until one of `delims` is encountered.
 
         If the request is not satisfied before ``timeout`` seconds pass,
         all data is buffered and an empty string (``''``) is returned.
@@ -337,7 +337,7 @@ class tube(Timeout, Logger):
     def recvlines(self, numlines=2**20, keepends = False, timeout = default):
         r"""recvlines(numlines, keepends = False, timeout = default) -> str list
 
-        Recieve up to ``numlines`` lines.
+        Receive up to ``numlines`` lines.
 
         A "line" is any sequence of bytes terminated by the byte sequence
         set by :attr:`newline`, which defaults to ``'\n'``.
@@ -516,8 +516,8 @@ class tube(Timeout, Logger):
     def recvline_startswith(self, delims, keepends = False, timeout = default):
         r"""recvline_startswith(delims, keepends = False, timeout = default) -> str
 
-        Keep recieving lines until one is found that starts with one of
-        `delims`.  Returns the last line recieved.
+        Keep receiving lines until one is found that starts with one of
+        `delims`.  Returns the last line received.
 
         If the request is not satisfied before ``timeout`` seconds pass,
         all data is buffered and an empty string (``''``) is returned.
@@ -554,8 +554,8 @@ class tube(Timeout, Logger):
     def recvline_endswith(self, delims, keepends = False, timeout = default):
         r"""recvline_endswith(delims, keepends = False, timeout = default) -> str
 
-        Keep recieving lines until one is found that starts with one of
-        `delims`.  Returns the last line recieved.
+        Keep receiving lines until one is found that starts with one of
+        `delims`.  Returns the last line received.
 
         If the request is not satisfied before ``timeout`` seconds pass,
         all data is buffered and an empty string (``''``) is returned.
@@ -669,7 +669,7 @@ class tube(Timeout, Logger):
         Receives data until EOF is reached.
         """
 
-        with self.waitfor('Recieving all data') as h:
+        with self.waitfor('Receiving all data') as h:
             l = len(self.buffer)
             with self.local(timeout):
                 try:
@@ -858,7 +858,7 @@ class tube(Timeout, Logger):
     def clean_and_log(self, timeout = 0.05):
         r"""clean_and_log(timeout = 0.05)
 
-        Works exactly as :meth:`pwnlib.tubes.tube.tube.clean`, but logs recieved
+        Works exactly as :meth:`pwnlib.tubes.tube.tube.clean`, but logs received
         data with :meth:`pwnlib.self.info`.
 
         Returns:

pwnlib/update.py

@@ -1,42 +1,83 @@
+"""
+# Pwntools Update
+
+In order to ensure that Pwntools users always have the latest and
+greatest version, Pwntools automatically checks for updates.
+
+Since this update check takes a moment, it is only performed once
+every week.  It can be permanently disabled via:
+
+.. code-block:: bash
+
+    $ echo never > ~/.pwntools-cache/update
+
+"""
 import datetime
 import json
-import pip
-import pkg_resources
 import os
 import time
+import xmlrpclib
+
+import packaging.version
 
 from .context import context
 from .log import getLogger
-from .version import __version__
 from .util.misc import read
 from .util.misc import write
 from .util.web import wget
+from .version import __version__
 
 log = getLogger(__name__)
 
-current_version = pkg_resources.parse_version(__version__)
+current_version = packaging.version.Version(__version__)
 package_name    = 'pwntools'
 package_repo    = 'Gallopsled/pwntools'
 update_freq     = datetime.timedelta(days=7).total_seconds()
 
-def available_on_github():
-    """Return True if an update is available on Github."""
+def available_on_github(prerelease=current_version.is_prerelease):
+    """Return True if an update is available on Github.
+
+    >>> available_on_github() # doctest: +ELLIPSIS
+    <Version('...')>
+    >>> available_on_github(prerelease=False).is_prerelease
+    False
+    """
     url = 'https://api.github.com/repos/%s/tags' % package_repo
 
     with context.quiet:
         tags = json.loads(wget(url))
 
-    return max(map(pkg_resources.parse_version, [t['name'] for t in tags]))
+    # 'pwntools-ancient' is a tag, but not a valid version.
+    # Handle this here, and for all potential tags which cause
+    # issues.
+    versions = []
+    for tag in [t['name'] for t in tags]:
+        try:
+            versions.append(packaging.version.Version(tag))
+        except Exception:
+            pass
 
-def available_on_pypi():
-    """Return True if an update is available on PyPI."""
-    search_command = pip.commands.search.SearchCommand()
-    options, _ = search_command.parse_args([package_name])
-    pypi_hits = search_command.search(package_name, options)
-    for hit in pip.commands.search.transform_hits(pypi_hits):
-        if hit['name'] != 'pwntools':
-            continue
-        return max(map(pkg_resources.parse_version, hit['versions']))
+    if not prerelease:
+        versions = filter(lambda v: not v.is_prerelease, versions)
+
+    return max(versions)
+
+def available_on_pypi(prerelease=current_version.is_prerelease):
+    """Return True if an update is available on PyPI.
+
+    >>> available_on_pypi() # doctest: +ELLIPSIS
+    <Version('...')>
+    >>> available_on_pypi(prerelease=False).is_prerelease
+    False
+    """
+    client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
+    versions = client.package_releases('pwntools', True)
+    versions = map(packaging.version.Version, versions)
+
+    if not prerelease:
+        versions = filter(lambda v: not v.is_prerelease, versions)
+
+    return max(versions)
 
 def cache_file():
     """Returns the path of the file used to cache update data, and ensures that it exists."""
@@ -61,18 +102,50 @@ def should_check():
         return False
     return time.time() > (last_check() + update_freq)
 
-def perform_check():
-    """Perform the update check, and report to the user."""
+def perform_check(prerelease=current_version.is_prerelease):
+    """Perform the update check, and report to the user.
+
+    Arguments:
+        prerelease(bool): Whether or not to include pre-release versions.
+
+    Returns:
+        A list of arguments to the update command.
+
+    >>> from packaging.version import Version
+    >>> pwnlib.update.current_version = Version("999.0.0")
+    >>> print perform_check()
+    None
+    >>> pwnlib.update.current_version = Version("0.0.0")
+    >>> perform_check() # doctest: +ELLIPSIS
+    ['pip', 'install', '-U', ...]
+
+    >>> def bail(*a): raise Exception()
+    >>> github = pwnlib.update.available_on_github
+    >>> pypi   = pwnlib.update.available_on_pypi
+
+    >>> pwnlib.update.available_on_github = bail
+    >>> perform_check(prerelease=False)
+    ['pip', 'install', '-U', 'pwntools']
+    >>> perform_check(prerelease=True)  # doctest: +ELLIPSIS
+    ['pip', 'install', '-U', 'pwntools...']
+    >>> pwnlib.update.available_on_github = github
+
+    >>> pwnlib.update.available_on_pypi = bail
+    >>> perform_check(prerelease=False)
+    ['pip', 'install', '-U', 'git+https://github.com/Gallopsled/pwntools.git@...']
+    >>> perform_check(prerelease=True)  # doctest: +ELLIPSIS
+    ['pip', 'install', '-U', 'git+https://github.com/Gallopsled/pwntools.git@...']
+    """
     pypi = current_version
     try:
-        pypi = available_on_pypi()
-    except:
+        pypi = available_on_pypi(prerelease)
+    except Exception:
         log.warning("An issue occurred while checking PyPI")
 
     github = current_version
     try:
-        github = available_on_github()
-    except:
+        github = available_on_github(prerelease)
+    except Exception:
         log.warning("An issue occurred while checking Github")
 
     best = max(pypi, github, current_version)
@@ -85,16 +158,28 @@ def perform_check():
         log.info("You have the latest version of Pwntools (%s)" % best)
         return
 
+    command = [
+        'pip',
+        'install',
+        '-U'
+    ]
+
     if best == pypi:
         where = 'pypi'
-        command = 'pip install -U %s' % package_name
+        pypi_package = package_name
+        if best.is_prerelease:
+            pypi_package += '==%s' % (best)
+        command += [pypi_package]
     else:
         where = 'GitHub'
-        command = 'pip install -U git+https://github.com/%s.git@%s' % (package_repo, github)
+        command += ['git+https://github.com/%s.git@%s' % (package_repo, github)]
+
+    command_str = ' '.join(command)
 
     log.info("A newer version of %s is available on %s (%s --> %s).\n" % (package_name, where, current_version, best) +
-             "Update with: $ %s" % command)
+             "Update with: $ %s" % command_str)
 
+    return command
 
 def check_automatically():
     if should_check():

pwnlib/util/fiddling.py

@@ -3,6 +3,7 @@ import base64
 import random
 import re
 import string
+import StringIO
 
 from . import lists
 from . import packing
@@ -487,7 +488,7 @@ def rol(n, k, word_size = None):
 def ror(n, k, word_size = None):
     """A simple wrapper around :func:`rol`, which negates the values of `k`."""
 
-    return ror(n, -k, word_size)
+    return rol(n, -k, word_size)
 
 def naf(n):
     """naf(int) -> int generator
@@ -566,15 +567,16 @@ def update_cyclic_pregenerated(size):
     global cyclic_pregen
     cyclic_pregen = cyclic(size)
 
-def hexdump_iter(s, width = 16, skip = True, hexii = False, begin = 0,
-                 style = None, highlight = None, cyclic=False):
+def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
+                 highlight=None, cyclic=False):
     """hexdump_iter(s, width = 16, skip = True, hexii = False, begin = 0,
-                    style = {}, highlight = []) -> str generator
+                    style = None, highlight = None, cyclic = False) -> str generator
 
-    Return a hexdump-dump of a string as a generator of lines.
+    Return a hexdump-dump of a string as a generator of lines.  Unless you have
+    massive amounts of data you probably want to use :meth:`hexdump`.
 
     Arguments:
-        s(str): The string to dump
+        fd(file): File object to dump.  Use :meth:`StringIO.StringIO` or :meth:`hexdump` to dump a string.
         width(int): The number of characters per line
         skip(bool): Set to True, if repeated lines should be replaced by a "*"
         hexii(bool): Set to True, if a hexii-dump should be returned instead of a hexdump.
@@ -584,7 +586,7 @@ def hexdump_iter(s, width = 16, skip = True, hexii = False, begin = 0,
         cyclic(bool): Attempt to skip consecutive, unmodified cyclic lines
 
     Returns:
-        A hexdump-dump in the form of a string.
+        A generator producing the hexdump-dump one line at a time.
     """
     style     = style or {}
     highlight = highlight or []
@@ -628,16 +630,18 @@ def hexdump_iter(s, width = 16, skip = True, hexii = False, begin = 0,
     if cyclic:
         update_cyclic_pregenerated(len(s))
 
-    chunks = lists.group(width, s)
-
-    for line, chunk in enumerate(chunks):
+    numb = 0
+    while True:
+        offset = begin + numb
+        chunk = fd.read(width)
+        if chunk == '':
+            break
+        numb += len(chunk)
         # If this chunk is the same as the last unique chunk,
         # use a '*' instead.
-        if line != 0 \
-        and line != len(chunks)-1 \
-        and skip \
-        and (last_unique == chunk \
-            or (cyclic and sequential_lines(last_unique, chunk))):
+        if skip and \
+           (last_unique == chunk or \
+            (cyclic and sequential_lines(last_unique, chunk))):
             last_unique = chunk
             if not skipping:
                 yield '*'
@@ -648,8 +652,7 @@ def hexdump_iter(s, width = 16, skip = True, hexii = False, begin = 0,
         last_unique = chunk
         skipping = False
 
-        # Cenerate contents for line
-        offset    = begin+line*width
+        # Generate contents for line
         hexbytes = ''
         printable = ''
         for i, b in enumerate(chunk):
@@ -672,13 +675,38 @@ def hexdump_iter(s, width = 16, skip = True, hexii = False, begin = 0,
         line = line_fmt % {'offset': offset, 'hexbytes': hexbytes, 'printable': printable}
         yield line
 
-    line = "%08x" % (len(s) + begin)
+    line = "%08x" % (begin + numb)
     yield line
 
-def hexdump(s, width = 16, skip = True, hexii = False, begin = 0,
-            style = None, highlight = None, cyclic=False):
+def hexdump(s, width=16, skip=True, hexii=False, begin=0,
+            style=None, highlight=None, cyclic=False):
+    """hexdump(s, width = 16, skip = True, hexii = False, begin = 0,
+               style = None, highlight = None, cyclic = False) -> str generator
+
+    Return a hexdump-dump of a string.
+
+    Arguments:
+        s(str): The data to hexdump.
+        width(int): The number of characters per line
+        skip(bool): Set to True, if repeated lines should be replaced by a "*"
+        hexii(bool): Set to True, if a hexii-dump should be returned instead of a hexdump.
+        begin(int):  Offset of the first byte to print in the left column
+        style(dict): Color scheme to use.
+        highlight(iterable): Byte values to highlight.
+        cyclic(bool): Attempt to skip consecutive, unmodified cyclic lines
+
+    Returns:
+        A hexdump-dump in the form of a string.
+"""
     s = packing.flat(s)
-    return '\n'.join(hexdump_iter(s, width, skip, hexii, begin, style, highlight, cyclic))
+    return '\n'.join(hexdump_iter(StringIO.StringIO(s),
+                                  width,
+                                  skip,
+                                  hexii,
+                                  begin,
+                                  style,
+                                  highlight,
+                                  cyclic))
 
 def negate(value, width = None):
     """

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '3.0.0'
+__version__ = '3.0.3'

setup.py

@@ -22,6 +22,7 @@ for scheme in INSTALL_SCHEMES.values():
 
 # Find all of the console scripts
 console_scripts = []
+
 for filename in glob.glob('pwnlib/commandline/*'):
     filename = os.path.basename(filename)
     filename, ext = os.path.splitext(filename)
@@ -34,7 +35,7 @@ for filename in glob.glob('pwnlib/commandline/*'):
 
 install_requires     = ['paramiko>=1.15.2',
                         'mako>=1.0.0',
-                        'pyelftools>=0.2.3',
+                        'pyelftools>=0.2.4',
                         'capstone',
                         'ropgadget>=5.3',
                         'pyserial>=2.7',
@@ -43,7 +44,9 @@ install_requires     = ['paramiko>=1.15.2',
                         'tox>=1.8.1',
                         'pygments>=2.0',
                         'pysocks',
-                        'python-dateutil']
+                        'python-dateutil',
+                        'pypandoc',
+                        'packaging']
 
 # This is a hack until somebody ports psutil to OpenBSD
 if platform.system() != 'OpenBSD':
@@ -56,10 +59,23 @@ if not os.path.exists(PythonH):
     print >> sys.stderr, "$ apt-get install python-dev"
     sys.exit(-1)
 
+# Convert README.md to reStructuredText for PyPI
+long_description = ''
+try:
+    import pypandoc
+    try:
+        pypandoc.get_pandoc_path()
+    except OSError:
+        pypandoc.download_pandoc()
+    long_description = pypandoc.convert_file('README.md', 'rst')
+except ImportError:
+    pass
+
+
 setup(
     name                 = 'pwntools',
     packages             = find_packages(),
-    version              = '3.0.0',
+    version              = '3.0.3',
     data_files           = [('',
                              ['LICENSE-pwntools.txt',
                              ]),
@@ -75,19 +91,32 @@ setup(
     },
     entry_points = {'console_scripts': console_scripts},
     scripts              = glob.glob("bin/*"),
-    description          = "CTF framework and exploit development library.",
-    author               = "Zach Riggle",
-    author_email         = "[email protected]",
+    description          = "Pwntools CTF framework and exploit development library.",
+    long_description     = long_description,
+    author               = "Gallopsled et al.",
+    author_email         = "#pwntools @ freenode.net",
     url                  = 'https://pwntools.com',
-    download_url         = "https://pwntools.com/tarball/master",
+    download_url         = "https://github.com/Gallopsled/pwntools/releases",
     install_requires     = install_requires,
-    license              = "Mostly MIT, some GPL/BSD, see LICENSE-pwntools.txt and LICENSE-pwntools.txt",
+    license              = "Mostly MIT, some GPL/BSD, see LICENSE-pwntools.txt",
+    keywords             = 'pwntools exploit ctf capture the flag binary wargame overflow stack heap defcon',
     classifiers          = [
-        'Topic :: Security',
+        'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
-        'Operating System :: OS Independent',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Science/Research',
+        'Intended Audience :: System Administrators',
         'License :: OSI Approved :: MIT License',
+        'Natural Language :: English',
+        'Operating System :: POSIX :: Linux',
         'Programming Language :: Python :: 2.7',
-        'Intended Audience :: Developers'
+        'Topic :: Security',
+        'Topic :: Software Development :: Assemblers',
+        'Topic :: Software Development :: Debuggers',
+        'Topic :: Software Development :: Disassemblers',
+        'Topic :: Software Development :: Embedded Systems',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Topic :: System :: System Shells',
+        'Topic :: Utilities',
     ]
 )