Release 3.0.4

Add all MD and TXT files to binary distribution; include correct file…

.travis_install.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash -e
+set -e
 local_deb_extract()
 {
     wget $1
@@ -17,7 +18,8 @@ get_binutils()
 get_qemu()
 {
     echo "Installing qemu"
-    QEMU_URL='https://mirrors.kernel.org/ubuntu/pool/universe/q/qemu/qemu-user-static_2.6%2bdfsg-3ubuntu1_amd64.deb'
+    QEMU_INDEX='http://packages.ubuntu.com/en/yakkety/amd64/qemu-user-static/download'
+    QEMU_URL=$(curl "$QEMU_INDEX" | grep kernel.org | grep -Eo 'http.*\.deb')
     local_deb_extract "$QEMU_URL"
 }
 
@@ -26,35 +28,35 @@ setup_travis()
     export PATH=$PWD/usr/bin:$PATH
     export LD_LIBRARY_PATH=$PWD/usr/lib
 
-    if [ ! -d usr/bin ];
+    # Install libbfd-multiarch and libopcodes-multiarch if not found in the cache
+    if [ ! -f usr/lib/libbfd-2.22-multiarch.so ];
     then
-        # Install our custom binutils
-        which arm-linux-as     || get_binutils arm
-        which mips-linux-as    || get_binutils mips
-        which powerpc-linux-as || get_binutils powerpc
-        which aarch64-linux-as || get_binutils aarch64
-
         # Install the multiarch binutils
-        local_deb_extract http://mirrors.mit.edu/ubuntu/ubuntu/pool/universe/b/binutils/binutils-multiarch_2.22-6ubuntu1_amd64.deb
+        local_deb_extract http://mirrors.mit.edu/ubuntu/pool/universe/b/binutils/binutils-multiarch_2.22-6ubuntu1_amd64.deb
+        pushd usr/lib
+        ln -sf libbfd-2.22-multiarch.so libbfd-2.22.so
+        ln -sf libopcodes-2.22-multiarch.so libopcodes-2.22.so
+        popd
     fi
 
-    if ! (which qemu-arm-static && qemu-arm-static -version | grep 2.6.0); then
+    # Install/upgrade qemu
+    if ! (which qemu-arm-static && qemu-arm-static -version | grep 2.6.1); then
         get_qemu
     fi
 
-    # Get rid of files we don't want cached
-    rm -rf usr/share
-
-    pushd usr/lib
-    ln -sf libbfd-2.22-multiarch.so libbfd-2.22.so
-    ln -sf libopcodes-2.22-multiarch.so libopcodes-2.22.so
-    popd
+    # Install our custom binutils
+    which arm-linux-gnu-as     || get_binutils arm
+    which mips-linux-gnu-as    || get_binutils mips
+    which powerpc-linux-gnu-as || get_binutils powerpc
 
+    # Test that the installs worked
     which arm-linux-gnu-as
     which mips-linux-gnu-as
     which powerpc-linux-gnu-as
-    which aarch64-linux-gnu-as
     which qemu-arm-static
+
+    # Get rid of files we don't want cached
+    rm -rf usr/share
 }
 
 setup_linux()

.travis_ssh_setup.sh

@@ -4,6 +4,8 @@
 #
 # All of the "conditional sudo" is to do container-based builds on
 # Travis which are much, much faster.
+set -e
+
 U=travis
 H=/home/$U
 
@@ -36,7 +38,7 @@ pubkey=$(cat ~/.ssh/$U.pub)
 
 # Set the authorized_keys entry to only permit login from localhost,
 # and only with
-USUDO mkdir $H/.ssh
+USUDO mkdir $H/.ssh || true
 USUDO tee -a $H/.ssh/authorized_keys <<EOF
 from="127.0.0.1" $pubkey
 EOF

CHANGELOG.md

@@ -3,17 +3,30 @@
 This changelog only includes added major features and changes. Bugfixes and
 minor changes are omitted.
 
-## 3.0.2 (September 6 2016)
+## 3.0.4 (September 19, 2016)
+
+**Current release**
+
+- Fixed a bug that made 3.0.3 uninstallable (Issue: #751, PR: #752)
+
+## 3.0.3 (September 18, 2016, broken release)
+
+- Fixed some performance and usability problems with the update system (Issues:
+  #723, #724, #736. PRs: #729, #738, #747).
+- Fixed a bug related to internals in pyelftools (PRs: #730, #746).
+- Fixed an issue with travis (Issue: #741, PRs: #743, #744, #745).
+
+## 3.0.2 (September 6, 2016)
 
 - Cherry-pick #695, as this was a regression-fix.
 - Added a fix for the update checker, as it would suggest prereleases as updates to stable releases.
 - Various documentation fixes.
 
-## 3.0.1 (August 20 2016)
+## 3.0.1 (August 20, 2016)
 
 A small bugfix release. There were a lot of references to the `master`-branch, however after 3.0.0 we use the names `stable`, `beta` and `dev` for our branches.
 
-## 3.0.0 (August 20 2016)
+## 3.0.0 (August 20, 2016)
 
 This was a large release (1305 commits since 2.2.0) with a lot of bugfixes and changes.  The Binjitsu project, a fork of Pwntools, was merged back into Pwntools.  As such, its features are now available here.
 
@@ -112,7 +125,7 @@ Android support via a new `adb` module, `context.device`, `context.adb_host`, an
 - Added a multi-threaded `bruteforce()` implementation, `mbruteforce()`.
 - Added `dealarm_shell()` helper to remove the effects of `alarm()` after you've popped a shell.
 
-## 2.2.0
+## 2.2.0 (January 5, 2015)
 
 This was a large release with a lot of bugfixes and changes. Only the most significant
 are mentioned here.

MANIFEST.in

@@ -1 +1,7 @@
-include *.txt
+graft examples
+graft extra
+include *.md *.txt *.sh *.yml MANIFEST.in
+recursive-include docs   *.rst *.png Makefile *.py *.txt
+recursive-include pwnlib *.py *.asm *.rst *.md *.txt *.sh
+recursive-include pwn    *.py *.asm *.rst *.md *.txt *.sh
+recursive-exclude *.pyc

pwnlib/elf/corefile.py

@@ -6,7 +6,6 @@ from elftools.common.py3compat import bytes2str
 from elftools.common.utils import roundup
 from elftools.common.utils import struct_parse
 from elftools.construct import CString
-
 from ..context import context
 from ..log import getLogger
 from ..tubes.tube import tube
@@ -30,11 +29,11 @@ def iter_notes(self):
     end = self['p_offset'] + self['p_filesz']
     while offset < end:
         note = struct_parse(
-            self._elfstructs.Elf_Nhdr,
+            self.elffile.structs.Elf_Nhdr,
             self.stream,
             stream_pos=offset)
         note['n_offset'] = offset
-        offset += self._elfstructs.Elf_Nhdr.sizeof()
+        offset += self.elffile.structs.Elf_Nhdr.sizeof()
         self.stream.seek(offset)
         # n_namesz is 4-byte aligned.
         disk_namesz = roundup(note['n_namesz'], 2)

pwnlib/update.py

@@ -1,54 +1,83 @@
+"""
+# Pwntools Update
+
+In order to ensure that Pwntools users always have the latest and
+greatest version, Pwntools automatically checks for updates.
+
+Since this update check takes a moment, it is only performed once
+every week.  It can be permanently disabled via:
+
+.. code-block:: bash
+
+    $ echo never > ~/.pwntools-cache/update
+
+"""
 import datetime
 import json
-import pip
-import pkg_resources
 import os
 import time
+import xmlrpclib
+
+import packaging.version
 
 from .context import context
 from .log import getLogger
-from .version import __version__
 from .util.misc import read
 from .util.misc import write
 from .util.web import wget
+from .version import __version__
 
 log = getLogger(__name__)
 
-current_version = pkg_resources.parse_version(__version__)
+current_version = packaging.version.Version(__version__)
 package_name    = 'pwntools'
 package_repo    = 'Gallopsled/pwntools'
 update_freq     = datetime.timedelta(days=7).total_seconds()
 
-def available_on_github(prerelease=False):
-    """Return True if an update is available on Github."""
+def available_on_github(prerelease=current_version.is_prerelease):
+    """Return True if an update is available on Github.
+
+    >>> available_on_github() # doctest: +ELLIPSIS
+    <Version('...')>
+    >>> available_on_github(prerelease=False).is_prerelease
+    False
+    """
     url = 'https://api.github.com/repos/%s/tags' % package_repo
 
     with context.quiet:
-        tags = json.loads(wget(url))
+        tags = json.loads(wget(url, timeout = 60))
 
-
-    versions = map(pkg_resources.parse_version, [t['name'] for t in tags])
+    # 'pwntools-ancient' is a tag, but not a valid version.
+    # Handle this here, and for all potential tags which cause
+    # issues.
+    versions = []
+    for tag in [t['name'] for t in tags]:
+        try:
+            versions.append(packaging.version.Version(tag))
+        except Exception:
+            pass
 
     if not prerelease:
         versions = filter(lambda v: not v.is_prerelease, versions)
 
     return max(versions)
 
-def available_on_pypi(prerelease=False):
-    """Return True if an update is available on PyPI."""
-    search_command = pip.commands.search.SearchCommand()
-    options, _ = search_command.parse_args([package_name])
-    pypi_hits = search_command.search(package_name, options)
-    for hit in pip.commands.search.transform_hits(pypi_hits):
-        if hit['name'] != 'pwntools':
-            continue
+def available_on_pypi(prerelease=current_version.is_prerelease):
+    """Return True if an update is available on PyPI.
 
-        versions = map(pkg_resources.parse_version, hit['versions'])
+    >>> available_on_pypi() # doctest: +ELLIPSIS
+    <Version('...')>
+    >>> available_on_pypi(prerelease=False).is_prerelease
+    False
+    """
+    client = xmlrpclib.ServerProxy('https://pypi.python.org/pypi')
+    versions = client.package_releases('pwntools', True)
+    versions = map(packaging.version.Version, versions)
 
-        if not prerelease:
-            versions = filter(lambda v: not v.is_prerelease, versions)
+    if not prerelease:
+        versions = filter(lambda v: not v.is_prerelease, versions)
 
-        return max(versions)
+    return max(versions)
 
 def cache_file():
     """Returns the path of the file used to cache update data, and ensures that it exists."""
@@ -73,18 +102,50 @@ def should_check():
         return False
     return time.time() > (last_check() + update_freq)
 
-def perform_check(prerelease=False):
-    """Perform the update check, and report to the user."""
+def perform_check(prerelease=current_version.is_prerelease):
+    """Perform the update check, and report to the user.
+
+    Arguments:
+        prerelease(bool): Whether or not to include pre-release versions.
+
+    Returns:
+        A list of arguments to the update command.
+
+    >>> from packaging.version import Version
+    >>> pwnlib.update.current_version = Version("999.0.0")
+    >>> print perform_check()
+    None
+    >>> pwnlib.update.current_version = Version("0.0.0")
+    >>> perform_check() # doctest: +ELLIPSIS
+    ['pip', 'install', '-U', ...]
+
+    >>> def bail(*a): raise Exception()
+    >>> github = pwnlib.update.available_on_github
+    >>> pypi   = pwnlib.update.available_on_pypi
+
+    >>> pwnlib.update.available_on_github = bail
+    >>> perform_check(prerelease=False)
+    ['pip', 'install', '-U', 'pwntools']
+    >>> perform_check(prerelease=True)  # doctest: +ELLIPSIS
+    ['pip', 'install', '-U', 'pwntools...']
+    >>> pwnlib.update.available_on_github = github
+
+    >>> pwnlib.update.available_on_pypi = bail
+    >>> perform_check(prerelease=False)
+    ['pip', 'install', '-U', 'git+https://github.com/Gallopsled/pwntools.git@...']
+    >>> perform_check(prerelease=True)  # doctest: +ELLIPSIS
+    ['pip', 'install', '-U', 'git+https://github.com/Gallopsled/pwntools.git@...']
+    """
     pypi = current_version
     try:
         pypi = available_on_pypi(prerelease)
-    except:
+    except Exception:
         log.warning("An issue occurred while checking PyPI")
 
     github = current_version
     try:
         github = available_on_github(prerelease)
-    except:
+    except Exception:
         log.warning("An issue occurred while checking Github")
 
     best = max(pypi, github, current_version)
@@ -97,15 +158,26 @@ def perform_check(prerelease=False):
         log.info("You have the latest version of Pwntools (%s)" % best)
         return
 
+    command = [
+        'pip',
+        'install',
+        '-U'
+    ]
+
     if best == pypi:
         where = 'pypi'
-        command = 'pip install -U %s' % package_name
+        pypi_package = package_name
+        if best.is_prerelease:
+            pypi_package += '==%s' % (best)
+        command += [pypi_package]
     else:
         where = 'GitHub'
-        command = 'pip install -U git+https://github.com/%s.git@%s' % (package_repo, github)
+        command += ['git+https://github.com/%s.git@%s' % (package_repo, github)]
+
+    command_str = ' '.join(command)
 
     log.info("A newer version of %s is available on %s (%s --> %s).\n" % (package_name, where, current_version, best) +
-             "Update with: $ %s" % command)
+             "Update with: $ %s" % command_str)
 
     return command
 

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '3.0.2'
+__version__ = '3.0.4'

setup.py

@@ -22,6 +22,7 @@ for scheme in INSTALL_SCHEMES.values():
 
 # Find all of the console scripts
 console_scripts = []
+
 for filename in glob.glob('pwnlib/commandline/*'):
     filename = os.path.basename(filename)
     filename, ext = os.path.splitext(filename)
@@ -34,7 +35,7 @@ for filename in glob.glob('pwnlib/commandline/*'):
 
 install_requires     = ['paramiko>=1.15.2',
                         'mako>=1.0.0',
-                        'pyelftools>=0.2.3',
+                        'pyelftools>=0.2.4',
                         'capstone',
                         'ropgadget>=5.3',
                         'pyserial>=2.7',
@@ -43,7 +44,9 @@ install_requires     = ['paramiko>=1.15.2',
                         'tox>=1.8.1',
                         'pygments>=2.0',
                         'pysocks',
-                        'python-dateutil']
+                        'python-dateutil',
+                        'pypandoc',
+                        'packaging']
 
 # This is a hack until somebody ports psutil to OpenBSD
 if platform.system() != 'OpenBSD':
@@ -56,13 +59,25 @@ if not os.path.exists(PythonH):
     print >> sys.stderr, "$ apt-get install python-dev"
     sys.exit(-1)
 
+# Convert README.md to reStructuredText for PyPI
+long_description = ''
+try:
+    import pypandoc
+    try:
+        pypandoc.get_pandoc_path()
+    except OSError:
+        pypandoc.download_pandoc()
+    long_description = pypandoc.convert_file('README.md', 'rst')
+except ImportError:
+    pass
+
+
 setup(
     name                 = 'pwntools',
     packages             = find_packages(),
-    version              = '3.0.2',
+    version              = '3.0.4',
     data_files           = [('',
-                             ['LICENSE-pwntools.txt',
-                             ]),
+                             glob.glob('*.md') + glob.glob('*.txt')),
                             ],
     package_data         = {
         'pwnlib': [
@@ -75,19 +90,32 @@ setup(
     },
     entry_points = {'console_scripts': console_scripts},
     scripts              = glob.glob("bin/*"),
-    description          = "CTF framework and exploit development library.",
+    description          = "Pwntools CTF framework and exploit development library.",
+    long_description     = long_description,
     author               = "Gallopsled et al.",
     author_email         = "#pwntools @ freenode.net",
     url                  = 'https://pwntools.com',
     download_url         = "https://github.com/Gallopsled/pwntools/releases",
     install_requires     = install_requires,
     license              = "Mostly MIT, some GPL/BSD, see LICENSE-pwntools.txt",
+    keywords             = 'pwntools exploit ctf capture the flag binary wargame overflow stack heap defcon',
     classifiers          = [
-        'Topic :: Security',
+        'Development Status :: 5 - Production/Stable',
         'Environment :: Console',
-        'Operating System :: OS Independent',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Science/Research',
+        'Intended Audience :: System Administrators',
         'License :: OSI Approved :: MIT License',
+        'Natural Language :: English',
+        'Operating System :: POSIX :: Linux',
         'Programming Language :: Python :: 2.7',
-        'Intended Audience :: Developers'
+        'Topic :: Security',
+        'Topic :: Software Development :: Assemblers',
+        'Topic :: Software Development :: Debuggers',
+        'Topic :: Software Development :: Disassemblers',
+        'Topic :: Software Development :: Embedded Systems',
+        'Topic :: Software Development :: Libraries :: Python Modules',
+        'Topic :: System :: System Shells',
+        'Topic :: Utilities',
     ]
 )