Release 4.7.1

This fixes a misbehaviour in environments where `XDG_CACHE_DIR` is not
set and `~/.cache` does not exist, e.g. in the docker image. We simply
attempt to create a temporary directory which will be registered with
`atexit` to be deleted on exit using `shutil.rmtree()`. A Python 3
approach for future would be to use a `TemporaryDirectory` instead which
would get cleaned up when the held object gets dropped out of the TLS
stack.

It is still possible for the `context.cache_dir` to be `None` if the
default directory exists but is not writable, which means some code
which uses this property is liable to blow up in such a circumstance.

.github/workflows/android.yml

@@ -3,7 +3,7 @@ name: Android Tests
 on: [push, pull_request]
 
 jobs:
-  test:
+  android-test:
     strategy:
       matrix:
         python-version: [3.8]

.github/workflows/ci.yml

@@ -81,6 +81,7 @@ jobs:
     - name: Install dependencies
       run: |
         pip install --upgrade pip
+        pip install --upgrade wheel
         pip install --upgrade flake8 appdirs
         python setup.py egg_info
         pip install --upgrade --editable .

CHANGELOG.md

@@ -11,7 +11,8 @@ The table below shows which release corresponds to each branch, and what date th
 | ---------------- | -------- | ---------------------- |
 | [4.9.0](#490)    | `dev`    |
 | [4.8.0](#480)    | `beta`   |
-| [4.7.0](#470)    | `stable` | Nov 15, 2021
+| [4.7.1](#471)    | `stable` | Apr 20, 2022
+| [4.7.0](#470)    |          | Nov 15, 2021
 | [4.6.0](#460)    |          | Jul 12, 2021
 | [4.5.1](#451)    |          | May 30, 2021
 | [4.5.0](#450)    |          | Apr 30, 2021
@@ -64,7 +65,15 @@ The table below shows which release corresponds to each branch, and what date th
 
 ## 4.9.0 (`dev`)
 
+- [#2011][2011] Fix tube's debug output of same byte compression
+- [#2023][2023] Support KDE Konsole in run_in_new_terminal function
+- [#2027][2027] Fix ELF.libc_start_main_return with glibc 2.34
+- [#2035][2035] Change Buffer's parent class to object
 
+[2011]: https://github.com/Gallopsled/pwntools/pull/2011
+[2023]: https://github.com/Gallopsled/pwntools/pull/2023
+[2027]: https://github.com/Gallopsled/pwntools/pull/2027
+[2035]: https://github.com/Gallopsled/pwntools/pull/2035
 
 ## 4.8.0 (`beta`)
 
@@ -78,7 +87,24 @@ The table below shows which release corresponds to each branch, and what date th
 [1939]: https://github.com/Gallopsled/pwntools/pull/1939
 [1981]: https://github.com/Gallopsled/pwntools/pull/1981
 
-## 4.7.0 (`stable`)
+## 4.7.1 (`stable`)
+
+- [#1784][1784] Use temporary cache directory when persistent cache cannot be used
+- [#1973][1973] ELF symbols can be looked up by bytes values
+- several bugfixes ([#2012][2012], [#2031][2031], [#1912][1912], [#1961][1961],
+[#2007][2007], [#2040][2040], [#2051][2051])
+
+[1784]: https://github.com/Gallopsled/pwntools/pull/1784
+[1912]: https://github.com/Gallopsled/pwntools/pull/1912
+[1961]: https://github.com/Gallopsled/pwntools/pull/1961
+[1973]: https://github.com/Gallopsled/pwntools/pull/1973
+[2007]: https://github.com/Gallopsled/pwntools/pull/2007
+[2012]: https://github.com/Gallopsled/pwntools/pull/2012
+[2031]: https://github.com/Gallopsled/pwntools/pull/2031
+[2040]: https://github.com/Gallopsled/pwntools/pull/2040
+[2051]: https://github.com/Gallopsled/pwntools/pull/2051
+
+## 4.7.0
 
 - [#1733][1733] Update libc headers -> more syscalls available!
 - [#1876][1876] add `self.message` and change `sys.exc_type` to `sys.exec_info()` in PwnlibException

docs/requirements.txt

@@ -12,7 +12,8 @@ pypandoc
 pyserial>=2.7
 requests>=2.5.1
 ropgadget>=5.3
-sphinx<3.4
+sphinx==1.8.6; python_version < '3.0.0'
+sphinx==4.5.0; python_version >= '3.0.0'
 sphinx_rtd_theme
 sphinxcontrib-napoleon
 sphinxcontrib-autoprogram<=0.1.5

pwnlib/asm.py

@@ -779,6 +779,11 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
            0:   60 00 00 00     nop
         >>> print(disasm(unhex('00000000'), arch='mips64'))
            0:   00000000        nop
+        >>> print(disasm(unhex('48b84141414141414100c3'), arch='amd64'))
+           0:   48 b8 41 41 41 41 41 41 41 00   movabs rax, 0x41414141414141
+           a:   c3                      ret
+        >>> print(disasm(unhex('00000000'), vma=0x80000000, arch='mips'))
+        80000000:       00000000        nop
     """
     result = ''
 
@@ -790,7 +795,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
 
     bfdarch = _bfdarch()
     bfdname = _bfdname()
-    objdump = _objdump() + ['-d', '--adjust-vma', str(vma), '-b', bfdname]
+    objdump = _objdump() + ['-w', '-d', '--adjust-vma', str(vma), '-b', bfdname]
     objcopy = _objcopy() + [
         '-I', 'binary',
         '-O', bfdname,
@@ -815,7 +820,7 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
         _run(objcopy + [step1, step2])
 
         output0 = _run(objdump + [step2])
-        output1 = output0.split('<.text>:\n')
+        output1 = re.split(r'<\.text(?:\+0x0)?>:\n', output0, flags=re.MULTILINE)
 
         if len(output1) != 2:
             log.error('Could not find .text in objdump output:\n%s' % output0)

pwnlib/context/__init__.py

@@ -6,17 +6,20 @@ contexts work properly and as expected.
 from __future__ import absolute_import
 from __future__ import division
 
+import atexit
 import collections
+import errno
 import functools
 import logging
 import os
+import os.path
 import platform
+import shutil
 import six
 import socket
-import stat
 import string
-import subprocess
 import sys
+import tempfile
 import threading
 import time
 
@@ -345,6 +348,10 @@ class ContextType(object):
         'binary': None,
         'bits': 32,
         'buffer_size': 4096,
+        'cache_dir_base': os.environ.get(
+            'XDG_CACHE_HOME',
+            os.path.join(os.path.expanduser('~'), '.cache')
+        ),
         'cyclic_alphabet': string.ascii_lowercase.encode(),
         'cyclic_size': 4,
         'delete_corefiles': False,
@@ -1269,6 +1276,21 @@ class ContextType(object):
         """
         return int(size)
 
+    @_validator
+    def cache_dir_base(self, new_base):
+        """Base directory to use for caching content.
+
+        Changing this to a different value will clear the `cache_dir` path
+        stored in TLS since a new path will need to be generated to respect the
+        new `cache_dir_base` value.
+        """
+
+        if new_base != self.cache_dir_base:
+            del self._tls["cache_dir"]
+        if os.access(new_base, os.F_OK) and not os.access(new_base, os.W_OK):
+            raise OSError(errno.EPERM, "Cache base dir is not writable")
+        return new_base
+
     @property
     def cache_dir(self):
         """Directory used for caching data.
@@ -1282,32 +1304,59 @@ class ContextType(object):
             >>> cache_dir is not None
             True
             >>> os.chmod(cache_dir, 0o000)
+            >>> del context._tls['cache_dir']
             >>> context.cache_dir is None
             True
             >>> os.chmod(cache_dir, 0o755)
             >>> cache_dir == context.cache_dir
             True
         """
-        xdg_cache_home = os.environ.get('XDG_CACHE_HOME') or \
-                         os.path.join(os.path.expanduser('~'), '.cache')
+        try:
+            # If the TLS already has a cache directory path, we return it
+            # without any futher checks since it must have been valid when it
+            # was set and if that has changed, hiding the TOCTOU here would be
+            # potentially confusing
+            return self._tls["cache_dir"]
+        except KeyError:
+            pass
 
-        if not os.access(xdg_cache_home, os.W_OK):
+        # Attempt to create a Python version specific cache dir and its parents
+        cache_dirname = '.pwntools-cache-%d.%d' % sys.version_info[:2]
+        cache_dirpath = os.path.join(self.cache_dir_base, cache_dirname)
+        try:
+            os.makedirs(cache_dirpath)
+        except OSError as exc:
+            # If we failed for any reason other than the cache directory
+            # already existing then we'll fall back to a temporary directory
+            # object which doesn't respect the `cache_dir_base`
+            if exc.errno != errno.EEXIST:
+                try:
+                    cache_dirpath = tempfile.mkdtemp(prefix=".pwntools-tmp")
+                except IOError:
+                    # This implies no good candidates for temporary files so we
+                    # have to return `None`
+                    return None
+                else:
+                    # Ensure the temporary cache dir is cleaned up on exit. A
+                    # `TemporaryDirectory` would do this better upon garbage
+                    # collection but this is necessary for Python 2 support.
+                    atexit.register(shutil.rmtree, cache_dirpath)
+        # By this time we have a cache directory which exists but we don't know
+        # if it is actually writable. Some wargames e.g. pwnable.kr have
+        # created dummy directories which cannot be modified by the user
+        # account (owned by root).
+        if os.access(cache_dirpath, os.W_OK):
+            # Stash this in TLS for later reuse
+            self._tls["cache_dir"] = cache_dirpath
+            return cache_dirpath
+        else:
             return None
 
-        cache = os.path.join(xdg_cache_home, '.pwntools-cache-%d.%d' % sys.version_info[:2])
-
-        if not os.path.exists(cache):
-            try:
-                os.mkdir(cache)
-            except OSError:
-                return None
-
-        # Some wargames e.g. pwnable.kr have created dummy directories
-        # which cannot be modified by the user account (owned by root).
-        if not os.access(cache, os.W_OK):
-            return None
-
-        return cache
+    @cache_dir.setter
+    def cache_dir(self, v):
+        if os.access(v, os.W_OK):
+            # Stash this in TLS for later reuse
+            self._tls["cache_dir"] = v
 
     @_validator
     def delete_corefiles(self, v):

pwnlib/elf/elf.py

@@ -157,6 +157,12 @@ class dotdict(dict):
         >>> x.bar.baz
         4
     """
+    def __missing__(self, name):
+        if isinstance(name, (bytes, bytearray)):
+            name = packing._decode(name)
+            return self[name]
+        raise KeyError(name)
+
     def __getattr__(self, name):
         if name in self:
             return self[name]

pwnlib/filepointer.py

@@ -193,7 +193,8 @@ class FileStructure(object):
             if isinstance(self.__getattr__(val), bytes):
                 structure += self.__getattr__(val).ljust(context.bytes, b'\x00')
             else:
-                structure += pack(self.__getattr__(val), self.length[val]*8)
+                if self.length[val] > 0:
+                    structure += pack(self.__getattr__(val), self.length[val]*8)
         return structure
 
     def struntil(self,v):

pwnlib/gdb.py

@@ -303,7 +303,7 @@ def _gdbserver_args(pid=None, path=None, args=None, which=None, env=None):
                 env_args.append(b'%s=%s' % (key, env.pop(key)))
             else:
                 env_args.append(b'%s=%s' % (key, env[key]))
-        gdbserver_args += ['--wrapper', 'env', '-i'] + env_args + ['--']
+        gdbserver_args += ['--wrapper', which('env'), '-i'] + env_args + ['--']
 
     gdbserver_args += ['localhost:0']
     gdbserver_args += args
@@ -915,7 +915,7 @@ def attach(target, gdbscript = '', exe = None, gdb_args = None, ssh = None, sysr
             cmd = ['sshpass', '-p', shell.password] + cmd
         if shell.keyfile:
             cmd += ['-i', shell.keyfile]
-        cmd += ['gdb', '-q', target.executable, target.pid, '-x', tmpfile]
+        cmd += ['gdb', '-q', target.executable, str(target.pid), '-x', tmpfile]
 
         misc.run_in_new_terminal(cmd)
         return

pwnlib/tubes/ssh.py

@@ -1523,29 +1523,24 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             local: Local directory
             remote: Remote directory
         """
-        remote   = remote or self.cwd
-
+        remote = packing._encode(remote or self.cwd)
 
         if self.sftp:
-            remote = str(self.sftp.normalize(remote))
+            remote = packing._encode(self.sftp.normalize(remote))
         else:
             with context.local(log_level='error'):
-                remote = self.system('readlink -f ' + sh_string(remote))
+                remote = self.system(b'readlink -f ' + sh_string(remote))
 
-        basename = os.path.basename(remote)
+        local = local or '.'
+        local = os.path.expanduser(local)
 
-
-        local    = local or '.'
-        local    = os.path.expanduser(local)
-
-        self.info("Downloading %r to %r" % (basename,local))
+        self.info("Downloading %r to %r" % (remote, local))
 
         with context.local(log_level='error'):
             remote_tar = self.mktemp()
-            cmd = 'tar -C %s -czf %s %s' % \
+            cmd = b'tar -C %s -czf %s .' % \
                   (sh_string(remote),
-                   sh_string(remote_tar),
-                   sh_string(basename))
+                   sh_string(remote_tar))
             tar = self.system(cmd)
 
             if 0 != tar.wait():
@@ -1686,10 +1681,8 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
             local(str): Local path to store the data.
                 By default, uses the current directory.
         """
-        if not self.sftp:
-            self.error("Cannot determine remote file type without SFTP")
-
-        with self.system('test -d ' + sh_string(file_or_directory)) as io:
+        file_or_directory = packing._encode(file_or_directory)
+        with self.system(b'test -d ' + sh_string(file_or_directory)) as io:
             is_dir = io.wait()
 
         if 0 == is_dir:

pwnlib/util/packing.py

@@ -1040,7 +1040,7 @@ def _need_text(s, level=1):
     if encoding == 'auto':
         for encoding in 'ASCII', 'UTF-8', 'ISO-8859-1':
             try:
-                s = s.decode(encoding)
+                s.decode(encoding)
             except UnicodeDecodeError:
                 pass
             else:

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '4.7.0'
+__version__ = '4.7.1'

setup.py

@@ -89,7 +89,7 @@ setup(
     name                 = 'pwntools',
     python_requires      = '>=2.7',
     packages             = find_packages(),
-    version              = '4.7.0',
+    version              = '4.7.1',
     data_files           = [('pwntools-doc',
                              glob.glob('*.md') + glob.glob('*.txt')),
                             ],