Merge pull request #2796 from peace-maker/stable_backports

Backport bugfixes from 5.0 to stable

.github/workflows/ci.yml

@@ -5,7 +5,7 @@ jobs:
   test:
     strategy:
       matrix:
-        python_version: ['3.10', '3.12']
+        python_version: ['3.10', '3.12', '3.13', '3.14', '3.15']
         os: [ubuntu-latest]
         include:
           - python_version: '2.7'
@@ -150,6 +150,8 @@ jobs:
         pip install --upgrade wheel build
         pip install --upgrade flake8 appdirs
         pip install --upgrade --editable .
+        # Pin to known working version until 2.1.5 is out. https://github.com/unicorn-engine/unicorn/issues/2134
+        pip install 'unicorn==2.1.2'
 
     - name: Sanity checks
       run:  PWNLIB_NOTERM=1 python -bb -c 'from pwn import *; print(pwnlib.term.term_mode)'
@@ -285,7 +287,7 @@ jobs:
       run: |
         pip install --upgrade pip
         pip install --upgrade --editable .
-    
+
     - name: Sanity checks
       run: |
         python -bb -c 'from pwn import *'

.github/workflows/pylint.yml

@@ -24,7 +24,7 @@ jobs:
     - name: PyLint
       run: |
         set -x
-        pip install pylint
+        pip install 'pylint<4'
         pip install --upgrade -e .
         run_pylint() { pylint --exit-zero --errors-only pwnlib -f parseable | cut -d ' ' -f2- | sed 's/line [0-9]\+/line XXXX/g'; }
         run_pylint > current.txt

CHANGELOG.md

@@ -75,18 +75,14 @@ The table below shows which release corresponds to each branch, and what date th
 
 ## 5.0.0 (`dev`)
 
-- [#2598][2598] aarch64: Fix ABI definition
 - [#2419][2419] riscv: avoid compressed instructions (if you need compressed, use .option rvc)
 - [#2551][2551] Detect when kitty is being used as terminal
 - [#2519][2519] Drop Python 2.7 support / Require Python 3.10
 - [#2507][2507] Add `+LINUX` and `+WINDOWS` doctest options and start proper testing on Windows
 - [#2522][2522] Support starting a kitty debugging window with the 'kitten' command
-- [#2524][2524] Raise EOFError during `process.recv` when stdout closes on Windows
 - [#2526][2526] Properly make use of extra arguments in `packing` utilities. `sign` parameter requires keyword syntax to specify it.
 - [#2517][2517] Allow to passthru kwargs on `ssh.__getattr__` convenience function to fix SSH motd problems
-- [#2530][2530] Do NOT error when passing directory arguments in `checksec` commandline tool.
 - [#2529][2529] Add LoongArch64 support
-- [#2506][2506] ROP: fix `ROP(ELF(exe)).leave` is `None` in some ELF
 - [#2504][2504] doc: add example case for `tuple` (host, port pair) in `gdb.attach`
 - [#2546][2546] ssh: Allow passing disabled_algorithms keyword argument from ssh to paramiko
 - [#2538][2538] Add `ssh -L` / `ssh.connect_remote()` workaround when `AllowTcpForwarding` is disabled
@@ -96,24 +92,18 @@ The table below shows which release corresponds to each branch, and what date th
 - [#2590][2590] Add support for finding corefiles under WSL2
 - [#2496][2496] Add linux ko file search support
 - [#2542][2542] Decode `_IO_*` flags in `FileStructure` member
-- [#2592][2592] pwnlib.config: Fix customization of `context.timeout`
 - [#2608][2608] Abort on `libcdb file libc.so --unstrip` if eu-unstrip is not installed
 - [#2611][2611] Cleanup `pwnlib.lexer` exports and imports
-- [#2610][2610] Fix `log.progress` ignoring `context.log_console`
 - [#2615][2615] tube/process: Fix redirecting stderr to stdout on Windows
 
-[2598]: https://github.com/Gallopsled/pwntools/pull/2598
 [2419]: https://github.com/Gallopsled/pwntools/pull/2419
 [2551]: https://github.com/Gallopsled/pwntools/pull/2551
 [2519]: https://github.com/Gallopsled/pwntools/pull/2519
 [2507]: https://github.com/Gallopsled/pwntools/pull/2507
 [2522]: https://github.com/Gallopsled/pwntools/pull/2522
-[2524]: https://github.com/Gallopsled/pwntools/pull/2524
 [2526]: https://github.com/Gallopsled/pwntools/pull/2526
 [2517]: https://github.com/Gallopsled/pwntools/pull/2517
-[2530]: https://github.com/Gallopsled/pwntools/pull/2530
 [2529]: https://github.com/Gallopsled/pwntools/pull/2529
-[2506]: https://github.com/Gallopsled/pwntools/pull/2506
 [2504]: https://github.com/Gallopsled/pwntools/pull/2504
 [2546]: https://github.com/Gallopsled/pwntools/pull/2546
 [2538]: https://github.com/Gallopsled/pwntools/pull/2538
@@ -123,12 +113,66 @@ The table below shows which release corresponds to each branch, and what date th
 [2590]: https://github.com/Gallopsled/pwntools/pull/2590
 [2496]: https://github.com/Gallopsled/pwntools/pull/2496
 [2542]: https://github.com/Gallopsled/pwntools/pull/2542
-[2592]: https://github.com/Gallopsled/pwntools/pull/2592
 [2608]: https://github.com/Gallopsled/pwntools/pull/2608
 [2611]: https://github.com/Gallopsled/pwntools/pull/2611
-[2610]: https://github.com/Gallopsled/pwntools/pull/2610
 [2615]: https://github.com/Gallopsled/pwntools/pull/2615
 
+## 4.15.1
+
+- [#2694][2694] fix: pad bytes fields to correct field size in FileStructure
+- [#2701][2701] Fix `adb._build_date()` crash on devices with non-standard locale date strings
+- [#2721][2721] Allow running with Unicorn 2.1.[34] but throw when emulating MIPS
+- [#2773][2773] Disable debuginfod doctests in libcdb.unstrip_libc
+- [#2772][2772] fix(packing): `dd()` no longer crashes on a `bytearray` destination
+- [#2787][2787] gdb.corefile: Run gdb directly to prevent detach race
+- [#2795][2795] regsort: Fix self-assignment detection through register aliases
+- [#2524][2524] Raise EOFError during `process.recv` when stdout closes on Windows
+- [#2530][2530] Do NOT error when passing directory arguments in `checksec` commandline tool.
+- [#2506][2506] ROP: fix `ROP(ELF(exe)).leave` is `None` in some ELF
+- [#2592][2592] pwnlib.config: Fix customization of `context.timeout`
+- [#2598][2598] aarch64: Fix ABI definition
+- [#2610][2610] Fix `log.progress` ignoring `context.log_console`
+- [#2627][2627] Remove `pwnlib.util.iters.lookahead` (broken anyway, fails on Python 3.14+)
+- [#2653][2653] Add aarch64_be qemu mapping
+- [#2642][2642] Remove `pip` as explicit dependency
+- [#2646][2646] fix(libcdb-cli): return early if no matched libc found
+- [#2669][2669] asm: try native binutils before fallback architectures
+- [#2677][2677] refactor: replace unsafe eval with safeeval.const in ROP cache loading
+- [#2699][2699] Fix `tty` and `raw` arguments in `ssh.process()`
+- [#2688][2688] Close SSH client connection when authentication failed
+- [#2704][2704] ssh: Fix distro lookup on Ubuntu 24.04+
+- [#2762][2762] fix(srop): correct amd64 SigreturnFrame `uc_sigmask` offset
+- [#2767][2767] Support paginated libc.rip API requests to include all search results
+- [#2739][2739] shellcraft: migrate lazy importer to find_spec for Python 3.12+
+- [#2792][2792] Fix deprecation warning for not using `RLIM_INFINITY` in `resource.setrlimit` (Python 3.15)
+
+[2694]: https://github.com/Gallopsled/pwntools/pull/2694
+[2701]: https://github.com/Gallopsled/pwntools/pull/2701
+[2721]: https://github.com/Gallopsled/pwntools/pull/2721
+[2773]: https://github.com/Gallopsled/pwntools/pull/2773
+[2772]: https://github.com/Gallopsled/pwntools/pull/2772
+[2787]: https://github.com/Gallopsled/pwntools/pull/2787
+[2795]: https://github.com/Gallopsled/pwntools/pull/2795
+[2524]: https://github.com/Gallopsled/pwntools/pull/2524
+[2530]: https://github.com/Gallopsled/pwntools/pull/2530
+[2506]: https://github.com/Gallopsled/pwntools/pull/2506
+[2592]: https://github.com/Gallopsled/pwntools/pull/2592
+[2598]: https://github.com/Gallopsled/pwntools/pull/2598
+[2610]: https://github.com/Gallopsled/pwntools/pull/2610
+[2627]: https://github.com/Gallopsled/pwntools/pull/2627
+[2653]: https://github.com/Gallopsled/pwntools/pull/2653
+[2642]: https://github.com/Gallopsled/pwntools/pull/2642
+[2646]: https://github.com/Gallopsled/pwntools/pull/2646
+[2669]: https://github.com/Gallopsled/pwntools/pull/2669
+[2677]: https://github.com/Gallopsled/pwntools/pull/2677
+[2699]: https://github.com/Gallopsled/pwntools/pull/2699
+[2688]: https://github.com/Gallopsled/pwntools/pull/2688
+[2704]: https://github.com/Gallopsled/pwntools/pull/2704
+[2762]: https://github.com/Gallopsled/pwntools/pull/2762
+[2767]: https://github.com/Gallopsled/pwntools/pull/2767
+[2739]: https://github.com/Gallopsled/pwntools/pull/2739
+[2792]: https://github.com/Gallopsled/pwntools/pull/2792
+
 ## 4.15.0 (`stable`)
 
 - [#2508][2508] Ignore a warning when compiling with asm on nix
@@ -193,7 +237,7 @@ The table below shows which release corresponds to each branch, and what date th
 [2612]: https://github.com/Gallopsled/pwntools/pull/2612
 [2624]: https://github.com/Gallopsled/pwntools/pull/2624
 
-## 4.14.1 (`stable`)
+## 4.14.1
 
 - [#2451][2451] Show symbols defined to value 0 (start of file)
 - [#2533][2533] Fix installation on Python 3.5 and lower
@@ -644,7 +688,7 @@ In memoriam — [Zach Riggle][zach] — long time contributor and maintainer of
 [fecf9f]: http://github.com/Gallopsled/pwntools/commit/fecf9f
 [1454]: https://github.com/Gallopsled/pwntools/pull/1454
 
-## 4.1.7 (`stable`)
+## 4.1.7
 
 - [#1615][1615] Fix aarch64 pushstr and pushstr_array
 

docs/requirements.txt

@@ -8,7 +8,6 @@ intervaltree
 isort
 mako>=1.0.0
 paramiko>=1.15.2
-pip>=6.0.8
 pyelftools>=0.29, <0.30; python_version<'3'
 pyelftools>=0.29; python_version>='3'
 pygments>=2.0
@@ -19,6 +18,6 @@ psutil
 requests>=2.5.1
 ropgadget>=5.3
 sphinx==1.8.6; python_version<'3'
-sphinx>=7.0.0; python_version>='3'
+sphinx>=8.1.3, <9; python_version>='3'
 sphinx_rtd_theme
 sphinxcontrib-autoprogram<=0.1.5

docs/source/intro.rst

@@ -34,6 +34,8 @@ This exposes a standard interface to talk to processes, sockets, serial ports,
 and all manner of things, along with some nifty helpers for common tasks.
 For example, remote connections via :mod:`pwnlib.tubes.remote`.
 
+::
+
     >>> conn = remote('ftp.ubuntu.com',21)
     >>> conn.recvline() # doctest: +ELLIPSIS
     b'220 ...'

pwnlib/abi.py

@@ -142,7 +142,7 @@ class SigreturnABI(SyscallABI):
 linux_i386   = ABI('esp', [], 4, 0)
 linux_amd64  = ABI('rsp', ['rdi','rsi','rdx','rcx','r8','r9'], 8, 0)
 linux_arm    = ABI('sp', ['r0', 'r1', 'r2', 'r3'], 8, 0)
-linux_aarch64 = ABI('sp', ['x0', 'x1', 'x2', 'x3'], 16, 0)
+linux_aarch64 = ABI('sp', ['x0', 'x1', 'x2', 'x3', 'x4', 'x5', 'x6', 'x7'], 16, 0)
 linux_mips  = ABI('$sp', ['$a0','$a1','$a2','$a3'], 4, 0)
 linux_ppc = ABI('sp', ['r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10'], 4, 0)
 linux_ppc64 = ABI('sp', ['r3', 'r4', 'r5', 'r6', 'r7', 'r8', 'r9', 'r10'], 8, 0)

pwnlib/adb/adb.py

@@ -1275,8 +1275,28 @@ properties = Property()
 
 def _build_date():
     """Returns the build date in the form YYYY-MM-DD as a string"""
+    import datetime
+
+    # Use ro.build.date.utc (integer epoch seconds) which is set by the
+    # AOSP build system and available on all standard Android devices.
+    # This avoids ro.build.date which is locale-dependent and can contain
+    # non-ASCII characters that dateutil cannot parse.  See #2513.
+    utc = getprop('ro.build.date.utc')
+    if utc and utc.strip().isdigit():
+        try:
+            as_datetime = datetime.datetime.fromtimestamp(int(utc.strip()), dateutil.tz.UTC)
+            return as_datetime.strftime('%Y-%b-%d')
+        except (OSError, OverflowError, ValueError):
+            pass
+
+    # Fallback for non-standard builds missing ro.build.date.utc.
     as_string = getprop('ro.build.date')
-    as_datetime =  dateutil.parser.parse(as_string)
+    if not as_string:
+        return ''
+    try:
+        as_datetime = dateutil.parser.parse(as_string)
+    except (ValueError, OverflowError):
+        return as_string
     return as_datetime.strftime('%Y-%b-%d')
 
 def find_ndk_project_root(source):

pwnlib/asm.py

@@ -188,29 +188,36 @@ def which_binutils(util, check_version=False):
 
     # Fix up pwntools vs Debian triplet naming, and account
     # for 'thumb' being its own pwntools architecture.
-    arches = [arch] + {
+    aliases = {
         'thumb':  ['arm',    'aarch64'],
-        'i386':   ['x86_64', 'amd64'],
         'i686':   ['x86_64', 'amd64'],
-        'amd64':  ['x86_64', 'i386'],
+        'amd64':  ['x86_64'],
+    }.get(arch, [])
+
+    # Some binutils can support multiple architectures. Try them as fallbacks.
+    fallback_arches = {
+        'i386':   ['x86_64', 'amd64'],
+        'amd64':  ['i386'],
+        'arm':  ['aarch64'],
         'mips': ['mipsel'],
         'mipsel': ['mips'],
-        'mips64': ['mips', 'mipsel'],
-        'mips64el': ['mipsel', 'mips'],
+        'mips64': ['mips64el', 'mips', 'mipsel'],
+        'mips64el': ['mips64', 'mipsel', 'mips'],
         'powerpc64': ['powerpc'],
         'sparc64': ['sparc'],
         'riscv32': ['riscv64', 'riscv'],
         'riscv64': ['riscv32', 'riscv'],
     }.get(arch, [])
+    arches = [arch] + aliases + fallback_arches
 
     # If one of the candidate architectures matches the native
-    # architecture, use that as a last resort.
+    # architecture, use that as a last resort before trying fallbacks.
     machine = platform.machine()
     machine = 'i386' if machine == 'i686' else machine
     try:
         with context.local(arch = machine):
             if context.arch in arches:
-                arches.append(None)
+                arches = [arch] + aliases + [None] + fallback_arches
     except AttributeError:
         log.warn_once("Your local binutils won't be used because architecture %r is not supported." % machine)
 
@@ -223,6 +230,8 @@ def which_binutils(util, check_version=False):
     if platform.system() == 'Windows':
         utils = [util + '.exe']
 
+    # Try the explicit tools for the target architecture first,
+    # then the native one optionally, then fallbacks.
     for arch in arches:
         for gutil in utils:
             # e.g. objdump

pwnlib/commandline/checksec.py

@@ -15,7 +15,6 @@ parser = common.parser_commands.add_parser(
 parser.add_argument(
     'elf',
     nargs='*',
-    type=argparse.FileType('rb'),
     help='Files to check'
 )
 parser.add_argument(
@@ -23,12 +22,11 @@ parser.add_argument(
     nargs='*',
     dest='elf2',
     metavar='elf',
-    type=argparse.FileType('rb'),
     help='File to check (for compatibility with checksec.sh)'
 )
 
 def main(args):
-    files  = args.elf or args.elf2 or []
+    files = args.elf or args.elf2 or []
 
     if not files:
         parser.print_usage()
@@ -36,9 +34,9 @@ def main(args):
 
     for f in files:
         try:
-            e = ELF(f.name)
+            e = ELF(f)
         except Exception as e:
-            print("{name}: {error}".format(name=f.name, error=e))
+            print("{name}: {error}".format(name=f, error=e))
 
 if __name__ == '__main__':
     common.main(__file__, main)

pwnlib/commandline/libcdb.py

@@ -240,6 +240,9 @@ def main(args):
         symbols = {pairs[i]:pairs[i+1] for i in range(0, len(pairs), 2)}
         matched_libcs = libcdb.search_by_symbol_offsets(symbols, offline_only=args.offline_only, return_raw=True)
 
+        if not matched_libcs:
+            return
+
         for libc in matched_libcs:
             print_libc_info(libc)
             if args.download_libc:

pwnlib/commandline/template.py

@@ -89,7 +89,7 @@ def get_docker_image_libraries():
             if not (libc and ld):
                 progress.failure("Could not find libraries")
                 return None, None
-            
+
             progress.status("Copying libraries to current directory")
             for filename, basename in zip((libc, ld), (libc_basename, ld_basename)):
                 cat_command = ["-c", "chroot %s /bin/sh -c '/bin/cat %s'" % (chroot_dir, filename)]
@@ -108,8 +108,9 @@ def get_docker_image_libraries():
                 write(basename, contents)
 
         except subprocess.CalledProcessError as e:
-            print(e.stderr.decode())
-            log.error("docker failed with status: %d" % e.returncode)
+            print(e.stderr.decode(), file=sys.stderr)
+            progress.failure("docker failed with status: %d" % e.returncode)
+            return None, None
 
         progress.success("Retrieved libraries from Docker image")
     return libc_basename, ld_basename

pwnlib/context/__init__.py

@@ -1748,7 +1748,7 @@ def update_context_defaults(section):
 
         default = ContextType.defaults[key]
 
-        if isinstance(default, six.string_types + six.integer_types + (tuple, list, dict)):
+        if isinstance(default, six.string_types + six.integer_types + (float, tuple, list, dict)):
             value = safeeval.expr(value)
         else:
             log.warn("Unsupported configuration option %r in section %r" % (key, 'context'))

pwnlib/elf/elf.py

@@ -764,7 +764,7 @@ class ELF(ELFFile):
             (see :attr:`.ELF.libs`).
         """
         for lib in self.libs:
-            if '/libc.' in lib or '/libc-' in lib:
+            if ('/libc.' in lib or '/libc-' in lib) and lib != self.path:
                 return ELF(lib, self._print_checksec)
 
     def _populate_libraries(self):
@@ -2123,7 +2123,7 @@ class ELF(ELFFile):
             "NX:".ljust(12) + {
                 True:  green("NX enabled"),
                 False: red("NX disabled"),
-                None: yellow("NX unknown - GNU_STACK missing"),
+                None:  yellow("NX enabled on new kernels"),
             }[self.nx],
             "PIE:".ljust(12) + {
                 True: green("PIE enabled"),

pwnlib/elf/plt.py

@@ -78,6 +78,13 @@ def __ensure_memory_to_run_unicorn():
 def prepare_unicorn_and_context(elf, got, address, data):
     import unicorn as U
 
+    if elf.arch in ('mips', 'mips64'):
+        # Unicorn 2.1.3 and 2.1.4 have a crashing bug when trying to emulate MIPS.
+        # https://github.com/unicorn-engine/unicorn/issues/2134
+        if U.UC_VERSION_MAJOR == 2 and U.UC_VERSION_MINOR == 1 and U.UC_VERSION_PATCH in (3, 4):
+            raise OSError("Unicorn Engine 2.1.3 and 2.1.4 have a crashing bug when emulating MIPS, please install an earlier or later version: https://github.com/unicorn-engine/unicorn/issues/2134")
+
+
     __ensure_memory_to_run_unicorn()
 
     # Instantiate the emulator with the correct arguments for the current

pwnlib/filepointer.py

@@ -156,6 +156,18 @@ class FileStructure(object):
          _wide_data: 0xdeadbeef
          unknown2: 0x0
          vtable: 0x0}
+
+        Bytes fields are padded to the correct field size.
+        For example, unknown2 is 56 bytes on i386, so a short value gets
+        zero-padded to 56, not to context.bytes (4):
+
+        >>> context.clear(arch='i386')
+        >>> fileStr2 = FileStructure(null=0)
+        >>> fileStr2.vtable = 0x561859f0
+        >>> old_len = len(bytes(fileStr2))
+        >>> fileStr2.unknown2 = b'AB'
+        >>> len(bytes(fileStr2)) == old_len
+        True
     """
 
     vars_=[]
@@ -186,7 +198,7 @@ class FileStructure(object):
         structure = b''
         for val in self.vars_:
             if isinstance(getattr(self, val), bytes):
-                structure += getattr(self, val).ljust(context.bytes, b'\x00')
+                structure += getattr(self, val).ljust(self.length[val], b'\x00')
             else:
                 if self.length[val] > 0:
                     structure += pack(getattr(self, val), self.length[val]*8)
@@ -215,7 +227,7 @@ class FileStructure(object):
         structure = b''
         for val in self.vars_:
             if isinstance(getattr(self, val), bytes):
-                structure += getattr(self, val).ljust(context.bytes, b'\x00')
+                structure += getattr(self, val).ljust(self.length[val], b'\x00')
             else:
                 structure += pack(getattr(self, val), self.length[val]*8)
             if val == v:

pwnlib/gdb.py

@@ -143,7 +143,6 @@ from __future__ import division
 
 import os
 import platform
-import psutil
 import random
 import re
 import six
@@ -1522,28 +1521,33 @@ def corefile(process):
                       'contain all of the data in the process.\n'
                       'Upgrade to GDB >= 7.11 for better core-dumps.' % binary())
 
-    # This is effectively the same as what the 'gcore' binary does
-    gdb_args = ['-batch',
-                '-q',
-                '-nx',
-                '-ex', 'set pagination off',
-                '-ex', 'set height 0',
-                '-ex', 'set width 0',
-                '-ex', 'set use-coredump-filter on',
-                '-ex', 'generate-core-file %s' % corefile_path,
-                '-ex', 'detach']
+    # This is effectively the same as what the 'gcore' binary does.
+    # Run GDB directly instead of going through attach(), which has to
+    # race GDB to observe the tracer before it already detached again.
+    cmd = [binary(),
+           '-batch',
+           '-q',
+           '-nx',
+           '-ex', 'set pagination off',
+           '-ex', 'set height 0',
+           '-ex', 'set width 0',
+           '-ex', 'set use-coredump-filter on',
+           '-ex', 'attach %d' % process.pid,
+           '-ex', 'generate-core-file %s' % corefile_path,
+           '-ex', 'detach']
 
-    with context.local(terminal = ['sh', '-c']):
-        with context.quiet:
-            pid = attach(process, gdb_args=gdb_args)
-            log.debug("Got GDB pid %d", pid)
-            try:
-                psutil.Process(pid).wait()
-            except psutil.Error:
-                pass
+    with context.quiet:
+        gdb = tubes.process.process(cmd, stdout=tubes.process.PIPE)
+        gdb.wait_for_close(timeout=60)
+        timed_out = gdb.poll() is None
+        output = gdb.recvallS(timeout=0)
+
+    if timed_out:
+        log.error("GDB did not finish dumping process %d within 60 seconds", process.pid)
 
     if not os.path.exists(corefile_path):
-        log.error("Could not generate a corefile for process %d", process.pid)
+        log.error("Could not generate a corefile for process %d (maybe check /proc/sys/kernel/yama/ptrace_scope)\n%s",
+                  process.pid, output)
 
     return elf.corefile.Core(corefile_path)
 

pwnlib/libcdb.py

@@ -119,16 +119,32 @@ def query_libc_rip(params):
     import requests
 
     url = "{}/api/find".format(LIBC_RIP_URL)
-    try:
-        result = requests.post(url, json=params, timeout=20)
-        result.raise_for_status()
-        if result.status_code != 200:
-            log.debug("Error: %s", result.text)
+    # Handle pagination transparently and fetch all results
+    offset = 0
+    count = 1
+    results = []
+    while offset < count:
+        try:
+            pagination_params = {'offset': offset, 'limit': 100}
+            log.debug("Querying libc.rip at %s (pagination %s) with parameters: %s", url, pagination_params, params)
+            result = requests.post(url, params=pagination_params, json=params, timeout=20)
+            result.raise_for_status()
+            if result.status_code != 200:
+                log.debug("Error: %s", result.text)
+                return None
+            imm_results = result.json()
+            # non-paginated result. old libc-database API?
+            if not isinstance(imm_results, dict):
+                return imm_results
+            results.extend(imm_results.get('results', []))
+            count = imm_results.get('total', 0)
+            offset += imm_results['count']
+            if not imm_results['has_more']:
+                break
+        except requests.RequestException as e:
+            log.warn_once("Failed to fetch libc info from libc.rip: %s", e)
             return None
-        return result.json()
-    except requests.RequestException as e:
-        log.warn_once("Failed to fetch libc info from libc.rip: %s", e)
-        return None
+    return results
 
 # https://libc.rip/
 def provider_libc_rip(search_target, search_type):
@@ -354,15 +370,15 @@ def unstrip_libc(filename):
         >>> libc = ELF(filename)
         >>> 'main_arena' in libc.symbols
         False
-        >>> unstrip_libc(filename)
+        >>> unstrip_libc(filename) # doctest: +SKIP
         True
         >>> libc = ELF(filename)
-        >>> hex(libc.symbols.main_arena)
+        >>> hex(libc.symbols.main_arena) # doctest: +SKIP
         '0x219c80'
         >>> unstrip_libc(pwnlib.data.elf.get('test-x86'))
         False
         >>> filename = search_by_build_id('d1704d25fbbb72fa95d517b883131828c0883fe9', unstrip=True)
-        >>> 'main_arena' in ELF(filename).symbols
+        >>> 'main_arena' in ELF(filename).symbols # doctest: +SKIP
         True
     """
     if not which('eu-unstrip'):
@@ -697,8 +713,8 @@ def search_by_symbol_offsets(symbols, select_index=None, unstrip=True, offline_o
         >>> matched_libcs = search_by_symbol_offsets({'__libc_start_main_ret': '7f89ad926550'}, return_as_list=True)
         >>> len(matched_libcs) > 1
         True
-        >>> for buildid in matched_libcs: # doctest +SKIP
-        ...     libc = ELF(search_by_build_id(buildid)) # doctest +SKIP
+        >>> for buildid in matched_libcs: # doctest: +SKIP
+        ...     libc = ELF(search_by_build_id(buildid)) # doctest: +SKIP
     """
     assert search_type in TYPES, search_type
 

pwnlib/log.py

@@ -546,7 +546,8 @@ class Handler(logging.StreamHandler):
 
         # if the record originates from a `Progress` object and term handling
         # is enabled we can have animated spinners! so check that
-        if progress is None or not term.term_mode:
+        # Don't show spinners if the output is being redirected
+        if progress is None or not term.term_mode or not os.path.sameopenfile(context.log_console.fileno(), term.term.fd.fileno()):
             super(Handler, self).emit(record)
             return
 

pwnlib/qemu.py

@@ -105,6 +105,7 @@ def archname():
         ('powerpc64', 'little'): 'ppc64le',
         ('thumb', 'little'):     'arm',
         ('thumb', 'big'):        'armeb',
+        ('aarch64', 'big'):      'aarch64_be',
     }.get((context.arch, context.endian), context.arch)
 
 @LocalContext

pwnlib/regsort.py

@@ -190,6 +190,15 @@ def regsort(in_out, all_regs, mapping = None, tmp = None, xchg = True, randomize
 
         >>> regsort({'a': 1, 'b': 2}, R)
         [('mov', 'a', 1), ('mov', 'b', 2)]
+        >>> M = {'ax': 'eax', 'eax': 'eax', 'ebx': 'ebx'}
+
+        Registers which are assigned one of their own aliases
+        are dropped, like same-name self-assignments.
+
+        >>> regsort({'ax': 'eax'}, M)
+        []
+        >>> regsort({'ax': 'eax'}, M, xchg=False)
+        []
         >>> regsort({'a': 'b', 'b': 'a'}, R)
         [('xchg', 'a', 'b')]
         >>> regsort({'a': 'b', 'b': 'a'}, R, tmp='X') #doctest: +NORMALIZE_WHITESPACE
@@ -253,7 +262,9 @@ def regsort(in_out, all_regs, mapping = None, tmp = None, xchg = True, randomize
     # Drop all registers which will be set to themselves.
     #
     # For example, {'eax': 'eax'}
-    in_out = {k:v for k,v in in_out.items() if k != v}
+    # This must go through the mapping, so that {'ax': 'eax'} is
+    # recognized as a self-assignment as well.
+    in_out = {k:v for k,v in in_out.items() if mapping.get(k, k) != mapping.get(v, v)}
 
     # Collapse constant values
     #

pwnlib/rop/rop.py

@@ -390,6 +390,7 @@ from pwnlib.rop.call import Unresolved
 from pwnlib.rop.gadgets import Gadget
 from pwnlib.util import lists
 from pwnlib.util import packing
+from pwnlib.util import safeeval
 from pwnlib.util.cyclic import cyclic
 from pwnlib.util.packing import pack
 from pwnlib.util.misc import python_2_bytes_compatible
@@ -1272,7 +1273,7 @@ class ROP(object):
         filename = self.__get_cachefile_name(elf)
         if filename is None or not os.path.exists(filename):
             return None
-        gadgets = eval(open(filename).read())
+        gadgets = safeeval.const(open(filename).read())
         gadgets = {k - elf.load_addr + elf.address:v for k, v in gadgets.items()}
         log.info_once('Loaded %s cached gadgets for %r', len(gadgets), elf.file.name)
         return gadgets
@@ -1391,6 +1392,8 @@ class ROP(object):
                 if pop.match(insn):
                     regs.append(pop.match(insn).group(1))
                     sp_move += context.bytes
+                    if 'sp' in insn:
+                        sp_move += 9999999
                 elif add.match(insn):
                     arg = int(add.match(insn).group(1), 16)
                     sp_move += arg
@@ -1418,7 +1421,7 @@ class ROP(object):
             if not set(['rsp', 'esp']) & set(regs):
                 self.pivots[sp_move] = addr
 
-        leave = self.search(regs=frame_regs, order='regs')
+        leave = self.search(regs=frame_regs, order='leav')
         if leave and leave.regs != frame_regs:
             leave = None
         self.leave = leave
@@ -1451,7 +1454,7 @@ class ROP(object):
                 pointer is adjusted.
             regs(list): Minimum list of registers which are popped off the
                 stack.
-            order(str): Either the string 'size' or 'regs'. Decides how to
+            order(str): Either the string 'size', 'leav' or 'regs'. Decides how to
                 order multiple gadgets the fulfill the requirements.
 
         The search will try to minimize the number of bytes popped more than
@@ -1459,7 +1462,9 @@ class ROP(object):
         the address.
 
         If ``order == 'size'``, then gadgets are compared lexicographically
-        by ``(total_moves, total_regs, addr)``, otherwise by ``(total_regs, total_moves, addr)``.
+        by ``(total_moves, total_regs, addr)``, if ``order == 'regs'``,
+        then by ``(total_regs, total_moves, addr)``. ``order == 'leav'``
+        is specifically for ``leave`` insn.
 
         Returns:
             A :class:`.Gadget` object
@@ -1471,8 +1476,9 @@ class ROP(object):
         # Search for an exact match, save the closest match
         key = {
             'size': lambda g: (g.move, len(g.regs), g.address),
-            'regs': lambda g: (len(g.regs), g.move, g.address)
-        }[order]
+            'regs': lambda g: (len(g.regs), g.move, g.address),
+            'leav': lambda g: ('leave' not in g.insns, len(g.regs), g.address)
+        }[order]                # False is prior than True
 
         try:
             result = min(matches, key=key)

pwnlib/rop/srop.py

@@ -185,7 +185,7 @@ registers = {
                   80: 'r13', 88: 'r14', 96: 'r15', 104: 'rdi', 112: 'rsi', 120: 'rbp', 128: 'rbx',
                   136: 'rdx', 144: 'rax', 152: 'rcx', 160: 'rsp', 168: 'rip', 176: 'eflags',
                   184: 'csgsfs', 192: 'err', 200: 'trapno', 208: 'oldmask', 216: 'cr2',
-                  224: '&fpstate', 232: '__reserved', 240: 'sigmask'},
+                  224: '&fpstate', 232: '__reserved', 296: 'sigmask'},
 # Reference : http://lxr.free-electrons.com/source/arch/arm/include/uapi/asm/sigcontext.h#L15
         'arm' : {0: 'uc_flags', 4: 'uc_link', 8: 'uc_stack.ss_sp', 12: 'uc_stack.ss_flags',
                  16: 'uc_stack.ss_size', 20: 'trap_no', 24: 'error_code', 28: 'oldmask', 32: 'r0',
@@ -270,15 +270,15 @@ class SigreturnFrame(dict):
         >>> context.clear(arch='amd64')
         >>> s = SigreturnFrame()
         >>> unpack_many(bytes(s))
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0]
-        >>> assert len(s) == 248
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+        >>> assert len(s) == 304
         >>> s.rax = 0xa
         >>> s.rdi = 0x00601000
         >>> s.rsi = 0x1000
         >>> s.rdx = 0x7
-        >>> assert len(bytes(s)) == 248
+        >>> assert len(bytes(s)) == 304
         >>> unpack_many(bytes(s))
-        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6295552, 4096, 0, 0, 7, 10, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0]
+        [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6295552, 4096, 0, 0, 7, 10, 0, 0, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
 
         Crafting a SigreturnFrame that calls mprotect on i386
 

pwnlib/shellcraft/__init__.py

@@ -170,21 +170,47 @@ tether = sys.modules[__name__]
 # Create the module structure
 shellcraft = module(__name__, '')
 
-class LazyImporter:
-    def find_module(self, fullname, path=None):
-        if not fullname.startswith('pwnlib.shellcraft.'):
-            return None
-
-        parts = fullname.split('.')[2:]
-        cur = shellcraft
-        for part in parts:
-            cur = getattr(cur, part, None)
-            if not isinstance(cur, ModuleType):
+if sys.version_info >= (3, 4):
+    import importlib.abc
+    import importlib.util
+    class LazyImporter(importlib.abc.MetaPathFinder, importlib.abc.Loader):
+        def find_spec(self, fullname, path=None, target=None):
+            if not fullname.startswith('pwnlib.shellcraft.'):
                 return None
 
-        return self
+            parts = fullname.split('.')[2:]
+            cur = shellcraft
+            for part in parts:
+                cur = getattr(cur, part, None)
+                if not isinstance(cur, ModuleType):
+                    return None
 
-    def load_module(self, fullname):
-        return sys.modules[fullname]
+            return importlib.util.spec_from_loader(fullname, self)
+
+        def create_module(self, spec):
+            # The submodule was already built and registered in sys.modules by
+            # the getattr walk in find_spec, so hand that object back instead
+            # of letting the import system make a fresh empty module.
+            return sys.modules.get(spec.name)
+
+        def exec_module(self, module):
+            pass
+else:
+    class LazyImporter:
+        def find_module(self, fullname, path=None):
+            if not fullname.startswith('pwnlib.shellcraft.'):
+                return None
+
+            parts = fullname.split('.')[2:]
+            cur = shellcraft
+            for part in parts:
+                cur = getattr(cur, part, None)
+                if not isinstance(cur, ModuleType):
+                    return None
+
+            return self
+
+        def load_module(self, fullname):
+            return sys.modules[fullname]
 
 sys.meta_path.append(LazyImporter())

pwnlib/term/readline.py

@@ -380,7 +380,7 @@ def readline(_size=-1, prompt='', float=True, priority=10):
     from pwnlib.term import term_mode
     if not term_mode:
         six.print_(prompt, end='', flush=True)
-        return getattr(sys.stdin, 'buffer', sys.stdin).readline(_size).rstrip(b'\n')
+        return force_to_bytes(getattr(sys.stdin, 'buffer', sys.stdin).readline(_size)).rstrip(b'\n')
     show_suggestions = False
     eof = False
     if prompt:

pwnlib/tubes/process.py

@@ -767,9 +767,13 @@ class process(tube):
 
         if IS_WINDOWS:
             with self.countdown(timeout=timeout):
-                while self.timeout and self._read_queue.empty():
+                while self.timeout and self._read_queue.empty() and self._read_thread.is_alive():
                     time.sleep(0.01)
-                return not self._read_queue.empty()
+                if not self._read_queue.empty():
+                    return True
+                if not self._read_thread.is_alive():
+                    raise EOFError
+                return False
 
         try:
             if timeout is None:

pwnlib/tubes/remote.py

@@ -86,7 +86,7 @@ class remote(sock):
             except socket.gaierror as e:
                 if e.errno != socket.EAI_NONAME:
                     raise
-                self.error('Could not resolve hostname: %r', host)
+                self.exception('Could not resolve hostname: %r', host)
         if self.sock:
             self.settimeout(self.timeout)
             self.lhost, self.lport = self.sock.getsockname()[:2]

pwnlib/tubes/ssh.py

@@ -692,6 +692,7 @@ class ssh(Timeout, Logger):
                 if user and auth_none and str(e) == "No authentication methods available":
                     self.client.get_transport().auth_none(user)
                 else:
+                    self.close()
                     raise
 
             self.transport = self.client.get_transport()
@@ -913,6 +914,13 @@ class ssh(Timeout, Logger):
             >>> io.recvline()
             b''
 
+            >>> io = s.process(['tty'], tty=True)
+            >>> io.recvline() # doctest: +ELLIPSIS
+            b'/dev/pts/...\n'
+            >>> io = s.process(['tty'], tty=False)
+            >>> io.recvline()
+            b'not a tty\n'
+
             >>> # Testing that empty argv works
             >>> io = s.process([], executable='sh')
             >>> io.sendline(b'echo $0')
@@ -970,7 +978,7 @@ class ssh(Timeout, Logger):
 
             script = 'echo PWNTOOLS; for py in python3 python2.7 python2 python; do test -x "$(command -v $py 2>&1)" && echo $py && exec $py -c %s check; done; echo 2' % sh_string(script)
             with context.quiet:
-                python = ssh_process(self, script, tty=True, cwd=cwd, raw=True, level=self.level, timeout=timeout)
+                python = ssh_process(self, script, tty=tty, cwd=cwd, raw=raw, level=self.level, timeout=timeout)
 
             try:
                 python.recvline_contains(b'PWNTOOLS')   # Magic flag so that any sh/bash initialization errors are swallowed
@@ -1898,7 +1906,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
                 if not self.which('lsb_release'):
                     return
 
-                with self.process(['lsb_release', '-irs']) as io:
+                with self.process(['lsb_release', '-irs'], stderr='/dev/null') as io:
                     lsb_info = io.recvall().strip().decode()
                     self._platform_info['distro'], self._platform_info['distro_ver'] = lsb_info.split()
             except Exception:
@@ -2081,7 +2089,7 @@ from ctypes import *; libc = CDLL('libc.so.6'); print(libc.getenv(%r))
         Example:
 
             >>> s = ssh("travis", "example.pwnme")
-            >>> s.ibt
+            >>> s.ibt # doctest: +SKIP
             False
         """
         if self._ibt is None:

pwnlib/util/iters.py

@@ -6,10 +6,12 @@ from __future__ import division
 
 import collections
 import copy
+import marshal
 import multiprocessing
 import operator
 import random
 import time
+import types
 from itertools import *
 from six.moves import map, filter, filterfalse, range, zip, zip_longest
 
@@ -27,7 +29,6 @@ __all__ = [
     'group'                                  ,
     'iter_except'                            ,
     'lexicographic'                          ,
-    'lookahead'                              ,
     'nth'                                    ,
     'pad'                                    ,
     'pairwise'                               ,
@@ -683,37 +684,6 @@ def random_combination_with_replacement(iterable, r):
     indices = sorted(random.randrange(n) for i in range(r))
     return tuple(pool[i] for i in indices)
 
-def lookahead(n, iterable):
-    """lookahead(n, iterable) -> object
-
-    Inspects the upcoming element at index `n` without advancing the iterator.
-    Raises ``IndexError`` if `iterable` has too few elements.
-
-    Arguments:
-      n(int):  Index of the element to return.
-      iterable:  An iterable.
-
-    Returns:
-      The element in `iterable` at index `n`.
-
-    Examples:
-
-      >>> i = count()
-      >>> lookahead(4, i)
-      4
-      >>> next(i)
-      0
-      >>> i = count()
-      >>> nth(4, i)
-      4
-      >>> next(i)
-      5
-      >>> lookahead(4, i)
-      10
-    """
-    for value in islice(copy.copy(iterable), n, None):
-        return value
-    raise IndexError(n)
 
 def lexicographic(alphabet):
     """lexicographic(alphabet) -> iterator
@@ -877,6 +847,54 @@ def _mbruteforcewrap(func, alphabet, length, method, start, databag):
     databag["result"] = res
 
 
+class CustomPickler:
+    def __init__(self, *reduction):
+        self.reduction = reduction
+
+    def __reduce__(self):
+        return self.reduction
+
+    def __repr__(self):
+        return self.__class__.__name__ + repr(self.reduction)
+
+
+def construct_func(*args):
+    return types.FunctionType(*args)
+
+
+class PicklableFunc:
+    def __init__(self, f):
+        self.f = f
+
+    def __call__(self, *a, **kw):
+        return self.f(*a, **kw)
+
+    def __getattr__(self, a):
+        return getattr(self.f, a)
+
+    def __reduce__(self):
+        # Constructor types.FunctionType tries to be __main__.function;
+        # globs could likewise be <module>.__dict__.  Neither
+        #   type('FunctionType', (), {'__module__':'types'})
+        # nor
+        #   def __reduce__(self): return "FunctionType"
+        # works because of checks for identity :(
+        globs = CustomPickler(vars, (CustomPickler(__import__, (self.f.__module__ or '__main__',)),))
+        return type(self), (CustomPickler(
+            construct_func,  # ideally self.f.__class__
+                             # but types.FunctionType tries to be __main__.function
+            (
+                CustomPickler(marshal.loads, (marshal.dumps(self.f.__code__),)),
+                globs,
+                self.f.__name__,
+                self.f.__defaults__,
+                self.f.__closure__,
+                self.f.__kwdefaults__,
+            ),
+            self.f.__dict__,
+        ),)
+
+
 def mbruteforce(func, alphabet, length, method = 'upto', start = None, threads = None):
     """mbruteforce(func, alphabet, length, method = 'upto', start = None, threads = None)
 
@@ -914,6 +932,9 @@ def mbruteforce(func, alphabet, length, method = 'upto', start = None, threads =
     (i2, N2) = start
     totalchunks = threads * N2
 
+    if isinstance(func, types.FunctionType):
+        func = PicklableFunc(func)
+
     for i in range(threads):
         shareddata[i] = multiprocessing.Manager().dict()
         shareddata[i]['result'] = None

pwnlib/util/misc.py

@@ -868,6 +868,7 @@ if sys.argv[-1] == 'check':
     sys.stdout.write(str(os.getgid()) + "\n")
     sys.stdout.write(str(suid) + "\n")
     sys.stdout.write(str(sgid) + "\n")
+    sys.stdout.flush()
     getattr(sys.stdout, 'buffer', sys.stdout).write(os.path.realpath(exe) + b'\x00')
     sys.stdout.flush()
 
@@ -886,7 +887,7 @@ if not %(aslr)r:
         ADDR_NO_RANDOMIZE = 0x0040000
         ctypes.CDLL('libc.so.6').personality(ADDR_NO_RANDOMIZE)
 
-    resource.setrlimit(resource.RLIMIT_STACK, (-1, -1))
+    resource.setrlimit(resource.RLIMIT_STACK, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
 
 # Attempt to dump ALL core file regions
 try:
@@ -897,7 +898,7 @@ except Exception:
 
 # Assume that the user would prefer to have core dumps.
 try:
-    resource.setrlimit(resource.RLIMIT_CORE, (-1, -1))
+    resource.setrlimit(resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
 except Exception:
     pass
 

pwnlib/util/packing.py

@@ -988,6 +988,11 @@ def dd(dst, src, count = 0, skip = 0, seek = 0, truncate = False):
         ('H', 'e', 'l', 'l', 'o', b'?')
         >>> dd(list('Hello!'), (63,), skip = 5)
         ['H', 'e', 'l', 'l', 'o', b'?']
+        >>> buf = bytearray(b'Hello!')
+        >>> dd(buf, b'?', skip = 5)
+        bytearray(b'Hello?')
+        >>> buf
+        bytearray(b'Hello?')
         >>> _ = open('/tmp/foo', 'w').write('A' * 10)
         >>> dd(open('/tmp/foo'), open('/dev/zero'), skip = 3, count = 4).read()
         'AAA\\x00\\x00\\x00\\x00AAA'
@@ -1105,7 +1110,12 @@ def dd(dst, src, count = 0, skip = 0, seek = 0, truncate = False):
         dst = real_dst
 
     elif isinstance(dst, (list, bytearray)):
-        dst[skip : skip + len(src)] = list(map(p8, bytearray(src)))
+        # A bytearray slice takes a byte string, while a list slice takes the
+        # single-byte bytes objects that p8() returns.
+        if isinstance(dst, bytearray):
+            dst[skip : skip + len(src)] = src
+        else:
+            dst[skip : skip + len(src)] = list(map(p8, bytearray(src)))
         if truncate:
             while len(dst) > truncate:
                 dst.pop()

pwnlib/util/safeeval.py

@@ -4,8 +4,8 @@ _const_codes = [
     'POP_TOP','ROT_TWO','ROT_THREE','ROT_FOUR','DUP_TOP',
     'BUILD_LIST','BUILD_MAP', 'MAP_ADD', 'BUILD_TUPLE','BUILD_SET',
     'BUILD_CONST_KEY_MAP', 'BUILD_STRING',
-    'LOAD_CONST','RETURN_VALUE','STORE_SUBSCR', 'STORE_MAP',
-    'LIST_TO_TUPLE', 'LIST_EXTEND', 'SET_UPDATE', 'DICT_UPDATE', 'DICT_MERGE',
+    'LOAD_CONST','LOAD_SMALL_INT','RETURN_VALUE','STORE_SUBSCR', 'STORE_MAP',
+    'LIST_TO_TUPLE', 'LIST_APPEND', 'LIST_EXTEND', 'SET_ADD', 'SET_UPDATE', 'DICT_UPDATE', 'DICT_MERGE',
     'COPY', 'RESUME', 'RETURN_CONST'
     ]
 
@@ -29,7 +29,7 @@ def _get_opcodes(codeobj):
     Extract the actual opcodes as a list from a code object
 
     >>> c = compile("[1 + 2, (1,2)]", "", "eval")
-    >>> _get_opcodes(c)  # doctest: +ELLIPSIS
+    >>> _get_opcodes(c)  # doctest: +SKIP
     [...100, 100, 103, 83]
     """
     import dis
@@ -85,6 +85,10 @@ def const(expr):
         Traceback (most recent call last):
         ...
         ValueError: opcode BINARY_ADD not allowed
+        >>> const("[" + ",".join(str(i) for i in range(40)) + "]") == list(range(40))
+        True
+        >>> const("{" + ",".join(str(i) for i in range(40)) + "}") == set(range(40))
+        True
     """
 
     c = test_expr(expr, _const_codes)

pyproject.toml

@@ -42,7 +42,6 @@ dependencies = [
     "ropgadget>=5.3",
     "pyserial>=2.7",
     "requests>=2.0",
-    "pip>=6.0.8",
     "pygments>=2.0",
     "pysocks",
     "python-dateutil",
@@ -50,7 +49,7 @@ dependencies = [
     "psutil>=3.3.0",
     "intervaltree>=3.0",
     "sortedcontainers",
-    "unicorn>=2.0.1, !=2.1.3, !=2.1.4",  # see https://github.com/unicorn-engine/unicorn/issues/2134
+    "unicorn>=2.0.1",
     "six>=1.12.0",
     "rpyc",
     "colored_traceback<0.4; python_version < '3'",

travis/libcdb_nginx_cache.conf

@@ -8,19 +8,26 @@ http {
                     '"$request" $status $body_bytes_sent '
                     '"$http_referer" "$http_user_agent"';
     access_log /dev/stdout cache_st;
+    # Use Docker's embedded DNS server to resolve hostnames at runtime
+    # instead of at startup.
+    resolver 127.0.0.11 8.8.8.8 ipv6=off valid=30s;
+    # Set default resolver for all upstreams to avoid startup-time DNS resolution
+    # This is a workaround for the occasional 'host not found in upstream' error
+    # in Docker environments where DNS may not be immediately available.
 
     server {
         listen 3000;
         proxy_cache my_cache;
 
         location / {
-            proxy_set_header Host debuginfod.elfutils.org;
+            set $upstream_host "debuginfod.elfutils.org";
+            proxy_set_header Host $upstream_host;
             proxy_ssl_server_name on;
             proxy_cache_revalidate on;
             proxy_cache_key $scheme://$host$uri$is_args$query_string;
             proxy_cache_valid 200 404 12w;
             proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429;
-            proxy_pass https://debuginfod.elfutils.org/;
+            proxy_pass https://$upstream_host$request_uri;
         }
     }
 
@@ -29,14 +36,15 @@ http {
         proxy_cache my_cache;
 
         location / {
-            proxy_set_header Host libc.rip;
+            set $upstream_host "libc.rip";
+            proxy_set_header Host $upstream_host;
             proxy_ssl_server_name on;
             proxy_cache_methods GET HEAD POST;
             proxy_cache_revalidate on;
             proxy_cache_key $scheme://$host$uri$is_args$query_string$request_body;
             proxy_cache_valid 200 404 12w;
             proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429;
-            proxy_pass https://libc.rip/;
+            proxy_pass https://$upstream_host$request_uri;
         }
     }
 
@@ -45,12 +53,13 @@ http {
         proxy_cache my_cache;
 
         location / {
-            proxy_set_header Host archive.ubuntu.com;
+            set $upstream_host "archive.ubuntu.com";
+            proxy_set_header Host $upstream_host;
             proxy_cache_revalidate on;
             proxy_cache_key $scheme://$host$uri$is_args$query_string;
             proxy_cache_valid 200 404 12w;
             proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429;
-            proxy_pass http://archive.ubuntu.com/;
+            proxy_pass http://$upstream_host$request_uri;
         }
     }
 
@@ -59,13 +68,15 @@ http {
         proxy_cache my_cache;
 
         location / {
-            proxy_set_header Host gitlab.com;
+            set $upstream_host "gitlab.com";
+            proxy_set_header Host $upstream_host;
             proxy_ssl_server_name on;
             proxy_cache_revalidate on;
             proxy_cache_key $scheme://$host$uri$is_args$query_string;
             proxy_cache_valid 200 404 12w;
+            proxy_ignore_headers Cache-Control Set-Cookie;
             proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429;
-            proxy_pass https://gitlab.com/;
+            proxy_pass https://$upstream_host$request_uri;
         }
     }
 
@@ -74,13 +85,15 @@ http {
         proxy_cache my_cache;
 
         location / {
-            proxy_set_header Host debuginfod.ubuntu.com;
+            set $upstream_host "debuginfod.ubuntu.com";
+            proxy_set_header Host $upstream_host;
             proxy_ssl_server_name on;
             proxy_cache_revalidate on;
             proxy_cache_key $scheme://$host$uri$is_args$query_string;
             proxy_cache_valid 200 404 12w;
             proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504 http_429;
-            proxy_pass https://debuginfod.ubuntu.com/;
+            proxy_pass https://$upstream_host$request_uri;
         }
     }
 }
+

travis/test_nginx_cache.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+docker run \
+    --name pwntools-test-nginx-cache \
+    --rm \
+    -v ./libcdb_nginx_cache.conf:/etc/nginx/nginx.conf:ro \
+    -p 3000:3000 \
+    -p 3001:3001 \
+    -p 3002:3002 \
+    -p 3003:3003 \
+    nginx
+
+# curl -v localhost:3000/buildid/8e9fd827446c24067541ac5390e6f527fb5947bb/debuginfo
+# curl -v localhost:3003/libcdb/libcdb/raw/master/hashes/build_id/fe136e485814fee2268cf19e5c124ed0f73f4400
+# curl -v localhost:3003/libcdb/libcdb/raw/master/libc/libc6-i386-2.18-6/lib32/libc-2.18.so
+# curl -v localhost:3003/libcdb/libcdb/raw/master/hashes/build_id/XX