Release 3.13.0

* Reduce cache filename length on rop.py (fix #1180)

* ROP cache elfs sorting

.travis.yml

@@ -1,4 +1,4 @@
-dist: trusty
+dist: xenial
 language: python
 addons:
   apt:
@@ -13,15 +13,22 @@ addons:
     - zsh
     - pandoc
     - gdb
+    - binutils
+    - qemu-user-static
+    - binutils-multiarch
+    - binutils-aarch64-linux-gnu
+    - binutils-arm-linux-gnueabihf
+    - binutils-mips-linux-gnu
+    - binutils-powerpc-linux-gnu
 cache:
     - pip
     - directories:
         - usr
-        - /home/travis/virtualenv/python2.7.13/lib/python2.7/site-packages/
-        - /home/travis/virtualenv/python2.7.13/bin/
+        - /home/travis/virtualenv/python2.7.15/lib/python2.7/site-packages/
+        - /home/travis/virtualenv/python2.7.15/bin/
 sudo: required
 python:
-  - "2.7"
+  - "2.7.15"
 before_install:
   - echo $-
   - source travis/install.sh
@@ -30,6 +37,7 @@ before_install:
 install:
   - pip install --upgrade pip
   - pip install --upgrade appdirs # https://github.com/ActiveState/appdirs/issues/89#issuecomment-282326570
+  - python setup.py egg_info      # pypa/pip#6275
   - pip install --upgrade --editable .
   - pip install --upgrade --requirement docs/requirements.txt
 before_script:

CHANGELOG.md

@@ -9,9 +9,10 @@ The table below shows which release corresponds to each branch, and what date th
 
 | Version          | Branch   | Release Date           |
 | ---------------- | -------- | ---------------------- |
-| [3.14.0](#3130)  | `dev`    | Jun 12, 2018 (planned)
-| [3.13.0](#3130)  | `beta`   | Mar 31, 2018 (planned)
-| [3.12.0](#3120)  | `stable` | Feb 22, 2018
+| [4.1.0](#410)    | `dev`    | Jan 20, 2020 (planned)
+| [4.0.0](#400)    | `beta`   | Dec 20, 2019 (planned)
+| [3.13.0](#3130)  | `stable` | Nov 5, 2019
+| [3.12.0](#3120)  |          | Feb 22, 2018
 | [3.11.0](#3110)  |          | Jan 3, 2018
 | [3.10.0](#3100)  |          | Oct 25, 2017
 | [3.9.2](#392)    |          | Oct 5, 2017
@@ -42,13 +43,44 @@ The table below shows which release corresponds to each branch, and what date th
 | [3.0.0](#300)    |          | Aug 20, 2016
 | [2.2.0](#220)    |          | Jan 5, 2015
 
-## 3.14.0
+## 4.1.0
 
-To be released on Jun 12, 2018.
+To be released on Jan 20, 2020.
+
+## 4.0.0
+
+To be released on Dec 20, 2019.
+
+- **Python 3 support! <3**
+- [#1317][1317] Tubes with `context.encoding`
+- [#1216][1216] Improve format string generator
+- [#1285][1285] Add freebsd generic syscall templates
+- [76413f][76413f] Add pwnlib.adb.bootimg for 'ANDROID!' format boot.img images
+- [#1202][1202] Docker: Kill 14 layers in pwntools base images
+- [#1182][1182] shellcraft.dupio() for mips
+
+[1317]: https://github.com/Gallopsled/pwntools/pulls/1317
+[1285]: https://github.com/Gallopsled/pwntools/pulls/1285
+[1216]: https://github.com/Gallopsled/pwntools/pulls/1216
+[1202]: https://github.com/Gallopsled/pwntools/pulls/1202
+[1182]: https://github.com/Gallopsled/pwntools/pulls/1182
+[76413f]: https://github.com/Gallopsled/pwntools/commit/76413f
 
 ## 3.13.0
 
-To be released on Mar 31, 2018.
+- [#1204][1204] Reduce ROP cache filename length
+- [#1175][1175] Fix nested SSH connectors
+- [#1355][1355] Fix 'break' syscall
+- [#1277][1277] Fix timeout parameter passing in sendlineafter and other similar functions
+- [#1292][1292] Provide correct arch name to gdb for sparc64
+
+[1175]: https://github.com/Gallopsled/pwntools/pulls/1175
+[1204]: https://github.com/Gallopsled/pwntools/pulls/1355
+[1277]: https://github.com/Gallopsled/pwntools/pulls/1277
+[1292]: https://github.com/Gallopsled/pwntools/pulls/1292
+[1355]: https://github.com/Gallopsled/pwntools/pulls/1355
+
+## 3.12.1
 
 - [#1104][1104] Add `DynELF.dump()` for dumping remote ELF files
 - [#1101][1101] Set `context.os` via `context.binary`, useful for Android exploitation
@@ -56,13 +88,11 @@ To be released on Mar 31, 2018.
 - [63dfed][63dfed] Print warning when Corefile deletion fails instead of throwing an exception
 - [#1094][1094] Make hexdump output alignment more consistent
 - [#1096][1096] `flat()` and `fit()` are now the same function
-- [#1068][1068] Work around very old OpenSSL versions which don't have sha256 support *AND* don't exit with an error code when trying to use it
 
 [1104]: https://github.com/Gallopsled/pwntools/pull/1104
 [1101]: https://github.com/Gallopsled/pwntools/pull/1101
 [1094]: https://github.com/Gallopsled/pwntools/pull/1094
 [1096]: https://github.com/Gallopsled/pwntools/pull/1096
-[1068]: https://github.com/Gallopsled/pwntools/pull/1068
 [5fdc08]: https://github.com/Gallopsled/pwntools/commit/5fdc08
 [63dfed]: https://github.com/Gallopsled/pwntools/commit/63dfed
 
@@ -72,6 +102,7 @@ To be released on Mar 31, 2018.
 - [#1085][1085] Add support for extracting Android `BOOTLDR!` images
 - [#1075][1075] Add support for detecting GNU Screen for `run_in_new_terminal`
 - [#1074][1074] Add support for running `pwntools-gdb` wrapper script instead of `gdb`
+- [#1068][1068] Work around very old OpenSSL versions which don't have sha256 support *AND* don't exit with an error code when trying to use it
 - [#1067][1067] Add `pwnlib.tubes.server` module, which adds a reusable `server` listener
 - [#1063][1063] Add support for labels in `fit()`, allowing dynamic contents to be injected.  (This feature is really cool, check out the pull request!)
 
@@ -79,6 +110,7 @@ To be released on Mar 31, 2018.
 [1085]: https://github.com/Gallopsled/pwntools/pull/1085
 [1075]: https://github.com/Gallopsled/pwntools/pull/1075
 [1074]: https://github.com/Gallopsled/pwntools/pull/1074
+[1068]: https://github.com/Gallopsled/pwntools/pull/1068
 [1067]: https://github.com/Gallopsled/pwntools/pull/1067
 [1063]: https://github.com/Gallopsled/pwntools/pull/1063
 

README.md

@@ -2,7 +2,7 @@
 ![pwntools logo](https://github.com/Gallopsled/pwntools/blob/stable/docs/source/logo.png?raw=true)
 
 [![Docs](https://readthedocs.org/projects/pwntools/badge/?version=stable)](https://docs.pwntools.com/)
-[![PyPI](https://img.shields.io/badge/pypi-v3.12.1-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
+[![PyPI](https://img.shields.io/badge/pypi-v3.13.0-green.svg?style=flat)](https://pypi.python.org/pypi/pwntools/)
 [![Travis](https://travis-ci.org/Gallopsled/pwntools.svg)](https://travis-ci.org/Gallopsled/pwntools)
 [![Coveralls](https://img.shields.io/coveralls/Gallopsled/pwntools/dev.svg)](https://coveralls.io/github/Gallopsled/pwntools?branch=dev)
 [![Twitter](https://img.shields.io/badge/twitter-pwntools-4099FF.svg?style=flat)](https://twitter.com/pwntools)

docs/source/conf.py

@@ -378,27 +378,25 @@ from sphinx.util.inspect import safe_getmembers, safe_getattr
 def dont_skip_any_doctests(app, what, name, obj, skip, options):
     return False
 
-# Test non-exported members
-class ModuleDocumenter(sphinx.ext.autodoc.ModuleDocumenter):
-    def get_object_members(self, want_all):
-        if want_all:
-            # if not hasattr(self.object, '__all__'):
-            #     for implicit module members, check __module__ to avoid
-            #     documenting imported objects
-                return True, safe_getmembers(self.object)
-            # else:
-            #     memberlist = self.object.__all__
-            #     # Sometimes __all__ is broken...
-            #     if not isinstance(memberlist, (list, tuple)) or not \
-            #        all(isinstance(entry, string_types) for entry in memberlist):
-            #         self.directive.warn(
-            #             '__all__ should be a list of strings, not %r '
-            #             '(in module %s) -- ignoring __all__' %
-            #             (memberlist, self.fullname))
-            #         # fall back to all members
-            #         return True, safe_getmembers(self.object)
-        else:
-            memberlist = self.options.members or []
+def get_object_members_all(self, want_all):
+    if want_all:
+        # if not hasattr(self.object, '__all__'):
+        #     for implicit module members, check __module__ to avoid
+        #     documenting imported objects
+        return True, safe_getmembers(self.object)
+    # else:
+    #     memberlist = self.object.__all__
+    #     # Sometimes __all__ is broken...
+    #     if not isinstance(memberlist, (list, tuple)) or not \
+        #        all(isinstance(entry, string_types) for entry in memberlist):
+    #         self.directive.warn(
+    #             '__all__ should be a list of strings, not %r '
+    #             '(in module %s) -- ignoring __all__' %
+    #             (memberlist, self.fullname))
+    #         # fall back to all members
+    #         return True, safe_getmembers(self.object)
+    else:
+        memberlist = self.options.members or []
         ret = []
         for mname in memberlist:
             try:
@@ -414,4 +412,4 @@ if 'doctest' in sys.argv:
     def setup(app):
         app.connect('autodoc-skip-member', dont_skip_any_doctests)
 
-    sphinx.ext.autodoc.ModuleDocumenter = ModuleDocumenter
+    sphinx.ext.autodoc.ModuleDocumenter.get_object_members = get_object_members_all

docs/source/util/proc.rst

@@ -1,6 +1,7 @@
 .. testsetup:: *
 
    from pwnlib.util.proc import *
+   from pwnlib.tubes.process import process
    import os, sys
 
 

pwnlib/adb/adb.py

@@ -881,9 +881,16 @@ def logcat(stream=False):
 def pidof(name):
     """Returns a list of PIDs for the named process."""
     with context.quiet:
-        io = process(['pidof', name])
-        data = io.recvall().split()
-    return list(map(int, data))
+        # Older devices have a broken 'pidof', apparently.
+        # Try pgrep first.
+        io = process(['pgrep', name])
+        data = io.recvall()
+
+        if 'not found' in data:
+            io = process(['pidof', name])
+            data = io.recvall()
+
+    return list(map(int, data.split()))
 
 @with_device
 def proc_exe(pid):

pwnlib/args.py

@@ -194,7 +194,7 @@ def initialize():
         value = 'True'
 
         if '=' in arg:
-            arg, value = arg.split('=')
+            arg, value = arg.split('=', 1)
 
         if arg in hooks:
             sys.argv.remove(orig)

pwnlib/asm.py

@@ -36,7 +36,7 @@ Disassembly
     To disassemble code, simply invoke :func:`disasm` on the bytes to disassemble.
 
     >>> disasm('\xb8\x0b\x00\x00\x00')
-    '   0:   b8 0b 00 00 00          mov    eax,0xb'
+    '   0:   b8 0b 00 00 00          mov    eax, 0xb'
 
 """
 from __future__ import absolute_import
@@ -614,7 +614,7 @@ def asm(shellcode, vma = 0, extract = True, shared = False):
     Runs :func:`cpp` over a given shellcode and then assembles it into bytes.
 
     To see which architectures or operating systems are supported,
-    look in :mod:`pwnlib.contex`.
+    look in :mod:`pwnlib.context`.
 
     Assembling shellcode requires that the GNU assembler is installed
     for the target architecture.
@@ -737,15 +737,15 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
     Examples:
 
         >>> print disasm('b85d000000'.decode('hex'), arch = 'i386')
-           0:   b8 5d 00 00 00          mov    eax,0x5d
+           0:   b8 5d 00 00 00          mov    eax, 0x5d
         >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0)
-           0:   mov    eax,0x5d
+           0:   mov    eax, 0x5d
         >>> print disasm('b85d000000'.decode('hex'), arch = 'i386', byte = 0, offset = 0)
-        mov    eax,0x5d
+        mov    eax, 0x5d
         >>> print disasm('b817000000'.decode('hex'), arch = 'amd64')
-           0:   b8 17 00 00 00          mov    eax,0x17
+           0:   b8 17 00 00 00          mov    eax, 0x17
         >>> print disasm('48c7c017000000'.decode('hex'), arch = 'amd64')
-           0:   48 c7 c0 17 00 00 00    mov    rax,0x17
+           0:   48 c7 c0 17 00 00 00    mov    rax, 0x17
         >>> print disasm('04001fe552009000'.decode('hex'), arch = 'arm')
            0:   e51f0004        ldr     r0, [pc, #-4]   ; 0x4
            4:   00900052        addseq  r0, r0, r2, asr r0
@@ -816,4 +816,4 @@ def disasm(data, vma = 0, byte = True, offset = True, instructions = True):
             line += i
         lines.append(line)
 
-    return '\n'.join(lines)
+    return re.sub(',([^ ])', r', \1', '\n'.join(lines))

pwnlib/commandline/debug.py

@@ -78,7 +78,12 @@ def main(args):
         # pidof() returns a list
         if not target:
             log.error("Could not find a PID for %r", args.process)
+
         target = target[0]
+
+        # pidof will sometimes return all PIDs, including init
+        if target == 1:
+            log.error("Got PID 1 from pidof.  Check the process name, or use --pid 1 to debug init")
     else:
         parser.print_usage()
         return 1

pwnlib/context/__init__.py

@@ -777,6 +777,7 @@ class ContextType(object):
         self.arch   = binary.arch
         self.bits   = binary.bits
         self.endian = binary.endian
+        self.os     = binary.os
 
         return binary
 

pwnlib/data/useragents/LICENSE.txt

@@ -1,13 +1,12 @@
-This directory contains a script `download-useragents.py' which downloads a list
-of user agent strings from the Internet.
+This directory contains a script `download-useragents.py' which downloads a
+list of user agent strings from the Internet.
 
 The user agent strings are downloaded from:
- - http://www.useragentstring.com/
- - http://www.user-agents.org/
+ - http://www.user-agents.org
  - http://techpatterns.com/downloads/firefox/useragentswitcher.xml
 
-The latter list is part of the Firefox `User Agent Switcher' add-on and released
-under BSD 2 Clause.
+The latter list is part of the Firefox `User Agent Switcher' add-on and
+released under BSD 2 Clause.
 
 
 As mentioned elsewhere we are not lawyers and do not know the legal status of

pwnlib/data/useragents/download-useragents.py

@@ -1,13 +1,9 @@
 #!/usr/bin/env python2
-"""
-Script for downloading lists of user agent strings
-"""
+"""Script for downloading lists of user agent strings."""
 import os
-import re
 import urllib
 
 from bs4 import BeautifulSoup
-
 from pwn import *
 
 uas = set()
@@ -23,39 +19,9 @@ def getxml(url):
     f.close()
     return xml
 
-with log.waitfor('Fetching from from http://www.useragentstring.com') as l:
-    html = getxml('http://www.useragentstring.com/pages/All/')
-    soup = BeautifulSoup(html)
-    l.success()
-
-with log.waitfor('Parsing list') as l:
-    liste = soup.select('#liste a')
-    for a in liste:
-        href = a.get('href')
-        if not href or href[0] != '/':
-            continue
-        m = re.match('More (.+) user agents strings -->>', a.string)
-        if m:
-            target = m.group(1)
-            with log.waitfor('Fetching additional user agents for %s' % target) as l:
-                html = getxml('http://www.useragentstring.com' + href)
-                soup = BeautifulSoup(html)
-                subliste = soup.select('#liste a')
-                for a in subliste:
-                    ua = a.string.strip()
-                    try:
-                        ua = ua.decode('utf-8')
-                    except UnicodeEncodeError:
-                        continue
-                    uas.add(ua)
-                l.success()
-        else:
-            uas.add(a.string.strip())
-    l.success()
-
 with log.waitfor('Fetching from from http://techpatterns.com') as l:
     xml = getxml('http://techpatterns.com/downloads/firefox/useragentswitcher.xml')
-    soup = BeautifulSoup(xml)
+    soup = BeautifulSoup(xml, 'html.parser')
     l.success()
 
 def loop(xml):
@@ -67,16 +33,16 @@ def loop(xml):
             uas.add(item['useragent'].strip())
 
 with log.waitfor('Parsing list') as l:
-    loop(soup.body.useragentswitcher)
+    loop(soup.useragentswitcher)
     l.success()
 
 with log.waitfor('Fetching from from http://www.user-agents.org') as l:
     xml = getxml('http://www.user-agents.org/allagents.xml')
-    soup = BeautifulSoup(xml)
+    soup = BeautifulSoup(xml, 'html.parser')
     l.success()
 
 with log.waitfor('Parsing list') as l:
-    for item in soup.body.__getattr__('user-agents'):
+    for item in soup.__getattr__('user-agents'):
         if item.name == 'user-agent':
             ua = item.select('string')[0].string.strip()
             try:

pwnlib/data/useragents/useragents.txt

@@ -8211,6 +8211,7 @@ Mozilla/5.0 (BlackBerry; U; BlackBerry 9850; en-US) AppleWebKit/534.11+ (KHTML,
 Mozilla/5.0 (BlackBerry; U; BlackBerry 9850; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0.254 Mobile Safari/534.11+
 Mozilla/5.0 (BlackBerry; U; BlackBerry 9860; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.0.0.254 Mobile Safari/534.11+
 Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+
+Mozilla/5.0 (BlackBerry; U; BlackBerry 9930; en-US) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.267 Mobile Safari/534.11+
 Mozilla/5.0 (Clustered-Search-Bot/1.0; [email protected]; http://www.clush.com/)
 Mozilla/5.0 (Darwin; FreeBSD 5.6; en-GB; rv:1.8.1.17pre) Gecko/20080716 K-Meleon/1.5.0
 Mozilla/5.0 (Darwin; FreeBSD 5.6; en-GB; rv:1.9.1b3pre)Gecko/20081211 K-Meleon/1.5.2
@@ -8244,13 +8245,26 @@ Mozilla/5.0 (Linux; Android 4.4.4; XT1032 Build/KXB21.14-L1.61) AppleWebKit/537.
 Mozilla/5.0 (Linux; Android 4.4; Nexus 5 Build/BuildID) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/30.0.0.0 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 5.0.1; SCH-R970 Build/LRX22C) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.84 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 5.0.2; SAMSUNG SM-T530NU Build/LRX22G) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/3.2 Chrome/38.0.2125.102 Safari/537.36
+Mozilla/5.0 (Linux; Android 5.1.1; Coolpad 3622A Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 5.1.1; Nexus 7 Build/LMY47V) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.2357.78 Safari/537.36 OPR/30.0.1856.93524
 Mozilla/5.0 (Linux; Android 5.1; C6740N Build/LMY47O) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.111 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 6.0.1; SM-G900H Build/MMB29K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 6.0; ALE-L21 Build/HuaweiALE-L21) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.89 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 6.0; HTC One M9 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.98 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 6.0; LG-D850 Build/MRA58K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.97 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 6.0; Nexus 5X Build/MDB08L) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.0; LG-H918 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.0; LGL84VL Build/NRD90U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.0; LGUS997 Build/NRD90U) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.0; Moto G (5) Plus Build/NPNS25.137-35-5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36
 Mozilla/5.0 (Linux; Android 7.0; Nexus 9 Build/NRD90R) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.124 Safari/537.36
+Mozilla/5.0 (Linux; Android 7.0; SAMSUNG SM-G925R6 Build/NRD90M) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/5.4 Chrome/51.0.2704.106 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.1.1; BBB100-1 Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.1.1; Coolpad 3632A Build/NMF26F) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.1.1; XT1710-02 Build/NDS26.74-36) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.125 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.1.2; Nexus 6P Build/N2G48C) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 7.1.2; Pixel Build/NHG47N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Mobile Safari/537.36
+Mozilla/5.0 (Linux; Android 8.0.0; Pixel XL Build/OPR6.170623.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.107 Mobile Safari/537.36
 Mozilla/5.0 (Linux; U) Opera 6.02  [en]
 Mozilla/5.0 (Linux; U; Android 0.5; en-us) AppleWebKit/522  (KHTML, like Gecko) Safari/419.3
 Mozilla/5.0 (Linux; U; Android 1.0; en-us; dream) AppleWebKit/525.10  (KHTML, like Gecko) Version/3.0.4 Mobile Safari/523.12.2
@@ -8369,6 +8383,7 @@ Mozilla/5.0 (Macintosh; I; Intel Mac OS X 11_7_9; de-LI; rv:1.9b4) Gecko/2012010
 Mozilla/5.0 (Macintosh; I; PPC Mac OS X Mach-O; en-US; rv:1.9a1) Gecko/20061204 Firefox/3.0a1
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.10; rv:40.0) Gecko/20100101 Firefox/40.0
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10.13; rv:55.0) Gecko/20100101 Firefox/55.0
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:10.0.1) Gecko/20100101 Firefox/10.0.1 SeaMonkey/2.7.1
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.5; rv:5.0) Gecko/20100101 Firefox/5.0
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:15.0) Gecko/20120909 Firefox/15.0.1 SeaMonkey/2.12.1
@@ -8405,17 +8420,20 @@ Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.1.25 (KHTML, like Gecko)
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.118 Safari/537.36 OPR/28.0.1750.51
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.82 Safari/537.36 OPR/29.0.1795.41
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.105 Safari/537.36 Vivaldi/1.0.162.9
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2859.0 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.49 Safari/537.36
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Maxthon/4.5.2
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/600.8.9 (KHTML, like Gecko) Version/8.0.8 Safari/600.8.9
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Safari/602.1.50
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11) AppleWebKit/601.1.56 (KHTML, like Gecko) Version/9.0 Safari/601.1.56
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.8 (KHTML, like Gecko) Version/10.1 Safari/603.1.30
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.68 Safari/534.24
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/534.31 (KHTML, like Gecko) Chrome/13.0.748.0 Safari/534.31
 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_5_8) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.801.0 Safari/535.1
@@ -10005,6 +10023,8 @@ Mozilla/5.0 (OS/2; Warp 4.5; rv:31.0) Gecko/20100101 Firefox/31.0
 Mozilla/5.0 (OS/2; Warp 4.5; rv:31.0) Gecko/20100101 Firefox/31.0 SeaMonkey/2.28
 Mozilla/5.0 (OS/2; Warp 4.5; rv:38.0) Gecko/20100101 Firefox/38.0
 Mozilla/5.0 (OS/2; Warp 4.5; rv:38.0) Gecko/20100101 Firefox/38.0 SeaMonkey/2.35
+Mozilla/5.0 (OS/2; Warp 4.5; rv:45.0) Gecko/20100101 Firefox/45.0
+Mozilla/5.0 (OS/2; Warp 4.5; rv:45.0) Gecko/20100101 Firefox/45.0 SeaMonkey/2.42.9esr
 Mozilla/5.0 (PLAYSTATION 3; 1.0)
 Mozilla/5.0 (PLAYSTATION 3; 1.00)
 Mozilla/5.0 (PLAYSTATION 3; 1.10)
@@ -10018,6 +10038,7 @@ Mozilla/5.0 (PlayBook; U; RIM Tablet OS 2.1.0; en-US) AppleWebKit/536.2+ (KHTML
 Mozilla/5.0 (S60; SymbOS; Opera Mobi/1181; U; en-GB; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.1
 Mozilla/5.0 (S60; SymbOS; Opera Mobi/1209; U; it; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.1
 Mozilla/5.0 (S60; SymbOS; Opera Mobi/SYB-1103211396; U; es-LA; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.00
+Mozilla/5.0 (SMART-TV; X11; Linux armv7l) AppleWebkit/537.42 (KHTML, like Gecko) Chromium/25.0.1349.2 Chrome/25.0.1349.2 Safari/537.42
 Mozilla/5.0 (Sage)
 Mozilla/5.0 (Slurp/cat; [email protected]; http://www.inktomi.com/slurp.html)
 Mozilla/5.0 (Slurp/si; [email protected]; http://www.inktomi.com/slurp.html)
@@ -10072,18 +10093,27 @@ Mozilla/5.0 (Windows ME; U) Opera 6.05  [de]
 Mozilla/5.0 (Windows ME; U; en) Opera 8.51
 Mozilla/5.0 (Windows ME; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.63
 Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.135 Safari/537.36 Edge/12.10240
+Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063
+Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.91 Safari/537.36 Vivaldi/1.92.917.39
 Mozilla/5.0 (Windows NT 10.0; ARM; Lumia 950 Dual SIM) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393
 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.93 Safari/537.36
 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 UBrowser/5.6.13705.206 Safari/537.36
 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.87 Safari/537.36 OPR/36.0.2130.46
 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36
 Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.90 Safari/537.36 Vivaldi/1.4.589.11
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 AOL/11.0 AOLBUILD/11.0.1305 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36 OPR/47.0.2631.55
+Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Maxthon/5.0.4.3000 Chrome/47.0.2526.73 Safari/537.36
 Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; MATBJS; rv:11.0) like Gecko
+Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; Touch; MALNJS; rv:11.0) like Gecko
 Mozilla/5.0 (Windows NT 10.0; WOW64; rv:40.0) Gecko/20100101 Firefox/40.0
 Mozilla/5.0 (Windows NT 10.0; WOW64; rv:47.0) Gecko/20100101 Firefox/47.0
 Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71 Safari/537.36 Edge/12.0
 Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2869.0 Safari/537.36
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3191.0 Safari/537.36
 Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:52.0) Gecko/20100101 Firefox/52.0
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:57.0) Gecko/20100101 Firefox/57.0
+Mozilla/5.0 (Windows NT 10.0; rv:45.9) Gecko/20100101 Goanna/3.2 Firefox/45.9 PaleMoon/27.4.0
 Mozilla/5.0 (Windows NT 4.0; U) Opera 6.05  [en]
 Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36
 Mozilla/5.0 (Windows NT 5.0) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1150.1  Iron/20.0.1150.1 Safari/536.11
@@ -10145,6 +10175,8 @@ Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.
 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36
 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36
 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36
+Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 YaBrowser/17.3.0.1785 Yowser/2.5 Safari/537.36
+Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) MxBrowser/4.5.10.7000 Chrome/30.0.1551.0 Safari/537.36
 Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.4 (KHTML, like Gecko) Chrome/22.0.1250.0 Iron/22.0.2150.0 Safari/537.4
 Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0
 Mozilla/5.0 (Windows NT 5.1; U) Opera 7.03  [de]
@@ -15192,6 +15224,7 @@ Mozilla/5.0 (X11; ; Linux i686; en-US; rv:1.8.1.3) Gecko
 Mozilla/5.0 (X11; ; Linux i686; rv:1.9.2.20) Gecko/20110805
 Mozilla/5.0 (X11; ; Linux x86_64; rv:1.8.1.6) Gecko/20070802 Firefox
 Mozilla/5.0 (X11; Arch Linux i686; rv:2.0) Gecko/20110321 Firefox/4.0
+Mozilla/5.0 (X11; CentOS; Linux x86_64; rv:36.0) Gecko/20100101 Firefox/36.0
 Mozilla/5.0 (X11; CrOS i686 0.13.507) AppleWebKit/534.35 (KHTML, like Gecko) Chrome/13.0.763.0 Safari/534.35
 Mozilla/5.0 (X11; CrOS i686 0.13.587) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.14 Safari/535.1
 Mozilla/5.0 (X11; CrOS i686 1193.158.0) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.75 Safari/535.7
@@ -15205,6 +15238,7 @@ Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) C
 Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36
 Mozilla/5.0 (X11; CrOS x86_64 5841.83.0) AppleWebKit/537.36 (KHTML like Gecko) Chrome/36.0.1985.138 Safari/537.36
 Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36
+Mozilla/5.0 (X11; Fedora; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3178.0 Safari/537.36
 Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0
 Mozilla/5.0 (X11; FreeBSD 7.1-RELEASE i386; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.63
 Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.65 Safari/535.11
@@ -15215,6 +15249,7 @@ Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/3
 Mozilla/5.0 (X11; FreeBSD amd64) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.79 Safari/537.4
 Mozilla/5.0 (X11; FreeBSD amd64; rv:27.0) Gecko/20100101 Firefox/27.0 SeaMonkey/2.24
 Mozilla/5.0 (X11; FreeBSD amd64; rv:5.0) Gecko/20100101 Firefox/5.0
+Mozilla/5.0 (X11; FreeBSD amd64; rv:54.0) Gecko/20100101 Firefox/54.0
 Mozilla/5.0 (X11; FreeBSD amd64; rv:6.0) Gecko/20110818 Firefox/6.0 SeaMonkey/2.3
 Mozilla/5.0 (X11; FreeBSD i386) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2
 Mozilla/5.0 (X11; FreeBSD i386; rv:28.0) Gecko/20100101 Firefox/28.0 SeaMonkey/2.25
@@ -15239,6 +15274,7 @@ Mozilla/5.0 (X11; Linux armv7l; rv:2.0b4pre) Gecko/20100818 Firefox/4.0b4pre Fen
 Mozilla/5.0 (X11; Linux i386; U) Opera 7.60  [en-GB]
 Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0
 Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.130 Safari/537.36
+Mozilla/5.0 (X11; Linux i686 (x86_64)) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3187.0 Safari/537.366
 Mozilla/5.0 (X11; Linux i686 on x86_64; rv:10.0.6) Gecko/20100101 Firefox/10.0.6 Iceweasel/10.0.6
 Mozilla/5.0 (X11; Linux i686 on x86_64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1
 Mozilla/5.0 (X11; Linux i686 on x86_64; rv:2.0.1) Gecko/20100101 Firefox/4.0.1 Fennec/2.0.1
@@ -15284,6 +15320,7 @@ Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0
 Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2166.2 Safari/537.36
 Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.101 Safari/537.36 OPR/40.0.2308.62
 Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/51.0.2704.79 Chrome/51.0.2704.79 Safari/537.36
+Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/60.0.3112.78 Chrome/60.0.3112.78 Safari/537.36
 Mozilla/5.0 (X11; Linux i686) AppleWebKit/538.1 (KHTML, like Gecko) QupZilla/1.8.6 Safari/538.1
 Mozilla/5.0 (X11; Linux i686; Debian Testing; rv:7.0.1) Gecko/20100101 Firefox/7.0.1 Iceweasel/7.0.1
 Mozilla/5.0 (X11; Linux i686; U) Opera 7.54  [en]
@@ -15387,6 +15424,7 @@ Mozilla/5.0 (X11; Linux i686; rv:40.0) Gecko/20100101 Firefox/40.0
 Mozilla/5.0 (X11; Linux i686; rv:43.0) Gecko/20100101 Firefox/43.0
 Mozilla/5.0 (X11; Linux i686; rv:46.0) Gecko/20100101 Firefox/46.0
 Mozilla/5.0 (X11; Linux i686; rv:49.0) Gecko/20100101 Firefox/49.0
+Mozilla/5.0 (X11; Linux i686; rv:49.0) Gecko/20100101 Firefox/49.0 SeaMonkey/2.46
 Mozilla/5.0 (X11; Linux i686; rv:5.0) Gecko/20100101 Firefox/5.0
 Mozilla/5.0 (X11; Linux i686; rv:5.0) Gecko/20100101 Firefox/5.0 Iceweasel/5.0
 Mozilla/5.0 (X11; Linux i686; rv:6.0) Gecko/20100101 Firefox/6.0
@@ -15453,6 +15491,7 @@ Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/33.0.1750.152 Chrome/33.0.1750.152 Safari/537.36
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.4 (KHTML like Gecko) Chrome/22.0.1229.56 Safari/537.4
 Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/538.1 (KHTML, like Gecko) QupZilla/1.9.0 Safari/538.1
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/604.1 (KHTML, like Gecko) Version/11.0 Safari/604.1 Ubuntu/17.04 (3.24.1-0ubuntu1) Epiphany/3.24.1
 Mozilla/5.0 (X11; Linux x86_64) Gecko Firefox/5.0
 Mozilla/5.0 (X11; Linux x86_64; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 10.62
 Mozilla/5.0 (X11; Linux x86_64; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0
@@ -15516,6 +15555,7 @@ Mozilla/5.0 (X11; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0 Iceweasel/3
 Mozilla/5.0 (X11; Linux x86_64; rv:5.0) Gecko/20100101 Firefox/5.0 FirePHP/0.5
 Mozilla/5.0 (X11; Linux x86_64; rv:5.0) Gecko/20100101 Firefox/5.0 Firefox/5.0
 Mozilla/5.0 (X11; Linux x86_64; rv:5.0) Gecko/20100101 Firefox/5.0 Iceweasel/5.0
+Mozilla/5.0 (X11; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0
 Mozilla/5.0 (X11; Linux x86_64; rv:6.0.1) Gecko/20110831 conkeror/0.9.3
 Mozilla/5.0 (X11; Linux x86_64; rv:7.0.1) Gecko/20100101 Firefox/7.0.1 Iceweasel/7.0.1 Debian
 Mozilla/5.0 (X11; Linux x86_64; rv:7.0a1) Gecko/20110602 Firefox/7.0a1 SeaMonkey/2.2a1pre Lightning/1.1a1pre
@@ -18563,6 +18603,7 @@ Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0
 Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0
 Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:35.0) Gecko/20100101 Firefox/35.0
 Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:49.0) Gecko/20100101 Firefox/49.0
+Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:55.0) Gecko/20100101 Firefox/55.0
 Mozilla/5.0 (X11; debian; Linux x86_64; rv:15.0) Gecko/20100101 Iceweasel/15.0
 Mozilla/5.0 (X11;U; Linux i686; en-GB; rv:1.9.1) Gecko/20090624 Ubuntu/9.04 (jaunty) Firefox/3.5
 Mozilla/5.0 (X11;U;Linux i686;en-US;rv:1.8.1) Gecko/2006101022 Firefox/2.0
@@ -19300,15 +19341,19 @@ Mozilla/5.0 (compatible; Yahoo! Slurp China; http://misc.yahoo.com.cn/help.html)
 Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)
 Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)
 Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)
+Mozilla/5.0 (compatible; YandexNews/4.0; +http://yandex.com/bots)
 Mozilla/5.0 (compatible; YesupBot/1.0; +http://www.yesup.net/bot.html)
 Mozilla/5.0 (compatible; YodaoBot/1.0; http://www.yodao.com/help/webmaster/spider/; )
 Mozilla/5.0 (compatible; Yoono; http://www.yoono.com/)
 Mozilla/5.0 (compatible; YoudaoBot/1.0; http://www.youdao.com/help/webmaster/spider/; )
 Mozilla/5.0 (compatible; Zenbot/1.3; +http://zen.co.za/webmasters/)
 Mozilla/5.0 (compatible; aiHitBot/1.0; +http://www.aihit.com/)
+Mozilla/5.0 (compatible; alexa site audit/1.0; +http://www.alexa.com/help/webmasters; )
+Mozilla/5.0 (compatible; archive.org_bot +http://www.archive.org/details/archive.org_bot)
 Mozilla/5.0 (compatible; archive.org_bot/1.10.0 +http://www.loc.gov/minerva/crawl.html)
 Mozilla/5.0 (compatible; archive.org_bot/1.13.1x http://crawler.archive.org)
 Mozilla/5.0 (compatible; archive.org_bot/1.5.0-200506132127 http://crawler.archive.org) Hurricane Katrina
+Mozilla/5.0 (compatible; archive.org_bot; Wayback Machine Live Record; +http://archive.org/details/archive.org_bot)
 Mozilla/5.0 (compatible; bingbot/2.0  http://www.bing.com/bingbot.htm)
 Mozilla/5.0 (compatible; bingbot/2.0 +http://www.bing.com/bingbot.htm)
 Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)
@@ -19352,6 +19397,7 @@ Mozilla/5.0 (en-US;rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2
 Mozilla/5.0 (en-us) AppleWebKit/525.13 (KHTML, like Gecko; Google Web Preview) Version/3.1 Safari/525.13
 Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.2; U; de-DE) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/234.40.1 Safari/534.6 TouchPad/1.0
 Mozilla/5.0 (iPad; CPU OS 10_0 like Mac OS X) AppleWebKit/601.1 (KHTML, like Gecko) CriOS/49.0.2623.109 Mobile/14A5335b Safari/601.1.46
+Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A5362a Safari/604.1
 Mozilla/5.0 (iPad; CPU OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Mobile/9B176
 Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3
 Mozilla/5.0 (iPad; CPU OS 5_1_1 like Mac OS X; ru-ru) AppleWebKit/534.46.0 (KHTML, like Gecko) CriOS/21.0.1180.80 Mobile/9B206 Safari/7534.48.3
@@ -19390,6 +19436,8 @@ Mozilla/5.0 (iPhone Simulator; U; CPU iPhone OS 3_2 like Mac OS X; en-us) AppleW
 Mozilla/5.0 (iPhone; CPU  iPhone OS 3_0 like Mac OS X; en-us) AppleWebKit/528.18 (KHTML, like Gecko) Version/4.0 Mobile/7A341 Safari/528.16
 Mozilla/5.0 (iPhone; CPU iPhone OS 10_0 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) GSA/18.0.130791545 Mobile/14A5345a Safari/600.1.4
 Mozilla/5.0 (iPhone; CPU iPhone OS 10_0 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14A346 Safari/602.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_3 like Mac OS X) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1
+Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A5362a Safari/604.1
 Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A403 Safari/8536.25
 Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML like Gecko) Version/7.0 Mobile/11D257 Safari/9537.53
 Mozilla/5.0 (iPhone; CPU iPhone OS 8_3 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12F70 Safari/600.1.4
@@ -20490,6 +20538,7 @@ Opera/9.80 (Android; Opera Mini/7.29530/27.1407; U; en) Presto/2.8.119 Version/1
 Opera/9.80 (Android; Opera Mini/7.5.33361/31.1350; U; en) Presto/2.8.119 Version/11.10
 Opera/9.80 (Android; Opera Mini/7.5.33361/31.1543; U; en) Presto/2.8.119 Version/11.1010
 Opera/9.80 (Android; Opera Mini/7.6.35766/35.5706; U; en) Presto/2.8.119 Version/11.10
+Opera/9.80 (Android; Opera Mini/9.0.1829/66.318; U; en) Presto/2.12.423 Version/12.16
 Opera/9.80 (Android;Opera Mini/6.0.24212/24.746 U;en) Presto/2.5.25 Version/10.5454
 Opera/9.80 (BlackBerry; Opera Mini/5.1.22303/22.387; U; en) Presto/2.5.25 Version/10.54
 Opera/9.80 (BlackBerry; Opera Mini/6.24209/27.1366; U; en) Presto/2.8.119 Version/11.10
@@ -21055,6 +21104,7 @@ Scooter_trk3-3.0.3
 Scope (Mars+)
 ScoutAbout
 ScoutAnt/0.1; +http://www.ant.com/what_is_ant.com/
+Screaming Frog SEO Spider/8.1
 Scrubby/2.1 (http://www.scrubtheweb.com/)
 Scrubby/2.2 (http://www.scrubtheweb.com/)
 Scrubby/2.x (http://www.scrubtheweb.com/)
@@ -21311,6 +21361,7 @@ Trampelpfad-Spider-v0.1
 TulipChain/5.x (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://java.sun.com/) Linux/2.4.17
 TulipChain/5.xx (http://ostermiller.org/tulipchain/) Java/1.x.1_0x (http://apple.com/) Mac_OS_X/10.2.8
 Tumblr/1.0 RSS syndication (+http://www.tumblr.com/) ([email protected])
+TurnitinBot (https://turnitin.com/robot/crawlerinfo.html)
 TurnitinBot/1.5 (http://www.turnitin.com/robot/crawlerinfo.html)
 TurnitinBot/1.5 http://www.turnitin.com/robot/crawlerinfo.html
 TurnitinBot/2.0 http://www.turnitin.com/robot/crawlerinfo.html
@@ -21323,6 +21374,7 @@ TwengaBot
 Twiceler www.cuill.com/robots.html
 Twiceler-0.9 http://www.cuill.com/twiceler/robot.html
 Twisted PageGetter
+Twitterbot/1.0
 Twitturly / v0.x
 Twotrees Reactive Filter V2.0
 Tycoon Agent/Nutch-1.0-dev
@@ -21584,6 +21636,7 @@ Xenu Link Sleuth 1.2h
 Xenu Link Sleuth 1.2i
 Xenu Link Sleuth 1.xx
 Xenu Link Sleuth/1.3.7
+Xenu Link Sleuth/1.3.8
 Xenu's Link Sleuth 1.x[a-z]
 Xerka WebBot v1.0.0 [UPVOpenDir]
 Xylix

pwnlib/dynelf.py

@@ -910,3 +910,65 @@ class DynELF(object):
         self.success('*curbrk: %#x' % brk)
 
         return brk
+
+    def _find_mapped_pages(self, readonly = False, page_size = 0x1000):
+        """
+        A generator of all mapped pages, as found using the Program Headers.
+
+        Yields tuples of the form: (virtual address, memory size)
+        """
+        leak  = self.leak
+        base  = self.libbase
+
+        Ehdr  = {32: elf.Elf32_Ehdr, 64: elf.Elf64_Ehdr}[self.elfclass]
+        Phdr  = {32: elf.Elf32_Phdr, 64: elf.Elf64_Phdr}[self.elfclass]
+
+        phead = base + leak.field(base, Ehdr.e_phoff)
+        phnum = leak.field(base, Ehdr.e_phnum)
+
+        for i in range(phnum):
+            if leak.field_compare(phead, Phdr.p_type, constants.PT_LOAD) :
+                # the interesting pages are those that are aligned to PAGE_SIZE
+                if leak.field_compare(phead, Phdr.p_align, page_size) and \
+                    (readonly or leak.field(phead, Phdr.p_flags) & 0x02 != 0):
+                    vaddr = leak.field(phead, Phdr.p_vaddr)
+                    memsz = leak.field(phead, Phdr.p_memsz)
+                    # fix relative offsets
+                    if vaddr < base :
+                        vaddr += base
+                    yield vaddr, memsz
+            phead += sizeof(Phdr)
+
+    def dump(self, libs = False, readonly = False):
+        """dump(libs = False, readonly = False)
+
+        Dumps the ELF's memory pages to allow further analysis.
+
+        Arguments:
+            libs(bool, optional): True if should dump the libraries too (False by default)
+            readonly(bool, optional): True if should dump read-only pages (False by default)
+
+        Returns:
+            a dictionary of the form: { address : bytes }
+        """
+        leak      = self.leak
+        page_size = 0x1000
+        pages     = {}
+
+        for vaddr, memsz in self._find_mapped_pages(readonly, page_size) :
+            offset    = vaddr % page_size
+            if offset != 0 :
+                memsz += offset
+                vaddr -= offset
+            memsz += (page_size - (memsz % page_size)) % page_size
+            pages[vaddr] = leak.n(vaddr, memsz)
+
+        if libs :
+            for lib_name in self.bases() :
+                if len(lib_name) == 0 :
+                    continue
+                dyn_lib = self._dynamic_load_dynelf(lib_name)
+                if dyn_lib is not None :
+                    pages.update(dyn_lib.dump(readonly = readonly))
+
+        return pages

pwnlib/elf/corefile.py

@@ -1111,7 +1111,7 @@ class Coredump(Corefile):
 class CorefileFinder(object):
     def __init__(self, proc):
         if proc.poll() is None:
-            log.error("Process %i has not exited" % (process.pid))
+            log.error("Process %i has not exited" % (proc.pid))
 
         self.process = proc
         self.pid = proc.pid
@@ -1171,7 +1171,10 @@ class CorefileFinder(object):
             new_path = 'core.%i' % core_pid
             if core_pid > 0 and new_path != self.core_path:
                 write(new_path, self.read(self.core_path))
-                self.unlink(self.core_path)
+                try:
+                    self.unlink(self.core_path)
+                except (IOError, OSError):
+                    log.warn("Could not delete %r" % self.core_path)
                 self.core_path = new_path
 
         # Check the PID

pwnlib/elf/elf.py

@@ -188,6 +188,7 @@ class ELF(ELFFile):
     functions = {}
     endian = 'little'
     address = 0x400000
+    linker = None
 
     # Whether to fill gaps in memory with zeroed pages
     _fill_gaps = True
@@ -313,8 +314,20 @@ class ELF(ELFFile):
 
         for seg in self.iter_segments_by_type('PT_INTERP'):
             self.executable = True
+
+            #: ``True`` if the ELF is statically linked
             self.statically_linked = False
 
+            #: Path to the linker for the ELF
+            self.linker = self.read(seg.header.p_vaddr, seg.header.p_memsz)
+            self.linker = self.linker.rstrip('\x00')
+
+        #: Operating system of the ELF
+        self.os = 'linux'
+
+        if self.linker and self.linker.startswith('/system/bin/linker'):
+            self.os = 'android'
+
         #: ``True`` if the ELF is a shared library
         self.library = not self.executable and self.elftype == 'DYN'
 

pwnlib/gdb.py

@@ -475,7 +475,8 @@ def get_gdb_arch():
         'powerpc': 'powerpc:common',
         'powerpc64': 'powerpc:common64',
         'mips64': 'mips:isa64',
-        'thumb': 'arm'
+        'thumb': 'arm',
+        'sparc64': 'sparc:v9'
     }.get(context.arch, context.arch)
 
 def binary():

pwnlib/rop/rop.py

@@ -1011,13 +1011,11 @@ class ROP(object):
         if isinstance(files, ELF):
             files = [files]
 
-        hashes = []
+        sha256 = hashlib.sha256()
+        for elf_data in sorted(elf.get_data() for elf in self.elfs):
+            sha256.update(elf_data)
 
-        for elf in self.elfs:
-            sha256 = hashlib.sha256(elf.get_data()).hexdigest()
-            hashes.append(sha256)
-
-        return os.path.join(cachedir, '_'.join(hashes))
+        return os.path.join(cachedir, sha256.hexdigest())
 
     def __cache_load(self, elf):
         filename = self.__get_cachefile_name(elf)

pwnlib/shellcraft/__init__.py

@@ -165,7 +165,7 @@ tether = sys.modules[__name__]
 shellcraft = module(__name__, '')
 
 class LazyImporter:
-    def find_module(self, fullname, path):
+    def find_module(self, fullname, path=None):
         if not fullname.startswith('pwnlib.shellcraft.'):
             return None
 

pwnlib/shellcraft/templates/common/linux/syscalls/break_.asm

@@ -4,7 +4,7 @@ import pwnlib.abi
 import pwnlib.constants
 import pwnlib.shellcraft
 %>
-<%docstring>break(vararg_0, vararg_1, vararg_2, vararg_3, vararg_4) -> str
+<%docstring>break_(vararg_0, vararg_1, vararg_2, vararg_3, vararg_4) -> str
 
 Invokes the syscall break.
 
@@ -83,7 +83,7 @@ Returns:
     else:
         raise Exception("Could not locate any syscalls: %r" % syscalls)
 %>
-    /* break(${', '.join(syscall_repr)}) */
+    /* break_(${', '.join(syscall_repr)}) */
 %for name, arg in string_arguments.items():
     ${pwnlib.shellcraft.pushstr(arg, append_null=('\x00' not in arg))}
     ${pwnlib.shellcraft.mov(regs[argument_names.index(name)], abi.stack)}
@@ -95,4 +95,4 @@ Returns:
     ${pwnlib.shellcraft.push(arg)}
 %endfor
     ${pwnlib.shellcraft.setregs(register_arguments)}
-    ${pwnlib.shellcraft.syscall(syscall)}
+    ${pwnlib.shellcraft.syscall(syscall)}

pwnlib/shellcraft/templates/thumb/linux/findpeer.asm

@@ -1,4 +1,4 @@
-<% from pwnlib.shellcraft.thumb.linux import mov %>
+<% from pwnlib.shellcraft.thumb import mov %>
 <% from socket import htons %>
 <%page args="port = None"/>
 <%docstring>
@@ -6,6 +6,10 @@
 
     Finds a connected socket. If port is specified it is checked
     against the peer port. Resulting socket is left in r6.
+
+    Example:
+        >>> enhex(asm(shellcraft.findpeer(1337)))
+        '6ff00006ee4606f101064ff001074fea072707f11f07f54630461fb401a96a4601df0130efdd01994fea11414ff039024fea022202f105029142e4d1'
 </%docstring>
 findpeer:
     /* File descriptor in r6 */
@@ -52,7 +56,7 @@ compare_port:
     ldr r1, [sp, #4]
     lsr r1, #16
 
-    /* Put the port (${port}) to search for into r1 */
+    /* Put the port (${port}) to search for into r2 */
     ${mov('r2', htons(int(port)))}
 
     /* Is it the one we have been searching for? */

pwnlib/shellcraft/templates/thumb/linux/listen.asm

@@ -1,4 +1,4 @@
-<% from pwnlib.shellcraft.thumb.linux import mov %>
+<% from pwnlib.shellcraft.thumb import mov %>
 <% from socket import htons %>
 <%page args="port, network='ipv4'"/>
 <%docstring>
@@ -6,6 +6,10 @@
 
     Listens on a TCP port, accept a client and leave his socket in r6.
     Port is the TCP port to listen on, network is either 'ipv4' or 'ipv6'.
+
+    Example:
+        >>> enhex(asm(shellcraft.listen(1337, 'ipv4')))
+        '4ff001074fea072707f119074ff002004ff0010182ea020201df0646004901e00200053906b469464ff0100207f1010701df30464ff0010107f1020701df304681ea010182ea020207f1010701df0646'
 </%docstring>
     /* First create listening socket */
     ${mov('r7', 'SYS_socket')}

pwnlib/tubes/ssh.py

@@ -146,13 +146,17 @@ class ssh_channel(sock):
         # However, we need to wait for the return value to propagate,
         # which may not happen by the time .close() is called by tube.recvall()
         tmp_sock = self.sock
+        tmp_close = self.close
+        self.close = lambda: None
 
         timeout = self.maximum if self.timeout is self.forever else self.timeout
         data = super(ssh_channel, self).recvall(timeout)
 
         # Restore self.sock to be able to call wait()
+        self.close = tmp_close
         self.sock = tmp_sock
         self.wait()
+        self.close()
 
         # Again set self.sock to None
         self.sock = None
@@ -418,9 +422,18 @@ class ssh_connecter(sock):
                 self.exception(e.message)
                 raise
 
-            sockname = self.sock.get_transport().sock.getsockname()
-            self.lhost = sockname[0]
-            self.lport = sockname[1]
+            try:
+                # Iterate all layers of proxying to get to base-level Socket object
+                curr = self.sock.get_transport().sock
+                while getattr(curr, "get_transport", None):
+                    curr = curr.get_transport().sock
+
+                sockname = curr.getsockname()
+                self.lhost = sockname[0]
+                self.lport = sockname[1]
+            except Exception as e:
+                self.exception("Could not find base-level Socket object.")
+                raise e
 
             h.success()
 
@@ -530,7 +543,21 @@ class ssh(Timeout, Logger):
             ssh_agent: If :const:`True`, enable usage of keys via ssh-agent
 
         NOTE: The proxy_command and proxy_sock arguments is only available if a
-        fairly new version of paramiko is used."""
+        fairly new version of paramiko is used.
+
+        Example proxying:
+
+            >>> s1 = ssh(host='example.pwnme',
+            ...          user='travis',
+            ...          password='demopass')
+            >>> r1 = s1.remote('localhost', 22)
+            >>> s2 = ssh(host='example.pwnme',
+            ...          user='travis',
+            ...          password='demopass',
+            ...          proxy_sock=r1.sock)
+            >>> r2 = s2.remote('localhost', 22) # and so on...
+            >>> for x in r2, s2, r1, s1: x.close()
+        """
         super(ssh, self).__init__(*a, **kw)
 
         Logger.__init__(self)

pwnlib/tubes/tube.py

@@ -732,37 +732,37 @@ class tube(Timeout, Logger):
     def sendafter(self, delim, data, timeout = default):
         """sendafter(delim, data, timeout = default) -> str
 
-        A combination of ``recvuntil(delim, timeout)`` and ``send(data)``.
+        A combination of ``recvuntil(delim, timeout=timeout)`` and ``send(data)``.
         """
 
-        res = self.recvuntil(delim, timeout)
+        res = self.recvuntil(delim, timeout=timeout)
         self.send(data)
         return res
 
     def sendlineafter(self, delim, data, timeout = default):
         """sendlineafter(delim, data, timeout = default) -> str
 
-        A combination of ``recvuntil(delim, timeout)`` and ``sendline(data)``."""
+        A combination of ``recvuntil(delim, timeout=timeout)`` and ``sendline(data)``."""
 
-        res = self.recvuntil(delim, timeout)
+        res = self.recvuntil(delim, timeout=timeout)
         self.sendline(data)
         return res
 
     def sendthen(self, delim, data, timeout = default):
         """sendthen(delim, data, timeout = default) -> str
 
-        A combination of ``send(data)`` and ``recvuntil(delim, timeout)``."""
+        A combination of ``send(data)`` and ``recvuntil(delim, timeout=timeout)``."""
 
         self.send(data)
-        return self.recvuntil(delim, timeout)
+        return self.recvuntil(delim, timeout=timeout)
 
     def sendlinethen(self, delim, data, timeout = default):
         """sendlinethen(delim, data, timeout = default) -> str
 
-        A combination of ``sendline(data)`` and ``recvuntil(delim, timeout)``."""
+        A combination of ``sendline(data)`` and ``recvuntil(delim, timeout=timeout)``."""
 
         self.send(data + self.newline)
-        return self.recvuntil(delim, timeout)
+        return self.recvuntil(delim, timeout=timeout)
 
     def interactive(self, prompt = term.text.bold_red('$') + ' '):
         """interactive(prompt = pwnlib.term.text.bold_red('$') + ' ')

pwnlib/util/fiddling.py

@@ -574,9 +574,9 @@ def update_cyclic_pregenerated(size):
         cyclic_pregen += de_bruijn_gen.next()
 
 def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
-                 highlight=None, cyclic=False):
+                 highlight=None, cyclic=False, groupsize=4):
     r"""hexdump_iter(s, width = 16, skip = True, hexii = False, begin = 0,
-                    style = None, highlight = None, cyclic = False) -> str generator
+                    style = None, highlight = None, cyclic = False, groupsize=4) -> str generator
 
     Return a hexdump-dump of a string as a generator of lines.  Unless you have
     massive amounts of data you probably want to use :meth:`hexdump`.
@@ -584,6 +584,7 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
     Arguments:
         fd(file): File object to dump.  Use :meth:`StringIO.StringIO` or :meth:`hexdump` to dump a string.
         width(int): The number of characters per line
+        groupsize(int): The number of characters per group
         skip(bool): Set to True, if repeated lines should be replaced by a "*"
         hexii(bool): Set to True, if a hexii-dump should be returned instead of a hexdump.
         begin(int):  Offset of the first byte to print in the left column
@@ -601,7 +602,7 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
         >>> tmp.flush()
         >>> tmp.seek(4)
         >>> print '\n'.join(hexdump_iter(tmp))
-        00000000  48 45 4c 4c  4f 2c 20 57  4f 52 4c 44               │HELL│O, W│ORLD││
+        00000000  48 45 4c 4c  4f 2c 20 57  4f 52 4c 44               │HELL│O, W│ORLD│
         0000000c
 
         >>> t = tube()
@@ -613,6 +614,9 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
     style     = style or {}
     highlight = highlight or []
 
+    if groupsize < 1:
+        groupsize = width
+
     for b in highlight:
         if isinstance(b, str):
             b = ord(b)
@@ -625,15 +629,10 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
     lines       = []
     last_unique = ''
     byte_width  = len('00 ')
-    column_sep  = '  '
-    line_fmt    = '%%(offset)08x  %%(hexbytes)-%is │%%(printable)s│' % (len(column_sep)+(width*byte_width))
     spacer      = ' '
     marker      = (style.get('marker') or (lambda s:s))('│')
 
-    if hexii:
-        column_sep = ''
-        line_fmt   = '%%(offset)08x  %%(hexbytes)-%is│' % (len(column_sep)+(width*byte_width))
-    else:
+    if not hexii:
         def style_byte(b):
             hbyte = '%02x' % ord(b)
             abyte = b if isprint(b) else '·'
@@ -696,31 +695,38 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
         # Generate contents for line
         hexbytes = ''
         printable = ''
+        color_chars = 0
+        abyte = abyte_previous = ''
         for i, b in enumerate(chunk):
             if not hexii:
+                abyte_previous = abyte
                 hbyte, abyte = cache[ord(b)]
+                color_chars += len(hbyte) - 2
             else:
                 hbyte, abyte = _hexiichar(b), ''
 
-            if i % 4 == 3 and i < width - 1:
+            if (i + 1) % groupsize == 0 and i < width - 1:
                 hbyte += spacer
-                abyte += marker
+                abyte_previous += abyte
+                abyte = marker
 
             hexbytes += hbyte + ' '
+            printable += abyte_previous
+
+        if abyte != marker:
             printable += abyte
 
-        if i + 1 < width:
-            delta = width - i - 1
+        dividers_per_line = (width // groupsize)
+        if width % groupsize == 0:
+            dividers_per_line -= 1
 
-            # How many hex-bytes would we have printed?
-            count = byte_width * delta
-
-            # How many dividers do we need to fill out the line?
-            dividers_per_line = (width // 4) - (1 if width % 4 == 0 else 0)
-            dividers_printed = (i // 4) + (1 if i % 4 == 3 else 0)
-            count += dividers_per_line - dividers_printed
-
-            hexbytes += ' ' * count
+        if hexii:
+            line_fmt = '%%(offset)08x  %%(hexbytes)-%is│' % (width*byte_width)
+        else:
+            line_fmt = '%%(offset)08x  %%(hexbytes)-%is │%%(printable)s│' % (
+                 (width * byte_width)
+                + color_chars
+                + dividers_per_line )
 
         line = line_fmt % {'offset': offset, 'hexbytes': hexbytes, 'printable': printable}
         yield line
@@ -729,15 +735,16 @@ def hexdump_iter(fd, width=16, skip=True, hexii=False, begin=0, style=None,
     yield line
 
 def hexdump(s, width=16, skip=True, hexii=False, begin=0,
-            style=None, highlight=None, cyclic=False):
+            style=None, highlight=None, cyclic=False, groupsize=4):
     r"""hexdump(s, width = 16, skip = True, hexii = False, begin = 0,
-               style = None, highlight = None, cyclic = False) -> str generator
+               style = None, highlight = None, cyclic = False, groupsize=4) -> str generator
 
     Return a hexdump-dump of a string.
 
     Arguments:
         s(str): The data to hexdump.
         width(int): The number of characters per line
+        groupsize(int): The number of characters per group
         skip(bool): Set to True, if repeated lines should be replaced by a "*"
         hexii(bool): Set to True, if a hexii-dump should be returned instead of a hexdump.
         begin(int):  Offset of the first byte to print in the left column
@@ -760,7 +767,7 @@ def hexdump(s, width=16, skip=True, hexii=False, begin=0,
         00000020
 
         >>> print hexdump('A'*32, width=8)
-        00000000  41 41 41 41  41 41 41 41   │AAAA│AAAA│
+        00000000  41 41 41 41  41 41 41 41  │AAAA│AAAA│
         *
         00000020
 
@@ -880,7 +887,7 @@ def hexdump(s, width=16, skip=True, hexii=False, begin=0,
         00000010
         >>> print hexdump('A'*16, width=12)
         00000000  41 41 41 41  41 41 41 41  41 41 41 41  │AAAA│AAAA│AAAA│
-        0000000c  41 41 41 41                            │AAAA││
+        0000000c  41 41 41 41                            │AAAA│
         00000010
         >>> print hexdump('A'*16, width=13)
         00000000  41 41 41 41  41 41 41 41  41 41 41 41  41  │AAAA│AAAA│AAAA│A│
@@ -894,6 +901,15 @@ def hexdump(s, width=16, skip=True, hexii=False, begin=0,
         00000000  41 41 41 41  41 41 41 41  41 41 41 41  41 41 41  │AAAA│AAAA│AAAA│AAA│
         0000000f  41                                               │A│
         00000010
+
+        >>> print hexdump('A'*24, width=16, groupsize=8)
+        00000000  41 41 41 41 41 41 41 41  41 41 41 41 41 41 41 41  │AAAAAAAA│AAAAAAAA│
+        00000010  41 41 41 41 41 41 41 41                           │AAAAAAAA│
+        00000018
+        >>> print hexdump('A'*24, width=16, groupsize=-1)
+        00000000  41 41 41 41 41 41 41 41 41 41 41 41 41 41 41 41  │AAAAAAAAAAAAAAAA│
+        00000010  41 41 41 41 41 41 41 41                          │AAAAAAAA│
+        00000018
     """
     s = packing.flat(s)
     return '\n'.join(hexdump_iter(StringIO.StringIO(s),
@@ -903,7 +919,8 @@ def hexdump(s, width=16, skip=True, hexii=False, begin=0,
                                   begin,
                                   style,
                                   highlight,
-                                  cyclic))
+                                  cyclic,
+                                  groupsize))
 
 def negate(value, width = None):
     """

pwnlib/util/packing.py

@@ -479,43 +479,121 @@ def make_unpacker(word_size = None, endianness = None, sign = None, **kwargs):
     else:
         return lambda number: unpack(number, word_size, endianness, sign)
 
+def _fit(pieces, preprocessor, packer, filler):
+    # Pulls bytes from `filler` and adds them to `pad` until it ends in `key`.
+    # Returns the index of `key` in `pad`.
+    pad = []
+    def fill(key):
+        key = list(key)
+        while len(pad) < len(key) or pad[-len(key):] != key:
+            pad.append(filler.next())
+        return len(pad) - len(key)
 
+    # Key conversion:
+    # - convert str/unicode keys to offsets
+    # - convert large int (no null-bytes in a machine word) keys to offsets
+    pieces_ = dict()
+    large_key = 2**(context.word_size-8)
+    for k, v in pieces.items():
+        if isinstance(k, (int, long)):
+            if k >= large_key:
+                k = fill(pack(k))
+        elif isinstance(k, unicode):
+            k = fill(k.encode('utf8'))
+        elif isinstance(k, bytearray):
+            k = fill(str(k))
+        elif isinstance(k, str):
+            k = fill(k)
+        else:
+            raise TypeError("flat(): offset must be of type int or str, but got '%s'" % type(k))
+        if k in pieces_:
+            raise ValueError("flag(): multiple values at offset %d" % k)
+        pieces_[k] = v
+    pieces = pieces_
 
-def _flat(args, preprocessor, packer):
+    # We must "roll back" `filler` so each recursive call to `_flat` gets it in
+    # the right position
+    filler = iters.chain(pad, filler)
+
+    # Build output
+    out = ''
+    for k, v in sorted(pieces.items()):
+        if k < len(out):
+            raise ValueError("flat(): data at offset %d overlaps with previous data which ends at offset %d" % (k, len(out)))
+
+        # Fill up to offset
+        while len(out) < k:
+            out += filler.next()
+
+        # Recursively flatten data
+        out += _flat([v], preprocessor, packer, filler)
+
+    return filler, out
+
+def _flat(args, preprocessor, packer, filler):
     out = []
     for arg in args:
 
-        if not isinstance(arg, (list, tuple)):
+        if not isinstance(arg, (list, tuple, dict)):
             arg_ = preprocessor(arg)
             if arg_ != None:
                 arg = arg_
 
         if hasattr(arg, '__flat__'):
-            out.append(arg.__flat__())
+            val = arg.__flat__()
         elif isinstance(arg, (list, tuple)):
-            out.append(_flat(arg, preprocessor, packer))
+            val = _flat(arg, preprocessor, packer, filler)
+        elif isinstance(arg, dict):
+            filler, val = _fit(arg, preprocessor, packer, filler)
         elif isinstance(arg, str):
-            out.append(arg)
+            val = arg
         elif isinstance(arg, unicode):
-            out.append(arg.encode('utf8'))
+            val = arg.encode('utf8')
         elif isinstance(arg, (int, long)):
-            out.append(packer(arg))
+            val = packer(arg)
         elif isinstance(arg, bytearray):
-            out.append(str(arg))
+            val = str(arg)
         else:
             raise ValueError("flat(): Flat does not support values of type %s" % type(arg))
+
+        out.append(val)
+
+        # Advance `filler` for "non-recursive" values
+        if not isinstance(arg, (list, tuple, dict)):
+            for _ in xrange(len(val)):
+                filler.next()
+
     return ''.join(out)
 
 @LocalContext
 def flat(*args, **kwargs):
-    """flat(*args, preprocessor = None, word_size = None, endianness = None, sign = None)
+    """flat(*args, preprocessor = None, length = None, filler = de_bruijn(),
+     word_size = None, endianness = None, sign = None) -> str
 
     Flattens the arguments into a string.
 
-    This function takes an arbitrary number of arbitrarily nested lists and
-    tuples. It will then find every string and number inside those and flatten
-    them out. Strings are inserted directly while numbers are packed using the
-    :func:`pack` function.
+    This function takes an arbitrary number of arbitrarily nested lists, tuples
+    and dictionaries.  It will then find every string and number inside those
+    and flatten them out.  Strings are inserted directly while numbers are
+    packed using the :func:`pack` function.  Unicode strings are UTF-8 encoded.
+
+    Dictionary keys give offsets at which to place the corresponding values
+    (which are recursively flattened).  Offsets are relative to where the
+    flattened dictionary occurs in the output (i.e. `{0: 'foo'}` is equivalent
+    to `'foo'`).  Offsets can be integers, unicode strings or regular strings.
+    Integer offsets >= `2**(word_size-8)` are converted to a string using
+    `:func:pack`.  Unicode strings are UTF-8 encoded.  After these conversions
+    offsets are either integers or strings.  In the latter case, the offset will
+    be the lowest index at which the string occurs in `filler`.  See examples
+    below.
+
+    Space between pieces of data is filled out using the iterable `filler`.  The
+    `n`'th byte in the output will be byte at index ``n % len(iterable)`` byte
+    in `filler` if it has finite length or the byte at index `n` otherwise.
+
+    If `length` is given, the output will be padded with bytes from `filler` to
+    be this size.  If the output is longer than `length`, a :py:exc:`ValueError`
+    exception is raised.
 
     The three kwargs `word_size`, `endianness` and `sign` will default to using
     values in :mod:`pwnlib.context` if not specified as an argument.
@@ -524,8 +602,10 @@ def flat(*args, **kwargs):
       args: Values to flatten
       preprocessor (function): Gets called on every element to optionally
          transform the element before flattening. If :const:`None` is
-         returned, then the original value is uded.
-      word_size (int): Word size of the converted integer (in bits).
+         returned, then the original value is used.
+      length: The length of the output.
+      filler: Iterable to use for padding.
+      word_size (int): Word size of the converted integer.
       endianness (str): Endianness of the converted integer ("little"/"big").
       sign (str): Signedness of the converted integer (False/True)
 
@@ -534,21 +614,48 @@ def flat(*args, **kwargs):
       '\\x01\\x00testABABABABABAB'
       >>> flat([1, [2, 3]], preprocessor = lambda x: str(x+1))
       '234'
-"""
+      >>> flat({12: 0x41414141,
+      ...       24: 'Hello',
+      ...      })
+      'aaaabaaacaaaAAAAeaaafaaaHello'
+      >>> flat({'caaa': ''})
+      'aaaabaaa'
+      >>> flat({12: 'XXXX'}, filler = 'AB', length = 20)
+      'ABABABABABABXXXXABAB'
+      >>> flat({ 8: [0x41414141, 0x42424242],
+      ...       20: 'CCCC'})
+      'aaaabaaaAAAABBBBeaaaCCCC'
+      >>> flat({ 0x61616162: 'X'})
+      'aaaaX'
+      >>> flat({4: {0: 'X', 4: 'Y'}})
+      'aaaaXaaaY'
+
+    """
+    # HACK: To avoid circular imports we need to delay the import of `cyclic`
+    from pwnlib.util import cyclic
 
     preprocessor = kwargs.pop('preprocessor', lambda x: None)
-    word_size    = kwargs.pop('word_size', None)
+    filler       = kwargs.pop('filler', cyclic.de_bruijn())
+    length       = kwargs.pop('length', None)
 
     if kwargs != {}:
         raise TypeError("flat() does not support argument %r" % kwargs.popitem()[0])
 
-    return _flat(args, preprocessor, make_packer(word_size))
+    filler = iters.cycle(filler)
+    out = _flat(args, preprocessor, make_packer(), filler)
 
+    if length:
+        if len(out) > length:
+            raise ValueError("flat(): Arguments does not fit within `length` (= %d) bytes" % length)
+        out += ''.join(filler.next() for _ in xrange(length - len(out)))
 
-@LocalContext
-def fit(pieces=None, **kwargs):
-    """fit(pieces, filler = de_bruijn(), length = None, preprocessor = None) -> str
+    return out
 
+def fit(*args, **kwargs):
+    """Legacy alias for `:func:flat`"""
+    return flat(*args, **kwargs)
+
+"""
     Generates a string from a dictionary mapping offsets to data to place at
     that offset.
 
@@ -582,97 +689,8 @@ def fit(pieces=None, **kwargs):
       sign (str): Signedness of the converted integer (False/True)
 
     Examples:
-      >>> fit({12: 0x41414141,
-      ...      24: 'Hello',
-      ...     })
-      'aaaabaaacaaaAAAAeaaafaaaHello'
-      >>> fit({'caaa': ''})
-      'aaaabaaa'
-      >>> fit({12: 'XXXX'}, filler = 'AB', length = 20)
-      'ABABABABABABXXXXABAB'
-      >>> fit({ 8: [0x41414141, 0x42424242],
-      ...      20: 'CCCC'})
-      'aaaabaaaAAAABBBBeaaaCCCC'
-      >>> fit({ 0x61616162: 'X'})
-      'aaaaX'
 
     """
-    # HACK: To avoid circular imports we need to delay the import of `cyclic`
-    from pwnlib.util import cyclic
-
-    filler       = kwargs.pop('filler', cyclic.de_bruijn())
-    length       = kwargs.pop('length', None)
-    preprocessor = kwargs.pop('preprocessor', lambda x: None)
-
-    if kwargs != {}:
-        raise TypeError("fit() does not support argument %r" % kwargs.popitem()[0])
-
-    packer = make_packer()
-    filler = iters.cycle(filler)
-    out = ''
-
-    if not length and not pieces:
-        return ''
-
-    if not pieces:
-        return ''.join(filler.next() for f in range(length))
-
-    def fill(out, value):
-        while value not in out:
-            out += filler.next()
-        return out, out.index(value)
-
-    # convert str keys to offsets
-    # convert large int keys to offsets
-    pieces_ = dict()
-    for k, v in pieces.items():
-        if isinstance(k, (int, long)):
-            # cyclic() generally starts with 'aaaa'
-            if k >= 0x61616161:
-                out, k = fill(out, pack(k))
-        elif isinstance(k, str):
-            out, k = fill(out, k)
-        else:
-            raise TypeError("fit(): offset must be of type int or str, but got '%s'" % type(k))
-        pieces_[k] = v
-    pieces = pieces_
-
-    # convert values to their flattened forms
-    for k,v in pieces.items():
-        pieces[k] = _flat([v], preprocessor, packer)
-
-    # if we were provided a length, make sure everything fits
-    last = max(pieces)
-    if length and last and (last + len(pieces[last]) > length):
-        raise ValueError("fit(): Pieces do not fit within `length` (= %d) bytes" % length)
-
-    # insert data into output
-    out = list(out)
-    l = 0
-    for k, v in sorted(pieces.items()):
-        if k < l:
-            raise ValueError("fit(): data at offset %d overlaps with previous data which ends at offset %d" % (k, l))
-        while len(out) < k:
-            out.append(filler.next())
-        v = _flat([v], preprocessor, packer)
-        l = k + len(v)
-
-        # consume the filler for each byte of actual data
-        for i in range(len(out), l):
-            filler.next()
-
-        out[k:l] = v
-
-    # truncate/pad output
-    if length:
-        if l > length:
-            raise ValueError("fit(): Pieces does not fit within `length` (= %d) bytes" % length)
-        while len(out) < length:
-            out.append(filler.next())
-    else:
-        out = out[:l]
-
-    return ''.join(out)
 
 def signed(integer):
     return unpack(pack(integer), signed=True)

pwnlib/util/proc.py

@@ -99,9 +99,9 @@ def name(pid):
         Name of process as listed in ``/proc/<pid>/status``.
 
     Example:
-        >>> pid = pidof('init')[0]
-        >>> name(pid) == 'init'
-        True
+        >>> p = process('cat')
+        >>> name(p.pid)
+        'cat'
     """
     return psutil.Process(pid).name()
 

pwnlib/util/safeeval.py

@@ -1,6 +1,6 @@
 _const_codes = [
     'POP_TOP','ROT_TWO','ROT_THREE','ROT_FOUR','DUP_TOP',
-    'BUILD_LIST','BUILD_MAP','BUILD_TUPLE',
+    'BUILD_LIST','BUILD_MAP','BUILD_TUPLE','BUILD_SET',
     'LOAD_CONST','RETURN_VALUE','STORE_SUBSCR', 'STORE_MAP'
     ]
 

pwnlib/version.py

@@ -1 +1 @@
-__version__ = '3.12.1'
+__version__ = '3.13.0'

setup.py

@@ -2,6 +2,7 @@
 import glob
 import os
 import platform
+import subprocess
 import sys
 import traceback
 from distutils.command.install import INSTALL_SCHEMES
@@ -54,7 +55,6 @@ install_requires     = ['paramiko>=1.15.2',
                         'pygments>=2.0',
                         'pysocks',
                         'python-dateutil',
-                        'pypandoc',
                         'packaging',
                         'psutil>=3.3.0',
                         'intervaltree<3.0', # See Gallopsled/pwntools#1238
@@ -71,23 +71,16 @@ if not os.path.exists(PythonH):
 # Convert README.md to reStructuredText for PyPI
 long_description = ''
 try:
-    import pypandoc
-    try:
-        pypandoc.get_pandoc_path()
-    except OSError:
-        pypandoc.download_pandoc()
-    long_description = pypandoc.convert_file('README.md', 'rst')
-except ImportError:
-    pass
+    long_description = subprocess.check_output(['pandoc', 'README.md', '--to=rst'])
 except Exception as e:
     print >>sys.stderr, "Failed to convert README.md through pandoc, proceeding anyway"
     traceback.print_exc()
 
-
 setup(
     name                 = 'pwntools',
+    python_requires      = '~=2.7',
     packages             = find_packages(),
-    version              = '3.12.1',
+    version              = '3.13.0',
     data_files           = [('',
                              glob.glob('*.md') + glob.glob('*.txt')),
                             ],

travis/install.sh

@@ -184,7 +184,7 @@ setup_osx()
 }
 
 if [[ "$USER" == "travis" ]]; then
-    setup_travis
+#   setup_travis
     setup_android_emulator
 elif [[ "$USER" == "shippable" ]]; then
     sudo apt-get update