Bug 1522931 - Vendor biplist; r=froydnj

Differential Revision: https://phabricator.services.mozilla.com/D26390

--HG--
extra : moz-landing-system : lando

build/virtualenv_packages.txt

@@ -8,6 +8,7 @@ mozilla.pth:python/mozversioncontrol
 mozilla.pth:python/l10n
 mozilla.pth:third_party/python/atomicwrites
 mozilla.pth:third_party/python/attrs/src
+mozilla.pth:third_party/python/biplist
 mozilla.pth:third_party/python/blessings
 mozilla.pth:third_party/python/Click
 mozilla.pth:third_party/python/compare-locales

third_party/python/biplist/AUTHORS

@@ -0,0 +1,3 @@
+Andrew Wooster (andrew@planetaryscale.com)
+
+Ported to Python 3 by Kevin Kelley (kelleyk@kelleyk.net)

third_party/python/biplist/LICENSE

@@ -0,0 +1,25 @@
+Copyright (c) 2010, Andrew Wooster
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright notice,
+      this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+    * Neither the name of biplist nor the names of its contributors may be
+      used to endorse or promote products derived from this software without 
+      specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

third_party/python/biplist/MANIFEST.in

@@ -0,0 +1,4 @@
+include LICENSE
+include AUTHORS
+include README.md
+recursive-include tests *.py *.plist

third_party/python/biplist/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 1.1
+Name: biplist
+Version: 1.0.3
+Summary: biplist is a library for reading/writing binary plists.
+Home-page: https://bitbucket.org/wooster/biplist
+Author: Andrew Wooster
+Author-email: andrew@planetaryscale.com
+License: BSD
+Download-URL: https://bitbucket.org/wooster/biplist/downloads/biplist-1.0.3.tar.gz
+Description: `biplist` is a binary plist parser/generator for Python.
+        
+        Binary Property List (plist) files provide a faster and smaller serialization
+        format for property lists on OS X. This is a library for generating binary
+        plists which can be read by OS X, iOS, or other clients.
+        
+        This module requires Python 2.6 or higher or Python 3.4 or higher.
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Text Processing :: Markup

third_party/python/biplist/README.md

@@ -0,0 +1,62 @@
+biplist
+=======
+`biplist` is a binary plist parser/generator for Python.
+
+## About
+
+Binary Property List (plist) files provide a faster and smaller serialization
+format for property lists on OS X. This is a library for generating binary
+plists which can be read by OS X, iOS, or other clients.
+
+## API
+
+The API models the `plistlib` API, and will call through to plistlib when
+XML serialization or deserialization is required.
+
+To generate plists with UID values, wrap the values with the `Uid` object. The
+value must be an int.
+
+To generate plists with `NSData`/`CFData` values, wrap the values with the
+`Data` object. The value must be a string.
+
+Date values can only be `datetime.datetime` objects.
+
+The exceptions `InvalidPlistException` and `NotBinaryPlistException` may be 
+thrown to indicate that the data cannot be serialized or deserialized as
+a binary plist.
+
+## Installation
+
+To install the latest release version:
+
+`sudo easy_install biplist`
+
+## Examples
+
+Plist generation example:
+
+```python
+from biplist import *
+from datetime import datetime
+plist = {'aKey':'aValue',
+         '0':1.322,
+         'now':datetime.now(),
+         'list':[1,2,3],
+         'tuple':('a','b','c')
+         }
+try:
+    writePlist(plist, "example.plist")
+except (InvalidPlistException, NotBinaryPlistException), e:
+    print "Something bad happened:", e
+```
+
+Plist parsing example:
+
+```python
+from biplist import *
+try:
+    plist = readPlist("example.plist")
+    print plist
+except (InvalidPlistException, NotBinaryPlistException), e:
+    print "Not a plist:", e
+```

third_party/python/biplist/biplist/__init__.py

@@ -0,0 +1,977 @@
+"""biplist -- a library for reading and writing binary property list files.
+
+Binary Property List (plist) files provide a faster and smaller serialization
+format for property lists on OS X. This is a library for generating binary
+plists which can be read by OS X, iOS, or other clients.
+
+The API models the plistlib API, and will call through to plistlib when
+XML serialization or deserialization is required.
+
+To generate plists with UID values, wrap the values with the Uid object. The
+value must be an int.
+
+To generate plists with NSData/CFData values, wrap the values with the
+Data object. The value must be a string.
+
+Date values can only be datetime.datetime objects.
+
+The exceptions InvalidPlistException and NotBinaryPlistException may be 
+thrown to indicate that the data cannot be serialized or deserialized as
+a binary plist.
+
+Plist generation example:
+    
+    from biplist import *
+    from datetime import datetime
+    plist = {'aKey':'aValue',
+             '0':1.322,
+             'now':datetime.now(),
+             'list':[1,2,3],
+             'tuple':('a','b','c')
+             }
+    try:
+        writePlist(plist, "example.plist")
+    except (InvalidPlistException, NotBinaryPlistException), e:
+        print "Something bad happened:", e
+
+Plist parsing example:
+
+    from biplist import *
+    try:
+        plist = readPlist("example.plist")
+        print plist
+    except (InvalidPlistException, NotBinaryPlistException), e:
+        print "Not a plist:", e
+"""
+
+from collections import namedtuple
+import datetime
+import io
+import math
+import plistlib
+from struct import pack, unpack, unpack_from
+from struct import error as struct_error
+import sys
+import time
+
+try:
+    unicode
+    unicodeEmpty = r''
+except NameError:
+    unicode = str
+    unicodeEmpty = ''
+try:
+    long
+except NameError:
+    long = int
+try:
+    {}.iteritems
+    iteritems = lambda x: x.iteritems()
+except AttributeError:
+    iteritems = lambda x: x.items()
+
+__all__ = [
+    'Uid', 'Data', 'readPlist', 'writePlist', 'readPlistFromString',
+    'writePlistToString', 'InvalidPlistException', 'NotBinaryPlistException'
+]
+
+# Apple uses Jan 1, 2001 as a base for all plist date/times.
+apple_reference_date = datetime.datetime.utcfromtimestamp(978307200)
+
+class Uid(object):
+    """Wrapper around integers for representing UID values. This
+       is used in keyed archiving."""
+    integer = 0
+    def __init__(self, integer):
+        self.integer = integer
+    
+    def __repr__(self):
+        return "Uid(%d)" % self.integer
+    
+    def __eq__(self, other):
+        if isinstance(self, Uid) and isinstance(other, Uid):
+            return self.integer == other.integer
+        return False
+    
+    def __cmp__(self, other):
+        return self.integer - other.integer
+    
+    def __lt__(self, other):
+        return self.integer < other.integer
+    
+    def __hash__(self):
+        return self.integer
+    
+    def __int__(self):
+        return int(self.integer)
+
+class Data(bytes):
+    """Wrapper around bytes to distinguish Data values."""
+
+class InvalidPlistException(Exception):
+    """Raised when the plist is incorrectly formatted."""
+
+class NotBinaryPlistException(Exception):
+    """Raised when a binary plist was expected but not encountered."""
+
+def readPlist(pathOrFile):
+    """Raises NotBinaryPlistException, InvalidPlistException"""
+    didOpen = False
+    result = None
+    if isinstance(pathOrFile, (bytes, unicode)):
+        pathOrFile = open(pathOrFile, 'rb')
+        didOpen = True
+    try:
+        reader = PlistReader(pathOrFile)
+        result = reader.parse()
+    except NotBinaryPlistException as e:
+        try:
+            pathOrFile.seek(0)
+            result = None
+            if hasattr(plistlib, 'loads'):
+                contents = None
+                if isinstance(pathOrFile, (bytes, unicode)):
+                    with open(pathOrFile, 'rb') as f:
+                        contents = f.read()
+                else:
+                    contents = pathOrFile.read()
+                result = plistlib.loads(contents)
+            else:
+                result = plistlib.readPlist(pathOrFile)
+            result = wrapDataObject(result, for_binary=True)
+        except Exception as e:
+            raise InvalidPlistException(e)
+    finally:
+        if didOpen:
+            pathOrFile.close()
+    return result
+
+def wrapDataObject(o, for_binary=False):
+    if isinstance(o, Data) and not for_binary:
+        v = sys.version_info
+        if not (v[0] >= 3 and v[1] >= 4):
+            o = plistlib.Data(o)
+    elif isinstance(o, (bytes, plistlib.Data)) and for_binary:
+        if hasattr(o, 'data'):
+            o = Data(o.data)
+    elif isinstance(o, tuple):
+        o = wrapDataObject(list(o), for_binary)
+        o = tuple(o)
+    elif isinstance(o, list):
+        for i in range(len(o)):
+            o[i] = wrapDataObject(o[i], for_binary)
+    elif isinstance(o, dict):
+        for k in o:
+            o[k] = wrapDataObject(o[k], for_binary)
+    return o
+
+def writePlist(rootObject, pathOrFile, binary=True):
+    if not binary:
+        rootObject = wrapDataObject(rootObject, binary)
+        if hasattr(plistlib, "dump"):
+            if isinstance(pathOrFile, (bytes, unicode)):
+                with open(pathOrFile, 'wb') as f:
+                    return plistlib.dump(rootObject, f)
+            else:
+                return plistlib.dump(rootObject, pathOrFile)
+        else:
+            return plistlib.writePlist(rootObject, pathOrFile)
+    else:
+        didOpen = False
+        if isinstance(pathOrFile, (bytes, unicode)):
+            pathOrFile = open(pathOrFile, 'wb')
+            didOpen = True
+        writer = PlistWriter(pathOrFile)
+        result = writer.writeRoot(rootObject)
+        if didOpen:
+            pathOrFile.close()
+        return result
+
+def readPlistFromString(data):
+    return readPlist(io.BytesIO(data))
+
+def writePlistToString(rootObject, binary=True):
+    if not binary:
+        rootObject = wrapDataObject(rootObject, binary)
+        if hasattr(plistlib, "dumps"):
+            return plistlib.dumps(rootObject)
+        elif hasattr(plistlib, "writePlistToBytes"):
+            return plistlib.writePlistToBytes(rootObject)
+        else:
+            return plistlib.writePlistToString(rootObject)
+    else:
+        ioObject = io.BytesIO()
+        writer = PlistWriter(ioObject)
+        writer.writeRoot(rootObject)
+        return ioObject.getvalue()
+
+def is_stream_binary_plist(stream):
+    stream.seek(0)
+    header = stream.read(7)
+    if header == b'bplist0':
+        return True
+    else:
+        return False
+
+PlistTrailer = namedtuple('PlistTrailer', 'offsetSize, objectRefSize, offsetCount, topLevelObjectNumber, offsetTableOffset')
+PlistByteCounts = namedtuple('PlistByteCounts', 'nullBytes, boolBytes, intBytes, realBytes, dateBytes, dataBytes, stringBytes, uidBytes, arrayBytes, setBytes, dictBytes')
+
+class PlistReader(object):
+    file = None
+    contents = ''
+    offsets = None
+    trailer = None
+    currentOffset = 0
+    # Used to detect recursive object references.
+    offsetsStack = []
+    
+    def __init__(self, fileOrStream):
+        """Raises NotBinaryPlistException."""
+        self.reset()
+        self.file = fileOrStream
+    
+    def parse(self):
+        return self.readRoot()
+    
+    def reset(self):
+        self.trailer = None
+        self.contents = ''
+        self.offsets = []
+        self.currentOffset = 0
+        self.offsetsStack = []
+    
+    def readRoot(self):
+        result = None
+        self.reset()
+        # Get the header, make sure it's a valid file.
+        if not is_stream_binary_plist(self.file):
+            raise NotBinaryPlistException()
+        self.file.seek(0)
+        self.contents = self.file.read()
+        if len(self.contents) < 32:
+            raise InvalidPlistException("File is too short.")
+        trailerContents = self.contents[-32:]
+        try:
+            self.trailer = PlistTrailer._make(unpack("!xxxxxxBBQQQ", trailerContents))
+            
+            if pow(2, self.trailer.offsetSize*8) < self.trailer.offsetTableOffset:
+                raise InvalidPlistException("Offset size insufficient to reference all objects.")
+            
+            if pow(2, self.trailer.objectRefSize*8) < self.trailer.offsetCount:
+                raise InvalidPlistException("Too many offsets to represent in size of object reference representation.")
+            
+            offset_size = self.trailer.offsetSize * self.trailer.offsetCount
+            offset = self.trailer.offsetTableOffset
+            
+            if offset + offset_size > pow(2, 64):
+                raise InvalidPlistException("Offset table is excessively long.")
+            
+            if self.trailer.offsetSize > 16:
+                raise InvalidPlistException("Offset size is greater than maximum integer size.")
+            
+            if self.trailer.objectRefSize == 0:
+                raise InvalidPlistException("Object reference size is zero.")
+            
+            if offset >= len(self.contents) - 32:
+                raise InvalidPlistException("Offset table offset is too large.")
+            
+            if offset < len("bplist00x"):
+                raise InvalidPlistException("Offset table offset is too small.")
+            
+            if self.trailer.topLevelObjectNumber >= self.trailer.offsetCount:
+                raise InvalidPlistException("Top level object number is larger than the number of objects.")
+            
+            offset_contents = self.contents[offset:offset+offset_size]
+            offset_i = 0
+            offset_table_length = len(offset_contents)
+            
+            while offset_i < self.trailer.offsetCount:
+                begin = self.trailer.offsetSize*offset_i
+                end = begin+self.trailer.offsetSize
+                if end > offset_table_length:
+                    raise InvalidPlistException("End of object is at invalid offset %d in offset table of length %d" % (end, offset_table_length))
+                tmp_contents = offset_contents[begin:end]
+                tmp_sized = self.getSizedInteger(tmp_contents, self.trailer.offsetSize)
+                self.offsets.append(tmp_sized)
+                offset_i += 1
+            self.setCurrentOffsetToObjectNumber(self.trailer.topLevelObjectNumber)
+            result = self.readObject()
+        except TypeError as e:
+            raise InvalidPlistException(e)
+        return result
+    
+    def setCurrentOffsetToObjectNumber(self, objectNumber):
+        if objectNumber > len(self.offsets) - 1:
+            raise InvalidPlistException("Invalid offset number: %d" % objectNumber)
+        self.currentOffset = self.offsets[objectNumber]
+        if self.currentOffset in self.offsetsStack:
+            raise InvalidPlistException("Recursive data structure detected in object: %d" % objectNumber)
+    
+    def beginOffsetProtection(self):
+        self.offsetsStack.append(self.currentOffset)
+        return self.currentOffset
+    
+    def endOffsetProtection(self, offset):
+        try:
+            index = self.offsetsStack.index(offset)
+            self.offsetsStack = self.offsetsStack[:index]
+        except ValueError as e:
+            pass
+    
+    def readObject(self):
+        protection = self.beginOffsetProtection()
+        result = None
+        tmp_byte = self.contents[self.currentOffset:self.currentOffset+1]
+        if len(tmp_byte) != 1:
+            raise InvalidPlistException("No object found at offset: %d" % self.currentOffset)
+        marker_byte = unpack("!B", tmp_byte)[0]
+        format = (marker_byte >> 4) & 0x0f
+        extra = marker_byte & 0x0f
+        self.currentOffset += 1
+        
+        def proc_extra(extra):
+            if extra == 0b1111:
+                extra = self.readObject()
+            return extra
+        
+        # bool, null, or fill byte
+        if format == 0b0000:
+            if extra == 0b0000:
+                result = None
+            elif extra == 0b1000:
+                result = False
+            elif extra == 0b1001:
+                result = True
+            elif extra == 0b1111:
+                pass # fill byte
+            else:
+                raise InvalidPlistException("Invalid object found at offset: %d" % (self.currentOffset - 1))
+        # int
+        elif format == 0b0001:
+            result = self.readInteger(pow(2, extra))
+        # real
+        elif format == 0b0010:
+            result = self.readReal(extra)
+        # date
+        elif format == 0b0011 and extra == 0b0011:
+            result = self.readDate()
+        # data
+        elif format == 0b0100:
+            extra = proc_extra(extra)
+            result = self.readData(extra)
+        # ascii string
+        elif format == 0b0101:
+            extra = proc_extra(extra)
+            result = self.readAsciiString(extra)
+        # Unicode string
+        elif format == 0b0110:
+            extra = proc_extra(extra)
+            result = self.readUnicode(extra)
+        # uid
+        elif format == 0b1000:
+            result = self.readUid(extra)
+        # array
+        elif format == 0b1010:
+            extra = proc_extra(extra)
+            result = self.readArray(extra)
+        # set
+        elif format == 0b1100:
+            extra = proc_extra(extra)
+            result = set(self.readArray(extra))
+        # dict
+        elif format == 0b1101:
+            extra = proc_extra(extra)
+            result = self.readDict(extra)
+        else:    
+            raise InvalidPlistException("Invalid object found: {format: %s, extra: %s}" % (bin(format), bin(extra)))
+        self.endOffsetProtection(protection)
+        return result
+    
+    def readContents(self, length, description="Object contents"):
+        end = self.currentOffset + length
+        if end >= len(self.contents) - 32:
+            raise InvalidPlistException("%s extends into trailer" % description)
+        elif length < 0:
+            raise InvalidPlistException("%s length is less than zero" % length)
+        data = self.contents[self.currentOffset:end]
+        return data
+    
+    def readInteger(self, byteSize):
+        data = self.readContents(byteSize, "Integer")
+        self.currentOffset = self.currentOffset + byteSize
+        return self.getSizedInteger(data, byteSize, as_number=True)
+    
+    def readReal(self, length):
+        to_read = pow(2, length)
+        data = self.readContents(to_read, "Real")
+        if length == 2: # 4 bytes
+            result = unpack('>f', data)[0]
+        elif length == 3: # 8 bytes
+            result = unpack('>d', data)[0]
+        else:
+            raise InvalidPlistException("Unknown Real of length %d bytes" % to_read)
+        return result
+    
+    def readRefs(self, count):    
+        refs = []
+        i = 0
+        while i < count:
+            fragment = self.readContents(self.trailer.objectRefSize, "Object reference")
+            ref = self.getSizedInteger(fragment, len(fragment))
+            refs.append(ref)
+            self.currentOffset += self.trailer.objectRefSize
+            i += 1
+        return refs
+    
+    def readArray(self, count):
+        if not isinstance(count, (int, long)):
+            raise InvalidPlistException("Count of entries in dict isn't of integer type.")
+        result = []
+        values = self.readRefs(count)
+        i = 0
+        while i < len(values):
+            self.setCurrentOffsetToObjectNumber(values[i])
+            value = self.readObject()
+            result.append(value)
+            i += 1
+        return result
+    
+    def readDict(self, count):
+        if not isinstance(count, (int, long)):
+            raise InvalidPlistException("Count of keys/values in dict isn't of integer type.")
+        result = {}
+        keys = self.readRefs(count)
+        values = self.readRefs(count)
+        i = 0
+        while i < len(keys):
+            self.setCurrentOffsetToObjectNumber(keys[i])
+            key = self.readObject()
+            self.setCurrentOffsetToObjectNumber(values[i])
+            value = self.readObject()
+            result[key] = value
+            i += 1
+        return result
+    
+    def readAsciiString(self, length):
+        if not isinstance(length, (int, long)):
+            raise InvalidPlistException("Length of ASCII string isn't of integer type.")
+        data = self.readContents(length, "ASCII string")
+        result = unpack("!%ds" % length, data)[0]
+        self.currentOffset += length
+        return str(result.decode('ascii'))
+    
+    def readUnicode(self, length):
+        if not isinstance(length, (int, long)):
+            raise InvalidPlistException("Length of Unicode string isn't of integer type.")
+        actual_length = length*2
+        data = self.readContents(actual_length, "Unicode string")
+        self.currentOffset += actual_length
+        return data.decode('utf_16_be')
+    
+    def readDate(self):
+        data = self.readContents(8, "Date")
+        x = unpack(">d", data)[0]
+        if math.isnan(x):
+            raise InvalidPlistException("Date is NaN")
+        # Use timedelta to workaround time_t size limitation on 32-bit python.
+        try:
+            result = datetime.timedelta(seconds=x) + apple_reference_date
+        except OverflowError:
+            if x > 0:
+                result = datetime.datetime.max
+            else:
+                result = datetime.datetime.min
+        self.currentOffset += 8
+        return result
+    
+    def readData(self, length):
+        if not isinstance(length, (int, long)):
+            raise InvalidPlistException("Length of data isn't of integer type.")
+        result = self.readContents(length, "Data")
+        self.currentOffset += length
+        return Data(result)
+    
+    def readUid(self, length):
+        if not isinstance(length, (int, long)):
+            raise InvalidPlistException("Uid length isn't of integer type.")
+        return Uid(self.readInteger(length+1))
+    
+    def getSizedInteger(self, data, byteSize, as_number=False):
+        """Numbers of 8 bytes are signed integers when they refer to numbers, but unsigned otherwise."""
+        result = 0
+        if byteSize == 0:
+            raise InvalidPlistException("Encountered integer with byte size of 0.")
+        # 1, 2, and 4 byte integers are unsigned
+        elif byteSize == 1:
+            result = unpack('>B', data)[0]
+        elif byteSize == 2:
+            result = unpack('>H', data)[0]
+        elif byteSize == 4:
+            result = unpack('>L', data)[0]
+        elif byteSize == 8:
+            if as_number:
+                result = unpack('>q', data)[0]
+            else:
+                result = unpack('>Q', data)[0]
+        elif byteSize <= 16:
+            # Handle odd-sized or integers larger than 8 bytes
+            # Don't naively go over 16 bytes, in order to prevent infinite loops.
+            result = 0
+            if hasattr(int, 'from_bytes'):
+                result = int.from_bytes(data, 'big')
+            else:
+                for byte in data:
+                    if not isinstance(byte, int): # Python3.0-3.1.x return ints, 2.x return str
+                        byte = unpack_from('>B', byte)[0]
+                    result = (result << 8) | byte
+        else:
+            raise InvalidPlistException("Encountered integer longer than 16 bytes.")
+        return result
+
+class HashableWrapper(object):
+    def __init__(self, value):
+        self.value = value
+    def __repr__(self):
+        return "<HashableWrapper: %s>" % [self.value]
+
+class BoolWrapper(object):
+    def __init__(self, value):
+        self.value = value
+    def __repr__(self):
+        return "<BoolWrapper: %s>" % self.value
+
+class FloatWrapper(object):
+    _instances = {}
+    def __new__(klass, value):
+        # Ensure FloatWrapper(x) for a given float x is always the same object
+        wrapper = klass._instances.get(value)
+        if wrapper is None:
+            wrapper = object.__new__(klass)
+            wrapper.value = value
+            klass._instances[value] = wrapper
+        return wrapper
+    def __repr__(self):
+        return "<FloatWrapper: %s>" % self.value
+
+class StringWrapper(object):
+    __instances = {}
+    
+    encodedValue = None
+    encoding = None
+    
+    def __new__(cls, value):
+        '''Ensure we only have a only one instance for any string,
+         and that we encode ascii as 1-byte-per character when possible'''
+        
+        encodedValue = None
+        
+        for encoding in ('ascii', 'utf_16_be'):
+            try:
+               encodedValue = value.encode(encoding)
+            except: pass
+            if encodedValue is not None:
+                if encodedValue not in cls.__instances:
+                    cls.__instances[encodedValue] = super(StringWrapper, cls).__new__(cls)
+                    cls.__instances[encodedValue].encodedValue = encodedValue
+                    cls.__instances[encodedValue].encoding = encoding
+                return cls.__instances[encodedValue]
+        
+        raise ValueError('Unable to get ascii or utf_16_be encoding for %s' % repr(value))
+    
+    def __len__(self):
+        '''Return roughly the number of characters in this string (half the byte length)'''
+        if self.encoding == 'ascii':
+            return len(self.encodedValue)
+        else:
+            return len(self.encodedValue)//2
+    
+    def __lt__(self, other):
+        return self.encodedValue < other.encodedValue
+    
+    @property
+    def encodingMarker(self):
+        if self.encoding == 'ascii':
+            return 0b0101
+        else:
+            return 0b0110
+    
+    def __repr__(self):
+        return '<StringWrapper (%s): %s>' % (self.encoding, self.encodedValue)
+
+class PlistWriter(object):
+    header = b'bplist00bybiplist1.0'
+    file = None
+    byteCounts = None
+    trailer = None
+    computedUniques = None
+    writtenReferences = None
+    referencePositions = None
+    wrappedTrue = None
+    wrappedFalse = None
+    # Used to detect recursive object references.
+    objectsStack = []
+    
+    def __init__(self, file):
+        self.reset()
+        self.file = file
+        self.wrappedTrue = BoolWrapper(True)
+        self.wrappedFalse = BoolWrapper(False)
+
+    def reset(self):
+        self.byteCounts = PlistByteCounts(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
+        self.trailer = PlistTrailer(0, 0, 0, 0, 0)
+        
+        # A set of all the uniques which have been computed.
+        self.computedUniques = set()
+        # A list of all the uniques which have been written.
+        self.writtenReferences = {}
+        # A dict of the positions of the written uniques.
+        self.referencePositions = {}
+        
+        self.objectsStack = []
+        
+    def positionOfObjectReference(self, obj):
+        """If the given object has been written already, return its
+           position in the offset table. Otherwise, return None."""
+        return self.writtenReferences.get(obj)
+        
+    def writeRoot(self, root):
+        """
+        Strategy is:
+        - write header
+        - wrap root object so everything is hashable
+        - compute size of objects which will be written
+          - need to do this in order to know how large the object refs
+            will be in the list/dict/set reference lists
+        - write objects
+          - keep objects in writtenReferences
+          - keep positions of object references in referencePositions
+          - write object references with the length computed previously
+        - computer object reference length
+        - write object reference positions
+        - write trailer
+        """
+        output = self.header
+        wrapped_root = self.wrapRoot(root)
+        self.computeOffsets(wrapped_root, asReference=True, isRoot=True)
+        self.trailer = self.trailer._replace(**{'objectRefSize':self.intSize(len(self.computedUniques))})
+        self.writeObjectReference(wrapped_root, output)
+        output = self.writeObject(wrapped_root, output, setReferencePosition=True)
+        
+        # output size at this point is an upper bound on how big the
+        # object reference offsets need to be.
+        self.trailer = self.trailer._replace(**{
+            'offsetSize':self.intSize(len(output)),
+            'offsetCount':len(self.computedUniques),
+            'offsetTableOffset':len(output),
+            'topLevelObjectNumber':0
+            })
+        
+        output = self.writeOffsetTable(output)
+        output += pack('!xxxxxxBBQQQ', *self.trailer)
+        self.file.write(output)
+    
+    def beginRecursionProtection(self, obj):
+        if not isinstance(obj, (set, dict, list, tuple)):
+            return
+        if id(obj) in self.objectsStack:
+            raise InvalidPlistException("Recursive containers are not allowed in plists.")
+        self.objectsStack.append(id(obj))
+    
+    def endRecursionProtection(self, obj):
+        if not isinstance(obj, (set, dict, list, tuple)):
+            return
+        try:
+            index = self.objectsStack.index(id(obj))
+            self.objectsStack = self.objectsStack[:index]
+        except ValueError as e:
+            pass
+
+    def wrapRoot(self, root):
+        result = None
+        self.beginRecursionProtection(root)
+        
+        if isinstance(root, bool):
+            if root is True:
+                result = self.wrappedTrue
+            else:
+                result = self.wrappedFalse
+        elif isinstance(root, float):
+            result = FloatWrapper(root)
+        elif isinstance(root, set):
+            n = set()
+            for value in root:
+                n.add(self.wrapRoot(value))
+            result = HashableWrapper(n)
+        elif isinstance(root, dict):
+            n = {}
+            for key, value in iteritems(root):
+                n[self.wrapRoot(key)] = self.wrapRoot(value)
+            result = HashableWrapper(n)
+        elif isinstance(root, list):
+            n = []
+            for value in root:
+                n.append(self.wrapRoot(value))
+            result = HashableWrapper(n)
+        elif isinstance(root, tuple):
+            n = tuple([self.wrapRoot(value) for value in root])
+            result = HashableWrapper(n)
+        elif isinstance(root, (str, unicode)) and not isinstance(root, Data):
+            result =  StringWrapper(root)
+        elif isinstance(root, bytes):
+            result = Data(root)
+        else:
+            result = root
+        
+        self.endRecursionProtection(root)
+        return result
+
+    def incrementByteCount(self, field, incr=1):
+        self.byteCounts = self.byteCounts._replace(**{field:self.byteCounts.__getattribute__(field) + incr})
+
+    def computeOffsets(self, obj, asReference=False, isRoot=False):
+        def check_key(key):
+            if key is None:
+                raise InvalidPlistException('Dictionary keys cannot be null in plists.')
+            elif isinstance(key, Data):
+                raise InvalidPlistException('Data cannot be dictionary keys in plists.')
+            elif not isinstance(key, StringWrapper):
+                raise InvalidPlistException('Keys must be strings.')
+        
+        def proc_size(size):
+            if size > 0b1110:
+                size += self.intSize(size)
+            return size
+        # If this should be a reference, then we keep a record of it in the
+        # uniques table.
+        if asReference:
+            if obj in self.computedUniques:
+                return
+            else:
+                self.computedUniques.add(obj)
+        
+        if obj is None:
+            self.incrementByteCount('nullBytes')
+        elif isinstance(obj, BoolWrapper):
+            self.incrementByteCount('boolBytes')
+        elif isinstance(obj, Uid):
+            size = self.intSize(obj.integer)
+            self.incrementByteCount('uidBytes', incr=1+size)
+        elif isinstance(obj, (int, long)):
+            size = self.intSize(obj)
+            self.incrementByteCount('intBytes', incr=1+size)
+        elif isinstance(obj, FloatWrapper):
+            size = self.realSize(obj)
+            self.incrementByteCount('realBytes', incr=1+size)
+        elif isinstance(obj, datetime.datetime):    
+            self.incrementByteCount('dateBytes', incr=2)
+        elif isinstance(obj, Data):
+            size = proc_size(len(obj))
+            self.incrementByteCount('dataBytes', incr=1+size)
+        elif isinstance(obj, StringWrapper):
+            size = proc_size(len(obj))
+            self.incrementByteCount('stringBytes', incr=1+size)
+        elif isinstance(obj, HashableWrapper):
+            obj = obj.value
+            if isinstance(obj, set):
+                size = proc_size(len(obj))
+                self.incrementByteCount('setBytes', incr=1+size)
+                for value in obj:
+                    self.computeOffsets(value, asReference=True)
+            elif isinstance(obj, (list, tuple)):
+                size = proc_size(len(obj))
+                self.incrementByteCount('arrayBytes', incr=1+size)
+                for value in obj:
+                    asRef = True
+                    self.computeOffsets(value, asReference=True)
+            elif isinstance(obj, dict):
+                size = proc_size(len(obj))
+                self.incrementByteCount('dictBytes', incr=1+size)
+                for key, value in iteritems(obj):
+                    check_key(key)
+                    self.computeOffsets(key, asReference=True)
+                    self.computeOffsets(value, asReference=True)
+        else:
+            raise InvalidPlistException("Unknown object type: %s (%s)" % (type(obj).__name__, repr(obj)))
+
+    def writeObjectReference(self, obj, output):
+        """Tries to write an object reference, adding it to the references
+           table. Does not write the actual object bytes or set the reference
+           position. Returns a tuple of whether the object was a new reference
+           (True if it was, False if it already was in the reference table)
+           and the new output.
+        """
+        position = self.positionOfObjectReference(obj)
+        if position is None:
+            self.writtenReferences[obj] = len(self.writtenReferences)
+            output += self.binaryInt(len(self.writtenReferences) - 1, byteSize=self.trailer.objectRefSize)
+            return (True, output)
+        else:
+            output += self.binaryInt(position, byteSize=self.trailer.objectRefSize)
+            return (False, output)
+
+    def writeObject(self, obj, output, setReferencePosition=False):
+        """Serializes the given object to the output. Returns output.
+           If setReferencePosition is True, will set the position the
+           object was written.
+        """
+        def proc_variable_length(format, length):
+            result = b''
+            if length > 0b1110:
+                result += pack('!B', (format << 4) | 0b1111)
+                result = self.writeObject(length, result)
+            else:
+                result += pack('!B', (format << 4) | length)
+            return result
+        
+        def timedelta_total_seconds(td):
+            # Shim for Python 2.6 compatibility, which doesn't have total_seconds.
+            # Make one argument a float to ensure the right calculation.
+            return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10.0**6) / 10.0**6
+       
+        if setReferencePosition:
+            self.referencePositions[obj] = len(output)
+        
+        if obj is None:
+            output += pack('!B', 0b00000000)
+        elif isinstance(obj, BoolWrapper):
+            if obj.value is False:
+                output += pack('!B', 0b00001000)
+            else:
+                output += pack('!B', 0b00001001)
+        elif isinstance(obj, Uid):
+            size = self.intSize(obj.integer)
+            output += pack('!B', (0b1000 << 4) | size - 1)
+            output += self.binaryInt(obj.integer)
+        elif isinstance(obj, (int, long)):
+            byteSize = self.intSize(obj)
+            root = math.log(byteSize, 2)
+            output += pack('!B', (0b0001 << 4) | int(root))
+            output += self.binaryInt(obj, as_number=True)
+        elif isinstance(obj, FloatWrapper):
+            # just use doubles
+            output += pack('!B', (0b0010 << 4) | 3)
+            output += self.binaryReal(obj)
+        elif isinstance(obj, datetime.datetime):
+            try:
+                timestamp = (obj - apple_reference_date).total_seconds()
+            except AttributeError:
+                timestamp = timedelta_total_seconds(obj - apple_reference_date)
+            output += pack('!B', 0b00110011)
+            output += pack('!d', float(timestamp))
+        elif isinstance(obj, Data):
+            output += proc_variable_length(0b0100, len(obj))
+            output += obj
+        elif isinstance(obj, StringWrapper):
+            output += proc_variable_length(obj.encodingMarker, len(obj))
+            output += obj.encodedValue
+        elif isinstance(obj, bytes):
+            output += proc_variable_length(0b0101, len(obj))
+            output += obj
+        elif isinstance(obj, HashableWrapper):
+            obj = obj.value
+            if isinstance(obj, (set, list, tuple)):
+                if isinstance(obj, set):
+                    output += proc_variable_length(0b1100, len(obj))
+                else:
+                    output += proc_variable_length(0b1010, len(obj))
+            
+                objectsToWrite = []
+                for objRef in sorted(obj) if isinstance(obj, set) else obj:
+                    (isNew, output) = self.writeObjectReference(objRef, output)
+                    if isNew:
+                        objectsToWrite.append(objRef)
+                for objRef in objectsToWrite:
+                    output = self.writeObject(objRef, output, setReferencePosition=True)
+            elif isinstance(obj, dict):
+                output += proc_variable_length(0b1101, len(obj))
+                keys = []
+                values = []
+                objectsToWrite = []
+                for key, value in sorted(iteritems(obj)):
+                    keys.append(key)
+                    values.append(value)
+                for key in keys:
+                    (isNew, output) = self.writeObjectReference(key, output)
+                    if isNew:
+                        objectsToWrite.append(key)
+                for value in values:
+                    (isNew, output) = self.writeObjectReference(value, output)
+                    if isNew:
+                        objectsToWrite.append(value)
+                for objRef in objectsToWrite:
+                    output = self.writeObject(objRef, output, setReferencePosition=True)
+        return output
+    
+    def writeOffsetTable(self, output):
+        """Writes all of the object reference offsets."""
+        all_positions = []
+        writtenReferences = list(self.writtenReferences.items())
+        writtenReferences.sort(key=lambda x: x[1])
+        for obj,order in writtenReferences:
+            # Porting note: Elsewhere we deliberately replace empty unicdoe strings
+            # with empty binary strings, but the empty unicode string
+            # goes into writtenReferences.  This isn't an issue in Py2
+            # because u'' and b'' have the same hash; but it is in
+            # Py3, where they don't.
+            if bytes != str and obj == unicodeEmpty:
+                obj = b''
+            position = self.referencePositions.get(obj)
+            if position is None:
+                raise InvalidPlistException("Error while writing offsets table. Object not found. %s" % obj)
+            output += self.binaryInt(position, self.trailer.offsetSize)
+            all_positions.append(position)
+        return output
+    
+    def binaryReal(self, obj):
+        # just use doubles
+        result = pack('>d', obj.value)
+        return result
+    
+    def binaryInt(self, obj, byteSize=None, as_number=False):
+        result = b''
+        if byteSize is None:
+            byteSize = self.intSize(obj)
+        if byteSize == 1:
+            result += pack('>B', obj)
+        elif byteSize == 2:
+            result += pack('>H', obj)
+        elif byteSize == 4:
+            result += pack('>L', obj)
+        elif byteSize == 8:
+            if as_number:
+                result += pack('>q', obj)
+            else:
+                result += pack('>Q', obj)
+        elif byteSize <= 16:
+            try:
+                result = pack('>Q', 0) + pack('>Q', obj)
+            except struct_error as e:
+                raise InvalidPlistException("Unable to pack integer %d: %s" % (obj, e))
+        else:
+            raise InvalidPlistException("Core Foundation can't handle integers with size greater than 16 bytes.")
+        return result
+    
+    def intSize(self, obj):
+        """Returns the number of bytes necessary to store the given integer."""
+        # SIGNED
+        if obj < 0: # Signed integer, always 8 bytes
+            return 8
+        # UNSIGNED
+        elif obj <= 0xFF: # 1 byte
+            return 1
+        elif obj <= 0xFFFF: # 2 bytes
+            return 2
+        elif obj <= 0xFFFFFFFF: # 4 bytes
+            return 4
+        # SIGNED
+        # 0x7FFFFFFFFFFFFFFF is the max.
+        elif obj <= 0x7FFFFFFFFFFFFFFF: # 8 bytes signed
+            return 8
+        elif obj <= 0xffffffffffffffff: # 8 bytes unsigned
+            return 16
+        else:
+            raise InvalidPlistException("Core Foundation can't handle integers with size greater than 8 bytes.")
+    
+    def realSize(self, obj):
+        return 8

third_party/python/biplist/setup.cfg

@@ -0,0 +1,5 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+tag_svn_revision = 0
+

third_party/python/biplist/setup.py

@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+
+try:
+    from setuptools import setup, find_packages
+except ImportError:
+    import ez_setup
+    ez_setup.use_setuptools()
+    from setuptools import setup, find_packages
+
+import os
+import sys
+
+major, minor, micro, releaselevel, serial = sys.version_info
+
+if major <= 1 or (major == 2 and minor < 6) or (major == 3 and minor < 4):
+    # N.B.: Haven't tested with older py3k versions.
+    print('This module supports Python 2 >= 2.6 and Python 3 >= 3.4.')
+    sys.exit(1)
+
+author = 'Andrew Wooster'
+email = 'andrew@planetaryscale.com'
+version = '1.0.3'
+desc = 'biplist is a library for reading/writing binary plists.'
+
+setup(
+    name = 'biplist',
+    version = version,
+    url = 'https://bitbucket.org/wooster/biplist',
+    download_url = 'https://bitbucket.org/wooster/biplist/downloads/biplist-%s.tar.gz' % version,
+    license = 'BSD',
+    description = desc,
+    long_description = 
+    """`biplist` is a binary plist parser/generator for Python.
+
+Binary Property List (plist) files provide a faster and smaller serialization
+format for property lists on OS X. This is a library for generating binary
+plists which can be read by OS X, iOS, or other clients.
+
+This module requires Python 2.6 or higher or Python 3.4 or higher.""",
+    author = author,
+    author_email = email,
+    packages = find_packages(),
+    include_package_data = True,
+    zip_safe = False,
+    classifiers = [
+        'Development Status :: 5 - Production/Stable',
+        'Intended Audience :: Developers',
+        'License :: OSI Approved :: BSD License',
+        'Operating System :: OS Independent',
+        'Programming Language :: Python',
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        "Topic :: Text Processing :: Markup",
+    ],
+    test_suite = 'nose.collector',
+    install_requires = [],
+    requires = [],
+    tests_require = ['nose', 'coverage'],
+)

third_party/python/biplist/tests/data/bogus_file.plist

@@ -0,0 +1 @@
+a;djflkadjsflsakdjf

third_party/python/biplist/tests/test_fuzz_results.py

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+
+from biplist import *
+import os
+from test_utils import *
+import unittest
+
+class TestFuzzResults(unittest.TestCase):
+    def setUp(self):
+        pass
+    
+    def testCurrentOffsetOutOfRange(self):
+        try:
+            readPlist(fuzz_data_path('list_index_out_of_range.plist'))
+            self.fail("list index out of range, should fail")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidMarkerByteUnpack(self):
+        try:
+            readPlist(fuzz_data_path('no_marker_byte.plist'))
+            self.fail("No marker byte at object offset")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidObjectOffset(self):
+        try:
+            readPlist(fuzz_data_path('invalid_object_offset.plist'))
+            self.fail("Invalid object offset in offsets table")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testRecursiveObjectOffset(self):
+        try:
+            readPlist(fuzz_data_path('recursive_object_offset.plist'))
+            self.fail("Recursive object found")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testExcessivelyLongAsciiString(self):
+        try:
+            readPlist(fuzz_data_path('ascii_string_too_long.plist'))
+            self.fail("ASCII string extends into trailer")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testNegativelyLongAsciiString(self):
+        try:
+            readPlist(fuzz_data_path('ascii_string_negative_length.plist'))
+            self.fail("ASCII string length less than zero")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidOffsetEnding(self):
+        # The end of the offset is past the end of the offset table.
+        try:
+            readPlist(fuzz_data_path('invalid_offset_ending.plist'))
+            self.fail("End of offset after end of offset table")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidDictionaryObjectCount(self):
+        try:
+            readPlist(fuzz_data_path('dictionary_invalid_count.plist'))
+            self.fail("Dictionary object count is not of integer type")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidArrayObjectCount(self):
+        try:
+            readPlist(fuzz_data_path('array_invalid_count.plist'))
+            self.fail("Array object count is not of integer type")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidRealLength(self):
+        # We shouldn't have been checking for extra length reals, anyway.
+        try:
+            readPlist(fuzz_data_path('real_invalid_length.plist'))
+            self.fail("Real length is not of integer type")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testNaNDateSeconds(self):
+        try:
+            readPlist(fuzz_data_path('date_seconds_is_nan.plist'))
+            self.fail("Date seconds is NaN")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testIntegerWithZeroByteLength(self):
+        try:
+            readPlist(fuzz_data_path('integer_zero_byte_length.plist'))
+            self.fail("Integer has byte size of 0.")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+

third_party/python/biplist/tests/test_invalid.py

@@ -0,0 +1,173 @@
+# -*- coding: utf-8 -*-
+
+from biplist import *
+from biplist import PlistTrailer
+from collections import namedtuple
+from math import pow
+import os
+from struct import pack
+from test_utils import *
+import unittest
+
+def trailerToString(trailer):
+    # Trailer is:
+    # 6 padding bytes
+    # 1 byte offsetSize
+    # 1 byte objectRefSize
+    # 8 bytes offsetCount (number of objects)
+    # 8 bytes topObjectNumber
+    # 8 bytes offsetTableOffset
+    # = 32 byte trailer
+    return pack('!xxxxxxBBQQQ', *trailer)
+
+class TestInvalidPlistFile(unittest.TestCase):
+    def setUp(self):
+        pass
+    def testEmptyFile(self):
+        try:
+            readPlist(data_path('empty_file.plist'))
+            self.fail("Should not successfully read empty plist.")
+        except NotBinaryPlistException as e:
+            pass
+        except InvalidPlistException as e:
+            pass
+    
+    def testTooShort(self):
+        try:
+            readPlistFromString(b"bplist0")
+            self.fail("Should not successfully read plist which is too short.")
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalid(self):
+        try:
+            readPlistFromString(b"bplist0-------------------------------------")
+            self.fail("Should not successfully read invalid plist.")
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidTemplate(self):
+        # Test that our template plists for range tests are valid.
+        trailer = PlistTrailer(1, 1, 1, 0, 9)
+        contents = b''.join([b'bplist00', # header
+                             b'\x08',     # bool false
+                             b'\x08',     # object at offset 8
+                             trailerToString(trailer)
+                         ])
+        result = readPlistFromString(contents)
+        self.assertEqual(result, False)
+        
+        trailer = PlistTrailer(1, 1, 3, 0, 25)
+        contents = b''.join([b'bplist00bybiplist1.0', # header
+                             b'\xA2',     # array with two entries
+                             b'\x01',     # object entry 1
+                             b'\x02',     # object entry 2
+                             b'\x09',     # boolean false
+                             b'\x08',     # boolean true
+                             b'\x14',     # offset at 20
+                             b'\x17',     # offset at 23
+                             b'\x18',     # offset at 24
+                             trailerToString(trailer)
+                         ])
+        result = readPlistFromString(contents)
+        self.assertEqual(result, [True, False])
+    
+    def testInvalidOffsetSize(self):
+        # offset size can't be zero
+        try:
+            trailer = PlistTrailer(0, 1, 1, 0, 9)
+            contents = b''.join([b'bplist00', # header
+                                 b'\x08',     # bool false
+                                 b'\x08',     # object at offset 8
+                                 trailerToString(trailer)
+                             ])
+            readPlistFromString(contents)
+            self.fail("Offset size can't be zero")
+        except InvalidPlistException as e:
+            pass
+        
+        # offset size can't be greater than 16
+        try:
+            trailer = PlistTrailer(17, 1, 1, 0, 9)
+            contents = b''.join([b'bplist00', # header
+                                 b'\x08',     # bool false
+                                 b'\x08',     # object at offset 8
+                                 trailerToString(trailer)
+                             ])
+            readPlistFromString(contents)
+            self.fail("Offset size can't be greater than 16")
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidOffsetOverflow(self):
+        # The offsets can't overflow the number of bytes used to represent them.
+        try:
+            c = readPlist(data_path('invalid_object_offset_size.plist'))
+            self.fail("Object offset size too small to reference all objects")
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidObjectRefSize(self):
+        # object reference size can't be zero
+        try:
+            trailer = PlistTrailer(1, 0, 1, 0, 9)
+            contents = b''.join([b'bplist00', # header
+                                 b'\x08',     # bool false
+                                 b'\x08',     # object at offset 8
+                                 trailerToString(trailer)
+                             ])
+            readPlistFromString(contents)
+            self.fail("Object reference size can't be zero")
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidObjectRefOverflow(self):
+        try:
+            readPlist(data_path('invalid_object_ref_size.plist'))
+            self.fail("Object ref size too small to reference all objects in the object table")
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidOffsestTableOffset(self):
+        # offsetTableOffset too large, extending into trailer
+        try:
+            trailer = PlistTrailer(1, 1, 1, 0, 10)
+            contents = b''.join([b'bplist00', # header
+                                 b'\x08',     # bool false
+                                 b'\x08',     # object at offset 8
+                                 trailerToString(trailer)
+                             ])
+            readPlistFromString(contents)
+            self.fail("Should not read plist when offsetTableOffset is too large")
+        except InvalidPlistException as e:
+            pass
+        
+        # offsetTableOffset too small, extending into header or objects
+        try:
+            trailer = PlistTrailer(1, 1, 1, 0, 8)
+            contents = b''.join([b'bplist00', # header
+                                 b'\x08',     # bool false
+                                 b'\x08',     # object at offset 8
+                                 trailerToString(trailer)
+                             ])
+            readPlistFromString(contents)
+            self.fail("Should not read plist when offsetTableOffset is too small")
+        except InvalidPlistException as e:
+            pass
+    
+    def testInvalidTopObjectNumber(self):
+        # topObjectNumber can't be greater than number of objects
+        try:
+            trailer = PlistTrailer(1, 1, 1, 1, 9)
+            contents = b''.join([b'bplist00', # header
+                                 b'\x08',     # bool false
+                                 b'\x08',     # object at offset 8
+                                 trailerToString(trailer)
+                             ])
+            readPlistFromString(contents)
+            self.fail("Top object number should not be greater than number of objects")
+        except InvalidPlistException as e:
+            pass
+        
+if __name__ == '__main__':
+    unittest.main()

third_party/python/biplist/tests/test_utils.py

@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+
+import os
+import subprocess
+import sys
+
+def data_path(path):
+    return os.path.join(os.path.dirname(globals()["__file__"]), 'data', path)
+
+def fuzz_data_path(path):
+    return os.path.join(os.path.dirname(globals()["__file__"]), 'fuzz_data', path)
+
+def run_command(args, verbose = False):
+    """Runs the command and returns the status and the output."""
+    if verbose:
+        sys.stderr.write("Running: %s\n" % command)
+    p = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+    stdin, stdout = (p.stdin, p.stdout)
+    output = stdout.read()
+    output = output.strip(b'\n')
+    status = stdin.close()
+    stdout.close()
+    p.wait()
+    return p.returncode, output

third_party/python/biplist/tests/test_valid.py

@@ -0,0 +1,125 @@
+# -*- coding: utf-8 -*-
+
+from biplist import *
+import datetime
+import os
+from test_utils import *
+import unittest
+
+try:
+    unicode
+    toUnicode = lambda x: x.decode('unicode-escape')
+except NameError:
+    unicode = str
+    toUnicode = lambda x: x
+
+class TestValidPlistFile(unittest.TestCase):
+    def setUp(self):
+        pass
+    
+    def validateSimpleBinaryRoot(self, root):
+        self.assertTrue(type(root) == dict, "Root should be dictionary.")
+        self.assertTrue(type(root['dateItem']) == datetime.datetime, "date should be datetime")
+        us = root['dateItem'].microsecond
+        if us == 385448:
+            # Python 3 doesn't round microseconds to the nearest value.
+            self.assertEqual(root['dateItem'], datetime.datetime(2010, 8, 19, 22, 27, 30, 385448), "dates not equal" )
+        else:
+            self.assertEqual(root['dateItem'], datetime.datetime(2010, 8, 19, 22, 27, 30, 385449), "dates not equal" )
+        self.assertEqual(root['numberItem'], -10000000000000000, "number not of expected value")
+        self.assertEqual(root['unicodeItem'], toUnicode('abc\u212cdef\u2133'))
+        self.assertEqual(root['stringItem'], 'Hi there')
+        self.assertEqual(root['realItem'], 0.47)
+        self.assertEqual(root['boolItem'], True)
+        self.assertEqual(root['arrayItem'], ['item0'])
+        
+    def testFileRead(self):
+        try:
+            result = readPlist(data_path('simple_binary.plist'))
+            self.validateSimpleBinaryRoot(result)
+        except NotBinaryPlistException as e:
+            self.fail("NotBinaryPlistException: %s" % e)
+        except InvalidPlistException as e:
+            self.fail("InvalidPlistException: %s" % e)
+    
+    def testUnicodeRoot(self):
+        result = readPlist(data_path('unicode_root.plist'))
+        self.assertEqual(result, toUnicode("Mirror's Edge\u2122 for iPad"))
+    
+    def testEmptyUnicodeRoot(self):
+        # Porting note: this test was tricky; it was only passing in
+        # Python 2 because the empty byte-string returned by
+        # readPlist() is considered equal to the empty unicode-string
+        # in the assertion.  Confusingly enough, given the name of the
+        # test, the value in unicode_empty.plist has the format byte
+        # 0b0101 (ASCII string), so the value being asserted against
+        # appears to be what is wrong.
+        result = readPlist(data_path('unicode_empty.plist'))
+        self.assertEqual(result, '')
+    
+    def testBoolOnly(self):
+        result = readPlist(data_path('bool_only_binary.plist'))
+        self.assertEqual(result, False)
+    
+    def testSmallReal(self):
+        result = readPlist(data_path('small_real.plist'))
+        self.assertEqual(result, {'4 byte real':0.5})
+    
+    def testLargeIntegers(self):
+        result = readPlist(data_path('large_int_limits.plist'))
+        self.assertEqual(result['Max 8 Byte Unsigned Integer'], 18446744073709551615)
+        self.assertEqual(result['Min 8 Byte Signed Integer'], -9223372036854775808)
+        self.assertEqual(result['Max 8 Byte Signed Integer'], 9223372036854775807)
+    
+    def testLargeDates(self):
+        result = readPlist(data_path("BFPersistentEventInfo.plist"))
+        self.assertEqual(result['lastShownRatePromptDate'], datetime.datetime(1, 12, 30, 0, 0, 0))
+
+    def testSmallDates(self):
+        result = readPlist(data_path("small_date.plist"))
+        # Date stored in plist is 0000-12-30T00:00:00Z
+        self.assertEqual(result, {'MyDate': datetime.datetime(1, 1, 1, 0, 0)})
+
+    def testKeyedArchiverPlist(self):
+        """
+        Archive is created with class like this:
+        @implementation Archived
+        ...
+        - (void)encodeWithCoder:(NSCoder *)coder {
+            [coder encodeObject:@"object value as string" forKey:@"somekey"];
+        }
+        @end
+        
+        Archived *test = [[Archived alloc] init];
+        NSData *data = [NSKeyedArchiver archivedDataWithRootObject:test]
+        ...
+        """
+        result = readPlist(data_path('nskeyedarchiver_example.plist'))
+        self.assertEqual(result, {
+            '$version': 100000, 
+            '$objects':
+                [
+                	'$null',
+                 	{'$class':Uid(3), 'somekey':Uid(2)}, 
+                 	'object value as string',
+                 	{'$classes':['Archived', 'NSObject'], '$classname':'Archived'}
+                ],
+            '$top': {'root':Uid(1)},
+            '$archiver':'NSKeyedArchiver'
+        })
+        self.assertEqual("Uid(1)", repr(Uid(1)))
+    
+    def testUidComparisons(self):
+        self.assertTrue(Uid(-2) < Uid(-1))
+        self.assertTrue(Uid(-1) < Uid(0))
+        self.assertTrue(Uid(1) > Uid(0))
+        self.assertTrue(Uid(1) > Uid(-2))
+        self.assertTrue(Uid(-1) == Uid(-1))
+        self.assertTrue(Uid(0) == Uid(0))
+        self.assertTrue(Uid(1) == Uid(1))
+        
+        self.assertFalse(1 == Uid(1))
+        self.assertFalse(Uid(0) == 0)
+    
+if __name__ == '__main__':
+    unittest.main()

third_party/python/biplist/tests/test_write.py

@@ -0,0 +1,358 @@
+# -*- coding: utf-8 -*-
+
+import datetime
+import io
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+
+from biplist import *
+from biplist import PlistWriter
+from test_utils import *
+
+try:
+    unicode
+    unicodeStr = lambda x: x.decode('utf-8')
+    toUnicode = lambda x: x.decode('unicode-escape')
+except NameError:
+    unicode = str
+    unicodeStr = lambda x: x
+    toUnicode = lambda x: x
+try:
+    xrange
+except NameError:
+    xrange = range
+
+class TestWritePlist(unittest.TestCase):
+    def setUp(self):
+        pass
+    
+    def roundTrip(self, case, xml=False, expected=None, reprTest=True):
+        # reprTest may fail randomly if True and the values being encoded include a dictionary with more
+        # than one key.
+        
+        # convert to plist string
+        plist = writePlistToString(case, binary=(not xml))
+        self.assertTrue(len(plist) > 0)
+        
+        # confirm that lint is happy with the result
+        self.lintPlist(plist)        
+        
+        # convert back
+        readResult = readPlistFromString(plist)
+        
+        # test equality
+        if reprTest is True:
+            self.assertEqual(repr(case if expected is None else expected), repr(readResult))
+        else:
+            self.assertEqual((case if expected is None else expected), readResult)
+        
+        # write to file
+        plistFile = tempfile.NamedTemporaryFile(mode='wb+', suffix='.plist')
+        writePlist(case, plistFile, binary=(xml is False))
+        plistFile.seek(0)
+        
+        # confirm that lint is happy with the result
+        self.lintPlist(plistFile)
+        
+        # read back from file
+        fileResult = readPlist(plistFile)
+        
+        # test equality
+        if reprTest is True:
+            self.assertEqual(repr(case if expected is None else expected), repr(fileResult))
+        else:
+            self.assertEqual((case if expected is None else expected), fileResult)
+    
+    def lintPlist(self, plist):
+        if os.access('/usr/bin/plutil', os.X_OK):
+            plistFile = None
+            plistFilePath = None
+            
+            if hasattr(plist, 'name'):
+                plistFilePath = plist.name
+            else:
+                if hasattr(plist, 'read'):
+                    plistFile = tempfile.NamedTemporaryFile('w%s' % ('b' if 'b' in plist.mode else ''))
+                    plistFile.write(plist.read())
+                else:
+                    plistFile = tempfile.NamedTemporaryFile('w%s' % ('b' if isinstance(plist, bytes) else ''))
+                    plistFile.write(plist)
+                plistFilePath = plistFile.name
+                plistFile.flush()
+
+            status, output = run_command(['/usr/bin/plutil', '-lint', plistFilePath])
+            if status != 0:
+                self.fail("plutil verification failed (status %d): %s" % (status, output))
+    
+    def testXMLPlist(self):
+        self.roundTrip({'hello':'world'}, xml=True)
+
+    def testXMLPlistWithData(self):
+        for binmode in (True, False):
+            binplist = writePlistToString({'data': Data(b'\x01\xac\xf0\xff')}, binary=binmode)
+            plist = readPlistFromString(binplist)
+            self.assertTrue(isinstance(plist['data'], (Data, bytes)), \
+                "unable to encode then decode Data into %s plist" % ("binary" if binmode else "XML"))
+
+    def testConvertToXMLPlistWithData(self):
+        binplist = writePlistToString({'data': Data(b'\x01\xac\xf0\xff')})
+        plist = readPlistFromString(binplist)
+        xmlplist = writePlistToString(plist, binary=False)
+        self.assertTrue(len(xmlplist) > 0, "unable to convert plist with Data from binary to XML")
+    
+    def testBoolRoot(self):
+        self.roundTrip(True)
+        self.roundTrip(False)
+    
+    def testDuplicate(self):
+        l = ["foo" for i in xrange(0, 100)]
+        self.roundTrip(l)
+        
+    def testListRoot(self):
+        self.roundTrip([1, 2, 3])
+    
+    def testDictRoot(self):
+        self.roundTrip({'a':1, 'B':'d'}, reprTest=False)
+    
+    def mixedNumericTypesHelper(self, cases):
+        result = readPlistFromString(writePlistToString(cases))
+        for i in xrange(0, len(cases)):
+            self.assertTrue(cases[i] == result[i])
+            self.assertEqual(type(cases[i]), type(result[i]), "Type mismatch on %d: %s != %s" % (i, repr(cases[i]), repr(result[i])))
+    
+    def testBoolsAndIntegersMixed(self):
+        self.mixedNumericTypesHelper([0, 1, True, False, None])
+        self.mixedNumericTypesHelper([False, True, 0, 1, None])
+        self.roundTrip({'1':[True, False, 1, 0], '0':[1, 2, 0, {'2':[1, 0, False]}]}, reprTest=False)
+        self.roundTrip([1, 1, 1, 1, 1, True, True, True, True])
+    
+    def testFloatsAndIntegersMixed(self):
+        self.mixedNumericTypesHelper([0, 1, 1.0, 0.0, None])
+        self.mixedNumericTypesHelper([0.0, 1.0, 0, 1, None])
+        self.roundTrip({'1':[1.0, 0.0, 1, 0], '0':[1, 2, 0, {'2':[1, 0, 0.0]}]}, reprTest=False)
+        self.roundTrip([1, 1, 1, 1, 1, 1.0, 1.0, 1.0, 1.0])
+    
+    def testSetRoot(self):
+        self.roundTrip(set((1, 2, 3)))
+    
+    def testDatetime(self):
+        now = datetime.datetime.utcnow()
+        now = now.replace(microsecond=0)
+        self.roundTrip([now])
+    
+    def testFloat(self):
+        self.roundTrip({'aFloat':1.23})
+    
+    def testTuple(self):
+        result = writePlistToString({'aTuple':(1, 2.0, 'a'), 'dupTuple':('a', 'a', 'a', 'b', 'b')})
+        self.assertTrue(len(result) > 0)
+        readResult = readPlistFromString(result)
+        self.assertEqual(readResult['aTuple'], [1, 2.0, 'a'])
+        self.assertEqual(readResult['dupTuple'], ['a', 'a', 'a', 'b', 'b'])
+    
+    def testComplicated(self):
+        root = {'preference':[1, 2, {'hi there':['a', 1, 2, {'yarrrr':123}]}]}
+        self.lintPlist(writePlistToString(root))
+        self.roundTrip(root)
+    
+    def testBytes(self):
+        self.roundTrip(b'0')
+        self.roundTrip(b'')
+        
+        self.roundTrip([b'0'])
+        self.roundTrip([b''])
+        
+        self.roundTrip({'a': b'0'})
+        self.roundTrip({'a': b''})
+    
+    def testString(self):
+        self.roundTrip('')
+        self.roundTrip('a')
+        self.roundTrip('1')
+        
+        self.roundTrip([''])
+        self.roundTrip(['a'])
+        self.roundTrip(['1'])
+        
+        self.roundTrip({'a':''})
+        self.roundTrip({'a':'a'})
+        self.roundTrip({'1':'a'})
+        
+        self.roundTrip({'a':'a'})
+        self.roundTrip({'a':'1'})
+    
+    def testUnicode(self):
+        # defaulting to 1 byte strings
+        if str != unicode:
+            self.roundTrip(unicodeStr(r''), expected='')
+            self.roundTrip(unicodeStr(r'a'), expected='a')
+            
+            self.roundTrip([unicodeStr(r'a')], expected=['a'])
+            
+            self.roundTrip({'a':unicodeStr(r'a')}, expected={'a':'a'})
+            self.roundTrip({unicodeStr(r'a'):'a'}, expected={'a':'a'})
+            self.roundTrip({unicodeStr(r''):unicodeStr(r'')}, expected={'':''})
+        
+        # TODO: need a 4-byte unicode character
+        self.roundTrip(unicodeStr(r'ü'))
+        self.roundTrip([unicodeStr(r'ü')])
+        self.roundTrip({'a':unicodeStr(r'ü')})
+        self.roundTrip({unicodeStr(r'ü'):'a'})
+        
+        self.roundTrip(toUnicode('\u00b6'))
+        self.roundTrip([toUnicode('\u00b6')])
+        self.roundTrip({toUnicode('\u00b6'):toUnicode('\u00b6')})
+        
+        self.roundTrip(toUnicode('\u1D161'))
+        self.roundTrip([toUnicode('\u1D161')])
+        self.roundTrip({toUnicode('\u1D161'):toUnicode('\u1D161')})
+        
+        # Smiley face emoji
+        self.roundTrip(toUnicode('\U0001f604'))
+        self.roundTrip([toUnicode('\U0001f604'), toUnicode('\U0001f604')])
+        self.roundTrip({toUnicode('\U0001f604'):toUnicode('\U0001f604')})
+    
+    def testNone(self):
+        self.roundTrip(None)
+        self.roundTrip({'1':None})
+        self.roundTrip([None, None, None])
+    
+    def testBools(self):
+        self.roundTrip(True)
+        self.roundTrip(False)
+        
+        self.roundTrip([True, False])
+        
+        self.roundTrip({'a':True, 'b':False}, reprTest=False)
+    
+    def testUniques(self):
+        root = {'hi':'there', 'halloo':'there'}
+        self.roundTrip(root, reprTest=False)
+    
+    def testAllEmpties(self):
+        '''Primarily testint that an empty unicode and bytes are not mixed up'''
+        self.roundTrip([unicodeStr(''), '', b'', [], {}], expected=['', '', b'', [], {}])
+    
+    def testLargeDict(self):
+        d = dict((str(x), str(x)) for x in xrange(0, 1000))
+        self.roundTrip(d, reprTest=False)
+        
+    def testWriteToFile(self):
+        for is_binary in [True, False]:
+            with tempfile.NamedTemporaryFile(mode='w%s' % ('b' if is_binary else ''), suffix='.plist') as plistFile:
+                # clear out the created file
+                os.unlink(plistFile.name)
+                self.assertFalse(os.path.exists(plistFile.name))
+                
+                # write to disk
+                writePlist([1, 2, 3], plistFile.name, binary=is_binary)
+                self.assertTrue(os.path.exists(plistFile.name))
+                
+                with open(plistFile.name, 'r%s' % ('b' if is_binary else '')) as f:
+                    fileContents = f.read()
+                    self.lintPlist(fileContents)
+    
+    def testBadKeys(self):
+        try:
+            self.roundTrip({None:1})
+            self.fail("None is not a valid key in Cocoa.")
+        except InvalidPlistException as e:
+            pass
+        try:
+            self.roundTrip({Data(b"hello world"):1})
+            self.fail("Data is not a valid key in Cocoa.")
+        except InvalidPlistException as e:
+            pass
+        try:
+            self.roundTrip({1:1})
+            self.fail("Number is not a valid key in Cocoa.")
+        except InvalidPlistException as e:
+            pass
+    
+    def testIntBoundaries(self):
+        edges = [0xff, 0xffff, 0xffffffff]
+        for edge in edges:
+            cases = [edge, edge-1, edge+1, edge-2, edge+2, edge*2, edge/2]
+            self.roundTrip(cases)
+        edges = [-pow(2, 7), pow(2, 7) - 1, 
+                 -pow(2, 15), pow(2, 15) - 1, 
+                 -pow(2, 31), pow(2, 31) - 1, 
+                 -pow(2, 63), pow(2, 64) - 1]
+        self.roundTrip(edges, reprTest=False)
+        
+        ioBytes = io.BytesIO()
+        writer = PlistWriter(ioBytes)
+        bytes = [(1, [pow(2, 7) - 1]),
+                 (2, [pow(2, 15) - 1]),
+                 (4, [pow(2, 31) - 1]),
+                 (8, [-pow(2, 7), -pow(2, 15), -pow(2, 31), -pow(2, 63), pow(2, 63) - 1]),
+                 (16, [pow(2, 64) - 1])
+            ]
+        for bytelen, tests in bytes:
+            for test in tests:
+                got = writer.intSize(test)
+                self.assertEqual(bytelen, got, "Byte size is wrong. Expected %d, got %d" % (bytelen, got))
+        
+        bytes_lists = [list(x) for x in bytes]
+        self.roundTrip(bytes_lists, reprTest=False)
+        
+        try:
+            self.roundTrip([0x10000000000000000, pow(2, 64)])
+            self.fail("2^64 should be too large for Core Foundation to handle.")
+        except InvalidPlistException as e:
+            pass
+    
+    def testUnicode2(self):
+        unicodeRoot = toUnicode("Mirror's Edge\u2122 for iPad")
+        self.roundTrip(unicodeRoot)
+        unicodeStrings = [toUnicode("Mirror's Edge\u2122 for iPad"), toUnicode('Weightbot \u2014 Track your Weight in Style')]
+        self.roundTrip(unicodeStrings)
+        self.roundTrip({toUnicode(""):toUnicode("")}, expected={'':''})
+        self.roundTrip(toUnicode(""), expected='')
+    
+    def testWriteData(self):
+        self.roundTrip(Data(b"woohoo"))
+
+    def testEmptyData(self):
+        data = Data(b'')
+        binplist = writePlistToString(data)
+        plist = readPlistFromString(binplist)
+        self.assertEqual(plist, data)
+        self.assertEqual(type(plist), type(data))
+        
+    def testUidWrite(self):
+        self.roundTrip({'$version': 100000, 
+            '$objects': 
+                ['$null', 
+                 {'$class': Uid(3), 'somekey': Uid(2)}, 
+                 'object value as string', 
+                 {'$classes': ['Archived', 'NSObject'], '$classname': 'Archived'}
+                 ], 
+            '$top': {'root': Uid(1)}, '$archiver': 'NSKeyedArchiver'}, reprTest=False)
+    
+    def testUidRoundTrip(self):
+        # Per https://github.com/wooster/biplist/issues/9
+        self.roundTrip(Uid(1))
+        self.roundTrip([Uid(1), 1])
+        self.roundTrip([1, Uid(1)])
+        self.roundTrip([Uid(1), Uid(1)])
+    
+    def testRecursiveWrite(self):
+        # Apple libraries disallow recursive containers, so we should fail on
+        # trying to write those.
+        root = []
+        child = [root]
+        root.extend(child)
+        try:
+            writePlistToString(root)
+            self.fail("Should not be able to write plists with recursive containers.")
+        except InvalidPlistException as e:
+            pass
+        except:
+            self.fail("Should get an invalid plist exception for recursive containers.")
+
+if __name__ == '__main__':
+    unittest.main()

third_party/python/requirements.in

@@ -1,4 +1,5 @@
 attrs==18.1.0
+biplist==1.0.3
 blessings==1.7
 jsmin==2.1.0
 json-e==2.7.0

third_party/python/requirements.txt

@@ -5,6 +5,8 @@ atomicwrites==1.1.5 \
 attrs==18.1.0 \
     --hash=sha256:4b90b09eeeb9b88c35bc642cbac057e45a5fd85367b985bd2809c62b7b939265 \
     --hash=sha256:e0d0eb91441a3b53dab4d9b743eafc1ac44476296a2053b6ca3af0b139faf87b
+biplist==1.0.3 \
+    --hash=sha256:4c0549764c5fe50b28042ec21aa2e14fe1a2224e239a1dae77d9e7f3932aa4c6
 blessings==1.7 \
     --hash=sha256:98e5854d805f50a5b58ac2333411b0482516a8210f23f43308baeb58d77c157d \
     --hash=sha256:b1fdd7e7a675295630f9ae71527a8ebc10bfefa236b3d6aa4932ee4462c17ba3 \