Bug 1437593 - Vendor virtualenv-clone 0.3.0; r=ted

MozReview-Commit-ID: JkPPc9xcGyU

--HG--
extra : rebase_source : ed49b3dc695aaa232c53eb75485fcda7b3bf3a60

third_party/python/virtualenv-clone/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2011, Edward George, based on code contained within the
+virtualenv project.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

third_party/python/virtualenv-clone/MANIFEST.in

@@ -0,0 +1,2 @@
+include README
+include LICENSE

third_party/python/virtualenv-clone/PKG-INFO

@@ -0,0 +1,21 @@
+Metadata-Version: 1.1
+Name: virtualenv-clone
+Version: 0.3.0
+Summary: script to clone virtualenvs.
+Home-page: http://github.com/edwardgeorge/virtualenv-clone
+Author: Edward George
+Author-email: edwardgeorge@gmail.com
+License: MIT
+Description-Content-Type: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python
+Classifier: Intended Audience :: Developers
+Classifier: Development Status :: 3 - Alpha
+Classifier: Programming Language :: Python :: 2.6
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.3
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6

third_party/python/virtualenv-clone/README

@@ -0,0 +1,33 @@
+virtualenv cloning script.
+
+A script for cloning a non-relocatable virtualenv.
+
+Virtualenv provides a way to make virtualenv's relocatable which could then be
+copied as we wanted. However making a virtualenv relocatable this way breaks
+the no-site-packages isolation of the virtualenv as well as other aspects that
+come with relative paths and '/usr/bin/env' shebangs that may be undesirable.
+
+Also, the .pth and .egg-link rewriting doesn't seem to work as intended. This
+attempts to overcome these issues and provide a way to easily clone an
+existing virtualenv.
+
+It performs the following:
+
+- copies sys.argv[1] dir to sys.argv[2]
+- updates the hardcoded VIRTUAL_ENV variable in the activate script to the
+  new repo location. (--relocatable doesn't touch this)
+- updates the shebangs of the various scripts in bin to the new python if
+  they pointed to the old python. (version numbering is retained.)
+
+    it can also change '/usr/bin/env python' shebangs to be absolute too,
+    though this functionality is not exposed at present.
+
+- checks sys.path of the cloned virtualenv and if any of the paths are from
+  the old environment it finds any .pth or .egg-link files within sys.path
+  located in the new environment and makes sure any absolute paths to the
+  old environment are updated to the new environment.
+
+- finally it double checks sys.path again and will fail if there are still
+  paths from the old environment present.
+
+NOTE: This script requires Python >= 2.5

third_party/python/virtualenv-clone/clonevirtualenv.py

@@ -0,0 +1,323 @@
+#!/usr/bin/env python
+from __future__ import with_statement
+
+import logging
+import optparse
+import os
+import os.path
+import re
+import shutil
+import subprocess
+import sys
+import itertools
+
+version_info = (0, 2, 6)
+__version__ = '.'.join(map(str, version_info))
+
+
+logger = logging.getLogger()
+
+
+if sys.version_info < (2, 6):
+    next = lambda gen: gen.next()
+
+
+env_bin_dir = 'bin'
+if sys.platform == 'win32':
+    env_bin_dir = 'Scripts'
+
+
+class UserError(Exception):
+    pass
+
+
+def _dirmatch(path, matchwith):
+    """Check if path is within matchwith's tree.
+
+    >>> _dirmatch('/home/foo/bar', '/home/foo/bar')
+    True
+    >>> _dirmatch('/home/foo/bar/', '/home/foo/bar')
+    True
+    >>> _dirmatch('/home/foo/bar/etc', '/home/foo/bar')
+    True
+    >>> _dirmatch('/home/foo/bar2', '/home/foo/bar')
+    False
+    >>> _dirmatch('/home/foo/bar2/etc', '/home/foo/bar')
+    False
+    """
+    matchlen = len(matchwith)
+    if (path.startswith(matchwith)
+        and path[matchlen:matchlen + 1] in [os.sep, '']):
+        return True
+    return False
+
+
+def _virtualenv_sys(venv_path):
+    "obtain version and path info from a virtualenv."
+    executable = os.path.join(venv_path, env_bin_dir, 'python')
+    # Must use "executable" as the first argument rather than as the
+    # keyword argument "executable" to get correct value from sys.path
+    p = subprocess.Popen([executable,
+        '-c', 'import sys;'
+              'print (sys.version[:3]);'
+              'print ("\\n".join(sys.path));'],
+        env={},
+        stdout=subprocess.PIPE)
+    stdout, err = p.communicate()
+    assert not p.returncode and stdout
+    lines = stdout.decode('utf-8').splitlines()
+    return lines[0], filter(bool, lines[1:])
+
+
+def clone_virtualenv(src_dir, dst_dir):
+    if not os.path.exists(src_dir):
+        raise UserError('src dir %r does not exist' % src_dir)
+    if os.path.exists(dst_dir):
+        raise UserError('dest dir %r exists' % dst_dir)
+    #sys_path = _virtualenv_syspath(src_dir)
+    logger.info('cloning virtualenv \'%s\' => \'%s\'...' %
+            (src_dir, dst_dir))
+    shutil.copytree(src_dir, dst_dir, symlinks=True,
+            ignore=shutil.ignore_patterns('*.pyc'))
+    version, sys_path = _virtualenv_sys(dst_dir)
+    logger.info('fixing scripts in bin...')
+    fixup_scripts(src_dir, dst_dir, version)
+
+    has_old = lambda s: any(i for i in s if _dirmatch(i, src_dir))
+
+    if has_old(sys_path):
+        # only need to fix stuff in sys.path if we have old
+        # paths in the sys.path of new python env. right?
+        logger.info('fixing paths in sys.path...')
+        fixup_syspath_items(sys_path, src_dir, dst_dir)
+    v_sys = _virtualenv_sys(dst_dir)
+    remaining = has_old(v_sys[1])
+    assert not remaining, v_sys
+    fix_symlink_if_necessary(src_dir, dst_dir)
+
+def fix_symlink_if_necessary(src_dir, dst_dir):
+    #sometimes the source virtual environment has symlinks that point to itself
+    #one example is $OLD_VIRTUAL_ENV/local/lib points to $OLD_VIRTUAL_ENV/lib
+    #this function makes sure
+    #$NEW_VIRTUAL_ENV/local/lib will point to $NEW_VIRTUAL_ENV/lib
+    #usually this goes unnoticed unless one tries to upgrade a package though pip, so this bug is hard to find.
+    logger.info("scanning for internal symlinks that point to the original virtual env")
+    for dirpath, dirnames, filenames in os.walk(dst_dir):
+        for a_file in itertools.chain(filenames, dirnames):
+            full_file_path = os.path.join(dirpath, a_file)
+            if os.path.islink(full_file_path):
+                target = os.path.realpath(full_file_path)
+                if target.startswith(src_dir):
+                    new_target = target.replace(src_dir, dst_dir)
+                    logger.debug('fixing symlink in %s' % (full_file_path,))
+                    os.remove(full_file_path)
+                    os.symlink(new_target, full_file_path)
+
+
+def fixup_scripts(old_dir, new_dir, version, rewrite_env_python=False):
+    bin_dir = os.path.join(new_dir, env_bin_dir)
+    root, dirs, files = next(os.walk(bin_dir))
+    pybinre = re.compile(r'pythonw?([0-9]+(\.[0-9]+(\.[0-9]+)?)?)?$')
+    for file_ in files:
+        filename = os.path.join(root, file_)
+        if file_ in ['python', 'python%s' % version, 'activate_this.py']:
+            continue
+        elif file_.startswith('python') and pybinre.match(file_):
+            # ignore other possible python binaries
+            continue
+        elif file_.endswith('.pyc'):
+            # ignore compiled files
+            continue
+        elif file_ == 'activate' or file_.startswith('activate.'):
+            fixup_activate(os.path.join(root, file_), old_dir, new_dir)
+        elif os.path.islink(filename):
+            fixup_link(filename, old_dir, new_dir)
+        elif os.path.isfile(filename):
+            fixup_script_(root, file_, old_dir, new_dir, version,
+                rewrite_env_python=rewrite_env_python)
+
+
+def fixup_script_(root, file_, old_dir, new_dir, version,
+                  rewrite_env_python=False):
+    old_shebang = '#!%s/bin/python' % os.path.normcase(os.path.abspath(old_dir))
+    new_shebang = '#!%s/bin/python' % os.path.normcase(os.path.abspath(new_dir))
+    env_shebang = '#!/usr/bin/env python'
+
+    filename = os.path.join(root, file_)
+    with open(filename, 'rb') as f:
+        if f.read(2) != b'#!':
+            # no shebang
+            return
+        f.seek(0)
+        lines = f.readlines()
+
+    if not lines:
+        # warn: empty script
+        return
+
+    def rewrite_shebang(version=None):
+        logger.debug('fixing %s' % filename)
+        shebang = new_shebang
+        if version:
+            shebang = shebang + version
+        shebang = (shebang + '\n').encode('utf-8')
+        with open(filename, 'wb') as f:
+            f.write(shebang)
+            f.writelines(lines[1:])
+
+    try:
+        bang = lines[0].decode('utf-8').strip()
+    except UnicodeDecodeError:
+        # binary file
+        return
+
+    # This takes care of the scheme in which shebang is of type
+    # '#!/venv/bin/python3' while the version of system python
+    # is of type 3.x e.g. 3.5.
+    short_version = bang[len(old_shebang):]
+
+    if not bang.startswith('#!'):
+        return
+    elif bang == old_shebang:
+        rewrite_shebang()
+    elif (bang.startswith(old_shebang)
+          and bang[len(old_shebang):] == version):
+        rewrite_shebang(version)
+    elif (bang.startswith(old_shebang)
+          and short_version
+          and bang[len(old_shebang):] == short_version):
+        rewrite_shebang(short_version)
+    elif rewrite_env_python and bang.startswith(env_shebang):
+        if bang == env_shebang:
+            rewrite_shebang()
+        elif bang[len(env_shebang):] == version:
+            rewrite_shebang(version)
+    else:
+        # can't do anything
+        return
+
+
+def fixup_activate(filename, old_dir, new_dir):
+    logger.debug('fixing %s' % filename)
+    with open(filename, 'rb') as f:
+        data = f.read().decode('utf-8')
+
+    data = data.replace(old_dir, new_dir)
+    with open(filename, 'wb') as f:
+        f.write(data.encode('utf-8'))
+
+
+def fixup_link(filename, old_dir, new_dir, target=None):
+    logger.debug('fixing %s' % filename)
+    if target is None:
+        target = os.readlink(filename)
+
+    origdir = os.path.dirname(os.path.abspath(filename)).replace(
+        new_dir, old_dir)
+    if not os.path.isabs(target):
+        target = os.path.abspath(os.path.join(origdir, target))
+        rellink = True
+    else:
+        rellink = False
+
+    if _dirmatch(target, old_dir):
+        if rellink:
+            # keep relative links, but don't keep original in case it
+            # traversed up out of, then back into the venv.
+            # so, recreate a relative link from absolute.
+            target = target[len(origdir):].lstrip(os.sep)
+        else:
+            target = target.replace(old_dir, new_dir, 1)
+
+    # else: links outside the venv, replaced with absolute path to target.
+    _replace_symlink(filename, target)
+
+
+def _replace_symlink(filename, newtarget):
+    tmpfn = "%s.new" % filename
+    os.symlink(newtarget, tmpfn)
+    os.rename(tmpfn, filename)
+
+
+def fixup_syspath_items(syspath, old_dir, new_dir):
+    for path in syspath:
+        if not os.path.isdir(path):
+            continue
+        path = os.path.normcase(os.path.abspath(path))
+        if _dirmatch(path, old_dir):
+            path = path.replace(old_dir, new_dir, 1)
+            if not os.path.exists(path):
+                continue
+        elif not _dirmatch(path, new_dir):
+            continue
+        root, dirs, files = next(os.walk(path))
+        for file_ in files:
+            filename = os.path.join(root, file_)
+            if filename.endswith('.pth'):
+                fixup_pth_file(filename, old_dir, new_dir)
+            elif filename.endswith('.egg-link'):
+                fixup_egglink_file(filename, old_dir, new_dir)
+
+
+def fixup_pth_file(filename, old_dir, new_dir):
+    logger.debug('fixup_pth_file %s' % filename)
+
+    with open(filename, 'r') as f:
+        lines = f.readlines()
+
+    has_change = False
+
+    for num, line in enumerate(lines):
+        line = (line.decode('utf-8') if hasattr(line, 'decode') else line).strip()
+
+        if not line or line.startswith('#') or line.startswith('import '):
+            continue
+        elif _dirmatch(line, old_dir):
+            lines[num] = line.replace(old_dir, new_dir, 1)
+            has_change = True
+
+    if has_change:
+        with open(filename, 'w') as f:
+            payload = os.linesep.join([l.strip() for l in lines]) + os.linesep
+            f.write(payload)
+
+
+def fixup_egglink_file(filename, old_dir, new_dir):
+    logger.debug('fixing %s' % filename)
+    with open(filename, 'rb') as f:
+        link = f.read().decode('utf-8').strip()
+    if _dirmatch(link, old_dir):
+        link = link.replace(old_dir, new_dir, 1)
+        with open(filename, 'wb') as f:
+            link = (link + '\n').encode('utf-8')
+            f.write(link)
+
+
+def main():
+    parser = optparse.OptionParser("usage: %prog [options]"
+        " /path/to/existing/venv /path/to/cloned/venv")
+    parser.add_option('-v',
+            action="count",
+            dest='verbose',
+            default=False,
+            help='verbosity')
+    options, args = parser.parse_args()
+    try:
+        old_dir, new_dir = args
+    except ValueError:
+        print("virtualenv-clone %s" % (__version__,))
+        parser.error("not enough arguments given.")
+    old_dir = os.path.realpath(old_dir)
+    new_dir = os.path.realpath(new_dir)
+    loglevel = (logging.WARNING, logging.INFO, logging.DEBUG)[min(2,
+            options.verbose)]
+    logging.basicConfig(level=loglevel, format='%(message)s')
+    try:
+        clone_virtualenv(old_dir, new_dir)
+    except UserError:
+        e = sys.exc_info()[1]
+        parser.error(str(e))
+
+
+if __name__ == '__main__':
+    main()

third_party/python/virtualenv-clone/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

third_party/python/virtualenv-clone/setup.py

@@ -0,0 +1,53 @@
+import sys
+from setuptools.command.test import test as TestCommand
+from setuptools import setup
+
+
+if __name__ == '__main__' and sys.version_info < (2, 5):
+    raise SystemExit("Python >= 2.5 required for virtualenv-clone")
+
+test_requirements = [
+    'virtualenv',
+    'tox',
+    'pytest'
+]
+
+
+class ToxTest(TestCommand):
+    def finalize_options(self):
+        TestCommand.finalize_options(self)
+        self.test_args = []
+        self.test_suite = True
+
+    def run_tests(self):
+        import tox
+        tox.cmdline()
+
+
+setup(name="virtualenv-clone",
+    version='0.3.0',
+    description='script to clone virtualenvs.',
+    author='Edward George',
+    author_email='edwardgeorge@gmail.com',
+    url='http://github.com/edwardgeorge/virtualenv-clone',
+    license="MIT",
+    py_modules=["clonevirtualenv"],
+    entry_points={
+        'console_scripts': [
+            'virtualenv-clone=clonevirtualenv:main',
+    ]},
+    classifiers=[
+        "License :: OSI Approved :: MIT License",
+        "Programming Language :: Python",
+        "Intended Audience :: Developers",
+        "Development Status :: 3 - Alpha",
+        "Programming Language :: Python :: 2.6",
+        "Programming Language :: Python :: 2.7",
+        "Programming Language :: Python :: 3.3",
+        "Programming Language :: Python :: 3.4",
+        "Programming Language :: Python :: 3.5",
+        "Programming Language :: Python :: 3.6",
+    ],
+    tests_require=test_requirements,
+    cmdclass={'test': ToxTest}
+)