Bug 830996 - implement a way to DRY mozbase packages for m-c;r=gps

build/virtualenv/packages.txt

@@ -1,15 +1,4 @@
 simplejson.pth:python/simplejson-2.1.1
-manifestdestiny.pth:testing/mozbase/manifestdestiny
-mozcrash.pth:testing/mozbase/mozcrash
-mozdevice.pth:testing/mozbase/mozdevice
-mozfile.pth:testing/mozbase/mozfile
-mozhttpd.pth:testing/mozbase/mozhttpd
-mozinfo.pth:testing/mozbase/mozinfo
-mozinstall.pth:testing/mozbase/mozinstall
-mozlog.pth:testing/mozbase/mozlog
-mozprocess.pth:testing/mozbase/mozprocess
-mozprofile.pth:testing/mozbase/mozprofile
-mozrunner.pth:testing/mozbase/mozrunner
 marionette.pth:testing/marionette/client
 blessings.pth:python/blessings
 mach.pth:python/mach
@@ -23,3 +12,4 @@ mozilla.pth:build
 mozilla.pth:config
 mozilla.pth:xpcom/typelib/xpt/tools
 copy:build/buildconfig.py
+packages.txt:testing/mozbase/packages.txt

build/virtualenv/populate_virtualenv.py

@@ -22,7 +22,7 @@ MINIMUM_PYTHON_MINOR = 7
 class VirtualenvManager(object):
     """Contains logic for managing virtualenvs for building the tree."""
 
-    def __init__(self, topsrcdir, virtualenv_path, log_handle):
+    def __init__(self, topsrcdir, virtualenv_path, log_handle, manifest_path):
         """Create a new manager.
 
         Each manager is associated with a source directory, a path where you
@@ -31,6 +31,7 @@ class VirtualenvManager(object):
         self.topsrcdir = topsrcdir
         self.virtualenv_root = virtualenv_path
         self.log_handle = log_handle
+        self.manifest_path = manifest_path
 
     @property
     def virtualenv_script_path(self):
@@ -38,11 +39,6 @@ class VirtualenvManager(object):
         return os.path.join(self.topsrcdir, 'python', 'virtualenv',
             'virtualenv.py')
 
-    @property
-    def manifest_path(self):
-        return os.path.join(self.topsrcdir, 'build', 'virtualenv',
-            'packages.txt')
-
     @property
     def python_path(self):
         if sys.platform in ('win32', 'cygwin'):
@@ -58,6 +54,36 @@ class VirtualenvManager(object):
 
         return os.path.join(self.virtualenv_root, 'bin', 'activate_this.py')
 
+    def up_to_date(self):
+        """Returns whether the virtualenv is present and up to date."""
+
+        deps = [self.manifest_path, __file__]
+
+        # check if virtualenv exists
+        if not os.path.exists(self.virtualenv_root) or \
+            not os.path.exists(self.activate_path):
+
+            return False
+
+        # check modification times
+        activate_mtime = os.path.getmtime(self.activate_path)
+        dep_mtime = max(os.path.getmtime(p) for p in deps)
+        if dep_mtime > activate_mtime:
+            return False
+
+        # recursively check sub packages.txt files
+        submanifests = [i for i in self.packages()
+                        if i[0] == 'packages.txt']
+        for submanifest in submanifests:
+            submanager = VirtualenvManager(self.topsrcdir,
+                                           self.virtualenv_root,
+                                           self.log_handle,
+                                           src)
+            if not submanager.up_to_date():
+                return False
+
+        return True
+
     def ensure(self):
         """Ensure the virtualenv is present and up to date.
 
@@ -67,20 +93,9 @@ class VirtualenvManager(object):
         This should be the main API used from this class as it is the
         highest-level.
         """
-        deps = [self.manifest_path, __file__]
-
-        if not os.path.exists(self.virtualenv_root) or \
-            not os.path.exists(self.activate_path):
-
-            return self.build()
-
-        activate_mtime = os.path.getmtime(self.activate_path)
-        dep_mtime = max(os.path.getmtime(p) for p in deps)
-
-        if dep_mtime > activate_mtime:
-            return self.build()
-
-        return self.virtualenv_root
+        if self.up_to_date():
+            return self.virtualenv_root
+        return self.build()
 
     def create(self):
         """Create a new, empty virtualenv.
@@ -90,10 +105,7 @@ class VirtualenvManager(object):
         write output to.
         """
         env = dict(os.environ)
-        try:
-            del env['PYTHONDONTWRITEBYTECODE']
-        except KeyError:
-            pass
+        env.pop('PYTHONDONTWRITEBYTECODE', None)
 
         args = [sys.executable, self.virtualenv_script_path,
             '--system-site-packages', self.virtualenv_root]
@@ -101,11 +113,17 @@ class VirtualenvManager(object):
         result = subprocess.call(args, stdout=self.log_handle,
             stderr=subprocess.STDOUT, env=env)
 
-        if result != 0:
+        if result:
             raise Exception('Error creating virtualenv.')
 
         return self.virtualenv_root
 
+    def packages(self):
+        with file(self.manifest_path, 'rU') as fh:
+            packages = [line.rstrip().split(':')
+                        for line in fh]
+        return packages
+
     def populate(self):
         """Populate the virtualenv.
 
@@ -136,11 +154,8 @@ class VirtualenvManager(object):
         environment is not configured properly, packages could be installed
         into the wrong place. This is how virtualenv's work.
         """
-        packages = []
-        fh = open(self.manifest_path, 'rU')
-        for line in fh:
-            packages.append(line.rstrip().split(':'))
-        fh.close()
+
+        packages = self.packages()
 
         def handle_package(package):
             python_lib = distutils.sysconfig.get_python_lib()
@@ -162,6 +177,19 @@ class VirtualenvManager(object):
 
                 return True
 
+            if package[0] == 'packages.txt':
+                assert len(package) == 2
+
+                src = os.path.join(self.topsrcdir, package[1])
+                assert os.path.isfile(src), "'%s' does not exist" % src
+                submanager = VirtualenvManager(self.topsrcdir,
+                                               self.virtualenv_root,
+                                               self.log_handle,
+                                               src)
+                submanager.populate()
+
+                return True
+
             if package[0].endswith('.pth'):
                 assert len(package) == 2
 
@@ -221,17 +249,12 @@ class VirtualenvManager(object):
             for package in packages:
                 handle_package(package)
         finally:
-            try:
-                del os.environ['MACOSX_DEPLOYMENT_TARGET']
-            except KeyError:
-                pass
+            os.environ.pop('MACOSX_DEPLOYMENT_TARGET', None)
 
             if old_target is not None:
                 os.environ['MACOSX_DEPLOYMENT_TARGET'] = old_target
 
-            for k in old_env_variables:
-                os.environ[k] = old_env_variables[k]
-
+            os.environ.update(old_env_variables)
 
     def call_setup(self, directory, arguments):
         """Calls setup.py in a directory."""
@@ -320,7 +343,10 @@ if __name__ == '__main__':
         topsrcdir = sys.argv[2]
         virtualenv_path = sys.argv[3]
 
-    manager = VirtualenvManager(topsrcdir, virtualenv_path, sys.stdout)
+    # path to default packages.txt
+    manifest_path = os.path.join(topsrcdir, 'build', 'virtualenv', 'packages.txt')
+
+    manager = VirtualenvManager(topsrcdir, virtualenv_path, sys.stdout, manifest_path)
 
     if populate:
         manager.populate()