Bug 1237706 - Extend download_unzip() and move to testbase.py. r=jlund

testing/mozharness/mozharness/base/script.py

@@ -45,6 +45,7 @@ except ImportError:
 
 from mozprocess import ProcessHandler
 from mozharness.base.config import BaseConfig
+from mozharness.base.errors import ZipErrorList
 from mozharness.base.log import SimpleFileLogger, MultiFileLogger, \
     LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL
 
@@ -450,6 +451,40 @@ class ScriptMixin(PlatformMixin):
             **retry_args
         )
 
+    def download_unzip(self, url, parent_dir, target_unzip_dirs=None, halt_on_failure=True):
+        """Generic method to download and extract a zip file.
+
+        The downloaded file will always be saved to the working directory and is not getting
+        deleted after extracting.
+
+        Args:
+            url (str): URL where the file to be downloaded is located.
+            parent_dir (str): directory where the downloaded file will
+                              be extracted to.
+            target_unzip_dirs (list, optional): directories inside the zip file to extract.
+                                                Defaults to `None`.
+            halt_on_failure (bool, optional): whether or not to redefine the
+                                              log level as `FATAL` on errors. Defaults to True.
+
+        """
+        dirs = self.query_abs_dirs()
+        zipfile = self.download_file(url, parent_dir=dirs['abs_work_dir'],
+                                     error_level=FATAL)
+
+        command = self.query_exe('unzip', return_type='list')
+        # Always overwrite to not get an input in a hidden pipe if files already exist
+        command.extend(['-q', '-o', zipfile, '-d', parent_dir])
+        if target_unzip_dirs:
+            command.extend(target_unzip_dirs)
+        # TODO error_list: http://www.info-zip.org/mans/unzip.html#DIAGNOSTICS
+        # unzip return code 11 is 'no matching files were found'
+        self.run_command(command,
+                         error_list=ZipErrorList,
+                         halt_on_failure=halt_on_failure,
+                         fatal_exit_code=3,
+                         success_codes=[0, 11],
+                         )
+
     def load_json_url(self, url, error_level=None, *args, **kwargs):
         """ Returns a json object from a url (it retries). """
         contents = self._retry_download(

testing/mozharness/mozharness/mozilla/testing/talos.py

@@ -274,7 +274,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
         if self.query_pagesets_url():
             self.info("Downloading pageset...")
             src_talos_pageset = os.path.join(src_talos_webdir, 'tests')
-            self._download_unzip(self.pagesets_url, src_talos_pageset)
+            self.download_unzip(self.pagesets_url, src_talos_pageset)
 
     # Action methods. {{{1
     # clobber defined in BaseScript

testing/mozharness/mozharness/mozilla/testing/testbase.py

@@ -112,7 +112,6 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin,
     binary_path = None
     test_url = None
     test_packages_url = None
-    test_zip_path = None
     symbols_url = None
     symbols_path = None
     jsshell_url = None
@@ -380,18 +379,6 @@ You can set this by:
                         "You are currently using version %s. Please update to at least 6.0.\n"
                         "You can visit http://www.info-zip.org/UnZip.html" % version)
 
-    def _download_test_zip(self):
-        dirs = self.query_abs_dirs()
-        file_name = None
-        if self.test_zip_path:
-            file_name = self.test_zip_path
-        # try to use our proxxy servers
-        # create a proxxy object and get the binaries from it
-        source = self.download_file(self.test_url, file_name=file_name,
-                                    parent_dir=dirs['abs_work_dir'],
-                                    error_level=FATAL)
-        self.test_zip_path = os.path.realpath(source)
-
     def _read_packages_manifest(self):
         dirs = self.query_abs_dirs()
         source = self.download_file(self.test_packages_url,
@@ -449,41 +436,15 @@ You can set this by:
                     unzip_dirs = None
                     target_dir = dirs['abs_test_bin_dir']
                 url = self.query_build_dir_url(file_name)
-                self._download_unzip(url, target_dir,
+                self.download_unzip(url, target_dir,
                                      target_unzip_dirs=unzip_dirs)
 
-    def _download_unzip(self, url, parent_dir, target_unzip_dirs=None):
+    def _download_test_zip(self, target_unzip_dirs=None):
-        """Generic download+unzip.
-        This is hardcoded to halt on failure.
-        We should probably change some other methods to call this."""
         dirs = self.query_abs_dirs()
-        zipfile = self.download_file(url, parent_dir=dirs['abs_work_dir'],
-                                             error_level=FATAL)
-        command = self.query_exe('unzip', return_type='list')
-        command.extend(['-q', '-o', zipfile])
-        if target_unzip_dirs:
-            command.extend(target_unzip_dirs)
-        self.run_command(command, cwd=parent_dir, halt_on_failure=True,
-                         success_codes=[0, 11],
-                         fatal_exit_code=3, output_timeout=1760)
-
-    def _extract_test_zip(self, target_unzip_dirs=None):
-        dirs = self.query_abs_dirs()
-        unzip = self.query_exe("unzip")
         test_install_dir = dirs.get('abs_test_install_dir',
                                     os.path.join(dirs['abs_work_dir'], 'tests'))
-        self.mkdir_p(test_install_dir)
+        self.download_unzip(self.test_url, test_install_dir,
-        # adding overwrite flag otherwise subprocess.Popen hangs on waiting for
+                             target_unzip_dirs=target_unzip_dirs)
-        # input in a hidden pipe whenever this action is run twice without
-        # clobber
-        unzip_cmd = [unzip, '-q', '-o', self.test_zip_path]
-        if target_unzip_dirs:
-            unzip_cmd.extend(target_unzip_dirs)
-        # TODO error_list
-        # unzip return code 11 is 'no matching files were found'
-        self.run_command(unzip_cmd, cwd=test_install_dir,
-                         halt_on_failure=True, success_codes=[0, 11],
-                         fatal_exit_code=3)
 
     def structured_output(self, suite_category):
         """Defines whether structured logging is in use in this configuration. This
@@ -527,14 +488,10 @@ You can set this by:
             return
         if not self.symbols_path:
             self.symbols_path = os.path.join(dirs['abs_work_dir'], 'symbols')
-        self.mkdir_p(self.symbols_path)
+
-        source = self.download_file(self.symbols_url,
-                                            parent_dir=self.symbols_path,
-                                            error_level=FATAL)
         self.set_buildbot_property("symbols_url", self.symbols_url,
                                    write_to_file=True)
-        self.run_command(['unzip', '-q', source], cwd=self.symbols_path,
+        self.download_unzip(self.symbols_url, self.symbols_path)
-                         halt_on_failure=True, fatal_exit_code=3)
 
     def download_and_extract(self, target_unzip_dirs=None, suite_categories=None):
         """
@@ -559,8 +516,7 @@ You can set this by:
                            ' package data at "%s" will be ignored.' %
                            (self.config.get('test_url'), self.test_packages_url))
 
-            self._download_test_zip()
+            self._download_test_zip(target_unzip_dirs)
-            self._extract_test_zip(target_unzip_dirs=target_unzip_dirs)
         else:
             if not self.test_packages_url:
                 # The caller intends to download harness specific packages, but doesn't know

testing/mozharness/mozharness/mozilla/testing/unittest.py

@@ -221,7 +221,7 @@ class EmulatorMixin(object):
         dirs = self.query_abs_dirs()
         self.mkdir_p(dirs['abs_emulator_dir'])
         if self.config.get('emulator_url'):
-            self._download_unzip(self.config['emulator_url'], dirs['abs_emulator_dir'])
+            self.download_unzip(self.config['emulator_url'], dirs['abs_emulator_dir'])
         elif self.config.get('emulator_manifest'):
             manifest_path = self.create_tooltool_manifest(self.config['emulator_manifest'])
             do_unzip = True

testing/mozharness/scripts/android_emulator_unittest.py

@@ -594,7 +594,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
 
     def verify_emulator(self):
         '''
-        Check to see if the emulator can be contacted via adb, telnet, and sut, if configured. 
+        Check to see if the emulator can be contacted via adb, telnet, and sut, if configured.
         If any communication attempt fails, kill the emulator, re-launch, and re-check.
         '''
         self.mkdir_p(self.query_abs_dirs()['abs_blob_upload_dir'])
@@ -627,8 +627,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
             robocop_url = self.installer_url[:self.installer_url.rfind('/')] + '/robocop.apk'
             self.info("Downloading robocop...")
             self.download_file(robocop_url, 'robocop.apk', dirs['abs_work_dir'], error_level=FATAL)
-        self.mkdir_p(dirs['abs_xre_dir'])
+        self.download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
-        self._download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
 
     def install(self):
         """

testing/mozharness/scripts/android_panda.py

@@ -382,9 +382,8 @@ class PandaTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixin, Bu
         c = self.config
         dirs = self.query_abs_dirs()
         self.host_utils_url = c['hostutils_url']
-        #create the hostutils dir, get the zip and extract it
+        # get the zip and extract it
-        self.mkdir_p(dirs['abs_hostutils_dir'])
+        self.download_unzip(self.host_utils_url, dirs['abs_hostutils_dir'])
-        self._download_unzip(self.host_utils_url, dirs['abs_hostutils_dir'])
 
     def _install_app(self):
         c = self.config

testing/mozharness/scripts/android_panda_talos.py

@@ -299,8 +299,7 @@ class PandaTalosTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixi
         self.rmtree(dirs['abs_talosdata_dir'])
         self.mkdir_p(dirs['abs_talosdata_dir'])
         self.mkdir_p(dirs['abs_symbols_dir'])
-        self.mkdir_p(dirs['abs_fennec_dir'])
+        self.download_unzip(self.installer_url,
-        self._download_unzip(self.installer_url,
                              dirs['abs_fennec_dir'])
         #this is ugly but you can't specify a file in download_unzip to extract the file to, by default it's the abs_work_dir
         #should think of a better way
@@ -318,10 +317,10 @@ class PandaTalosTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixi
                            error_level=FATAL)
         self.symbols_url = self.query_symbols_url()
 
-        self._download_unzip(self.symbols_url,
+        self.download_unzip(self.symbols_url,
                              dirs['abs_symbols_dir'])
 
-        self._download_unzip(self.config['retry_url'],
+        self.download_unzip(self.config['retry_url'],
                              dirs['abs_talosdata_dir'])
 
         taloscode = self.config.get("talos_from_code_url")

testing/mozharness/scripts/androidx86_emulator_unittest.py

@@ -681,8 +681,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
                 self._download_robocop_apk()
                 break
 
-        self.mkdir_p(dirs['abs_xre_dir'])
+        self.download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
-        self._download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
 
     def install(self):
         assert self.installer_path is not None, \

testing/mozharness/scripts/b2g_emulator_unittest.py

@@ -208,8 +208,7 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
                          error_list=TarErrorList,
                          halt_on_failure=True, fatal_exit_code=3)
 
-        self.mkdir_p(dirs['abs_xre_dir'])
+        self.download_unzip(self.config['xre_url'],
-        self._download_unzip(self.config['xre_url'],
                              dirs['abs_xre_dir'])
 
         if self.config.get('busybox_url'):