Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-15 06:15:43 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1237706 - Extend download_unzip() and move to testbase.py. r=jlund

Browse Source
This commit is contained in:
Henrik Skupin 2016-01-12 22:42:31 +01:00
parent ad971d5294
commit eb8b1ccb76
9 changed files with 53 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
testing/mozharness/mozharness/base/script.py
View File

@ -45,6 +45,7 @@ except ImportError:
from mozprocess import ProcessHandler
from mozharness.base.config import BaseConfig
from mozharness.base.errors import ZipErrorList
from mozharness.base.log import SimpleFileLogger, MultiFileLogger, \
LogMixin, OutputParser, DEBUG, INFO, ERROR, FATAL
@ -450,6 +451,40 @@ class ScriptMixin(PlatformMixin):
**retry_args
)
def download_unzip(self, url, parent_dir, target_unzip_dirs=None, halt_on_failure=True):
"""Generic method to download and extract a zip file.
The downloaded file will always be saved to the working directory and is not getting
deleted after extracting.
Args:
url (str): URL where the file to be downloaded is located.
parent_dir (str): directory where the downloaded file will
be extracted to.
target_unzip_dirs (list, optional): directories inside the zip file to extract.
Defaults to `None`.
halt_on_failure (bool, optional): whether or not to redefine the
log level as `FATAL` on errors. Defaults to True.
"""
dirs = self.query_abs_dirs()
zipfile = self.download_file(url, parent_dir=dirs['abs_work_dir'],
error_level=FATAL)
command = self.query_exe('unzip', return_type='list')
# Always overwrite to not get an input in a hidden pipe if files already exist
command.extend(['-q', '-o', zipfile, '-d', parent_dir])
if target_unzip_dirs:
command.extend(target_unzip_dirs)
# TODO error_list: http://www.info-zip.org/mans/unzip.html#DIAGNOSTICS
# unzip return code 11 is 'no matching files were found'
self.run_command(command,
error_list=ZipErrorList,
halt_on_failure=halt_on_failure,
fatal_exit_code=3,
success_codes=[0, 11],
)
def load_json_url(self, url, error_level=None, *args, **kwargs):
""" Returns a json object from a url (it retries). """
contents = self._retry_download(

2
testing/mozharness/mozharness/mozilla/testing/talos.py
View File

@ -274,7 +274,7 @@ class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
if self.query_pagesets_url():
self.info("Downloading pageset...")
src_talos_pageset = os.path.join(src_talos_webdir, 'tests')
self._download_unzip(self.pagesets_url, src_talos_pageset)
self.download_unzip(self.pagesets_url, src_talos_pageset)
# Action methods. {{{1
# clobber defined in BaseScript

58
testing/mozharness/mozharness/mozilla/testing/testbase.py
View File

@ -112,7 +112,6 @@ class TestingMixin(VirtualenvMixin, BuildbotMixin, ResourceMonitoringMixin,
binary_path = None
test_url = None
test_packages_url = None
test_zip_path = None
symbols_url = None
symbols_path = None
jsshell_url = None
@ -380,18 +379,6 @@ You can set this by:
"You are currently using version %s. Please update to at least 6.0.\n"
"You can visit http://www.info-zip.org/UnZip.html" % version)
def _download_test_zip(self):
dirs = self.query_abs_dirs()
file_name = None
if self.test_zip_path:
file_name = self.test_zip_path
# try to use our proxxy servers
# create a proxxy object and get the binaries from it
source = self.download_file(self.test_url, file_name=file_name,
parent_dir=dirs['abs_work_dir'],
error_level=FATAL)
self.test_zip_path = os.path.realpath(source)
def _read_packages_manifest(self):
dirs = self.query_abs_dirs()
source = self.download_file(self.test_packages_url,
@ -449,41 +436,15 @@ You can set this by:
unzip_dirs = None
target_dir = dirs['abs_test_bin_dir']
url = self.query_build_dir_url(file_name)
self._download_unzip(url, target_dir,
self.download_unzip(url, target_dir,
target_unzip_dirs=unzip_dirs)
def _download_unzip(self, url, parent_dir, target_unzip_dirs=None):
"""Generic download+unzip.
This is hardcoded to halt on failure.
We should probably change some other methods to call this."""
def _download_test_zip(self, target_unzip_dirs=None):
dirs = self.query_abs_dirs()
zipfile = self.download_file(url, parent_dir=dirs['abs_work_dir'],
error_level=FATAL)
command = self.query_exe('unzip', return_type='list')
command.extend(['-q', '-o', zipfile])
if target_unzip_dirs:
command.extend(target_unzip_dirs)
self.run_command(command, cwd=parent_dir, halt_on_failure=True,
success_codes=[0, 11],
fatal_exit_code=3, output_timeout=1760)
def _extract_test_zip(self, target_unzip_dirs=None):
dirs = self.query_abs_dirs()
unzip = self.query_exe("unzip")
test_install_dir = dirs.get('abs_test_install_dir',
os.path.join(dirs['abs_work_dir'], 'tests'))
self.mkdir_p(test_install_dir)
# adding overwrite flag otherwise subprocess.Popen hangs on waiting for
# input in a hidden pipe whenever this action is run twice without
# clobber
unzip_cmd = [unzip, '-q', '-o', self.test_zip_path]
if target_unzip_dirs:
unzip_cmd.extend(target_unzip_dirs)
# TODO error_list
# unzip return code 11 is 'no matching files were found'
self.run_command(unzip_cmd, cwd=test_install_dir,
halt_on_failure=True, success_codes=[0, 11],
fatal_exit_code=3)
self.download_unzip(self.test_url, test_install_dir,
target_unzip_dirs=target_unzip_dirs)
def structured_output(self, suite_category):
"""Defines whether structured logging is in use in this configuration. This
@ -527,14 +488,10 @@ You can set this by:
return
if not self.symbols_path:
self.symbols_path = os.path.join(dirs['abs_work_dir'], 'symbols')
self.mkdir_p(self.symbols_path)
source = self.download_file(self.symbols_url,
parent_dir=self.symbols_path,
error_level=FATAL)
self.set_buildbot_property("symbols_url", self.symbols_url,
write_to_file=True)
self.run_command(['unzip', '-q', source], cwd=self.symbols_path,
halt_on_failure=True, fatal_exit_code=3)
self.download_unzip(self.symbols_url, self.symbols_path)
def download_and_extract(self, target_unzip_dirs=None, suite_categories=None):
"""
@ -559,8 +516,7 @@ You can set this by:
' package data at "%s" will be ignored.' %
(self.config.get('test_url'), self.test_packages_url))
self._download_test_zip()
self._extract_test_zip(target_unzip_dirs=target_unzip_dirs)
self._download_test_zip(target_unzip_dirs)
else:
if not self.test_packages_url:
# The caller intends to download harness specific packages, but doesn't know

2
testing/mozharness/mozharness/mozilla/testing/unittest.py
View File

@ -221,7 +221,7 @@ class EmulatorMixin(object):
dirs = self.query_abs_dirs()
self.mkdir_p(dirs['abs_emulator_dir'])
if self.config.get('emulator_url'):
self._download_unzip(self.config['emulator_url'], dirs['abs_emulator_dir'])
self.download_unzip(self.config['emulator_url'], dirs['abs_emulator_dir'])
elif self.config.get('emulator_manifest'):
manifest_path = self.create_tooltool_manifest(self.config['emulator_manifest'])
do_unzip = True

3
testing/mozharness/scripts/android_emulator_unittest.py
View File

@ -627,8 +627,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
robocop_url = self.installer_url[:self.installer_url.rfind('/')] + '/robocop.apk'
self.info("Downloading robocop...")
self.download_file(robocop_url, 'robocop.apk', dirs['abs_work_dir'], error_level=FATAL)
self.mkdir_p(dirs['abs_xre_dir'])
self._download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
self.download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
def install(self):
"""

5
testing/mozharness/scripts/android_panda.py
View File

@ -382,9 +382,8 @@ class PandaTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixin, Bu
c = self.config
dirs = self.query_abs_dirs()
self.host_utils_url = c['hostutils_url']
#create the hostutils dir, get the zip and extract it
self.mkdir_p(dirs['abs_hostutils_dir'])
self._download_unzip(self.host_utils_url, dirs['abs_hostutils_dir'])
# get the zip and extract it
self.download_unzip(self.host_utils_url, dirs['abs_hostutils_dir'])
def _install_app(self):
c = self.config

7
testing/mozharness/scripts/android_panda_talos.py
View File

@ -299,8 +299,7 @@ class PandaTalosTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixi
self.rmtree(dirs['abs_talosdata_dir'])
self.mkdir_p(dirs['abs_talosdata_dir'])
self.mkdir_p(dirs['abs_symbols_dir'])
self.mkdir_p(dirs['abs_fennec_dir'])
self._download_unzip(self.installer_url,
self.download_unzip(self.installer_url,
dirs['abs_fennec_dir'])
#this is ugly but you can't specify a file in download_unzip to extract the file to, by default it's the abs_work_dir
#should think of a better way
@ -318,10 +317,10 @@ class PandaTalosTest(TestingMixin, MercurialScript, BlobUploadMixin, MozpoolMixi
error_level=FATAL)
self.symbols_url = self.query_symbols_url()
self._download_unzip(self.symbols_url,
self.download_unzip(self.symbols_url,
dirs['abs_symbols_dir'])
self._download_unzip(self.config['retry_url'],
self.download_unzip(self.config['retry_url'],
dirs['abs_talosdata_dir'])
taloscode = self.config.get("talos_from_code_url")

3
testing/mozharness/scripts/androidx86_emulator_unittest.py
View File

@ -681,8 +681,7 @@ class AndroidEmulatorTest(BlobUploadMixin, TestingMixin, EmulatorMixin, VCSMixin
self._download_robocop_apk()
break
self.mkdir_p(dirs['abs_xre_dir'])
self._download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
self.download_unzip(self.host_utils_url, dirs['abs_xre_dir'])
def install(self):
assert self.installer_path is not None, \

3
testing/mozharness/scripts/b2g_emulator_unittest.py
View File

@ -208,8 +208,7 @@ class B2GEmulatorTest(TestingMixin, VCSMixin, BaseScript, BlobUploadMixin):
error_list=TarErrorList,
halt_on_failure=True, fatal_exit_code=3)
self.mkdir_p(dirs['abs_xre_dir'])
self._download_unzip(self.config['xre_url'],
self.download_unzip(self.config['xre_url'],
dirs['abs_xre_dir'])
if self.config.get('busybox_url'):