diff --git a/taskcluster/ci/test/test-platforms.yml b/taskcluster/ci/test/test-platforms.yml index 9633f7c66818..692fc0bed2e1 100644 --- a/taskcluster/ci/test/test-platforms.yml +++ b/taskcluster/ci/test/test-platforms.yml @@ -34,6 +34,7 @@ linux32-nightly/opt: - linux32-opt-tests - external-media-tests - external-media-tests-slow + - awsy linux64/debug: build-platform: linux64/debug @@ -60,6 +61,7 @@ linux64-nightly/opt: - opt-only-tests - desktop-screenshot-capture - talos + - awsy # TODO: use 'pgo' and 'asan' labels here, instead of -pgo/opt linux64-pgo/opt: diff --git a/taskcluster/ci/test/test-sets.yml b/taskcluster/ci/test/test-sets.yml index 527605511898..ca1afb0f8ac1 100644 --- a/taskcluster/ci/test/test-sets.yml +++ b/taskcluster/ci/test/test-sets.yml @@ -65,6 +65,9 @@ talos: - talos-svgr - talos-tp5o +awsy: + - awsy + ## # Limited test sets for specific platforms diff --git a/taskcluster/ci/test/tests.yml b/taskcluster/ci/test/tests.yml index 2e36ef3deefe..8f1d049a9296 100644 --- a/taskcluster/ci/test/tests.yml +++ b/taskcluster/ci/test/tests.yml @@ -1341,6 +1341,24 @@ talos-tp5o: extra-options: - --suite=tp5o +awsy: + description: "Are we slim yet" + suite: awsy + treeherder-symbol: tc-SY(sy) + run-on-projects: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try'] + docker-image: {"in-tree": "desktop1604-test"} + max-run-time: 7200 + e10s: true + instance-size: xlarge + allow-software-gl-layers: false + mozharness: + script: awsy_script.py + config: + by-test-platform: + default: + - awsy/linux_config.py + - remove_executables.py + web-platform-tests: description: "Web platform test run" suite: web-platform-tests diff --git a/taskcluster/taskgraph/transforms/task.py b/taskcluster/taskgraph/transforms/task.py index 3494044a3331..0686bf90dda6 100644 --- a/taskcluster/taskgraph/transforms/task.py +++ b/taskcluster/taskgraph/transforms/task.py @@ -362,6 +362,7 @@ GROUP_NAMES = { 'tc-R-e10s': 'Reftests executed by TaskCluster with e10s', 'tc-T': 'Talos performance tests executed by TaskCluster', 'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s', + 'tc-SY-e10s': 'Are we slim yet tests by TaskCluster with e10s', 'tc-VP': 'VideoPuppeteer tests executed by TaskCluster', 'tc-W': 'Web platform tests executed by TaskCluster', 'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s', diff --git a/taskcluster/taskgraph/transforms/tests.py b/taskcluster/taskgraph/transforms/tests.py index ac6ab2beab61..e24c1f2ba235 100644 --- a/taskcluster/taskgraph/transforms/tests.py +++ b/taskcluster/taskgraph/transforms/tests.py @@ -592,7 +592,7 @@ def remove_linux_pgo_try_talos(config, tests): def predicate(test): return not( test['test-platform'] == 'linux64-pgo/opt' - and test['suite'] == 'talos' + and (test['suite'] == 'talos' or test['suite'] == 'awsy') and config.params['project'] == 'try' ) for test in filter(predicate, tests): diff --git a/testing/mozharness/configs/awsy/linux_config.py b/testing/mozharness/configs/awsy/linux_config.py new file mode 100644 index 000000000000..ff3d9f01c863 --- /dev/null +++ b/testing/mozharness/configs/awsy/linux_config.py @@ -0,0 +1,50 @@ +import os +import platform + +PYTHON = "/usr/bin/env python" +VENV_PATH = '%s/build/venv' % os.getcwd() +if platform.architecture()[0] == '64bit': + TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest" + MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk" +else: + TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest" + MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk" +ABS_WORK_DIR = os.path.join(os.getcwd(), "build") +BINARY_PATH = os.path.join(ABS_WORK_DIR, "application", "firefox", "firefox-bin") +INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2") + +config = { + "log_name": "awsy", + "binary_path": BINARY_PATH, + "download_tooltool": True, + "installer_path": INSTALLER_PATH, + "virtualenv_path": VENV_PATH, + "find_links": [ + "http://pypi.pvt.build.mozilla.org/pub", + "http://pypi.pub.build.mozilla.org/pub", + ], + "cmd_timeout": 6500, + "pip_index": False, + "exes": { + 'python': PYTHON, + 'virtualenv': [PYTHON, '/tools/misc-python/virtualenv.py'], + 'tooltool.py': "/tools/tooltool.py", + }, + "title": os.uname()[1].lower().split('.')[0], + "default_actions": [ + "clobber", + "download-and-extract", + "populate-webroot", + "create-virtualenv", + "install", + "run-tests", + ], + "default_blob_upload_servers": [ + "https://blobupload.elasticbeanstalk.com", + ], + "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"), + "download_minidump_stackwalk": True, + "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH, + "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH, + "tooltool_cache": os.path.join(os.getcwd(), "tooltool_cache"), +} diff --git a/testing/mozharness/scripts/awsy_script.py b/testing/mozharness/scripts/awsy_script.py new file mode 100644 index 000000000000..714f203e1adb --- /dev/null +++ b/testing/mozharness/scripts/awsy_script.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python +# ***** BEGIN LICENSE BLOCK ***** +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# ***** END LICENSE BLOCK ***** +""" +run awsy tests in a virtualenv +""" + +import os +import sys +import copy + +# load modules from parent dir +sys.path.insert(1, os.path.dirname(sys.path[0])) + +from mozharness.base.script import PreScriptAction +from mozharness.base.log import INFO, ERROR, WARNING, CRITICAL +from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options +from mozharness.base.vcs.vcsbase import MercurialScript +from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options +from mozharness.mozilla.tooltool import TooltoolMixin +from mozharness.mozilla.structuredlog import StructuredOutputParser + + +class AWSY(TestingMixin, MercurialScript, BlobUploadMixin,TooltoolMixin): + config_options = [ + [["--e10s"], + {"action": "store_true", + "dest": "e10s", + "default": False, + "help": "Run tests with multiple processes. (Desktop builds only)", + }] + ] + testing_config_options + copy.deepcopy(blobupload_config_options) + + def __init__(self, **kwargs): + + kwargs.setdefault('config_options', self.config_options) + kwargs.setdefault('all_actions', ['clobber', + 'download-and-extract', + 'populate-webroot', + 'create-virtualenv', + 'install', + 'run-tests', + ]) + kwargs.setdefault('default_actions', ['clobber', + 'download-and-extract', + 'populate-webroot', + 'create-virtualenv', + 'install', + 'run-tests', + ]) + kwargs.setdefault('config', {}) + super(AWSY, self).__init__(**kwargs) + self.installer_url = self.config.get("installer_url") + self.tests = None + + self.workdir = self.query_abs_dirs()['abs_work_dir'] + self.testdir = os.path.join(self.workdir, 'tests') + self.awsy_path = os.path.join( + self.testdir, 'awsy' + ) + self.awsy_libdir = os.path.join(self.awsy_path, 'awsy') + self.awsy_testdir = os.path.join(self.awsy_path, 'page_load_test') + self.binary_path = self.config.get('binary_path') + + def query_abs_dirs(self): + if self.abs_dirs: + return self.abs_dirs + abs_dirs = super(AWSY, self).query_abs_dirs() + + dirs = {} + dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir') + abs_dirs.update(dirs) + self.abs_dirs = abs_dirs + return self.abs_dirs + + def download_and_extract(self, extract_dirs=None, suite_categories=None): + ret = super(AWSY, self).download_and_extract( + suite_categories=['common', 'awsy'] + ) + return ret + + @PreScriptAction('create-virtualenv') + def _pre_create_virtualenv(self, action): + requirements_files = [os.path.join(self.testdir, + 'config', + 'marionette_requirements.txt')] + + for requirements_file in requirements_files: + self.register_virtualenv_module(requirements=[requirements_file], + two_pass=True) + + self.register_virtualenv_module('awsy', self.awsy_path) + + + def populate_webroot(self): + """Populate the production test slaves' webroots""" + self.info("Downloading pageset with tooltool...") + manifest_file = os.path.join(self.awsy_path, 'tp5n-pageset.manifest') + if not os.path.isdir(self.awsy_testdir): + self.mkdir_p(self.awsy_testdir) + self.tooltool_fetch( + manifest_file, + output_dir=self.awsy_testdir, + cache=self.config.get('tooltool_cache') + ) + archive = os.path.join(self.awsy_testdir, 'tp5n.zip') + unzip = self.query_exe('unzip') + unzip_cmd = [unzip, '-q', '-o', archive, '-d', self.awsy_testdir] + self.run_command(unzip_cmd, halt_on_failure=True) + self.run_command("ls %s" % self.awsy_testdir) + + + def run_tests(self, args=None, **kw): + ''' + AWSY test should be implemented here + ''' + dirs = self.abs_dirs + env = {} + error_summary_file = os.path.join(dirs['abs_blob_upload_dir'], + 'marionette_errorsummary.log') + + cmd = ['marionette'] + cmd.append("--preferences=%s" % os.path.join(self.awsy_path, "conf", "prefs.json")) + cmd.append("--testvars=%s" % os.path.join(self.awsy_path, "conf", "testvars.json")) + cmd.append("--log-raw=-") + cmd.append("--log-errorsummary=%s" % error_summary_file) + cmd.append("--binary=%s" % self.binary_path) + cmd.append("--profile=%s" % (os.path.join(dirs['abs_work_dir'], 'profile'))) + if not self.config['e10s']: + cmd.append('--disable-e10s') + cmd.append('--gecko-log=%s' % os.path.join(dirs["abs_blob_upload_dir"], + 'gecko.log')) + + test_file = os.path.join(self.awsy_libdir, 'test_memory_usage.py') + cmd.append(test_file) + + env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir'] + if not os.path.isdir(env['MOZ_UPLOAD_DIR']): + self.mkdir_p(env['MOZ_UPLOAD_DIR']) + env = self.query_env(partial_env=env) + parser = StructuredOutputParser(config=self.config, + log_obj=self.log_obj) + return_code = self.run_command(command=cmd, + cwd=self.awsy_path, + output_timeout=self.config.get("cmd_timeout"), + env=env, + output_parser=parser) + + level = INFO + tbpl_status, log_level = parser.evaluate_parser( + return_code=return_code) + + self.log("AWSY exited with return code %s: %s" % (return_code, tbpl_status), + level=level) + self.buildbot_status(tbpl_status) + + +if __name__ == '__main__': + awsy_test = AWSY() + awsy_test.run_and_exit()