Bug 1272113: Creating awsy task and mozharness script r=jmaher

* * *
[mq]: test_only

MozReview-Commit-ID: AawtpSUIj0P

--HG--
extra : rebase_source : 147a97e07f00e352f30aa9d5d39aa34500ed2abb

taskcluster/ci/test/test-platforms.yml

@@ -34,6 +34,7 @@ linux32-nightly/opt:
         -  linux32-opt-tests
         - external-media-tests
         - external-media-tests-slow
+        - awsy
 
 linux64/debug:
     build-platform: linux64/debug
@@ -60,6 +61,7 @@ linux64-nightly/opt:
         - opt-only-tests
         - desktop-screenshot-capture
         - talos
+        - awsy
 
 # TODO: use 'pgo' and 'asan' labels here, instead of -pgo/opt
 linux64-pgo/opt:

taskcluster/ci/test/test-sets.yml

@@ -65,6 +65,9 @@ talos:
     - talos-svgr
     - talos-tp5o
 
+awsy:
+    - awsy
+
 ##
 # Limited test sets for specific platforms
 

taskcluster/ci/test/tests.yml

@@ -1341,6 +1341,24 @@ talos-tp5o:
         extra-options:
             - --suite=tp5o
 
+awsy:
+    description: "Are we slim yet"
+    suite: awsy
+    treeherder-symbol: tc-SY(sy)
+    run-on-projects: ['mozilla-beta', 'mozilla-aurora', 'mozilla-central', 'mozilla-inbound', 'autoland', 'try']
+    docker-image: {"in-tree": "desktop1604-test"}
+    max-run-time: 7200
+    e10s: true
+    instance-size: xlarge
+    allow-software-gl-layers: false
+    mozharness:
+        script: awsy_script.py
+        config:
+            by-test-platform:
+                default:
+                    - awsy/linux_config.py
+                    - remove_executables.py
+
 web-platform-tests:
     description: "Web platform test run"
     suite: web-platform-tests

taskcluster/taskgraph/transforms/task.py

@@ -362,6 +362,7 @@ GROUP_NAMES = {
     'tc-R-e10s': 'Reftests executed by TaskCluster with e10s',
     'tc-T': 'Talos performance tests executed by TaskCluster',
     'tc-T-e10s': 'Talos performance tests executed by TaskCluster with e10s',
+    'tc-SY-e10s': 'Are we slim yet tests by TaskCluster with e10s',
     'tc-VP': 'VideoPuppeteer tests executed by TaskCluster',
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',

taskcluster/taskgraph/transforms/tests.py

@@ -592,7 +592,7 @@ def remove_linux_pgo_try_talos(config, tests):
     def predicate(test):
         return not(
             test['test-platform'] == 'linux64-pgo/opt'
-            and test['suite'] == 'talos'
+            and (test['suite'] == 'talos' or test['suite'] == 'awsy')
             and config.params['project'] == 'try'
         )
     for test in filter(predicate, tests):

testing/mozharness/configs/awsy/linux_config.py

@@ -0,0 +1,50 @@
+import os
+import platform
+
+PYTHON = "/usr/bin/env python"
+VENV_PATH = '%s/build/venv' % os.getcwd()
+if platform.architecture()[0] == '64bit':
+    TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux64/releng.manifest"
+    MINIDUMP_STACKWALK_PATH = "linux64-minidump_stackwalk"
+else:
+    TOOLTOOL_MANIFEST_PATH = "config/tooltool-manifests/linux32/releng.manifest"
+    MINIDUMP_STACKWALK_PATH = "linux32-minidump_stackwalk"
+ABS_WORK_DIR = os.path.join(os.getcwd(), "build")
+BINARY_PATH = os.path.join(ABS_WORK_DIR, "application", "firefox", "firefox-bin")
+INSTALLER_PATH = os.path.join(ABS_WORK_DIR, "installer.tar.bz2")
+
+config = {
+    "log_name": "awsy",
+    "binary_path": BINARY_PATH,
+    "download_tooltool": True,
+    "installer_path": INSTALLER_PATH,
+    "virtualenv_path": VENV_PATH,
+    "find_links": [
+        "http://pypi.pvt.build.mozilla.org/pub",
+        "http://pypi.pub.build.mozilla.org/pub",
+    ],
+    "cmd_timeout": 6500,
+    "pip_index": False,
+    "exes": {
+        'python': PYTHON,
+        'virtualenv': [PYTHON, '/tools/misc-python/virtualenv.py'],
+        'tooltool.py': "/tools/tooltool.py",
+    },
+    "title": os.uname()[1].lower().split('.')[0],
+    "default_actions": [
+        "clobber",
+        "download-and-extract",
+        "populate-webroot",
+        "create-virtualenv",
+        "install",
+        "run-tests",
+    ],
+    "default_blob_upload_servers": [
+        "https://blobupload.elasticbeanstalk.com",
+    ],
+    "blob_uploader_auth_file": os.path.join(os.getcwd(), "oauth.txt"),
+    "download_minidump_stackwalk": True,
+    "minidump_stackwalk_path": MINIDUMP_STACKWALK_PATH,
+    "minidump_tooltool_manifest_path": TOOLTOOL_MANIFEST_PATH,
+    "tooltool_cache": os.path.join(os.getcwd(), "tooltool_cache"),
+}

testing/mozharness/scripts/awsy_script.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+"""
+run awsy tests in a virtualenv
+"""
+
+import os
+import sys
+import copy
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.script import PreScriptAction
+from mozharness.base.log import INFO, ERROR, WARNING, CRITICAL
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
+from mozharness.mozilla.tooltool import TooltoolMixin
+from mozharness.mozilla.structuredlog import StructuredOutputParser
+
+
+class AWSY(TestingMixin, MercurialScript, BlobUploadMixin,TooltoolMixin):
+    config_options = [
+        [["--e10s"],
+        {"action": "store_true",
+         "dest": "e10s",
+         "default": False,
+         "help": "Run tests with multiple processes. (Desktop builds only)",
+         }]
+    ] + testing_config_options + copy.deepcopy(blobupload_config_options)
+
+    def __init__(self, **kwargs):
+
+        kwargs.setdefault('config_options', self.config_options)
+        kwargs.setdefault('all_actions', ['clobber',
+                                          'download-and-extract',
+                                          'populate-webroot',
+                                          'create-virtualenv',
+                                          'install',
+                                          'run-tests',
+                                          ])
+        kwargs.setdefault('default_actions', ['clobber',
+                                              'download-and-extract',
+                                              'populate-webroot',
+                                              'create-virtualenv',
+                                              'install',
+                                              'run-tests',
+                                              ])
+        kwargs.setdefault('config', {})
+        super(AWSY, self).__init__(**kwargs)
+        self.installer_url = self.config.get("installer_url")
+        self.tests = None
+
+        self.workdir = self.query_abs_dirs()['abs_work_dir']
+        self.testdir = os.path.join(self.workdir, 'tests')
+        self.awsy_path = os.path.join(
+            self.testdir, 'awsy'
+        )
+        self.awsy_libdir = os.path.join(self.awsy_path, 'awsy')
+        self.awsy_testdir = os.path.join(self.awsy_path, 'page_load_test')
+        self.binary_path = self.config.get('binary_path')
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(AWSY, self).query_abs_dirs()
+
+        dirs = {}
+        dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
+        abs_dirs.update(dirs)
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    def download_and_extract(self, extract_dirs=None, suite_categories=None):
+        ret = super(AWSY, self).download_and_extract(
+            suite_categories=['common', 'awsy']
+        )
+        return ret
+
+    @PreScriptAction('create-virtualenv')
+    def _pre_create_virtualenv(self, action):
+        requirements_files = [os.path.join(self.testdir,
+                                           'config',
+                                           'marionette_requirements.txt')]
+
+        for requirements_file in requirements_files:
+            self.register_virtualenv_module(requirements=[requirements_file],
+                                            two_pass=True)
+
+        self.register_virtualenv_module('awsy', self.awsy_path)
+
+
+    def populate_webroot(self):
+        """Populate the production test slaves' webroots"""
+        self.info("Downloading pageset with tooltool...")
+        manifest_file = os.path.join(self.awsy_path, 'tp5n-pageset.manifest')
+        if not os.path.isdir(self.awsy_testdir):
+            self.mkdir_p(self.awsy_testdir)
+        self.tooltool_fetch(
+            manifest_file,
+            output_dir=self.awsy_testdir,
+            cache=self.config.get('tooltool_cache')
+        )
+        archive = os.path.join(self.awsy_testdir, 'tp5n.zip')
+        unzip = self.query_exe('unzip')
+        unzip_cmd = [unzip, '-q', '-o', archive, '-d', self.awsy_testdir]
+        self.run_command(unzip_cmd, halt_on_failure=True)
+        self.run_command("ls %s" % self.awsy_testdir)
+
+
+    def run_tests(self, args=None, **kw):
+        '''
+        AWSY test should be implemented here
+        '''
+        dirs = self.abs_dirs
+        env = {}
+        error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
+                                          'marionette_errorsummary.log')
+
+        cmd = ['marionette']
+        cmd.append("--preferences=%s" % os.path.join(self.awsy_path, "conf", "prefs.json"))
+        cmd.append("--testvars=%s" % os.path.join(self.awsy_path, "conf", "testvars.json"))
+        cmd.append("--log-raw=-")
+        cmd.append("--log-errorsummary=%s" % error_summary_file)
+        cmd.append("--binary=%s" % self.binary_path)
+        cmd.append("--profile=%s" % (os.path.join(dirs['abs_work_dir'], 'profile')))
+        if not self.config['e10s']:
+            cmd.append('--disable-e10s')
+        cmd.append('--gecko-log=%s' % os.path.join(dirs["abs_blob_upload_dir"],
+                                                   'gecko.log'))
+
+        test_file = os.path.join(self.awsy_libdir, 'test_memory_usage.py')
+        cmd.append(test_file)
+
+        env['MOZ_UPLOAD_DIR'] = dirs['abs_blob_upload_dir']
+        if not os.path.isdir(env['MOZ_UPLOAD_DIR']):
+            self.mkdir_p(env['MOZ_UPLOAD_DIR'])
+        env = self.query_env(partial_env=env)
+        parser = StructuredOutputParser(config=self.config,
+                                        log_obj=self.log_obj)
+        return_code = self.run_command(command=cmd,
+                                       cwd=self.awsy_path,
+                                       output_timeout=self.config.get("cmd_timeout"),
+                                       env=env,
+                                       output_parser=parser)
+
+        level = INFO
+        tbpl_status, log_level = parser.evaluate_parser(
+            return_code=return_code)
+
+        self.log("AWSY exited with return code %s: %s" % (return_code, tbpl_status),
+                 level=level)
+        self.buildbot_status(tbpl_status)
+
+
+if __name__ == '__main__':
+    awsy_test = AWSY()
+    awsy_test.run_and_exit()