Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-18 07:45:30 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1460743 - Add 'speedometer' benchmark to raptor for google chrome; r=jmaher

Browse Source 
Includes a temporary downloading/installing Chrome 'hack' just so we can get going on try, until 'official' chrome builds are available in CI. Also some small raptor fixes/changes to support speedometer on chrome.

MozReview-Commit-ID: 8ZBlOg0BOE4

--HG--
extra : rebase_source : 8271a7b9efd6b1d30e363eb47876d82f243d5ca7
This commit is contained in:
Rob Wood 2018-06-04 15:42:44 -04:00
parent 25a3c8254c
commit 978d9b823c
9 changed files with 163 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
testing/mozharness/configs/raptor/mac_config.py
View File

@ -26,6 +26,7 @@ config = {
"clobber",
"download-and-extract",
"populate-webroot",
"install-chrome",
"create-virtualenv",
"install",
"run-tests",

141
testing/mozharness/mozharness/mozilla/testing/raptor.py
View File

@ -10,6 +10,7 @@ import os
import re
import sys
import subprocess
import time
from shutil import copyfile
@ -27,6 +28,7 @@ from mozharness.mozilla.testing.codecoverage import (
scripts_path = os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__)))
external_tools_path = os.path.join(scripts_path, 'external_tools')
here = os.path.abspath(os.path.dirname(__file__))
RaptorErrorList = PythonErrorList + [
{'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
@ -52,6 +54,12 @@ class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin
"dest": "test",
"help": "Raptor test to run"
}],
[["--app"],
{"default": "firefox",
"choices": ["firefox", "chrome"],
"dest": "app",
"help": "name of the application we are testing (default: firefox)"
}],
[["--branch-name"],
{"action": "store",
"dest": "branch",
@ -59,7 +67,7 @@ class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin
}],
[["--add-option"],
{"action": "extend",
"dest": "raptor_extra_options",
"dest": "raptor_cmd_line_args",
"default": None,
"help": "extra options to raptor"
}],
@ -70,6 +78,7 @@ class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin
kwargs.setdefault('all_actions', ['clobber',
'download-and-extract',
'populate-webroot',
'install-chrome',
'create-virtualenv',
'install',
'run-tests',
@ -77,6 +86,7 @@ class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin
kwargs.setdefault('default_actions', ['clobber',
'download-and-extract',
'populate-webroot',
'install-chrome',
'create-virtualenv',
'install',
'run-tests',
@ -87,28 +97,37 @@ class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin
self.workdir = self.query_abs_dirs()['abs_work_dir'] # convenience
self.run_local = self.config.get('run_local')
# app (browser testing on) defaults to firefox
self.app = "firefox"
if self.run_local:
# raptor initiated locally, get app from command line args
# which are passed in from mach inside 'raptor_cmd_line_args'
self.app = "firefox"
if 'raptor_cmd_line_args' in self.config:
for next_arg in self.config['raptor_cmd_line_args']:
if "chrome" in next_arg:
self.app = "chrome"
break
else:
# raptor initiated in production via mozharness
self.test = self.config['test']
self.app = self.config.get("app", "firefox")
self.installer_url = self.config.get("installer_url")
self.raptor_json_url = self.config.get("raptor_json_url")
self.raptor_json = self.config.get("raptor_json")
self.raptor_json_config = self.config.get("raptor_json_config")
self.repo_path = self.config.get("repo_path")
self.obj_path = self.config.get("obj_path")
self.tests = None
self.test = None
self.gecko_profile = self.config.get('gecko_profile')
self.gecko_profile_interval = self.config.get('gecko_profile_interval')
# some platforms download a mitmproxy release binary
self.mitmproxy_rel_bin = None
# zip file found on tooltool that contains all of the mitmproxy recordings
self.mitmproxy_pageset = None
# files inside the recording set
self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None)
# path to mitmdump tool itself, in py3 venv
self.mitmdump = None
# We accept some configuration options from the try commit message in the
# format mozharness: <options>
# Example try commit message:
# mozharness: --geckoProfile try: <stuff>
# format mozharness: <options>. Example try commit message: mozharness:
# --geckoProfile try: <stuff>
def query_gecko_profile_options(self):
gecko_results = []
# if gecko_profile is set, we add that to the raptor options
@ -130,19 +149,77 @@ class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin
self.abs_dirs = abs_dirs
return self.abs_dirs
def install_chrome(self):
# temporary hack to install google chrome in production; until chrome is in our CI
if self.app != "chrome":
self.info("Google Chrome is not required")
return
if self.config.get("run_local"):
self.info("expecting Google Chrome to be pre-installed locally")
return
chrome_url = "https://dl.google.com/chrome/mac/stable/GGRO/googlechrome.dmg"
# in production we can put the chrome build in mozharness/mozilla/testing/chrome
self.chrome_dest = os.path.join(here, 'chrome')
chrome_dmg = os.path.join(self.chrome_dest, 'googlechrome.dmg')
self.info("installing google chrome - temporary install hack")
self.info("chrome_dest is: %s" % self.chrome_dest)
self.chrome_path = os.path.join(self.chrome_dest, 'Google Chrome.app',
'Contents', 'MacOS', 'Google Chrome')
if os.path.exists(self.chrome_path):
self.info("google chrome binary already exists at: %s" % self.chrome_path)
return
if not os.path.exists(chrome_dmg):
# download the chrome dmg
self.download_file(chrome_url, parent_dir=self.chrome_dest)
command = ["open", "googlechrome.dmg"]
return_code = self.run_command(command, cwd=self.chrome_dest)
if return_code not in [0]:
self.info("abort: failed to open %s/googlechrome.dmg" % self.chrome_dest)
return
# give 30 sec for open cmd to finish
time.sleep(30)
# now that the googlechrome dmg is mounted, extract/copy app from mnt to our folder
command = ["cp", "-r", "/Volumes/Google Chrome/Google Chrome.app", "."]
return_code = self.run_command(command, cwd=self.chrome_dest)
if return_code not in [0]:
self.info("abort: failed to open %s/googlechrome.dmg" % self.chrome_dest)
return
# now ensure chrome binary exists
if os.path.exists(self.chrome_path):
self.info("successfully installed Google Chrome to: %s" % self.chrome_path)
else:
self.info("abort: failed to install Google Chrome")
def raptor_options(self, args=None, **kw):
"""return options to raptor"""
# binary path
binary_path = self.binary_path or self.config.get('binary_path')
if not binary_path:
msg = """Raptor requires a path to the binary. You can specify binary_path or add
download-and-extract to your action list."""
self.fatal(msg)
# raptor options
if binary_path.endswith('.exe'):
binary_path = binary_path[:-4]
options = []
kw_options = {'binary': binary_path}
kw_options = {}
# binary path; if testing on firefox the binary path already came from mozharness/pro;
# otherwise the binary path is forwarded from cmd line arg (raptor_cmd_line_args)
kw_options['app'] = self.app
if self.app == "firefox":
binary_path = self.binary_path or self.config.get('binary_path')
if not binary_path:
self.fatal("Raptor requires a path to the binary.")
if binary_path.endswith('.exe'):
binary_path = binary_path[:-4]
kw_options['binary'] = binary_path
else:
if not self.run_local:
# in production we aready installed google chrome, so set the binary path for arg
# when running locally a --binary arg as passed in, already in raptor_cmd_line_args
kw_options['binary'] = self.chrome_path
# options overwritten from **kw
if 'test' in self.config:
kw_options['test'] = self.config['test']
@ -160,12 +237,13 @@ class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin
options += args
if self.config.get('run_local', False):
options.extend(['--run-local'])
if 'raptor_extra_options' in self.config:
options += self.config['raptor_extra_options']
if 'raptor_cmd_line_args' in self.config:
options += self.config['raptor_cmd_line_args']
if self.config.get('code_coverage', False):
options.extend(['--code-coverage'])
for key, value in kw_options.items():
options.extend(['--%s' % key, value])
return options
def populate_webroot(self):
@ -173,21 +251,8 @@ class Raptor(TestingMixin, MercurialScript, Python3Virtualenv, CodeCoverageMixin
self.raptor_path = os.path.join(
self.query_abs_dirs()['abs_test_install_dir'], 'raptor'
)
if self.config.get('run_local'):
# raptor initiated locally, get and verify test from cmd line
self.raptor_path = os.path.join(self.repo_path, 'testing', 'raptor')
if 'raptor_extra_options' in self.config:
if '--test' in self.config['raptor_extra_options']:
# --test specified, get test from cmd line and ensure is valid
test_name_index = self.config['raptor_extra_options'].index('--test') + 1
if test_name_index < len(self.config['raptor_extra_options']):
self.test = self.config['raptor_extra_options'][test_name_index]
else:
self.fatal("Test name not provided")
else:
# raptor initiated in production via mozharness
self.test = self.config['test']
# Action methods. {{{1
# clobber defined in BaseScript

4
testing/raptor/mach_commands.py
View File

@ -49,7 +49,7 @@ class RaptorRunner(MozbuildObject):
self.raptor_args = raptor_args
def make_config(self):
default_actions = ['populate-webroot', 'create-virtualenv', 'run-tests']
default_actions = ['populate-webroot', 'install-chrome', 'create-virtualenv', 'run-tests']
self.config = {
'run_local': True,
'binary_path': self.binary_path,
@ -66,7 +66,7 @@ class RaptorRunner(MozbuildObject):
},
'title': socket.gethostname(),
'default_actions': default_actions,
'raptor_extra_options': self.raptor_args,
'raptor_cmd_line_args': self.raptor_args,
'python3_manifest': {
'win32': 'python3.manifest',
'win64': 'python3_x64.manifest',

14
testing/raptor/raptor/cmdline.py
View File

@ -15,13 +15,13 @@ def create_parser(mach_interface=False):
add_arg('-t', '--test', required=True, dest='test',
help="name of raptor test to run")
add_arg('--app', default='firefox', dest='app',
help="name of the application we are testing (default: firefox)",
choices=['firefox', 'chrome'])
add_arg('-b', '--binary', dest='binary',
help="path to the browser executable that we are testing")
if not mach_interface:
add_arg('--app', default='firefox', dest='app',
help="name of the application we are testing (default: firefox)",
choices=['firefox', 'chrome'])
add_arg('-b', '--binary', required=True, dest='binary',
help="path to the browser executable that we are testing")
add_arg('--branchName', dest="branch_name", default=None,
add_arg('--branchName', dest="branch_name", default='',
help="Name of the branch we are testing on")
add_arg('--symbolsPath', dest='symbols_path',
help="Path to the symbols for the build we are testing")
@ -36,6 +36,8 @@ def create_parser(mach_interface=False):
def verify_options(parser, args):
ctx = vars(args)
if args.binary is None:
parser.error("--binary is required!")
if not os.path.isfile(args.binary):
parser.error("{binary} does not exist!".format(**ctx))

2
testing/raptor/raptor/raptor.ini
View File

@ -1,3 +1,3 @@
# raptor tests
[include:tests/raptor-firefox-tp6.ini]
[include:tests/raptor-tp6.ini]
[include:tests/raptor-speedometer.ini]

12
testing/raptor/raptor/raptor.py
View File

@ -120,7 +120,9 @@ class Raptor(object):
raptor_webext = os.path.join(webext_dir, 'raptor')
self.log.info("installing webext %s" % raptor_webext)
self.profile.addons.install(raptor_webext)
webext_id = self.profile.addons.addon_details(raptor_webext)['id']
# on firefox we can get an addon id; chrome addon actually is just cmd line arg
if self.config['app'] == "firefox":
webext_id = self.profile.addons.addon_details(raptor_webext)['id']
# some tests require tools to playback the test pages
if test.get('playback', None) is not None:
@ -146,8 +148,10 @@ class Raptor(object):
self.playback.stop()
# remove the raptor webext; as it must be reloaded with each subtest anyway
self.log.info("removing webext %s" % raptor_webext)
self.profile.addons.remove_addon(webext_id)
# applies to firefox only; chrome the addon is actually just cmd line arg
if self.config['app'] == "firefox":
self.log.info("removing webext %s" % raptor_webext)
self.profile.addons.remove_addon(webext_id)
if self.runner.is_running():
self.log("Application timed out after {} seconds".format(timeout))
@ -179,6 +183,8 @@ def main(args=sys.argv[1:]):
commandline.setup_logging('raptor', args, {'tbpl': sys.stdout})
LOG = get_default_logger(component='raptor-main')
LOG.info("received command line arguments: %s" % str(args))
# if a test name specified on command line, and it exists, just run that one
# otherwise run all available raptor tests that are found for this browser
raptor_test_list = get_raptor_test_list(args)

11
testing/raptor/raptor/tests/raptor-speedometer.ini
View File

@ -4,12 +4,17 @@
# speedometer benchmark for firefox and chrome
[raptor-speedometer]
apps = firefox
[DEFAULT]
type = benchmark
test_url = http://localhost:<port>/Speedometer/index.html?raptor
page_cycles = 5
page_cycles = 1
page_timeout = 120000
unit = score
lower_is_better = false
alert_threshold = 2.0
[raptor-speedometer-firefox]
apps = firefox
[raptor-speedometer-chrome]
apps = chrome

32
testing/raptor/raptor/tests/raptor-firefox-tp6.ini → testing/raptor/raptor/tests/raptor-tp6.ini
View File

@ -2,10 +2,9 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
# raptor tp6 on firefox
# raptor tp6
[DEFAULT]
apps = firefox
type = pageload
playback = mitmproxy
playback_binary_manifest = mitmproxy-rel-bin-osx.manifest
@ -18,22 +17,51 @@ lower_is_better = true
alert_threshold = 2.0
[raptor-firefox-tp6-amazon]
apps = firefox
test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
playback_recordings = mitmproxy-recording-amazon.mp
measure = fnbpaint
[raptor-firefox-tp6-facebook]
apps = firefox
test_url = https://www.facebook.com
playback_recordings = mitmproxy-recording-facebook.mp
measure = fnbpaint
[raptor-firefox-tp6-google]
apps = firefox
test_url = https://www.google.com/#hl=en&q=barack+obama
playback_recordings = mitmproxy-recording-google.mp
measure = fnbpaint, hero
hero = hero
[raptor-firefox-tp6-youtube]
apps = firefox
test_url = https://www.youtube.com
playback_recordings = mitmproxy-recording-youtube.mp
measure = fnbpaint
[raptor-chrome-tp6-amazon]
apps = chrome
test_url = https://www.amazon.com/s/url=search-alias%3Daps&field-keywords=laptop
playback_recordings = mitmproxy-recording-amazon.mp
measure = fcp
[raptor-chrome-tp6-facebook]
apps = chrome
test_url = https://www.facebook.com
playback_recordings = mitmproxy-recording-facebook.mp
measure = fcp
[raptor-chrome-tp6-google]
apps = chrome
test_url = https://www.google.com/#hl=en&q=barack+obama
playback_recordings = mitmproxy-recording-google.mp
measure = fcp, hero
hero = hero
[raptor-chrome-tp6-youtube]
apps = chrome
test_url = https://www.youtube.com
playback_recordings = mitmproxy-recording-youtube.mp
measure = fcp

1
testing/raptor/webext/raptor/manifest.json
View File

@ -16,6 +16,7 @@
"matches": ["*://*.amazon.com/*",
"*://*.facebook.com/*",
"*://*.google.com/*",
"*://*.google.ca/*",
"*://*.youtube.com/*"],
"js": ["measure.js"]
},