Bug 1375073 - Modify talos startup test framework to support receiving multiple values from a single iteration; r=jmaher

MozReview-Commit-ID: EUGt71dB6f6

--HG--
extra : rebase_source : 81e1139a7556f17dc5e26f649305a1ede9a976bf

testing/talos/talos/cmdline.py

@@ -80,6 +80,10 @@ def create_parser(mach_interface=False):
                  " an environment variable")
     add_arg("--mozAfterPaint", action='store_true', dest="tpmozafterpaint",
             help="wait for MozAfterPaint event before recording the time")
+    add_arg("--firstPaint", action='store_true', dest="firstpaint",
+            help="Also report the first paint value in supported tests")
+    add_arg("--userReady", action='store_true', dest="userready",
+            help="Also report the user ready value in supported tests")
     add_arg('--spsProfile', action="store_true", dest="gecko_profile",
             help="(Deprecated - Use --geckoProfile instead.) Profile the "
                  "run and output the results in $MOZ_UPLOAD_DIR.")

testing/talos/talos/config.py

@@ -39,6 +39,9 @@ DEFAULTS = dict(
         tpchrome=True,
         tpcycles=10,
         tpmozafterpaint=False,
+        firstpaint=False,
+        userready=False,
+        testeventmap=[],
         tpdisable_e10s=False,
         tpnoisy=True,
         tppagecycles=1,
@@ -201,6 +204,8 @@ GLOBAL_OVERRIDES = (
     'tpmanifest',
     'tptimeout',
     'tpmozafterpaint',
+    'firstpaint',
+    'userready',
 )
 
 
@@ -354,6 +359,8 @@ def build_manifest(config, manifestName):
 
 def get_test(config, global_overrides, counters, test_instance):
     mozAfterPaint = getattr(test_instance, 'tpmozafterpaint', None)
+    firstPaint = getattr(test_instance, 'firstpaint', None)
+    userReady = getattr(test_instance, 'userready', None)
 
     test_instance.update(**global_overrides)
 
@@ -361,6 +368,10 @@ def get_test(config, global_overrides, counters, test_instance):
     # so check for None
     if mozAfterPaint is not None:
         test_instance.tpmozafterpaint = mozAfterPaint
+    if firstPaint is not None:
+        test_instance.firstpaint = firstPaint
+    if userReady is not None:
+        test_instance.userready = userReady
 
     # fix up url
     url = getattr(test_instance, 'url', None)

testing/talos/talos/results.py

@@ -34,7 +34,6 @@ class TalosResults(object):
         output all results to appropriate URLs
         - output_formats: a dict mapping formats to a list of URLs
         """
-
         tbpl_output = {}
         try:
 
@@ -174,22 +173,33 @@ class TsResults(Results):
         self.results = []
         index = 0
 
-        # Handle the case where we support a pagename in the results
-        # (new format)
-        for line in lines:
-            result = {}
-            r = line.strip().split(',')
-            r = [i for i in r if i]
-            if len(r) <= 1:
-                continue
+        # Case where one test iteration may report multiple event values i.e. ts_paint
+        if string.startswith('{'):
+            jsonResult = json.loads(string)
+            result = {'runs': {}}
             result['index'] = index
-            result['page'] = r[0]
-            # note: if we have len(r) >1, then we have pagename,raw_results
-            result['runs'] = [float(i) for i in r[1:]]
-            self.results.append(result)
-            index += 1
+            result['page'] = 'NULL'
 
-        # The original case where we just have numbers and no pagename
+            for event_label in jsonResult:
+                result['runs'][str(event_label)] = [jsonResult[event_label]]
+            self.results.append(result)
+
+        # Case where we support a pagename in the results
+        if not self.results:
+            for line in lines:
+                result = {}
+                r = line.strip().split(',')
+                r = [i for i in r if i]
+                if len(r) <= 1:
+                    continue
+                result['index'] = index
+                result['page'] = r[0]
+                # note: if we have len(r) >1, then we have pagename,raw_results
+                result['runs'] = [float(i) for i in r[1:]]
+                self.results.append(result)
+                index += 1
+
+        # Original case where we just have numbers and no pagename
         if not self.results:
             result = {}
             result['index'] = index

testing/talos/talos/run_tests.py

@@ -4,6 +4,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import copy
 import mozversion
 import os
 import sys
@@ -152,8 +153,8 @@ def run_tests(config, browser_config):
         browser_config['sourcestamp'] = version_info['application_changeset']
     except KeyError:
         if not browser_config['develop']:
-            print("unable to find changeset or repository: %s" % version_info)
-            sys.exit()
+            print("Abort: unable to find changeset or repository: %s" % version_info)
+            sys.exit(1)
         else:
             browser_config['repository'] = 'develop'
             browser_config['sourcestamp'] = 'develop'
@@ -201,8 +202,7 @@ def run_tests(config, browser_config):
         mitmdump_path = config.get('mitmdumpPath', False)
         if mitmdump_path is False:
             # cannot continue, need path for mitmdump playback tool
-            LOG.error('Aborting: mitmdumpPath was not provided on cmd line but is required')
-            sys.exit()
+            raise TalosError('Aborting: mitmdumpPath not provided on cmd line but is required')
 
         mitmproxy_recording_path = os.path.join(here, 'mitmproxy')
         mitmproxy_proc = mitmproxy.start_mitmproxy_playback(mitmdump_path,
@@ -232,7 +232,32 @@ def run_tests(config, browser_config):
             LOG.test_start(testname)
 
             mytest = TTest()
-            talos_results.add(mytest.runTest(browser_config, test))
+
+            # some tests like ts_paint return multiple results in a single iteration
+            if test.get('firstpaint', False) or test.get('userready', None):
+                # we need a 'testeventmap' to tell us which tests each event should map to
+                multi_value_result = None
+                separate_results_list = []
+
+                test_event_map = test.get('testeventmap', None)
+                if test_event_map is None:
+                    raise TalosError("Need 'testeventmap' in test.py for %s" % test.get('name'))
+
+                # run the test
+                multi_value_result = mytest.runTest(browser_config, test)
+                if multi_value_result is None:
+                    raise TalosError("Abort: no results returned for %s" % test.get('name'))
+
+                # parse out the multi-value results, and 'fake it' to appear like separate tests
+                separate_results_list = convert_to_separate_test_results(multi_value_result,
+                                                                         test_event_map)
+
+                # now we have three separate test results, store them
+                for test_result in separate_results_list:
+                    talos_results.add(test_result)
+            else:
+                # just expecting regular test - one result value per iteration
+                talos_results.add(mytest.runTest(browser_config, test))
 
             LOG.test_end(testname, status='OK')
 
@@ -273,6 +298,44 @@ def run_tests(config, browser_config):
     return 0
 
 
+def convert_to_separate_test_results(multi_value_result, test_event_map):
+    ''' Receive a test result that actually contains multiple values in a single iteration, and
+    parse it out in order to 'fake' three seprate test results.
+
+    Incoming result looks like this:
+
+    [{'index': 0, 'runs': {'event_1': [1338, ...], 'event_2': [1438, ...], 'event_3':
+    [1538, ...]}, 'page': 'NULL'}]
+
+    We want to parse it out such that we have 'faked' three separate tests, setting test names
+    and taking the run values for each. End goal is to have results reported as three separate
+    tests, like this:
+
+    PERFHERDER_DATA: {"framework": {"name": "talos"}, "suites": [{"subtests": [{"replicates":
+    [1338, ...], "name": "ts_paint", "value": 1338}], "extraOptions": ["e10s"], "name":
+    "ts_paint"}, {"subtests": [{"replicates": [1438, ...], "name": "ts_first_paint", "value":
+    1438}], "extraOptions": ["e10s"], "name": "ts_first_paint"}, {"subtests": [{"replicates":
+    [1538, ...], "name": "ts_user_ready", "value": 1538}], "extraOptions": ["e10s"], "name":
+    "ts_user_ready"}]}
+    '''
+    list_of_separate_tests = []
+
+    for next_test in test_event_map:
+        # copy the original test result that has multiple values per iteration
+        separate_test = copy.deepcopy(multi_value_result)
+        # set the name of the new 'faked' test
+        separate_test.test_config['name'] = next_test['name']
+        # set the run values for the new test
+        for x in separate_test.results:
+            for item in x.results:
+                all_runs = item['runs']
+                item['runs'] = all_runs[next_test['label']]
+        # add it to our list of results to return
+        list_of_separate_tests.append(separate_test)
+
+    return list_of_separate_tests
+
+
 def main(args=sys.argv[1:]):
     try:
         config, browser_config = get_configs()

testing/talos/talos/test.py

@@ -104,6 +104,9 @@ class TsBase(Test):
         'xperf_user_providers',
         'xperf_stackwalk',
         'tpmozafterpaint',
+        'firstpaint',
+        'userready',
+        'testeventmap',
         'extensions',
         'filters',
         'setup',
@@ -235,12 +238,12 @@ class PageloaderTest(Test):
     cycles = None
     timeout = None
     keys = ['tpmanifest', 'tpcycles', 'tppagecycles', 'tprender', 'tpchrome',
-            'tpmozafterpaint', 'tploadnocache', 'rss', 'mainthread',
-            'resolution', 'cycles', 'gecko_profile', 'gecko_profile_interval',
-            'gecko_profile_entries', 'tptimeout', 'win_counters', 'w7_counters',
-            'linux_counters', 'mac_counters', 'tpscrolltest', 'xperf_counters',
-            'timeout', 'shutdown', 'responsiveness', 'profile_path',
-            'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
+            'tpmozafterpaint', 'tploadnocache', 'firstpaint', 'userready',
+            'testeventmap', 'rss', 'mainthread', 'resolution', 'cycles',
+            'gecko_profile', 'gecko_profile_interval', 'gecko_profile_entries',
+            'tptimeout', 'win_counters', 'w7_counters', 'linux_counters', 'mac_counters',
+            'tpscrolltest', 'xperf_counters', 'timeout', 'shutdown', 'responsiveness',
+            'profile_path', 'xperf_providers', 'xperf_user_providers', 'xperf_stackwalk',
             'filters', 'preferences', 'extensions', 'setup', 'cleanup',
             'lower_is_better', 'alert_threshold', 'unit', 'webextensions']