From 2ee61397d40dabbda8528bfb4c15b28f4226a7ed Mon Sep 17 00:00:00 2001 From: Dave Hunt Date: Tue, 28 Jan 2014 15:28:56 -0800 Subject: [PATCH] Bug 952530 - List all skipped/disabled tests rather than just the conditionally skipped tests. r=jgriffin --- .../client/marionette/runner/base.py | 77 +++++++++++++------ .../marionette/runner/mixins/reporting.py | 45 +++++++---- 2 files changed, 85 insertions(+), 37 deletions(-) diff --git a/testing/marionette/client/marionette/runner/base.py b/testing/marionette/client/marionette/runner/base.py index a4621562c4db..36bc7ab141f4 100644 --- a/testing/marionette/client/marionette/runner/base.py +++ b/testing/marionette/client/marionette/runner/base.py @@ -532,6 +532,7 @@ class BaseMarionetteTestRunner(object): self.shuffle = shuffle self.sdcard = sdcard self.mixin_run_tests = [] + self.manifest_skipped_tests = [] if testvars: if not os.path.exists(testvars): @@ -806,22 +807,30 @@ class BaseMarionetteTestRunner(object): manifest = TestManifest() manifest.read(filepath) - all_tests = manifest.active_tests(exists=False, disabled=False) manifest_tests = manifest.active_tests(exists=False, - disabled=False, + disabled=True, device=self.device, app=self.appName, **mozinfo.info) - skip_tests = list(set([x['path'] for x in all_tests]) - - set([x['path'] for x in manifest_tests])) - for skipped in skip_tests: - self.logger.info('TEST-SKIP | %s | device=%s, app=%s' % - (os.path.basename(skipped), - self.device, - self.appName)) + unfiltered_tests = [] + for test in manifest_tests: + if test.get('disabled'): + self.manifest_skipped_tests.append(test) + else: + unfiltered_tests.append(test) + + target_tests = manifest.get(tests=unfiltered_tests, **testargs) + for test in unfiltered_tests: + if test['path'] not in [x['path'] for x in target_tests]: + test.setdefault('disabled', 'filtered by type (%s)' % self.type) + self.manifest_skipped_tests.append(test) + + for test in self.manifest_skipped_tests: + self.logger.info('TEST-SKIP | %s | %s' % ( + os.path.basename(test['path']), + test['disabled'])) self.todo += 1 - target_tests = manifest.get(tests=manifest_tests, **testargs) if self.shuffle: random.shuffle(target_tests) for i in target_tests: @@ -874,17 +883,32 @@ class BaseMarionetteTestRunner(object): def generate_xml(self, results_list): - def _extract_xml(test, result='passed'): + def _extract_xml_from_result(test_result, result='passed'): + _extract_xml( + test_name=unicode(test_result.name).split()[0], + test_class=test_result.test_class, + duration=test_result.duration, + result=result, + output='\n'.join(test_result.output)) + + def _extract_xml_from_skipped_manifest_test(test): + _extract_xml( + test_name=test['name'], + result='skipped', + output=test['disabled']) + + def _extract_xml(test_name, test_class='', duration=0, + result='passed', output=''): testcase = doc.createElement('testcase') - testcase.setAttribute('classname', test.test_class) - testcase.setAttribute('name', unicode(test.name).split()[0]) - testcase.setAttribute('time', str(test.duration)) + testcase.setAttribute('classname', test_class) + testcase.setAttribute('name', test_name) + testcase.setAttribute('time', str(duration)) testsuite.appendChild(testcase) if result in ['failure', 'error', 'skipped']: f = doc.createElement(result) f.setAttribute('message', 'test %s' % result) - f.appendChild(doc.createTextNode(test.reason)) + f.appendChild(doc.createTextNode(output)) testcase.appendChild(f) doc = dom.Document() @@ -905,33 +929,38 @@ class BaseMarionetteTestRunner(object): for results in results_list]))) testsuite.setAttribute('errors', str(sum([len(results.errors) for results in results_list]))) - testsuite.setAttribute('skips', str(sum([len(results.skipped) + - len(results.expectedFailures) - for results in results_list]))) + testsuite.setAttribute( + 'skips', str(sum([len(results.skipped) + + len(results.expectedFailures) + for results in results_list]) + + len(self.manifest_skipped_tests))) for results in results_list: for result in results.errors: - _extract_xml(result, result='error') + _extract_xml_from_result(result, result='error') for result in results.failures: - _extract_xml(result, result='failure') + _extract_xml_from_result(result, result='failure') if hasattr(results, 'unexpectedSuccesses'): for test in results.unexpectedSuccesses: # unexpectedSuccesses is a list of Testcases only, no tuples - _extract_xml(test, result='failure') + _extract_xml_from_result(test, result='failure') if hasattr(results, 'skipped'): for result in results.skipped: - _extract_xml(result, result='skipped') + _extract_xml_from_result(result, result='skipped') if hasattr(results, 'expectedFailures'): for result in results.expectedFailures: - _extract_xml(result, result='skipped') + _extract_xml_from_result(result, result='skipped') for result in results.tests_passed: - _extract_xml(result) + _extract_xml_from_result(result) + + for test in self.manifest_skipped_tests: + _extract_xml_from_skipped_manifest_test(test) doc.appendChild(testsuite) return doc.toprettyxml(encoding='utf-8') diff --git a/testing/marionette/client/marionette/runner/mixins/reporting.py b/testing/marionette/client/marionette/runner/mixins/reporting.py index 8d965553b928..cc91bb57bba9 100644 --- a/testing/marionette/client/marionette/runner/mixins/reporting.py +++ b/testing/marionette/client/marionette/runner/mixins/reporting.py @@ -44,15 +44,31 @@ class HTMLReportingTestRunnerMixin(object): tests = sum([results.testsRun for results in results_list]) failures = sum([len(results.failures) for results in results_list]) expected_failures = sum([len(results.expectedFailures) for results in results_list]) - skips = sum([len(results.skipped) for results in results_list]) + skips = sum([len(results.skipped) for results in results_list]) + len(self.manifest_skipped_tests) errors = sum([len(results.errors) for results in results_list]) passes = sum([results.passed for results in results_list]) unexpected_passes = sum([len(results.unexpectedSuccesses) for results in results_list]) test_time = self.elapsedtime.total_seconds() test_logs = [] - def _extract_html(test): + def _extract_html_from_result(result): + _extract_html( + result=result.result, + test_name=result.name, + test_class=result.test_class, + debug=result.debug, + output='\n'.join(result.output)) + + def _extract_html_from_skipped_manifest_test(test): + _extract_html( + result='skipped', + test_name=test['name'], + output=test.get('disabled')) + + def _extract_html(result, test_name, test_class='', duration=0, + debug=None, output=''): additional_html = [] + debug = debug or {} links_html = [] result_map = { @@ -61,13 +77,13 @@ class HTMLReportingTestRunnerMixin(object): 'UNEXPECTED-FAIL': 'failure', 'UNEXPECTED-PASS': 'unexpected pass'} - if test.result in ['SKIPPED', 'UNEXPECTED-FAIL', 'KNOWN-FAIL', 'ERROR']: - if test.debug.get('screenshot'): - screenshot = 'data:image/png;base64,%s' % test.debug['screenshot'] + if result.upper() in ['SKIPPED', 'UNEXPECTED-FAIL', 'KNOWN-FAIL', 'ERROR']: + if debug.get('screenshot'): + screenshot = 'data:image/png;base64,%s' % debug['screenshot'] additional_html.append(html.div( html.a(html.img(src=screenshot), href="#"), class_='screenshot')) - for name, content in test.debug.items(): + for name, content in debug.items(): try: if 'screenshot' in name: href = '#' @@ -86,7 +102,7 @@ class HTMLReportingTestRunnerMixin(object): pass log = html.div(class_='log') - for line in '\n'.join(test.output).splitlines(): + for line in output.splitlines(): separator = line.startswith(' ' * 10) if separator: log.append(line[:80]) @@ -99,16 +115,19 @@ class HTMLReportingTestRunnerMixin(object): additional_html.append(log) test_logs.append(html.tr([ - html.td(result_map.get(test.result, test.result).title(), class_='col-result'), - html.td(test.test_class, class_='col-class'), - html.td(unicode(test.name), class_='col-name'), - html.td(int(test.duration), class_='col-duration'), + html.td(result_map.get(result, result).title(), class_='col-result'), + html.td(test_class, class_='col-class'), + html.td(test_name, class_='col-name'), + html.td(str(duration), class_='col-duration'), html.td(links_html, class_='col-links'), html.td(additional_html, class_='debug')], - class_=result_map.get(test.result, test.result).lower() + ' results-table-row')) + class_=result_map.get(result, result).lower() + ' results-table-row')) for results in results_list: - [_extract_html(test) for test in results.tests] + [_extract_html_from_result(test) for test in results.tests] + + for test in self.manifest_skipped_tests: + _extract_html_from_skipped_manifest_test(test) generated = datetime.datetime.now() doc = html.html(