Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-12 12:55:46 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 952530 - List all skipped/disabled tests rather than just the conditionally skipped tests. r=jgriffin

Browse Source
This commit is contained in:
Dave Hunt 2014-01-28 15:28:56 -08:00
parent a9430995cc
commit 2ee61397d4
2 changed files with 85 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
testing/marionette/client/marionette/runner/base.py
View File

@ -532,6 +532,7 @@ class BaseMarionetteTestRunner(object):
self.shuffle = shuffle
self.sdcard = sdcard
self.mixin_run_tests = []
self.manifest_skipped_tests = []
if testvars:
if not os.path.exists(testvars):
@ -806,22 +807,30 @@ class BaseMarionetteTestRunner(object):
manifest = TestManifest()
manifest.read(filepath)
all_tests = manifest.active_tests(exists=False, disabled=False)
manifest_tests = manifest.active_tests(exists=False,
disabled=False,
disabled=True,
device=self.device,
app=self.appName,
**mozinfo.info)
skip_tests = list(set([x['path'] for x in all_tests]) -
set([x['path'] for x in manifest_tests]))
for skipped in skip_tests:
self.logger.info('TEST-SKIP | %s | device=%s, app=%s' %
(os.path.basename(skipped),
self.device,
self.appName))
unfiltered_tests = []
for test in manifest_tests:
if test.get('disabled'):
self.manifest_skipped_tests.append(test)
else:
unfiltered_tests.append(test)
target_tests = manifest.get(tests=unfiltered_tests, **testargs)
for test in unfiltered_tests:
if test['path'] not in [x['path'] for x in target_tests]:
test.setdefault('disabled', 'filtered by type (%s)' % self.type)
self.manifest_skipped_tests.append(test)
for test in self.manifest_skipped_tests:
self.logger.info('TEST-SKIP | %s | %s' % (
os.path.basename(test['path']),
test['disabled']))
self.todo += 1
target_tests = manifest.get(tests=manifest_tests, **testargs)
if self.shuffle:
random.shuffle(target_tests)
for i in target_tests:
@ -874,17 +883,32 @@ class BaseMarionetteTestRunner(object):
def generate_xml(self, results_list):
def _extract_xml(test, result='passed'):
def _extract_xml_from_result(test_result, result='passed'):
_extract_xml(
test_name=unicode(test_result.name).split()[0],
test_class=test_result.test_class,
duration=test_result.duration,
result=result,
output='\n'.join(test_result.output))
def _extract_xml_from_skipped_manifest_test(test):
_extract_xml(
test_name=test['name'],
result='skipped',
output=test['disabled'])
def _extract_xml(test_name, test_class='', duration=0,
result='passed', output=''):
testcase = doc.createElement('testcase')
testcase.setAttribute('classname', test.test_class)
testcase.setAttribute('name', unicode(test.name).split()[0])
testcase.setAttribute('time', str(test.duration))
testcase.setAttribute('classname', test_class)
testcase.setAttribute('name', test_name)
testcase.setAttribute('time', str(duration))
testsuite.appendChild(testcase)
if result in ['failure', 'error', 'skipped']:
f = doc.createElement(result)
f.setAttribute('message', 'test %s' % result)
f.appendChild(doc.createTextNode(test.reason))
f.appendChild(doc.createTextNode(output))
testcase.appendChild(f)
doc = dom.Document()
@ -905,33 +929,38 @@ class BaseMarionetteTestRunner(object):
for results in results_list])))
testsuite.setAttribute('errors', str(sum([len(results.errors)
for results in results_list])))
testsuite.setAttribute('skips', str(sum([len(results.skipped) +
len(results.expectedFailures)
for results in results_list])))
testsuite.setAttribute(
'skips', str(sum([len(results.skipped) +
len(results.expectedFailures)
for results in results_list]) +
len(self.manifest_skipped_tests)))
for results in results_list:
for result in results.errors:
_extract_xml(result, result='error')
_extract_xml_from_result(result, result='error')
for result in results.failures:
_extract_xml(result, result='failure')
_extract_xml_from_result(result, result='failure')
if hasattr(results, 'unexpectedSuccesses'):
for test in results.unexpectedSuccesses:
# unexpectedSuccesses is a list of Testcases only, no tuples
_extract_xml(test, result='failure')
_extract_xml_from_result(test, result='failure')
if hasattr(results, 'skipped'):
for result in results.skipped:
_extract_xml(result, result='skipped')
_extract_xml_from_result(result, result='skipped')
if hasattr(results, 'expectedFailures'):
for result in results.expectedFailures:
_extract_xml(result, result='skipped')
_extract_xml_from_result(result, result='skipped')
for result in results.tests_passed:
_extract_xml(result)
_extract_xml_from_result(result)
for test in self.manifest_skipped_tests:
_extract_xml_from_skipped_manifest_test(test)
doc.appendChild(testsuite)
return doc.toprettyxml(encoding='utf-8')

45
testing/marionette/client/marionette/runner/mixins/reporting.py
View File

@ -44,15 +44,31 @@ class HTMLReportingTestRunnerMixin(object):
tests = sum([results.testsRun for results in results_list])
failures = sum([len(results.failures) for results in results_list])
expected_failures = sum([len(results.expectedFailures) for results in results_list])
skips = sum([len(results.skipped) for results in results_list])
skips = sum([len(results.skipped) for results in results_list]) + len(self.manifest_skipped_tests)
errors = sum([len(results.errors) for results in results_list])
passes = sum([results.passed for results in results_list])
unexpected_passes = sum([len(results.unexpectedSuccesses) for results in results_list])
test_time = self.elapsedtime.total_seconds()
test_logs = []
def _extract_html(test):
def _extract_html_from_result(result):
_extract_html(
result=result.result,
test_name=result.name,
test_class=result.test_class,
debug=result.debug,
output='\n'.join(result.output))
def _extract_html_from_skipped_manifest_test(test):
_extract_html(
result='skipped',
test_name=test['name'],
output=test.get('disabled'))
def _extract_html(result, test_name, test_class='', duration=0,
debug=None, output=''):
additional_html = []
debug = debug or {}
links_html = []
result_map = {
@ -61,13 +77,13 @@ class HTMLReportingTestRunnerMixin(object):
'UNEXPECTED-FAIL': 'failure',
'UNEXPECTED-PASS': 'unexpected pass'}
if test.result in ['SKIPPED', 'UNEXPECTED-FAIL', 'KNOWN-FAIL', 'ERROR']:
if test.debug.get('screenshot'):
screenshot = 'data:image/png;base64,%s' % test.debug['screenshot']
if result.upper() in ['SKIPPED', 'UNEXPECTED-FAIL', 'KNOWN-FAIL', 'ERROR']:
if debug.get('screenshot'):
screenshot = 'data:image/png;base64,%s' % debug['screenshot']
additional_html.append(html.div(
html.a(html.img(src=screenshot), href="#"),
class_='screenshot'))
for name, content in test.debug.items():
for name, content in debug.items():
try:
if 'screenshot' in name:
href = '#'
@ -86,7 +102,7 @@ class HTMLReportingTestRunnerMixin(object):
pass
log = html.div(class_='log')
for line in '\n'.join(test.output).splitlines():
for line in output.splitlines():
separator = line.startswith(' ' * 10)
if separator:
log.append(line[:80])
@ -99,16 +115,19 @@ class HTMLReportingTestRunnerMixin(object):
additional_html.append(log)
test_logs.append(html.tr([
html.td(result_map.get(test.result, test.result).title(), class_='col-result'),
html.td(test.test_class, class_='col-class'),
html.td(unicode(test.name), class_='col-name'),
html.td(int(test.duration), class_='col-duration'),
html.td(result_map.get(result, result).title(), class_='col-result'),
html.td(test_class, class_='col-class'),
html.td(test_name, class_='col-name'),
html.td(str(duration), class_='col-duration'),
html.td(links_html, class_='col-links'),
html.td(additional_html, class_='debug')],
class_=result_map.get(test.result, test.result).lower() + ' results-table-row'))
class_=result_map.get(result, result).lower() + ' results-table-row'))
for results in results_list:
[_extract_html(test) for test in results.tests]
[_extract_html_from_result(test) for test in results.tests]
for test in self.manifest_skipped_tests:
_extract_html_from_skipped_manifest_test(test)
generated = datetime.datetime.now()
doc = html.html(