From 0fa9f19bdf405a3b4a88d78da04d69413194adf2 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Wed, 15 Feb 2017 16:38:45 -0500 Subject: [PATCH] Bug 1335873 - Convert marionette harness unittests to standard python unittests, r=maja_zf This formats the marionette-harness python tests to be a regular |mach python-test| suite. Though we add subsuite=marionette, this is just for automation purposes. The new preferred way to run the marionette harness tests locally is: ./mach python-test testing/marionette They will also run if running the full suite. The mozbase packages.txt file modifies mozlog to use 'setup.py' instead of 'pth'. The reason for this is that the marionette-harness tests use the pytest_mozlog pytest plugin for formatting their results (converts pytest format into something resembling the standard tbpl logging format). In order for this plugin to get picked up however, mozlog's setup.py file needs to be processed. MozReview-Commit-ID: Ata99evHxbd --HG-- extra : rebase_source : 16ed70edd38a53c3279d8632d7cba3df4d5216c3 --- python/mach_commands.py | 37 ++--- taskcluster/ci/marionette-harness/kind.yml | 50 ------- taskcluster/ci/source-test/python-tests.yml | 29 ++++ taskcluster/docs/kinds.rst | 5 - .../scripts/tester/harness-test-linux.sh | 40 ----- taskcluster/taskgraph/try_option_syntax.py | 1 - .../marionette_harness_test_requirements.txt | 13 -- .../tests/harness_unit/python.ini | 9 ++ .../tests/harness_unit/test_httpd.py | 3 +- .../harness_unit/test_marionette_arguments.py | 3 +- .../harness_unit/test_marionette_harness.py | 3 +- .../harness_unit/test_marionette_runner.py | 3 +- .../test_marionette_test_result.py | 3 +- .../tests/harness_unit/test_serve.py | 3 +- testing/mozbase/packages.txt | 2 +- .../scripts/marionette_harness_tests.py | 141 ------------------ 16 files changed, 61 insertions(+), 284 deletions(-) delete mode 100644 taskcluster/ci/marionette-harness/kind.yml delete mode 100644 taskcluster/scripts/tester/harness-test-linux.sh delete mode 100644 testing/config/marionette_harness_test_requirements.txt create mode 100644 testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini delete mode 100644 testing/mozharness/scripts/marionette_harness_tests.py diff --git a/python/mach_commands.py b/python/mach_commands.py index ba162df228c2..f252b2a4ec17 100644 --- a/python/mach_commands.py +++ b/python/mach_commands.py @@ -57,11 +57,6 @@ class MachCommands(MachCommandBase): default=False, action='store_true', help='Stop running tests after the first error or failure.') - @CommandArgument('--path-only', - default=False, - action='store_true', - help=('Collect all tests under given path instead of default ' - 'test resolution. Supports pytest-style tests.')) @CommandArgument('-j', '--jobs', default=1, type=int, @@ -79,7 +74,6 @@ class MachCommands(MachCommandBase): test_objects=None, subsuite=None, verbose=False, - path_only=False, stop=False, jobs=1): self._activate_virtualenv() @@ -109,30 +103,19 @@ class MachCommands(MachCommandBase): # which produces output in the format Mozilla infrastructure expects. # Some tests are run via pytest. if test_objects is None: - # If we're not being called from `mach test`, do our own - # test resolution. - if path_only: - if tests: - test_objects = [{'path': p} for p in find_tests_by_path()] - else: - self.log(logging.WARN, 'python-test', {}, - 'TEST-UNEXPECTED-FAIL | No tests specified') - test_objects = [] + from mozbuild.testing import TestResolver + resolver = self._spawn(TestResolver) + if tests: + # If we were given test paths, try to find tests matching them. + test_objects = resolver.resolve_tests(paths=tests, + flavor='python') else: - from mozbuild.testing import TestResolver - resolver = self._spawn(TestResolver) - if tests: - # If we were given test paths, try to find tests matching them. - test_objects = resolver.resolve_tests(paths=tests, - flavor='python') - else: - # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS - test_objects = resolver.resolve_tests(flavor='python') + # Otherwise just run everything in PYTHON_UNITTEST_MANIFESTS + test_objects = resolver.resolve_tests(flavor='python') if not test_objects: - message = 'TEST-UNEXPECTED-FAIL | No tests collected' - if not path_only: - message += ' (Not in PYTHON_UNITTEST_MANIFESTS? Try --path-only?)' + message = 'TEST-UNEXPECTED-FAIL | No tests collected ' + \ + '(Not in PYTHON_UNITTEST_MANIFESTS?)' self.log(logging.WARN, 'python-test', {}, message) return 1 diff --git a/taskcluster/ci/marionette-harness/kind.yml b/taskcluster/ci/marionette-harness/kind.yml deleted file mode 100644 index 073e21a3366c..000000000000 --- a/taskcluster/ci/marionette-harness/kind.yml +++ /dev/null @@ -1,50 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -# NOTE: please write a description of this kind in taskcluster/docs/kinds.rst - -implementation: taskgraph.task.transform:TransformTask - -transforms: - - taskgraph.transforms.marionette_harness:transforms - - taskgraph.transforms.task:transforms - -# NOTE: this task should be refactored so that it is invoked as a job either -# with a run.using of "mozharness", and combined with the source-check kind. - -jobs: - marionette-harness/opt: - description: "Marionette harness unit test" - attributes: - build_platform: marionette-harness - build_type: opt - treeherder: - platform: linux64/opt - kind: test - tier: 2 - symbol: tc(Mn-h) - worker-type: aws-provisioner-v1/gecko-t-linux-xlarge - worker: - implementation: docker-worker - docker-image: {in-tree: desktop-build} # NOTE: better to use the lint image - env: - JOB_SCRIPT: "taskcluster/scripts/tester/harness-test-linux.sh" - MOZHARNESS_SCRIPT: "testing/mozharness/scripts/marionette_harness_tests.py" - TOOLS_DISABLE: "true" - artifacts: - - name: public/logs/ - path: /home/worker/workspace/mozharness_workspace/upload/logs/ - type: directory - command: - - "bash" - - "/home/worker/bin/build.sh" - - "--tests=testing/marionette/harness/marionette_harness/tests/harness_unit" - - "--work-dir=mozharness_workspace" - max-run-time: 1800 - when: - files-changed: - - "testing/marionette/harness/**" - - "testing/mozbase/mozlog/mozlog/pytest_mozlog/**" - - "testing/mozharness/scripts/marionette_harness_tests.py" - - "testing/config/marionette_harness_test_requirements.txt" diff --git a/taskcluster/ci/source-test/python-tests.yml b/taskcluster/ci/source-test/python-tests.yml index f930ff5958f5..da86fad29000 100644 --- a/taskcluster/ci/source-test/python-tests.yml +++ b/taskcluster/ci/source-test/python-tests.yml @@ -22,6 +22,35 @@ taskgraph-tests/opt: - 'config/mozunit.py' - 'python/mach/**/*.py' +marionette-harness/opt: + description: testing/marionette/harness unit tests + platforms: + - linux64/opt + treeherder: + symbol: py(mnh) + kind: test + tier: 2 + worker-type: + by-platform: + linux64.*: aws-provisioner-v1/b2gtest + worker: + by-platform: + linux64.*: + implementation: docker-worker + docker-image: {in-tree: "lint"} + max-run-time: 3600 + run: + using: mach + mach: python-test --subsuite marionette-harness + run-on-projects: + - integration + - release + when: + files-changed: + - 'testing/marionette/harness/**' + - 'testing/mozbase/mozlog/mozlog/pytest_mozlog/**' + - 'python/mach_commands.py' + mozbase/opt: description: testing/mozbase unit tests platforms: diff --git a/taskcluster/docs/kinds.rst b/taskcluster/docs/kinds.rst index 2493e457491a..9060f14f27e4 100644 --- a/taskcluster/docs/kinds.rst +++ b/taskcluster/docs/kinds.rst @@ -90,11 +90,6 @@ spidermonkey Spidermonkey tasks check out the full gecko source tree, then compile only the spidermonkey portion. Each task runs specific tests after the build. -marionette-harness ------------------- - -TBD (Maja) - Tests ----- diff --git a/taskcluster/scripts/tester/harness-test-linux.sh b/taskcluster/scripts/tester/harness-test-linux.sh deleted file mode 100644 index b38ffa1241f0..000000000000 --- a/taskcluster/scripts/tester/harness-test-linux.sh +++ /dev/null @@ -1,40 +0,0 @@ -#! /bin/bash -vex - -set -x -e - -echo "running as" $(id) - -#### -# Taskcluster friendly wrapper for running a script in -# testing/mozharness/scripts in a source checkout (no build). -# Example use: Python-only harness unit tests -#### - -: WORKSPACE ${WORKSPACE:=/home/worker/workspace} -: SRC_ROOT ${SRC_ROOT:=$WORKSPACE/build/src} -# These paths should be relative to $SRC_ROOT -: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} -: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} -: mozharness args "${@}" - -set -v -cd $WORKSPACE - -fail() { - echo # make sure error message is on a new line - echo "[harness-test-linux.sh:error]" "${@}" - exit 1 -} - -if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi - -# support multiple, space delimited, config files -config_cmds="" -for cfg in $MOZHARNESS_CONFIG; do - config_cmds="${config_cmds} --config-file ${SRC_ROOT}/${cfg}" -done - -python2.7 $SRC_ROOT/${MOZHARNESS_SCRIPT} ${config_cmds} "${@}" - - - diff --git a/taskcluster/taskgraph/try_option_syntax.py b/taskcluster/taskgraph/try_option_syntax.py index 18f7e2981084..53ca5abfd6ba 100644 --- a/taskcluster/taskgraph/try_option_syntax.py +++ b/taskcluster/taskgraph/try_option_syntax.py @@ -37,7 +37,6 @@ BUILD_KINDS = set([ JOB_KINDS = set([ 'source-test', 'toolchain', - 'marionette-harness', 'android-stuff', ]) diff --git a/testing/config/marionette_harness_test_requirements.txt b/testing/config/marionette_harness_test_requirements.txt deleted file mode 100644 index e75ec7d68d48..000000000000 --- a/testing/config/marionette_harness_test_requirements.txt +++ /dev/null @@ -1,13 +0,0 @@ --r mozbase_requirements.txt - -# TODO - if we structure common.tests.zip to match the in-tree structure of the -# testing directory, we could use ./marionette_requirements.txt instead -../web-platform/tests/tools/wptserve -../marionette/client -../marionette/harness/marionette_harness/runner/mixins/browsermob-proxy-py -../marionette/harness - -# pytest -../../python/py -../../python/pytest -../../python/mock-1.0.0 diff --git a/testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini b/testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini new file mode 100644 index 000000000000..a0864a129ac1 --- /dev/null +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini @@ -0,0 +1,9 @@ +[DEFAULT] +subsuite = marionette-harness + +[test_httpd.py] +[test_marionette_arguments.py] +[test_marionette_harness.py] +[test_marionette_runner.py] +[test_marionette_test_result.py] +[test_serve.py] diff --git a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py index bd86b2fffbc9..40812103cf84 100644 --- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_httpd.py @@ -87,4 +87,5 @@ def test_handler(server): if __name__ == "__main__": import sys - sys.exit(pytest.main(["--verbose", __file__])) + sys.exit(pytest.main( + ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) diff --git a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py index 1a0687028c88..f2b4df4f34fa 100644 --- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_arguments.py @@ -29,4 +29,5 @@ def test_parse_arg_socket_timeout(socket_timeout): if __name__ == '__main__': import sys - sys.exit(pytest.main(['--verbose', __file__])) + sys.exit(pytest.main( + ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) diff --git a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py index dfbbfb788274..1a20ce1dbe5e 100644 --- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_harness.py @@ -105,4 +105,5 @@ def test_harness_sets_up_default_test_handlers(mach_parsed_kwargs): if __name__ == '__main__': import sys - sys.exit(pytest.main(['--verbose', __file__])) + sys.exit(pytest.main( + ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) diff --git a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py index 07f62e8d43d2..5b5933eb9bfb 100644 --- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_runner.py @@ -439,4 +439,5 @@ def test_e10s_option_clash_raises(mock_runner): if __name__ == '__main__': import sys - sys.exit(pytest.main(['--verbose', __file__])) + sys.exit(pytest.main( + ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) diff --git a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py index a69b072cd5f4..86b8bba83222 100644 --- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_marionette_test_result.py @@ -51,4 +51,5 @@ def test_crash_is_recorded_as_error(empty_marionette_test, if __name__ == '__main__': import sys - sys.exit(pytest.main(['--verbose', __file__])) + sys.exit(pytest.main( + ['-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) diff --git a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py index 73684c0d6347..7402482da77b 100644 --- a/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py +++ b/testing/marionette/harness/marionette_harness/tests/harness_unit/test_serve.py @@ -64,4 +64,5 @@ def test_where_is(): if __name__ == "__main__": import sys - sys.exit(pytest.main(["-s", "--verbose", __file__])) + sys.exit(pytest.main( + ['-s', '-p', 'no:terminalreporter', '--log-tbpl=-', __file__])) diff --git a/testing/mozbase/packages.txt b/testing/mozbase/packages.txt index ba1f292ef38f..8259149a7b48 100644 --- a/testing/mozbase/packages.txt +++ b/testing/mozbase/packages.txt @@ -8,7 +8,7 @@ mozhttpd.pth:testing/mozbase/mozhttpd mozinfo.pth:testing/mozbase/mozinfo mozinstall.pth:testing/mozbase/mozinstall mozleak.pth:testing/mozbase/mozleak -mozlog.pth:testing/mozbase/mozlog +setup.py:testing/mozbase/mozlog:develop moznetwork.pth:testing/mozbase/moznetwork mozprocess.pth:testing/mozbase/mozprocess mozprofile.pth:testing/mozbase/mozprofile diff --git a/testing/mozharness/scripts/marionette_harness_tests.py b/testing/mozharness/scripts/marionette_harness_tests.py deleted file mode 100644 index 0811bef9cb37..000000000000 --- a/testing/mozharness/scripts/marionette_harness_tests.py +++ /dev/null @@ -1,141 +0,0 @@ -#!/usr/bin/env python -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this file, -# You can obtain one at http://mozilla.org/MPL/2.0/. -import copy -import os -import sys - -# load modules from parent dir -sys.path.insert(1, os.path.dirname(sys.path[0])) - -from mozharness.base.python import PreScriptAction -from mozharness.base.python import ( - VirtualenvMixin, - virtualenv_config_options, -) -from mozharness.base.script import BaseScript -from mozharness.mozilla.buildbot import ( - BuildbotMixin, TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE, - TBPL_EXCEPTION -) - -marionette_harness_tests_config_options = [ - [['--tests'], { - 'dest': 'test_path', - 'default': None, - 'help': 'Path to test_*.py or directory relative to src root.', - }], - [['--src-dir'], { - 'dest': 'rel_src_dir', - 'default': None, - 'help': 'Path to hg.mo source checkout relative to work dir.', - }], - -] + copy.deepcopy(virtualenv_config_options) - -marionette_harness_tests_config = { - "find_links": [ - "http://pypi.pub.build.mozilla.org/pub", - ], - "pip_index": False, - # relative to workspace - "rel_src_dir": os.path.join("build", "src"), -} - -class MarionetteHarnessTests(VirtualenvMixin, BuildbotMixin, BaseScript): - - def __init__(self, config_options=None, - all_actions=None, default_actions=None, - *args, **kwargs): - config_options = config_options or marionette_harness_tests_config_options - actions = [ - 'clobber', - 'create-virtualenv', - 'run-tests', - ] - super(MarionetteHarnessTests, self).__init__( - config_options=config_options, - all_actions=all_actions or actions, - default_actions=default_actions or actions, - config=marionette_harness_tests_config, - *args, **kwargs) - - @PreScriptAction('create-virtualenv') - def _pre_create_virtualenv(self, action): - dirs = self.query_abs_dirs() - c = self.config - requirements = os.path.join( - dirs['abs_src_dir'], - 'testing', 'config', - 'marionette_harness_test_requirements.txt' - ) - self.register_virtualenv_module( - requirements=[requirements], - two_pass=True - ) - - def query_abs_dirs(self): - if self.abs_dirs: - return self.abs_dirs - c = self.config - abs_dirs = super(MarionetteHarnessTests, self).query_abs_dirs() - dirs = { - 'abs_src_dir': os.path.abspath( - os.path.join(abs_dirs['base_work_dir'], c['rel_src_dir']) - ), - } - - for key in dirs: - if key not in abs_dirs: - abs_dirs[key] = dirs[key] - self.abs_dirs = abs_dirs - - return self.abs_dirs - - def _get_pytest_status(self, code): - """ - Translate pytest exit code to TH status - - Based on https://github.com/pytest-dev/pytest/blob/master/_pytest/main.py#L21-L26 - """ - if code == 0: - return TBPL_SUCCESS - elif code == 1: - return TBPL_WARNING - elif 1 < code < 6: - self.error("pytest returned exit code: %s" % code) - return TBPL_FAILURE - else: - return TBPL_EXCEPTION - - def run_tests(self): - """Run all the tests""" - dirs = self.query_abs_dirs() - test_relpath = self.config.get( - 'test_path', - os.path.join('testing', 'marionette', - 'harness', 'marionette_harness', 'tests', - 'harness_unit') - ) - test_path = os.path.join(dirs['abs_src_dir'], test_relpath) - self.activate_virtualenv() - import pytest - command = ['-p', 'no:terminalreporter', # disable pytest logging - test_path] - logs = {} - for fmt in ['tbpl', 'mach', 'raw']: - logs[fmt] = os.path.join(dirs['abs_log_dir'], - 'mn-harness_{}.log'.format(fmt)) - command.extend(['--log-'+fmt, logs[fmt]]) - self.info('Calling pytest.main with the following arguments: %s' % command) - status = self._get_pytest_status(pytest.main(command)) - self.read_from_file(logs['tbpl']) - for log in logs.values(): - self.copy_to_upload_dir(log, dest='logs/') - self.buildbot_status(status) - - -if __name__ == '__main__': - script = MarionetteHarnessTests() - script.run_and_exit()