Bug 1212609 - [mozharness] Update firefox-ui-tests script to use common.tests.zip file. r=jlund

testing/mozharness/mozharness/mozilla/testing/firefox_ui_tests.py

@@ -99,7 +99,8 @@ firefox_ui_update_config_options = firefox_ui_update_harness_config_options \
 
 class FirefoxUITests(TestingMixin, VCSToolsScript):
 
-    cli_script = 'runtests.py'
+    # Needs to be overwritten in sub classes
+    cli_script = None
 
     def __init__(self, config_options=None,
                  all_actions=None, default_actions=None,
@@ -108,7 +109,6 @@ class FirefoxUITests(TestingMixin, VCSToolsScript):
         actions = [
             'clobber',
             'download-and-extract',
-            'checkout',  # keep until firefox-ui-tests are located in tree
             'create-virtualenv',
             'install',
             'run-tests',
@@ -121,53 +121,60 @@ class FirefoxUITests(TestingMixin, VCSToolsScript):
             default_actions=default_actions or actions,
             *args, **kwargs)
 
+        # As long as we don't run on buildbot the following properties have be set on our own
+        self.installer_url = self.config.get('installer_url')
+        self.installer_path = self.config.get('installer_path')
+        self.test_packages_url = self.config.get('test_packages_url')
+        self.test_url = self.config.get('test_url')
+
         self.reports = {'html': 'report.html', 'xunit': 'report.xml'}
 
         self.firefox_ui_repo = self.config['firefox_ui_repo']
         self.firefox_ui_branch = self.config.get('firefox_ui_branch')
 
-        if not self.firefox_ui_branch:
+        if not self.test_url and not self.test_packages_url and not self.firefox_ui_branch:
             self.fatal(
-                'Please specify --firefox-ui-branch. Valid values can be found '
-                'in here https://github.com/mozilla/firefox-ui-tests/branches')
-
-        # As long as we don't run on buildbot the installers are not handled by TestingMixin
-        self.installer_url = self.config.get('installer_url')
-        self.installer_path = self.config.get('installer_path')
+                'You must use --test-url, --test-packages-url, or --firefox-ui-branch (valid '
+                'values can be found at: https://github.com/mozilla/firefox-ui-tests/branches)')
 
     @PreScriptAction('create-virtualenv')
     def _pre_create_virtualenv(self, action):
         dirs = self.query_abs_dirs()
 
-        # List of exact versions of mozbase packages which are known to work
-        requirements_file = os.path.join(dirs['fx_ui_dir'], 'requirements.txt')
-        if os.path.isfile(requirements_file):
-            self.register_virtualenv_module(requirements=[requirements_file])
+        # If tests are used from common.tests.zip install every Python package
+        # via the single requirements file
+        if self.test_packages_url or self.test_url:
+            test_install_dir = dirs.get('abs_test_install_dir',
+                                        os.path.join(dirs['abs_work_dir'], 'tests'))
+            requirements_file = os.path.join(test_install_dir,
+                                             'config', 'firefox_ui_requirements.txt')
+            if os.path.isfile(requirements_file):
+                self.register_virtualenv_module(requirements=[requirements_file])
 
-        # Optional packages to be installed, e.g. for Jenkins
-        if self.config.get('virtualenv_modules'):
-            for module in self.config['virtualenv_modules']:
-                self.register_virtualenv_module(module)
+        # We have a non-packaged version of Firefox UI tests. So install requirements
+        # and the firefox-ui-tests package separately
+        else:
+            # Register all modules for firefox-ui-tests including all dependencies
+            # as strict versions to ensure newer releases won't break something
+            requirements_file = os.path.join(dirs.get('abs_test_install_dir',
+                                                      os.path.join(dirs['abs_work_dir'], 'tests')),
+                                             'requirements.txt')
+            if os.path.isfile(requirements_file):
+                self.register_virtualenv_module(requirements=[requirements_file])
 
-        self.register_virtualenv_module('firefox-ui-tests', url=dirs['fx_ui_dir'])
-
-    @PreScriptAction('checkout')
-    def _pre_checkout(self, action):
-        if not self.firefox_ui_branch:
-            self.fatal(
-                'Please specify --firefox-ui-branch. Valid values can be found '
-                'in here https://github.com/mozilla/firefox-ui-tests/branches')
+            # Optional packages to be installed, e.g. for Jenkins
+            if self.config.get('virtualenv_modules'):
+                for module in self.config['virtualenv_modules']:
+                    self.register_virtualenv_module(module)
 
     def checkout(self):
-        """
-        We checkout firefox_ui_tests and update to the right branch
-        for it.
-        """
+        """Clone the firefox-ui-tests repository."""
         dirs = self.query_abs_dirs()
 
         self.vcs_checkout(
             repo=self.firefox_ui_repo,
-            dest=dirs['fx_ui_dir'],
+            dest=dirs.get('abs_test_install_dir',
+                          os.path.join(dirs['abs_work_dir'], 'tests')),
             branch=self.firefox_ui_branch,
             vcs='gittool',
             env=self.query_env(),
@@ -193,15 +200,22 @@ class FirefoxUITests(TestingMixin, VCSToolsScript):
                                     max_backups=self.config.get("log_max_rotate", 0))
 
     def download_and_extract(self):
-        """Overriding method from TestingMixin until firefox-ui-tests are in tree.
+        """Overriding method from TestingMixin until firefox-ui-tests are in tree and
+        supported across all branches.
 
-        Right now we only care about the installer and symbolds.
+        We use the test_packages_url command line argument to check where to get the
+        harness, puppeteer, and tests from and how to set them up.
 
         """
-        self._download_installer()
+        if self.test_packages_url or self.test_url:
+            super(FirefoxUITests, self).download_and_extract()
 
-        if self.config.get('download_symbols'):
-            self._download_and_extract_symbols()
+        else:
+            self.checkout()
+            self._download_installer()
+
+            if self.config.get('download_symbols'):
+                self._download_and_extract_symbols()
 
     def query_abs_dirs(self):
         if self.abs_dirs:
@@ -210,7 +224,6 @@ class FirefoxUITests(TestingMixin, VCSToolsScript):
         abs_dirs = VCSToolsScript.query_abs_dirs(self)
         abs_dirs.update({
             'abs_reports_dir': os.path.join(abs_dirs['base_work_dir'], 'reports'),
-            'fx_ui_dir': os.path.join(abs_dirs['abs_work_dir'], 'firefox_ui_tests'),
         })
         self.abs_dirs = abs_dirs
 
@@ -238,14 +251,16 @@ class FirefoxUITests(TestingMixin, VCSToolsScript):
         return args
 
     def query_minidump_stackwalk(self):
-        """We don't have an extracted test package available to get the manifest file.
+        """Download the minidump stackwalk binary.
 
-        So we have to explicitely download the latest version of the manifest from the
-        mozilla-central repository and feed it into the query_minidump_stackwalk() method.
-
-        We can remove this whole method once our tests are part of the tree.
+        We can remove this whole method once we no longer need the github repository.
 
         """
+        # If the test package is available use it
+        if self.test_packages_url or self.test_url:
+            return super(FirefoxUITests, self).query_minidump_stackwalk()
+
+        # Otherwise grab the manifest file from hg.mozilla.org
         manifest_path = None
 
         if self.config.get('download_minidump_stackwalk'):
@@ -271,9 +286,13 @@ class FirefoxUITests(TestingMixin, VCSToolsScript):
         """All required steps for running the tests against an installer."""
         dirs = self.query_abs_dirs()
 
+        # Import the harness to retrieve the location of the cli scripts
+        import firefox_ui_harness
+
         cmd = [
             self.query_python_path(),
-            os.path.join(dirs['fx_ui_dir'], 'firefox_ui_harness', self.cli_script),
+            os.path.join(os.path.dirname(firefox_ui_harness.__file__),
+                         self.cli_script),
             '--binary', binary_path,
             '--address', 'localhost:{}'.format(marionette_port),
 
@@ -330,6 +349,11 @@ class FirefoxUITests(TestingMixin, VCSToolsScript):
         )
 
 
+class FirefoxUIFunctionalTests(FirefoxUITests):
+
+    cli_script = 'cli_functional.py'
+
+
 class FirefoxUIUpdateTests(FirefoxUITests):
 
     cli_script = 'cli_update.py'

testing/mozharness/scripts/firefox_ui_tests/functional.py

@@ -12,9 +12,9 @@ import sys
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
 
-from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUITests
+from mozharness.mozilla.testing.firefox_ui_tests import FirefoxUIFunctionalTests
 
 
 if __name__ == '__main__':
-    myScript = FirefoxUITests()
+    myScript = FirefoxUIFunctionalTests()
     myScript.run_and_exit()