Bug 1193264 - Add support for saving and reusing try strings in mach try, r=chmanchester

Adds --save and --preset arguments that can be used to store and reuse
frequently used try strings.

testing/mach_commands.py

@@ -10,6 +10,7 @@ import sys
 import tempfile
 import subprocess
 import shutil
+from collections import defaultdict
 
 from mach.decorators import (
     CommandArgument,
@@ -437,33 +438,51 @@ class JsapiTestsCommand(MachCommandBase):
         return jsapi_tests_result
 
 def autotry_parser():
-    from autotry import parser
-    return parser()
+    from autotry import arg_parser
+    return arg_parser()
 
 @CommandProvider
 class PushToTry(MachCommandBase):
+    def normalise_list(self, items, allow_subitems=False):
+        from autotry import parse_arg
 
-    def validate_args(self, paths, tests, tags, builds, platforms):
-        if not any([len(paths), tests, tags]):
-            print("Paths, tests, or tags must be specified.")
+        rv = defaultdict(list)
+        for item in items:
+            parsed = parse_arg(item)
+            for key, values in parsed.iteritems():
+                rv[key].extend(values)
+
+        if not allow_subitems:
+            if not all(item == [] for item in rv.itervalues()):
+                raise ValueError("Unexpected subitems in argument")
+            return rv.keys()
+        else:
+            return rv
+
+    def validate_args(self, **kwargs):
+        if not kwargs["paths"] and not kwargs["tests"] and not kwargs["tags"]:
+            print("Paths, tags, or tests must be specified as an argument to autotry.")
             sys.exit(1)
 
-        if platforms is None:
-            platforms = os.environ['AUTOTRY_PLATFORM_HINT']
+        if kwargs["platforms"] is None:
+            print("Platforms must be specified as an argument to autotry")
+            sys.exit(1)
 
-        rv_platforms = []
-        for item in platforms:
-            for platform in item.split(","):
-                if platform:
-                    rv_platforms.append(platform)
+        try:
+            platforms = self.normalise_list(kwargs["platforms"])
+        except ValueError as e:
+            print("Error parsing -p argument:\n%s" % e.message)
+            sys.exit(1)
 
-        rv_tests = []
-        for item in tests:
-            for test in item.split(","):
-                if test:
-                    rv_tests.append(test)
+        try:
+            tests = (self.normalise_list(kwargs["tests"], allow_subitems=True)
+                     if kwargs["tests"] else {})
+        except ValueError as e:
+            print("Error parsing -u argument:\n%s" % e.message)
+            sys.exit(1)
 
-        for p in paths:
+        paths = []
+        for p in kwargs["paths"]:
             p = os.path.normpath(os.path.abspath(p))
             if not p.startswith(self.topsrcdir):
                 print('Specified path "%s" is outside of the srcdir, unable to'
@@ -473,15 +492,23 @@ class PushToTry(MachCommandBase):
                 print('Specified path "%s" is at the top of the srcdir and would'
                       ' select all tests.' % p)
                 sys.exit(1)
+            paths.append(os.path.relpath(p, self.topsrcdir))
+
+        try:
+            tags = self.normalise_list(kwargs["tags"]) if kwargs["tags"] else []
+        except ValueError as e:
+            print("Error parsing --tags argument:\n%s" % e.message)
+            sys.exit(1)
+
+        return kwargs["builds"], platforms, tests, paths, tags, kwargs["extra_args"]
 
-        return builds, rv_platforms, rv_tests
 
     @Command('try',
              category='testing',
              description='Push selected tests to the try server',
              parser=autotry_parser)
-    def autotry(self, builds=None, platforms=None, paths=None, verbose=None,
-                push=None, tags=None, tests=None, extra_args=None, intersection=False):
+
+    def autotry(self, **kwargs):
         """Autotry is in beta, please file bugs blocking 1149670.
 
         Push the current tree to try, with the specified syntax.
@@ -527,14 +554,23 @@ class PushToTry(MachCommandBase):
         from autotry import AutoTry
         print("mach try is under development, please file bugs blocking 1149670.")
 
-        if tests is None:
-            tests = []
-
-        builds, platforms, tests = self.validate_args(paths, tests, tags, builds, platforms)
         resolver = self._spawn(TestResolver)
-
         at = AutoTry(self.topsrcdir, resolver, self._mach_context)
-        if push and at.find_uncommited_changes():
+
+        if kwargs["load"] is not None:
+            defaults = at.load_config(kwargs["load"])
+
+            if defaults is None:
+                print("No saved configuration called %s found in autotry.ini" % kwargs["load"],
+                      file=sys.stderr)
+
+            for key, value in kwargs.iteritems():
+                if value in (None, []) and key in defaults:
+                    kwargs[key] = defaults[key]
+
+        builds, platforms, tests, paths, tags, extra_args = self.validate_args(**kwargs)
+
+        if kwargs["push"] and at.find_uncommited_changes():
             print('ERROR please commit changes before continuing')
             sys.exit(1)
 
@@ -551,30 +587,31 @@ class PushToTry(MachCommandBase):
                       paths)
                 sys.exit(1)
 
-            if not intersection:
+            if not kwargs["intersection"]:
                 paths_by_flavor = at.remove_duplicates(paths_by_flavor, tests)
         else:
             paths_by_flavor = {}
 
         try:
             msg = at.calc_try_syntax(platforms, tests, builds, paths_by_flavor, tags,
-                                     extra_args, intersection)
+                                     extra_args, kwargs["intersection"])
         except ValueError as e:
             print(e.message)
             sys.exit(1)
 
-        if verbose and paths_by_flavor:
+        if kwargs["verbose"] and paths_by_flavor:
             print('The following tests will be selected: ')
             for flavor, paths in paths_by_flavor.iteritems():
                 print("%s: %s" % (flavor, ",".join(paths)))
 
-        if verbose or not push:
+        if kwargs["verbose"] or not kwargs["push"]:
             print('The following try syntax was calculated:\n%s' % msg)
 
-        if push:
-            at.push_to_try(msg, verbose)
+        if kwargs["push"]:
+            at.push_to_try(msg, kwargs["verbose"])
 
-        return
+        if kwargs["save"] is not None:
+            at.save_config(kwargs["save"], msg)
 
 
 def get_parser(argv=None):

testing/tools/autotry/autotry.py

@@ -3,10 +3,11 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import argparse
-import sys
-import os
 import itertools
+import os
+import re
 import subprocess
+import sys
 import which
 
 from collections import defaultdict
@@ -14,7 +15,7 @@ from collections import defaultdict
 import ConfigParser
 
 
-def parser():
+def arg_parser():
     parser = argparse.ArgumentParser()
     parser.add_argument('paths', nargs='*', help='Paths to search for tests to run on try.')
     parser.add_argument('-p', dest='platforms', action="append",
@@ -31,6 +32,10 @@ def parser():
                         help='Do not push to try as a result of running this command (if '
                         'specified this command will only print calculated try '
                         'syntax and selection info).')
+    parser.add_argument('--save', dest="save", action='store',
+                        help="Save the command line arguments for future use with --preset")
+    parser.add_argument('--preset', dest="load", action='store',
+                        help="Load a saved set of arguments. Additional arguments will override saved ones")
     parser.add_argument('extra_args', nargs=argparse.REMAINDER,
                         help='Extra arguments to put in the try push')
     parser.add_argument('-v', "--verbose", dest='verbose', action='store_true', default=False,
@@ -38,6 +43,102 @@ def parser():
                         'and commands performed.')
     return parser
 
+class TryArgumentTokenizer(object):
+    symbols = [("seperator", ","),
+               ("list_start", "\["),
+               ("list_end", "\]"),
+               ("item", "([^,\[\]\s][^,\[\]]+)"),
+               ("space", "\s+")]
+    token_re = re.compile("|".join("(?P<%s>%s)" % item for item in symbols))
+
+    def tokenize(self, data):
+        for match in self.token_re.finditer(data):
+            symbol = match.lastgroup
+            data = match.group(symbol)
+            if symbol == "space":
+                pass
+            else:
+                yield symbol, data
+
+class TryArgumentParser(object):
+    """Simple three-state parser for handling expressions
+    of the from "foo[sub item, another], bar,baz". This takes
+    input from the TryArgumentTokenizer and runs through a small
+    state machine, returning a dictionary of {top-level-item:[sub_items]}
+    i.e. the above would result in
+    {"foo":["sub item", "another"], "bar": [], "baz": []}
+    In the case of invalid input a ValueError is raised."""
+
+    EOF = object()
+
+    def __init__(self):
+        self.reset()
+
+    def reset(self):
+        self.tokens = None
+        self.current_item = None
+        self.data = {}
+        self.token = None
+        self.state = None
+
+    def parse(self, tokens):
+        self.reset()
+        self.tokens = tokens
+        self.consume()
+        self.state = self.item_state
+        while self.token[0] != self.EOF:
+            self.state()
+        return self.data
+
+    def consume(self):
+        try:
+            self.token = self.tokens.next()
+        except StopIteration:
+            self.token = (self.EOF, None)
+
+    def expect(self, *types):
+        if self.token[0] not in types:
+            raise ValueError("Error parsing try string, unexpected %s" % (self.token[0]))
+
+    def item_state(self):
+        self.expect("item")
+        value = self.token[1].strip()
+        if value not in self.data:
+            self.data[value] = []
+        self.current_item = value
+        self.consume()
+        if self.token[0] == "seperator":
+            self.consume()
+        elif self.token[0] == "list_start":
+            self.consume()
+            self.state = self.subitem_state
+        elif self.token[0] == self.EOF:
+            pass
+        else:
+            raise ValueError
+
+    def subitem_state(self):
+        self.expect("item")
+        value = self.token[1].strip()
+        self.data[self.current_item].append(value)
+        self.consume()
+        if self.token[0] == "seperator":
+            self.consume()
+        elif self.token[0] == "list_end":
+            self.consume()
+            self.state = self.after_list_end_state
+        else:
+            raise ValueError
+
+    def after_list_end_state(self):
+        self.expect("seperator")
+        self.consume()
+        self.state = self.item_state
+
+def parse_arg(arg):
+    tokenizer = TryArgumentTokenizer()
+    parser = TryArgumentParser()
+    return parser.parse(tokenizer.tokenize(arg))
 
 class AutoTry(object):
 
@@ -76,6 +177,40 @@ class AutoTry(object):
         else:
             self._use_git = True
 
+    @property
+    def config_path(self):
+        return os.path.join(self.mach_context.state_dir, "autotry.ini")
+
+    def load_config(self, name):
+        config = ConfigParser.RawConfigParser()
+        success = config.read([self.config_path])
+        if not success:
+            return None
+
+        try:
+            data = config.get("try", name)
+        except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
+            return None
+
+        kwargs = vars(arg_parser().parse_args(data.split()))
+
+        return kwargs
+
+    def save_config(self, name, data):
+        assert data.startswith("try: ")
+        data = data[len("try: "):]
+
+        parser = ConfigParser.RawConfigParser()
+        parser.read([self.config_path])
+
+        if not parser.has_section("try"):
+            parser.add_section("try")
+
+        parser.set("try", name, data)
+
+        with open(self.config_path, "w") as f:
+            parser.write(f)
+
     def paths_by_flavor(self, paths=None, tags=None):
         paths_by_flavor = defaultdict(set)
 
@@ -113,7 +248,7 @@ class AutoTry(object):
                         intersection):
         parts = ["try:", "-b", builds, "-p", ",".join(platforms)]
 
-        suites = set(tests) if not intersection else set()
+        suites = tests if not intersection else {}
         paths = set()
         for flavor, flavor_tests in paths_by_flavor.iteritems():
             suite = self.flavor_suites[flavor]
@@ -121,13 +256,14 @@ class AutoTry(object):
                 for job_name in self.flavor_jobs[flavor]:
                     for test in flavor_tests:
                         paths.add("%s:%s" % (flavor, test))
-                    suites.add(job_name)
+                    suites[job_name] = tests.get(suite, [])
 
         if not suites:
             raise ValueError("No tests found matching filters")
 
         parts.append("-u")
-        parts.append(",".join(sorted(suites)))
+        parts.append(",".join("%s%s" % (k, "[%s]" % ",".join(v) if v else "")
+                              for k,v in sorted(suites.items())))
 
         if tags:
             parts.append(' '.join('--tag %s' % t for t in tags))