diff --git a/taskcluster/ci/build/linux.yml b/taskcluster/ci/build/linux.yml index c892908b28bf..d2e1e370f1b9 100644 --- a/taskcluster/ci/build/linux.yml +++ b/taskcluster/ci/build/linux.yml @@ -272,6 +272,7 @@ linux64-stylo/opt: secrets: true custom-build-variant-cfg: stylo tooltool-downloads: public + run-on-projects: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central' ] linux64-stylo/debug: description: "Linux64 Debug Stylo" @@ -296,6 +297,7 @@ linux64-stylo/debug: secrets: true custom-build-variant-cfg: stylo-debug tooltool-downloads: public + run-on-projects: [ 'stylo', 'autoland', 'mozilla-inbound', 'mozilla-central' ] linux64-jsdcov/opt: description: "Linux64-JSDCov Opt" diff --git a/taskcluster/ci/test/tests.yml b/taskcluster/ci/test/tests.yml index ba772a5a8c1b..7fe2f437256d 100644 --- a/taskcluster/ci/test/tests.yml +++ b/taskcluster/ci/test/tests.yml @@ -936,6 +936,17 @@ reftest-stylo: by-test-platform: linux64-stylo/opt: [] linux64-stylo/debug: [] + e10s: + by-test-platform: + linux64-stylo/opt: + by-project: + mozilla-inbound: false + default: both + linux64-stylo/debug: + # only e10s on inbound + by-project: + mozilla-inbound: true + default: both mozharness: script: desktop_unittest.py no-read-buildbot-config: true diff --git a/taskcluster/taskgraph/test/test_util_schema.py b/taskcluster/taskgraph/test/test_util_schema.py index 6e6845f493d3..e59ce2646ff7 100644 --- a/taskcluster/taskgraph/test/test_util_schema.py +++ b/taskcluster/taskgraph/test/test_util_schema.py @@ -53,6 +53,32 @@ class TestResolveKeyedBy(unittest.TestCase): resolve_keyed_by({'x': {'a': 10}}, 'x', 'n'), {'x': {'a': 10}}) + def test_nested(self): + x = { + 'by-foo': { + 'F1': { + 'by-bar': { + 'B1': 11, + 'B2': 12, + }, + }, + 'F2': 20, + 'default': 0, + }, + } + self.assertEqual( + resolve_keyed_by({'x': x}, 'x', 'x', foo='F1', bar='B1'), + {'x': 11}) + self.assertEqual( + resolve_keyed_by({'x': x}, 'x', 'x', foo='F1', bar='B2'), + {'x': 12}) + self.assertEqual( + resolve_keyed_by({'x': x}, 'x', 'x', foo='F2'), + {'x': 20}) + self.assertEqual( + resolve_keyed_by({'x': x}, 'x', 'x', foo='F99', bar='B1'), + {'x': 0}) + def test_no_by_empty_dict(self): self.assertEqual( resolve_keyed_by({'x': {}}, 'x', 'n'), diff --git a/taskcluster/taskgraph/transforms/tests.py b/taskcluster/taskgraph/transforms/tests.py index 6ea730eac5fb..ed7e063ca9d1 100644 --- a/taskcluster/taskgraph/transforms/tests.py +++ b/taskcluster/taskgraph/transforms/tests.py @@ -119,7 +119,7 @@ test_description_schema = Schema({ # one task without e10s. E10s tasks have "-e10s" appended to the test name # and treeherder group. Required('e10s', default='both'): optionally_keyed_by( - 'test-platform', + 'test-platform', 'project', Any(bool, 'both')), # The EC2 instance size to run these tests on. @@ -456,7 +456,8 @@ def handle_keyed_by(config, tests): ] for test in tests: for field in fields: - resolve_keyed_by(test, field, item_name=test['test-name']) + resolve_keyed_by(test, field, item_name=test['test-name'], + project=config.params['project']) yield test diff --git a/taskcluster/taskgraph/util/schema.py b/taskcluster/taskgraph/util/schema.py index d4e3135fd080..11a352976f5f 100644 --- a/taskcluster/taskgraph/util/schema.py +++ b/taskcluster/taskgraph/util/schema.py @@ -34,13 +34,22 @@ def optionally_keyed_by(*arguments): 'some-value': optionally_keyed_by( 'test-platform', 'build-platform', Any('a', 'b', 'c')) + + The resulting schema will allow nesting of `by-test-platform` and + `by-build-platform` in either order. """ - subschema = arguments[-1] + schema = arguments[-1] fields = arguments[:-1] - options = [subschema] - for field in fields: - options.append({'by-' + field: {basestring: subschema}}) - return voluptuous.Any(*options) + + # build the nestable schema by generating schema = Any(schema, + # by-fld1, by-fld2, by-fld3) once for each field. So we don't allow + # infinite nesting, but one level of nesting for each field. + for _ in arguments: + options = [schema] + for field in fields: + options.append({'by-' + field: {basestring: schema}}) + schema = voluptuous.Any(*options) + return schema def resolve_keyed_by(item, field, item_name, **extra_values): @@ -50,7 +59,7 @@ def resolve_keyed_by(item, field, item_name, **extra_values): (modifying `item` directly). The field is specified using dotted notation to traverse dictionaries. - For example, given item + For example, given item:: job: test-platform: linux128 @@ -61,7 +70,7 @@ def resolve_keyed_by(item, field, item_name, **extra_values): default: 12 a call to `resolve_keyed_by(item, 'job.chunks', item['thing-name']) - would mutate item in-place to + would mutate item in-place to:: job: chunks: 12 @@ -70,6 +79,17 @@ def resolve_keyed_by(item, field, item_name, **extra_values): If extra_values are supplied, they represent additional values available for reference from by-. + + Items can be nested as deeply as the schema will allow:: + + chunks: + by-test-platform: + win.*: + by-project: + ash: .. + cedar: .. + linux: 13 + default: 12 """ # find the field, returning the item unchanged if anything goes wrong container, subfield = item, field @@ -84,33 +104,35 @@ def resolve_keyed_by(item, field, item_name, **extra_values): if subfield not in container: return item value = container[subfield] - if not isinstance(value, dict) or len(value) != 1 or not value.keys()[0].startswith('by-'): - return item + while True: + if not isinstance(value, dict) or len(value) != 1 or not value.keys()[0].startswith('by-'): + return item - keyed_by = value.keys()[0][3:] # strip off 'by-' prefix - key = extra_values.get(keyed_by) if keyed_by in extra_values else item[keyed_by] - alternatives = value.values()[0] + keyed_by = value.keys()[0][3:] # strip off 'by-' prefix + key = extra_values.get(keyed_by) if keyed_by in extra_values else item[keyed_by] + alternatives = value.values()[0] - # exact match - if key in alternatives: - container[subfield] = alternatives[key] - return item + # exact match + if key in alternatives: + value = container[subfield] = alternatives[key] + continue + + # regular expression match + matches = [(k, v) for k, v in alternatives.iteritems() if re.match(k + '$', key)] + if len(matches) > 1: + raise Exception( + "Multiple matching values for {} {!r} found while " + "determining item {} in {}".format( + keyed_by, key, field, item_name)) + elif matches: + value = container[subfield] = matches[0][1] + continue + + # default + if 'default' in alternatives: + value = container[subfield] = alternatives['default'] + continue - # regular expression match - matches = [(k, v) for k, v in alternatives.iteritems() if re.match(k + '$', key)] - if len(matches) > 1: raise Exception( - "Multiple matching values for {} {!r} found while determining item {} in {}".format( + "No {} matching {!r} nor 'default' found while determining item {} in {}".format( keyed_by, key, field, item_name)) - elif matches: - container[subfield] = matches[0][1] - return item - - # default - if 'default' in alternatives: - container[subfield] = alternatives['default'] - return item - - raise Exception( - "No {} matching {!r} nor 'default' found while determining item {} in {}".format( - keyed_by, key, field, item_name))