Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-25 05:41:12 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1286075: add support for optimizing based on files changed in the push; r=gps

Browse Source 
MozReview-Commit-ID: 5di7TuL9X2P

--HG--
extra : rebase_source : 30b078178b028b22c0cd7b3d893aa12fd195cf26
This commit is contained in:
Dustin J. Mitchell 2016-09-12 18:40:12 +00:00
parent 6db7f5787e
commit 2fb0796e2b
15 changed files with 609 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
taskcluster/taskgraph/files_changed.py Normal file
View File

@ -0,0 +1,65 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Support for optimizing tasks based on the set of files that have changed.
"""
from __future__ import absolute_import, print_function, unicode_literals
import logging
import requests
from redo import retry
from mozpack.path import match as mozpackmatch
logger = logging.getLogger(__name__)
_cache = {}
def get_changed_files(repository, revision):
"""
Get the set of files changed in the push headed by the given revision.
Responses are cached, so multiple calls with the same arguments are OK.
"""
key = repository, revision
if key not in _cache:
url = '%s/json-automationrelevance/%s' % (repository.rstrip('/'), revision)
logger.debug("Querying version control for metadata: %s", url)
def get_automationrelevance():
response = requests.get(url, timeout=5)
return response.json()
contents = retry(get_automationrelevance, attempts=2, sleeptime=10)
logger.debug('{} commits influencing task scheduling:'
.format(len(contents['changesets'])))
changed_files = set()
for c in contents['changesets']:
logger.debug(" {cset} {desc}".format(
cset=c['node'][0:12],
desc=c['desc'].splitlines()[0].encode('ascii', 'ignore')))
changed_files |= set(c['files'])
_cache[key] = changed_files
return _cache[key]
def check(params, file_patterns):
"""Determine whether any of the files changed in the indicated push to
https://hg.mozilla.org match any of the given file patterns."""
repository = params.get('head_repository')
revision = params.get('head_rev')
if not repository or not revision:
logger.warning("Missing `head_repository` or `head_rev` parameters; "
"assuming all files have changed")
return True
changed_files = get_changed_files(repository, revision)
for pattern in file_patterns:
for path in changed_files:
if mozpackmatch(path, pattern):
return True
return False

1
taskcluster/taskgraph/generator.py
View File

@ -203,6 +203,7 @@ class TaskGraphGenerator(object):
if not self.parameters.get('optimize_target_tasks', True):
do_not_optimize = target_task_set.graph.nodes
optimized_task_graph, label_to_taskid = optimize_task_graph(target_task_graph,
self.parameters,
do_not_optimize)
yield 'label_to_taskid', label_to_taskid
yield 'optimized_task_graph', optimized_task_graph

7
taskcluster/taskgraph/optimize.py
View File

@ -14,7 +14,7 @@ logger = logging.getLogger(__name__)
TASK_REFERENCE_PATTERN = re.compile('<([^>]+)>')
def optimize_task_graph(target_task_graph, do_not_optimize, existing_tasks=None):
def optimize_task_graph(target_task_graph, params, do_not_optimize, existing_tasks=None):
"""
Perform task optimization, without optimizing tasks named in
do_not_optimize.
@ -28,6 +28,7 @@ def optimize_task_graph(target_task_graph, do_not_optimize, existing_tasks=None)
# only the non-optimized tasks, with all task labels resolved to taskIds
# and with task['dependencies'] populated.
annotate_task_graph(target_task_graph=target_task_graph,
params=params,
do_not_optimize=do_not_optimize,
named_links_dict=named_links_dict,
label_to_taskid=label_to_taskid,
@ -59,7 +60,7 @@ def resolve_task_references(label, task_def, taskid_for_edge_name):
return recurse(task_def)
def annotate_task_graph(target_task_graph, do_not_optimize,
def annotate_task_graph(target_task_graph, params, do_not_optimize,
named_links_dict, label_to_taskid, existing_tasks):
"""
Annotate each task in the graph with .optimized (boolean) and .task_id
@ -94,7 +95,7 @@ def annotate_task_graph(target_task_graph, do_not_optimize,
replacement_task_id = existing_tasks[label]
# otherwise, examine the task itself (which may be an expensive operation)
else:
optimized, replacement_task_id = task.optimize()
optimized, replacement_task_id = task.optimize(params)
task.optimized = optimized
task.task_id = replacement_task_id

2
taskcluster/taskgraph/task/base.py
View File

@ -79,7 +79,7 @@ class Task(object):
dependencies.
"""
def optimize(self):
def optimize(self, params):
"""
Determine whether this task can be optimized, and if it can, what taskId
it should be replaced with.

2
taskcluster/taskgraph/task/docker_image.py
View File

@ -118,7 +118,7 @@ class DockerImageTask(base.Task):
def get_dependencies(self, taskgraph):
return []
def optimize(self):
def optimize(self, params):
for index_path in self.index_paths:
try:
url = INDEX_URL.format(index_path)

2
taskcluster/taskgraph/task/legacy.py
View File

@ -618,7 +618,7 @@ class LegacyTask(base.Task):
return deps
def optimize(self):
def optimize(self, params):
# no optimization for the moment
return False, None

2
taskcluster/taskgraph/task/nightly_fennec.py
View File

@ -109,5 +109,5 @@ class NightlyFennecTask(base.Task):
return deps
def optimize(self):
def optimize(self, params):
return False, None

2
taskcluster/taskgraph/task/signing.py
View File

@ -51,5 +51,5 @@ class SigningTask(base.Task):
def get_dependencies(self, taskgraph):
return [('build-nightly-fennec', 'build-nightly-fennec')]
def optimize(self):
def optimize(self, params):
return False, None

15
taskcluster/taskgraph/task/transform.py
View File

@ -7,6 +7,7 @@ from __future__ import absolute_import, print_function, unicode_literals
import logging
from . import base
from .. import files_changed
from ..util.python_path import find_object
from ..util.yaml import load_yaml
from ..transforms.base import TransformSequence, TransformConfig
@ -18,8 +19,8 @@ class TransformTask(base.Task):
"""
Tasks of this class are generated by applying transformations to a sequence
of input entities. By default, it gets those inputs from YAML data in the
kind directory, but subclasses may override `get_inputs` to produce them
in some other way.
kind directory, but subclasses may override `get_inputs` to produce them in
some other way.
"""
@classmethod
@ -63,11 +64,19 @@ class TransformTask(base.Task):
def __init__(self, kind, task):
self.dependencies = task['dependencies']
self.when = task['when']
super(TransformTask, self).__init__(kind, task['label'],
task['attributes'], task['task'])
def get_dependencies(self, taskgraph):
return [(label, name) for name, label in self.dependencies.items()]
def optimize(self):
def optimize(self, params):
if 'files-changed' in self.when:
changed = files_changed.check(
params, self.when['files-changed'])
if not changed:
logger.debug('no files found matching a pattern in `when.files-changed` for '
+ self.label)
return True, None
return False, None

425
taskcluster/taskgraph/test/automationrelevance.json Normal file
View File

@ -0,0 +1,425 @@
{
"changesets": [
{
"author": "James Long <longster@gmail.com>",
"backsoutnodes": [],
"bugs": [
{
"no": "1300866",
"url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300866"
}
],
"date": [
1473196655.0,
14400
],
"desc": "Bug 1300866 - expose devtools require to new debugger r=jlast,bgrins",
"extra": {
"branch": "default"
},
"files": [
"devtools/client/debugger/new/index.html"
],
"node": "ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"parents": [
"37c9349b4e8167a61b08b7e119c21ea177b98942"
],
"perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"pushdate": [
1473261248,
0
],
"pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
"pushid": 30664,
"pushnodes": [
"ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
"16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
"99c542fa43a72ee863c813b5624048d1b443549b",
"a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
"541c9086c0f27fba60beecc9bc94543103895c86",
"041a925171e431bf51fb50193ab19d156088c89a",
"a14f88a9af7a59e677478694bafd9375ac53683e"
],
"pushuser": "cbook@mozilla.com",
"rev": 312890,
"reviewers": [
{
"name": "jlast",
"revset": "reviewer(jlast)"
},
{
"name": "bgrins",
"revset": "reviewer(bgrins)"
}
],
"treeherderrepo": "mozilla-central",
"treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
},
{
"author": "Wes Kocher <wkocher@mozilla.com>",
"backsoutnodes": [],
"bugs": [],
"date": [
1473208638.0,
25200
],
"desc": "Merge m-c to fx-team, a=merge",
"extra": {
"branch": "default"
},
"files": [
"taskcluster/scripts/builder/build-l10n.sh"
],
"node": "73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
"parents": [
"ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"91c2b9d5c1354ca79e5b174591dbb03b32b15bbf"
],
"perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"pushdate": [
1473261248,
0
],
"pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
"pushid": 30664,
"pushnodes": [
"ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
"16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
"99c542fa43a72ee863c813b5624048d1b443549b",
"a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
"541c9086c0f27fba60beecc9bc94543103895c86",
"041a925171e431bf51fb50193ab19d156088c89a",
"a14f88a9af7a59e677478694bafd9375ac53683e"
],
"pushuser": "cbook@mozilla.com",
"rev": 312891,
"reviewers": [
{
"name": "merge",
"revset": "reviewer(merge)"
}
],
"treeherderrepo": "mozilla-central",
"treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
},
{
"author": "Towkir Ahmed <towkir17@gmail.com>",
"backsoutnodes": [],
"bugs": [
{
"no": "1296648",
"url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1296648"
}
],
"date": [
1472957580.0,
14400
],
"desc": "Bug 1296648 - Fix direction of .ruleview-expander.theme-twisty in RTL locales. r=ntim",
"extra": {
"branch": "default"
},
"files": [
"devtools/client/themes/rules.css"
],
"node": "16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
"parents": [
"73a6a267a50a0e1c41e689b265ad3eebe43d7ac6"
],
"perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"pushdate": [
1473261248,
0
],
"pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
"pushid": 30664,
"pushnodes": [
"ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
"16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
"99c542fa43a72ee863c813b5624048d1b443549b",
"a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
"541c9086c0f27fba60beecc9bc94543103895c86",
"041a925171e431bf51fb50193ab19d156088c89a",
"a14f88a9af7a59e677478694bafd9375ac53683e"
],
"pushuser": "cbook@mozilla.com",
"rev": 312892,
"reviewers": [
{
"name": "ntim",
"revset": "reviewer(ntim)"
}
],
"treeherderrepo": "mozilla-central",
"treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
},
{
"author": "Oriol <oriol-bugzilla@hotmail.com>",
"backsoutnodes": [],
"bugs": [
{
"no": "1300336",
"url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1300336"
}
],
"date": [
1472921160.0,
14400
],
"desc": "Bug 1300336 - Allow pseudo-arrays to have a length property. r=fitzgen",
"extra": {
"branch": "default"
},
"files": [
"devtools/client/webconsole/test/browser_webconsole_output_06.js",
"devtools/server/actors/object.js"
],
"node": "99c542fa43a72ee863c813b5624048d1b443549b",
"parents": [
"16a1a91f9269ab95dd83eb29dc5d0227665f7d94"
],
"perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"pushdate": [
1473261248,
0
],
"pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
"pushid": 30664,
"pushnodes": [
"ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
"16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
"99c542fa43a72ee863c813b5624048d1b443549b",
"a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
"541c9086c0f27fba60beecc9bc94543103895c86",
"041a925171e431bf51fb50193ab19d156088c89a",
"a14f88a9af7a59e677478694bafd9375ac53683e"
],
"pushuser": "cbook@mozilla.com",
"rev": 312893,
"reviewers": [
{
"name": "fitzgen",
"revset": "reviewer(fitzgen)"
}
],
"treeherderrepo": "mozilla-central",
"treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
},
{
"author": "Ruturaj Vartak <ruturaj@gmail.com>",
"backsoutnodes": [],
"bugs": [
{
"no": "1295010",
"url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
}
],
"date": [
1472854020.0,
-7200
],
"desc": "Bug 1295010 - Don't move the eyedropper to the out of browser window by keyboard navigation. r=pbro\n\nMozReview-Commit-ID: vBwmSxVNXK",
"extra": {
"amend_source": "6885024ef00cfa33d73c59dc03c48ebcda9ccbdd",
"branch": "default",
"histedit_source": "c43167f0a7cbe9f4c733b15da726e5150a9529ba",
"rebase_source": "b74df421630fc46dab6b6cc026bf3e0ae6b4a651"
},
"files": [
"devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
"devtools/client/inspector/test/head.js",
"devtools/server/actors/highlighters/eye-dropper.js"
],
"node": "a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
"parents": [
"99c542fa43a72ee863c813b5624048d1b443549b"
],
"perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"pushdate": [
1473261248,
0
],
"pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
"pushid": 30664,
"pushnodes": [
"ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
"16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
"99c542fa43a72ee863c813b5624048d1b443549b",
"a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
"541c9086c0f27fba60beecc9bc94543103895c86",
"041a925171e431bf51fb50193ab19d156088c89a",
"a14f88a9af7a59e677478694bafd9375ac53683e"
],
"pushuser": "cbook@mozilla.com",
"rev": 312894,
"reviewers": [
{
"name": "pbro",
"revset": "reviewer(pbro)"
}
],
"treeherderrepo": "mozilla-central",
"treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
},
{
"author": "Matteo Ferretti <mferretti@mozilla.com>",
"backsoutnodes": [],
"bugs": [
{
"no": "1299154",
"url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1299154"
}
],
"date": [
1472629906.0,
-7200
],
"desc": "Bug 1299154 - added Set/GetOverrideDPPX to restorefromHistory; r=mstange\n\nMozReview-Commit-ID: AsyAcG3Igbn\n",
"extra": {
"branch": "default",
"committer": "Matteo Ferretti <mferretti@mozilla.com> 1473236511 -7200"
},
"files": [
"docshell/base/nsDocShell.cpp",
"dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html"
],
"node": "541c9086c0f27fba60beecc9bc94543103895c86",
"parents": [
"a6b6a93eb41a05e310a11f0172f01ba9b21d3eac"
],
"perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"pushdate": [
1473261248,
0
],
"pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
"pushid": 30664,
"pushnodes": [
"ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
"16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
"99c542fa43a72ee863c813b5624048d1b443549b",
"a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
"541c9086c0f27fba60beecc9bc94543103895c86",
"041a925171e431bf51fb50193ab19d156088c89a",
"a14f88a9af7a59e677478694bafd9375ac53683e"
],
"pushuser": "cbook@mozilla.com",
"rev": 312895,
"reviewers": [
{
"name": "mstange",
"revset": "reviewer(mstange)"
}
],
"treeherderrepo": "mozilla-central",
"treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
},
{
"author": "Patrick Brosset <pbrosset@mozilla.com>",
"backsoutnodes": [],
"bugs": [
{
"no": "1295010",
"url": "https://bugzilla.mozilla.org/show_bug.cgi?id=1295010"
}
],
"date": [
1473239449.0,
-7200
],
"desc": "Bug 1295010 - Removed testActor from highlighterHelper in inspector tests; r=me\n\nMozReview-Commit-ID: GMksl81iGcp",
"extra": {
"branch": "default"
},
"files": [
"devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js",
"devtools/client/inspector/test/head.js"
],
"node": "041a925171e431bf51fb50193ab19d156088c89a",
"parents": [
"541c9086c0f27fba60beecc9bc94543103895c86"
],
"perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"pushdate": [
1473261248,
0
],
"pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
"pushid": 30664,
"pushnodes": [
"ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
"16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
"99c542fa43a72ee863c813b5624048d1b443549b",
"a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
"541c9086c0f27fba60beecc9bc94543103895c86",
"041a925171e431bf51fb50193ab19d156088c89a",
"a14f88a9af7a59e677478694bafd9375ac53683e"
],
"pushuser": "cbook@mozilla.com",
"rev": 312896,
"reviewers": [
{
"name": "me",
"revset": "reviewer(me)"
}
],
"treeherderrepo": "mozilla-central",
"treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
},
{
"author": "Carsten \"Tomcat\" Book <cbook@mozilla.com>",
"backsoutnodes": [],
"bugs": [],
"date": [
1473261233.0,
-7200
],
"desc": "merge fx-team to mozilla-central a=merge",
"extra": {
"branch": "default"
},
"files": [],
"node": "a14f88a9af7a59e677478694bafd9375ac53683e",
"parents": [
"3d0b41fdd93bd8233745eadb4e0358e385bf2cb9",
"041a925171e431bf51fb50193ab19d156088c89a"
],
"perfherderurl": "https://treeherder.mozilla.org/perf.html#/compare?originalProject=mozilla-central&originalRevision=a14f88a9af7a59e677478694bafd9375ac53683e&newProject=mozilla-central&newRevision=ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"pushdate": [
1473261248,
0
],
"pushhead": "a14f88a9af7a59e677478694bafd9375ac53683e",
"pushid": 30664,
"pushnodes": [
"ae2144aa4356b65c2f8c0de8c9082dcb7e330e24",
"73a6a267a50a0e1c41e689b265ad3eebe43d7ac6",
"16a1a91f9269ab95dd83eb29dc5d0227665f7d94",
"99c542fa43a72ee863c813b5624048d1b443549b",
"a6b6a93eb41a05e310a11f0172f01ba9b21d3eac",
"541c9086c0f27fba60beecc9bc94543103895c86",
"041a925171e431bf51fb50193ab19d156088c89a",
"a14f88a9af7a59e677478694bafd9375ac53683e"
],
"pushuser": "cbook@mozilla.com",
"rev": 312897,
"reviewers": [
{
"name": "merge",
"revset": "reviewer(merge)"
}
],
"treeherderrepo": "mozilla-central",
"treeherderrepourl": "https://treeherder.mozilla.org/#/jobs?repo=mozilla-central"
}
],
"visible": true
}

73
taskcluster/taskgraph/test/test_files_changed.py Normal file
View File

@ -0,0 +1,73 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import json
import os
import unittest
from .. import files_changed
PARAMS = {
'head_repository': 'https://hg.mozilla.org/mozilla-central',
'head_rev': 'a14f88a9af7a',
}
FILES_CHANGED = [
'devtools/client/debugger/new/index.html',
'devtools/client/inspector/test/browser_inspector_highlighter-eyedropper-events.js',
'devtools/client/inspector/test/head.js',
'devtools/client/themes/rules.css',
'devtools/client/webconsole/test/browser_webconsole_output_06.js',
'devtools/server/actors/highlighters/eye-dropper.js',
'devtools/server/actors/object.js',
'docshell/base/nsDocShell.cpp',
'dom/tests/mochitest/general/test_contentViewer_overrideDPPX.html',
'taskcluster/scripts/builder/build-l10n.sh',
]
class FakeResponse:
def json(self):
with open(os.path.join(os.path.dirname(__file__), 'automationrelevance.json')) as f:
return json.load(f)
class TestGetChangedFiles(unittest.TestCase):
def setUp(self):
files_changed._cache.clear()
self.old_get = files_changed.requests.get
def fake_get(url, **kwargs):
return FakeResponse()
files_changed.requests.get = fake_get
def tearDown(self):
files_changed.requests.get = self.old_get
def test_get_changed_files(self):
"""Get_changed_files correctly gets the list of changed files in a push.
This tests against the production hg.mozilla.org so that it will detect
any changes in the format of the returned data."""
self.assertEqual(
sorted(files_changed.get_changed_files(PARAMS['head_repository'], PARAMS['head_rev'])),
FILES_CHANGED)
class TestCheck(unittest.TestCase):
def setUp(self):
files_changed._cache[PARAMS['head_repository'], PARAMS['head_rev']] = FILES_CHANGED
def test_check_no_params(self):
self.assertTrue(files_changed.check({}, ["ignored"]))
def test_check_no_match(self):
self.assertFalse(files_changed.check(PARAMS, ["nosuch/**"]))
def test_check_match(self):
self.assertTrue(files_changed.check(PARAMS, ["devtools/**"]))

2
taskcluster/taskgraph/test/test_generator.py
View File

@ -34,7 +34,7 @@ class FakeTask(base.Task):
else:
return []
def optimize(self):
def optimize(self, params):
return False, None

24
taskcluster/taskgraph/test/test_optimize.py
View File

@ -85,7 +85,7 @@ class TestOptimize(unittest.TestCase):
def test_annotate_task_graph_no_optimize(self):
"annotating marks everything as un-optimized if the kind returns that"
OptimizingTask.optimize = lambda self: (False, None)
OptimizingTask.optimize = lambda self, params: (False, None)
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
@ -93,7 +93,7 @@ class TestOptimize(unittest.TestCase):
('task2', 'task1', 'build'),
('task2', 'task3', 'image'),
)
annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
self.assert_annotations(
graph,
task1=(False, None),
@ -103,17 +103,17 @@ class TestOptimize(unittest.TestCase):
def test_annotate_task_graph_taskid_without_optimize(self):
"raises exception if kind returns a taskid without optimizing"
OptimizingTask.optimize = lambda self: (False, 'some-taskid')
OptimizingTask.optimize = lambda self, params: (False, 'some-taskid')
graph = self.make_graph(self.make_task('task1'))
self.assertRaises(
Exception,
lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
)
def test_annotate_task_graph_optimize_away_dependency(self):
"raises exception if kind optimizes away a task on which another depends"
OptimizingTask.optimize = \
lambda self: (True, None) if self.label == 'task1' else (False, None)
lambda self, params: (True, None) if self.label == 'task1' else (False, None)
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
@ -121,19 +121,19 @@ class TestOptimize(unittest.TestCase):
)
self.assertRaises(
Exception,
lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {}, None)
lambda: annotate_task_graph(graph, {}, set(), graph.graph.named_links_dict(), {}, None)
)
def test_annotate_task_graph_do_not_optimize(self):
"annotating marks everything as un-optimized if in do_not_optimize"
OptimizingTask.optimize = lambda self: (True, 'taskid')
OptimizingTask.optimize = lambda self, params: (True, 'taskid')
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
('task2', 'task1', 'build'),
)
label_to_taskid = {}
annotate_task_graph(graph, {'task1', 'task2'},
annotate_task_graph(graph, {}, {'task1', 'task2'},
graph.graph.named_links_dict(), label_to_taskid, None)
self.assert_annotations(
graph,
@ -145,7 +145,7 @@ class TestOptimize(unittest.TestCase):
def test_annotate_task_graph_nos_do_not_propagate(self):
"a task with a non-optimized dependency can be optimized"
OptimizingTask.optimize = \
lambda self: (False, None) if self.label == 'task1' else (True, 'taskid')
lambda self, params: (False, None) if self.label == 'task1' else (True, 'taskid')
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
@ -153,7 +153,7 @@ class TestOptimize(unittest.TestCase):
('task2', 'task1', 'build'),
('task2', 'task3', 'image'),
)
annotate_task_graph(graph, set(),
annotate_task_graph(graph, {}, set(),
graph.graph.named_links_dict(), {}, None)
self.assert_annotations(
graph,
@ -242,7 +242,7 @@ class TestOptimize(unittest.TestCase):
def test_optimize(self):
"optimize_task_graph annotates and extracts the subgraph from a simple graph"
OptimizingTask.optimize = \
lambda self: (True, 'dep1') if self.label == 'task1' else (False, None)
lambda self, params: (True, 'dep1') if self.label == 'task1' else (False, None)
input = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
@ -250,7 +250,7 @@ class TestOptimize(unittest.TestCase):
('task2', 'task1', 'build'),
('task2', 'task3', 'image'),
)
opt, label_to_taskid = optimize_task_graph(input, set())
opt, label_to_taskid = optimize_task_graph(input, {}, set())
self.assertEqual(opt.graph, graph.Graph(
{label_to_taskid['task2'], label_to_taskid['task3']},
{(label_to_taskid['task2'], label_to_taskid['task3'], 'image')}))

1
taskcluster/taskgraph/transforms/job/__init__.py
View File

@ -54,6 +54,7 @@ job_description_schema = Schema({
Optional('coalesce-name'): task_description_schema['coalesce-name'],
Optional('worker-type'): task_description_schema['worker-type'],
Required('worker'): task_description_schema['worker'],
Optional('when'): task_description_schema['when'],
# A description of how to run this job.
'run': {

10
taskcluster/taskgraph/transforms/task.py
View File

@ -212,6 +212,16 @@ task_description_schema = Schema({
Extra: basestring, # additional properties are allowed
},
}),
# The "when" section contains descriptions of the circumstances
# under which this task can be "optimized", that is, left out of the
# task graph because it is unnecessary.
Optional('when'): Any({
# This task only needs to be run if a file matching one of the given
# patterns has changed in the push. The patterns use the mozpack
# match function (python/mozbuild/mozpack/path.py).
Optional('files-changed'): [basestring],
}),
})
GROUP_NAMES = {