Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-09 11:25:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1280231: refactor task kinds to task classes; r=jonasfj

Browse Source 
MozReview-Commit-ID: 1cNukxBgfey

--HG--
extra : rebase_source : 4f0fcce2bcea0fb78ba70e7c052638ca2c5b8a3d
extra : intermediate-source : ba5cbf4e06a550993e5216f816dcf0ccd3938b2e
extra : source : f744bd2fbcd3ae9b90851dcd12307c15d04f8bea
This commit is contained in:
Dustin J. Mitchell 2016-06-27 22:57:44 +00:00
parent fae35df0c6
commit 49f6131a63
18 changed files with 201 additions and 194 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
taskcluster/ci/docker-image/kind.yml
View File

@ -2,7 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
implementation: 'taskgraph.kind.docker_image:DockerImageKind'
implementation: 'taskgraph.kind.docker_image:DockerImageTask'
images_path: '../../../testing/docker'
# make a task for each docker-image we might want. For the moment, since we

2
taskcluster/ci/legacy/kind.yml
View File

@ -2,5 +2,5 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
implementation: 'taskgraph.kind.legacy:LegacyKind'
implementation: 'taskgraph.kind.legacy:LegacyTask'
legacy_path: '.'

3
taskcluster/taskgraph/create.py
View File

@ -49,7 +49,8 @@ def create_tasks(taskgraph, label_to_taskid):
task_def['taskGroupId'] = task_group_id
# Wait for dependencies before submitting this.
deps_fs = [fs[dep] for dep in task_def['dependencies'] if dep in fs]
deps_fs = [fs[dep] for dep in task_def.get('dependencies', [])
if dep in fs]
for f in futures.as_completed(deps_fs):
f.result()

18
taskcluster/taskgraph/generator.py
View File

@ -115,8 +115,8 @@ class TaskGraphGenerator(object):
path = os.path.join(self.root_dir, path)
if not os.path.isdir(path):
continue
name = os.path.basename(path)
logger.debug("loading kind `{}` from `{}`".format(name, path))
kind_name = os.path.basename(path)
logger.debug("loading kind `{}` from `{}`".format(kind_name, path))
kind_yml = os.path.join(path, 'kind.yml')
with open(kind_yml) as f:
@ -138,16 +138,16 @@ class TaskGraphGenerator(object):
for a in impl_object.split('.'):
impl_class = getattr(impl_class, a)
yield impl_class(path, config)
for task in impl_class.load_tasks(kind_name, path, config, self.parameters):
yield task
def _run(self):
logger.info("Generating full task set")
all_tasks = {}
for kind in self._load_kinds():
for task in kind.load_tasks(self.parameters):
if task.label in all_tasks:
raise Exception("duplicate tasks with label " + task.label)
all_tasks[task.label] = task
for task in self._load_kinds():
if task.label in all_tasks:
raise Exception("duplicate tasks with label " + task.label)
all_tasks[task.label] = task
full_task_set = TaskGraph(all_tasks, Graph(set(all_tasks), set()))
yield 'full_task_set', full_task_set
@ -155,7 +155,7 @@ class TaskGraphGenerator(object):
logger.info("Generating full task graph")
edges = set()
for t in full_task_set:
for dep, depname in t.kind.get_task_dependencies(t, full_task_set):
for dep, depname in t.get_dependencies(full_task_set):
edges.add((t.label, dep, depname))
full_task_graph = TaskGraph(all_tasks,

51
taskcluster/taskgraph/kind/base.py
View File

@ -4,12 +4,23 @@
from __future__ import absolute_import, print_function, unicode_literals
import os
import abc
class Kind(object):
class Task(object):
"""
Representation of a task in a TaskGraph. Each Task has, at creation:
- kind: the name of the task kind
- label; the label for this task
- attributes: a dictionary of attributes for this task (used for filtering)
- task: the task definition (JSON-able dictionary)
And later, as the task-graph processing proceeds:
- task_id -- TaskCluster taskId under which this task will be created
- optimized -- true if this task need not be performed
A kind represents a collection of tasks that share common characteristics.
For example, all build jobs. Each instance of a kind is intialized with a
path from which it draws its task configuration. The instance is free to
@ -17,15 +28,32 @@ class Kind(object):
"""
__metaclass__ = abc.ABCMeta
def __init__(self, path, config):
self.name = os.path.basename(path)
self.path = path
self.config = config
def __init__(self, kind, label, attributes, task):
self.kind = kind
self.label = label
self.attributes = attributes
self.task = task
self.task_id = None
self.optimized = False
self.attributes['kind'] = kind
if not (all(isinstance(x, basestring) for x in self.attributes.iterkeys()) and
all(isinstance(x, basestring) for x in self.attributes.itervalues())):
raise TypeError("attribute names and values must be strings")
@classmethod
@abc.abstractmethod
def load_tasks(self, parameters):
def load_tasks(cls, kind, path, config, parameters):
"""
Get the set of tasks of this kind.
Load the tasks for a given kind.
The `kind` is the name of the kind; the configuration for that kind
named this class.
The `path` is the path to the configuration directory for the kind. This
can be used to load extra data, templates, etc.
The `parameters` give details on which to base the task generation.
See `taskcluster/docs/parameters.rst` for details.
@ -34,15 +62,16 @@ class Kind(object):
"""
@abc.abstractmethod
def get_task_dependencies(self, task, taskgraph):
def get_dependencies(self, taskgraph):
"""
Get the set of task labels this task depends on, by querying the task graph.
Get the set of task labels this task depends on, by querying the full
task set, given as `taskgraph`.
Returns a list of (task_label, dependency_name) pairs describing the
dependencies.
"""
def optimize_task(self, task):
def optimize(self):
"""
Determine whether this task can be optimized, and if it can, what taskId
it should be replaced with.

36
taskcluster/taskgraph/kind/docker_image.py
View File

@ -12,7 +12,6 @@ import tarfile
import time
from . import base
from ..types import Task
from taskgraph.util.docker import (
docker_image,
generate_context_hash
@ -29,9 +28,14 @@ ARTIFACT_URL = 'https://queue.taskcluster.net/v1/task/{}/artifacts/{}'
INDEX_URL = 'https://index.taskcluster.net/v1/task/{}'
class DockerImageKind(base.Kind):
class DockerImageTask(base.Task):
def load_tasks(self, params):
def __init__(self, *args, **kwargs):
self.index_paths = kwargs.pop('index_paths')
super(DockerImageTask, self).__init__(*args, **kwargs)
@classmethod
def load_tasks(cls, kind, path, config, params):
# TODO: make this match the pushdate (get it from a parameter rather than vcs)
pushdate = time.strftime('%Y%m%d%H%M%S', time.gmtime())
@ -57,8 +61,8 @@ class DockerImageKind(base.Kind):
}
tasks = []
templates = Templates(self.path)
for image_name in self.config['images']:
templates = Templates(path)
for image_name in config['images']:
context_path = os.path.join('testing', 'docker', image_name)
context_hash = generate_context_hash(context_path)
@ -76,7 +80,7 @@ class DockerImageKind(base.Kind):
"artifacts/decision_task/image_contexts/{}/context.tar.gz".format(image_name))
image_parameters['context_url'] = ARTIFACT_URL.format(
os.environ['TASK_ID'], image_artifact_path)
self.create_context_tar(context_path, destination, image_name)
cls.create_context_tar(context_path, destination, image_name)
else:
# skip context generation since this isn't a decision task
# TODO: generate context tarballs using subdirectory clones in
@ -85,10 +89,7 @@ class DockerImageKind(base.Kind):
image_task = templates.load('image.yml', image_parameters)
attributes = {
'kind': self.name,
'image_name': image_name,
}
attributes = {'image_name': image_name}
# As an optimization, if the context hash exists for mozilla-central, that image
# task ID will be used. The reasoning behind this is that eventually everything ends
@ -99,17 +100,17 @@ class DockerImageKind(base.Kind):
project, image_name, context_hash)
for project in ['mozilla-central', params['project']]]
tasks.append(Task(self, 'build-docker-image-' + image_name,
task=image_task['task'], attributes=attributes,
index_paths=index_paths))
tasks.append(cls(kind, 'build-docker-image-' + image_name,
task=image_task['task'], attributes=attributes,
index_paths=index_paths))
return tasks
def get_task_dependencies(self, task, taskgraph):
def get_dependencies(self, taskgraph):
return []
def optimize_task(self, task, taskgraph):
for index_path in task.extra['index_paths']:
def optimize(self):
for index_path in self.index_paths:
try:
url = INDEX_URL.format(index_path)
existing_task = json.load(urllib2.urlopen(url))
@ -130,7 +131,8 @@ class DockerImageKind(base.Kind):
return False, None
def create_context_tar(self, context_dir, destination, image_name):
@classmethod
def create_context_tar(cls, context_dir, destination, image_name):
'Creates a tar file of a particular context directory.'
destination = os.path.abspath(destination)
if not os.path.exists(os.path.dirname(destination)):

47
taskcluster/taskgraph/kind/legacy.py
View File

@ -13,7 +13,6 @@ import time
from collections import namedtuple
from . import base
from ..types import Task
from mozpack.path import match as mozpackmatch
from slugid import nice as slugid
from taskgraph.util.legacy_commit_parser import parse_commit
@ -71,10 +70,10 @@ def gaia_info():
# Just use the hg params...
return {
'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
'gaia_ref': gaia['revision'],
'gaia_rev': gaia['revision']
'gaia_base_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
'gaia_head_repository': 'https://hg.mozilla.org/{}'.format(gaia['repo_path']),
'gaia_ref': gaia['revision'],
'gaia_rev': gaia['revision']
}
else:
@ -292,7 +291,7 @@ def validate_build_task(task):
'task.extra.locations.tests_packages missing')
class LegacyKind(base.Kind):
class LegacyTask(base.Task):
"""
This kind generates a full task graph from the old YAML files in
`testing/taskcluster/tasks`. The tasks already have dependency links.
@ -302,8 +301,13 @@ class LegacyKind(base.Kind):
"TaskLabel==". These labels are unfortunately not stable from run to run.
"""
def load_tasks(self, params):
root = os.path.abspath(os.path.join(self.path, self.config['legacy_path']))
def __init__(self, *args, **kwargs):
self.task_dict = kwargs.pop('task_dict')
super(LegacyTask, self).__init__(*args, **kwargs)
@classmethod
def load_tasks(cls, kind, path, config, params):
root = os.path.abspath(os.path.join(path, config['legacy_path']))
project = params['project']
# NOTE: message is ignored here; we always use DEFAULT_TRY, then filter the
@ -383,7 +387,7 @@ class LegacyKind(base.Kind):
graph['metadata'] = {
'source': '{repo}file/{rev}/testing/taskcluster/mach_commands.py'.format(
repo=params['head_repository'], rev=params['head_rev']),
repo=params['head_repository'], rev=params['head_rev']),
'owner': params['owner'],
# TODO: Add full mach commands to this example?
'description': 'Task graph generated via ./mach taskcluster-graph',
@ -497,7 +501,7 @@ class LegacyKind(base.Kind):
graph['scopes'] |= set(build_task['task'].get('scopes', []))
route_scopes = map(
lambda route: 'queue:route:' + route, build_task['task'].get('routes', [])
)
)
graph['scopes'] |= set(route_scopes)
# Treeherder symbol configuration for the graph required for each
@ -613,28 +617,25 @@ class LegacyKind(base.Kind):
graph['scopes'] = sorted(graph['scopes'])
# save the graph for later, when taskgraph asks for additional information
# such as dependencies
self.graph = graph
self.tasks_by_label = {t['taskId']: t for t in self.graph['tasks']}
# Convert to a dictionary of tasks. The process above has invented a
# taskId for each task, and we use those as the *labels* for the tasks;
# taskgraph will later assign them new taskIds.
return [Task(self, t['taskId'], task=t['task'], attributes=t['attributes'])
for t in self.graph['tasks']]
return [
cls(kind, t['taskId'], task=t['task'], attributes=t['attributes'], task_dict=t)
for t in graph['tasks']
]
def get_task_dependencies(self, task, taskgraph):
def get_dependencies(self, taskgraph):
# fetch dependency information from the cached graph
taskdict = self.tasks_by_label[task.label]
deps = [(label, label) for label in taskdict.get('requires', [])]
deps = [(label, label) for label in self.task_dict.get('requires', [])]
# add a dependency on an image task, if needed
if 'docker-image' in taskdict:
deps.append(('build-docker-image-{docker-image}'.format(**taskdict), 'docker-image'))
if 'docker-image' in self.task_dict:
deps.append(('build-docker-image-{docker-image}'.format(**self.task_dict),
'docker-image'))
return deps
def optimize_task(self, task, taskgraph):
def optimize(self):
# no optimization for the moment
return False, None

2
taskcluster/taskgraph/optimize.py
View File

@ -88,7 +88,7 @@ def annotate_task_graph(target_task_graph, do_not_optimize, named_links_dict, la
optimized = False
# otherwise, examine the task itself (which may be an expensive operation)
else:
optimized, replacement_task_id = task.kind.optimize_task(task, named_task_dependencies)
optimized, replacement_task_id = task.optimize()
task.optimized = optimized
task.task_id = replacement_task_id

23
taskcluster/taskgraph/test/test_create.py
View File

@ -9,20 +9,12 @@ import os
from .. import create
from ..graph import Graph
from ..types import Task, TaskGraph
from ..types import TaskGraph
from .util import TestTask
from mozunit import main
class FakeKind(object):
def get_task_definition(self, task, deps_by_name):
# sanity-check the deps_by_name
for k, v in deps_by_name.iteritems():
assert k == 'edge'
return {'payload': 'hello world'}
class TestCreate(unittest.TestCase):
def setUp(self):
@ -44,11 +36,9 @@ class TestCreate(unittest.TestCase):
self.created_tasks[task_id] = task_def
def test_create_tasks(self):
os.environ['TASK_ID'] = 'decisiontask'
kind = FakeKind()
tasks = {
'tid-a': Task(kind=kind, label='a', task={'payload': 'hello world'}),
'tid-b': Task(kind=kind, label='b', task={'payload': 'hello world'}),
'tid-a': TestTask(label='a', task={'payload': 'hello world'}),
'tid-b': TestTask(label='b', task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a', 'b': 'tid-b'}
graph = Graph(nodes={'tid-a', 'tid-b'}, edges={('tid-a', 'tid-b', 'edge')})
@ -68,9 +58,8 @@ class TestCreate(unittest.TestCase):
def test_create_task_without_dependencies(self):
"a task with no dependencies depends on the decision task"
os.environ['TASK_ID'] = 'decisiontask'
kind = FakeKind()
tasks = {
'tid-a': Task(kind=kind, label='a', task={'payload': 'hello world'}),
'tid-a': TestTask(label='a', task={'payload': 'hello world'}),
}
label_to_taskid = {'a': 'tid-a'}
graph = Graph(nodes={'tid-a'}, edges=set())
@ -79,7 +68,7 @@ class TestCreate(unittest.TestCase):
create.create_tasks(taskgraph, label_to_taskid)
for tid, task in self.created_tasks.iteritems():
self.assertEqual(task['dependencies'], [os.environ['TASK_ID']])
self.assertEqual(task.get('dependencies'), [os.environ['TASK_ID']])
if __name__ == '__main__':

11
taskcluster/taskgraph/test/test_decision.py
View File

@ -13,7 +13,8 @@ import tempfile
from .. import decision
from ..graph import Graph
from ..types import Task, TaskGraph
from ..types import TaskGraph
from .util import TestTask
from mozunit import main
@ -21,8 +22,8 @@ class TestDecision(unittest.TestCase):
def test_taskgraph_to_json(self):
tasks = {
'a': Task(kind=None, label='a', attributes={'attr': 'a-task'}),
'b': Task(kind=None, label='b', task={'task': 'def'}),
'a': TestTask(label='a', attributes={'attr': 'a-task'}),
'b': TestTask(label='b', task={'task': 'def'}),
}
graph = Graph(nodes=set('ab'), edges={('a', 'b', 'edgelabel')})
taskgraph = TaskGraph(tasks, graph)
@ -32,13 +33,13 @@ class TestDecision(unittest.TestCase):
self.assertEqual(res, {
'a': {
'label': 'a',
'attributes': {'attr': 'a-task'},
'attributes': {'attr': 'a-task', 'kind': 'test'},
'task': {},
'dependencies': {'edgelabel': 'b'},
},
'b': {
'label': 'b',
'attributes': {},
'attributes': {'kind': 'test'},
'task': {'task': 'def'},
'dependencies': {},
}

37
taskcluster/taskgraph/test/test_generator.py
View File

@ -7,40 +7,41 @@ from __future__ import absolute_import, print_function, unicode_literals
import unittest
from ..generator import TaskGraphGenerator
from .. import types
from .. import graph
from ..kind import base
from mozunit import main
class FakeKind(object):
class FakeTask(base.Task):
def maketask(self, i):
return types.Task(
self,
label='t-{}'.format(i),
attributes={'tasknum': str(i)},
task={},
i=i)
def __init__(self, **kwargs):
self.i = kwargs.pop('i')
super(FakeTask, self).__init__(**kwargs)
def load_tasks(self, parameters):
self.tasks = [self.maketask(i) for i in range(3)]
return self.tasks
@classmethod
def load_tasks(cls, kind, path, config, parameters):
return [cls(kind=kind,
label='t-{}'.format(i),
attributes={'tasknum': str(i)},
task={},
i=i)
for i in range(3)]
def get_task_dependencies(self, task, full_task_set):
i = task.extra['i']
def get_dependencies(self, full_task_set):
i = self.i
if i > 0:
return [('t-{}'.format(i - 1), 'prev')]
else:
return []
def optimize_task(self, task, dependencies):
def optimize(self):
return False, None
class WithFakeKind(TaskGraphGenerator):
class WithFakeTask(TaskGraphGenerator):
def _load_kinds(self):
yield FakeKind()
return FakeTask.load_tasks('fake', '/fake', {}, {})
class TestGenerator(unittest.TestCase):
@ -50,7 +51,7 @@ class TestGenerator(unittest.TestCase):
def target_tasks_method(full_task_graph, parameters):
return self.target_tasks
self.tgg = WithFakeKind('/root', {}, target_tasks_method)
self.tgg = WithFakeTask('/root', {}, target_tasks_method)
def test_full_task_set(self):
"The full_task_set property has all tasks"

16
taskcluster/taskgraph/test/test_kind_docker_image.py
View File

@ -12,23 +12,29 @@ from ..kind import docker_image
from mozunit import main
KIND_PATH = os.path.join(docker_image.GECKO, 'taskcluster', 'ci', 'docker-image')
class TestDockerImageKind(unittest.TestCase):
def setUp(self):
self.kind = docker_image.DockerImageKind(
os.path.join(docker_image.GECKO, 'taskcluster', 'ci', 'docker-image'),
{})
self.task = docker_image.DockerImageTask(
'docker-image',
KIND_PATH,
{},
{},
index_paths=[])
def test_get_task_dependencies(self):
# this one's easy!
self.assertEqual(self.kind.get_task_dependencies(None, None), [])
self.assertEqual(self.task.get_dependencies(None), [])
# TODO: optimize_task
def test_create_context_tar(self):
image_dir = os.path.join(docker_image.GECKO, 'testing', 'docker', 'image_builder')
tarball = tempfile.mkstemp()[1]
self.kind.create_context_tar(image_dir, tarball, 'image_builder')
self.task.create_context_tar(image_dir, tarball, 'image_builder')
self.failUnless(os.path.exists(tarball))
os.unlink(tarball)

10
taskcluster/taskgraph/test/test_kind_legacy.py
View File

@ -7,22 +7,12 @@ from __future__ import absolute_import, print_function, unicode_literals
import unittest
from ..kind.legacy import (
LegacyKind,
validate_build_task,
BuildTaskValidationException
)
from mozunit import main
class TestLegacyKind(unittest.TestCase):
# NOTE: much of LegacyKind is copy-pasted from the old legacy code, which
# is emphatically *not* designed for testing, so this test class does not
# attempt to test the entire class.
def setUp(self):
self.kind = LegacyKind('/root', {})
class TestValidateBuildTask(unittest.TestCase):
def test_validate_missing_extra(self):

53
taskcluster/taskgraph/test/test_optimize.py
View File

@ -10,6 +10,7 @@ from ..optimize import optimize_task_graph, resolve_task_references
from ..optimize import annotate_task_graph, get_subgraph
from .. import types
from .. import graph
from .util import TestTask
class TestResolveTaskReferences(unittest.TestCase):
@ -49,32 +50,29 @@ class TestResolveTaskReferences(unittest.TestCase):
KeyError,
"task 'subject' has no dependency with label 'no-such'",
lambda: resolve_task_references('subject', {'task-reference': '<no-such>'}, {})
)
)
class FakeKind(object):
def __init__(self, optimize_task):
self.optimize_task = optimize_task
class OptimizingTask(TestTask):
# the `optimize` method on this class is overridden direclty in the tests
# below.
pass
class TestOptimize(unittest.TestCase):
kind = None
def make_kind(self, optimize_task):
self.kind = FakeKind(optimize_task)
def make_task(self, label, task_def=None, optimized=None, task_id=None):
task_def = task_def or {'sample': 'task-def'}
task = types.Task(self.kind, label=label, task=task_def)
task = OptimizingTask(label=label, task=task_def)
task.optimized = optimized
task.task_id = task_id
return task
def make_graph(self, *tasks_and_edges):
tasks = {t.label: t for t in tasks_and_edges if isinstance(t, types.Task)}
edges = {e for e in tasks_and_edges if not isinstance(e, types.Task)}
tasks = {t.label: t for t in tasks_and_edges if isinstance(t, OptimizingTask)}
edges = {e for e in tasks_and_edges if not isinstance(e, OptimizingTask)}
return types.TaskGraph(tasks, graph.Graph(set(tasks), edges))
def assert_annotations(self, graph, **annotations):
@ -82,12 +80,12 @@ class TestOptimize(unittest.TestCase):
return 'SLUGID' if task_id and len(task_id) == 22 else task_id
got_annotations = {
t.label: (t.optimized, repl(t.task_id)) for t in graph.tasks.itervalues()
}
}
self.assertEqual(got_annotations, annotations)
def test_annotate_task_graph_no_optimize(self):
"annotating marks everything as un-optimized if the kind returns that"
self.make_kind(lambda task, deps: (False, None))
OptimizingTask.optimize = lambda self: (False, None)
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
@ -101,20 +99,21 @@ class TestOptimize(unittest.TestCase):
task1=(False, None),
task2=(False, None),
task3=(False, None)
)
)
def test_annotate_task_graph_taskid_without_optimize(self):
"raises exception if kind returns a taskid without optimizing"
self.make_kind(lambda task, deps: (False, 'some-taskid'))
OptimizingTask.optimize = lambda self: (False, 'some-taskid')
graph = self.make_graph(self.make_task('task1'))
self.assertRaises(
Exception,
lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {})
)
)
def test_annotate_task_graph_optimize_away_dependency(self):
"raises exception if kind optimizes away a task on which another depends"
self.make_kind(lambda task, deps: (True, None) if task.label == 'task1' else (False, None))
OptimizingTask.optimize = \
lambda self: (True, None) if self.label == 'task1' else (False, None)
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
@ -123,11 +122,11 @@ class TestOptimize(unittest.TestCase):
self.assertRaises(
Exception,
lambda: annotate_task_graph(graph, set(), graph.graph.named_links_dict(), {})
)
)
def test_annotate_task_graph_do_not_optimize(self):
"annotating marks everything as un-optimized if in do_not_optimize"
self.make_kind(lambda task, deps: (True, 'taskid'))
OptimizingTask.optimize = lambda self: (True, 'taskid')
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
@ -140,14 +139,13 @@ class TestOptimize(unittest.TestCase):
graph,
task1=(False, None),
task2=(False, None)
)
)
self.assertEqual
def test_annotate_task_graph_nos_propagate(self):
"annotating marks a task with a non-optimized dependency as non-optimized"
self.make_kind(
lambda task, deps: (False, None) if task.label == 'task1' else (True, 'taskid')
)
OptimizingTask.optimize = \
lambda self: (False, None) if self.label == 'task1' else (True, 'taskid')
graph = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),
@ -162,7 +160,7 @@ class TestOptimize(unittest.TestCase):
task1=(False, None),
task2=(False, None), # kind would have returned (True, 'taskid') here
task3=(True, 'taskid')
)
)
def test_get_subgraph_single_dep(self):
"when a single dependency is optimized, it is omitted from the graph"
@ -225,7 +223,7 @@ class TestOptimize(unittest.TestCase):
'task2',
optimized=False,
task_def={'payload': {'task-reference': 'http://<build>/<test>'}}
),
),
('task2', 'task1', 'build'),
('task2', 'task3', 'test'),
self.make_task('task3', optimized=False),
@ -243,9 +241,8 @@ class TestOptimize(unittest.TestCase):
def test_optimize(self):
"optimize_task_graph annotates and extracts the subgraph from a simple graph"
self.make_kind(
lambda task, deps: (True, 'dep1') if task.label == 'task1' else (False, None)
)
OptimizingTask.optimize = \
lambda self: (True, 'dep1') if self.label == 'task1' else (False, None)
input = self.make_graph(
self.make_task('task1'),
self.make_task('task2'),

13
taskcluster/taskgraph/test/test_target_tasks.py
View File

@ -9,7 +9,8 @@ import unittest
from .. import target_tasks
from .. import try_option_syntax
from ..graph import Graph
from ..types import Task, TaskGraph
from ..types import TaskGraph
from .util import TestTask
from mozunit import main
@ -32,16 +33,16 @@ class TestTargetTasks(unittest.TestCase):
def test_all_builds_and_tests(self):
method = target_tasks.get_method('all_builds_and_tests')
graph = TaskGraph(tasks={
'a': Task(kind=None, label='a', attributes={'kind': 'legacy'}),
'b': Task(kind=None, label='b', attributes={'kind': 'legacy'}),
'boring': Task(kind=None, label='boring', attributes={'kind': 'docker-image'}),
'a': TestTask(kind='legacy', label='a'),
'b': TestTask(kind='legacy', label='b'),
'boring': TestTask(kind='docker', label='boring'),
}, graph=Graph(nodes={'a', 'b', 'boring'}, edges=set()))
self.assertEqual(sorted(method(graph, {})), sorted(['a', 'b']))
def test_try_option_syntax(self):
tasks = {
'a': Task(kind=None, label='a'),
'b': Task(kind=None, label='b', attributes={'at-at': 'yep'}),
'a': TestTask(kind=None, label='a'),
'b': TestTask(kind=None, label='b', attributes={'at-at': 'yep'}),
}
graph = Graph(nodes=set('ab'), edges=set())
tg = TaskGraph(tasks, graph)

11
taskcluster/taskgraph/test/test_try_option_syntax.py
View File

@ -8,7 +8,8 @@ import unittest
from ..try_option_syntax import TryOptionSyntax
from ..graph import Graph
from ..types import TaskGraph, Task
from ..types import TaskGraph
from .util import TestTask
from mozunit import main
# an empty graph, for things that don't look at it
@ -16,14 +17,14 @@ empty_graph = TaskGraph({}, Graph(set(), set()))
def unittest_task(n, tp):
return (n, Task('test', n, {
return (n, TestTask('test', n, {
'unittest_try_name': n,
'test_platform': tp,
}))
def talos_task(n, tp):
return (n, Task('test', n, {
return (n, TestTask('test', n, {
'talos_try_name': n,
'test_platform': tp,
}))
@ -258,8 +259,8 @@ class TestTryOptionSyntax(unittest.TestCase):
# -t shares an implementation with -u, so it's not tested heavily
def test_trigger_tests(self):
"--trigger-tests 10 sets trigger_tests"
tos = TryOptionSyntax('try: --trigger-tests 10', empty_graph)
"--rebuild 10 sets trigger_tests"
tos = TryOptionSyntax('try: --rebuild 10', empty_graph)
self.assertEqual(tos.trigger_tests, 10)
def test_interactive(self):

24
taskcluster/taskgraph/test/util.py Normal file
View File

@ -0,0 +1,24 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from ..kind import base
class TestTask(base.Task):
def __init__(self, kind=None, label=None, attributes=None, task=None):
super(TestTask, self).__init__(
kind or 'test',
label or 'test-label',
attributes or {},
task or {})
@classmethod
def load_tasks(cls, kind, path, config, parameters):
return []
def get_dependencies(self, taskgraph):
return []

36
taskcluster/taskgraph/types.py
View File

@ -5,42 +5,6 @@
from __future__ import absolute_import, print_function, unicode_literals
class Task(object):
"""
Representation of a task in a TaskGraph.
Each has, at creation:
- kind: Kind instance that created this task
- label; the label for this task
- attributes: a dictionary of attributes for this task (used for filtering)
- task: the task definition (JSON-able dictionary)
- extra: extra kind-specific metadata
And later, as the task-graph processing proceeds:
- optimization_key -- key for finding equivalent tasks in the TC index
- task_id -- TC taskId under which this task will be created
"""
def __init__(self, kind, label, attributes=None, task=None, **extra):
self.kind = kind
self.label = label
self.attributes = attributes or {}
self.task = task or {}
self.extra = extra
self.task_id = None
if not (all(isinstance(x, basestring) for x in self.attributes.iterkeys()) and
all(isinstance(x, basestring) for x in self.attributes.itervalues())):
raise TypeError("attribute names and values must be strings")
def __str__(self):
return "{} ({})".format(self.task_id or self.label,
self.task['metadata']['description'].strip())
class TaskGraph(object):
"""
Representation of a task graph.