mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-19 08:15:31 +00:00
Bug 1652086: Make kind_dependencies_tasks
a dictionary based on the label; r=Callek
I was looking at py-spy for generating the taskgraph, and found that a bunch of time was spent in taskgraph.transforms.job.use_fetches[1]. Use a dictionary there instead saves about 20-30s on my machine. [1] https://searchfox.org/mozilla-central/rev/622dbd3409610ad3f71b56c9a6a92da905dab0aa/taskcluster/taskgraph/transforms/job/__init__.py#243-247 Differential Revision: https://phabricator.services.mozilla.com/D83166
This commit is contained in:
parent
93c5659e4a
commit
51f106a3a3
@ -54,8 +54,8 @@ class Kind(object):
|
||||
config = copy.deepcopy(self.config)
|
||||
|
||||
kind_dependencies = config.get('kind-dependencies', [])
|
||||
kind_dependencies_tasks = [task for task in loaded_tasks
|
||||
if task.kind in kind_dependencies]
|
||||
kind_dependencies_tasks = {task.label: task for task in loaded_tasks
|
||||
if task.kind in kind_dependencies}
|
||||
|
||||
inputs = loader(self.name, self.path, config, parameters, loaded_tasks)
|
||||
|
||||
|
@ -54,7 +54,7 @@ def cache_task(config, tasks):
|
||||
return
|
||||
|
||||
digests = {}
|
||||
for task in config.kind_dependencies_tasks:
|
||||
for task in config.kind_dependencies_tasks.values():
|
||||
if 'cached_task' in task.attributes:
|
||||
digests[task.label] = format_task_digest(task.attributes['cached_task'])
|
||||
|
||||
|
@ -18,7 +18,7 @@ def add_dependencies(config, jobs):
|
||||
job.setdefault('soft-dependencies', [])
|
||||
job['soft-dependencies'] += [
|
||||
dep_task.label
|
||||
for dep_task in config.kind_dependencies_tasks
|
||||
for dep_task in config.kind_dependencies_tasks.values()
|
||||
if dep_task.attributes.get('code-review') is True
|
||||
]
|
||||
yield job
|
||||
|
@ -82,13 +82,6 @@ transforms.add_validate(docker_image_schema)
|
||||
|
||||
@transforms.add
|
||||
def fill_template(config, tasks):
|
||||
available_packages = set()
|
||||
for task in config.kind_dependencies_tasks:
|
||||
if task.kind != 'packages':
|
||||
continue
|
||||
name = task.label.replace('packages-', '')
|
||||
available_packages.add(name)
|
||||
|
||||
if not taskgraph.fast and config.write_artifacts:
|
||||
if not os.path.isdir(CONTEXTS_DIR):
|
||||
os.makedirs(CONTEXTS_DIR)
|
||||
@ -102,7 +95,7 @@ def fill_template(config, tasks):
|
||||
parent = task.pop('parent', None)
|
||||
|
||||
for p in packages:
|
||||
if p not in available_packages:
|
||||
if "packages-{}".format(p) not in config.kind_dependencies_tasks:
|
||||
raise Exception('Missing package job for {}-{}: {}'.format(
|
||||
config.kind, image_name, p))
|
||||
|
||||
|
@ -187,7 +187,7 @@ def use_fetches(config, jobs):
|
||||
if value:
|
||||
aliases['{}-{}'.format(config.kind, value)] = label
|
||||
|
||||
for task in config.kind_dependencies_tasks:
|
||||
for task in config.kind_dependencies_tasks.values():
|
||||
if task.kind in ('fetch', 'toolchain'):
|
||||
get_attribute(
|
||||
artifact_names, task.label, task.attributes,
|
||||
@ -240,25 +240,17 @@ def use_fetches(config, jobs):
|
||||
if dep_label in artifact_prefixes:
|
||||
prefix = artifact_prefixes[dep_label]
|
||||
else:
|
||||
dep_tasks = [
|
||||
task
|
||||
for task in config.kind_dependencies_tasks
|
||||
if task.label == dep_label
|
||||
]
|
||||
if len(dep_tasks) != 1:
|
||||
if dep_label not in config.kind_dependencies_tasks:
|
||||
raise Exception(
|
||||
"{name} can't fetch {kind} artifacts because "
|
||||
"there are {tasks} with label {label} in kind dependencies!".format(
|
||||
"there are no tasks with label {label} in kind dependencies!".format(
|
||||
name=name,
|
||||
kind=kind,
|
||||
label=dependencies[kind],
|
||||
tasks="no tasks"
|
||||
if len(dep_tasks) == 0
|
||||
else "multiple tasks",
|
||||
)
|
||||
)
|
||||
|
||||
prefix = get_artifact_prefix(dep_tasks[0])
|
||||
prefix = get_artifact_prefix(config.kind_dependencies_tasks[dep_label])
|
||||
|
||||
for artifact in artifacts:
|
||||
if isinstance(artifact, text_type):
|
||||
|
@ -25,7 +25,7 @@ def add_dependencies(config, jobs):
|
||||
continue
|
||||
|
||||
required_signoffs = set(job.setdefault('attributes', {}).get('required_signoffs', []))
|
||||
for dep_task in config.kind_dependencies_tasks:
|
||||
for dep_task in config.kind_dependencies_tasks.values():
|
||||
# Weed out unwanted tasks.
|
||||
# XXX we have run-on-projects which specifies the on-push behavior;
|
||||
# we need another attribute that specifies release promotion,
|
||||
|
@ -18,7 +18,7 @@ transforms = TransformSequence()
|
||||
@transforms.add
|
||||
def add_command(config, tasks):
|
||||
config_tasks = {}
|
||||
for dep in config.kind_dependencies_tasks:
|
||||
for dep in config.kind_dependencies_tasks.values():
|
||||
if 'update-verify-config' in dep.kind or 'update-verify-next-config' in dep.kind:
|
||||
config_tasks[dep.name] = dep
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user