Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-24 13:21:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1748929 - Upgrade taskgraph to version 1.4.0, r=releng-reviewers,jcristau

Browse Source 
Differential Revision: https://phabricator.services.mozilla.com/D144622
This commit is contained in:
Andrew Halberstadt 2022-04-26 13:17:48 +00:00
parent 5aae6d2f7b
commit a4e3dd8c6a
14 changed files with 202 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
third_party/python/poetry.lock generated vendored
View File

@ -606,7 +606,7 @@ test = ["pytest", "pytest-cov", "pytest-mock", "httmock", "mock", "setuptools-li
[[package]]
name = "taskcluster-taskgraph"
version = "1.2.0"
version = "1.4.0"
description = "Build taskcluster taskgraphs"
category = "main"
optional = false
@ -728,7 +728,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt
[metadata]
lock-version = "1.1"
python-versions = "^3.6"
content-hash = "373971de74385d2b0f2e060e38b05d923ee13d1d4f2974226ef98c68314697df"
content-hash = "2c8645955b35c4a1b90c8f5473f2b29a8d492613ea054cb17dbc20dadcaf19d6"
[metadata.files]
aiohttp = [
@ -1112,8 +1112,8 @@ taskcluster = [
{file = "taskcluster-44.2.2.tar.gz", hash = "sha256:0266a6a901e1a2ec838984a7f24e7adb6d58f9f2e221a7f613388f8f23f786fc"},
]
taskcluster-taskgraph = [
{file = "taskcluster-taskgraph-1.2.0.tar.gz", hash = "sha256:bce1ef59f15b317256214da4d82f19d745bdc1b4497561d753436f4efa32ea39"},
{file = "taskcluster_taskgraph-1.2.0-py3-none-any.whl", hash = "sha256:ac7b0345b5b8752c5b31c516c9d798d7168bfda8963774def972b0beee349f6e"},
{file = "taskcluster-taskgraph-1.4.0.tar.gz", hash = "sha256:237ed0399ef55e9eef537e6163b456c04d58a7b72bc1b55e8c61a05331170b1e"},
{file = "taskcluster_taskgraph-1.4.0-py3-none-any.whl", hash = "sha256:2be5ac94745180029dfad260486aafeb14f564f5a95d1d3269e0d17834273f11"},
]
taskcluster-urls = [
{file = "taskcluster-urls-13.0.1.tar.gz", hash = "sha256:b25e122ecec249c4299ac7b20b08db76e3e2025bdaeb699a9d444556de5fd367"},

2
third_party/python/requirements.in vendored
View File

@ -37,7 +37,7 @@ setuptools==51.2.0
six==1.13.0
slugid==2.0.0
taskcluster==44.2.2
taskcluster-taskgraph==1.2.0
taskcluster-taskgraph==1.4.0
taskcluster-urls==13.0.1
tqdm==4.62.3
urllib3==1.26

6
third_party/python/requirements.txt vendored
View File

@ -319,9 +319,9 @@ six==1.13.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (pyt
slugid==2.0.0 \
--hash=sha256:aec8b0e01c4ad32e38e12d609eab3ec912fd129aaf6b2ded0199b56a5f8fd67c \
--hash=sha256:a950d98b72691178bdd4d6c52743c4a2aa039207cf7a97d71060a111ff9ba297
taskcluster-taskgraph==1.2.0 \
--hash=sha256:bce1ef59f15b317256214da4d82f19d745bdc1b4497561d753436f4efa32ea39 \
--hash=sha256:ac7b0345b5b8752c5b31c516c9d798d7168bfda8963774def972b0beee349f6e
taskcluster-taskgraph==1.4.0 \
--hash=sha256:237ed0399ef55e9eef537e6163b456c04d58a7b72bc1b55e8c61a05331170b1e \
--hash=sha256:2be5ac94745180029dfad260486aafeb14f564f5a95d1d3269e0d17834273f11
taskcluster-urls==13.0.1 \
--hash=sha256:b25e122ecec249c4299ac7b20b08db76e3e2025bdaeb699a9d444556de5fd367 \
--hash=sha256:5e25e7e6818e8877178b175ff43d2e6548afad72694aa125f404a7329ece0973 \

4
third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.2.0.dist-info/METADATA → third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.4.0.dist-info/METADATA vendored
View File

@ -1,8 +1,8 @@
Metadata-Version: 2.1
Name: taskcluster-taskgraph
Version: 1.2.0
Version: 1.4.0
Summary: Build taskcluster taskgraphs
Home-page: https://hg.mozilla.org/ci/taskgraph
Home-page: https://github.com/taskcluster/taskgraph
License: UNKNOWN
Platform: UNKNOWN
Classifier: Development Status :: 5 - Production/Stable

33
third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.2.0.dist-info/RECORD → third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.4.0.dist-info/RECORD vendored
View File

@ -1,14 +1,14 @@
taskgraph/__init__.py,sha256=jwOtU7TkmU317LP_IsgIswpj2T1OPUXXgMRv4sIU7nE,707
taskgraph/config.py,sha256=Bm0RU1LyEj4RuX_lBnpQrr5fGmx7zCbKleN1Zy6fQ5w,4410
taskgraph/create.py,sha256=VeAYxtLK8f5HufgUSjzDRP7WBSwQza--_O7RzRRhzA4,5190
taskgraph/decision.py,sha256=GlK1rs1VoWqoTfHM9oJzf0LS608HJVtEYWIPmQ3iEhc,9418
taskgraph/decision.py,sha256=TpKJyWJIT6mCI_wPORKHqSqWVwmZGzpR7VJuf-3kj-M,9724
taskgraph/docker.py,sha256=HlCEtfW2coUAyGce8ToVfn8OdLfQJT7Vq_mYQntO4gQ,7485
taskgraph/files_changed.py,sha256=30ONF-m1hbGQnQ31E-TCsHG1jhdNjLTOolSXu2jzOCs,2159
taskgraph/filter_tasks.py,sha256=ty4SK0ujcZ8824F4ikwdCaGX1TRSq_90TFArDogJHTo,875
taskgraph/generator.py,sha256=Axwun0FeuEXKZpCKD3F4HHCk5K3STeYrR2gr15DY9DM,14549
taskgraph/generator.py,sha256=y89WzZmfeHNNxKM3k60x7L2IJMCciXad5u6lNhZ0mdY,15079
taskgraph/graph.py,sha256=W7I-lfDiBMqSj5ZKgqVmebwTaNxdmbKLx2MqFT_cD_I,4666
taskgraph/main.py,sha256=Dz-uH6QJ3Xtd_t-9gD5iwS4a0CmKypM9aQSW2eZCFao,23254
taskgraph/morph.py,sha256=EH_kP5FAMb0F8Oap4VZeVpMNXGr0QhDVz_lxiY-ksgA,9562
taskgraph/morph.py,sha256=xX6bU-4iJtwwMw60_0Qt9mG8N4i6zhp4fB_BUZOjQ4o,9567
taskgraph/optimize.py,sha256=qcYcxSmhN9joPZGFEdW6D27G5F4xorS3wmX6TO-gC1Q,16487
taskgraph/parameters.py,sha256=QSVVMhoOQ0Uck4rE9_McFkDByUvidAG0yJFLLxLGuEE,10744
taskgraph/target_tasks.py,sha256=mspItlKD-HCuQR1x_UD6HT_Qd1v5kEHvgRWIUbweRDg,3166
@ -29,10 +29,10 @@ taskgraph/run-task/robustcheckout.py,sha256=P41ZGrec5aa8hVTEqOkKJ9wTygsgaXtjoQnl
taskgraph/run-task/run-task,sha256=XDT8N9NPIYBZIptcLliHDlRVuVClXcmrCOlSflYd-7s,39869
taskgraph/test/__init__.py,sha256=7LTScvkVcLPqivjL-wCb_Znk2GGajaJi1fJ4SjcLMoA,289
taskgraph/test/automationrelevance.json,sha256=ttGejNZeVhcTWXFdoU0xME0YEYRYvaYcAr8zBz0ewrw,17980
taskgraph/test/conftest.py,sha256=urHJNFOw7mVSr8WzIDZtOZPbXykdR--iw3J_hiF2rjg,3584
taskgraph/test/conftest.py,sha256=GoaEjxBClxzByVNkhwq1UDBC9uzOr4vy8PoWtqqNyeU,3615
taskgraph/test/mockedopen.py,sha256=Ccr2qGJSLeWIUYd03Poy8eKKRSW2aTpRGI-0AA7xYrw,4055
taskgraph/test/test_create.py,sha256=oY7DeVW8usjVNe-QPPyTrE3hsvwMo9HvMxPSCllOsMQ,3657
taskgraph/test/test_decision.py,sha256=dxlMnRtpKZrOqSZo-znPLw6TwlFJOB9Yp0sadEnJdaQ,2658
taskgraph/test/test_decision.py,sha256=twp07KM4iaw0hPOnX0YkmNOXD1jVlAmWc2ry5sqLPSk,2681
taskgraph/test/test_files_changed.py,sha256=MoHr_M-qtbi9PbKQ9loDlKOOzecUUyE1N0SgxjXfP5Y,2608
taskgraph/test/test_generator.py,sha256=6-ZpG45F3YlTWTCILrktz7LeSs3tEfQWCzekN6OHYHw,4284
taskgraph/test/test_graph.py,sha256=IEly2SS9NZTN3F0AM4VhxxYx0WTj5u7x2UFyojr1Ddg,7064
@ -44,11 +44,11 @@ taskgraph/test/test_scripts_run_task.py,sha256=ucv3oVin-gTwf7DUtFPG_bA3cqvQdRIKa
taskgraph/test/test_target_tasks.py,sha256=AzvuEw1NI4b_f14rPFkWuK0bJyxknkPIQi5KI6k2K8A,12046
taskgraph/test/test_taskgraph.py,sha256=EA09CzcgZBUoXVqhAWeEDPrnUF8Vw2gbTPwHrZGCWDc,3760
taskgraph/test/test_transforms_base.py,sha256=Vo9slzCB2GePvMoLmkrSdhYVWh2nQYn5bRxMjsx40Mw,873
taskgraph/test/test_transforms_job.py,sha256=pHO_Ea32b3OjCSIYhRHK3CqHsugJizD43_a3jMFtG_A,4490
taskgraph/test/test_transforms_job.py,sha256=eEbbnet6Pvunz7lxiN09lIpTyVoXJDcDEPhkWYl0nAc,4656
taskgraph/test/test_util_attributes.py,sha256=K_Wro-p5oA-S4yrsLqT8HMBlOAN4L0krQQQ82WYgGAQ,3596
taskgraph/test/test_util_docker.py,sha256=TKe9D5d3q8rK0iA5rMhzbmhQ4c4OSnVKrjioeZ1uNuo,8800
taskgraph/test/test_util_memoize.py,sha256=yq-PTegHBdQZkRm6Iv1NdCRcKCVzsb4pmZPvIz0P6j8,2340
taskgraph/test/test_util_parameterization.py,sha256=IW8zWFMi7INVuAkpT5rk674nZVZ4BhHrKRTz8HKN-O0,7945
taskgraph/test/test_util_parameterization.py,sha256=TEI3WVpL9Z54xVk8RZk7u3NwzddJxdmuAWfZa1IPaFw,7485
taskgraph/test/test_util_path.py,sha256=icJyH1DyMNVuZ5xfPXvrEQwQ0pQYTg4ORlZq3RK6_V8,5906
taskgraph/test/test_util_python_path.py,sha256=VQo4hwsJ0It-jLIqe4nErPmZn9AQ7rliN25h2oO_zMg,1216
taskgraph/test/test_util_readonlydict.py,sha256=KRgjLvSBsZZj4EUhwcqeUsM1T--iGklVE0QJuC6Xv4o,1234
@ -57,7 +57,8 @@ taskgraph/test/test_util_taskcluster.py,sha256=hZqtF7OC58OO4nW4URxF5ZTyxnNiwh2Jk
taskgraph/test/test_util_templates.py,sha256=u3ckrzmx1eyk7vXiqRiQlETtVZvSx4FHEXN5xB4GZDQ,1676
taskgraph/test/test_util_time.py,sha256=SG4WmSupTQiL1UhE2UMsMNZEugfPdoaxO4aWSxLGXBM,1803
taskgraph/test/test_util_treeherder.py,sha256=20zzGcMd0BL0ayTFQj6Etj39afdxZPgtZxSGUZ0iL5M,912
taskgraph/test/test_util_vcs.py,sha256=zL9j_PaSBPTuncgCpjvR5SDL0wIAkblPSTabn9qU-qs,6625
taskgraph/test/test_util_vcs.py,sha256=gWQcbEUYW0ApaGrXJDplE2Oze1i1_LxIchXEAcJjn_Y,6838
taskgraph/test/test_util_verify.py,sha256=hTia5J9DkaGejCnHU9FafBSjhZdE7LfmedFeUcmn8Bc,2486
taskgraph/test/test_util_yaml.py,sha256=zymZxaAZBIBn5u-p91QsA---IqCH_CVVk3YqMoshLlQ,1019
taskgraph/transforms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
taskgraph/transforms/base.py,sha256=oIQdBKrHG_FZXAoRyiazxgLa8zbAI4TEVlduqz8H12I,5277
@ -65,11 +66,11 @@ taskgraph/transforms/cached_tasks.py,sha256=fMCbxqA-HPSMG6mILYCfdIpnUg9uSKlQGKkU
taskgraph/transforms/code_review.py,sha256=du7npIGOpVMK9QZtdkHrkNecRiaqd-fHjHavDVkfL58,698
taskgraph/transforms/docker_image.py,sha256=14FPWvQ1cAOpSMX1hDV2vTQw3-E99DKI78GnDBIWCo8,7590
taskgraph/transforms/fetch.py,sha256=z-SAZTQSOcVRiFa2E8z0dAEhkIUhdOJdwQgdUah2LzA,9400
taskgraph/transforms/task.py,sha256=zAfcqIVegJh2OScqQR0mgLPhxjD4WCzXmRgW4-jC0Ng,46793
taskgraph/transforms/task.py,sha256=u4P7vk5d_g9395HhspQc5xGhdogVptuUnQOfA38D-QM,47778
taskgraph/transforms/job/__init__.py,sha256=imtb3MHVQbKtcCngSnvgumtBfOwxOPiRsJDwHKUtYn0,16891
taskgraph/transforms/job/common.py,sha256=onHnerPcmmvbSk0oHt8mvJmOo7AnjHQya0ombgMNLG8,7106
taskgraph/transforms/job/index_search.py,sha256=zPldmHSalHJjvULAMF9_QAeOZzIeWpr89kOVeP2IJAE,1220
taskgraph/transforms/job/run_task.py,sha256=UjFkPaULuMW3-nyJJuzRNubLFnmnN4o9W7G5kdUFxvw,9017
taskgraph/transforms/job/run_task.py,sha256=p_sE5XfbO-EIRgJ5YseeY_YJAqsDl_MUdcCQyKXi0e4,8258
taskgraph/transforms/job/toolchain.py,sha256=z2Z7sxI4yn_dI8zzcMWcrcmfTHeK6mgfSNSM6MAgrCU,4649
taskgraph/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
taskgraph/util/archive.py,sha256=Hcm8YHsCtazX7efDSd8vXm8Pw87Z1UP_Bo-gIUEzb_4,2856
@ -91,11 +92,11 @@ taskgraph/util/templates.py,sha256=Dqxfl244u-PX7dnsk3_vYyzDwpDgJtANK6NmZwN3Qow,1
taskgraph/util/time.py,sha256=cMRYsBiz7rgPwgZk77p0P7h9JzeEJENBZCoetBaEHqY,3490
taskgraph/util/treeherder.py,sha256=XrdE-Je0ZvXe6_8f0DvvqNbrHherUk-hUuxirImPEIo,2138
taskgraph/util/vcs.py,sha256=p_cQ9iIqALzVNBVPUIG9nDz4IGMdqom_5o3eidL1vCY,6510
taskgraph/util/verify.py,sha256=e0j_Ec2gDDIVlnEO9tHQ0gjxzGm00NJbPVI5q4BuiHc,5653
taskgraph/util/verify.py,sha256=ie3JTNRfooGhd3-M8_FSGglJFk1-7jUp1ZJ0T2kreP4,7398
taskgraph/util/workertypes.py,sha256=5g2mgIbEKMzDpZNnmPMoMNyy7Wahi-jmWcV1amDAcPo,2341
taskgraph/util/yaml.py,sha256=hfKI_D8Q7dimq4_VvO3WEh8CJsTrsIMwN6set7HIQbY,990
taskcluster_taskgraph-1.2.0.dist-info/METADATA,sha256=eBSq9--hnqRlzQHhFQ4QT9x3Mi28aLf9bJqk604NSmk,990
taskcluster_taskgraph-1.2.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
taskcluster_taskgraph-1.2.0.dist-info/entry_points.txt,sha256=VoXNtZpN4LvyXYB1wq47AU9CO-DMYMJ0VktKxjugzbY,51
taskcluster_taskgraph-1.2.0.dist-info/top_level.txt,sha256=3JNeYn_hNiNXC7DrdH_vcv-WYSE7QdgGjdvUYvSjVp0,10
taskcluster_taskgraph-1.2.0.dist-info/RECORD,,
taskcluster_taskgraph-1.4.0.dist-info/METADATA,sha256=FT-KN56SvLSAGRPk2ZJbwGqE51fW1j5XEMkH--K1ooQ,995
taskcluster_taskgraph-1.4.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
taskcluster_taskgraph-1.4.0.dist-info/entry_points.txt,sha256=VoXNtZpN4LvyXYB1wq47AU9CO-DMYMJ0VktKxjugzbY,51
taskcluster_taskgraph-1.4.0.dist-info/top_level.txt,sha256=3JNeYn_hNiNXC7DrdH_vcv-WYSE7QdgGjdvUYvSjVp0,10
taskcluster_taskgraph-1.4.0.dist-info/RECORD,,

0
third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.2.0.dist-info/WHEEL → third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.4.0.dist-info/WHEEL vendored
View File

0
third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.2.0.dist-info/entry_points.txt → third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.4.0.dist-info/entry_points.txt vendored
View File

0
third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.2.0.dist-info/top_level.txt → third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.4.0.dist-info/top_level.txt vendored
View File

41
third_party/python/taskcluster_taskgraph/taskgraph/decision.py vendored
View File

@ -3,29 +3,31 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import os
import json
import logging
import os
import pathlib
import shutil
import time
import yaml
from pathlib import Path
from .actions import render_actions_json
from .create import create_tasks
from .generator import TaskGraphGenerator
from .parameters import Parameters
from .taskgraph import TaskGraph
from taskgraph.util.python_path import find_object
from taskgraph.util.vcs import get_repository
from .util.schema import validate_schema, Schema
from taskgraph.util.yaml import load_yaml
import yaml
from voluptuous import Optional
from taskgraph.actions import render_actions_json
from taskgraph.create import create_tasks
from taskgraph.generator import TaskGraphGenerator
from taskgraph.parameters import Parameters
from taskgraph.taskgraph import TaskGraph
from taskgraph.util.python_path import find_object
from taskgraph.util.schema import Schema, validate_schema
from taskgraph.util.vcs import get_repository
from taskgraph.util.yaml import load_yaml
logger = logging.getLogger(__name__)
ARTIFACTS_DIR = "artifacts"
ARTIFACTS_DIR = Path("artifacts")
# For each project, this gives a set of parameters specific to the project.
# See `taskcluster/docs/parameters.rst` for information on parameters.
@ -116,6 +118,11 @@ def taskgraph_decision(options, parameters=None):
write_artifact("task-graph.json", tgg.morphed_task_graph.to_json())
write_artifact("label-to-taskid.json", tgg.label_to_taskid)
# write out current run-task and fetch-content scripts
RUN_TASK_DIR = pathlib.Path(__file__).parent / "run-task"
shutil.copy2(RUN_TASK_DIR / "run-task", ARTIFACTS_DIR)
shutil.copy2(RUN_TASK_DIR / "fetch-content", ARTIFACTS_DIR)
# actually create the graph
create_tasks(
tgg.graph_config,
@ -249,7 +256,7 @@ def write_artifact(filename, data):
logger.info(f"writing artifact file `{filename}`")
if not os.path.isdir(ARTIFACTS_DIR):
os.mkdir(ARTIFACTS_DIR)
path = os.path.join(ARTIFACTS_DIR, filename)
path = ARTIFACTS_DIR / filename
if filename.endswith(".yml"):
with open(path, "w") as f:
yaml.safe_dump(data, f, allow_unicode=True, default_flow_style=False)
@ -266,7 +273,7 @@ def write_artifact(filename, data):
def read_artifact(filename):
path = os.path.join(ARTIFACTS_DIR, filename)
path = ARTIFACTS_DIR / filename
if filename.endswith(".yml"):
return load_yaml(path, filename)
elif filename.endswith(".json"):
@ -282,4 +289,4 @@ def read_artifact(filename):
def rename_artifact(src, dest):
os.rename(os.path.join(ARTIFACTS_DIR, src), os.path.join(ARTIFACTS_DIR, dest))
os.rename(ARTIFACTS_DIR / src, ARTIFACTS_DIR / dest)

30
third_party/python/taskcluster_taskgraph/taskgraph/generator.py vendored
View File

@ -77,11 +77,13 @@ class Kind:
Task(
self.name,
label=task_dict["label"],
description=task_dict["description"],
attributes=task_dict["attributes"],
task=task_dict["task"],
optimization=task_dict.get("optimization"),
dependencies=task_dict.get("dependencies"),
soft_dependencies=task_dict.get("soft-dependencies"),
if_dependencies=task_dict.get("if-dependencies"),
)
for task_dict in transforms(trans_config, inputs)
]
@ -248,6 +250,9 @@ class TaskGraphGenerator:
continue
def _run(self):
# Initial verifications that don't depend on any generation state.
verifications("initial")
logger.info("Loading graph configuration.")
graph_config = load_graph_config(self.root_dir)
@ -269,7 +274,7 @@ class TaskGraphGenerator:
filters.insert(0, "target_tasks_method")
filters = [filter_tasks.filter_task_functions[f] for f in filters]
yield ("parameters", parameters)
yield self.verify("parameters", parameters)
logger.info("Loading kinds")
# put the kinds into a graph and sort topologically so that kinds are loaded
@ -285,6 +290,7 @@ class TaskGraphGenerator:
kind.name: kind
for kind in self._load_kinds(graph_config, parameters.get("target-kind"))
}
verifications("kinds", kinds)
edges = set()
for kind in kinds.values():
@ -315,7 +321,7 @@ class TaskGraphGenerator:
all_tasks[task.label] = task
logger.info(f"Generated {len(new_tasks)} tasks for kind {kind_name}")
full_task_set = TaskGraph(all_tasks, Graph(set(all_tasks), set()))
yield verifications("full_task_set", full_task_set, graph_config)
yield self.verify("full_task_set", full_task_set, graph_config, parameters)
logger.info("Generating full task graph")
edges = set()
@ -328,7 +334,7 @@ class TaskGraphGenerator:
"Full task graph contains %d tasks and %d dependencies"
% (len(full_task_set.graph.nodes), len(edges))
)
yield verifications("full_task_graph", full_task_graph, graph_config)
yield self.verify("full_task_graph", full_task_graph, graph_config, parameters)
logger.info("Generating target task set")
target_task_set = TaskGraph(
@ -345,7 +351,7 @@ class TaskGraphGenerator:
% (fltr.__name__, old_len - len(target_tasks), len(target_tasks))
)
yield verifications("target_task_set", target_task_set, graph_config)
yield self.verify("target_task_set", target_task_set, graph_config, parameters)
logger.info("Generating target task graph")
# include all docker-image build tasks here, in case they are needed for a graph morph
@ -370,7 +376,9 @@ class TaskGraphGenerator:
target_task_graph = TaskGraph(
{l: all_tasks[l] for l in target_graph.nodes}, target_graph
)
yield verifications("target_task_graph", target_task_graph, graph_config)
yield self.verify(
"target_task_graph", target_task_graph, graph_config, parameters
)
logger.info("Generating optimized task graph")
existing_tasks = parameters.get("existing_tasks")
@ -385,14 +393,18 @@ class TaskGraphGenerator:
existing_tasks=existing_tasks,
)
yield verifications("optimized_task_graph", optimized_task_graph, graph_config)
yield self.verify(
"optimized_task_graph", optimized_task_graph, graph_config, parameters
)
morphed_task_graph, label_to_taskid = morph(
optimized_task_graph, label_to_taskid, parameters, graph_config
)
yield "label_to_taskid", label_to_taskid
yield verifications("morphed_task_graph", morphed_task_graph, graph_config)
yield self.verify(
"morphed_task_graph", morphed_task_graph, graph_config, parameters
)
def _run_until(self, name):
while name not in self._run_results:
@ -403,6 +415,10 @@ class TaskGraphGenerator:
self._run_results[k] = v
return self._run_results[name]
def verify(self, name, obj, *args, **kwargs):
verifications(name, obj, *args, **kwargs)
return name, obj
def load_tasks_for_kind(parameters, kind, root_dir=None):
"""

2
third_party/python/taskcluster_taskgraph/taskgraph/morph.py vendored
View File

@ -190,7 +190,7 @@ def _get_morph_url():
existing use case.
"""
taskgraph_repo = os.environ.get(
"TASKGRAPH_HEAD_REPOSITORY", "https://hg.mozilla.org/ci/taskgraph"
"TASKGRAPH_HEAD_REPOSITORY", "https://github.com/taskcluster/taskgraph"
)
taskgraph_rev = os.environ.get("TASKGRAPH_HEAD_REV", "default")
return f"{taskgraph_repo}/raw-file/{taskgraph_rev}/src/taskgraph/morph.py"

34
third_party/python/taskcluster_taskgraph/taskgraph/transforms/job/run_task.py vendored
View File

@ -110,32 +110,14 @@ worker_defaults = {
def script_url(config, script):
# This logic is a bit of a hack, and should be replaced by something better.
# TASK_ID is used as a proxy for running in automation. In that case, we
# want to use the run-task/fetch-content corresponding to the taskgraph
# version we are running, and otherwise, we aren't going to run the task we
# generate, so the exact version doesn't matter.
# TASK_ID is also set for test-action-callback, so we check for that with
# taskcluster.testing.
# If we checked out the taskgraph code with run-task in the decision task,
# we can use TASKGRAPH_* to find the right version, which covers the
# existing use case.
if "TASK_ID" in os.environ and not taskcluster.testing:
if (
"TASKGRAPH_HEAD_REPOSITORY" not in os.environ
or "TASKGRAPH_HEAD_REV" not in os.environ
):
raise Exception(
"Must specify 'TASKGRAPH_HEAD_REPOSITORY' and 'TASKGRAPH_HEAD_REV' "
"to use run-task on generic-worker."
)
taskgraph_repo = os.environ.get(
"TASKGRAPH_HEAD_REPOSITORY", "https://hg.mozilla.org/ci/taskgraph"
)
taskgraph_rev = os.environ.get("TASKGRAPH_HEAD_REV", "default")
return "{}/raw-file/{}/src/taskgraph/run-task/{}".format(
taskgraph_repo, taskgraph_rev, script
)
if "MOZ_AUTOMATION" in os.environ and "TASK_ID" not in os.environ:
raise Exception("TASK_ID must be defined to use run-task on generic-worker")
task_id = os.environ.get("TASK_ID", "<TASK_ID>")
# use_proxy = False to avoid having all generic-workers turn on proxy
# Assumes the cluster allows anonymous downloads of public artifacts
tc_url = taskcluster.get_root_url(False)
# TODO: Use util/taskcluster.py:get_artifact_url once hack for Bug 1405889 is removed
return f"{tc_url}/api/queue/v1/task/{task_id}/artifacts/public/{script}"
@run_job_using(

32
third_party/python/taskcluster_taskgraph/taskgraph/transforms/task.py vendored
View File

@ -72,6 +72,8 @@ task_description_schema = Schema(
},
# Soft dependencies of this task, as a list of tasks labels
Optional("soft-dependencies"): [str],
# Dependencies that must be scheduled in order for this task to run.
Optional("if-dependencies"): [str],
Optional("requires"): Any("all-completed", "all-resolved"),
# expiration and deadline times, relative to task creation, with units
# (e.g., "14 days"). Defaults are set based on the project.
@ -607,15 +609,20 @@ def build_generic_worker_payload(config, task, task_def):
"maxRunTime": worker["max-run-time"],
}
on_exit_status = {}
if "retry-exit-status" in worker:
on_exit_status["retry"] = worker["retry-exit-status"]
if worker["os"] == "windows":
task_def["payload"]["onExitStatus"] = {
"retry": [
on_exit_status.setdefault("retry", []).extend(
[
# These codes (on windows) indicate a process interruption,
# rather than a task run failure. See bug 1544403.
1073807364, # process force-killed due to system shutdown
3221225786, # sigint (any interrupt)
]
}
)
if on_exit_status:
task_def["payload"]["onExitStatus"] = on_exit_status
env = worker.get("env", {})
@ -1088,10 +1095,27 @@ def build_task(config, tasks):
env = payload.setdefault("env", {})
env["MOZ_AUTOMATION"] = "1"
dependencies = task.get("dependencies", {})
if_dependencies = task.get("if-dependencies", [])
if if_dependencies:
for i, dep in enumerate(if_dependencies):
if dep in dependencies:
if_dependencies[i] = dependencies[dep]
continue
raise Exception(
"{label} specifies '{dep}' in if-dependencies, "
"but {dep} is not a dependency!".format(
label=task["label"], dep=dep
)
)
yield {
"label": task["label"],
"description": task["description"],
"task": task_def,
"dependencies": task.get("dependencies", {}),
"dependencies": dependencies,
"if-dependencies": if_dependencies,
"soft-dependencies": task.get("soft-dependencies", []),
"attributes": attributes,
"optimization": task.get("optimization", None),

108
third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py vendored
View File

@ -5,12 +5,81 @@
import logging
import sys
from abc import ABC, abstractmethod
import attr
from taskgraph.config import GraphConfig
from taskgraph.parameters import Parameters
from taskgraph.taskgraph import TaskGraph
from taskgraph.util.attributes import match_run_on_projects
logger = logging.getLogger(__name__)
@attr.s(frozen=True)
class Verification(ABC):
func = attr.ib()
@abstractmethod
def verify(self, **kwargs) -> None:
pass
@attr.s(frozen=True)
class InitialVerification(Verification):
"""Verification that doesn't depend on any generation state."""
def verify(self):
self.func()
@attr.s(frozen=True)
class GraphVerification(Verification):
"""Verification for a TaskGraph object."""
run_on_projects = attr.ib(default=None)
def verify(
self, graph: TaskGraph, graph_config: GraphConfig, parameters: Parameters
):
if self.run_on_projects and not match_run_on_projects(
parameters["project"], self.run_on_projects
):
return
scratch_pad = {}
graph.for_each_task(
self.func,
scratch_pad=scratch_pad,
graph_config=graph_config,
parameters=parameters,
)
self.func(
None,
graph,
scratch_pad=scratch_pad,
graph_config=graph_config,
parameters=parameters,
)
@attr.s(frozen=True)
class ParametersVerification(Verification):
"""Verification for a set of parameters."""
def verify(self, parameters: Parameters):
self.func(parameters)
@attr.s(frozen=True)
class KindsVerification(Verification):
"""Verification for kinds."""
def verify(self, kinds: dict):
self.func(kinds)
@attr.s(frozen=True)
class VerificationSequence:
"""
@ -22,21 +91,22 @@ class VerificationSequence:
"""
_verifications = attr.ib(factory=dict)
_verification_types = {
"graph": GraphVerification,
"initial": InitialVerification,
"kinds": KindsVerification,
"parameters": ParametersVerification,
}
def __call__(self, graph_name, graph, graph_config):
for verification in self._verifications.get(graph_name, []):
scratch_pad = {}
graph.for_each_task(
verification, scratch_pad=scratch_pad, graph_config=graph_config
)
verification(
None, graph, scratch_pad=scratch_pad, graph_config=graph_config
)
return graph_name, graph
def __call__(self, name, *args, **kwargs):
for verification in self._verifications.get(name, []):
verification.verify(*args, **kwargs)
def add(self, name, **kwargs):
cls = self._verification_types.get(name, GraphVerification)
def add(self, graph_name):
def wrap(func):
self._verifications.setdefault(graph_name, []).append(func)
self._verifications.setdefault(name, []).append(cls(func, **kwargs))
return func
return wrap
@ -46,7 +116,7 @@ verifications = VerificationSequence()
@verifications.add("full_task_graph")
def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config):
def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config, parameters):
"""
This function verifies that tuple
(collection.keys(), machine.platform, groupSymbol, symbol) is unique
@ -77,7 +147,9 @@ def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config):
@verifications.add("full_task_graph")
def verify_trust_domain_v2_routes(task, taskgraph, scratch_pad, graph_config):
def verify_trust_domain_v2_routes(
task, taskgraph, scratch_pad, graph_config, parameters
):
"""
This function ensures that any two tasks have distinct ``index.{trust-domain}.v2`` routes.
"""
@ -100,7 +172,9 @@ def verify_trust_domain_v2_routes(task, taskgraph, scratch_pad, graph_config):
@verifications.add("full_task_graph")
def verify_routes_notification_filters(task, taskgraph, scratch_pad, graph_config):
def verify_routes_notification_filters(
task, taskgraph, scratch_pad, graph_config, parameters
):
"""
This function ensures that only understood filters for notifications are
specified.
@ -127,7 +201,7 @@ def verify_routes_notification_filters(task, taskgraph, scratch_pad, graph_confi
@verifications.add("full_task_graph")
def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config):
def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config, parameters):
tiers = scratch_pad
if task is not None:
tiers[task.label] = (
@ -159,7 +233,7 @@ def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config):
@verifications.add("optimized_task_graph")
def verify_always_optimized(task, taskgraph, scratch_pad, graph_config):
def verify_always_optimized(task, taskgraph, scratch_pad, graph_config, parameters):
"""
This function ensures that always-optimized tasks have been optimized.
"""