Bug 1748929 - Upgrade taskgraph to version 1.4.0, r=releng-reviewers,jcristau

Differential Revision: https://phabricator.services.mozilla.com/D144622

third_party/python/poetry.lock

@@ -606,7 +606,7 @@ test = ["pytest", "pytest-cov", "pytest-mock", "httmock", "mock", "setuptools-li
 
 [[package]]
 name = "taskcluster-taskgraph"
-version = "1.2.0"
+version = "1.4.0"
 description = "Build taskcluster taskgraphs"
 category = "main"
 optional = false
@@ -728,7 +728,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=1.2.3)", "pytest-flake8", "pyt
 [metadata]
 lock-version = "1.1"
 python-versions = "^3.6"
-content-hash = "373971de74385d2b0f2e060e38b05d923ee13d1d4f2974226ef98c68314697df"
+content-hash = "2c8645955b35c4a1b90c8f5473f2b29a8d492613ea054cb17dbc20dadcaf19d6"
 
 [metadata.files]
 aiohttp = [
@@ -1112,8 +1112,8 @@ taskcluster = [
     {file = "taskcluster-44.2.2.tar.gz", hash = "sha256:0266a6a901e1a2ec838984a7f24e7adb6d58f9f2e221a7f613388f8f23f786fc"},
 ]
 taskcluster-taskgraph = [
-    {file = "taskcluster-taskgraph-1.2.0.tar.gz", hash = "sha256:bce1ef59f15b317256214da4d82f19d745bdc1b4497561d753436f4efa32ea39"},
+    {file = "taskcluster-taskgraph-1.4.0.tar.gz", hash = "sha256:237ed0399ef55e9eef537e6163b456c04d58a7b72bc1b55e8c61a05331170b1e"},
-    {file = "taskcluster_taskgraph-1.2.0-py3-none-any.whl", hash = "sha256:ac7b0345b5b8752c5b31c516c9d798d7168bfda8963774def972b0beee349f6e"},
+    {file = "taskcluster_taskgraph-1.4.0-py3-none-any.whl", hash = "sha256:2be5ac94745180029dfad260486aafeb14f564f5a95d1d3269e0d17834273f11"},
 ]
 taskcluster-urls = [
     {file = "taskcluster-urls-13.0.1.tar.gz", hash = "sha256:b25e122ecec249c4299ac7b20b08db76e3e2025bdaeb699a9d444556de5fd367"},

third_party/python/requirements.in

@@ -37,7 +37,7 @@ setuptools==51.2.0
 six==1.13.0
 slugid==2.0.0
 taskcluster==44.2.2
-taskcluster-taskgraph==1.2.0
+taskcluster-taskgraph==1.4.0
 taskcluster-urls==13.0.1
 tqdm==4.62.3
 urllib3==1.26

third_party/python/requirements.txt

@@ -319,9 +319,9 @@ six==1.13.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (pyt
 slugid==2.0.0 \
     --hash=sha256:aec8b0e01c4ad32e38e12d609eab3ec912fd129aaf6b2ded0199b56a5f8fd67c \
     --hash=sha256:a950d98b72691178bdd4d6c52743c4a2aa039207cf7a97d71060a111ff9ba297
-taskcluster-taskgraph==1.2.0 \
+taskcluster-taskgraph==1.4.0 \
-    --hash=sha256:bce1ef59f15b317256214da4d82f19d745bdc1b4497561d753436f4efa32ea39 \
+    --hash=sha256:237ed0399ef55e9eef537e6163b456c04d58a7b72bc1b55e8c61a05331170b1e \
-    --hash=sha256:ac7b0345b5b8752c5b31c516c9d798d7168bfda8963774def972b0beee349f6e
+    --hash=sha256:2be5ac94745180029dfad260486aafeb14f564f5a95d1d3269e0d17834273f11
 taskcluster-urls==13.0.1 \
     --hash=sha256:b25e122ecec249c4299ac7b20b08db76e3e2025bdaeb699a9d444556de5fd367 \
     --hash=sha256:5e25e7e6818e8877178b175ff43d2e6548afad72694aa125f404a7329ece0973 \

third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.4.0.dist-info/METADATA

@@ -1,8 +1,8 @@
 Metadata-Version: 2.1
 Name: taskcluster-taskgraph
-Version: 1.2.0
+Version: 1.4.0
 Summary: Build taskcluster taskgraphs
-Home-page: https://hg.mozilla.org/ci/taskgraph
+Home-page: https://github.com/taskcluster/taskgraph
 License: UNKNOWN
 Platform: UNKNOWN
 Classifier: Development Status :: 5 - Production/Stable

third_party/python/taskcluster_taskgraph/taskcluster_taskgraph-1.4.0.dist-info/RECORD

@@ -1,14 +1,14 @@
 taskgraph/__init__.py,sha256=jwOtU7TkmU317LP_IsgIswpj2T1OPUXXgMRv4sIU7nE,707
 taskgraph/config.py,sha256=Bm0RU1LyEj4RuX_lBnpQrr5fGmx7zCbKleN1Zy6fQ5w,4410
 taskgraph/create.py,sha256=VeAYxtLK8f5HufgUSjzDRP7WBSwQza--_O7RzRRhzA4,5190
-taskgraph/decision.py,sha256=GlK1rs1VoWqoTfHM9oJzf0LS608HJVtEYWIPmQ3iEhc,9418
+taskgraph/decision.py,sha256=TpKJyWJIT6mCI_wPORKHqSqWVwmZGzpR7VJuf-3kj-M,9724
 taskgraph/docker.py,sha256=HlCEtfW2coUAyGce8ToVfn8OdLfQJT7Vq_mYQntO4gQ,7485
 taskgraph/files_changed.py,sha256=30ONF-m1hbGQnQ31E-TCsHG1jhdNjLTOolSXu2jzOCs,2159
 taskgraph/filter_tasks.py,sha256=ty4SK0ujcZ8824F4ikwdCaGX1TRSq_90TFArDogJHTo,875
-taskgraph/generator.py,sha256=Axwun0FeuEXKZpCKD3F4HHCk5K3STeYrR2gr15DY9DM,14549
+taskgraph/generator.py,sha256=y89WzZmfeHNNxKM3k60x7L2IJMCciXad5u6lNhZ0mdY,15079
 taskgraph/graph.py,sha256=W7I-lfDiBMqSj5ZKgqVmebwTaNxdmbKLx2MqFT_cD_I,4666
 taskgraph/main.py,sha256=Dz-uH6QJ3Xtd_t-9gD5iwS4a0CmKypM9aQSW2eZCFao,23254
-taskgraph/morph.py,sha256=EH_kP5FAMb0F8Oap4VZeVpMNXGr0QhDVz_lxiY-ksgA,9562
+taskgraph/morph.py,sha256=xX6bU-4iJtwwMw60_0Qt9mG8N4i6zhp4fB_BUZOjQ4o,9567
 taskgraph/optimize.py,sha256=qcYcxSmhN9joPZGFEdW6D27G5F4xorS3wmX6TO-gC1Q,16487
 taskgraph/parameters.py,sha256=QSVVMhoOQ0Uck4rE9_McFkDByUvidAG0yJFLLxLGuEE,10744
 taskgraph/target_tasks.py,sha256=mspItlKD-HCuQR1x_UD6HT_Qd1v5kEHvgRWIUbweRDg,3166
@@ -29,10 +29,10 @@ taskgraph/run-task/robustcheckout.py,sha256=P41ZGrec5aa8hVTEqOkKJ9wTygsgaXtjoQnl
 taskgraph/run-task/run-task,sha256=XDT8N9NPIYBZIptcLliHDlRVuVClXcmrCOlSflYd-7s,39869
 taskgraph/test/__init__.py,sha256=7LTScvkVcLPqivjL-wCb_Znk2GGajaJi1fJ4SjcLMoA,289
 taskgraph/test/automationrelevance.json,sha256=ttGejNZeVhcTWXFdoU0xME0YEYRYvaYcAr8zBz0ewrw,17980
-taskgraph/test/conftest.py,sha256=urHJNFOw7mVSr8WzIDZtOZPbXykdR--iw3J_hiF2rjg,3584
+taskgraph/test/conftest.py,sha256=GoaEjxBClxzByVNkhwq1UDBC9uzOr4vy8PoWtqqNyeU,3615
 taskgraph/test/mockedopen.py,sha256=Ccr2qGJSLeWIUYd03Poy8eKKRSW2aTpRGI-0AA7xYrw,4055
 taskgraph/test/test_create.py,sha256=oY7DeVW8usjVNe-QPPyTrE3hsvwMo9HvMxPSCllOsMQ,3657
-taskgraph/test/test_decision.py,sha256=dxlMnRtpKZrOqSZo-znPLw6TwlFJOB9Yp0sadEnJdaQ,2658
+taskgraph/test/test_decision.py,sha256=twp07KM4iaw0hPOnX0YkmNOXD1jVlAmWc2ry5sqLPSk,2681
 taskgraph/test/test_files_changed.py,sha256=MoHr_M-qtbi9PbKQ9loDlKOOzecUUyE1N0SgxjXfP5Y,2608
 taskgraph/test/test_generator.py,sha256=6-ZpG45F3YlTWTCILrktz7LeSs3tEfQWCzekN6OHYHw,4284
 taskgraph/test/test_graph.py,sha256=IEly2SS9NZTN3F0AM4VhxxYx0WTj5u7x2UFyojr1Ddg,7064
@@ -44,11 +44,11 @@ taskgraph/test/test_scripts_run_task.py,sha256=ucv3oVin-gTwf7DUtFPG_bA3cqvQdRIKa
 taskgraph/test/test_target_tasks.py,sha256=AzvuEw1NI4b_f14rPFkWuK0bJyxknkPIQi5KI6k2K8A,12046
 taskgraph/test/test_taskgraph.py,sha256=EA09CzcgZBUoXVqhAWeEDPrnUF8Vw2gbTPwHrZGCWDc,3760
 taskgraph/test/test_transforms_base.py,sha256=Vo9slzCB2GePvMoLmkrSdhYVWh2nQYn5bRxMjsx40Mw,873
-taskgraph/test/test_transforms_job.py,sha256=pHO_Ea32b3OjCSIYhRHK3CqHsugJizD43_a3jMFtG_A,4490
+taskgraph/test/test_transforms_job.py,sha256=eEbbnet6Pvunz7lxiN09lIpTyVoXJDcDEPhkWYl0nAc,4656
 taskgraph/test/test_util_attributes.py,sha256=K_Wro-p5oA-S4yrsLqT8HMBlOAN4L0krQQQ82WYgGAQ,3596
 taskgraph/test/test_util_docker.py,sha256=TKe9D5d3q8rK0iA5rMhzbmhQ4c4OSnVKrjioeZ1uNuo,8800
 taskgraph/test/test_util_memoize.py,sha256=yq-PTegHBdQZkRm6Iv1NdCRcKCVzsb4pmZPvIz0P6j8,2340
-taskgraph/test/test_util_parameterization.py,sha256=IW8zWFMi7INVuAkpT5rk674nZVZ4BhHrKRTz8HKN-O0,7945
+taskgraph/test/test_util_parameterization.py,sha256=TEI3WVpL9Z54xVk8RZk7u3NwzddJxdmuAWfZa1IPaFw,7485
 taskgraph/test/test_util_path.py,sha256=icJyH1DyMNVuZ5xfPXvrEQwQ0pQYTg4ORlZq3RK6_V8,5906
 taskgraph/test/test_util_python_path.py,sha256=VQo4hwsJ0It-jLIqe4nErPmZn9AQ7rliN25h2oO_zMg,1216
 taskgraph/test/test_util_readonlydict.py,sha256=KRgjLvSBsZZj4EUhwcqeUsM1T--iGklVE0QJuC6Xv4o,1234
@@ -57,7 +57,8 @@ taskgraph/test/test_util_taskcluster.py,sha256=hZqtF7OC58OO4nW4URxF5ZTyxnNiwh2Jk
 taskgraph/test/test_util_templates.py,sha256=u3ckrzmx1eyk7vXiqRiQlETtVZvSx4FHEXN5xB4GZDQ,1676
 taskgraph/test/test_util_time.py,sha256=SG4WmSupTQiL1UhE2UMsMNZEugfPdoaxO4aWSxLGXBM,1803
 taskgraph/test/test_util_treeherder.py,sha256=20zzGcMd0BL0ayTFQj6Etj39afdxZPgtZxSGUZ0iL5M,912
-taskgraph/test/test_util_vcs.py,sha256=zL9j_PaSBPTuncgCpjvR5SDL0wIAkblPSTabn9qU-qs,6625
+taskgraph/test/test_util_vcs.py,sha256=gWQcbEUYW0ApaGrXJDplE2Oze1i1_LxIchXEAcJjn_Y,6838
+taskgraph/test/test_util_verify.py,sha256=hTia5J9DkaGejCnHU9FafBSjhZdE7LfmedFeUcmn8Bc,2486
 taskgraph/test/test_util_yaml.py,sha256=zymZxaAZBIBn5u-p91QsA---IqCH_CVVk3YqMoshLlQ,1019
 taskgraph/transforms/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 taskgraph/transforms/base.py,sha256=oIQdBKrHG_FZXAoRyiazxgLa8zbAI4TEVlduqz8H12I,5277
@@ -65,11 +66,11 @@ taskgraph/transforms/cached_tasks.py,sha256=fMCbxqA-HPSMG6mILYCfdIpnUg9uSKlQGKkU
 taskgraph/transforms/code_review.py,sha256=du7npIGOpVMK9QZtdkHrkNecRiaqd-fHjHavDVkfL58,698
 taskgraph/transforms/docker_image.py,sha256=14FPWvQ1cAOpSMX1hDV2vTQw3-E99DKI78GnDBIWCo8,7590
 taskgraph/transforms/fetch.py,sha256=z-SAZTQSOcVRiFa2E8z0dAEhkIUhdOJdwQgdUah2LzA,9400
-taskgraph/transforms/task.py,sha256=zAfcqIVegJh2OScqQR0mgLPhxjD4WCzXmRgW4-jC0Ng,46793
+taskgraph/transforms/task.py,sha256=u4P7vk5d_g9395HhspQc5xGhdogVptuUnQOfA38D-QM,47778
 taskgraph/transforms/job/__init__.py,sha256=imtb3MHVQbKtcCngSnvgumtBfOwxOPiRsJDwHKUtYn0,16891
 taskgraph/transforms/job/common.py,sha256=onHnerPcmmvbSk0oHt8mvJmOo7AnjHQya0ombgMNLG8,7106
 taskgraph/transforms/job/index_search.py,sha256=zPldmHSalHJjvULAMF9_QAeOZzIeWpr89kOVeP2IJAE,1220
-taskgraph/transforms/job/run_task.py,sha256=UjFkPaULuMW3-nyJJuzRNubLFnmnN4o9W7G5kdUFxvw,9017
+taskgraph/transforms/job/run_task.py,sha256=p_sE5XfbO-EIRgJ5YseeY_YJAqsDl_MUdcCQyKXi0e4,8258
 taskgraph/transforms/job/toolchain.py,sha256=z2Z7sxI4yn_dI8zzcMWcrcmfTHeK6mgfSNSM6MAgrCU,4649
 taskgraph/util/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 taskgraph/util/archive.py,sha256=Hcm8YHsCtazX7efDSd8vXm8Pw87Z1UP_Bo-gIUEzb_4,2856
@@ -91,11 +92,11 @@ taskgraph/util/templates.py,sha256=Dqxfl244u-PX7dnsk3_vYyzDwpDgJtANK6NmZwN3Qow,1
 taskgraph/util/time.py,sha256=cMRYsBiz7rgPwgZk77p0P7h9JzeEJENBZCoetBaEHqY,3490
 taskgraph/util/treeherder.py,sha256=XrdE-Je0ZvXe6_8f0DvvqNbrHherUk-hUuxirImPEIo,2138
 taskgraph/util/vcs.py,sha256=p_cQ9iIqALzVNBVPUIG9nDz4IGMdqom_5o3eidL1vCY,6510
-taskgraph/util/verify.py,sha256=e0j_Ec2gDDIVlnEO9tHQ0gjxzGm00NJbPVI5q4BuiHc,5653
+taskgraph/util/verify.py,sha256=ie3JTNRfooGhd3-M8_FSGglJFk1-7jUp1ZJ0T2kreP4,7398
 taskgraph/util/workertypes.py,sha256=5g2mgIbEKMzDpZNnmPMoMNyy7Wahi-jmWcV1amDAcPo,2341
 taskgraph/util/yaml.py,sha256=hfKI_D8Q7dimq4_VvO3WEh8CJsTrsIMwN6set7HIQbY,990
-taskcluster_taskgraph-1.2.0.dist-info/METADATA,sha256=eBSq9--hnqRlzQHhFQ4QT9x3Mi28aLf9bJqk604NSmk,990
+taskcluster_taskgraph-1.4.0.dist-info/METADATA,sha256=FT-KN56SvLSAGRPk2ZJbwGqE51fW1j5XEMkH--K1ooQ,995
-taskcluster_taskgraph-1.2.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
+taskcluster_taskgraph-1.4.0.dist-info/WHEEL,sha256=G16H4A3IeoQmnOrYV4ueZGKSjhipXx8zc8nu9FGlvMA,92
-taskcluster_taskgraph-1.2.0.dist-info/entry_points.txt,sha256=VoXNtZpN4LvyXYB1wq47AU9CO-DMYMJ0VktKxjugzbY,51
+taskcluster_taskgraph-1.4.0.dist-info/entry_points.txt,sha256=VoXNtZpN4LvyXYB1wq47AU9CO-DMYMJ0VktKxjugzbY,51
-taskcluster_taskgraph-1.2.0.dist-info/top_level.txt,sha256=3JNeYn_hNiNXC7DrdH_vcv-WYSE7QdgGjdvUYvSjVp0,10
+taskcluster_taskgraph-1.4.0.dist-info/top_level.txt,sha256=3JNeYn_hNiNXC7DrdH_vcv-WYSE7QdgGjdvUYvSjVp0,10
-taskcluster_taskgraph-1.2.0.dist-info/RECORD,,
+taskcluster_taskgraph-1.4.0.dist-info/RECORD,,

third_party/python/taskcluster_taskgraph/taskgraph/decision.py

@@ -3,29 +3,31 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
-import os
 import json
 import logging
-
+import os
-
+import pathlib
+import shutil
 import time
-import yaml
+from pathlib import Path
 
-from .actions import render_actions_json
+import yaml
-from .create import create_tasks
-from .generator import TaskGraphGenerator
-from .parameters import Parameters
-from .taskgraph import TaskGraph
-from taskgraph.util.python_path import find_object
-from taskgraph.util.vcs import get_repository
-from .util.schema import validate_schema, Schema
-from taskgraph.util.yaml import load_yaml
 from voluptuous import Optional
 
+from taskgraph.actions import render_actions_json
+from taskgraph.create import create_tasks
+from taskgraph.generator import TaskGraphGenerator
+from taskgraph.parameters import Parameters
+from taskgraph.taskgraph import TaskGraph
+from taskgraph.util.python_path import find_object
+from taskgraph.util.schema import Schema, validate_schema
+from taskgraph.util.vcs import get_repository
+from taskgraph.util.yaml import load_yaml
 
 logger = logging.getLogger(__name__)
 
-ARTIFACTS_DIR = "artifacts"
+ARTIFACTS_DIR = Path("artifacts")
+
 
 # For each project, this gives a set of parameters specific to the project.
 # See `taskcluster/docs/parameters.rst` for information on parameters.
@@ -116,6 +118,11 @@ def taskgraph_decision(options, parameters=None):
     write_artifact("task-graph.json", tgg.morphed_task_graph.to_json())
     write_artifact("label-to-taskid.json", tgg.label_to_taskid)
 
+    # write out current run-task and fetch-content scripts
+    RUN_TASK_DIR = pathlib.Path(__file__).parent / "run-task"
+    shutil.copy2(RUN_TASK_DIR / "run-task", ARTIFACTS_DIR)
+    shutil.copy2(RUN_TASK_DIR / "fetch-content", ARTIFACTS_DIR)
+
     # actually create the graph
     create_tasks(
         tgg.graph_config,
@@ -249,7 +256,7 @@ def write_artifact(filename, data):
     logger.info(f"writing artifact file `{filename}`")
     if not os.path.isdir(ARTIFACTS_DIR):
         os.mkdir(ARTIFACTS_DIR)
-    path = os.path.join(ARTIFACTS_DIR, filename)
+    path = ARTIFACTS_DIR / filename
     if filename.endswith(".yml"):
         with open(path, "w") as f:
             yaml.safe_dump(data, f, allow_unicode=True, default_flow_style=False)
@@ -266,7 +273,7 @@ def write_artifact(filename, data):
 
 
 def read_artifact(filename):
-    path = os.path.join(ARTIFACTS_DIR, filename)
+    path = ARTIFACTS_DIR / filename
     if filename.endswith(".yml"):
         return load_yaml(path, filename)
     elif filename.endswith(".json"):
@@ -282,4 +289,4 @@ def read_artifact(filename):
 
 
 def rename_artifact(src, dest):
-    os.rename(os.path.join(ARTIFACTS_DIR, src), os.path.join(ARTIFACTS_DIR, dest))
+    os.rename(ARTIFACTS_DIR / src, ARTIFACTS_DIR / dest)

third_party/python/taskcluster_taskgraph/taskgraph/generator.py

@@ -77,11 +77,13 @@ class Kind:
             Task(
                 self.name,
                 label=task_dict["label"],
+                description=task_dict["description"],
                 attributes=task_dict["attributes"],
                 task=task_dict["task"],
                 optimization=task_dict.get("optimization"),
                 dependencies=task_dict.get("dependencies"),
                 soft_dependencies=task_dict.get("soft-dependencies"),
+                if_dependencies=task_dict.get("if-dependencies"),
             )
             for task_dict in transforms(trans_config, inputs)
         ]
@@ -248,6 +250,9 @@ class TaskGraphGenerator:
                     continue
 
     def _run(self):
+        # Initial verifications that don't depend on any generation state.
+        verifications("initial")
+
         logger.info("Loading graph configuration.")
         graph_config = load_graph_config(self.root_dir)
 
@@ -269,7 +274,7 @@ class TaskGraphGenerator:
             filters.insert(0, "target_tasks_method")
         filters = [filter_tasks.filter_task_functions[f] for f in filters]
 
-        yield ("parameters", parameters)
+        yield self.verify("parameters", parameters)
 
         logger.info("Loading kinds")
         # put the kinds into a graph and sort topologically so that kinds are loaded
@@ -285,6 +290,7 @@ class TaskGraphGenerator:
             kind.name: kind
             for kind in self._load_kinds(graph_config, parameters.get("target-kind"))
         }
+        verifications("kinds", kinds)
 
         edges = set()
         for kind in kinds.values():
@@ -315,7 +321,7 @@ class TaskGraphGenerator:
                 all_tasks[task.label] = task
             logger.info(f"Generated {len(new_tasks)} tasks for kind {kind_name}")
         full_task_set = TaskGraph(all_tasks, Graph(set(all_tasks), set()))
-        yield verifications("full_task_set", full_task_set, graph_config)
+        yield self.verify("full_task_set", full_task_set, graph_config, parameters)
 
         logger.info("Generating full task graph")
         edges = set()
@@ -328,7 +334,7 @@ class TaskGraphGenerator:
             "Full task graph contains %d tasks and %d dependencies"
             % (len(full_task_set.graph.nodes), len(edges))
         )
-        yield verifications("full_task_graph", full_task_graph, graph_config)
+        yield self.verify("full_task_graph", full_task_graph, graph_config, parameters)
 
         logger.info("Generating target task set")
         target_task_set = TaskGraph(
@@ -345,7 +351,7 @@ class TaskGraphGenerator:
                 % (fltr.__name__, old_len - len(target_tasks), len(target_tasks))
             )
 
-        yield verifications("target_task_set", target_task_set, graph_config)
+        yield self.verify("target_task_set", target_task_set, graph_config, parameters)
 
         logger.info("Generating target task graph")
         # include all docker-image build tasks here, in case they are needed for a graph morph
@@ -370,7 +376,9 @@ class TaskGraphGenerator:
         target_task_graph = TaskGraph(
             {l: all_tasks[l] for l in target_graph.nodes}, target_graph
         )
-        yield verifications("target_task_graph", target_task_graph, graph_config)
+        yield self.verify(
+            "target_task_graph", target_task_graph, graph_config, parameters
+        )
 
         logger.info("Generating optimized task graph")
         existing_tasks = parameters.get("existing_tasks")
@@ -385,14 +393,18 @@ class TaskGraphGenerator:
             existing_tasks=existing_tasks,
         )
 
-        yield verifications("optimized_task_graph", optimized_task_graph, graph_config)
+        yield self.verify(
+            "optimized_task_graph", optimized_task_graph, graph_config, parameters
+        )
 
         morphed_task_graph, label_to_taskid = morph(
             optimized_task_graph, label_to_taskid, parameters, graph_config
         )
 
         yield "label_to_taskid", label_to_taskid
-        yield verifications("morphed_task_graph", morphed_task_graph, graph_config)
+        yield self.verify(
+            "morphed_task_graph", morphed_task_graph, graph_config, parameters
+        )
 
     def _run_until(self, name):
         while name not in self._run_results:
@@ -403,6 +415,10 @@ class TaskGraphGenerator:
             self._run_results[k] = v
         return self._run_results[name]
 
+    def verify(self, name, obj, *args, **kwargs):
+        verifications(name, obj, *args, **kwargs)
+        return name, obj
+
 
 def load_tasks_for_kind(parameters, kind, root_dir=None):
     """

third_party/python/taskcluster_taskgraph/taskgraph/morph.py

@@ -190,7 +190,7 @@ def _get_morph_url():
     existing use case.
     """
     taskgraph_repo = os.environ.get(
-        "TASKGRAPH_HEAD_REPOSITORY", "https://hg.mozilla.org/ci/taskgraph"
+        "TASKGRAPH_HEAD_REPOSITORY", "https://github.com/taskcluster/taskgraph"
     )
     taskgraph_rev = os.environ.get("TASKGRAPH_HEAD_REV", "default")
     return f"{taskgraph_repo}/raw-file/{taskgraph_rev}/src/taskgraph/morph.py"

third_party/python/taskcluster_taskgraph/taskgraph/transforms/job/run_task.py

@@ -110,32 +110,14 @@ worker_defaults = {
 
 
 def script_url(config, script):
-    # This logic is a bit of a hack, and should be replaced by something better.
+    if "MOZ_AUTOMATION" in os.environ and "TASK_ID" not in os.environ:
-    # TASK_ID is used as a proxy for running in automation.  In that case, we
+        raise Exception("TASK_ID must be defined to use run-task on generic-worker")
-    # want to use the run-task/fetch-content corresponding to the taskgraph
+    task_id = os.environ.get("TASK_ID", "<TASK_ID>")
-    # version we are running, and otherwise, we aren't going to run the task we
+    # use_proxy = False to avoid having all generic-workers turn on proxy
-    # generate, so the exact version doesn't matter.
+    # Assumes the cluster allows anonymous downloads of public artifacts
-    # TASK_ID is also set for test-action-callback, so we check for that with
+    tc_url = taskcluster.get_root_url(False)
-    # taskcluster.testing.
+    # TODO: Use util/taskcluster.py:get_artifact_url once hack for Bug 1405889 is removed
-    # If we checked out the taskgraph code with run-task in the decision task,
+    return f"{tc_url}/api/queue/v1/task/{task_id}/artifacts/public/{script}"
-    # we can use TASKGRAPH_* to find the right version, which covers the
-    # existing use case.
-    if "TASK_ID" in os.environ and not taskcluster.testing:
-        if (
-            "TASKGRAPH_HEAD_REPOSITORY" not in os.environ
-            or "TASKGRAPH_HEAD_REV" not in os.environ
-        ):
-            raise Exception(
-                "Must specify 'TASKGRAPH_HEAD_REPOSITORY' and 'TASKGRAPH_HEAD_REV' "
-                "to use run-task on generic-worker."
-            )
-    taskgraph_repo = os.environ.get(
-        "TASKGRAPH_HEAD_REPOSITORY", "https://hg.mozilla.org/ci/taskgraph"
-    )
-    taskgraph_rev = os.environ.get("TASKGRAPH_HEAD_REV", "default")
-    return "{}/raw-file/{}/src/taskgraph/run-task/{}".format(
-        taskgraph_repo, taskgraph_rev, script
-    )
 
 
 @run_job_using(

third_party/python/taskcluster_taskgraph/taskgraph/transforms/task.py

@@ -72,6 +72,8 @@ task_description_schema = Schema(
         },
         # Soft dependencies of this task, as a list of tasks labels
         Optional("soft-dependencies"): [str],
+        # Dependencies that must be scheduled in order for this task to run.
+        Optional("if-dependencies"): [str],
         Optional("requires"): Any("all-completed", "all-resolved"),
         # expiration and deadline times, relative to task creation, with units
         # (e.g., "14 days").  Defaults are set based on the project.
@@ -607,15 +609,20 @@ def build_generic_worker_payload(config, task, task_def):
         "maxRunTime": worker["max-run-time"],
     }
 
+    on_exit_status = {}
+    if "retry-exit-status" in worker:
+        on_exit_status["retry"] = worker["retry-exit-status"]
     if worker["os"] == "windows":
-        task_def["payload"]["onExitStatus"] = {
+        on_exit_status.setdefault("retry", []).extend(
-            "retry": [
+            [
                 # These codes (on windows) indicate a process interruption,
                 # rather than a task run failure. See bug 1544403.
                 1073807364,  # process force-killed due to system shutdown
                 3221225786,  # sigint (any interrupt)
             ]
-        }
+        )
+    if on_exit_status:
+        task_def["payload"]["onExitStatus"] = on_exit_status
 
     env = worker.get("env", {})
 
@@ -1088,10 +1095,27 @@ def build_task(config, tasks):
                 env = payload.setdefault("env", {})
                 env["MOZ_AUTOMATION"] = "1"
 
+        dependencies = task.get("dependencies", {})
+        if_dependencies = task.get("if-dependencies", [])
+        if if_dependencies:
+            for i, dep in enumerate(if_dependencies):
+                if dep in dependencies:
+                    if_dependencies[i] = dependencies[dep]
+                    continue
+
+                raise Exception(
+                    "{label} specifies '{dep}' in if-dependencies, "
+                    "but {dep} is not a dependency!".format(
+                        label=task["label"], dep=dep
+                    )
+                )
+
         yield {
             "label": task["label"],
+            "description": task["description"],
             "task": task_def,
-            "dependencies": task.get("dependencies", {}),
+            "dependencies": dependencies,
+            "if-dependencies": if_dependencies,
             "soft-dependencies": task.get("soft-dependencies", []),
             "attributes": attributes,
             "optimization": task.get("optimization", None),

third_party/python/taskcluster_taskgraph/taskgraph/util/verify.py

@@ -5,12 +5,81 @@
 
 import logging
 import sys
+from abc import ABC, abstractmethod
 
 import attr
 
+from taskgraph.config import GraphConfig
+from taskgraph.parameters import Parameters
+from taskgraph.taskgraph import TaskGraph
+from taskgraph.util.attributes import match_run_on_projects
+
 logger = logging.getLogger(__name__)
 
 
+@attr.s(frozen=True)
+class Verification(ABC):
+    func = attr.ib()
+
+    @abstractmethod
+    def verify(self, **kwargs) -> None:
+        pass
+
+
+@attr.s(frozen=True)
+class InitialVerification(Verification):
+    """Verification that doesn't depend on any generation state."""
+
+    def verify(self):
+        self.func()
+
+
+@attr.s(frozen=True)
+class GraphVerification(Verification):
+    """Verification for a TaskGraph object."""
+
+    run_on_projects = attr.ib(default=None)
+
+    def verify(
+        self, graph: TaskGraph, graph_config: GraphConfig, parameters: Parameters
+    ):
+        if self.run_on_projects and not match_run_on_projects(
+            parameters["project"], self.run_on_projects
+        ):
+            return
+
+        scratch_pad = {}
+        graph.for_each_task(
+            self.func,
+            scratch_pad=scratch_pad,
+            graph_config=graph_config,
+            parameters=parameters,
+        )
+        self.func(
+            None,
+            graph,
+            scratch_pad=scratch_pad,
+            graph_config=graph_config,
+            parameters=parameters,
+        )
+
+
+@attr.s(frozen=True)
+class ParametersVerification(Verification):
+    """Verification for a set of parameters."""
+
+    def verify(self, parameters: Parameters):
+        self.func(parameters)
+
+
+@attr.s(frozen=True)
+class KindsVerification(Verification):
+    """Verification for kinds."""
+
+    def verify(self, kinds: dict):
+        self.func(kinds)
+
+
 @attr.s(frozen=True)
 class VerificationSequence:
     """
@@ -22,21 +91,22 @@ class VerificationSequence:
     """
 
     _verifications = attr.ib(factory=dict)
+    _verification_types = {
+        "graph": GraphVerification,
+        "initial": InitialVerification,
+        "kinds": KindsVerification,
+        "parameters": ParametersVerification,
+    }
 
-    def __call__(self, graph_name, graph, graph_config):
+    def __call__(self, name, *args, **kwargs):
-        for verification in self._verifications.get(graph_name, []):
+        for verification in self._verifications.get(name, []):
-            scratch_pad = {}
+            verification.verify(*args, **kwargs)
-            graph.for_each_task(
+
-                verification, scratch_pad=scratch_pad, graph_config=graph_config
+    def add(self, name, **kwargs):
-            )
+        cls = self._verification_types.get(name, GraphVerification)
-            verification(
-                None, graph, scratch_pad=scratch_pad, graph_config=graph_config
-            )
-        return graph_name, graph
 
-    def add(self, graph_name):
         def wrap(func):
-            self._verifications.setdefault(graph_name, []).append(func)
+            self._verifications.setdefault(name, []).append(cls(func, **kwargs))
             return func
 
         return wrap
@@ -46,7 +116,7 @@ verifications = VerificationSequence()
 
 
 @verifications.add("full_task_graph")
-def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config):
+def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config, parameters):
     """
     This function verifies that tuple
     (collection.keys(), machine.platform, groupSymbol, symbol) is unique
@@ -77,7 +147,9 @@ def verify_task_graph_symbol(task, taskgraph, scratch_pad, graph_config):
 
 
 @verifications.add("full_task_graph")
-def verify_trust_domain_v2_routes(task, taskgraph, scratch_pad, graph_config):
+def verify_trust_domain_v2_routes(
+    task, taskgraph, scratch_pad, graph_config, parameters
+):
     """
     This function ensures that any two tasks have distinct ``index.{trust-domain}.v2`` routes.
     """
@@ -100,7 +172,9 @@ def verify_trust_domain_v2_routes(task, taskgraph, scratch_pad, graph_config):
 
 
 @verifications.add("full_task_graph")
-def verify_routes_notification_filters(task, taskgraph, scratch_pad, graph_config):
+def verify_routes_notification_filters(
+    task, taskgraph, scratch_pad, graph_config, parameters
+):
     """
     This function ensures that only understood filters for notifications are
     specified.
@@ -127,7 +201,7 @@ def verify_routes_notification_filters(task, taskgraph, scratch_pad, graph_confi
 
 
 @verifications.add("full_task_graph")
-def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config):
+def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config, parameters):
     tiers = scratch_pad
     if task is not None:
         tiers[task.label] = (
@@ -159,7 +233,7 @@ def verify_dependency_tiers(task, taskgraph, scratch_pad, graph_config):
 
 
 @verifications.add("optimized_task_graph")
-def verify_always_optimized(task, taskgraph, scratch_pad, graph_config):
+def verify_always_optimized(task, taskgraph, scratch_pad, graph_config, parameters):
     """
     This function ensures that always-optimized tasks have been optimized.
     """