Bug 1406209 - Use .taskcluster.yml for action tasks templates r=dustin,jonasfj

MozReview-Commit-ID: 8I8lIouV6KF

--HG--
extra : rebase_source : 6961105d1a232e93cf2f73db9ca88f8b08278dc0

.taskcluster.yml

@@ -1,139 +1,193 @@
 # This file is rendered via JSON-e by
 # - mozilla-taskcluster - https://docs.taskcluster.net/reference/integrations/mozilla-taskcluster/docs/taskcluster-yml
 # - cron tasks - taskcluster/taskgraph/cron/decision.py
+# - action tasks - taskcluster/taskgraph/actions/registry.py
 version: 1
 tasks:
-  $let:
-    # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
-    ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
-    # ensure there's no trailing `/` on the repo URL
-    repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
-  in:
-  - taskId: '${as_slugid("decision")}'
-    taskGroupId: '${as_slugid("decision")}' # same as tsakId; this is how automation identifies a decision tsak
-    schedulerId: 'gecko-level-${repository.level}'
+  - $let:
+      # sometimes the push user is just `ffxbld` or the like, but we want an email-like field..
+      ownerEmail: {$if: '"@" in push.owner', then: '${push.owner}', else: '${push.owner}@noreply.mozilla.org'}
+      # ensure there's no trailing `/` on the repo URL
+      repoUrl: {$if: 'repository.url[-1] == "/"', then: {$eval: 'repository.url[:-1]'}, else: {$eval: 'repository.url'}}
+    in:
+      taskId: {$if: 'tasks_for != "action"', then: '${as_slugid("decision")}'}
+      taskGroupId:
+        $if: 'tasks_for == "action"'
+        then:
+          '${action.taskGroupId}'
+        else:
+          '${as_slugid("decision")}' # same as taskId; this is how automation identifies a decision tsak
+      schedulerId: 'gecko-level-${repository.level}'
 
-    created: {$fromNow: ''}
-    deadline: {$fromNow: '1 day'}
-    expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors
-    metadata:
-      $merge:
-        - owner: "${ownerEmail}"
-          source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
-        - $if: 'tasks_for == "hg-push"'
-          then:
-            name: "Gecko Decision Task"
-            description: 'The task that creates all of the other tasks in the task graph'
-          else:
-            name: "Decision Task for cron job ${cron.job_name}"
-            description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'
-
-    provisionerId: "aws-provisioner-v1"
-    workerType: "gecko-${repository.level}-decision"
-
-    tags:
-      $if: 'tasks_for == "hg-push"'
-      then: {createdForUser: "${ownerEmail}"}
-
-    routes:
-      $if: 'tasks_for == "hg-push"'
-      then:
-        - "index.gecko.v2.${repository.project}.latest.firefox.decision"
-        - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
-        - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
-        - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
-        - "notify.email.${ownerEmail}.on-failed"
-        - "notify.email.${ownerEmail}.on-exception"
-      else:
-        - "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
-        - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
-        - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
-
-    scopes:
-      $if: 'tasks_for == "hg-push"'
-      then:
-        - 'assume:repo:${repoUrl[8:]}:*'
-        - 'queue:route:notify.email.${ownerEmail}.*'
-      else:
-        - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
-
-    dependencies: []
-    requires: all-completed
-
-    priority: lowest
-    retries: 5
-
-    payload:
-      env:
-        # checkout-gecko uses these to check out the source; the inputs
-        # to `mach taskgraph decision` are all on the command line.
-        GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
-        GECKO_HEAD_REPOSITORY: '${repoUrl}'
-        GECKO_HEAD_REF: '${push.revision}'
-        GECKO_HEAD_REV: '${push.revision}'
-        GECKO_COMMIT_MSG: '${push.comment}'
-        HG_STORE_PATH: /builds/worker/checkouts/hg-store
-        TASKCLUSTER_CACHES: /builds/worker/checkouts
-
-      cache:
-        level-${repository.level}-checkouts-sparse-v1: /builds/worker/checkouts
-
-      features:
-        taskclusterProxy: true
-        chainOfTrust: true
-
-      # Note: This task is built server side without the context or tooling that
-      # exist in tree so we must hard code the hash
-      # XXX Changing this will break Chain of Trust without an associated puppet and
-      # scriptworker patch!
-      image: 'taskcluster/decision:2.0.0@sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef'
-
-      maxRunTime: 1800
-
-      # TODO use mozilla-unified for the base repository once the tc-vcs
-      # tar.gz archives are created or tc-vcs isn't being used.
-      command:
-        - /builds/worker/bin/run-task
-        - '--vcs-checkout=/builds/worker/checkouts/gecko'
-        - '--sparse-profile=build/sparse-profiles/taskgraph'
-        - '--'
-        - bash
-        - -cx
-        - $let:
-            extraArgs: {$if: 'tasks_for == "hg-push"', then: '', else: '${cron.quoted_args}'}
-          # NOTE: the explicit reference to mozilla-central below is required because android-stuff
-          # still uses tc-vcs, which does not support mozilla-unified
-          # https://bugzilla.mozilla.org/show_bug.cgi?id=1383973
-          in: >
-            cd /builds/worker/checkouts/gecko &&
-            ln -s /builds/worker/artifacts artifacts &&
-            ./mach --log-no-times taskgraph decision
-            --pushlog-id='${push.pushlog_id}'
-            --pushdate='${push.pushdate}'
-            --project='${repository.project}'
-            --message="$GECKO_COMMIT_MSG"
-            --owner='${ownerEmail}'
-            --level='${repository.level}'
-            --base-repository='https://hg.mozilla.org/mozilla-central'
-            --head-repository="$GECKO_HEAD_REPOSITORY"
-            --head-ref="$GECKO_HEAD_REF"
-            --head-rev="$GECKO_HEAD_REV"
-            ${extraArgs}
-
-      artifacts:
-        'public':
-          type: 'directory'
-          path: '/builds/worker/artifacts'
-          expires: {$fromNow: '1 year'}
-
-    extra:
-      treeherder:
+      created: {$fromNow: ''}
+      deadline: {$fromNow: '1 day'}
+      expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors
+      metadata:
         $merge:
-          - machine:
-              platform: gecko-decision
+          - owner: "${ownerEmail}"
+            source: "${repoUrl}/raw-file/${push.revision}/.taskcluster.yml"
           - $if: 'tasks_for == "hg-push"'
             then:
-              symbol: D
+              name: "Gecko Decision Task"
+              description: 'The task that creates all of the other tasks in the task graph'
             else:
-              groupSymbol: cron
-              symbol: "${cron.job_symbol}"
+              $if: 'tasks_for == "action"'
+              then:
+                name: "Action: ${action.title}"
+                description: '${action.description}'
+              else:
+                name: "Decision Task for cron job ${cron.job_name}"
+                description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'
+
+      provisionerId: "aws-provisioner-v1"
+      workerType: "gecko-${repository.level}-decision"
+
+      tags:
+        $if: 'tasks_for == "hg-push"'
+        then: {createdForUser: "${ownerEmail}"}
+        else:
+          $if: 'tasks_for == "action"'
+          then:
+            createdForUser: '${ownerEmail}'
+            kind: 'action-callback'
+
+      routes:
+        $if: 'tasks_for == "hg-push"'
+        then:
+          - "index.gecko.v2.${repository.project}.latest.firefox.decision"
+          - "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.decision"
+          - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+          - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+          - "notify.email.${ownerEmail}.on-failed"
+          - "notify.email.${ownerEmail}.on-exception"
+        else:
+          - "tc-treeherder.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+          - "tc-treeherder-stage.v2.${repository.project}.${push.revision}.${push.pushlog_id}"
+          - $if: 'tasks_for == "action"'
+            then: "index.gecko.v2.${repository.project}.pushlog-id.${push.pushlog_id}.actions.$ownTaskId"
+            else: "index.gecko.v2.${repository.project}.latest.firefox.decision-${cron.job_name}"
+
+      scopes:
+        $if: 'tasks_for == "hg-push"'
+        then:
+          - 'assume:repo:${repoUrl[8:]}:*'
+          - 'queue:route:notify.email.${ownerEmail}.*'
+        else:
+          $if: 'tasks_for == "action"'
+          then:
+            - '${action.repo_scope}'
+          else:
+            - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
+
+      dependencies: []
+      requires: all-completed
+
+      priority: lowest
+      retries: 5
+
+      payload:
+        env:
+          # checkout-gecko uses these to check out the source; the inputs
+          # to `mach taskgraph decision` are all on the command line.
+          $merge:
+            - GECKO_BASE_REPOSITORY: 'https://hg.mozilla.org/mozilla-unified'
+              GECKO_HEAD_REPOSITORY: '${repoUrl}'
+              GECKO_HEAD_REF: '${push.revision}'
+              GECKO_HEAD_REV: '${push.revision}'
+              GECKO_COMMIT_MSG: {$if: 'tasks_for != "action"', then: '${push.comment}'}
+              HG_STORE_PATH: /builds/worker/checkouts/hg-store
+              TASKCLUSTER_CACHES: /builds/worker/checkouts
+            - $if: 'tasks_for == "action"'
+              then:
+                ACTION_TASK_GROUP_ID: '${action.taskGroupId}'
+                ACTION_TASK_ID: {$json: {$eval: 'taskId'}}
+                ACTION_TASK: {$json: {$eval: 'task'}}
+                ACTION_INPUT: {$json: {$eval: 'input'}}
+                ACTION_CALLBACK: '${action.cb_name}'
+                ACTION_PARAMETERS: {$json: {$eval: 'parameters'}}
+
+        cache:
+          level-${repository.level}-checkouts-sparse-v1: /builds/worker/checkouts
+
+        features:
+          taskclusterProxy: true
+          chainOfTrust: true
+
+        # Note: This task is built server side without the context or tooling that
+        # exist in tree so we must hard code the hash
+        # XXX Changing this will break Chain of Trust without an associated puppet and
+        # scriptworker patch!
+        image: 'taskcluster/decision:2.0.0@sha256:4039fd878e5700b326d4a636e28c595c053fbcb53909c1db84ad1f513cf644ef'
+
+        maxRunTime: 1800
+
+        # TODO use mozilla-unified for the base repository once the tc-vcs
+        # tar.gz archives are created or tc-vcs isn't being used.
+        command:
+          - /builds/worker/bin/run-task
+          - '--vcs-checkout=/builds/worker/checkouts/gecko'
+          - '--sparse-profile=build/sparse-profiles/taskgraph'
+          - '--'
+          - bash
+          - -cx
+          - $let:
+              extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
+            # NOTE: the explicit reference to mozilla-central below is required because android-stuff
+            # still uses tc-vcs, which does not support mozilla-unified
+            # https://bugzilla.mozilla.org/show_bug.cgi?id=1383973
+            in:
+              $if: 'tasks_for == "action"'
+              then: >
+                cd /builds/worker/checkouts/gecko &&
+                ln -s /builds/worker/artifacts artifacts &&
+                ./mach --log-no-times taskgraph action-callback
+              else: >
+                cd /builds/worker/checkouts/gecko &&
+                ln -s /builds/worker/artifacts artifacts &&
+                ./mach --log-no-times taskgraph decision
+                --pushlog-id='${push.pushlog_id}'
+                --pushdate='${push.pushdate}'
+                --project='${repository.project}'
+                --message="$GECKO_COMMIT_MSG"
+                --owner='${ownerEmail}'
+                --level='${repository.level}'
+                --base-repository='https://hg.mozilla.org/mozilla-central'
+                --head-repository="$GECKO_HEAD_REPOSITORY"
+                --head-ref="$GECKO_HEAD_REF"
+                --head-rev="$GECKO_HEAD_REV"
+                ${extraArgs}
+
+        artifacts:
+          'public':
+            type: 'directory'
+            path: '/builds/worker/artifacts'
+            expires: {$fromNow: '1 year'}
+
+      extra:
+        $merge:
+          - treeherder:
+              $merge:
+                - machine:
+                    platform: gecko-decision
+                - $if: 'tasks_for == "hg-push"'
+                  then:
+                    symbol: D
+                  else:
+                    $if: 'tasks_for == "action"'
+                    then:
+                      groupName: 'action-callback'
+                      groupSymbol: AC
+                      symbol: "${action.symbol}"
+                    else:
+                      groupSymbol: cron
+                      symbol: "${cron.job_symbol}"
+          - $if: 'tasks_for == "action"'
+            then:
+              parent: '${action.taskGroupId}'
+              action:
+                name: '${action.name}'
+                context:
+                  taskGroupId: '${action.taskGroupId}'
+                  taskId: {$eval: 'taskId'}
+                  input: {$eval: 'input'}
+                  parameters: {$eval: 'parameters'}

build/sparse-profiles/taskgraph

@@ -36,3 +36,6 @@ glob:**/tooltool-manifests/**
 # For scheduling android-gradle-dependencies.
 path:mobile/android/config/
 glob:**/*.gradle
+
+# for action-task building
+path:.taskcluster.yml

taskcluster/taskgraph/actions/registry.py

@@ -8,21 +8,17 @@ from __future__ import absolute_import, print_function, unicode_literals
 
 import json
 import os
-import inspect
 import re
 import yaml
 from slugid import nice as slugid
 from mozbuild.util import memoize
 from types import FunctionType
 from collections import namedtuple
-from taskgraph import create
+from taskgraph import create, GECKO
 from taskgraph.util import taskcluster
-from taskgraph.util.docker import docker_image
 from taskgraph.parameters import Parameters
 
 
-GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..'))
-
 actions = []
 callbacks = {}
 
@@ -173,7 +169,6 @@ def register_callback_action(name, title, symbol, description, order=10000,
         assert 1 <= len(symbol) <= 25, 'symbol must be between 1 and 25 characters'
         assert not mem['registered'], 'register_callback_action must be used as decorator'
         assert cb.__name__ not in callbacks, 'callback name {} is not unique'.format(cb.__name__)
-        source_path = os.path.relpath(inspect.stack()[1][1], GECKO)
 
         @register_task_action(name, title, description, order, context, schema)
         def build_callback_action_task(parameters):
@@ -188,98 +183,41 @@ def register_callback_action(name, title, symbol, description, order=10000,
 
             task_group_id = os.environ.get('TASK_ID', slugid())
 
-            return {
-                'created': {'$fromNow': ''},
-                'deadline': {'$fromNow': '12 hours'},
-                'expires': {'$fromNow': '1 year'},
-                'metadata': {
-                    'owner': 'mozilla-taskcluster-maintenance@mozilla.com',
-                    'source': '{}/raw-file/{}/{}'.format(
-                        parameters['head_repository'], parameters['head_rev'], source_path,
-                    ),
-                    'name': 'Action: {}'.format(title),
-                    'description': 'Task executing callback for action.\n\n---\n' + description,
-                },
-                'workerType': 'gecko-{}-decision'.format(parameters['level']),
-                'provisionerId': 'aws-provisioner-v1',
-                'taskGroupId': task_group_id,
-                'schedulerId': 'gecko-level-{}'.format(parameters['level']),
-                'scopes': [
-                    repo_scope,
-                ],
-                'tags': {
-                    'createdForUser': parameters['owner'],
-                    'kind': 'action-callback',
-                },
-                'routes': [
-                    'tc-treeherder.v2.{}.{}.{}'.format(
-                        parameters['project'], parameters['head_rev'], parameters['pushlog_id']),
-                    'tc-treeherder-stage.v2.{}.{}.{}'.format(
-                        parameters['project'], parameters['head_rev'], parameters['pushlog_id']),
-                    'index.gecko.v2.{}.pushlog-id.{}.actions.${{ownTaskId}}'.format(
-                        parameters['project'], parameters['pushlog_id'])
-                ],
-                'payload': {
-                    'env': {
-                        'GECKO_BASE_REPOSITORY': 'https://hg.mozilla.org/mozilla-unified',
-                        'GECKO_HEAD_REPOSITORY': parameters['head_repository'],
-                        'GECKO_HEAD_REF': parameters['head_ref'],
-                        'GECKO_HEAD_REV': parameters['head_rev'],
-                        'HG_STORE_PATH': '/builds/worker/checkouts/hg-store',
-                        'ACTION_TASK_GROUP_ID': task_group_id,
-                        'ACTION_TASK_ID': {'$json': {'$eval': 'taskId'}},
-                        'ACTION_TASK': {'$json': {'$eval': 'task'}},
-                        'ACTION_INPUT': {'$json': {'$eval': 'input'}},
-                        'ACTION_CALLBACK': cb.__name__,
-                        'ACTION_PARAMETERS': {'$json': {'$eval': 'parameters'}},
-                        'TASKCLUSTER_CACHES': '/builds/worker/checkouts',
-                    },
-                    'artifacts': {
-                        'public': {
-                            'type': 'directory',
-                            'path': '/builds/worker/artifacts',
-                            'expires': {'$fromNow': '1 year'},
+            template = os.path.join(GECKO, '.taskcluster.yml')
+
+            with open(template, 'r') as f:
+                taskcluster_yml = yaml.safe_load(f)
+                if taskcluster_yml['version'] != 1:
+                    raise Exception('actions.json must be updated to work with .taskcluster.yml')
+                if not isinstance(taskcluster_yml['tasks'], list):
+                    raise Exception('.taskcluster.yml "tasks" must be a list for action tasks')
+
+                return {
+                    '$let': {
+                        'tasks_for': 'action',
+                        'repository': {
+                            'url': parameters['head_repository'],
+                            'project': parameters['project'],
+                            'level': parameters['level'],
                         },
-                    },
-                    'cache': {
-                        'level-{}-checkouts-sparse-v1'.format(parameters['level']):
-                            '/builds/worker/checkouts',
-                    },
-                    'features': {
-                        'taskclusterProxy': True,
-                        'chainOfTrust': True,
-                    },
-                    'image': docker_image('decision'),
-                    'maxRunTime': 1800,
-                    'command': [
-                        '/builds/worker/bin/run-task',
-                        '--vcs-checkout=/builds/worker/checkouts/gecko',
-                        '--sparse-profile=build/sparse-profiles/taskgraph',
-                        '--', 'bash', '-cx',
-                        """\
-cd /builds/worker/checkouts/gecko &&
-ln -s /builds/worker/artifacts artifacts &&
-./mach --log-no-times taskgraph action-callback""",
-                    ],
-                },
-                'extra': {
-                    'treeherder': {
-                        'groupName': 'action-callback',
-                        'groupSymbol': 'AC',
-                        'symbol': symbol,
-                    },
-                    'parent': task_group_id,
-                    'action': {
-                        'name': name,
-                        'context': {
+                        'push': {
+                            'owner': 'mozilla-taskcluster-maintenance@mozilla.com',
+                            'pushlog_id': parameters['pushlog_id'],
+                            'revision': parameters['head_rev'],
+                        },
+                        'action': {
+                            'name': name,
+                            'title': title,
+                            'description': description,
                             'taskGroupId': task_group_id,
-                            'taskId': {'$eval': 'taskId'},
-                            'input': {'$eval': 'input'},
-                            'parameters': {'$eval': 'parameters'},
+                            'repo_scope': repo_scope,
+                            'cb_name': cb.__name__,
+                            'symbol': symbol,
                         },
                     },
-                },
-            }
+                    'in': taskcluster_yml['tasks'][0]
+                }
+
         mem['registered'] = True
         callbacks[cb.__name__] = cb
     return register_callback